From 5b65eb82896fcf3abf2b557408ed8039b29728cd Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 29 Jul 2026 10:42:50 -0700 Subject: [PATCH 1/7] =?UTF-8?q?fix:=20metadata-only=20update()=20never=20r?= =?UTF-8?q?ewrites=20the=20noun=20record=20=E2=80=94=20the=20unconditional?= =?UTF-8?q?=20whole-vector=20save=20turned=20per-entity=20stat=20touches?= =?UTF-8?q?=20into=20full=20rewrites+fsync,=20amplifying=20read-heavy=20sw?= =?UTF-8?q?eeps=20into=20disk=20saturation=20on=20a=20production=20deploym?= =?UTF-8?q?ent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also: idle PathResolver stats tick no longer logs NaN% every minute (logs only on new traffic, via prodLog); graph-lsm-* key family recognized as system resources (kills the per-boot unknown-key warning on provider-backed brains). Four regression pins in tests/integration/update-write-granularity. --- src/brainy.ts | 27 ++-- src/storage/baseStorage.ts | 4 + src/vfs/PathResolver.ts | 13 +- .../update-write-granularity.test.ts | 126 ++++++++++++++++++ 4 files changed, 155 insertions(+), 15 deletions(-) create mode 100644 tests/integration/update-write-granularity.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 007cdb32..1b5321e9 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -3161,18 +3161,23 @@ export class Brainy implements BrainyInterface { new UpdateNounMetadataOperation(this.storage, params.id, updatedMetadata) ) - // Operation 2: Update vector data (will use updated type cache) - tx.addOperation( - new SaveNounOperation(this.storage, { - id: params.id, - vector, - connections: new Map(), - level: 0 - }) - ) - - // Operation 3-4: Update HNSW index (remove and re-add if reindexing needed) + // Operations 2-4: vector-record write + HNSW reindex — ONLY when the + // vector side actually changed (new data/vector/type). A metadata-only + // update must never rewrite the noun record: the record carries the + // full vector, so an unconditional save turned every metadata touch + // into a whole-vector rewrite + fsync — under a read-heavy consumer + // sweep that bumps per-entity stats, this amplified into disk + // saturation on a production deployment (SELF-ENGINE-RESTART-GRIND, + // 2026-07-29: 5.8GB written in 40min from ~50 recalls/min). if (needsReindexing) { + tx.addOperation( + new SaveNounOperation(this.storage, { + id: params.id, + vector, + connections: new Map(), + level: 0 + }) + ) tx.addOperation( new RemoveFromVectorIndexOperation(this.index, params.id, existing.vector) ) diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index 6daf09c0..1d3e245d 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -382,6 +382,10 @@ export abstract class BaseStorage extends BaseStorageAdapter { // identical to the unknown-key fallback these keys hit // before being listed here — this only kills the // per-boot "Unknown key format" warning) + id.startsWith('graph-lsm-') || // Graph-LSM store manifests written through storage by + // an active native graph provider — same + // warn-then-route fallback as above; listing the family + // silences the per-boot warning on provider-backed brains isSingletonSystemKey(id) // Known singletons (e.g. brainy:entityIdMapper) hit the // same warn-then-route fallback without this — the // routing below already handles them identically diff --git a/src/vfs/PathResolver.ts b/src/vfs/PathResolver.ts index e496c834..502c95f0 100644 --- a/src/vfs/PathResolver.ts +++ b/src/vfs/PathResolver.ts @@ -57,6 +57,7 @@ export class PathResolver { // Statistics private cacheHits = 0 private cacheMisses = 0 + private lastLoggedLookups = 0 // last total the maintenance tick logged stats at private metadataIndexHits = 0 private metadataIndexMisses = 0 private graphTraversalFallbacks = 0 @@ -519,10 +520,14 @@ export class PathResolver { } } - // Log cache statistics (in production, send to monitoring) - const hitRate = this.cacheHits / (this.cacheHits + this.cacheMisses) - if ((this.cacheHits + this.cacheMisses) % 1000 === 0) { - console.log(`[PathResolver] Cache stats: ${Math.round(hitRate * 100)}% hit rate, ${this.pathCache.size} entries, ${this.hotPaths.size} hot paths`) + // Log cache statistics only when there is new traffic to report — an + // idle resolver stays silent. 0/0 lookups previously rendered + // "NaN% hit rate" (and the %1000 gate passes at zero), which spammed + // production journals once a minute on every idle VFS. + const totalLookups = this.cacheHits + this.cacheMisses + if (totalLookups > 0 && totalLookups !== this.lastLoggedLookups && totalLookups % 1000 === 0) { + this.lastLoggedLookups = totalLookups + prodLog.debug(`[PathResolver] Cache stats: ${Math.round((this.cacheHits / totalLookups) * 100)}% hit rate, ${this.pathCache.size} entries, ${this.hotPaths.size} hot paths`) } }, 60000) // Every minute // Cache maintenance must never keep the host process alive. diff --git a/tests/integration/update-write-granularity.test.ts b/tests/integration/update-write-granularity.test.ts new file mode 100644 index 00000000..234df334 --- /dev/null +++ b/tests/integration/update-write-granularity.test.ts @@ -0,0 +1,126 @@ +/** + * @module tests/integration/update-write-granularity + * @description Write-granularity law for update() (SELF-ENGINE-RESTART-GRIND, + * 2026-07-29): a metadata-only update must NEVER rewrite the noun record — + * the record carries the full vector, so an unconditional save turns every + * metadata touch into a whole-vector rewrite + fsync. Under a read-heavy + * consumer sweep bumping per-entity stats this amplified into disk saturation + * on a production deployment. Laws: + * (1) metadata-only update() → zero saveNoun calls (metadata leg only); + * (2) data/vector/type-changing update() → saveNoun runs (the vector leg and + * HNSW reindex still happen when the vector side actually changed); + * (3) the metadata-only path still lands: merged metadata readable, _rev + * bumped, find() by the new field sees the entity. + */ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +const stubEmbedding = async (text: string): Promise => { + const hash = text.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0) + return new Array(384).fill(0).map((_, i) => Math.sin(hash + i)) +} + +describe('update() write granularity', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' as const }, + embeddingFunction: stubEmbedding + }) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + it('metadata-only update never rewrites the noun record (no vector rewrite)', async () => { + const id = await brain.add({ + data: 'granularity law subject', + type: NounType.Concept, + metadata: { touched: 0 } + }) + + const storage = (brain as any).storage + const saveNounSpy = vi.spyOn(storage, 'saveNoun') + + await brain.update({ id, metadata: { touched: 1 } }) + + expect(saveNounSpy).not.toHaveBeenCalled() + saveNounSpy.mockRestore() + + // The metadata leg still landed with full semantics. + const after = await brain.get(id, { includeVectors: true }) + expect(after?.metadata?.touched).toBe(1) + expect(after?._rev).toBe(2) + expect(Array.isArray(after?.vector) && after!.vector!.length).toBe(384) + + const found = await brain.find({ where: { touched: 1 } }) + expect(found.some((r: any) => r.id === id)).toBe(true) + }) + + it('confidence/weight/subtype-only updates also skip the noun record', async () => { + const id = await brain.add({ + data: 'reserved-field touch subject', + type: NounType.Concept, + metadata: {} + }) + + const storage = (brain as any).storage + const saveNounSpy = vi.spyOn(storage, 'saveNoun') + + await brain.update({ id, confidence: 0.5, weight: 2, subtype: 'note' }) + + expect(saveNounSpy).not.toHaveBeenCalled() + saveNounSpy.mockRestore() + + const after = await brain.get(id) + expect(after?.confidence).toBe(0.5) + expect(after?.subtype).toBe('note') + }) + + it('data-changing update still writes the noun record and reindexes', async () => { + const id = await brain.add({ + data: 'original embedded text', + type: NounType.Concept, + metadata: {} + }) + + const before = await brain.get(id, { includeVectors: true }) + + const storage = (brain as any).storage + const saveNounSpy = vi.spyOn(storage, 'saveNoun') + + await brain.update({ id, data: 'completely different embedded text' }) + + expect(saveNounSpy).toHaveBeenCalled() + saveNounSpy.mockRestore() + + const after = await brain.get(id, { includeVectors: true }) + expect(after?.data).toBe('completely different embedded text') + expect(after?.vector).not.toEqual(before?.vector) + }) + + it('explicit-vector update still writes the noun record', async () => { + const id = await brain.add({ + data: 'vector swap subject', + type: NounType.Concept, + metadata: {} + }) + + const storage = (brain as any).storage + const saveNounSpy = vi.spyOn(storage, 'saveNoun') + + const newVector = new Array(384).fill(0).map((_, i) => Math.cos(i)) + await brain.update({ id, vector: newVector }) + + expect(saveNounSpy).toHaveBeenCalled() + saveNounSpy.mockRestore() + + const after = await brain.get(id, { includeVectors: true }) + expect(after?.vector?.[0]).toBeCloseTo(1) // cos(0) + }) +}) From a0123b5b8bdc65f9a3411e5f83e50f4887d77170 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 29 Jul 2026 15:08:16 -0700 Subject: [PATCH 2/7] =?UTF-8?q?docs:=208.10.2=20consumer=20release=20notes?= =?UTF-8?q?=20=E2=80=94=20update()=20write=20granularity,=20PathResolver?= =?UTF-8?q?=20idle-log=20fix,=20graph-lsm=20key=20recognition?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- RELEASES.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/RELEASES.md b/RELEASES.md index 03d7283a..e68bcca1 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -31,6 +31,30 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the --- +## v8.10.2 — 2026-07-29 (metadata-only updates stop rewriting the vector record) + +From a production incident on a large deployment: a read-heavy sweep that bumped +per-entity stats (metadata-only `update()` calls) saturated the disk — 5.8GB written +in 40 minutes — because every `update()` unconditionally re-persisted the WHOLE noun +record, unchanged vector included, fsynced. + +- **`update()` write granularity fixed at the core.** A metadata-only update (no new + `data`, `vector`, or `type`) now writes the metadata leg and index deltas ONLY — + the vector-bearing noun record is never rewritten. Vector-side writes and HNSW + reindexing still happen exactly when the vector side actually changed. Regression + pins: `tests/integration/update-write-granularity.test.ts`. +- **Consumer guidance:** per-entity stat touches are now cheap, but batch them anyway + (one `transact()` instead of N `update()` calls) — granularity fixes the cost per + touch; batching fixes the count. +- Idle VFS `PathResolver` no longer logs `NaN% hit rate` once a minute (stats log + only on new traffic, at debug level). +- Native graph providers' `graph-lsm-*` storage keys are recognized as system + resources — the per-boot `Unknown key format` warning for them is gone. + +Pairs with the native accelerator's same-day patch release; adopt as one bump. + +--- + ## v8.10.1 — 2026-07-24 (the no-hot-retry contract + warm()'s metadata surface under native providers) From a production incident: a native-provider op ground 38-40s inside a transaction, From 3047ffa75cdeb500c72eecc378e4a31ad03b8080 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 29 Jul 2026 15:08:36 -0700 Subject: [PATCH 3/7] chore(release): 8.10.2 --- CHANGELOG.md | 6 ++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5f344cc..e0e47015 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [8.10.2](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.1...v8.10.2) (2026-07-29) + +- docs: 8.10.2 consumer release notes — update() write granularity, PathResolver idle-log fix, graph-lsm key recognition (a0123b5b) +- fix: metadata-only update() never rewrites the noun record — the unconditional whole-vector save turned per-entity stat touches into full rewrites+fsync, amplifying read-heavy sweeps into disk saturation on a production deployment (5b65eb82) + + ### [8.10.1](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.0...v8.10.1) (2026-07-24) - refactor: remove the orphaned transaction-result type left behind by the dead-path removal (edf123a5) diff --git a/package-lock.json b/package-lock.json index d0c7b9d9..a0fb6f6d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "8.10.1", + "version": "8.10.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "8.10.1", + "version": "8.10.2", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index ce670369..5d140743 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "8.10.1", + "version": "8.10.2", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From 099579f716baee5031fc483c33111996fabd78ff Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 29 Jul 2026 15:29:56 -0700 Subject: [PATCH 4/7] =?UTF-8?q?ci:=20carry=20the=20tag-push=20publish=20wo?= =?UTF-8?q?rkflow=20on=20the=20release=20branch=20=E2=80=94=20the=20forge?= =?UTF-8?q?=20discovers=20tag=20workflows=20from=20the=20tag's=20own=20tre?= =?UTF-8?q?e,=20so=20a=20backport=20tag=20must=20contain=20the=20workflow?= =?UTF-8?q?=20itself=20(completes=20the=208.11.0=20maiden-flight=20lesson)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .forgejo/workflows/publish-forge.yml | 67 ++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .forgejo/workflows/publish-forge.yml diff --git a/.forgejo/workflows/publish-forge.yml b/.forgejo/workflows/publish-forge.yml new file mode 100644 index 00000000..fb7428bf --- /dev/null +++ b/.forgejo/workflows/publish-forge.yml @@ -0,0 +1,67 @@ +name: Publish (forge) + +# Datacenter-side forge publish, moved off the laptop: an 87MB tarball PUT +# over the laptop's WAN times out; the forge's own runner does it in seconds. +# scripts/release.sh tags + pushes, then polls this workflow's result (npm +# view against the forge registry) before it ever touches the npmjs leg — +# see the "delegation contract" in scripts/release.sh's forge-publish step. + +on: + push: + tags: + - 'v*' + +jobs: + publish: + name: Publish to the forge registry + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + - run: npm ci + - run: npm run build + - name: Publish + readback-verify on the forge registry + env: + FORGE_NPM_TOKEN: ${{ secrets.FORGE_NPM_TOKEN }} + run: | + set -eo pipefail + + FORGE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/" + VERSION="$(node -p "require('./package.json').version")" + echo "Publishing @soulcraft/brainy@${VERSION} to the forge registry..." + + TMPRC="$(mktemp)" + chmod 600 "$TMPRC" + { + echo "@soulcraft:registry=${FORGE_NPM_REG}" + echo "//source.soulcraft.com/api/packages/soulcraft/npm/:_authToken=${FORGE_NPM_TOKEN}" + } > "$TMPRC" + + # The release script bumps package.json's version before it tags, so + # this tag's checkout already carries the version being published — + # nothing here re-derives it from the tag name. + PUBLISH_OK=true + if ! npm publish --tag latest --userconfig "$TMPRC"; then + PUBLISH_OK=false + fi + + # Readback verify is the source of truth, run regardless of the publish + # exit code: a benign duplicate publish (a prior run, or a mirror, already + # landed this exact version) reports failure even though the registry + # already holds the right content. + LANDED_VERSION="$(npm view "@soulcraft/brainy@${VERSION}" version --userconfig "$TMPRC" 2>/dev/null || echo "")" + rm -f "$TMPRC" + + if [ "$LANDED_VERSION" != "$VERSION" ]; then + echo "::error::Readback verify FAILED — the forge registry reports version '${LANDED_VERSION:-}', expected '${VERSION}'. This is a genuine publish failure, not a benign duplicate." + exit 1 + fi + + if [ "$PUBLISH_OK" = true ]; then + echo "Published and verified @soulcraft/brainy@${VERSION} on the forge registry." + else + echo "::warning::npm publish reported failure, but readback confirms @soulcraft/brainy@${VERSION} is already live on the forge (a prior run or mirror landed it) — treating this run as successful, since the registry content is correct. Any OTHER failure mode would have failed the readback check above instead." + fi From 958a0859262ad51f919e327e26e6f246b069291f Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 3 Aug 2026 11:57:32 -0700 Subject: [PATCH 5/7] =?UTF-8?q?fix:=20user=20metadata=20named=20'level'=20?= =?UTF-8?q?is=20a=20real=20field=20everywhere=20=E2=80=94=20the=20engine-i?= =?UTF-8?q?nternal=20node=20layer=20no=20longer=20shadows=20it=20in=20sort?= =?UTF-8?q?/filter/aggregation,=20and=20the=20indexing=20views=20stop=20st?= =?UTF-8?q?amping=20a=20phantom=200=20into=20its=20column;=20index=20epoch?= =?UTF-8?q?=202=20rebuilds=20existing=20brains=20at=20first=20open?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also completes the v8.10.2 write-granularity law for the transact() plan path: a metadata-only batch update never rewrites the vector-bearing noun record (planUpdate staged the unconditional save the update() fix removed). Seven pins in tests/integration/level-field-shadow.test.ts including the reporting consumer's exact repro rows; orderBy JSDoc documents the ordering contract and the announced field-addressing law. --- RELEASES.md | 55 +++++++ src/brainy.ts | 33 ++-- src/coreTypes.ts | 7 +- src/storage/brainFormat.ts | 7 +- src/types/brainy.types.ts | 18 ++- tests/integration/level-field-shadow.test.ts | 147 ++++++++++++++++++ tests/integration/orderby-sort-bug.test.ts | 5 +- tests/unit/brainy/migration-deference.test.ts | 4 +- 8 files changed, 259 insertions(+), 17 deletions(-) create mode 100644 tests/integration/level-field-shadow.test.ts diff --git a/RELEASES.md b/RELEASES.md index e68bcca1..03d2d427 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -55,6 +55,61 @@ Pairs with the native accelerator's same-day patch release; adopt as one bump. --- +## Unreleased (natural field names stop colliding with engine internals) + +From a production report: sorting by a user metadata field named `level` silently +returned insertion order — the engine's internal HNSW node layer (also called +`level`) shadowed the user's field in every by-name read, and the indexing path +stamped a hardcoded `0` into the same index column (multi-valued poison). `level` +is a perfectly natural field name (game characters, priorities, floors); the +engine was wrong, not the caller. + +- **`level` is user data now, everywhere.** Engine plumbing no longer resolves by + name, never shadows metadata, and never enters the indexed views. `orderBy: + 'level'`, `where: { level: 9 }`, `groupBy: ['level']` all read YOUR field. + Regression pins: `tests/integration/level-field-shadow.test.ts` (the reporting + consumer's exact repro rows). +- **Index epoch 2.** The derived posting set changed, so every existing brain + rebuilds its metadata index from canonical at first open — poisoned columns + heal automatically; no manual step. First open after upgrade pays one rebuild + (observable via `getIndexStatus()`); pair this release with the same-day + native-accelerator release, which makes `level` indexable on the native path. +- **`transact()` metadata-only updates stop rewriting the vector record** — the + v8.10.2 write-granularity law now covers the batch/plan path too (it was + fixed for `update()` but the transact plan builder still staged the + unconditional save). If you batch stat touches through `transact()`, this is + your write-amplification fix. +- Coming next (announced so parsers and call sites can prepare): one + field-addressing law — bare names = user metadata, `system.` for + engine fields, typed refusals for unresolvable names. Ships as its own + release with a migration advisory; nothing changes in this release. + +--- + +## v8.10.2 — 2026-07-29 (metadata-only updates stop rewriting the vector record) + +From a production incident on a large deployment: a read-heavy sweep that bumped +per-entity stats (metadata-only `update()` calls) saturated the disk — 5.8GB written +in 40 minutes — because every `update()` unconditionally re-persisted the WHOLE noun +record, unchanged vector included, fsynced. + +- **`update()` write granularity fixed at the core.** A metadata-only update (no new + `data`, `vector`, or `type`) now writes the metadata leg and index deltas ONLY — + the vector-bearing noun record is never rewritten. Vector-side writes and HNSW + reindexing still happen exactly when the vector side actually changed. Regression + pins: `tests/integration/update-write-granularity.test.ts`. +- **Consumer guidance:** per-entity stat touches are now cheap, but batch them anyway + (one `transact()` instead of N `update()` calls) — granularity fixes the cost per + touch; batching fixes the count. +- Idle VFS `PathResolver` no longer logs `NaN% hit rate` once a minute (stats log + only on new traffic, at debug level). +- Native graph providers' `graph-lsm-*` storage keys are recognized as system + resources — the per-boot `Unknown key format` warning for them is gone. + +Pairs with the native accelerator's same-day patch release; adopt as one bump. + +--- + ## v8.10.1 — 2026-07-24 (the no-hot-retry contract + warm()'s metadata surface under native providers) From a production incident: a native-provider op ground 38-40s inside a transaction, diff --git a/src/brainy.ts b/src/brainy.ts index 1b5321e9..1a501f0e 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -2123,11 +2123,13 @@ export class Brainy implements BrainyInterface { // If undefined values are included as explicit keys, extractIndexableFields indexes // them as '__NULL__' entries that removeFromIndex can never clean up (storageMetadata // omits those keys entirely via conditional spreading, so the fields don't match). + // No `level` here: engine plumbing never enters the indexing view — a + // hardcoded level:0 landed in the SAME flattened index column as user + // metadata named `level`, poisoning it multi-valued ([0, real]). const entityForIndexing = { id, vector, connections: new Map(), - level: 0, type: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), ...(params.visibility !== undefined && @@ -3102,12 +3104,13 @@ export class Brainy implements BrainyInterface { }) } - // Build entity structure for metadata index (with top-level fields) + // Build entity structure for metadata index (with top-level fields). + // No `level`: engine plumbing never enters the indexing view (it + // poisoned the flattened user `level` column — VENUE-BRAINY-ORDERBY-NOOP). const entityForIndexing = { id: params.id, vector, connections: new Map(), - level: 0, type: params.type || existing.type, subtype: params.subtype !== undefined ? params.subtype : existing.subtype, ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && { @@ -9338,7 +9341,7 @@ export class Brainy implements BrainyInterface { id, vector, connections: new Map(), - level: 0, + // no `level` — plumbing never enters the indexing view type: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), ...(params.visibility !== undefined && @@ -9489,7 +9492,7 @@ export class Brainy implements BrainyInterface { id: params.id, vector, connections: new Map(), - level: 0, + // no `level` — plumbing never enters the indexing view type: params.type || existing.type, subtype: params.subtype !== undefined ? params.subtype : existing.subtype, ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && { @@ -9517,16 +9520,22 @@ export class Brainy implements BrainyInterface { } plan.operations.push( - new UpdateNounMetadataOperation(this.storage, params.id, updatedMetadata), - new SaveNounOperation(this.storage, { - id: params.id, - vector, - connections: new Map(), - level: 0 - }) + new UpdateNounMetadataOperation(this.storage, params.id, updatedMetadata) ) + // Noun-record write + HNSW reindex ONLY when the vector side actually + // changed — the same write-granularity law as update(): a metadata-only + // patch must never rewrite the whole vector record. This plan path is the + // one transact() updates ride, so an unconditional save here would + // re-open the read-sweep disk-saturation amplifier for exactly the + // consumers batching their stat touches through transact(). if (needsReindexing) { plan.operations.push( + new SaveNounOperation(this.storage, { + id: params.id, + vector, + connections: new Map(), + level: 0 + }), new RemoveFromVectorIndexOperation(this.index, params.id, existing.vector), new AddToVectorIndexOperation(this.index, params.id, vector) ) diff --git a/src/coreTypes.ts b/src/coreTypes.ts index e0248d17..90fc4462 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -284,7 +284,12 @@ export const STANDARD_ENTITY_FIELDS: ReadonlySet = new Set([ 'id', 'vector', 'connections', - 'level', + // 'level' is deliberately ABSENT: it is HNSW plumbing, not an entity field. + // Listing it here made every by-name read of a user metadata field called + // `level` resolve to the engine's internal node layer instead — a silent + // shadow that broke sort/filter/aggregation on a perfectly natural field + // name (VENUE-BRAINY-ORDERBY-NOOP). Engine plumbing is invisible to the + // query surface; a bare `level` reads `entity.metadata.level`. 'type', 'subtype', 'visibility', diff --git a/src/storage/brainFormat.ts b/src/storage/brainFormat.ts index 2e6488e9..a1241fe0 100644 --- a/src/storage/brainFormat.ts +++ b/src/storage/brainFormat.ts @@ -69,7 +69,12 @@ export const BRAIN_FORMAT_PATH = '_system/brain-format.json' * (the 8.0 GA baseline). An on-disk `indexEpoch` that differs from this — or an * absent marker — triggers a full derived-index rebuild on open. */ -export const EXPECTED_INDEX_EPOCH = 1 +// Epoch 2 (2026-08-03, paired with the native accelerator's same-day release): +// user metadata fields named `level` become indexable on both engines — the +// derived posting set changed, so every pre-fix brain must rebuild its +// metadata index from canonical at first open (poisoned multi-valued `level` +// columns heal through this rebuild; no bespoke heal path). +export const EXPECTED_INDEX_EPOCH = 2 /** * @description The data-layer format string this build writes and runs as. diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index b5f286ed..89be78b9 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -551,7 +551,23 @@ export interface FindParams { cursor?: string // Cursor-based pagination // Sorting - orderBy?: string // Field to sort by (e.g., 'createdAt', 'title', 'metadata.priority') + /** + * Field to sort by. User metadata fields sort by their stored values — + * including natural names like `level`, `rank`, or `score` (an engine-internal + * field can never shadow your metadata; fixed 2026-08 after a production + * report). System timestamps (`createdAt`, `updatedAt`) sort by entity age. + * + * Ordering contract (identical on the pure-JS engine and the native + * accelerator): entities missing the field sort LAST in both directions — + * they are never dropped from the result; ties break deterministically. + * + * NOTE — the field-addressing law is changing (announced 2026-08): bare + * names will mean user metadata ALWAYS, and system fields will be reached + * explicitly as `system.` (e.g. `system.createdAt`), with typed + * refusals for unresolvable names. Until that release, bare `createdAt` + * and friends keep resolving to the system fields as documented above. + */ + orderBy?: string order?: 'asc' | 'desc' // Sort direction: 'asc' (default) or 'desc' // Advanced options diff --git a/tests/integration/level-field-shadow.test.ts b/tests/integration/level-field-shadow.test.ts new file mode 100644 index 00000000..d50593ff --- /dev/null +++ b/tests/integration/level-field-shadow.test.ts @@ -0,0 +1,147 @@ +/** + * @module tests/integration/level-field-shadow + * @description The reserved-name shadow fix (VENUE-BRAINY-ORDERBY-NOOP, + * 2026-08-03): `level` is HNSW plumbing, not an entity field — it must never + * shadow user metadata of the same name. Pre-fix, STANDARD_ENTITY_FIELDS + * listed `level`, so every by-name read returned the engine's internal 0 + * (all-equal → stable sort → insertion order, silently), and the indexing + * views stamped level:0 into the same flattened column as user values + * (multi-valued [0, real] poison). Laws: + * (1) venue's exact repro sorts: three adds with metadata.level 3/9/6 → + * find({orderBy:'level'}) returns 9,6,3 desc and 3,6,9 asc; + * (2) where {level: N} matches through filter AND egress guard; + * (3) the index column carries the user value only (no 0 poison); + * (4) update() keeps `level` readable (the update indexing view is clean too); + * (5) the transact() update path never rewrites the noun record on a + * metadata-only patch (the planUpdate granularity completion). + */ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import { EXPECTED_INDEX_EPOCH } from '../../src/storage/brainFormat.js' + +const stubEmbedding = async (text: string): Promise => { + const hash = text.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0) + return new Array(384).fill(0).map((_, i) => Math.sin(hash + i)) +} + +describe('level field shadow — user metadata named level is a real field', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' as const }, + embeddingFunction: stubEmbedding + }) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + async function addProbeRows(): Promise { + const ids: string[] = [] + for (const level of [3, 9, 6]) { + ids.push( + await brain.add({ + data: `probe character level ${level}`, + type: NounType.Person, + subtype: 'probe-char', + metadata: { name: `char-${level}`, level } + }) + ) + } + return ids + } + + it("venue's exact repro: orderBy 'level' sorts desc and asc", async () => { + await addProbeRows() + + const desc = await brain.find({ + type: NounType.Person, + subtype: 'probe-char', + orderBy: 'level', + order: 'desc', + limit: 100 + }) + expect(desc.map((r: any) => r.metadata?.level)).toEqual([9, 6, 3]) + + const asc = await brain.find({ + type: NounType.Person, + subtype: 'probe-char', + orderBy: 'level', + order: 'asc', + limit: 100 + }) + expect(asc.map((r: any) => r.metadata?.level)).toEqual([3, 6, 9]) + }) + + it('ordered reads are COMPLETE — no row dropped (the 2-of-3 face)', async () => { + const ids = await addProbeRows() + const desc = await brain.find({ + type: NounType.Person, + subtype: 'probe-char', + orderBy: 'level', + order: 'desc', + limit: 100 + }) + expect(desc).toHaveLength(3) + expect(new Set(desc.map((r: any) => r.id))).toEqual(new Set(ids)) + }) + + it('where {level: N} matches through the filter and the egress guard', async () => { + const ids = await addProbeRows() + const hit = await brain.find({ where: { level: 9 } }) + expect(hit).toHaveLength(1) + expect(hit[0].id).toBe(ids[1]) + expect(hit[0].metadata?.level).toBe(9) + }) + + it('the index column carries ONLY the user value (no 0 poison)', async () => { + const ids = await addProbeRows() + const metadataIndex = (brain as any).metadataIndex + const value = await metadataIndex.getFieldValueForEntity(ids[1], 'level') + expect(value).toBe(9) + + // Zero must not match anything — pre-fix every entity carried a phantom 0. + const phantom = await brain.find({ where: { level: 0 } }) + expect(phantom).toHaveLength(0) + }) + + it('update() keeps level readable (the update indexing view is clean)', async () => { + const ids = await addProbeRows() + await brain.update({ id: ids[0], metadata: { level: 12 } }) + const desc = await brain.find({ + type: NounType.Person, + subtype: 'probe-char', + orderBy: 'level', + order: 'desc', + limit: 100 + }) + expect(desc.map((r: any) => r.metadata?.level)).toEqual([12, 9, 6]) + }) + + it('transact() metadata-only update never rewrites the noun record', async () => { + const ids = await addProbeRows() + const storage = (brain as any).storage + const saveNounSpy = vi.spyOn(storage, 'saveNoun') + + await brain.transact([ + { op: 'update', id: ids[0], metadata: { level: 4 } }, + { op: 'update', id: ids[2], metadata: { level: 7 } } + ]) + + expect(saveNounSpy).not.toHaveBeenCalled() + saveNounSpy.mockRestore() + + const after = await brain.get(ids[0], { includeVectors: true }) + expect(after?.metadata?.level).toBe(4) + expect(Array.isArray(after?.vector) && after!.vector!.length).toBe(384) + }) + + it('this build runs index epoch 2 (the paired level-indexability rebuild)', () => { + expect(EXPECTED_INDEX_EPOCH).toBe(2) + }) +}) diff --git a/tests/integration/orderby-sort-bug.test.ts b/tests/integration/orderby-sort-bug.test.ts index 9c28b1c9..db40fe12 100644 --- a/tests/integration/orderby-sort-bug.test.ts +++ b/tests/integration/orderby-sort-bug.test.ts @@ -215,7 +215,6 @@ describe('resolveEntityField helper', () => { 'id', 'vector', 'connections', - 'level', 'type', 'confidence', 'weight', @@ -228,5 +227,9 @@ describe('resolveEntityField helper', () => { for (const field of expected) { expect(STANDARD_ENTITY_FIELDS.has(field)).toBe(true) } + // `level` is deliberately NOT resolvable: it is HNSW plumbing, and listing + // it here shadowed user metadata named `level` in every by-name read + // (the reserved-name shadow bug). Plumbing stays out of the resolver. + expect(STANDARD_ENTITY_FIELDS.has('level')).toBe(false) }) }) diff --git a/tests/unit/brainy/migration-deference.test.ts b/tests/unit/brainy/migration-deference.test.ts index 6471b4ef..f03bba9c 100644 --- a/tests/unit/brainy/migration-deference.test.ts +++ b/tests/unit/brainy/migration-deference.test.ts @@ -245,7 +245,9 @@ describe('rc.8 no-freeze migration deference (isMigrating / stampBrainFormat / b it('the brain-format marker module exports the compiled epoch + data-format constants', () => { // cor imports these from '@soulcraft/brainy/brain-format' (Hook 3) so both // sides share ONE source of truth — no duplicated constant to drift. - expect(EXPECTED_INDEX_EPOCH).toBe(1) + // Epoch 2: user metadata named `level` became indexable (the reserved-name + // shadow fix, 2026-08-03) — pre-fix brains rebuild derived indexes at open. + expect(EXPECTED_INDEX_EPOCH).toBe(2) expect(CURRENT_DATA_FORMAT).toBe('8.0') }) }) From 8c956608741f7a7d64ab01a6c9109c37660a72b9 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 3 Aug 2026 12:04:40 -0700 Subject: [PATCH 6/7] docs: dedupe the 8.10.2 release-notes entry the cherry doubled onto the branch --- RELEASES.md | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index 03d2d427..af31160c 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -31,30 +31,6 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the --- -## v8.10.2 — 2026-07-29 (metadata-only updates stop rewriting the vector record) - -From a production incident on a large deployment: a read-heavy sweep that bumped -per-entity stats (metadata-only `update()` calls) saturated the disk — 5.8GB written -in 40 minutes — because every `update()` unconditionally re-persisted the WHOLE noun -record, unchanged vector included, fsynced. - -- **`update()` write granularity fixed at the core.** A metadata-only update (no new - `data`, `vector`, or `type`) now writes the metadata leg and index deltas ONLY — - the vector-bearing noun record is never rewritten. Vector-side writes and HNSW - reindexing still happen exactly when the vector side actually changed. Regression - pins: `tests/integration/update-write-granularity.test.ts`. -- **Consumer guidance:** per-entity stat touches are now cheap, but batch them anyway - (one `transact()` instead of N `update()` calls) — granularity fixes the cost per - touch; batching fixes the count. -- Idle VFS `PathResolver` no longer logs `NaN% hit rate` once a minute (stats log - only on new traffic, at debug level). -- Native graph providers' `graph-lsm-*` storage keys are recognized as system - resources — the per-boot `Unknown key format` warning for them is gone. - -Pairs with the native accelerator's same-day patch release; adopt as one bump. - ---- - ## Unreleased (natural field names stop colliding with engine internals) From a production report: sorting by a user metadata field named `level` silently From 133d0d78d5d152c2aa04877612eaaf4e9dc95fc6 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 3 Aug 2026 12:43:38 -0700 Subject: [PATCH 7/7] chore(release): 8.10.3 --- CHANGELOG.md | 6 ++++++ RELEASES.md | 2 +- package-lock.json | 4 ++-- package.json | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0e47015..7b367663 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [8.10.3](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.2...v8.10.3) (2026-08-03) + +- docs: dedupe the 8.10.2 release-notes entry the cherry doubled onto the branch (8c956608) +- fix: user metadata named 'level' is a real field everywhere — the engine-internal node layer no longer shadows it in sort/filter/aggregation, and the indexing views stop stamping a phantom 0 into its column; index epoch 2 rebuilds existing brains at first open (958a0859) + + ### [8.10.2](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.1...v8.10.2) (2026-07-29) - docs: 8.10.2 consumer release notes — update() write granularity, PathResolver idle-log fix, graph-lsm key recognition (a0123b5b) diff --git a/RELEASES.md b/RELEASES.md index af31160c..ab0e0d58 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -31,7 +31,7 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the --- -## Unreleased (natural field names stop colliding with engine internals) +## v8.10.3 — 2026-08-03 (natural field names stop colliding with engine internals) From a production report: sorting by a user metadata field named `level` silently returned insertion order — the engine's internal HNSW node layer (also called diff --git a/package-lock.json b/package-lock.json index a0fb6f6d..823b2625 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "8.10.2", + "version": "8.10.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "8.10.2", + "version": "8.10.3", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 5d140743..0d8b266b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "8.10.2", + "version": "8.10.3", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js",