From 5b65eb82896fcf3abf2b557408ed8039b29728cd Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 29 Jul 2026 10:42:50 -0700 Subject: [PATCH 1/4] =?UTF-8?q?fix:=20metadata-only=20update()=20never=20r?= =?UTF-8?q?ewrites=20the=20noun=20record=20=E2=80=94=20the=20unconditional?= =?UTF-8?q?=20whole-vector=20save=20turned=20per-entity=20stat=20touches?= =?UTF-8?q?=20into=20full=20rewrites+fsync,=20amplifying=20read-heavy=20sw?= =?UTF-8?q?eeps=20into=20disk=20saturation=20on=20a=20production=20deploym?= =?UTF-8?q?ent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also: idle PathResolver stats tick no longer logs NaN% every minute (logs only on new traffic, via prodLog); graph-lsm-* key family recognized as system resources (kills the per-boot unknown-key warning on provider-backed brains). Four regression pins in tests/integration/update-write-granularity. --- src/brainy.ts | 27 ++-- src/storage/baseStorage.ts | 4 + src/vfs/PathResolver.ts | 13 +- .../update-write-granularity.test.ts | 126 ++++++++++++++++++ 4 files changed, 155 insertions(+), 15 deletions(-) create mode 100644 tests/integration/update-write-granularity.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 007cdb32..1b5321e9 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -3161,18 +3161,23 @@ export class Brainy implements BrainyInterface { new UpdateNounMetadataOperation(this.storage, params.id, updatedMetadata) ) - // Operation 2: Update vector data (will use updated type cache) - tx.addOperation( - new SaveNounOperation(this.storage, { - id: params.id, - vector, - connections: new Map(), - level: 0 - }) - ) - - // Operation 3-4: Update HNSW index (remove and re-add if reindexing needed) + // Operations 2-4: vector-record write + HNSW reindex — ONLY when the + // vector side actually changed (new data/vector/type). A metadata-only + // update must never rewrite the noun record: the record carries the + // full vector, so an unconditional save turned every metadata touch + // into a whole-vector rewrite + fsync — under a read-heavy consumer + // sweep that bumps per-entity stats, this amplified into disk + // saturation on a production deployment (SELF-ENGINE-RESTART-GRIND, + // 2026-07-29: 5.8GB written in 40min from ~50 recalls/min). if (needsReindexing) { + tx.addOperation( + new SaveNounOperation(this.storage, { + id: params.id, + vector, + connections: new Map(), + level: 0 + }) + ) tx.addOperation( new RemoveFromVectorIndexOperation(this.index, params.id, existing.vector) ) diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index 6daf09c0..1d3e245d 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -382,6 +382,10 @@ export abstract class BaseStorage extends BaseStorageAdapter { // identical to the unknown-key fallback these keys hit // before being listed here — this only kills the // per-boot "Unknown key format" warning) + id.startsWith('graph-lsm-') || // Graph-LSM store manifests written through storage by + // an active native graph provider — same + // warn-then-route fallback as above; listing the family + // silences the per-boot warning on provider-backed brains isSingletonSystemKey(id) // Known singletons (e.g. brainy:entityIdMapper) hit the // same warn-then-route fallback without this — the // routing below already handles them identically diff --git a/src/vfs/PathResolver.ts b/src/vfs/PathResolver.ts index e496c834..502c95f0 100644 --- a/src/vfs/PathResolver.ts +++ b/src/vfs/PathResolver.ts @@ -57,6 +57,7 @@ export class PathResolver { // Statistics private cacheHits = 0 private cacheMisses = 0 + private lastLoggedLookups = 0 // last total the maintenance tick logged stats at private metadataIndexHits = 0 private metadataIndexMisses = 0 private graphTraversalFallbacks = 0 @@ -519,10 +520,14 @@ export class PathResolver { } } - // Log cache statistics (in production, send to monitoring) - const hitRate = this.cacheHits / (this.cacheHits + this.cacheMisses) - if ((this.cacheHits + this.cacheMisses) % 1000 === 0) { - console.log(`[PathResolver] Cache stats: ${Math.round(hitRate * 100)}% hit rate, ${this.pathCache.size} entries, ${this.hotPaths.size} hot paths`) + // Log cache statistics only when there is new traffic to report — an + // idle resolver stays silent. 0/0 lookups previously rendered + // "NaN% hit rate" (and the %1000 gate passes at zero), which spammed + // production journals once a minute on every idle VFS. + const totalLookups = this.cacheHits + this.cacheMisses + if (totalLookups > 0 && totalLookups !== this.lastLoggedLookups && totalLookups % 1000 === 0) { + this.lastLoggedLookups = totalLookups + prodLog.debug(`[PathResolver] Cache stats: ${Math.round((this.cacheHits / totalLookups) * 100)}% hit rate, ${this.pathCache.size} entries, ${this.hotPaths.size} hot paths`) } }, 60000) // Every minute // Cache maintenance must never keep the host process alive. diff --git a/tests/integration/update-write-granularity.test.ts b/tests/integration/update-write-granularity.test.ts new file mode 100644 index 00000000..234df334 --- /dev/null +++ b/tests/integration/update-write-granularity.test.ts @@ -0,0 +1,126 @@ +/** + * @module tests/integration/update-write-granularity + * @description Write-granularity law for update() (SELF-ENGINE-RESTART-GRIND, + * 2026-07-29): a metadata-only update must NEVER rewrite the noun record — + * the record carries the full vector, so an unconditional save turns every + * metadata touch into a whole-vector rewrite + fsync. Under a read-heavy + * consumer sweep bumping per-entity stats this amplified into disk saturation + * on a production deployment. Laws: + * (1) metadata-only update() → zero saveNoun calls (metadata leg only); + * (2) data/vector/type-changing update() → saveNoun runs (the vector leg and + * HNSW reindex still happen when the vector side actually changed); + * (3) the metadata-only path still lands: merged metadata readable, _rev + * bumped, find() by the new field sees the entity. + */ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +const stubEmbedding = async (text: string): Promise => { + const hash = text.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0) + return new Array(384).fill(0).map((_, i) => Math.sin(hash + i)) +} + +describe('update() write granularity', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' as const }, + embeddingFunction: stubEmbedding + }) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + it('metadata-only update never rewrites the noun record (no vector rewrite)', async () => { + const id = await brain.add({ + data: 'granularity law subject', + type: NounType.Concept, + metadata: { touched: 0 } + }) + + const storage = (brain as any).storage + const saveNounSpy = vi.spyOn(storage, 'saveNoun') + + await brain.update({ id, metadata: { touched: 1 } }) + + expect(saveNounSpy).not.toHaveBeenCalled() + saveNounSpy.mockRestore() + + // The metadata leg still landed with full semantics. + const after = await brain.get(id, { includeVectors: true }) + expect(after?.metadata?.touched).toBe(1) + expect(after?._rev).toBe(2) + expect(Array.isArray(after?.vector) && after!.vector!.length).toBe(384) + + const found = await brain.find({ where: { touched: 1 } }) + expect(found.some((r: any) => r.id === id)).toBe(true) + }) + + it('confidence/weight/subtype-only updates also skip the noun record', async () => { + const id = await brain.add({ + data: 'reserved-field touch subject', + type: NounType.Concept, + metadata: {} + }) + + const storage = (brain as any).storage + const saveNounSpy = vi.spyOn(storage, 'saveNoun') + + await brain.update({ id, confidence: 0.5, weight: 2, subtype: 'note' }) + + expect(saveNounSpy).not.toHaveBeenCalled() + saveNounSpy.mockRestore() + + const after = await brain.get(id) + expect(after?.confidence).toBe(0.5) + expect(after?.subtype).toBe('note') + }) + + it('data-changing update still writes the noun record and reindexes', async () => { + const id = await brain.add({ + data: 'original embedded text', + type: NounType.Concept, + metadata: {} + }) + + const before = await brain.get(id, { includeVectors: true }) + + const storage = (brain as any).storage + const saveNounSpy = vi.spyOn(storage, 'saveNoun') + + await brain.update({ id, data: 'completely different embedded text' }) + + expect(saveNounSpy).toHaveBeenCalled() + saveNounSpy.mockRestore() + + const after = await brain.get(id, { includeVectors: true }) + expect(after?.data).toBe('completely different embedded text') + expect(after?.vector).not.toEqual(before?.vector) + }) + + it('explicit-vector update still writes the noun record', async () => { + const id = await brain.add({ + data: 'vector swap subject', + type: NounType.Concept, + metadata: {} + }) + + const storage = (brain as any).storage + const saveNounSpy = vi.spyOn(storage, 'saveNoun') + + const newVector = new Array(384).fill(0).map((_, i) => Math.cos(i)) + await brain.update({ id, vector: newVector }) + + expect(saveNounSpy).toHaveBeenCalled() + saveNounSpy.mockRestore() + + const after = await brain.get(id, { includeVectors: true }) + expect(after?.vector?.[0]).toBeCloseTo(1) // cos(0) + }) +}) From a0123b5b8bdc65f9a3411e5f83e50f4887d77170 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 29 Jul 2026 15:08:16 -0700 Subject: [PATCH 2/4] =?UTF-8?q?docs:=208.10.2=20consumer=20release=20notes?= =?UTF-8?q?=20=E2=80=94=20update()=20write=20granularity,=20PathResolver?= =?UTF-8?q?=20idle-log=20fix,=20graph-lsm=20key=20recognition?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- RELEASES.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/RELEASES.md b/RELEASES.md index 03d7283a..e68bcca1 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -31,6 +31,30 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the --- +## v8.10.2 — 2026-07-29 (metadata-only updates stop rewriting the vector record) + +From a production incident on a large deployment: a read-heavy sweep that bumped +per-entity stats (metadata-only `update()` calls) saturated the disk — 5.8GB written +in 40 minutes — because every `update()` unconditionally re-persisted the WHOLE noun +record, unchanged vector included, fsynced. + +- **`update()` write granularity fixed at the core.** A metadata-only update (no new + `data`, `vector`, or `type`) now writes the metadata leg and index deltas ONLY — + the vector-bearing noun record is never rewritten. Vector-side writes and HNSW + reindexing still happen exactly when the vector side actually changed. Regression + pins: `tests/integration/update-write-granularity.test.ts`. +- **Consumer guidance:** per-entity stat touches are now cheap, but batch them anyway + (one `transact()` instead of N `update()` calls) — granularity fixes the cost per + touch; batching fixes the count. +- Idle VFS `PathResolver` no longer logs `NaN% hit rate` once a minute (stats log + only on new traffic, at debug level). +- Native graph providers' `graph-lsm-*` storage keys are recognized as system + resources — the per-boot `Unknown key format` warning for them is gone. + +Pairs with the native accelerator's same-day patch release; adopt as one bump. + +--- + ## v8.10.1 — 2026-07-24 (the no-hot-retry contract + warm()'s metadata surface under native providers) From a production incident: a native-provider op ground 38-40s inside a transaction, From 3047ffa75cdeb500c72eecc378e4a31ad03b8080 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 29 Jul 2026 15:08:36 -0700 Subject: [PATCH 3/4] chore(release): 8.10.2 --- CHANGELOG.md | 6 ++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5f344cc..e0e47015 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [8.10.2](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.1...v8.10.2) (2026-07-29) + +- docs: 8.10.2 consumer release notes — update() write granularity, PathResolver idle-log fix, graph-lsm key recognition (a0123b5b) +- fix: metadata-only update() never rewrites the noun record — the unconditional whole-vector save turned per-entity stat touches into full rewrites+fsync, amplifying read-heavy sweeps into disk saturation on a production deployment (5b65eb82) + + ### [8.10.1](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.0...v8.10.1) (2026-07-24) - refactor: remove the orphaned transaction-result type left behind by the dead-path removal (edf123a5) diff --git a/package-lock.json b/package-lock.json index d0c7b9d9..a0fb6f6d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "8.10.1", + "version": "8.10.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "8.10.1", + "version": "8.10.2", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index ce670369..5d140743 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "8.10.1", + "version": "8.10.2", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From 099579f716baee5031fc483c33111996fabd78ff Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 29 Jul 2026 15:29:56 -0700 Subject: [PATCH 4/4] =?UTF-8?q?ci:=20carry=20the=20tag-push=20publish=20wo?= =?UTF-8?q?rkflow=20on=20the=20release=20branch=20=E2=80=94=20the=20forge?= =?UTF-8?q?=20discovers=20tag=20workflows=20from=20the=20tag's=20own=20tre?= =?UTF-8?q?e,=20so=20a=20backport=20tag=20must=20contain=20the=20workflow?= =?UTF-8?q?=20itself=20(completes=20the=208.11.0=20maiden-flight=20lesson)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .forgejo/workflows/publish-forge.yml | 67 ++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .forgejo/workflows/publish-forge.yml diff --git a/.forgejo/workflows/publish-forge.yml b/.forgejo/workflows/publish-forge.yml new file mode 100644 index 00000000..fb7428bf --- /dev/null +++ b/.forgejo/workflows/publish-forge.yml @@ -0,0 +1,67 @@ +name: Publish (forge) + +# Datacenter-side forge publish, moved off the laptop: an 87MB tarball PUT +# over the laptop's WAN times out; the forge's own runner does it in seconds. +# scripts/release.sh tags + pushes, then polls this workflow's result (npm +# view against the forge registry) before it ever touches the npmjs leg — +# see the "delegation contract" in scripts/release.sh's forge-publish step. + +on: + push: + tags: + - 'v*' + +jobs: + publish: + name: Publish to the forge registry + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + - run: npm ci + - run: npm run build + - name: Publish + readback-verify on the forge registry + env: + FORGE_NPM_TOKEN: ${{ secrets.FORGE_NPM_TOKEN }} + run: | + set -eo pipefail + + FORGE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/" + VERSION="$(node -p "require('./package.json').version")" + echo "Publishing @soulcraft/brainy@${VERSION} to the forge registry..." + + TMPRC="$(mktemp)" + chmod 600 "$TMPRC" + { + echo "@soulcraft:registry=${FORGE_NPM_REG}" + echo "//source.soulcraft.com/api/packages/soulcraft/npm/:_authToken=${FORGE_NPM_TOKEN}" + } > "$TMPRC" + + # The release script bumps package.json's version before it tags, so + # this tag's checkout already carries the version being published — + # nothing here re-derives it from the tag name. + PUBLISH_OK=true + if ! npm publish --tag latest --userconfig "$TMPRC"; then + PUBLISH_OK=false + fi + + # Readback verify is the source of truth, run regardless of the publish + # exit code: a benign duplicate publish (a prior run, or a mirror, already + # landed this exact version) reports failure even though the registry + # already holds the right content. + LANDED_VERSION="$(npm view "@soulcraft/brainy@${VERSION}" version --userconfig "$TMPRC" 2>/dev/null || echo "")" + rm -f "$TMPRC" + + if [ "$LANDED_VERSION" != "$VERSION" ]; then + echo "::error::Readback verify FAILED — the forge registry reports version '${LANDED_VERSION:-}', expected '${VERSION}'. This is a genuine publish failure, not a benign duplicate." + exit 1 + fi + + if [ "$PUBLISH_OK" = true ]; then + echo "Published and verified @soulcraft/brainy@${VERSION} on the forge registry." + else + echo "::warning::npm publish reported failure, but readback confirms @soulcraft/brainy@${VERSION} is already live on the forge (a prior run or mirror landed it) — treating this run as successful, since the registry content is correct. Any OTHER failure mode would have failed the readback check above instead." + fi