diff --git a/.forgejo/workflows/publish-forge.yml b/.forgejo/workflows/publish-forge.yml deleted file mode 100644 index fb7428bf..00000000 --- a/.forgejo/workflows/publish-forge.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: Publish (forge) - -# Datacenter-side forge publish, moved off the laptop: an 87MB tarball PUT -# over the laptop's WAN times out; the forge's own runner does it in seconds. -# scripts/release.sh tags + pushes, then polls this workflow's result (npm -# view against the forge registry) before it ever touches the npmjs leg — -# see the "delegation contract" in scripts/release.sh's forge-publish step. - -on: - push: - tags: - - 'v*' - -jobs: - publish: - name: Publish to the forge registry - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: '22' - cache: npm - - run: npm ci - - run: npm run build - - name: Publish + readback-verify on the forge registry - env: - FORGE_NPM_TOKEN: ${{ secrets.FORGE_NPM_TOKEN }} - run: | - set -eo pipefail - - FORGE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/" - VERSION="$(node -p "require('./package.json').version")" - echo "Publishing @soulcraft/brainy@${VERSION} to the forge registry..." - - TMPRC="$(mktemp)" - chmod 600 "$TMPRC" - { - echo "@soulcraft:registry=${FORGE_NPM_REG}" - echo "//source.soulcraft.com/api/packages/soulcraft/npm/:_authToken=${FORGE_NPM_TOKEN}" - } > "$TMPRC" - - # The release script bumps package.json's version before it tags, so - # this tag's checkout already carries the version being published — - # nothing here re-derives it from the tag name. - PUBLISH_OK=true - if ! npm publish --tag latest --userconfig "$TMPRC"; then - PUBLISH_OK=false - fi - - # Readback verify is the source of truth, run regardless of the publish - # exit code: a benign duplicate publish (a prior run, or a mirror, already - # landed this exact version) reports failure even though the registry - # already holds the right content. - LANDED_VERSION="$(npm view "@soulcraft/brainy@${VERSION}" version --userconfig "$TMPRC" 2>/dev/null || echo "")" - rm -f "$TMPRC" - - if [ "$LANDED_VERSION" != "$VERSION" ]; then - echo "::error::Readback verify FAILED — the forge registry reports version '${LANDED_VERSION:-}', expected '${VERSION}'. This is a genuine publish failure, not a benign duplicate." - exit 1 - fi - - if [ "$PUBLISH_OK" = true ]; then - echo "Published and verified @soulcraft/brainy@${VERSION} on the forge registry." - else - echo "::warning::npm publish reported failure, but readback confirms @soulcraft/brainy@${VERSION} is already live on the forge (a prior run or mirror landed it) — treating this run as successful, since the registry content is correct. Any OTHER failure mode would have failed the readback check above instead." - fi diff --git a/CHANGELOG.md b/CHANGELOG.md index e0e47015..a5f344cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,6 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. -### [8.10.2](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.1...v8.10.2) (2026-07-29) - -- docs: 8.10.2 consumer release notes — update() write granularity, PathResolver idle-log fix, graph-lsm key recognition (a0123b5b) -- fix: metadata-only update() never rewrites the noun record — the unconditional whole-vector save turned per-entity stat touches into full rewrites+fsync, amplifying read-heavy sweeps into disk saturation on a production deployment (5b65eb82) - - ### [8.10.1](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.0...v8.10.1) (2026-07-24) - refactor: remove the orphaned transaction-result type left behind by the dead-path removal (edf123a5) diff --git a/RELEASES.md b/RELEASES.md index e68bcca1..03d7283a 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -31,30 +31,6 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the --- -## v8.10.2 — 2026-07-29 (metadata-only updates stop rewriting the vector record) - -From a production incident on a large deployment: a read-heavy sweep that bumped -per-entity stats (metadata-only `update()` calls) saturated the disk — 5.8GB written -in 40 minutes — because every `update()` unconditionally re-persisted the WHOLE noun -record, unchanged vector included, fsynced. - -- **`update()` write granularity fixed at the core.** A metadata-only update (no new - `data`, `vector`, or `type`) now writes the metadata leg and index deltas ONLY — - the vector-bearing noun record is never rewritten. Vector-side writes and HNSW - reindexing still happen exactly when the vector side actually changed. Regression - pins: `tests/integration/update-write-granularity.test.ts`. -- **Consumer guidance:** per-entity stat touches are now cheap, but batch them anyway - (one `transact()` instead of N `update()` calls) — granularity fixes the cost per - touch; batching fixes the count. -- Idle VFS `PathResolver` no longer logs `NaN% hit rate` once a minute (stats log - only on new traffic, at debug level). -- Native graph providers' `graph-lsm-*` storage keys are recognized as system - resources — the per-boot `Unknown key format` warning for them is gone. - -Pairs with the native accelerator's same-day patch release; adopt as one bump. - ---- - ## v8.10.1 — 2026-07-24 (the no-hot-retry contract + warm()'s metadata surface under native providers) From a production incident: a native-provider op ground 38-40s inside a transaction, diff --git a/package-lock.json b/package-lock.json index a0fb6f6d..d0c7b9d9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "8.10.2", + "version": "8.10.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "8.10.2", + "version": "8.10.1", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 5d140743..ce670369 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "8.10.2", + "version": "8.10.1", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", diff --git a/src/brainy.ts b/src/brainy.ts index 1b5321e9..007cdb32 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -3161,23 +3161,18 @@ export class Brainy implements BrainyInterface { new UpdateNounMetadataOperation(this.storage, params.id, updatedMetadata) ) - // Operations 2-4: vector-record write + HNSW reindex — ONLY when the - // vector side actually changed (new data/vector/type). A metadata-only - // update must never rewrite the noun record: the record carries the - // full vector, so an unconditional save turned every metadata touch - // into a whole-vector rewrite + fsync — under a read-heavy consumer - // sweep that bumps per-entity stats, this amplified into disk - // saturation on a production deployment (SELF-ENGINE-RESTART-GRIND, - // 2026-07-29: 5.8GB written in 40min from ~50 recalls/min). + // Operation 2: Update vector data (will use updated type cache) + tx.addOperation( + new SaveNounOperation(this.storage, { + id: params.id, + vector, + connections: new Map(), + level: 0 + }) + ) + + // Operation 3-4: Update HNSW index (remove and re-add if reindexing needed) if (needsReindexing) { - tx.addOperation( - new SaveNounOperation(this.storage, { - id: params.id, - vector, - connections: new Map(), - level: 0 - }) - ) tx.addOperation( new RemoveFromVectorIndexOperation(this.index, params.id, existing.vector) ) diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index 1d3e245d..6daf09c0 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -382,10 +382,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { // identical to the unknown-key fallback these keys hit // before being listed here — this only kills the // per-boot "Unknown key format" warning) - id.startsWith('graph-lsm-') || // Graph-LSM store manifests written through storage by - // an active native graph provider — same - // warn-then-route fallback as above; listing the family - // silences the per-boot warning on provider-backed brains isSingletonSystemKey(id) // Known singletons (e.g. brainy:entityIdMapper) hit the // same warn-then-route fallback without this — the // routing below already handles them identically diff --git a/src/vfs/PathResolver.ts b/src/vfs/PathResolver.ts index 502c95f0..e496c834 100644 --- a/src/vfs/PathResolver.ts +++ b/src/vfs/PathResolver.ts @@ -57,7 +57,6 @@ export class PathResolver { // Statistics private cacheHits = 0 private cacheMisses = 0 - private lastLoggedLookups = 0 // last total the maintenance tick logged stats at private metadataIndexHits = 0 private metadataIndexMisses = 0 private graphTraversalFallbacks = 0 @@ -520,14 +519,10 @@ export class PathResolver { } } - // Log cache statistics only when there is new traffic to report — an - // idle resolver stays silent. 0/0 lookups previously rendered - // "NaN% hit rate" (and the %1000 gate passes at zero), which spammed - // production journals once a minute on every idle VFS. - const totalLookups = this.cacheHits + this.cacheMisses - if (totalLookups > 0 && totalLookups !== this.lastLoggedLookups && totalLookups % 1000 === 0) { - this.lastLoggedLookups = totalLookups - prodLog.debug(`[PathResolver] Cache stats: ${Math.round((this.cacheHits / totalLookups) * 100)}% hit rate, ${this.pathCache.size} entries, ${this.hotPaths.size} hot paths`) + // Log cache statistics (in production, send to monitoring) + const hitRate = this.cacheHits / (this.cacheHits + this.cacheMisses) + if ((this.cacheHits + this.cacheMisses) % 1000 === 0) { + console.log(`[PathResolver] Cache stats: ${Math.round(hitRate * 100)}% hit rate, ${this.pathCache.size} entries, ${this.hotPaths.size} hot paths`) } }, 60000) // Every minute // Cache maintenance must never keep the host process alive. diff --git a/tests/integration/update-write-granularity.test.ts b/tests/integration/update-write-granularity.test.ts deleted file mode 100644 index 234df334..00000000 --- a/tests/integration/update-write-granularity.test.ts +++ /dev/null @@ -1,126 +0,0 @@ -/** - * @module tests/integration/update-write-granularity - * @description Write-granularity law for update() (SELF-ENGINE-RESTART-GRIND, - * 2026-07-29): a metadata-only update must NEVER rewrite the noun record — - * the record carries the full vector, so an unconditional save turns every - * metadata touch into a whole-vector rewrite + fsync. Under a read-heavy - * consumer sweep bumping per-entity stats this amplified into disk saturation - * on a production deployment. Laws: - * (1) metadata-only update() → zero saveNoun calls (metadata leg only); - * (2) data/vector/type-changing update() → saveNoun runs (the vector leg and - * HNSW reindex still happen when the vector side actually changed); - * (3) the metadata-only path still lands: merged metadata readable, _rev - * bumped, find() by the new field sees the entity. - */ -import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' - -const stubEmbedding = async (text: string): Promise => { - const hash = text.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0) - return new Array(384).fill(0).map((_, i) => Math.sin(hash + i)) -} - -describe('update() write granularity', () => { - let brain: Brainy - - beforeEach(async () => { - brain = new Brainy({ - requireSubtype: false, - storage: { type: 'memory' as const }, - embeddingFunction: stubEmbedding - }) - await brain.init() - }) - - afterEach(async () => { - await brain.close() - }) - - it('metadata-only update never rewrites the noun record (no vector rewrite)', async () => { - const id = await brain.add({ - data: 'granularity law subject', - type: NounType.Concept, - metadata: { touched: 0 } - }) - - const storage = (brain as any).storage - const saveNounSpy = vi.spyOn(storage, 'saveNoun') - - await brain.update({ id, metadata: { touched: 1 } }) - - expect(saveNounSpy).not.toHaveBeenCalled() - saveNounSpy.mockRestore() - - // The metadata leg still landed with full semantics. - const after = await brain.get(id, { includeVectors: true }) - expect(after?.metadata?.touched).toBe(1) - expect(after?._rev).toBe(2) - expect(Array.isArray(after?.vector) && after!.vector!.length).toBe(384) - - const found = await brain.find({ where: { touched: 1 } }) - expect(found.some((r: any) => r.id === id)).toBe(true) - }) - - it('confidence/weight/subtype-only updates also skip the noun record', async () => { - const id = await brain.add({ - data: 'reserved-field touch subject', - type: NounType.Concept, - metadata: {} - }) - - const storage = (brain as any).storage - const saveNounSpy = vi.spyOn(storage, 'saveNoun') - - await brain.update({ id, confidence: 0.5, weight: 2, subtype: 'note' }) - - expect(saveNounSpy).not.toHaveBeenCalled() - saveNounSpy.mockRestore() - - const after = await brain.get(id) - expect(after?.confidence).toBe(0.5) - expect(after?.subtype).toBe('note') - }) - - it('data-changing update still writes the noun record and reindexes', async () => { - const id = await brain.add({ - data: 'original embedded text', - type: NounType.Concept, - metadata: {} - }) - - const before = await brain.get(id, { includeVectors: true }) - - const storage = (brain as any).storage - const saveNounSpy = vi.spyOn(storage, 'saveNoun') - - await brain.update({ id, data: 'completely different embedded text' }) - - expect(saveNounSpy).toHaveBeenCalled() - saveNounSpy.mockRestore() - - const after = await brain.get(id, { includeVectors: true }) - expect(after?.data).toBe('completely different embedded text') - expect(after?.vector).not.toEqual(before?.vector) - }) - - it('explicit-vector update still writes the noun record', async () => { - const id = await brain.add({ - data: 'vector swap subject', - type: NounType.Concept, - metadata: {} - }) - - const storage = (brain as any).storage - const saveNounSpy = vi.spyOn(storage, 'saveNoun') - - const newVector = new Array(384).fill(0).map((_, i) => Math.cos(i)) - await brain.update({ id, vector: newVector }) - - expect(saveNounSpy).toHaveBeenCalled() - saveNounSpy.mockRestore() - - const after = await brain.get(id, { includeVectors: true }) - expect(after?.vector?.[0]).toBeCloseTo(1) // cos(0) - }) -})