Compare commits
4 commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 099579f716 | |||
| 3047ffa75c | |||
| a0123b5b8b | |||
| 5b65eb8289 |
9 changed files with 255 additions and 18 deletions
67
.forgejo/workflows/publish-forge.yml
Normal file
67
.forgejo/workflows/publish-forge.yml
Normal file
|
|
@ -0,0 +1,67 @@
|
||||||
|
name: Publish (forge)
|
||||||
|
|
||||||
|
# Datacenter-side forge publish, moved off the laptop: an 87MB tarball PUT
|
||||||
|
# over the laptop's WAN times out; the forge's own runner does it in seconds.
|
||||||
|
# scripts/release.sh tags + pushes, then polls this workflow's result (npm
|
||||||
|
# view against the forge registry) before it ever touches the npmjs leg —
|
||||||
|
# see the "delegation contract" in scripts/release.sh's forge-publish step.
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- 'v*'
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
publish:
|
||||||
|
name: Publish to the forge registry
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: '22'
|
||||||
|
cache: npm
|
||||||
|
- run: npm ci
|
||||||
|
- run: npm run build
|
||||||
|
- name: Publish + readback-verify on the forge registry
|
||||||
|
env:
|
||||||
|
FORGE_NPM_TOKEN: ${{ secrets.FORGE_NPM_TOKEN }}
|
||||||
|
run: |
|
||||||
|
set -eo pipefail
|
||||||
|
|
||||||
|
FORGE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/"
|
||||||
|
VERSION="$(node -p "require('./package.json').version")"
|
||||||
|
echo "Publishing @soulcraft/brainy@${VERSION} to the forge registry..."
|
||||||
|
|
||||||
|
TMPRC="$(mktemp)"
|
||||||
|
chmod 600 "$TMPRC"
|
||||||
|
{
|
||||||
|
echo "@soulcraft:registry=${FORGE_NPM_REG}"
|
||||||
|
echo "//source.soulcraft.com/api/packages/soulcraft/npm/:_authToken=${FORGE_NPM_TOKEN}"
|
||||||
|
} > "$TMPRC"
|
||||||
|
|
||||||
|
# The release script bumps package.json's version before it tags, so
|
||||||
|
# this tag's checkout already carries the version being published —
|
||||||
|
# nothing here re-derives it from the tag name.
|
||||||
|
PUBLISH_OK=true
|
||||||
|
if ! npm publish --tag latest --userconfig "$TMPRC"; then
|
||||||
|
PUBLISH_OK=false
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Readback verify is the source of truth, run regardless of the publish
|
||||||
|
# exit code: a benign duplicate publish (a prior run, or a mirror, already
|
||||||
|
# landed this exact version) reports failure even though the registry
|
||||||
|
# already holds the right content.
|
||||||
|
LANDED_VERSION="$(npm view "@soulcraft/brainy@${VERSION}" version --userconfig "$TMPRC" 2>/dev/null || echo "")"
|
||||||
|
rm -f "$TMPRC"
|
||||||
|
|
||||||
|
if [ "$LANDED_VERSION" != "$VERSION" ]; then
|
||||||
|
echo "::error::Readback verify FAILED — the forge registry reports version '${LANDED_VERSION:-<none>}', expected '${VERSION}'. This is a genuine publish failure, not a benign duplicate."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$PUBLISH_OK" = true ]; then
|
||||||
|
echo "Published and verified @soulcraft/brainy@${VERSION} on the forge registry."
|
||||||
|
else
|
||||||
|
echo "::warning::npm publish reported failure, but readback confirms @soulcraft/brainy@${VERSION} is already live on the forge (a prior run or mirror landed it) — treating this run as successful, since the registry content is correct. Any OTHER failure mode would have failed the readback check above instead."
|
||||||
|
fi
|
||||||
|
|
@ -2,6 +2,12 @@
|
||||||
|
|
||||||
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
||||||
|
|
||||||
|
### [8.10.2](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.1...v8.10.2) (2026-07-29)
|
||||||
|
|
||||||
|
- docs: 8.10.2 consumer release notes — update() write granularity, PathResolver idle-log fix, graph-lsm key recognition (a0123b5b)
|
||||||
|
- fix: metadata-only update() never rewrites the noun record — the unconditional whole-vector save turned per-entity stat touches into full rewrites+fsync, amplifying read-heavy sweeps into disk saturation on a production deployment (5b65eb82)
|
||||||
|
|
||||||
|
|
||||||
### [8.10.1](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.0...v8.10.1) (2026-07-24)
|
### [8.10.1](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.0...v8.10.1) (2026-07-24)
|
||||||
|
|
||||||
- refactor: remove the orphaned transaction-result type left behind by the dead-path removal (edf123a5)
|
- refactor: remove the orphaned transaction-result type left behind by the dead-path removal (edf123a5)
|
||||||
|
|
|
||||||
24
RELEASES.md
24
RELEASES.md
|
|
@ -31,6 +31,30 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
## v8.10.2 — 2026-07-29 (metadata-only updates stop rewriting the vector record)
|
||||||
|
|
||||||
|
From a production incident on a large deployment: a read-heavy sweep that bumped
|
||||||
|
per-entity stats (metadata-only `update()` calls) saturated the disk — 5.8GB written
|
||||||
|
in 40 minutes — because every `update()` unconditionally re-persisted the WHOLE noun
|
||||||
|
record, unchanged vector included, fsynced.
|
||||||
|
|
||||||
|
- **`update()` write granularity fixed at the core.** A metadata-only update (no new
|
||||||
|
`data`, `vector`, or `type`) now writes the metadata leg and index deltas ONLY —
|
||||||
|
the vector-bearing noun record is never rewritten. Vector-side writes and HNSW
|
||||||
|
reindexing still happen exactly when the vector side actually changed. Regression
|
||||||
|
pins: `tests/integration/update-write-granularity.test.ts`.
|
||||||
|
- **Consumer guidance:** per-entity stat touches are now cheap, but batch them anyway
|
||||||
|
(one `transact()` instead of N `update()` calls) — granularity fixes the cost per
|
||||||
|
touch; batching fixes the count.
|
||||||
|
- Idle VFS `PathResolver` no longer logs `NaN% hit rate` once a minute (stats log
|
||||||
|
only on new traffic, at debug level).
|
||||||
|
- Native graph providers' `graph-lsm-*` storage keys are recognized as system
|
||||||
|
resources — the per-boot `Unknown key format` warning for them is gone.
|
||||||
|
|
||||||
|
Pairs with the native accelerator's same-day patch release; adopt as one bump.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## v8.10.1 — 2026-07-24 (the no-hot-retry contract + warm()'s metadata surface under native providers)
|
## v8.10.1 — 2026-07-24 (the no-hot-retry contract + warm()'s metadata surface under native providers)
|
||||||
|
|
||||||
From a production incident: a native-provider op ground 38-40s inside a transaction,
|
From a production incident: a native-provider op ground 38-40s inside a transaction,
|
||||||
|
|
|
||||||
4
package-lock.json
generated
4
package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
||||||
{
|
{
|
||||||
"name": "@soulcraft/brainy",
|
"name": "@soulcraft/brainy",
|
||||||
"version": "8.10.1",
|
"version": "8.10.2",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "@soulcraft/brainy",
|
"name": "@soulcraft/brainy",
|
||||||
"version": "8.10.1",
|
"version": "8.10.2",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@msgpack/msgpack": "^3.1.2",
|
"@msgpack/msgpack": "^3.1.2",
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "@soulcraft/brainy",
|
"name": "@soulcraft/brainy",
|
||||||
"version": "8.10.1",
|
"version": "8.10.2",
|
||||||
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
|
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"module": "dist/index.js",
|
"module": "dist/index.js",
|
||||||
|
|
|
||||||
|
|
@ -3161,7 +3161,15 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
new UpdateNounMetadataOperation(this.storage, params.id, updatedMetadata)
|
new UpdateNounMetadataOperation(this.storage, params.id, updatedMetadata)
|
||||||
)
|
)
|
||||||
|
|
||||||
// Operation 2: Update vector data (will use updated type cache)
|
// Operations 2-4: vector-record write + HNSW reindex — ONLY when the
|
||||||
|
// vector side actually changed (new data/vector/type). A metadata-only
|
||||||
|
// update must never rewrite the noun record: the record carries the
|
||||||
|
// full vector, so an unconditional save turned every metadata touch
|
||||||
|
// into a whole-vector rewrite + fsync — under a read-heavy consumer
|
||||||
|
// sweep that bumps per-entity stats, this amplified into disk
|
||||||
|
// saturation on a production deployment (SELF-ENGINE-RESTART-GRIND,
|
||||||
|
// 2026-07-29: 5.8GB written in 40min from ~50 recalls/min).
|
||||||
|
if (needsReindexing) {
|
||||||
tx.addOperation(
|
tx.addOperation(
|
||||||
new SaveNounOperation(this.storage, {
|
new SaveNounOperation(this.storage, {
|
||||||
id: params.id,
|
id: params.id,
|
||||||
|
|
@ -3170,9 +3178,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
level: 0
|
level: 0
|
||||||
})
|
})
|
||||||
)
|
)
|
||||||
|
|
||||||
// Operation 3-4: Update HNSW index (remove and re-add if reindexing needed)
|
|
||||||
if (needsReindexing) {
|
|
||||||
tx.addOperation(
|
tx.addOperation(
|
||||||
new RemoveFromVectorIndexOperation(this.index, params.id, existing.vector)
|
new RemoveFromVectorIndexOperation(this.index, params.id, existing.vector)
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -382,6 +382,10 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
// identical to the unknown-key fallback these keys hit
|
// identical to the unknown-key fallback these keys hit
|
||||||
// before being listed here — this only kills the
|
// before being listed here — this only kills the
|
||||||
// per-boot "Unknown key format" warning)
|
// per-boot "Unknown key format" warning)
|
||||||
|
id.startsWith('graph-lsm-') || // Graph-LSM store manifests written through storage by
|
||||||
|
// an active native graph provider — same
|
||||||
|
// warn-then-route fallback as above; listing the family
|
||||||
|
// silences the per-boot warning on provider-backed brains
|
||||||
isSingletonSystemKey(id) // Known singletons (e.g. brainy:entityIdMapper) hit the
|
isSingletonSystemKey(id) // Known singletons (e.g. brainy:entityIdMapper) hit the
|
||||||
// same warn-then-route fallback without this — the
|
// same warn-then-route fallback without this — the
|
||||||
// routing below already handles them identically
|
// routing below already handles them identically
|
||||||
|
|
|
||||||
|
|
@ -57,6 +57,7 @@ export class PathResolver {
|
||||||
// Statistics
|
// Statistics
|
||||||
private cacheHits = 0
|
private cacheHits = 0
|
||||||
private cacheMisses = 0
|
private cacheMisses = 0
|
||||||
|
private lastLoggedLookups = 0 // last total the maintenance tick logged stats at
|
||||||
private metadataIndexHits = 0
|
private metadataIndexHits = 0
|
||||||
private metadataIndexMisses = 0
|
private metadataIndexMisses = 0
|
||||||
private graphTraversalFallbacks = 0
|
private graphTraversalFallbacks = 0
|
||||||
|
|
@ -519,10 +520,14 @@ export class PathResolver {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Log cache statistics (in production, send to monitoring)
|
// Log cache statistics only when there is new traffic to report — an
|
||||||
const hitRate = this.cacheHits / (this.cacheHits + this.cacheMisses)
|
// idle resolver stays silent. 0/0 lookups previously rendered
|
||||||
if ((this.cacheHits + this.cacheMisses) % 1000 === 0) {
|
// "NaN% hit rate" (and the %1000 gate passes at zero), which spammed
|
||||||
console.log(`[PathResolver] Cache stats: ${Math.round(hitRate * 100)}% hit rate, ${this.pathCache.size} entries, ${this.hotPaths.size} hot paths`)
|
// production journals once a minute on every idle VFS.
|
||||||
|
const totalLookups = this.cacheHits + this.cacheMisses
|
||||||
|
if (totalLookups > 0 && totalLookups !== this.lastLoggedLookups && totalLookups % 1000 === 0) {
|
||||||
|
this.lastLoggedLookups = totalLookups
|
||||||
|
prodLog.debug(`[PathResolver] Cache stats: ${Math.round((this.cacheHits / totalLookups) * 100)}% hit rate, ${this.pathCache.size} entries, ${this.hotPaths.size} hot paths`)
|
||||||
}
|
}
|
||||||
}, 60000) // Every minute
|
}, 60000) // Every minute
|
||||||
// Cache maintenance must never keep the host process alive.
|
// Cache maintenance must never keep the host process alive.
|
||||||
|
|
|
||||||
126
tests/integration/update-write-granularity.test.ts
Normal file
126
tests/integration/update-write-granularity.test.ts
Normal file
|
|
@ -0,0 +1,126 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/update-write-granularity
|
||||||
|
* @description Write-granularity law for update() (SELF-ENGINE-RESTART-GRIND,
|
||||||
|
* 2026-07-29): a metadata-only update must NEVER rewrite the noun record —
|
||||||
|
* the record carries the full vector, so an unconditional save turns every
|
||||||
|
* metadata touch into a whole-vector rewrite + fsync. Under a read-heavy
|
||||||
|
* consumer sweep bumping per-entity stats this amplified into disk saturation
|
||||||
|
* on a production deployment. Laws:
|
||||||
|
* (1) metadata-only update() → zero saveNoun calls (metadata leg only);
|
||||||
|
* (2) data/vector/type-changing update() → saveNoun runs (the vector leg and
|
||||||
|
* HNSW reindex still happen when the vector side actually changed);
|
||||||
|
* (3) the metadata-only path still lands: merged metadata readable, _rev
|
||||||
|
* bumped, find() by the new field sees the entity.
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||||
|
import { Brainy } from '../../src/brainy.js'
|
||||||
|
import { NounType } from '../../src/types/graphTypes.js'
|
||||||
|
|
||||||
|
const stubEmbedding = async (text: string): Promise<number[]> => {
|
||||||
|
const hash = text.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0)
|
||||||
|
return new Array(384).fill(0).map((_, i) => Math.sin(hash + i))
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('update() write granularity', () => {
|
||||||
|
let brain: Brainy
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
brain = new Brainy({
|
||||||
|
requireSubtype: false,
|
||||||
|
storage: { type: 'memory' as const },
|
||||||
|
embeddingFunction: stubEmbedding
|
||||||
|
})
|
||||||
|
await brain.init()
|
||||||
|
})
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('metadata-only update never rewrites the noun record (no vector rewrite)', async () => {
|
||||||
|
const id = await brain.add({
|
||||||
|
data: 'granularity law subject',
|
||||||
|
type: NounType.Concept,
|
||||||
|
metadata: { touched: 0 }
|
||||||
|
})
|
||||||
|
|
||||||
|
const storage = (brain as any).storage
|
||||||
|
const saveNounSpy = vi.spyOn(storage, 'saveNoun')
|
||||||
|
|
||||||
|
await brain.update({ id, metadata: { touched: 1 } })
|
||||||
|
|
||||||
|
expect(saveNounSpy).not.toHaveBeenCalled()
|
||||||
|
saveNounSpy.mockRestore()
|
||||||
|
|
||||||
|
// The metadata leg still landed with full semantics.
|
||||||
|
const after = await brain.get(id, { includeVectors: true })
|
||||||
|
expect(after?.metadata?.touched).toBe(1)
|
||||||
|
expect(after?._rev).toBe(2)
|
||||||
|
expect(Array.isArray(after?.vector) && after!.vector!.length).toBe(384)
|
||||||
|
|
||||||
|
const found = await brain.find({ where: { touched: 1 } })
|
||||||
|
expect(found.some((r: any) => r.id === id)).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('confidence/weight/subtype-only updates also skip the noun record', async () => {
|
||||||
|
const id = await brain.add({
|
||||||
|
data: 'reserved-field touch subject',
|
||||||
|
type: NounType.Concept,
|
||||||
|
metadata: {}
|
||||||
|
})
|
||||||
|
|
||||||
|
const storage = (brain as any).storage
|
||||||
|
const saveNounSpy = vi.spyOn(storage, 'saveNoun')
|
||||||
|
|
||||||
|
await brain.update({ id, confidence: 0.5, weight: 2, subtype: 'note' })
|
||||||
|
|
||||||
|
expect(saveNounSpy).not.toHaveBeenCalled()
|
||||||
|
saveNounSpy.mockRestore()
|
||||||
|
|
||||||
|
const after = await brain.get(id)
|
||||||
|
expect(after?.confidence).toBe(0.5)
|
||||||
|
expect(after?.subtype).toBe('note')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('data-changing update still writes the noun record and reindexes', async () => {
|
||||||
|
const id = await brain.add({
|
||||||
|
data: 'original embedded text',
|
||||||
|
type: NounType.Concept,
|
||||||
|
metadata: {}
|
||||||
|
})
|
||||||
|
|
||||||
|
const before = await brain.get(id, { includeVectors: true })
|
||||||
|
|
||||||
|
const storage = (brain as any).storage
|
||||||
|
const saveNounSpy = vi.spyOn(storage, 'saveNoun')
|
||||||
|
|
||||||
|
await brain.update({ id, data: 'completely different embedded text' })
|
||||||
|
|
||||||
|
expect(saveNounSpy).toHaveBeenCalled()
|
||||||
|
saveNounSpy.mockRestore()
|
||||||
|
|
||||||
|
const after = await brain.get(id, { includeVectors: true })
|
||||||
|
expect(after?.data).toBe('completely different embedded text')
|
||||||
|
expect(after?.vector).not.toEqual(before?.vector)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('explicit-vector update still writes the noun record', async () => {
|
||||||
|
const id = await brain.add({
|
||||||
|
data: 'vector swap subject',
|
||||||
|
type: NounType.Concept,
|
||||||
|
metadata: {}
|
||||||
|
})
|
||||||
|
|
||||||
|
const storage = (brain as any).storage
|
||||||
|
const saveNounSpy = vi.spyOn(storage, 'saveNoun')
|
||||||
|
|
||||||
|
const newVector = new Array(384).fill(0).map((_, i) => Math.cos(i))
|
||||||
|
await brain.update({ id, vector: newVector })
|
||||||
|
|
||||||
|
expect(saveNounSpy).toHaveBeenCalled()
|
||||||
|
saveNounSpy.mockRestore()
|
||||||
|
|
||||||
|
const after = await brain.get(id, { includeVectors: true })
|
||||||
|
expect(after?.vector?.[0]).toBeCloseTo(1) // cos(0)
|
||||||
|
})
|
||||||
|
})
|
||||||
Loading…
Add table
Add a link
Reference in a new issue