Compare commits
7 commits
main
...
release/8.
| Author | SHA1 | Date | |
|---|---|---|---|
| 133d0d78d5 | |||
| 8c95660874 | |||
| 958a085926 | |||
| 099579f716 | |||
| 3047ffa75c | |||
| a0123b5b8b | |||
| 5b65eb8289 |
15 changed files with 496 additions and 35 deletions
67
.forgejo/workflows/publish-forge.yml
Normal file
67
.forgejo/workflows/publish-forge.yml
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
name: Publish (forge)
|
||||
|
||||
# Datacenter-side forge publish, moved off the laptop: an 87MB tarball PUT
|
||||
# over the laptop's WAN times out; the forge's own runner does it in seconds.
|
||||
# scripts/release.sh tags + pushes, then polls this workflow's result (npm
|
||||
# view against the forge registry) before it ever touches the npmjs leg —
|
||||
# see the "delegation contract" in scripts/release.sh's forge-publish step.
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v*'
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: Publish to the forge registry
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- run: npm run build
|
||||
- name: Publish + readback-verify on the forge registry
|
||||
env:
|
||||
FORGE_NPM_TOKEN: ${{ secrets.FORGE_NPM_TOKEN }}
|
||||
run: |
|
||||
set -eo pipefail
|
||||
|
||||
FORGE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/"
|
||||
VERSION="$(node -p "require('./package.json').version")"
|
||||
echo "Publishing @soulcraft/brainy@${VERSION} to the forge registry..."
|
||||
|
||||
TMPRC="$(mktemp)"
|
||||
chmod 600 "$TMPRC"
|
||||
{
|
||||
echo "@soulcraft:registry=${FORGE_NPM_REG}"
|
||||
echo "//source.soulcraft.com/api/packages/soulcraft/npm/:_authToken=${FORGE_NPM_TOKEN}"
|
||||
} > "$TMPRC"
|
||||
|
||||
# The release script bumps package.json's version before it tags, so
|
||||
# this tag's checkout already carries the version being published —
|
||||
# nothing here re-derives it from the tag name.
|
||||
PUBLISH_OK=true
|
||||
if ! npm publish --tag latest --userconfig "$TMPRC"; then
|
||||
PUBLISH_OK=false
|
||||
fi
|
||||
|
||||
# Readback verify is the source of truth, run regardless of the publish
|
||||
# exit code: a benign duplicate publish (a prior run, or a mirror, already
|
||||
# landed this exact version) reports failure even though the registry
|
||||
# already holds the right content.
|
||||
LANDED_VERSION="$(npm view "@soulcraft/brainy@${VERSION}" version --userconfig "$TMPRC" 2>/dev/null || echo "")"
|
||||
rm -f "$TMPRC"
|
||||
|
||||
if [ "$LANDED_VERSION" != "$VERSION" ]; then
|
||||
echo "::error::Readback verify FAILED — the forge registry reports version '${LANDED_VERSION:-<none>}', expected '${VERSION}'. This is a genuine publish failure, not a benign duplicate."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$PUBLISH_OK" = true ]; then
|
||||
echo "Published and verified @soulcraft/brainy@${VERSION} on the forge registry."
|
||||
else
|
||||
echo "::warning::npm publish reported failure, but readback confirms @soulcraft/brainy@${VERSION} is already live on the forge (a prior run or mirror landed it) — treating this run as successful, since the registry content is correct. Any OTHER failure mode would have failed the readback check above instead."
|
||||
fi
|
||||
12
CHANGELOG.md
12
CHANGELOG.md
|
|
@ -2,6 +2,18 @@
|
|||
|
||||
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
||||
|
||||
### [8.10.3](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.2...v8.10.3) (2026-08-03)
|
||||
|
||||
- docs: dedupe the 8.10.2 release-notes entry the cherry doubled onto the branch (8c956608)
|
||||
- fix: user metadata named 'level' is a real field everywhere — the engine-internal node layer no longer shadows it in sort/filter/aggregation, and the indexing views stop stamping a phantom 0 into its column; index epoch 2 rebuilds existing brains at first open (958a0859)
|
||||
|
||||
|
||||
### [8.10.2](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.1...v8.10.2) (2026-07-29)
|
||||
|
||||
- docs: 8.10.2 consumer release notes — update() write granularity, PathResolver idle-log fix, graph-lsm key recognition (a0123b5b)
|
||||
- fix: metadata-only update() never rewrites the noun record — the unconditional whole-vector save turned per-entity stat touches into full rewrites+fsync, amplifying read-heavy sweeps into disk saturation on a production deployment (5b65eb82)
|
||||
|
||||
|
||||
### [8.10.1](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.0...v8.10.1) (2026-07-24)
|
||||
|
||||
- refactor: remove the orphaned transaction-result type left behind by the dead-path removal (edf123a5)
|
||||
|
|
|
|||
55
RELEASES.md
55
RELEASES.md
|
|
@ -31,6 +31,61 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the
|
|||
|
||||
---
|
||||
|
||||
## v8.10.3 — 2026-08-03 (natural field names stop colliding with engine internals)
|
||||
|
||||
From a production report: sorting by a user metadata field named `level` silently
|
||||
returned insertion order — the engine's internal HNSW node layer (also called
|
||||
`level`) shadowed the user's field in every by-name read, and the indexing path
|
||||
stamped a hardcoded `0` into the same index column (multi-valued poison). `level`
|
||||
is a perfectly natural field name (game characters, priorities, floors); the
|
||||
engine was wrong, not the caller.
|
||||
|
||||
- **`level` is user data now, everywhere.** Engine plumbing no longer resolves by
|
||||
name, never shadows metadata, and never enters the indexed views. `orderBy:
|
||||
'level'`, `where: { level: 9 }`, `groupBy: ['level']` all read YOUR field.
|
||||
Regression pins: `tests/integration/level-field-shadow.test.ts` (the reporting
|
||||
consumer's exact repro rows).
|
||||
- **Index epoch 2.** The derived posting set changed, so every existing brain
|
||||
rebuilds its metadata index from canonical at first open — poisoned columns
|
||||
heal automatically; no manual step. First open after upgrade pays one rebuild
|
||||
(observable via `getIndexStatus()`); pair this release with the same-day
|
||||
native-accelerator release, which makes `level` indexable on the native path.
|
||||
- **`transact()` metadata-only updates stop rewriting the vector record** — the
|
||||
v8.10.2 write-granularity law now covers the batch/plan path too (it was
|
||||
fixed for `update()` but the transact plan builder still staged the
|
||||
unconditional save). If you batch stat touches through `transact()`, this is
|
||||
your write-amplification fix.
|
||||
- Coming next (announced so parsers and call sites can prepare): one
|
||||
field-addressing law — bare names = user metadata, `system.<field>` for
|
||||
engine fields, typed refusals for unresolvable names. Ships as its own
|
||||
release with a migration advisory; nothing changes in this release.
|
||||
|
||||
---
|
||||
|
||||
## v8.10.2 — 2026-07-29 (metadata-only updates stop rewriting the vector record)
|
||||
|
||||
From a production incident on a large deployment: a read-heavy sweep that bumped
|
||||
per-entity stats (metadata-only `update()` calls) saturated the disk — 5.8GB written
|
||||
in 40 minutes — because every `update()` unconditionally re-persisted the WHOLE noun
|
||||
record, unchanged vector included, fsynced.
|
||||
|
||||
- **`update()` write granularity fixed at the core.** A metadata-only update (no new
|
||||
`data`, `vector`, or `type`) now writes the metadata leg and index deltas ONLY —
|
||||
the vector-bearing noun record is never rewritten. Vector-side writes and HNSW
|
||||
reindexing still happen exactly when the vector side actually changed. Regression
|
||||
pins: `tests/integration/update-write-granularity.test.ts`.
|
||||
- **Consumer guidance:** per-entity stat touches are now cheap, but batch them anyway
|
||||
(one `transact()` instead of N `update()` calls) — granularity fixes the cost per
|
||||
touch; batching fixes the count.
|
||||
- Idle VFS `PathResolver` no longer logs `NaN% hit rate` once a minute (stats log
|
||||
only on new traffic, at debug level).
|
||||
- Native graph providers' `graph-lsm-*` storage keys are recognized as system
|
||||
resources — the per-boot `Unknown key format` warning for them is gone.
|
||||
|
||||
Pairs with the native accelerator's same-day patch release; adopt as one bump.
|
||||
|
||||
---
|
||||
|
||||
## v8.10.1 — 2026-07-24 (the no-hot-retry contract + warm()'s metadata surface under native providers)
|
||||
|
||||
From a production incident: a native-provider op ground 38-40s inside a transaction,
|
||||
|
|
|
|||
4
package-lock.json
generated
4
package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"name": "@soulcraft/brainy",
|
||||
"version": "8.10.1",
|
||||
"version": "8.10.3",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@soulcraft/brainy",
|
||||
"version": "8.10.1",
|
||||
"version": "8.10.3",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@msgpack/msgpack": "^3.1.2",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@soulcraft/brainy",
|
||||
"version": "8.10.1",
|
||||
"version": "8.10.3",
|
||||
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
|
||||
"main": "dist/index.js",
|
||||
"module": "dist/index.js",
|
||||
|
|
|
|||
|
|
@ -2123,11 +2123,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// If undefined values are included as explicit keys, extractIndexableFields indexes
|
||||
// them as '__NULL__' entries that removeFromIndex can never clean up (storageMetadata
|
||||
// omits those keys entirely via conditional spreading, so the fields don't match).
|
||||
// No `level` here: engine plumbing never enters the indexing view — a
|
||||
// hardcoded level:0 landed in the SAME flattened index column as user
|
||||
// metadata named `level`, poisoning it multi-valued ([0, real]).
|
||||
const entityForIndexing = {
|
||||
id,
|
||||
vector,
|
||||
connections: new Map(),
|
||||
level: 0,
|
||||
type: params.type,
|
||||
...(params.subtype !== undefined && { subtype: params.subtype }),
|
||||
...(params.visibility !== undefined &&
|
||||
|
|
@ -3102,12 +3104,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
})
|
||||
}
|
||||
|
||||
// Build entity structure for metadata index (with top-level fields)
|
||||
// Build entity structure for metadata index (with top-level fields).
|
||||
// No `level`: engine plumbing never enters the indexing view (it
|
||||
// poisoned the flattened user `level` column — VENUE-BRAINY-ORDERBY-NOOP).
|
||||
const entityForIndexing = {
|
||||
id: params.id,
|
||||
vector,
|
||||
connections: new Map(),
|
||||
level: 0,
|
||||
type: params.type || existing.type,
|
||||
subtype: params.subtype !== undefined ? params.subtype : existing.subtype,
|
||||
...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && {
|
||||
|
|
@ -3161,18 +3164,23 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
new UpdateNounMetadataOperation(this.storage, params.id, updatedMetadata)
|
||||
)
|
||||
|
||||
// Operation 2: Update vector data (will use updated type cache)
|
||||
tx.addOperation(
|
||||
new SaveNounOperation(this.storage, {
|
||||
id: params.id,
|
||||
vector,
|
||||
connections: new Map(),
|
||||
level: 0
|
||||
})
|
||||
)
|
||||
|
||||
// Operation 3-4: Update HNSW index (remove and re-add if reindexing needed)
|
||||
// Operations 2-4: vector-record write + HNSW reindex — ONLY when the
|
||||
// vector side actually changed (new data/vector/type). A metadata-only
|
||||
// update must never rewrite the noun record: the record carries the
|
||||
// full vector, so an unconditional save turned every metadata touch
|
||||
// into a whole-vector rewrite + fsync — under a read-heavy consumer
|
||||
// sweep that bumps per-entity stats, this amplified into disk
|
||||
// saturation on a production deployment (SELF-ENGINE-RESTART-GRIND,
|
||||
// 2026-07-29: 5.8GB written in 40min from ~50 recalls/min).
|
||||
if (needsReindexing) {
|
||||
tx.addOperation(
|
||||
new SaveNounOperation(this.storage, {
|
||||
id: params.id,
|
||||
vector,
|
||||
connections: new Map(),
|
||||
level: 0
|
||||
})
|
||||
)
|
||||
tx.addOperation(
|
||||
new RemoveFromVectorIndexOperation(this.index, params.id, existing.vector)
|
||||
)
|
||||
|
|
@ -9333,7 +9341,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
id,
|
||||
vector,
|
||||
connections: new Map(),
|
||||
level: 0,
|
||||
// no `level` — plumbing never enters the indexing view
|
||||
type: params.type,
|
||||
...(params.subtype !== undefined && { subtype: params.subtype }),
|
||||
...(params.visibility !== undefined &&
|
||||
|
|
@ -9484,7 +9492,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
id: params.id,
|
||||
vector,
|
||||
connections: new Map(),
|
||||
level: 0,
|
||||
// no `level` — plumbing never enters the indexing view
|
||||
type: params.type || existing.type,
|
||||
subtype: params.subtype !== undefined ? params.subtype : existing.subtype,
|
||||
...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && {
|
||||
|
|
@ -9512,16 +9520,22 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
|
||||
plan.operations.push(
|
||||
new UpdateNounMetadataOperation(this.storage, params.id, updatedMetadata),
|
||||
new SaveNounOperation(this.storage, {
|
||||
id: params.id,
|
||||
vector,
|
||||
connections: new Map(),
|
||||
level: 0
|
||||
})
|
||||
new UpdateNounMetadataOperation(this.storage, params.id, updatedMetadata)
|
||||
)
|
||||
// Noun-record write + HNSW reindex ONLY when the vector side actually
|
||||
// changed — the same write-granularity law as update(): a metadata-only
|
||||
// patch must never rewrite the whole vector record. This plan path is the
|
||||
// one transact() updates ride, so an unconditional save here would
|
||||
// re-open the read-sweep disk-saturation amplifier for exactly the
|
||||
// consumers batching their stat touches through transact().
|
||||
if (needsReindexing) {
|
||||
plan.operations.push(
|
||||
new SaveNounOperation(this.storage, {
|
||||
id: params.id,
|
||||
vector,
|
||||
connections: new Map(),
|
||||
level: 0
|
||||
}),
|
||||
new RemoveFromVectorIndexOperation(this.index, params.id, existing.vector),
|
||||
new AddToVectorIndexOperation(this.index, params.id, vector)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -284,7 +284,12 @@ export const STANDARD_ENTITY_FIELDS: ReadonlySet<string> = new Set([
|
|||
'id',
|
||||
'vector',
|
||||
'connections',
|
||||
'level',
|
||||
// 'level' is deliberately ABSENT: it is HNSW plumbing, not an entity field.
|
||||
// Listing it here made every by-name read of a user metadata field called
|
||||
// `level` resolve to the engine's internal node layer instead — a silent
|
||||
// shadow that broke sort/filter/aggregation on a perfectly natural field
|
||||
// name (VENUE-BRAINY-ORDERBY-NOOP). Engine plumbing is invisible to the
|
||||
// query surface; a bare `level` reads `entity.metadata.level`.
|
||||
'type',
|
||||
'subtype',
|
||||
'visibility',
|
||||
|
|
|
|||
|
|
@ -382,6 +382,10 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
// identical to the unknown-key fallback these keys hit
|
||||
// before being listed here — this only kills the
|
||||
// per-boot "Unknown key format" warning)
|
||||
id.startsWith('graph-lsm-') || // Graph-LSM store manifests written through storage by
|
||||
// an active native graph provider — same
|
||||
// warn-then-route fallback as above; listing the family
|
||||
// silences the per-boot warning on provider-backed brains
|
||||
isSingletonSystemKey(id) // Known singletons (e.g. brainy:entityIdMapper) hit the
|
||||
// same warn-then-route fallback without this — the
|
||||
// routing below already handles them identically
|
||||
|
|
|
|||
|
|
@ -69,7 +69,12 @@ export const BRAIN_FORMAT_PATH = '_system/brain-format.json'
|
|||
* (the 8.0 GA baseline). An on-disk `indexEpoch` that differs from this — or an
|
||||
* absent marker — triggers a full derived-index rebuild on open.
|
||||
*/
|
||||
export const EXPECTED_INDEX_EPOCH = 1
|
||||
// Epoch 2 (2026-08-03, paired with the native accelerator's same-day release):
|
||||
// user metadata fields named `level` become indexable on both engines — the
|
||||
// derived posting set changed, so every pre-fix brain must rebuild its
|
||||
// metadata index from canonical at first open (poisoned multi-valued `level`
|
||||
// columns heal through this rebuild; no bespoke heal path).
|
||||
export const EXPECTED_INDEX_EPOCH = 2
|
||||
|
||||
/**
|
||||
* @description The data-layer format string this build writes and runs as.
|
||||
|
|
|
|||
|
|
@ -551,7 +551,23 @@ export interface FindParams<T = any> {
|
|||
cursor?: string // Cursor-based pagination
|
||||
|
||||
// Sorting
|
||||
orderBy?: string // Field to sort by (e.g., 'createdAt', 'title', 'metadata.priority')
|
||||
/**
|
||||
* Field to sort by. User metadata fields sort by their stored values —
|
||||
* including natural names like `level`, `rank`, or `score` (an engine-internal
|
||||
* field can never shadow your metadata; fixed 2026-08 after a production
|
||||
* report). System timestamps (`createdAt`, `updatedAt`) sort by entity age.
|
||||
*
|
||||
* Ordering contract (identical on the pure-JS engine and the native
|
||||
* accelerator): entities missing the field sort LAST in both directions —
|
||||
* they are never dropped from the result; ties break deterministically.
|
||||
*
|
||||
* NOTE — the field-addressing law is changing (announced 2026-08): bare
|
||||
* names will mean user metadata ALWAYS, and system fields will be reached
|
||||
* explicitly as `system.<field>` (e.g. `system.createdAt`), with typed
|
||||
* refusals for unresolvable names. Until that release, bare `createdAt`
|
||||
* and friends keep resolving to the system fields as documented above.
|
||||
*/
|
||||
orderBy?: string
|
||||
order?: 'asc' | 'desc' // Sort direction: 'asc' (default) or 'desc'
|
||||
|
||||
// Advanced options
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ export class PathResolver {
|
|||
// Statistics
|
||||
private cacheHits = 0
|
||||
private cacheMisses = 0
|
||||
private lastLoggedLookups = 0 // last total the maintenance tick logged stats at
|
||||
private metadataIndexHits = 0
|
||||
private metadataIndexMisses = 0
|
||||
private graphTraversalFallbacks = 0
|
||||
|
|
@ -519,10 +520,14 @@ export class PathResolver {
|
|||
}
|
||||
}
|
||||
|
||||
// Log cache statistics (in production, send to monitoring)
|
||||
const hitRate = this.cacheHits / (this.cacheHits + this.cacheMisses)
|
||||
if ((this.cacheHits + this.cacheMisses) % 1000 === 0) {
|
||||
console.log(`[PathResolver] Cache stats: ${Math.round(hitRate * 100)}% hit rate, ${this.pathCache.size} entries, ${this.hotPaths.size} hot paths`)
|
||||
// Log cache statistics only when there is new traffic to report — an
|
||||
// idle resolver stays silent. 0/0 lookups previously rendered
|
||||
// "NaN% hit rate" (and the %1000 gate passes at zero), which spammed
|
||||
// production journals once a minute on every idle VFS.
|
||||
const totalLookups = this.cacheHits + this.cacheMisses
|
||||
if (totalLookups > 0 && totalLookups !== this.lastLoggedLookups && totalLookups % 1000 === 0) {
|
||||
this.lastLoggedLookups = totalLookups
|
||||
prodLog.debug(`[PathResolver] Cache stats: ${Math.round((this.cacheHits / totalLookups) * 100)}% hit rate, ${this.pathCache.size} entries, ${this.hotPaths.size} hot paths`)
|
||||
}
|
||||
}, 60000) // Every minute
|
||||
// Cache maintenance must never keep the host process alive.
|
||||
|
|
|
|||
147
tests/integration/level-field-shadow.test.ts
Normal file
147
tests/integration/level-field-shadow.test.ts
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
/**
|
||||
* @module tests/integration/level-field-shadow
|
||||
* @description The reserved-name shadow fix (VENUE-BRAINY-ORDERBY-NOOP,
|
||||
* 2026-08-03): `level` is HNSW plumbing, not an entity field — it must never
|
||||
* shadow user metadata of the same name. Pre-fix, STANDARD_ENTITY_FIELDS
|
||||
* listed `level`, so every by-name read returned the engine's internal 0
|
||||
* (all-equal → stable sort → insertion order, silently), and the indexing
|
||||
* views stamped level:0 into the same flattened column as user values
|
||||
* (multi-valued [0, real] poison). Laws:
|
||||
* (1) venue's exact repro sorts: three adds with metadata.level 3/9/6 →
|
||||
* find({orderBy:'level'}) returns 9,6,3 desc and 3,6,9 asc;
|
||||
* (2) where {level: N} matches through filter AND egress guard;
|
||||
* (3) the index column carries the user value only (no 0 poison);
|
||||
* (4) update() keeps `level` readable (the update indexing view is clean too);
|
||||
* (5) the transact() update path never rewrites the noun record on a
|
||||
* metadata-only patch (the planUpdate granularity completion).
|
||||
*/
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { Brainy } from '../../src/brainy.js'
|
||||
import { NounType } from '../../src/types/graphTypes.js'
|
||||
import { EXPECTED_INDEX_EPOCH } from '../../src/storage/brainFormat.js'
|
||||
|
||||
const stubEmbedding = async (text: string): Promise<number[]> => {
|
||||
const hash = text.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0)
|
||||
return new Array(384).fill(0).map((_, i) => Math.sin(hash + i))
|
||||
}
|
||||
|
||||
describe('level field shadow — user metadata named level is a real field', () => {
|
||||
let brain: Brainy
|
||||
|
||||
beforeEach(async () => {
|
||||
brain = new Brainy({
|
||||
requireSubtype: false,
|
||||
storage: { type: 'memory' as const },
|
||||
embeddingFunction: stubEmbedding
|
||||
})
|
||||
await brain.init()
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await brain.close()
|
||||
})
|
||||
|
||||
async function addProbeRows(): Promise<string[]> {
|
||||
const ids: string[] = []
|
||||
for (const level of [3, 9, 6]) {
|
||||
ids.push(
|
||||
await brain.add({
|
||||
data: `probe character level ${level}`,
|
||||
type: NounType.Person,
|
||||
subtype: 'probe-char',
|
||||
metadata: { name: `char-${level}`, level }
|
||||
})
|
||||
)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
it("venue's exact repro: orderBy 'level' sorts desc and asc", async () => {
|
||||
await addProbeRows()
|
||||
|
||||
const desc = await brain.find({
|
||||
type: NounType.Person,
|
||||
subtype: 'probe-char',
|
||||
orderBy: 'level',
|
||||
order: 'desc',
|
||||
limit: 100
|
||||
})
|
||||
expect(desc.map((r: any) => r.metadata?.level)).toEqual([9, 6, 3])
|
||||
|
||||
const asc = await brain.find({
|
||||
type: NounType.Person,
|
||||
subtype: 'probe-char',
|
||||
orderBy: 'level',
|
||||
order: 'asc',
|
||||
limit: 100
|
||||
})
|
||||
expect(asc.map((r: any) => r.metadata?.level)).toEqual([3, 6, 9])
|
||||
})
|
||||
|
||||
it('ordered reads are COMPLETE — no row dropped (the 2-of-3 face)', async () => {
|
||||
const ids = await addProbeRows()
|
||||
const desc = await brain.find({
|
||||
type: NounType.Person,
|
||||
subtype: 'probe-char',
|
||||
orderBy: 'level',
|
||||
order: 'desc',
|
||||
limit: 100
|
||||
})
|
||||
expect(desc).toHaveLength(3)
|
||||
expect(new Set(desc.map((r: any) => r.id))).toEqual(new Set(ids))
|
||||
})
|
||||
|
||||
it('where {level: N} matches through the filter and the egress guard', async () => {
|
||||
const ids = await addProbeRows()
|
||||
const hit = await brain.find({ where: { level: 9 } })
|
||||
expect(hit).toHaveLength(1)
|
||||
expect(hit[0].id).toBe(ids[1])
|
||||
expect(hit[0].metadata?.level).toBe(9)
|
||||
})
|
||||
|
||||
it('the index column carries ONLY the user value (no 0 poison)', async () => {
|
||||
const ids = await addProbeRows()
|
||||
const metadataIndex = (brain as any).metadataIndex
|
||||
const value = await metadataIndex.getFieldValueForEntity(ids[1], 'level')
|
||||
expect(value).toBe(9)
|
||||
|
||||
// Zero must not match anything — pre-fix every entity carried a phantom 0.
|
||||
const phantom = await brain.find({ where: { level: 0 } })
|
||||
expect(phantom).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('update() keeps level readable (the update indexing view is clean)', async () => {
|
||||
const ids = await addProbeRows()
|
||||
await brain.update({ id: ids[0], metadata: { level: 12 } })
|
||||
const desc = await brain.find({
|
||||
type: NounType.Person,
|
||||
subtype: 'probe-char',
|
||||
orderBy: 'level',
|
||||
order: 'desc',
|
||||
limit: 100
|
||||
})
|
||||
expect(desc.map((r: any) => r.metadata?.level)).toEqual([12, 9, 6])
|
||||
})
|
||||
|
||||
it('transact() metadata-only update never rewrites the noun record', async () => {
|
||||
const ids = await addProbeRows()
|
||||
const storage = (brain as any).storage
|
||||
const saveNounSpy = vi.spyOn(storage, 'saveNoun')
|
||||
|
||||
await brain.transact([
|
||||
{ op: 'update', id: ids[0], metadata: { level: 4 } },
|
||||
{ op: 'update', id: ids[2], metadata: { level: 7 } }
|
||||
])
|
||||
|
||||
expect(saveNounSpy).not.toHaveBeenCalled()
|
||||
saveNounSpy.mockRestore()
|
||||
|
||||
const after = await brain.get(ids[0], { includeVectors: true })
|
||||
expect(after?.metadata?.level).toBe(4)
|
||||
expect(Array.isArray(after?.vector) && after!.vector!.length).toBe(384)
|
||||
})
|
||||
|
||||
it('this build runs index epoch 2 (the paired level-indexability rebuild)', () => {
|
||||
expect(EXPECTED_INDEX_EPOCH).toBe(2)
|
||||
})
|
||||
})
|
||||
|
|
@ -215,7 +215,6 @@ describe('resolveEntityField helper', () => {
|
|||
'id',
|
||||
'vector',
|
||||
'connections',
|
||||
'level',
|
||||
'type',
|
||||
'confidence',
|
||||
'weight',
|
||||
|
|
@ -228,5 +227,9 @@ describe('resolveEntityField helper', () => {
|
|||
for (const field of expected) {
|
||||
expect(STANDARD_ENTITY_FIELDS.has(field)).toBe(true)
|
||||
}
|
||||
// `level` is deliberately NOT resolvable: it is HNSW plumbing, and listing
|
||||
// it here shadowed user metadata named `level` in every by-name read
|
||||
// (the reserved-name shadow bug). Plumbing stays out of the resolver.
|
||||
expect(STANDARD_ENTITY_FIELDS.has('level')).toBe(false)
|
||||
})
|
||||
})
|
||||
|
|
|
|||
126
tests/integration/update-write-granularity.test.ts
Normal file
126
tests/integration/update-write-granularity.test.ts
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
/**
|
||||
* @module tests/integration/update-write-granularity
|
||||
* @description Write-granularity law for update() (SELF-ENGINE-RESTART-GRIND,
|
||||
* 2026-07-29): a metadata-only update must NEVER rewrite the noun record —
|
||||
* the record carries the full vector, so an unconditional save turns every
|
||||
* metadata touch into a whole-vector rewrite + fsync. Under a read-heavy
|
||||
* consumer sweep bumping per-entity stats this amplified into disk saturation
|
||||
* on a production deployment. Laws:
|
||||
* (1) metadata-only update() → zero saveNoun calls (metadata leg only);
|
||||
* (2) data/vector/type-changing update() → saveNoun runs (the vector leg and
|
||||
* HNSW reindex still happen when the vector side actually changed);
|
||||
* (3) the metadata-only path still lands: merged metadata readable, _rev
|
||||
* bumped, find() by the new field sees the entity.
|
||||
*/
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { Brainy } from '../../src/brainy.js'
|
||||
import { NounType } from '../../src/types/graphTypes.js'
|
||||
|
||||
const stubEmbedding = async (text: string): Promise<number[]> => {
|
||||
const hash = text.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0)
|
||||
return new Array(384).fill(0).map((_, i) => Math.sin(hash + i))
|
||||
}
|
||||
|
||||
describe('update() write granularity', () => {
|
||||
let brain: Brainy
|
||||
|
||||
beforeEach(async () => {
|
||||
brain = new Brainy({
|
||||
requireSubtype: false,
|
||||
storage: { type: 'memory' as const },
|
||||
embeddingFunction: stubEmbedding
|
||||
})
|
||||
await brain.init()
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await brain.close()
|
||||
})
|
||||
|
||||
it('metadata-only update never rewrites the noun record (no vector rewrite)', async () => {
|
||||
const id = await brain.add({
|
||||
data: 'granularity law subject',
|
||||
type: NounType.Concept,
|
||||
metadata: { touched: 0 }
|
||||
})
|
||||
|
||||
const storage = (brain as any).storage
|
||||
const saveNounSpy = vi.spyOn(storage, 'saveNoun')
|
||||
|
||||
await brain.update({ id, metadata: { touched: 1 } })
|
||||
|
||||
expect(saveNounSpy).not.toHaveBeenCalled()
|
||||
saveNounSpy.mockRestore()
|
||||
|
||||
// The metadata leg still landed with full semantics.
|
||||
const after = await brain.get(id, { includeVectors: true })
|
||||
expect(after?.metadata?.touched).toBe(1)
|
||||
expect(after?._rev).toBe(2)
|
||||
expect(Array.isArray(after?.vector) && after!.vector!.length).toBe(384)
|
||||
|
||||
const found = await brain.find({ where: { touched: 1 } })
|
||||
expect(found.some((r: any) => r.id === id)).toBe(true)
|
||||
})
|
||||
|
||||
it('confidence/weight/subtype-only updates also skip the noun record', async () => {
|
||||
const id = await brain.add({
|
||||
data: 'reserved-field touch subject',
|
||||
type: NounType.Concept,
|
||||
metadata: {}
|
||||
})
|
||||
|
||||
const storage = (brain as any).storage
|
||||
const saveNounSpy = vi.spyOn(storage, 'saveNoun')
|
||||
|
||||
await brain.update({ id, confidence: 0.5, weight: 2, subtype: 'note' })
|
||||
|
||||
expect(saveNounSpy).not.toHaveBeenCalled()
|
||||
saveNounSpy.mockRestore()
|
||||
|
||||
const after = await brain.get(id)
|
||||
expect(after?.confidence).toBe(0.5)
|
||||
expect(after?.subtype).toBe('note')
|
||||
})
|
||||
|
||||
it('data-changing update still writes the noun record and reindexes', async () => {
|
||||
const id = await brain.add({
|
||||
data: 'original embedded text',
|
||||
type: NounType.Concept,
|
||||
metadata: {}
|
||||
})
|
||||
|
||||
const before = await brain.get(id, { includeVectors: true })
|
||||
|
||||
const storage = (brain as any).storage
|
||||
const saveNounSpy = vi.spyOn(storage, 'saveNoun')
|
||||
|
||||
await brain.update({ id, data: 'completely different embedded text' })
|
||||
|
||||
expect(saveNounSpy).toHaveBeenCalled()
|
||||
saveNounSpy.mockRestore()
|
||||
|
||||
const after = await brain.get(id, { includeVectors: true })
|
||||
expect(after?.data).toBe('completely different embedded text')
|
||||
expect(after?.vector).not.toEqual(before?.vector)
|
||||
})
|
||||
|
||||
it('explicit-vector update still writes the noun record', async () => {
|
||||
const id = await brain.add({
|
||||
data: 'vector swap subject',
|
||||
type: NounType.Concept,
|
||||
metadata: {}
|
||||
})
|
||||
|
||||
const storage = (brain as any).storage
|
||||
const saveNounSpy = vi.spyOn(storage, 'saveNoun')
|
||||
|
||||
const newVector = new Array(384).fill(0).map((_, i) => Math.cos(i))
|
||||
await brain.update({ id, vector: newVector })
|
||||
|
||||
expect(saveNounSpy).toHaveBeenCalled()
|
||||
saveNounSpy.mockRestore()
|
||||
|
||||
const after = await brain.get(id, { includeVectors: true })
|
||||
expect(after?.vector?.[0]).toBeCloseTo(1) // cos(0)
|
||||
})
|
||||
})
|
||||
|
|
@ -245,7 +245,9 @@ describe('rc.8 no-freeze migration deference (isMigrating / stampBrainFormat / b
|
|||
it('the brain-format marker module exports the compiled epoch + data-format constants', () => {
|
||||
// cor imports these from '@soulcraft/brainy/brain-format' (Hook 3) so both
|
||||
// sides share ONE source of truth — no duplicated constant to drift.
|
||||
expect(EXPECTED_INDEX_EPOCH).toBe(1)
|
||||
// Epoch 2: user metadata named `level` became indexable (the reserved-name
|
||||
// shadow fix, 2026-08-03) — pre-fix brains rebuild derived indexes at open.
|
||||
expect(EXPECTED_INDEX_EPOCH).toBe(2)
|
||||
expect(CURRENT_DATA_FORMAT).toBe('8.0')
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue