Compare commits

..

No commits in common. "main" and "v10.4.0" have entirely different histories.

129 changed files with 732 additions and 9688 deletions

View file

@ -2,7 +2,7 @@
## What Is Brainy ## What Is Brainy
@soulcraftlabs/brainy (v7.17.0) is a Universal Knowledge Protocol -- a Triple Intelligence database combining vector search, graph traversal, and metadata filtering in a single library. Published to npm as a public MIT-licensed package. @soulcraft/brainy (v7.17.0) is a Universal Knowledge Protocol -- a Triple Intelligence database combining vector search, graph traversal, and metadata filtering in a single library. Published to npm as a public MIT-licensed package.
## Core Architecture ## Core Architecture

View file

@ -5,10 +5,6 @@ name: CI
# sequential, so tag-triggered matrix jobs (~22 min) would queue AHEAD of the # sequential, so tag-triggered matrix jobs (~22 min) would queue AHEAD of the
# tag's publish-source run and starve every release (observed on 8.10.3 and # tag's publish-source run and starve every release (observed on 8.10.3 and
# 9.0.0: the publish sat behind the tag's own redundant CI). # 9.0.0: the publish sat behind the tag's own redundant CI).
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: true
on: on:
push: push:
branches: ['**'] branches: ['**']

View file

@ -12,11 +12,6 @@ on:
push: push:
tags: tags:
- 'v*' - 'v*'
workflow_dispatch:
inputs:
ref_reason:
description: 'why this manual run (e.g. tag event dropped)'
required: false
jobs: jobs:
publish: publish:
@ -37,7 +32,7 @@ jobs:
run: | run: |
set -eo pipefail set -eo pipefail
SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraftlabs/npm/" SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/"
VERSION="$(node -p "require('./package.json').version")" VERSION="$(node -p "require('./package.json').version")"
# The dist-tag follows the version: a prerelease (any hyphen — # The dist-tag follows the version: a prerelease (any hyphen —
# 10.4.0-rc.1) publishes under 'rc' and must NEVER move 'latest' — # 10.4.0-rc.1) publishes under 'rc' and must NEVER move 'latest' —
@ -48,13 +43,13 @@ jobs:
case "$VERSION" in case "$VERSION" in
*-*) NPM_TAG="rc" ;; *-*) NPM_TAG="rc" ;;
esac esac
echo "Publishing @soulcraftlabs/brainy@${VERSION} to The Source registry (dist-tag: ${NPM_TAG})..." echo "Publishing @soulcraft/brainy@${VERSION} to The Source registry (dist-tag: ${NPM_TAG})..."
TMPRC="$(mktemp)" TMPRC="$(mktemp)"
chmod 600 "$TMPRC" chmod 600 "$TMPRC"
{ {
echo "@soulcraftlabs:registry=${SOURCE_NPM_REG}" echo "@soulcraft:registry=${SOURCE_NPM_REG}"
echo "//source.soulcraft.com/api/packages/soulcraftlabs/npm/:_authToken=${FORGE_NPM_TOKEN}" echo "//source.soulcraft.com/api/packages/soulcraft/npm/:_authToken=${FORGE_NPM_TOKEN}"
} > "$TMPRC" } > "$TMPRC"
# The release script bumps package.json's version before it tags, so # The release script bumps package.json's version before it tags, so
@ -69,7 +64,7 @@ jobs:
# exit code: a benign duplicate publish (a prior run, or a mirror, already # exit code: a benign duplicate publish (a prior run, or a mirror, already
# landed this exact version) reports failure even though the registry # landed this exact version) reports failure even though the registry
# already holds the right content. # already holds the right content.
LANDED_VERSION="$(npm view "@soulcraftlabs/brainy@${VERSION}" version --userconfig "$TMPRC" 2>/dev/null || echo "")" LANDED_VERSION="$(npm view "@soulcraft/brainy@${VERSION}" version --userconfig "$TMPRC" 2>/dev/null || echo "")"
rm -f "$TMPRC" rm -f "$TMPRC"
if [ "$LANDED_VERSION" != "$VERSION" ]; then if [ "$LANDED_VERSION" != "$VERSION" ]; then
@ -78,7 +73,7 @@ jobs:
fi fi
if [ "$PUBLISH_OK" = true ]; then if [ "$PUBLISH_OK" = true ]; then
echo "Published and verified @soulcraftlabs/brainy@${VERSION} on The Source registry." echo "Published and verified @soulcraft/brainy@${VERSION} on The Source registry."
else else
echo "::warning::npm publish reported failure, but readback confirms @soulcraftlabs/brainy@${VERSION} is already live on The Source (a prior run or mirror landed it) — treating this run as successful, since the registry content is correct. Any OTHER failure mode would have failed the readback check above instead." echo "::warning::npm publish reported failure, but readback confirms @soulcraft/brainy@${VERSION} is already live on The Source (a prior run or mirror landed it) — treating this run as successful, since the registry content is correct. Any OTHER failure mode would have failed the readback check above instead."
fi fi

View file

@ -2,81 +2,17 @@
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
### [10.4.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.3...v10.4.4) (2026-08-28) ### [10.4.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.4...v10.4.0) (2026-08-26)
- fix(vfs): the old-root sweep narrates only when it has something to say (d49148e1)
- fix(tests): the health-gate pin follows the verdict, and the VFS suite uses its own store (42e2da25)
- Merge branch 'next/open-lazy-open-and-counts' (5ebd3b40)
- docs: the contract manifest stands alone; public docs describe this engine only (a8c724a2)
- docs(releases): 10.4.4 consumer notes — correctness and observability, with the performance line stated exactly (61a46927)
- docs: measurements in public history carry numbers, not provenance (02c61636)
- feat(open): name the two steps that hold the vfs-bootstrap phase (2cf38010)
- fix(storage): a dead flush watch falls back to the 500ms poll, not the 30s sweep (5c22f950)
- fix(storage): the flush watcher cannot arm twice in its async window (16d2e1a9)
- perf(idle): the flush-request watch is event-driven; the heartbeat is observability (fb1da1c5)
- perf(open): answer "are there any entities?" with one directory read (417ddb51)
- perf(generations): discover generations by directory name, not by walking the log (9dd39921)
- fix(flush): clear() and repairIndex() set the dirty witness themselves (e4c27fbc)
- feat(open): the open names the STEP that cost the time, not just the phase (5a091cca)
- perf(vfs): the old-root sweep runs once per store, not once per open (4a67aa0f)
- chore: keep the generated neural stamps at main's values (c1f09723)
- feat(contract): declare contract 1, serve three operators, refuse four by name (48802ba3)
- fix(open): a provider rebuilding itself is a third state, not a CRITICAL (50676c02)
- feat(open): open never waits for a provider that is rebuilding itself (131daa08)
- perf(flush): an idle brain does no work — no periodic flush without a write (f5a6cb3f)
- feat(repair): repairIndex narrates every phase and its receipt carries the walls (3fffd9c6)
- fix(storage): a suspect count ledger heals itself, and counts.json is written atomically (f4e2d34b)
- feat(open): the open narrates itself, on a channel production cannot clamp (afe08a1f)
- fix(storage): a clean close is recorded, and the writer lock is always given up (e652162c)
- docs: repository links point at soulcraftlabs/open-brainy — the soulcraft/brainy path becomes the native engine's repo tonight (38c3397b)
### [10.4.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.2...v10.4.3) (2026-08-27)
- Merge branch 'next/open-brainy-rename' (a58372f0)
- chore: rename to @soulcraftlabs/brainy for Open Brainy on The Source (a99b1e83)
- docs(releases): 10.4.3 — Open Brainy's first release under the new name, same engine as 10.4.2; The Source is the one registry (9f248b24)
### [10.4.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.2-rc.1...v10.4.2) (2026-08-27)
- docs(releases): 10.4.1 and 10.4.2 consumer notes; 10.4.2 is the last MIT release under this name, Open Brainy continues at @soulcraftlabs/brainy (a082e0ef)
### [10.4.2-rc.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.1...v10.4.2-rc.1) (2026-08-27)
- Merge branch 'next/zero-norm-unvector-door' (9b84ef5b)
- fix(vectors): a zero-norm vector is not a vector, canonical side included, plus the sanctioned unvector door (0de76659)
- fix(hnsw): skip unvectored rows on rebuild; refuse empty vectors in the index (8fc553b1)
- fix(storage): derive the canonical count ledger from identity records, stamp the derivation rule, and mark legacy-derived ledgers suspect at load (fd6b4ce4)
- Merge branch 'next/enumeration-identity-rekey' (204d74c1)
- fix(storage): enumeration re-keys on the identity record, not the vector leg (f8d8ce16)
- fix(init): rethrow plugin activation failures with the original error as cause so the originating frame survives to the caller (2496e09a)
- Merge branch 'next/vfs-root-zero-norm' (4c7b0fab)
- fix(vfs): the VFS root never persists a zero-norm vector (c6cc0de9)
- build: derive generated-file stamps from git commit time, not wall clock (8a5c1245)
- Merge remote-tracking branch 'origin/release/10.4.1' (aad9e2ee)
- docs(concepts): the serving law — a failure is graded by whether an answer could be wrong, never by the cost of the fix; reads refuse per family (2914e0eb)
- chore(release): 10.4.1-rc.1 (7870dc40)
### [10.4.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0...v10.4.1) (2026-08-26)
- fix(reads): the read gate is per-family; a write carrying unchanged data never re-embeds (c039411e)
- docs(guide): the docs pipeline publishes through the ingest API — the separate deploy step is retired (21e506e8)
### [10.4.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.4...v10.4.0) (2026-08-26)
- docs(releases): the 10.4.0 entry catches up to the late trains — repair routing, the vector ledger and open-gate leg, the loud config guard, the JSON-safe crossing (834149ed) - docs(releases): the 10.4.0 entry catches up to the late trains — repair routing, the vector ledger and open-gate leg, the loud config guard, the JSON-safe crossing (834149ed)
### [10.4.0-rc.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.3...v10.4.0-rc.4) (2026-08-25) ### [10.4.0-rc.4](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.3...v10.4.0-rc.4) (2026-08-25)
- feat(vector): the vectored-noun scalar joins the count ledger; the open gate closes the vector leg (9730835b) - feat(vector): the vectored-noun scalar joins the count ledger; the open gate closes the vector leg (9730835b)
### [10.4.0-rc.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.2...v10.4.0-rc.3) (2026-08-25) ### [10.4.0-rc.3](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.2...v10.4.0-rc.3) (2026-08-25)
- fix(update-seam): the metadata crossing never carries BigInt endpoint ints (f4780c8e) - fix(update-seam): the metadata crossing never carries BigInt endpoint ints (f4780c8e)
- Merge branch 'worktree-agent-ad3aff0dffd17a6eb' (f14da34b) - Merge branch 'worktree-agent-ad3aff0dffd17a6eb' (f14da34b)
@ -85,7 +21,7 @@ All notable changes to this project will be documented in this file. See [standa
- feat(open-path): init never gates on the embedding model; open goes concurrent; slow opens narrate (96624f40) - feat(open-path): init never gates on the embedding model; open goes concurrent; slow opens narrate (96624f40)
### [10.4.0-rc.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.1...v10.4.0-rc.2) (2026-08-25) ### [10.4.0-rc.2](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.1...v10.4.0-rc.2) (2026-08-25)
- test(readiness): the report helper's clock freezes — two independently-built reports compared across a millisecond tick made the plant lane red (39b916a3) - test(readiness): the report helper's clock freezes — two independently-built reports compared across a millisecond tick made the plant lane red (39b916a3)
- feat(repair): a heal:'repair' verdict routes to the provider's own incremental repair() (553e0d97) - feat(repair): a heal:'repair' verdict routes to the provider's own incremental repair() (553e0d97)
@ -96,7 +32,7 @@ All notable changes to this project will be documented in this file. See [standa
- feat(health): the gate reads the named report — reads refuse loudly, never rebuild; open serves before it returns; the ceremony door (f8f64780) - feat(health): the gate reads the named report — reads refuse loudly, never rebuild; open serves before it returns; the ceremony door (f8f64780)
### [10.4.0-rc.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.3.1...v10.4.0-rc.1) (2026-08-24) ### [10.4.0-rc.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.3.1...v10.4.0-rc.1) (2026-08-24)
- ci(publish): the home dist-tag follows the version — a prerelease publishes under 'rc' and never moves 'latest' (a1376e4a) - ci(publish): the home dist-tag follows the version — a prerelease publishes under 'rc' and never moves 'latest' (a1376e4a)
- chore(release): --source-only — a home-only prerelease mode (The Source, never the storefront) (dcbad176) - chore(release): --source-only — a home-only prerelease mode (The Source, never the storefront) (dcbad176)
@ -109,13 +45,13 @@ All notable changes to this project will be documented in this file. See [standa
- ci(gate): the machine-health preflight and the truncation verdict guard (1e046aa1) - ci(gate): the machine-health preflight and the truncation verdict guard (1e046aa1)
### [10.3.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.3.0...v10.3.1) (2026-08-18) ### [10.3.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.3.0...v10.3.1) (2026-08-18)
- docs(releases): the 10.3.1 consumer entry — the fold that behaves (900cc895) - docs(releases): the 10.3.1 consumer entry — the fold that behaves (900cc895)
- fix(recovery): the fold streams and narrates; the checkpoint chain arms at the flip (ed7d1db9) - fix(recovery): the fold streams and narrates; the checkpoint chain arms at the flip (ed7d1db9)
### [10.3.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.2.0...v10.3.0) (2026-08-18) ### [10.3.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.2.0...v10.3.0) (2026-08-18)
- docs(releases): the 10.3.0 consumer entry — the trust-and-provenance release (97d75649) - docs(releases): the 10.3.0 consumer entry — the trust-and-provenance release (97d75649)
- fix(locks): the fence keys ownership on pid+hostname — a same-process re-open never fences its predecessor (0991cf28) - fix(locks): the fence keys ownership on pid+hostname — a same-process re-open never fences its predecessor (0991cf28)
@ -124,14 +60,14 @@ All notable changes to this project will be documented in this file. See [standa
- feat(log): system commits carry their origin; the attested per-id reconcile door (9ac9e706) - feat(log): system commits carry their origin; the attested per-id reconcile door (9ac9e706)
### [10.2.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.1.0...v10.2.0) (2026-08-17) ### [10.2.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.1.0...v10.2.0) (2026-08-17)
- docs(releases): the 10.2.0 consumer entry — adoption completes in one call (97538e1f) - docs(releases): the 10.2.0 consumer entry — adoption completes in one call (97538e1f)
- ci: the correctness plant runs integration + conformance on every push — a release never waits on a second machine (b17fdc8e) - ci: the correctness plant runs integration + conformance on every push — a release never waits on a second machine (b17fdc8e)
- fix(adoption): the baseline backfill runs to completion — one call adopts a pre-log baseline of any size (a5a18838) - fix(adoption): the baseline backfill runs to completion — one call adopts a pre-log baseline of any size (a5a18838)
### [10.1.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.0.0...v10.1.0) (2026-08-13) ### [10.1.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.0.0...v10.1.0) (2026-08-13)
- docs(releases): the 10.1.0 consumer entry — bounded recovery, restore founding, the two write-path cures (7d3c8696) - docs(releases): the 10.1.0 consumer entry — bounded recovery, restore founding, the two write-path cures (7d3c8696)
- fix(restore): a restore is an unclean event — the swap runs quiesced and the snapshot's durability stamps never survive it (9ca80667) - fix(restore): a restore is an unclean event — the swap runs quiesced and the snapshot's durability stamps never survive it (9ca80667)
@ -140,7 +76,7 @@ All notable changes to this project will be documented in this file. See [standa
- feat(query): the sparse-store cut — where on a never-carried field serves operator truth, never a refusal (7b67db4d) - feat(query): the sparse-store cut — where on a never-carried field serves operator truth, never a refusal (7b67db4d)
### [10.0.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v9.0.0...v10.0.0) (2026-08-12) ### [10.0.0](https://source.soulcraft.com/soulcraft/brainy/compare/v9.0.0...v10.0.0) (2026-08-12)
- fix(adoption): the baseline backfill cures hydration-law drift — existing brains reach the crash-safe default with zero operator steps (25f0dd96) - fix(adoption): the baseline backfill cures hydration-law drift — existing brains reach the crash-safe default with zero operator steps (25f0dd96)
- fix(adoption): the reserved-root mint exemption — int 0 is legitimate for exactly one id (2abe8b38) - fix(adoption): the reserved-root mint exemption — int 0 is legitimate for exactly one id (2abe8b38)
@ -172,7 +108,7 @@ All notable changes to this project will be documented in this file. See [standa
- test: version-coupling pins go major-agnostic — the 8.x literals broke at the 9.0.0 bump while the coupling law itself behaved correctly (8a6807e8) - test: version-coupling pins go major-agnostic — the 8.x literals broke at the 9.0.0 bump while the coupling law itself behaved correctly (8a6807e8)
### [9.0.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.11.0...v9.0.0) (2026-08-04) ### [9.0.0](https://source.soulcraft.com/soulcraft/brainy/compare/v8.11.0...v9.0.0) (2026-08-04)
- docs: 9.0 namespace-migration guide — the simple story + the mechanical sweep checklist, published for humans and tooling alike (61ab9db2) - docs: 9.0 namespace-migration guide — the simple story + the mechanical sweep checklist, published for humans and tooling alike (61ab9db2)
- fix(release): storefront leg republishes CI's exact forge artifact — byte-identity by construction, verified by cross-registry shasum before the ceremony reports success (d89df2ed) - fix(release): storefront leg republishes CI's exact forge artifact — byte-identity by construction, verified by cross-registry shasum before the ceremony reports success (d89df2ed)
@ -207,7 +143,7 @@ All notable changes to this project will be documented in this file. See [standa
- feat: scanFacts liveness contract — first batch or loud failure within a documented bound (f8e6da2b) - feat: scanFacts liveness contract — first batch or loud failure within a documented bound (f8e6da2b)
### [8.11.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.1...v8.11.0) (2026-07-27) ### [8.11.0](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.1...v8.11.0) (2026-07-27)
- docs: the last two archived-host links point home (91ef1c8b) - docs: the last two archived-host links point home (91ef1c8b)
- feat: includeHidden — export carries every visibility tier for migration-grade canon completeness (63c1eeb9) - feat: includeHidden — export carries every visibility tier for migration-grade canon completeness (63c1eeb9)
@ -216,19 +152,19 @@ All notable changes to this project will be documented in this file. See [standa
- ci: run the pipeline on the forge (999d0ebb) - ci: run the pipeline on the forge (999d0ebb)
### [8.10.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.2...v8.10.3) (2026-08-03) ### [8.10.3](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.2...v8.10.3) (2026-08-03)
- docs: dedupe the 8.10.2 release-notes entry the cherry doubled onto the branch (8c956608) - docs: dedupe the 8.10.2 release-notes entry the cherry doubled onto the branch (8c956608)
- fix: user metadata named 'level' is a real field everywhere — the engine-internal node layer no longer shadows it in sort/filter/aggregation, and the indexing views stop stamping a phantom 0 into its column; index epoch 2 rebuilds existing brains at first open (958a0859) - fix: user metadata named 'level' is a real field everywhere — the engine-internal node layer no longer shadows it in sort/filter/aggregation, and the indexing views stop stamping a phantom 0 into its column; index epoch 2 rebuilds existing brains at first open (958a0859)
### [8.10.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.1...v8.10.2) (2026-07-29) ### [8.10.2](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.1...v8.10.2) (2026-07-29)
- docs: 8.10.2 consumer release notes — update() write granularity, PathResolver idle-log fix, graph-lsm key recognition (a0123b5b) - docs: 8.10.2 consumer release notes — update() write granularity, PathResolver idle-log fix, graph-lsm key recognition (a0123b5b)
- fix: metadata-only update() never rewrites the noun record — the unconditional whole-vector save turned per-entity stat touches into full rewrites+fsync, amplifying read-heavy sweeps into disk saturation on a production deployment (5b65eb82) - fix: metadata-only update() never rewrites the noun record — the unconditional whole-vector save turned per-entity stat touches into full rewrites+fsync, amplifying read-heavy sweeps into disk saturation on a production deployment (5b65eb82)
### [8.10.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.0...v8.10.1) (2026-07-24) ### [8.10.1](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.0...v8.10.1) (2026-07-24)
- refactor: remove the orphaned transaction-result type left behind by the dead-path removal (edf123a5) - refactor: remove the orphaned transaction-result type left behind by the dead-path removal (edf123a5)
- fix: warm() metadata surface routes through the active provider (warm hook added to the metadata contract); add maintenanceDebt() observability surface (5b2cbf74) - fix: warm() metadata surface routes through the active provider (warm hook added to the metadata contract); add maintenanceDebt() observability surface (5b2cbf74)

View file

@ -12,13 +12,13 @@ Handoff file: `/home/dpsifr/.strategy/PLATFORM-HANDOFF.md`
**Brainy's current open actions:** None. MIT open-source — no platform-specific actions. **Brainy's current open actions:** None. MIT open-source — no platform-specific actions.
**Current version:** run `npm view @soulcraftlabs/brainy version --registry https://source.soulcraft.com/api/packages/soulcraftlabs/npm/` (never trust a hardcoded number here — this line went stale for months); consumer-facing changes tracked in `RELEASES.md` **Current version:** run `npm view @soulcraft/brainy version` (never trust a hardcoded number here — this line went stale for months); consumer-facing changes tracked in `RELEASES.md`
--- ---
## Project Overview ## Project Overview
Brainy is a Universal Knowledge Protocol -- a Triple Intelligence database that combines vector similarity search, graph traversal, and metadata filtering into a single TypeScript library. Published as `@soulcraftlabs/brainy` on The Source (source.soulcraft.com registry) under the MIT license. Brainy is a Universal Knowledge Protocol -- a Triple Intelligence database that combines vector similarity search, graph traversal, and metadata filtering into a single TypeScript library. Published as `@soulcraft/brainy` on npm under the MIT license.
## Getting Started ## Getting Started
@ -91,7 +91,7 @@ test: add/update tests (patch version bump)
## Docs Pipeline — soulcraft.com/docs ## Docs Pipeline — soulcraft.com/docs
Docs in `docs/**/*.md` are published with the npm package (included in `files`) and go live on soulcraft.com/docs via the docs ingest API: the release script's `scripts/push-docs.js` step POSTs every public doc to `https://soulcraft.com/api/docs/ingest` (auth: `DOCS_INGEST_SECRET` in the environment). No separate deploy step is involved (the old deploy-to-publish flow was retired in a platform change, 2026-08). Frontmatter controls what appears publicly. Docs in `docs/**/*.md` are published with the npm package (included in `files`) and synced to soulcraft.com/docs on every portal deploy. Frontmatter controls what appears publicly.
### Docs check triggers ### Docs check triggers
@ -161,9 +161,9 @@ npm run release:major # Breaking changes (rare, manual decision)
The script: verifies clean git state, builds, tests, bumps version, updates CHANGELOG.md, commits, tags, pushes, publishes to npm, and creates a GitHub release. The script: verifies clean git state, builds, tests, bumps version, updates CHANGELOG.md, commits, tags, pushes, publishes to npm, and creates a GitHub release.
After a successful release, remind the user: After a successful release, remind the user:
> "Published. Docs are live on soulcraft.com/docs (pushed via the ingest API during the release) — spot-check a changed page with curl." > "Published. Deploy portal to pick up the new docs → go to the portal project and deploy."
There is no separate deploy step anymore. If the docs push failed (the script warns loudly), re-run `node scripts/push-docs.js` with `DOCS_INGEST_SECRET` set. Do NOT deploy portal from here. Portal is always deployed separately from within the portal project.
## Closed-Source Product Names — HARD RULE ## Closed-Source Product Names — HARD RULE

View file

@ -6,7 +6,7 @@ may find elsewhere in the repo's history.
## Where the project lives ## Where the project lives
The source of truth is a self-hosted forge: **source.soulcraft.com/soulcraftlabs/open-brainy**. The source of truth is a self-hosted forge: **source.soulcraft.com/soulcraft/brainy**.
It's anonymously readable and cloneable — no account needed to browse, clone, It's anonymously readable and cloneable — no account needed to browse, clone,
or build. or build.
@ -31,7 +31,7 @@ fine) to talk through the approach saves everyone rework.
## Development setup ## Development setup
```bash ```bash
git clone https://source.soulcraft.com/soulcraftlabs/open-brainy.git git clone https://source.soulcraft.com/soulcraft/brainy.git
cd brainy cd brainy
npm install npm install
npm run build npm run build
@ -57,17 +57,6 @@ see `package.json` for `test:integration`, `test:coverage`, and friends.
description states a number, cite the benchmark that produced it (see description states a number, cite the benchmark that produced it (see
[docs/performance-envelopes.md](docs/performance-envelopes.md) for the [docs/performance-envelopes.md](docs/performance-envelopes.md) for the
pattern). Don't state an estimate as if it were measured. pattern). Don't state an estimate as if it were measured.
- **Measurements carry numbers, not provenance.** Public commit messages and
docs give the SHAPE a number was taken at and never where it was taken: no
hostnames, no store or deployment identities, no operational anecdotes about
someone's running system. "A 14,056-noun / 72,679-verb production-shaped
store, measured solo under an exclusive lock" tells a reader everything the
number depends on; the machine it ran on and whose data it was tell them
nothing except where somebody's infrastructure lives.
- **Documents that answer or reference a confidential specification never enter
this repository, even summarized.** The public docs describe THIS engine and
the published contract, and nothing else — a summary of a private document is
still that document's contents.
## License ## License

View file

@ -1,5 +1,5 @@
<p align="center"> <p align="center">
<img src="https://source.soulcraft.com/soulcraftlabs/open-brainy/raw/branch/main/brainy.png" alt="Brainy" width="180"> <img src="https://source.soulcraft.com/soulcraft/brainy/raw/branch/main/brainy.png" alt="Brainy" width="180">
</p> </p>
<h1 align="center">Brainy</h1> <h1 align="center">Brainy</h1>
@ -11,9 +11,9 @@
</p> </p>
<p align="center"> <p align="center">
<a href="https://source.soulcraft.com/soulcraftlabs/-/packages/npm/brainy"><img src="https://img.shields.io/badge/package-The%20Source-2c3e50.svg" alt="Package on The Source"></a> <a href="https://www.npmjs.com/package/@soulcraft/brainy"><img src="https://img.shields.io/npm/v/@soulcraft/brainy.svg" alt="npm version"></a>
<a href="https://source.soulcraft.com/soulcraftlabs/open-brainy"><img src="https://img.shields.io/badge/repo-open--brainy-2c3e50.svg" alt="Repository"></a> <a href="https://www.npmjs.com/package/@soulcraft/brainy"><img src="https://img.shields.io/npm/dm/@soulcraft/brainy.svg" alt="npm downloads"></a>
<a href="https://source.soulcraft.com/soulcraftlabs/open-brainy/actions"><img src="https://source.soulcraft.com/soulcraftlabs/open-brainy/actions/workflows/ci.yml/badge.svg?branch=main" alt="CI"></a> <a href="https://source.soulcraft.com/soulcraft/brainy/actions"><img src="https://source.soulcraft.com/soulcraft/brainy/actions/workflows/ci.yml/badge.svg?branch=main" alt="CI"></a>
<a href="https://soulcraft.com/docs"><img src="https://img.shields.io/badge/docs-soulcraft.com-blue.svg" alt="Documentation"></a> <a href="https://soulcraft.com/docs"><img src="https://img.shields.io/badge/docs-soulcraft.com-blue.svg" alt="Documentation"></a>
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="MIT License"></a> <a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="MIT License"></a>
<a href="https://www.typescriptlang.org/"><img src="https://img.shields.io/badge/%3C%2F%3E-TypeScript-%230074c1.svg" alt="TypeScript"></a> <a href="https://www.typescriptlang.org/"><img src="https://img.shields.io/badge/%3C%2F%3E-TypeScript-%230074c1.svg" alt="TypeScript"></a>
@ -30,8 +30,6 @@
--- ---
**Open Brainy** is the MIT engine — the open API, client library, types, and protocol; an openly specified canonical on-disk format; and this TypeScript reference engine, scoped as a single-node engine for stores up to roughly one million rows. `@soulcraft/brainy` 10.4.2 was the last release under the old package name — the name passes to the native engine, **Brainy**, at 11.0.0: the same API over the same open format at production scale, and it requires a license.
Built because we were tired of stitching a vector store to a graph database to a document store — and spending weeks on plumbing before writing a line of business logic. Brainy indexes every fact **three ways at once** and lets one call query them together: Built because we were tired of stitching a vector store to a graph database to a document store — and spending weeks on plumbing before writing a line of business logic. Brainy indexes every fact **three ways at once** and lets one call query them together:
| You write | Brainy indexes it as | You query it with | | You write | Brainy indexes it as | You query it with |
@ -47,14 +45,12 @@ It runs **inside your process** — no server, no Docker, nothing to operate —
## Quick start ## Quick start
```bash ```bash
bun add @soulcraftlabs/brainy # Bun ≥ 1.1 — recommended bun add @soulcraft/brainy # Bun ≥ 1.1 — recommended
npm install @soulcraftlabs/brainy # Node.js ≥ 22 npm install @soulcraft/brainy # Node.js ≥ 22
``` ```
> **Registry**: add `@soulcraftlabs:registry=https://source.soulcraft.com/api/packages/soulcraftlabs/npm/` to your `.npmrc` (anonymous read).
```javascript ```javascript
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
const brain = new Brainy() // in-memory; one line swaps to disk const brain = new Brainy() // in-memory; one line swaps to disk
await brain.init() await brain.init()

View file

@ -1,14 +1,7 @@
# @soulcraft/brainy — Release Notes for Consumers # @soulcraft/brainy — Release Notes for Consumers
Machine-readable release notes are published at
https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/open-brainy.json
(this engine) and
https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/brainy.json
(the product engine) — read by HQ's `/hq/releases` door, and the source of
truth ahead of this file.
This file is the **quick reference for downstream sessions** tracking Brainy changes. This file is the **quick reference for downstream sessions** tracking Brainy changes.
Full auto-generated changelog: `CHANGELOG.md` · Releases: https://source.soulcraft.com/soulcraftlabs/open-brainy/releases Full auto-generated changelog: `CHANGELOG.md` · Releases: https://source.soulcraft.com/soulcraft/brainy/releases
**How to use:** Brainy is the underlying data engine for downstream applications. Read this when: **How to use:** Brainy is the underlying data engine for downstream applications. Read this when:
- Upgrading `@soulcraft/brainy` in your application - Upgrading `@soulcraft/brainy` in your application
@ -38,227 +31,6 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the
--- ---
## v10.4.4 — 2026-08-28
**A correctness and observability release.** The headline is not speed: it is that a
restart now tells you the truth about itself, a store stops lying about how much it
holds, and the engine stops doing work nobody asked for. There is a performance
improvement and it is modest; it is stated exactly below rather than rounded up.
### The dark restart — fixed at the root
A service could stop cleanly, exit 0, having awaited `close()` on every store it held,
and its next boot would announce `Overwriting stale writer lock … appears dead` for
every one of them. Nothing had crashed. Two deployments hit this; the same defect also
made those boots pay a crash-recovery fold they did not owe.
The cause was not the lock. `close()` released it correctly — when it got there. A
failure part-way through close skipped both the release AND the clean-shutdown marker,
and "the recorded pid is gone" reads identically for an orderly restart and a crash.
- `close()` is now two parts and the second is unconditional: the flush-request watcher,
the **writer lock**, the VFS timers and the terminal `closed` flag are released whether
the durable steps succeeded or not. The original failure is narrated with what it costs
the next open, then rethrown.
- Releasing the lock writes a **clean-close record** naming the lock generation it gave
up. The next open reads that record instead of guessing: recorded → nothing to recover;
absent → it says so, and names the recovery it is about to run. This also ends two
long-standing false alarms — a recycled pid locking a store out of its own reopen, and
`Re-acquiring writer lock … this is a bug` after a perfectly clean close.
- The signal path stopped failing in a batch. One store's failing flush used to strand
every remaining store's lock and markers — at exit code 0. Now: per-store isolation, the
generation store's close (the marker) is part of shutdown, the lock goes in a `finally`,
and the handler no longer calls `process.exit()` when the host application has its own
signal handler, a race that truncated the host's own shutdown mid-flight.
### The count ledger stops lying, and `counts.json` is written atomically
The all-tier scalars are the denominator a coverage check subtracts against. A ledger
derived under the old rule — one entity per id DIRECTORY — counted ghost and scar
containers as rows, and was only FLAGGED suspect: it went on serving wrong numbers for
the life of the store. Two copies of one archive could disagree, and a downstream index
heal reported remaining work that did not exist.
- Such a ledger now derives itself honestly **in the background** after the open, counting
identity records, and persists the correction stamped. Nothing waits for it, because no
read is served from a denominator.
- A derivation that raced a write refuses to stamp its number: one retry on a quiet store,
then the ledger stays SUSPECT and names `repairIndex()` as the door that recounts under
a barrier.
- `counts.json` is written temp+rename. A truncating write left a window in which a
concurrent reader saw the file EMPTY — and an unparseable ledger sends the next open
down the full-rescan path, so the cheapest file in the store was buying the most
expensive recovery.
### An open and a repair narrate themselves — on a channel a log level cannot silence
A store could open for three minutes and print nothing at all. The phase timings existed;
they were written to a channel that every production-looking environment clamps away.
- Narration moved to an always-visible channel. An open now heartbeats the phase it is in,
names each phase as it ends with what it was paying for, and names the expensive STEP
inside a phase. `repairIndex()` does the same and its receipt carries a per-family
`durationMs` — a repair that ran for half an hour with no output could only be watched
through `top`.
- A brain nobody has written to now does nothing: a flush over a clean store is a no-op
and says nothing, the graph index's auto-flush asks before it acts, and the
cross-process flush-request watch is **event-driven** (`fs.watch`) instead of polling a
directory every 500 ms per store forever, with a slow safety sweep behind it and a
narrated fall back to polling where a filesystem cannot be watched.
- A provider that is REBUILDING ITSELF is no longer confused with a broken one. `init()`
does not wait for it, every other family serves, and that family's doors refuse **by
name, carrying the provider's own progress**, saying plainly that they open by
themselves and no action is needed. Health narration dedupes by content, so an unchanged
verdict is silent however a provider's generation counter moves.
### For operators — one behaviour change
**Four `where` operators that previously returned an empty page now raise
`INVALID_QUERY`:** `startsWith`, `endsWith`, `matches` and `length`. An equality/range
posting index cannot evaluate a substring, a pattern or an array length without reading
every row, and it now refuses by name instead of answering with an empty result that
looks like an answer.
**Three that previously returned an empty page are now SERVED:** `hasAll`, `noneOf` and
`excludes`. All 25 accepted operator tokens now agree between this engine and its
accelerated counterpart.
### Performance — stated exactly
Measured on a 14,056-noun / 72,679-verb production-shaped store, both builds solo under
an exclusive lock:
- **Warm reopen after a clean close: 85.7 s → 77.0 s (10.2%).** The whole of that gain is
one fix — generation discovery reads directory NAMES instead of recursively walking the
entire generation log (9.2 s, and it scales with history rather than row count). The
VFS phase is **unchanged**.
- **Cold open: 31.4 s** (518.1 s → 486.7 s), of which the count-ledger derivation moving
off the critical path accounts for storage-init dropping 5,941 ms → 25 ms.
- **A dominant ~38 s remains, diagnosed and NOT fixed.** It is not the VFS — the VFS's own
init is under 2 s of that phase. It is the log-authority adoption and/or the
pending-embed log recovery, both now instrumented so the next measurement names the
culprit outright.
Continuing work, named so nobody has to rediscover it: that ~38 s term; making the
generation store's committed-range set lazy; the hydration path that substitutes
`Date.now()` for an unreadable stored timestamp (inventing data); and a VFS path-prefix
filter built with a `$startsWith` spelling no operator set accepts, so
`searchFiles({ path })` throws today.
---
## v10.4.3 — 2026-08-27 (Open Brainy's first release)
**`@soulcraftlabs/brainy` 10.4.3 is the same engine as `@soulcraft/brainy` 10.4.2, byte for
byte — only the name, the registry, and the pointers changed.** Install:
```bash
npm install @soulcraftlabs/brainy
```
with the registry line in your `.npmrc` (anonymous read):
```
@soulcraftlabs:registry=https://source.soulcraft.com/api/packages/soulcraftlabs/npm/
```
- **The Source is the one registry.** Open Brainy publishes to source.soulcraft.com only; the
npmjs republish step is retired from the release rail. Existing npmjs versions of
`@soulcraft/brainy` stay as they are and receive no new versions.
- **The repository moved** to `soulcraftlabs/open-brainy` on The Source; the old path redirects.
- **No engine change.** Everything in the 10.4.2 notes applies unchanged; adoption is one
install-line change (`@soulcraft/brainy``@soulcraftlabs/brainy`), which downstream
applications make together with their native-engine bump.
## v10.4.2 — 2026-08-27 (a zero-norm vector is not a vector)
**This is the last release of the MIT engine under the `@soulcraft/brainy` name.**
The MIT package continues as **Open Brainy**`@soulcraftlabs/brainy`: the open API,
client library, types and protocol, an openly specified canonical format, and the TypeScript
reference engine, scoped honestly as a single-node engine for stores up to roughly one
million rows. The `@soulcraft/brainy` name passes to the native engine, **Brainy**, at a
major version bump; that engine implements the same API over the same open format at
production scale, requires a license, and refuses loudly without one. Nothing changes
for existing installs until that major ships; the move is announced with it.
Six fixes, one law: a vector with no magnitude carries no information, so it must
never reach a vector index — in any engine — and the canonical store must say so.
- **The permanently-unvectored row.** `add({ ..., vector: [] })` (and the same item
shape in `addMany` / `transact`) is now the sanctioned "no vector" row: persisted
with an empty vector leg, never embedded, never indexed, counted as unvectored in
the canonical ledger. Metadata-only rows — telemetry tallies, counters, plumbing —
no longer need a placeholder vector and never enter the vector leg. `vector: []`
together with `deferEmbedding: true` is refused with a typed error (a supplied
vector has nothing to defer). Previously `vector: []` threw a dimension error.
- **The unvector door.** `update({ id, vector: [] })` (and its `transact()` twin) is
the sanctioned way to strip a vector from an existing row: canonical vector → `[]`,
removal from the vector index, the vectored ledger decremented exactly once — and
idempotent, so a resumed cleanup pass may simply re-issue. It never re-embeds, and
it clears a pending deferred-embed marker durably so the background worker cannot
re-vector the row later. Note that a rebuild never sheds vectors (it re-derives the
index from canonical rows); shedding historical vectors needs this door.
- **Zero-norm vectors are normalized at the write.** An explicit all-zero vector on
any write path is persisted as unvectored (`[]`) with one warning naming the row;
the vector-index operations keep their own refusal as a second line. The engine's
own VFS root, which used to persist a deliberate all-zero placeholder (harmless
under cosine distance, a false attractor under a downstream engine's
squared-euclidean serving — a production incident this week), is now created
unvectored, and an existing store's legacy root is migrated on open by a single
fixed-path read before the health gate runs — never a walk.
- **Enumeration keys on the identity record.** `getNouns()` / `getVerbs()` and the
cursor walks behind them enumerate by the metadata record, the same key the
canonical ledger counts by — previously the walk keyed on the vector file, so a
row holding metadata but no vector was counted yet never yielded (a permanent
"missing" phantom in coverage math), while an orphaned vector-only directory
could be yielded as a phantom id. The recovery fold also never deletes an existing
vector when it replays a metadata-only after-image (preserve-if-absent). One
documented gap remains: a verb's endpoints live only in its vector leg, so a
metadata-only verb is counted and loudly skipped, never fabricated — the fix is a
canonical-format change and lands with the open format.
- **The ledger's one-time derivation counts identity records.** Stores upgraded from
pre-ledger versions derived their ALL-visibility scalars once by counting id
directories, which included ghost and scar containers left by an old partial-delete
defect — an inflated denominator whose coverage row could never reach exact. The
derivation now counts only directories holding a metadata record, `counts.json`
carries a derivation-rule stamp, and a ledger derived under the old rule is marked
`suspect` at open (one O(1) field read, one warning) so the online `repairIndex()`
path clears it with a real recount.
- **The vector index refuses what it cannot hold.** `rebuild()` skips unvectored and
zero-norm rows (one summary line), re-pins the vector dimension from the first real
vector after a restart (previously a restart left the pin unset, so a wrong-length
insert became the new pin instead of being rejected), and `addItem` / `updateItem`
throw a typed `EmptyVectorIndexError` on a length-0 vector instead of ever storing
a vector-less node.
- **Smaller:** a failing plugin activation now rethrows with the original error as
`cause` (the originating file and line survive to the caller's log); build
generators stamp from the repository history of their inputs instead of wall clock,
so two builds of the same tree are byte-identical.
Adoption: one restart, paired with its native-engine release. The first open of an
existing store runs the legacy-root migration (one narrated line) and, on stores that
upgraded from pre-ledger versions, marks the ledger suspect until the next sanctioned
recount — no rebuild in either case.
## v10.4.1 — 2026-08-26 (reads refuse per family; an unchanged write never re-embeds)
Two production defects from the same week, fixed together as a patch to 10.4.0.
- **The read gate is per family.** A read now refuses only when the index family it
actually consults is unhealthy: a metadata filter is served while the vector leg is
rebuilding; a semantic query is refused only by the vector family; a graph
traversal only by the graph family. Previously any unhealthy family refused every
read on the brain — under a long vector rebuild, a production deployment's
metadata-only reads were refused for the duration, and the retries became a write
pump of their own.
- **Unchanged data never re-embeds.** `update()` compares the incoming `data`
structurally with the stored record; an update carrying identical data (a common
shape for periodic upserts) no longer embeds again and no longer churns the vector
leg. Previously every such update re-embedded and re-inserted, which under load
saturated the vector index with near-identical vectors.
Adoption: one restart, paired with its native-engine release.
## v10.4.0 — 2026-08-25 (the health report has a name) ## v10.4.0 — 2026-08-25 (the health report has a name)
Three related cures, one root cause: an index deciding whether it could be trusted Three related cures, one root cause: an index deciding whether it could be trusted

View file

@ -30,7 +30,7 @@ commit to backporting fixes to unsupported lines.
## Scope ## Scope
This policy covers the `@soulcraftlabs/brainy` package itself — the code in This policy covers the `@soulcraft/brainy` package itself — the code in
this repository. If you're evaluating a deployment that also uses this repository. If you're evaluating a deployment that also uses
`@soulcraft/cor`, report issues in that package the same way, to the same `@soulcraft/cor`, report issues in that package the same way, to the same
address; we'll route internally. address; we'll route internally.

View file

@ -3,7 +3,7 @@
/** /**
* Modern TypeScript CLI Runner * Modern TypeScript CLI Runner
* *
* This is the entry point after npm install @soulcraftlabs/brainy * This is the entry point after npm install @soulcraft/brainy
* It runs the compiled TypeScript CLI code * It runs the compiled TypeScript CLI code
*/ */

View file

@ -3,7 +3,7 @@
"configVersion": 0, "configVersion": 0,
"workspaces": { "workspaces": {
"": { "": {
"name": "@soulcraftlabs/brainy", "name": "@soulcraft/brainy",
"dependencies": { "dependencies": {
"@aws-sdk/client-s3": "^3.540.0", "@aws-sdk/client-s3": "^3.540.0",
"@azure/identity": "^4.0.0", "@azure/identity": "^4.0.0",

View file

@ -25,13 +25,13 @@
### Prerequisites ### Prerequisites
```bash ```bash
npm install @soulcraftlabs/brainy npm install @soulcraft/brainy
``` ```
### Your First Neural Database ### Your First Neural Database
```typescript ```typescript
import { Brainy, NounType } from '@soulcraftlabs/brainy' import { Brainy, NounType } from '@soulcraft/brainy'
// Step 1: Create and initialize Brainy // Step 1: Create and initialize Brainy
const brain = new Brainy({ const brain = new Brainy({
@ -143,7 +143,7 @@ Once you're comfortable with basic operations, move to **Level 2** to learn abou
### Building a Knowledge Graph ### Building a Knowledge Graph
```typescript ```typescript
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
const brain = new Brainy({ storage: { type: 'memory' } }) const brain = new Brainy({ storage: { type: 'memory' } })
await brain.init() await brain.init()
@ -314,7 +314,7 @@ Ready for AI-powered search and clustering? Move to **Level 3**.
### Triple Intelligence in Action ### Triple Intelligence in Action
```typescript ```typescript
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
const brain = new Brainy({ storage: { type: 'memory' } }) const brain = new Brainy({ storage: { type: 'memory' } })
await brain.init() await brain.init()
@ -529,7 +529,7 @@ Want to treat files as intelligent entities? Learn the **Virtual Filesystem** in
### Files as Intelligent Entities ### Files as Intelligent Entities
```typescript ```typescript
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
const brain = new Brainy({ storage: { type: 'memory' } }) const brain = new Brainy({ storage: { type: 'memory' } })
await brain.init() await brain.init()
@ -832,7 +832,7 @@ Ready for production deployment? Level 5 covers **planet-scale architecture**.
### Production-Ready Deployment ### Production-Ready Deployment
```typescript ```typescript
import { Brainy, NounType } from '@soulcraftlabs/brainy' import { Brainy, NounType } from '@soulcraft/brainy'
// 1. PRODUCTION STORAGE - Filesystem with off-site snapshots // 1. PRODUCTION STORAGE - Filesystem with off-site snapshots
console.log('Initializing production storage...\n') console.log('Initializing production storage...\n')

View file

@ -1217,7 +1217,7 @@ where: {
await brain.find({ type: 'Document' }) await brain.find({ type: 'Document' })
// ✅ Correct: Use NounType enum // ✅ Correct: Use NounType enum
import { NounType } from '@soulcraftlabs/brainy' import { NounType } from '@soulcraft/brainy'
await brain.find({ type: NounType.Document }) await brain.find({ type: NounType.Document })
// ❌ Error: Operator not recognized // ❌ Error: Operator not recognized

View file

@ -153,13 +153,13 @@ brainy-data/
### Step 1: Update Brainy Package ### Step 1: Update Brainy Package
```bash ```bash
npm install @soulcraftlabs/brainy@latest npm install @soulcraft/brainy@latest
``` ```
**Check your version:** **Check your version:**
```bash ```bash
npm list @soulcraftlabs/brainy npm list @soulcraft/brainy
# Should show: @soulcraftlabs/brainy@4.0.0 # Should show: @soulcraft/brainy@4.0.0
``` ```
### Step 2: No Code Changes Required! ✅ ### Step 2: No Code Changes Required! ✅
@ -374,7 +374,7 @@ If you encounter issues, you can rollback:
```bash ```bash
# Reinstall v3 # Reinstall v3
npm install @soulcraftlabs/brainy@^3.50.0 npm install @soulcraft/brainy@^3.50.0
# Restart application # Restart application
``` ```
@ -389,7 +389,7 @@ rm -rf ./data
cp -r ./data-backup ./data cp -r ./data-backup ./data
# Reinstall v3 # Reinstall v3
npm install @soulcraftlabs/brainy@^3.50.0 npm install @soulcraft/brainy@^3.50.0
``` ```
## Common Migration Scenarios ## Common Migration Scenarios
@ -539,7 +539,7 @@ console.log('Storage type:', status.type)
**Migration Checklist:** **Migration Checklist:**
- ✅ Backup data - ✅ Backup data
- ✅ Update npm package (`npm install @soulcraftlabs/brainy@latest`) - ✅ Update npm package (`npm install @soulcraft/brainy@latest`)
- ✅ Restart application (automatic migration) - ✅ Restart application (automatic migration)
- ✅ Verify data integrity - ✅ Verify data integrity
- ✅ Enable lifecycle policies - ✅ Enable lifecycle policies

View file

@ -46,7 +46,7 @@ If no plugin provides a given key, brainy uses its built-in JavaScript implement
### 1. Implement the `BrainyPlugin` interface ### 1. Implement the `BrainyPlugin` interface
```typescript ```typescript
import type { BrainyPlugin, BrainyPluginContext } from '@soulcraftlabs/brainy/plugin' import type { BrainyPlugin, BrainyPluginContext } from '@soulcraft/brainy/plugin'
const myPlugin: BrainyPlugin = { const myPlugin: BrainyPlugin = {
name: 'my-brainy-plugin', // Must be unique (typically your npm package name) name: 'my-brainy-plugin', // Must be unique (typically your npm package name)
@ -90,7 +90,7 @@ await brain.init()
**Programmatic registration:** For plugins not installed as npm packages, use `brain.use()`: **Programmatic registration:** For plugins not installed as npm packages, use `brain.use()`:
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
import myPlugin from './my-plugin.js' import myPlugin from './my-plugin.js'
const brain = new Brainy() const brain = new Brainy()
@ -272,10 +272,10 @@ When provided by an optional native acceleration plugin (such as `@soulcraft/cor
#### `cache` #### `cache`
**Type:** `UnifiedCache` **Type:** `UnifiedCache`
Replaces the global `UnifiedCache` singleton used for VFS path resolution, semantic caching, and vector index caching. Must implement the `UnifiedCache` interface (available from `@soulcraftlabs/brainy/internals`). Replaces the global `UnifiedCache` singleton used for VFS path resolution, semantic caching, and vector index caching. Must implement the `UnifiedCache` interface (available from `@soulcraft/brainy/internals`).
```typescript ```typescript
import type { UnifiedCache } from '@soulcraftlabs/brainy/internals' import type { UnifiedCache } from '@soulcraft/brainy/internals'
context.registerProvider('cache', myNativeCache) context.registerProvider('cache', myNativeCache)
``` ```
@ -325,8 +325,8 @@ Plugins can register custom storage backends that users reference by name.
### Implementing a Storage Adapter ### Implementing a Storage Adapter
```typescript ```typescript
import type { StorageAdapterFactory } from '@soulcraftlabs/brainy/plugin' import type { StorageAdapterFactory } from '@soulcraft/brainy/plugin'
import type { StorageAdapter } from '@soulcraftlabs/brainy' import type { StorageAdapter } from '@soulcraft/brainy'
class MyStorageAdapter implements StorageAdapter { class MyStorageAdapter implements StorageAdapter {
async init(): Promise<void> { /* ... */ } async init(): Promise<void> { /* ... */ }
@ -360,9 +360,9 @@ Brainy provides three entry points for plugin developers:
| Import Path | Contents | Stability | | Import Path | Contents | Stability |
|-------------|----------|-----------| |-------------|----------|-----------|
| `@soulcraftlabs/brainy` | Public API, types, StorageAdapter | Stable (semver) | | `@soulcraft/brainy` | Public API, types, StorageAdapter | Stable (semver) |
| `@soulcraftlabs/brainy/plugin` | BrainyPlugin, BrainyPluginContext, StorageAdapterFactory | Stable (semver) | | `@soulcraft/brainy/plugin` | BrainyPlugin, BrainyPluginContext, StorageAdapterFactory | Stable (semver) |
| `@soulcraftlabs/brainy/internals` | UnifiedCache, EntityIdMapper, logger utilities | Internal (may change between minor versions) | | `@soulcraft/brainy/internals` | UnifiedCache, EntityIdMapper, logger utilities | Internal (may change between minor versions) |
## Diagnostics ## Diagnostics
@ -440,7 +440,7 @@ A minimal but useful plugin that provides SIMD-accelerated distance calculations
```typescript ```typescript
// simd-distance-plugin/src/plugin.ts // simd-distance-plugin/src/plugin.ts
import type { BrainyPlugin, BrainyPluginContext } from '@soulcraftlabs/brainy/plugin' import type { BrainyPlugin, BrainyPluginContext } from '@soulcraft/brainy/plugin'
// Hypothetical native module // Hypothetical native module
import { simdCosineDistance } from './native.js' import { simdCosineDistance } from './native.js'
@ -470,7 +470,7 @@ export default simdDistancePlugin
"main": "./dist/plugin.js", "main": "./dist/plugin.js",
"types": "./dist/plugin.d.ts", "types": "./dist/plugin.d.ts",
"peerDependencies": { "peerDependencies": {
"@soulcraftlabs/brainy": ">=7.0.0" "@soulcraft/brainy": ">=7.0.0"
} }
} }
``` ```
@ -478,7 +478,7 @@ export default simdDistancePlugin
Usage: Usage:
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brain = new Brainy({ plugins: ['brainy-simd-distance'] }) const brain = new Brainy({ plugins: ['brainy-simd-distance'] })
await brain.init() await brain.init()

View file

@ -54,7 +54,7 @@ After 40 API calls:
```typescript ```typescript
// server.ts // server.ts
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
// SINGLETON INSTANCE // SINGLETON INSTANCE
let brainInstance: Brainy | null = null let brainInstance: Brainy | null = null
@ -174,7 +174,7 @@ process.on('SIGTERM', async () => {
```typescript ```typescript
// server.ts - Clean Bun implementation // server.ts - Clean Bun implementation
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
let brain: Brainy | null = null let brain: Brainy | null = null

View file

@ -5,7 +5,7 @@
## Quick Start ## Quick Start
```typescript ```typescript
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()

View file

@ -99,7 +99,7 @@ Examples:
```bash ```bash
# 1. Deprecate wrong version on npm # 1. Deprecate wrong version on npm
npm deprecate @soulcraftlabs/brainy@X.X.X "Incorrect version - use Y.Y.Y" npm deprecate @soulcraft/brainy@X.X.X "Incorrect version - use Y.Y.Y"
# 2. Fix version in package.json # 2. Fix version in package.json
# 3. Republish correct version # 3. Republish correct version

View file

@ -13,7 +13,7 @@
### In-Memory ### In-Memory
```typescript ```typescript
import Brainy from '@soulcraftlabs/brainy' import Brainy from '@soulcraft/brainy'
const brain = new Brainy({ storage: { type: 'memory' } }) const brain = new Brainy({ storage: { type: 'memory' } })
``` ```
@ -43,7 +43,7 @@ The native vector provider (via the optional `@soulcraft/cor` package) extends t
Numbers below are **measured** by `tests/benchmarks/find-composition-scale.js` (a single Numbers below are **measured** by `tests/benchmarks/find-composition-scale.js` (a single
Node 22 process, in-memory storage, 384-dim vectors, `balanced` recall). They are the Node 22 process, in-memory storage, 384-dim vectors, `balanced` recall). They are the
open-core (pure-TypeScript) path — what you get from `@soulcraftlabs/brainy` with no native open-core (pure-TypeScript) path — what you get from `@soulcraft/brainy` with no native
provider installed. Run it yourself: `node --max-old-space-size=8192 tests/benchmarks/find-composition-scale.js 100000`. provider installed. Run it yourself: `node --max-old-space-size=8192 tests/benchmarks/find-composition-scale.js 100000`.
`find()` query latency, p50 / p95 (200 queries each): `find()` query latency, p50 / p95 (200 queries each):

File diff suppressed because it is too large Load diff

View file

@ -24,7 +24,7 @@ next:
## Quick Start ## Quick Start
```typescript ```typescript
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
const brain = new Brainy() // Zero config! const brain = new Brainy() // Zero config!
await brain.init() // VFS auto-initialized! await brain.init() // VFS auto-initialized!
@ -1010,7 +1010,7 @@ await db.release() // unpin + free cached materialization
### Db API errors ### Db API errors
All exported from `@soulcraftlabs/brainy`: All exported from `@soulcraft/brainy`:
| Error | Thrown by | Meaning | | Error | Thrown by | Meaning |
|---|---|---| |---|---|---|
@ -1918,11 +1918,11 @@ isn't serving throws instead of rebuilding mid-query:
| `MetadataIndexNotReadyError` | `find({ where })` | Metadata/field index isn't serving | | `MetadataIndexNotReadyError` | `find({ where })` | Metadata/field index isn't serving |
| `VectorIndexNotReadyError` | `find({ query })`, `similar()` | Vector index isn't serving | | `VectorIndexNotReadyError` | `find({ query })`, `similar()` | Vector index isn't serving |
All three are exported from `@soulcraftlabs/brainy`. Catch them to distinguish All three are exported from `@soulcraft/brainy`. Catch them to distinguish
"index not ready" from a genuine empty result: "index not ready" from a genuine empty result:
```typescript ```typescript
import { MetadataIndexNotReadyError } from '@soulcraftlabs/brainy' import { MetadataIndexNotReadyError } from '@soulcraft/brainy'
try { try {
const rows = await brain.find({ where: { status: 'active' } }) const rows = await brain.find({ where: { status: 'active' } })
@ -2208,7 +2208,7 @@ For the full taxonomy with all 169 types and their descriptions, see:
- **📖 Documentation:** [Full Documentation](../) - **📖 Documentation:** [Full Documentation](../)
- **🐛 Issues:** [GitHub Issues](https://github.com/soulcraftlabs/brainy/issues) - **🐛 Issues:** [GitHub Issues](https://github.com/soulcraftlabs/brainy/issues)
- **💬 Discussions:** [GitHub Discussions](https://github.com/soulcraftlabs/brainy/discussions) - **💬 Discussions:** [GitHub Discussions](https://github.com/soulcraftlabs/brainy/discussions)
- **📦 NPM:** [@soulcraftlabs/brainy](https://www.npmjs.com/package/@soulcraftlabs/brainy) - **📦 NPM:** [@soulcraft/brainy](https://www.npmjs.com/package/@soulcraft/brainy)
- **⭐ GitHub:** [Star us](https://github.com/soulcraftlabs/brainy) - **⭐ GitHub:** [Star us](https://github.com/soulcraftlabs/brainy)
--- ---

View file

@ -268,7 +268,7 @@ locks/_flush_responses/ # writer answers with <uuid>.ack
| **Counts/statistics** | Per-type and per-subtype maps | `_system/{type,subtype,verb-subtype}-statistics.json.gz`, `counts.json` | Recomputable by scanning entities (`brainy inspect repair`) | | **Counts/statistics** | Per-type and per-subtype maps | `_system/{type,subtype,verb-subtype}-statistics.json.gz`, `counts.json` | Recomputable by scanning entities (`brainy inspect repair`) |
A pluggable index provider (the 8.0 plugin contract in A pluggable index provider (the 8.0 plugin contract in
`@soulcraftlabs/brainy/plugin`) may replace any of the JS implementations; the `@soulcraft/brainy/plugin`) may replace any of the JS implementations; the
persisted formats above are contract-bound so JS and native implementations persisted formats above are contract-bound so JS and native implementations
can interleave on the same directory. can interleave on the same directory.

View file

@ -126,7 +126,7 @@ class TypeAwareMetadataIndex {
**The Design**: Specify types clearly in your API calls: **The Design**: Specify types clearly in your API calls:
```typescript ```typescript
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
// Add entity with explicit type // Add entity with explicit type
await brain.add({ await brain.add({
@ -231,7 +231,7 @@ class OrgEnrichmentAugmentation {
**Brainy's Approach**: Extract **typed** concepts: **Brainy's Approach**: Extract **typed** concepts:
```typescript ```typescript
import { NaturalLanguageProcessor } from '@soulcraftlabs/brainy' import { NaturalLanguageProcessor } from '@soulcraft/brainy'
const nlp = new NaturalLanguageProcessor() const nlp = new NaturalLanguageProcessor()
const concepts = await nlp.extractConcepts("Alice works at Google in San Francisco") const concepts = await nlp.extractConcepts("Alice works at Google in San Francisco")
@ -382,7 +382,7 @@ import {
getVerbTypes, getVerbTypes,
BrainyTypes, BrainyTypes,
suggestType suggestType
} from '@soulcraftlabs/brainy' } from '@soulcraft/brainy'
// Get all available noun types // Get all available noun types
const nounTypes = getNounTypes() const nounTypes = getNounTypes()

View file

@ -127,7 +127,7 @@ For reference, a clean migration path:
`isMultiProcessSafe` type-guard. Keep `hasStorageMethod` for `isMultiProcessSafe` type-guard. Keep `hasStorageMethod` for
build/install artifact protection. build/install artifact protection.
5. Document the new contract in `concepts/storage-adapters.md`. 5. Document the new contract in `concepts/storage-adapters.md`.
6. Major-version-bump the `@soulcraftlabs/brainy` peerDep range expected by 6. Major-version-bump the `@soulcraft/brainy` peerDep range expected by
plugins. plugins.
Estimated work: ~half a day of code, ~2 hours of doc/example updates, Estimated work: ~half a day of code, ~2 hours of doc/example updates,

View file

@ -20,7 +20,7 @@ next:
Every example on this page is written against the real Brainy 8.0 API. The setup is always the same: Every example on this page is written against the real Brainy 8.0 API. The setup is always the same:
```typescript ```typescript
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()
@ -40,7 +40,7 @@ Brainy's **Noun-Verb Taxonomy** achieves broad coverage of human knowledge throu
- **Multi-hop Graph Traversals = Relationship Complexity** - **Multi-hop Graph Traversals = Relationship Complexity**
- **Result: Model data across virtually any industry** - **Result: Model data across virtually any industry**
Every piece of information can be represented as entities (nouns) connected by relationships (verbs) carrying properties (metadata). The standardized type system from `@soulcraftlabs/brainy` (`NounType`, `VerbType`) gives those nouns and verbs a stable, shared name. Every piece of information can be represented as entities (nouns) connected by relationships (verbs) carrying properties (metadata). The standardized type system from `@soulcraft/brainy` (`NounType`, `VerbType`) gives those nouns and verbs a stable, shared name.
## The Power of Standardization: Universal Interoperability ## The Power of Standardization: Universal Interoperability

View file

@ -35,7 +35,7 @@ constructor and `init()`.
## Instant Start ## Instant Start
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
// That's it. No config needed. // That's it. No config needed.
const brain = new Brainy() const brain = new Brainy()

View file

@ -167,7 +167,7 @@ await brain.find({ orderBy: 'createdAt' })
`UnresolvableFieldError` is exported from the package root: `UnresolvableFieldError` is exported from the package root:
```typescript ```typescript
import { UnresolvableFieldError } from '@soulcraftlabs/brainy' import { UnresolvableFieldError } from '@soulcraft/brainy'
try { try {
await brain.find({ orderBy: 'createdAt' }) await brain.find({ orderBy: 'createdAt' })

View file

@ -68,19 +68,6 @@ a maintenance window, a divergence `repairIndex()` will clean up on its own
schedule. `serving: false` is not benign. It means this provider is refusing to schedule. `serving: false` is not benign. It means this provider is refusing to
answer, on its own word, right now. answer, on its own word, right now.
**How a failure gets its grade — the serving law.** A provider grades `heal` by
one question only: *could an answer be wrong?* — never *how expensive is the
fix?* A missing-postings shortfall, however large, is `heal: 'repair'` (re-post
exactly what the ledger names, reads serving throughout); it can never withhold
serving just because healing it takes work. `serving` is withheld only by a
small, named set of rebuild-graded conditions — the index not initialized, its
durable state absent, a manifest naming files that are not resident, a replay
that did not complete cleanly — the states in which an answer could genuinely be
wrong. And a read is only ever refused by the family it actually consults: a
metadata filter is answered by the metadata index alone, vector search by the
vector index, traversal by the graph index — one family's refusal never blocks
another family's reads.
## Reads refuse — they never rebuild ## Reads refuse — they never rebuild
A query that reaches a not-serving provider does not trigger a rebuild from inside A query that reaches a not-serving provider does not trigger a rebuild from inside
@ -95,7 +82,7 @@ catchable error naming the reason:
| `MetadataIndexNotReadyError` | `find({ where })` | The metadata/field index isn't serving — a filtered read would otherwise return `[]` indistinguishable from "no matches" | | `MetadataIndexNotReadyError` | `find({ where })` | The metadata/field index isn't serving — a filtered read would otherwise return `[]` indistinguishable from "no matches" |
| `VectorIndexNotReadyError` | `find({ query })`, `similar()` | The vector index isn't serving — a semantic search would otherwise return `[]` indistinguishable from "nothing similar" | | `VectorIndexNotReadyError` | `find({ query })`, `similar()` | The vector index isn't serving — a semantic search would otherwise return `[]` indistinguishable from "nothing similar" |
All three are exported from `@soulcraftlabs/brainy`. Catch them where your application All three are exported from `@soulcraft/brainy`. Catch them where your application
needs to distinguish "this index isn't ready yet" from "there's genuinely nothing needs to distinguish "this index isn't ready yet" from "there's genuinely nothing
here" — a health dashboard, a retry policy, an operator alert. The fix is always here" — a health dashboard, a retry policy, an operator alert. The fix is always
the same: reconcile the index, either by reopening the brain (which brings every the same: reconcile the index, either by reopening the brain (which brings every

View file

@ -61,7 +61,7 @@ The only required override is the capability flag. Returning `true` from
to call `acquireWriterLock()` at init. to call `acquireWriterLock()` at init.
```typescript ```typescript
import { FileSystemStorage } from '@soulcraftlabs/brainy' import { FileSystemStorage } from '@soulcraft/brainy'
export class MmapFileSystemStorage extends FileSystemStorage { export class MmapFileSystemStorage extends FileSystemStorage {
public supportsMultiProcessLocking(): boolean { public supportsMultiProcessLocking(): boolean {
@ -79,7 +79,7 @@ If your storage is **not filesystem-backed** (a custom
network backend), extend `BaseStorage` directly: network backend), extend `BaseStorage` directly:
```typescript ```typescript
import { BaseStorage } from '@soulcraftlabs/brainy' import { BaseStorage } from '@soulcraft/brainy'
export class MyCloudStorage extends BaseStorage { export class MyCloudStorage extends BaseStorage {
// BaseStorage's default no-op implementations of the multi-process // BaseStorage's default no-op implementations of the multi-process
@ -101,7 +101,7 @@ The defensive check at every new-storage-method call site (`brainy.ts`,
`hasStorageMethod(name)`) does **not** exist to handle "plugin bundles a `hasStorageMethod(name)`) does **not** exist to handle "plugin bundles a
stale BaseStorage." Plugins ship a dist that preserves the dynamic ESM stale BaseStorage." Plugins ship a dist that preserves the dynamic ESM
import (verify in your plugin's `dist/`: `import { FileSystemStorage } from import (verify in your plugin's `dist/`: `import { FileSystemStorage } from
'@soulcraftlabs/brainy'` is not rewritten to a vendored copy). The prototype '@soulcraft/brainy'` is not rewritten to a vendored copy). The prototype
chain at runtime resolves to whatever Brainy version your consumer has chain at runtime resolves to whatever Brainy version your consumer has
installed. installed.
@ -109,8 +109,8 @@ installed.
the prototype chain at the consumer-app level: the prototype chain at the consumer-app level:
- **Stale `node_modules`** — a lingering install from before the consumer - **Stale `node_modules`** — a lingering install from before the consumer
upgraded Brainy. The package.json says `@soulcraftlabs/brainy@7.22.0` but upgraded Brainy. The package.json says `@soulcraft/brainy@7.22.0` but
`node_modules/@soulcraftlabs/brainy` is still 7.20.x. `node_modules/@soulcraft/brainy` is still 7.20.x.
- **Lockfile drift**`bun.lockb` / `package-lock.json` pins a brainy - **Lockfile drift**`bun.lockb` / `package-lock.json` pins a brainy
version older than the package.json range, and `bun install` honors the version older than the package.json range, and `bun install` honors the
lockfile. lockfile.
@ -131,7 +131,7 @@ and the warning names the adapter class plus a remediation hint:
methods on its prototype chain. Writer locking and the flush-request RPC are methods on its prototype chain. Writer locking and the flush-request RPC are
disabled for this directory. Likely fix: clean install (`rm -rf node_modules disabled for this directory. Likely fix: clean install (`rm -rf node_modules
bun.lockb && bun install`) or rebuild your container image to refresh bun.lockb && bun install`) or rebuild your container image to refresh
`@soulcraftlabs/brainy` to ≥7.21. See docs/concepts/storage-adapters.md. `@soulcraft/brainy` to ≥7.21. See docs/concepts/storage-adapters.md.
``` ```
## Authoring a new storage adapter — minimum checklist ## Authoring a new storage adapter — minimum checklist
@ -168,7 +168,7 @@ bun.lockb && bun install`) or rebuild your container image to refresh
install time — fix install, not your plugin. install time — fix install, not your plugin.
6. **Pin your peer dep generously.** `"peerDependencies": { 6. **Pin your peer dep generously.** `"peerDependencies": {
"@soulcraftlabs/brainy": "^7.21.0" }` accepts any compatible 7.x. Don't pin "@soulcraft/brainy": "^7.21.0" }` accepts any compatible 7.x. Don't pin
to an exact patch unless you're tracking a known regression. to an exact patch unless you're tracking a known regression.
## Future direction ## Future direction
@ -185,5 +185,5 @@ follow-up; consumers don't need to anticipate the change.
heartbeat semantics, what the lock protects. heartbeat semantics, what the lock protects.
- [`guides/inspection`](../guides/inspection.md) — `brainy inspect` and the - [`guides/inspection`](../guides/inspection.md) — `brainy inspect` and the
read-only mode. read-only mode.
- `node_modules/@soulcraftlabs/brainy/dist/storage/baseStorage.d.ts` — the - `node_modules/@soulcraft/brainy/dist/storage/baseStorage.d.ts` — the
authoritative type signatures for every method this page references. authoritative type signatures for every method this page references.

View file

@ -22,7 +22,7 @@ they share a single scan.
## Quick Start ## Quick Start
```typescript ```typescript
import { Brainy, NounType } from '@soulcraftlabs/brainy' import { Brainy, NounType } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()

View file

@ -8,7 +8,7 @@ Brainy is **framework-friendly** - designed to drop into the server side of any
Brainy embeds an HNSW vector index, a graph engine, and a filesystem-backed persistence layer. These belong on the server: Brainy embeds an HNSW vector index, a graph engine, and a filesystem-backed persistence layer. These belong on the server:
- **Zero configuration**: Just `import { Brainy } from '@soulcraftlabs/brainy'` - **Zero configuration**: Just `import { Brainy } from '@soulcraft/brainy'`
- **Auto storage detection**: `new Brainy()` auto-selects filesystem persistence on Node - **Auto storage detection**: `new Brainy()` auto-selects filesystem persistence on Node
- **Cleaner code**: No browser polyfills, no conditional client/server imports - **Cleaner code**: No browser polyfills, no conditional client/server imports
- **Better DX**: One instance shared across your server routes - **Better DX**: One instance shared across your server routes
@ -18,13 +18,13 @@ Brainy embeds an HNSW vector index, a graph engine, and a filesystem-backed pers
### Install Brainy ### Install Brainy
```bash ```bash
npm install @soulcraftlabs/brainy npm install @soulcraft/brainy
``` ```
### Basic Integration ### Basic Integration
```javascript ```javascript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
// Run on the server (API route, server component, backend service) // Run on the server (API route, server component, backend service)
// new Brainy() auto-detects filesystem persistence on Node // new Brainy() auto-detects filesystem persistence on Node
@ -105,7 +105,7 @@ On the server, create one Brainy instance and reuse it across requests. This mod
```javascript ```javascript
// lib/brain.server.js // lib/brain.server.js
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
let brainPromise let brainPromise
@ -163,7 +163,7 @@ On the server, create one Brainy instance and reuse it across requests:
```javascript ```javascript
// server/brain.js (server-only module) // server/brain.js (server-only module)
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
let brainPromise let brainPromise
@ -248,7 +248,7 @@ The matching backend endpoint uses Brainy directly (Node/Bun):
```typescript ```typescript
// server: api/search // server: api/search
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brain = new Brainy() // auto-detects filesystem persistence on Node const brain = new Brainy() // auto-detects filesystem persistence on Node
await brain.init() await brain.init()
@ -266,7 +266,7 @@ In Next.js, Brainy lives in server code only: API routes, server components, or
```javascript ```javascript
// lib/brain.server.js (imported only by server code) // lib/brain.server.js (imported only by server code)
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
let brainPromise let brainPromise
@ -318,7 +318,7 @@ Brainy runs in a server-only module (`*.server.js`); the component fetches resul
```javascript ```javascript
// src/lib/server/brain.js (server-only — note the .server suffix) // src/lib/server/brain.js (server-only — note the .server suffix)
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
let brainPromise let brainPromise
@ -432,7 +432,7 @@ import { defineConfig } from 'vite'
export default defineConfig({ export default defineConfig({
ssr: { ssr: {
external: ['@soulcraftlabs/brainy'] external: ['@soulcraft/brainy']
} }
}) })
``` ```
@ -440,7 +440,7 @@ export default defineConfig({
```javascript ```javascript
// rollup.config.js (server bundle) // rollup.config.js (server bundle)
export default { export default {
external: ['@soulcraftlabs/brainy', 'node:fs', 'node:path', 'node:crypto'] external: ['@soulcraft/brainy', 'node:fs', 'node:path', 'node:crypto']
} }
``` ```
@ -466,7 +466,7 @@ export async function load({ url }) {
```javascript ```javascript
// For build-time usage (runs in Node during the build) // For build-time usage (runs in Node during the build)
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
export async function generateStaticProps() { export async function generateStaticProps() {
const brain = new Brainy({ const brain = new Brainy({
@ -513,7 +513,7 @@ export async function generateStaticProps() {
### Issue: Large client bundle size ### Issue: Large client bundle size
**Cause**: A client module is pulling in Brainy. **Cause**: A client module is pulling in Brainy.
**Solution**: Move the `import { Brainy } from '@soulcraftlabs/brainy'` into a server-only module so it never reaches the browser bundle. **Solution**: Move the `import { Brainy } from '@soulcraft/brainy'` into a server-only module so it never reaches the browser bundle.
### Issue: SSR hydration mismatch ### Issue: SSR hydration mismatch
**Solution**: Run the search on the server (loader / server action / API route) and pass the results down as props, so server and client render the same markup. **Solution**: Run the search on the server (loader / server action / API route) and pass the results down as props, so server and client render the same markup.

View file

@ -9,7 +9,7 @@ Brainy's import is **ONE magical method** that understands EVERYTHING:
## The Ultimate Simplicity ## The Ultimate Simplicity
```javascript ```javascript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()

View file

@ -13,7 +13,7 @@ Brainy provides real-time progress tracking for **all 7 supported file formats**
### Basic Progress Tracking ### Basic Progress Tracking
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
import * as fs from 'fs' import * as fs from 'fs'
const brain = await Brainy.create() const brain = await Brainy.create()

View file

@ -7,7 +7,7 @@
## Basic Import ## Basic Import
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()
@ -187,7 +187,7 @@ await brain.import(file, {
## Complete Example ## Complete Example
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
import * as fs from 'fs' import * as fs from 'fs'
async function importCatalog() { async function importCatalog() {

View file

@ -108,7 +108,7 @@ check fails — useful for piping into monitoring or CI.
## Programmatic inspection ## Programmatic inspection
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const reader = await Brainy.openReadOnly({ const reader = await Brainy.openReadOnly({
storage: { type: 'filesystem', path: '/data/brain' } storage: { type: 'filesystem', path: '/data/brain' }

View file

@ -21,21 +21,21 @@ next:
## Install ## Install
```bash ```bash
npm install @soulcraftlabs/brainy npm install @soulcraft/brainy
``` ```
Or with your preferred package manager: Or with your preferred package manager:
```bash ```bash
bun add @soulcraftlabs/brainy bun add @soulcraft/brainy
yarn add @soulcraftlabs/brainy yarn add @soulcraft/brainy
pnpm add @soulcraftlabs/brainy pnpm add @soulcraft/brainy
``` ```
## Verify ## Verify
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()
@ -52,7 +52,7 @@ npm install @soulcraft/cor
``` ```
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brain = new Brainy({ plugins: ['@soulcraft/cor'] }) const brain = new Brainy({ plugins: ['@soulcraft/cor'] })
await brain.init() // native providers registered during init await brain.init() // native providers registered during init
@ -71,7 +71,7 @@ remains available on npm if you need it.
Brainy ships with full TypeScript types. No `@types/` package needed: Brainy ships with full TypeScript types. No `@types/` package needed:
```typescript ```typescript
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()

View file

@ -66,7 +66,7 @@ const results = await brain.search("query")
**New diagnostics for capacity planning and performance tuning.** **New diagnostics for capacity planning and performance tuning.**
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()
@ -112,7 +112,7 @@ Recommendations: ${stats.recommendations.join(', ')}
### Step 1: Update Package ### Step 1: Update Package
```bash ```bash
npm install @soulcraftlabs/brainy@latest npm install @soulcraft/brainy@latest
``` ```
### Step 2: Restart Your Application ### Step 2: Restart Your Application
@ -134,7 +134,7 @@ npm run start
### Check Adaptive Sizing is Working ### Check Adaptive Sizing is Working
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()
@ -218,7 +218,7 @@ For debugging or compatibility testing:
If you need to rollback to v3.35.0: If you need to rollback to v3.35.0:
```bash ```bash
npm install @soulcraftlabs/brainy@3.35.0 npm install @soulcraft/brainy@3.35.0
``` ```
**Note:** We don't anticipate any issues, but rollback is straightforward if needed. **Note:** We don't anticipate any issues, but rollback is straightforward if needed.
@ -367,7 +367,7 @@ if (stats.fairness.fairnessViolation) {
## Next Steps ## Next Steps
1. ✅ **Upgrade:** `npm install @soulcraftlabs/brainy@latest` 1. ✅ **Upgrade:** `npm install @soulcraft/brainy@latest`
2. 📊 **Monitor:** Use `getCacheStats()` to verify performance improvements 2. 📊 **Monitor:** Use `getCacheStats()` to verify performance improvements
3. 🎯 **Tune:** Adjust based on recommendations (if needed) 3. 🎯 **Tune:** Adjust based on recommendations (if needed)
4. 📖 **Read:** [Operations Guide](../operations/capacity-planning.md) for capacity planning 4. 📖 **Read:** [Operations Guide](../operations/capacity-planning.md) for capacity planning

View file

@ -37,7 +37,7 @@ This single WASM file contains everything needed for sentence embeddings.
```bash ```bash
# Bun as a runtime — supported and recommended # Bun as a runtime — supported and recommended
bun add @soulcraftlabs/brainy bun add @soulcraft/brainy
bun run server.ts bun run server.ts
``` ```

View file

@ -80,7 +80,7 @@ If you read raw stored records (fact-log scanners, export tooling), use
the exported shape-aware splitters — they handle both record eras: the exported shape-aware splitters — they handle both record eras:
```typescript ```typescript
import { splitNounMetadataRecord } from '@soulcraftlabs/brainy' import { splitNounMetadataRecord } from '@soulcraft/brainy'
const { reserved, custom } = splitNounMetadataRecord(rawRecord) const { reserved, custom } = splitNounMetadataRecord(rawRecord)
// reserved = engine fields · custom = the user's bag, ANY names // reserved = engine fields · custom = the user's bag, ANY names
``` ```
@ -88,7 +88,7 @@ const { reserved, custom } = splitNounMetadataRecord(rawRecord)
Feature detection (never version-sniff): Feature detection (never version-sniff):
```typescript ```typescript
import * as brainy from '@soulcraftlabs/brainy' import * as brainy from '@soulcraft/brainy'
const lawActive = 'FIELD_ADDRESSING_CAPABILITY' in brainy // 'field-addressing/v1' const lawActive = 'FIELD_ADDRESSING_CAPABILITY' in brainy // 'field-addressing/v1'
``` ```

View file

@ -9,7 +9,7 @@ Complete guide to integrating Brainy with Next.js applications, covering App Rou
```bash ```bash
npx create-next-app@latest my-brainy-app npx create-next-app@latest my-brainy-app
cd my-brainy-app cd my-brainy-app
npm install @soulcraftlabs/brainy npm install @soulcraft/brainy
``` ```
### Basic Setup ### Basic Setup
@ -18,7 +18,7 @@ npm install @soulcraftlabs/brainy
// app/components/BrainyProvider.jsx // app/components/BrainyProvider.jsx
'use client' 'use client'
import { createContext, useContext, useEffect, useState } from 'react' import { createContext, useContext, useEffect, useState } from 'react'
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const BrainyContext = createContext() const BrainyContext = createContext()
@ -271,7 +271,7 @@ export default function SearchPage() {
```javascript ```javascript
// app/api/search/route.js (App Router) // app/api/search/route.js (App Router)
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
let brain = null let brain = null
@ -332,7 +332,7 @@ export async function GET() {
```javascript ```javascript
// pages/api/search.js (Pages Router) // pages/api/search.js (Pages Router)
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
let brain = null let brain = null
@ -374,7 +374,7 @@ export default async function handler(req, res) {
```javascript ```javascript
// app/api/data/route.js // app/api/data/route.js
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
let brain = null let brain = null
@ -418,7 +418,7 @@ export async function POST(request) {
```jsx ```jsx
// app/actions/brainy.js // app/actions/brainy.js
'use server' 'use server'
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
let brain = null let brain = null
@ -630,7 +630,7 @@ CMD ["npm", "start"]
/** @type {import('next').NextConfig} */ /** @type {import('next').NextConfig} */
const nextConfig = { const nextConfig = {
experimental: { experimental: {
serverComponentsExternalPackages: ['@soulcraftlabs/brainy'] serverComponentsExternalPackages: ['@soulcraft/brainy']
}, },
webpack: (config, { isServer }) => { webpack: (config, { isServer }) => {
if (!isServer) { if (!isServer) {
@ -797,7 +797,7 @@ export function rateLimit(req, limit = 100, window = 60000) {
// app/contexts/BrainyContext.jsx // app/contexts/BrainyContext.jsx
'use client' 'use client'
import { createContext, useContext, useReducer, useEffect } from 'react' import { createContext, useContext, useReducer, useEffect } from 'react'
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const BrainyContext = createContext() const BrainyContext = createContext()
@ -873,7 +873,7 @@ import { BrainyProvider } from '../app/components/BrainyProvider'
import { Search } from '../app/components/Search' import { Search } from '../app/components/Search'
// Mock Brainy // Mock Brainy
jest.mock('@soulcraftlabs/brainy', () => ({ jest.mock('@soulcraft/brainy', () => ({
Brainy: jest.fn().mockImplementation(() => ({ Brainy: jest.fn().mockImplementation(() => ({
init: jest.fn().mockResolvedValue(undefined), init: jest.fn().mockResolvedValue(undefined),
find: jest.fn().mockResolvedValue([ find: jest.fn().mockResolvedValue([

View file

@ -32,7 +32,7 @@ Brainy 7.31.0 adds a per-entity revision counter so multiple writers can coordin
Every distributed-job scheduler eventually wants this exact loop: Every distributed-job scheduler eventually wants this exact loop:
```ts ```ts
import { Brainy, RevisionConflictError } from '@soulcraftlabs/brainy' import { Brainy, RevisionConflictError } from '@soulcraft/brainy'
const LOCK_ID = '...uuid for this job slot...' const LOCK_ID = '...uuid for this job slot...'
@ -137,7 +137,7 @@ await brain.addIfMissing({ // ← not a real API
It's race-prone as a plain read-then-write: two concurrent imports both see "not found," both insert, you get duplicates. Without a unique-index primitive (which Brainy doesn't have today), close the race with whole-store CAS — read at a pinned generation, then commit only if nothing moved: It's race-prone as a plain read-then-write: two concurrent imports both see "not found," both insert, you get duplicates. Without a unique-index primitive (which Brainy doesn't have today), close the race with whole-store CAS — read at a pinned generation, then commit only if nothing moved:
```ts ```ts
import { GenerationConflictError } from '@soulcraftlabs/brainy' import { GenerationConflictError } from '@soulcraft/brainy'
async function addIfMissingByEmail(email: string, data: string) { async function addIfMissingByEmail(email: string, data: string) {
for (let attempt = 0; attempt < 5; attempt++) { for (let attempt = 0; attempt < 5; attempt++) {

View file

@ -18,13 +18,13 @@ Get Brainy running in under a minute.
## 1. Install ## 1. Install
```bash ```bash
npm install @soulcraftlabs/brainy npm install @soulcraft/brainy
``` ```
## 2. Initialize ## 2. Initialize
```typescript ```typescript
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()
@ -67,7 +67,7 @@ await brain.relate({
## 5. Query with Triple Intelligence ## 5. Query with Triple Intelligence
```typescript ```typescript
import type { Result } from '@soulcraftlabs/brainy' import type { Result } from '@soulcraft/brainy'
// All three search paradigms in one call // All three search paradigms in one call
const results: Result[] = await brain.find({ const results: Result[] = await brain.find({

View file

@ -11,7 +11,7 @@
### One Interface for Everything ### One Interface for Everything
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brain = await Brainy.create() const brain = await Brainy.create()
@ -78,7 +78,7 @@ interface ImportProgress {
```typescript ```typescript
import { useState } from 'react' import { useState } from 'react'
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
function UniversalImportProgress({ file }: { file: File }) { function UniversalImportProgress({ file }: { file: File }) {
const [progress, setProgress] = useState({ const [progress, setProgress] = useState({
@ -177,7 +177,7 @@ function UniversalImportProgress({ file }: { file: File }) {
```typescript ```typescript
import ora from 'ora' import ora from 'ora'
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
async function importWithProgress(filePath: string) { async function importWithProgress(filePath: string) {
const spinner = ora('Starting import...').start() const spinner = ora('Starting import...').start()

View file

@ -28,7 +28,7 @@ on-disk layout (memory's "disk" is a JS Map).
## Quick start ## Quick start
```ts ```ts
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
// Filesystem (recommended for any persistent workload): // Filesystem (recommended for any persistent workload):
const brain = new Brainy({ const brain = new Brainy({
@ -134,7 +134,7 @@ config; the `type` is optional.
If you want to skip the factory: If you want to skip the factory:
```ts ```ts
import { FileSystemStorage, MemoryStorage } from '@soulcraftlabs/brainy' import { FileSystemStorage, MemoryStorage } from '@soulcraft/brainy'
const fsStorage = new FileSystemStorage('./brainy-data') const fsStorage = new FileSystemStorage('./brainy-data')
const memStorage = new MemoryStorage() const memStorage = new MemoryStorage()

View file

@ -34,7 +34,7 @@ Three layers solve this:
### Write ### Write
```typescript ```typescript
import { Brainy, NounType } from '@soulcraftlabs/brainy' import { Brainy, NounType } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()
@ -240,7 +240,7 @@ await brain.migrateField({
A realistic adoption sequence for a brain that started without these primitives: A realistic adoption sequence for a brain that started without these primitives:
```typescript ```typescript
import { Brainy, NounType } from '@soulcraftlabs/brainy' import { Brainy, NounType } from '@soulcraft/brainy'
const brain = new Brainy({ storage: { type: 'filesystem', path: './brain-data' } }) const brain = new Brainy({ storage: { type: 'filesystem', path: './brain-data' } })
await brain.init() await brain.init()

View file

@ -25,7 +25,7 @@ content — and how 8.0 recovers it for you.
## TL;DR ## TL;DR
- **Just upgrade to `@soulcraftlabs/brainy@8.0.12` (or later) and open the store.** - **Just upgrade to `@soulcraft/brainy@8.0.12` (or later) and open the store.**
If a previous upgrade left VFS content stranded, 8.0.12 **heals it on open**, If a previous upgrade left VFS content stranded, 8.0.12 **heals it on open**,
with no operator action. with no operator action.
- Want to force or script it? Call **`await brain.vfs.adoptOrphanedBlobs()`**. - Want to force or script it? Call **`await brain.vfs.adoptOrphanedBlobs()`**.
@ -90,7 +90,7 @@ So the operator action for a stranded store is simply: **upgrade to 8.0.12 and
open it.** open it.**
```ts ```ts
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
// Opening the store is all that is required — recovery runs during init(). // Opening the store is all that is required — recovery runs during init().
const brain = new Brainy({ storage: { type: 'filesystem', path: '/data/my-store' } }) const brain = new Brainy({ storage: { type: 'filesystem', path: '/data/my-store' } })
@ -182,5 +182,5 @@ and opening each store is sufficient.
The recovery is copy-only, so no rollback of the recovery itself is ever needed. The recovery is copy-only, so no rollback of the recovery itself is ever needed.
If you need to roll back the **whole** 7→8 upgrade, restore the directory from If you need to roll back the **whole** 7→8 upgrade, restore the directory from
your pre-upgrade backup (retained automatically while recovery is incomplete, or your pre-upgrade backup (retained automatically while recovery is incomplete, or
your own snapshot) and pin `@soulcraftlabs/brainy@7.x`. 8.0 does not keep the old your own snapshot) and pin `@soulcraft/brainy@7.x`. 8.0 does not keep the old
branch layout in place, so a directory-level restore is the rollback path. branch layout in place, so a directory-level restore is the rollback path.

View file

@ -12,7 +12,7 @@ Complete guide to integrating Brainy with Vue.js applications, covering Vue 3, N
npm create vue@latest my-brainy-app npm create vue@latest my-brainy-app
cd my-brainy-app cd my-brainy-app
npm install npm install
npm install @soulcraftlabs/brainy npm install @soulcraft/brainy
``` ```
### Basic Setup ### Basic Setup
@ -574,7 +574,7 @@ Nuxt's server engine (Nitro) is the natural home for Brainy: it runs on Node/Bun
```javascript ```javascript
// server/utils/brain.js (server-only — Nitro never bundles this into the client) // server/utils/brain.js (server-only — Nitro never bundles this into the client)
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
let brainPromise let brainPromise
@ -1201,7 +1201,7 @@ import vue from '@vitejs/plugin-vue'
export default defineConfig({ export default defineConfig({
plugins: [vue()], plugins: [vue()],
ssr: { ssr: {
external: ['@soulcraftlabs/brainy'] external: ['@soulcraft/brainy']
} }
}) })
``` ```

View file

@ -24,7 +24,7 @@ Brainy's neural extraction system uses a **4-signal ensemble architecture** to c
### Method 1: Brain Instance (Recommended) ### Method 1: Brain Instance (Recommended)
```typescript ```typescript
import { Brainy, NounType } from '@soulcraftlabs/brainy' import { Brainy, NounType } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()
@ -62,9 +62,9 @@ const people = await brain.extractEntities('...', {
import { import {
SmartExtractor, SmartExtractor,
SmartRelationshipExtractor SmartRelationshipExtractor
} from '@soulcraftlabs/brainy' } from '@soulcraft/brainy'
// Or use subpath imports: // Or use subpath imports:
import { SmartExtractor } from '@soulcraftlabs/brainy/neural/SmartExtractor' import { SmartExtractor } from '@soulcraft/brainy/neural/SmartExtractor'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()
@ -176,7 +176,7 @@ const withVectors = await brain.extractEntities(text, {
**Direct entity type classifier.** Use when you have pre-detected candidates or need custom configuration. **Direct entity type classifier.** Use when you have pre-detected candidates or need custom configuration.
```typescript ```typescript
import { SmartExtractor, FormatContext } from '@soulcraftlabs/brainy' import { SmartExtractor, FormatContext } from '@soulcraft/brainy'
const extractor = new SmartExtractor(brain, { const extractor = new SmartExtractor(brain, {
minConfidence: 0.7, // Threshold minConfidence: 0.7, // Threshold
@ -229,7 +229,7 @@ interface ExtractionResult {
**Relationship type classifier.** Determines verb/relationship types between entities. **Relationship type classifier.** Determines verb/relationship types between entities.
```typescript ```typescript
import { SmartRelationshipExtractor } from '@soulcraftlabs/brainy' import { SmartRelationshipExtractor } from '@soulcraft/brainy'
const relExtractor = new SmartRelationshipExtractor(brain, { const relExtractor = new SmartRelationshipExtractor(brain, {
minConfidence: 0.6, minConfidence: 0.6,
@ -286,7 +286,7 @@ const rel = await relExtractor.infer(
**Full extraction orchestrator.** Handles candidate detection, classification, and deduplication. **Full extraction orchestrator.** Handles candidate detection, classification, and deduplication.
```typescript ```typescript
import { NeuralEntityExtractor } from '@soulcraftlabs/brainy' import { NeuralEntityExtractor } from '@soulcraft/brainy'
const extractor = new NeuralEntityExtractor(brain) const extractor = new NeuralEntityExtractor(brain)
@ -607,7 +607,7 @@ const locations = entities.filter(e => e.type === NounType.Location)
### Example 2: Excel Data Classification ### Example 2: Excel Data Classification
```typescript ```typescript
import { SmartExtractor } from '@soulcraftlabs/brainy' import { SmartExtractor } from '@soulcraft/brainy'
const extractor = new SmartExtractor(brain) const extractor = new SmartExtractor(brain)
@ -629,7 +629,7 @@ for (let i = 0; i < cells.length; i++) {
### Example 3: Relationship Extraction ### Example 3: Relationship Extraction
```typescript ```typescript
import { SmartRelationshipExtractor } from '@soulcraftlabs/brainy' import { SmartRelationshipExtractor } from '@soulcraft/brainy'
const relExtractor = new SmartRelationshipExtractor(brain) const relExtractor = new SmartRelationshipExtractor(brain)

View file

@ -204,8 +204,8 @@ await brain.add({ data: { name: 'Entity' }, type: NounType.Thing })
### Basic Add Operation ### Basic Add Operation
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
import { NounType } from '@soulcraftlabs/brainy/types' import { NounType } from '@soulcraft/brainy/types'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()
@ -428,7 +428,7 @@ await brain.relate({ ... }) // a crash here leaves the entity unlinked
```typescript ```typescript
import { describe, it, expect } from 'vitest' import { describe, it, expect } from 'vitest'
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
describe('Transaction Tests', () => { describe('Transaction Tests', () => {
it('should rollback on failure', async () => { it('should rollback on failure', async () => {

View file

@ -23,7 +23,7 @@ The Universal Display Augmentation is a powerful AI-powered system that automati
### Basic Usage ### Basic Usage
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brainy = new Brainy() const brainy = new Brainy()
await brainy.init() await brainy.init()

View file

@ -71,9 +71,9 @@ Let's build a projection that organizes files by priority (high, medium, low):
### Step 1: Create the Strategy Class ### Step 1: Create the Strategy Class
```typescript ```typescript
import { BaseProjectionStrategy } from '@soulcraftlabs/brainy/vfs/semantic' import { BaseProjectionStrategy } from '@soulcraft/brainy/vfs/semantic'
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
import { VirtualFileSystem, VFSEntity } from '@soulcraftlabs/brainy/vfs' import { VirtualFileSystem, VFSEntity } from '@soulcraft/brainy/vfs'
export class PriorityProjection extends BaseProjectionStrategy { export class PriorityProjection extends BaseProjectionStrategy {
readonly name = 'priority' readonly name = 'priority'
@ -141,7 +141,7 @@ export class PriorityProjection extends BaseProjectionStrategy {
### Step 2: Register the Strategy ### Step 2: Register the Strategy
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
import { PriorityProjection } from './PriorityProjection' import { PriorityProjection } from './PriorityProjection'
const brain = new Brainy() const brain = new Brainy()
@ -537,7 +537,7 @@ Use the projection's resolve cache:
```typescript ```typescript
import { describe, it, expect, beforeAll } from 'vitest' import { describe, it, expect, beforeAll } from 'vitest'
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
import { PriorityProjection } from './PriorityProjection' import { PriorityProjection } from './PriorityProjection'
describe('PriorityProjection', () => { describe('PriorityProjection', () => {
@ -714,7 +714,7 @@ async resolve(brain, vfs, value: string) {
3. Use appropriate limits: Don't fetch more than needed 3. Use appropriate limits: Don't fetch more than needed
### Type errors ### Type errors
1. Import correct types: `import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy'` 1. Import correct types: `import { Brainy, VirtualFileSystem } from '@soulcraft/brainy'`
2. Use `as VFSEntity` when mapping results 2. Use `as VFSEntity` when mapping results
3. Check BaseProjectionStrategy import 3. Check BaseProjectionStrategy import

View file

@ -14,11 +14,11 @@ A file explorer that:
## ⚡ Step 1: Basic Setup (1 minute) ## ⚡ Step 1: Basic Setup (1 minute)
```bash ```bash
npm install @soulcraftlabs/brainy npm install @soulcraft/brainy
``` ```
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
// ✅ CORRECT: Use filesystem storage for production // ✅ CORRECT: Use filesystem storage for production
const brain = new Brainy({ const brain = new Brainy({
@ -115,7 +115,7 @@ Here's a complete React component using the correct patterns:
```tsx ```tsx
import React, { useState, useEffect } from 'react' import React, { useState, useEffect } from 'react'
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
export function FileExplorer() { export function FileExplorer() {
const [brain, setBrain] = useState(null) const [brain, setBrain] = useState(null)
@ -288,8 +288,8 @@ Your file explorer is now working! Here's what to explore next:
### "Module not found" errors ### "Module not found" errors
```bash ```bash
# Make sure you're using the right import # Make sure you're using the right import
npm ls @soulcraftlabs/brainy # Check version npm ls @soulcraft/brainy # Check version
npm install @soulcraftlabs/brainy@latest # Update if needed npm install @soulcraft/brainy@latest # Update if needed
``` ```
### "VFS not initialized" errors ### "VFS not initialized" errors

View file

@ -24,7 +24,7 @@ Brainy VFS is a revolutionary virtual filesystem that runs on top of Brainy's ne
## Quick Start ## Quick Start
```javascript ```javascript
import { VirtualFileSystem } from '@soulcraftlabs/brainy/vfs' import { VirtualFileSystem } from '@soulcraft/brainy/vfs'
// Initialize the VFS // Initialize the VFS
const vfs = new VirtualFileSystem({ const vfs = new VirtualFileSystem({
@ -381,7 +381,7 @@ Brainy VFS fully leverages Brainy's revolutionary Triple Intelligence system:
## Installation ## Installation
```bash ```bash
npm install @soulcraftlabs/brainy npm install @soulcraft/brainy
``` ```
## Requirements ## Requirements

View file

@ -135,7 +135,7 @@ Mount VFS as a native filesystem on Linux/Mac/Windows.
```typescript ```typescript
// Planned (research phase) // Planned (research phase)
import { mountVFS } from '@soulcraftlabs/brainy/vfs/fuse' import { mountVFS } from '@soulcraft/brainy/vfs/fuse'
await mountVFS(vfs, { await mountVFS(vfs, {
mountPoint: '/mnt/brainy', mountPoint: '/mnt/brainy',
@ -160,7 +160,7 @@ These features would benefit from community contributions. If you're interested
### Express.js Static Middleware ### Express.js Static Middleware
```typescript ```typescript
// Wanted: Community contribution // Wanted: Community contribution
import { createStaticMiddleware } from '@soulcraftlabs/brainy/vfs/express' import { createStaticMiddleware } from '@soulcraft/brainy/vfs/express'
app.use('/files', createStaticMiddleware(vfs, { app.use('/files', createStaticMiddleware(vfs, {
index: ['index.html', 'index.md'], index: ['index.html', 'index.md'],
@ -172,7 +172,7 @@ app.use('/files', createStaticMiddleware(vfs, {
### VSCode Extension ### VSCode Extension
```typescript ```typescript
// Wanted: Community contribution // Wanted: Community contribution
import { VFSProvider } from '@soulcraftlabs/brainy/vfs/vscode' import { VFSProvider } from '@soulcraft/brainy/vfs/vscode'
const provider = new VFSProvider(vfs) const provider = new VFSProvider(vfs)
vscode.workspace.registerFileSystemProvider('brainy', provider) vscode.workspace.registerFileSystemProvider('brainy', provider)

View file

@ -327,7 +327,7 @@ console.log(id1 === id2 && id2 === id3) // true
Create your own semantic dimensions: Create your own semantic dimensions:
```typescript ```typescript
import { BaseProjectionStrategy } from '@soulcraftlabs/brainy/vfs/semantic' import { BaseProjectionStrategy } from '@soulcraft/brainy/vfs/semantic'
class PriorityProjection extends BaseProjectionStrategy { class PriorityProjection extends BaseProjectionStrategy {
readonly name = 'priority' readonly name = 'priority'

View file

@ -7,7 +7,7 @@ Brainy's Virtual Filesystem (VFS) provides a POSIX-like filesystem interface tha
## Quick Start ## Quick Start
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
// Initialize Brainy // Initialize Brainy
const brain = new Brainy({ const brain = new Brainy({
@ -598,7 +598,7 @@ const user = await store.findById('users', 'user123')
VFS uses standard POSIX-style errors: VFS uses standard POSIX-style errors:
```typescript ```typescript
import { VFSError, VFSErrorCode } from '@soulcraftlabs/brainy' import { VFSError, VFSErrorCode } from '@soulcraft/brainy'
try { try {
await vfs.readFile('/nonexistent.txt') await vfs.readFile('/nonexistent.txt')

View file

@ -280,7 +280,7 @@ GitBridge provides Git import/export capabilities:
#### GitBridge Usage #### GitBridge Usage
```javascript ```javascript
// Import and instantiate GitBridge // Import and instantiate GitBridge
import { GitBridge } from '@soulcraftlabs/brainy' import { GitBridge } from '@soulcraft/brainy'
const gitBridge = new GitBridge(vfs, brain) const gitBridge = new GitBridge(vfs, brain)
// Export VFS to Git repository structure // Export VFS to Git repository structure
@ -452,7 +452,7 @@ This ordering prevents race conditions where file writes might fail because pare
## Complete Example ## Complete Example
```javascript ```javascript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
async function vfsExample() { async function vfsExample() {
// Initialize // Initialize

View file

@ -196,5 +196,5 @@ await brain.relate({
Always import and use the type enums: Always import and use the type enums:
```javascript ```javascript
import { NounType, VerbType } from '@soulcraftlabs/brainy' import { NounType, VerbType } from '@soulcraft/brainy'
``` ```

View file

@ -5,7 +5,7 @@
The Brainy VFS is automatically initialized during `brain.init()`. No separate initialization needed! The Brainy VFS is automatically initialized during `brain.init()`. No separate initialization needed!
```javascript ```javascript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
// Create and initialize Brainy // Create and initialize Brainy
const brain = new Brainy({ const brain = new Brainy({
@ -71,7 +71,7 @@ VFS stores files as entities and relationships in the same graph as everything e
## Complete Example ## Complete Example
```javascript ```javascript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
async function useVFS() { async function useVFS() {
// Initialize Brainy // Initialize Brainy
@ -100,7 +100,7 @@ useVFS().catch(console.error)
## TypeScript Usage ## TypeScript Usage
```typescript ```typescript
import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy' import { Brainy, VirtualFileSystem } from '@soulcraft/brainy'
class FileManager { class FileManager {
private brain: Brainy private brain: Brainy

View file

@ -37,7 +37,7 @@ Brainy VFS provides safe, tree-aware methods that prevent these issues:
### Method 1: Use `getDirectChildren()` (Recommended) ### Method 1: Use `getDirectChildren()` (Recommended)
```typescript ```typescript
import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy' import { Brainy, VirtualFileSystem } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()
@ -97,7 +97,7 @@ Here's a complete example using React:
```tsx ```tsx
import React, { useState, useEffect } from 'react' import React, { useState, useEffect } from 'react'
import { VirtualFileSystem } from '@soulcraftlabs/brainy' import { VirtualFileSystem } from '@soulcraft/brainy'
interface FileNode { interface FileNode {
name: string name: string
@ -177,7 +177,7 @@ function TreeView({ node, onToggle, expanded }) {
If you must build trees manually from flat lists, use the `VFSTreeUtils`: If you must build trees manually from flat lists, use the `VFSTreeUtils`:
```typescript ```typescript
import { VFSTreeUtils } from '@soulcraftlabs/brainy/vfs' import { VFSTreeUtils } from '@soulcraft/brainy/vfs'
// Get all entities somehow // Get all entities somehow
const allEntities = await vfs.getDescendants('/root') const allEntities = await vfs.getDescendants('/root')

View file

@ -7,7 +7,7 @@
* the Bluesky firehose with Brainy's distributed architecture * the Bluesky firehose with Brainy's distributed architecture
*/ */
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
import { WebSocket } from 'ws' import { WebSocket } from 'ws'
// ===================================================== // =====================================================

View file

@ -14,7 +14,7 @@
* ts-node examples/monitor-cache-performance.ts * ts-node examples/monitor-cache-performance.ts
*/ */
import { Brainy, NounType } from '@soulcraftlabs/brainy' import { Brainy, NounType } from '@soulcraft/brainy'
// ANSI color codes for pretty output // ANSI color codes for pretty output
const colors = { const colors = {

View file

@ -5,7 +5,7 @@ Connect Brainy to spreadsheets, BI tools, and external systems with zero configu
## Quick Start ## Quick Start
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brain = new Brainy({ integrations: true }) const brain = new Brainy({ integrations: true })
await brain.init() await brain.init()
@ -178,7 +178,7 @@ Webhooks include `X-Brainy-Signature` header with HMAC-SHA256 signature.
### Minimal (in-memory): ### Minimal (in-memory):
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brain = new Brainy({ integrations: true }) const brain = new Brainy({ integrations: true })
await brain.init() await brain.init()
@ -194,7 +194,7 @@ console.log(brain.hub.getInstructions())
```typescript ```typescript
import express from 'express' import express from 'express'
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const app = express() const app = express()
const brain = new Brainy({ const brain = new Brainy({
@ -232,7 +232,7 @@ app.listen(3000, () => {
```typescript ```typescript
import { Hono } from 'hono' import { Hono } from 'hono'
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const app = new Hono() const app = new Hono()

View file

@ -99,7 +99,7 @@ Add the `BRAINY_URL` script property in Apps Script settings.
The simplest way to enable all integrations: The simplest way to enable all integrations:
```javascript ```javascript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brain = new Brainy({ integrations: true }) const brain = new Brainy({ integrations: true })
await brain.init() await brain.init()
@ -112,7 +112,7 @@ With Express:
```javascript ```javascript
import express from 'express' import express from 'express'
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const app = express() const app = express()
const brain = new Brainy({ integrations: true }) const brain = new Brainy({ integrations: true })

8
package-lock.json generated
View file

@ -1,12 +1,12 @@
{ {
"name": "@soulcraftlabs/brainy", "name": "@soulcraft/brainy",
"version": "10.4.4", "version": "10.4.0",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "@soulcraftlabs/brainy", "name": "@soulcraft/brainy",
"version": "10.4.4", "version": "10.4.0",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"@msgpack/msgpack": "^3.1.2", "@msgpack/msgpack": "^3.1.2",

View file

@ -1,7 +1,6 @@
{ {
"name": "@soulcraftlabs/brainy", "name": "@soulcraft/brainy",
"version": "10.4.4", "version": "10.4.0",
"brainyContract": 1,
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
"main": "dist/index.js", "main": "dist/index.js",
"module": "dist/index.js", "module": "dist/index.js",
@ -127,16 +126,15 @@
"license": "MIT", "license": "MIT",
"private": false, "private": false,
"publishConfig": { "publishConfig": {
"access": "public", "access": "public"
"registry": "https://source.soulcraft.com/api/packages/soulcraftlabs/npm/"
}, },
"homepage": "https://source.soulcraft.com/soulcraftlabs/open-brainy", "homepage": "https://source.soulcraft.com/soulcraft/brainy",
"bugs": { "bugs": {
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/issues" "url": "https://source.soulcraft.com/soulcraft/brainy/issues"
}, },
"repository": { "repository": {
"type": "git", "type": "git",
"url": "git+https://source.soulcraft.com/soulcraftlabs/open-brainy.git" "url": "git+https://source.soulcraft.com/soulcraft/brainy.git"
}, },
"files": [ "files": [
"dist/**/*.js", "dist/**/*.js",

View file

@ -10,7 +10,6 @@ import { TransformerEmbedding } from '../src/utils/embedding.js'
import * as fs from 'fs/promises' import * as fs from 'fs/promises'
import * as path from 'path' import * as path from 'path'
import { fileURLToPath } from 'url' import { fileURLToPath } from 'url'
import { resolveDeterministicStamp } from './lib/deterministicStamp.js'
const __dirname = path.dirname(fileURLToPath(import.meta.url)) const __dirname = path.dirname(fileURLToPath(import.meta.url))
@ -98,22 +97,13 @@ async function buildEmbeddedPatterns() {
// Convert to base64 for embedding in TypeScript // Convert to base64 for embedding in TypeScript
const uint8 = new Uint8Array(buffer) const uint8 = new Uint8Array(buffer)
const base64 = Buffer.from(uint8).toString('base64') const base64 = Buffer.from(uint8).toString('base64')
// Deterministic stamp: derived from the git commit time of this
// generator's inputs, never from wall-clock time — two builds of the
// same source tree must produce byte-identical output.
const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedPatterns.ts')
const generatedStamp = resolveDeterministicStamp(
[path.join(__dirname, 'buildEmbeddedPatterns.ts'), libraryPath],
outputPath
)
// Generate TypeScript file with everything embedded // Generate TypeScript file with everything embedded
const tsContent = `/** const tsContent = `/**
* 🧠 BRAINY EMBEDDED PATTERNS * 🧠 BRAINY EMBEDDED PATTERNS
* *
* AUTO-GENERATED - DO NOT EDIT * AUTO-GENERATED - DO NOT EDIT
* Generated: ${generatedStamp} * Generated: ${new Date().toISOString()}
* Patterns: ${libraryData.patterns.length} * Patterns: ${libraryData.patterns.length}
* Coverage: 94-98% of all queries * Coverage: 94-98% of all queries
* *
@ -207,6 +197,7 @@ prodLog.info(\`🧠 Brainy Pattern Library loaded: \${EMBEDDED_PATTERNS.length}
` `
// Write the TypeScript file // Write the TypeScript file
const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedPatterns.ts')
await fs.writeFile(outputPath, tsContent) await fs.writeFile(outputPath, tsContent)
// Report statistics // Report statistics

View file

@ -11,7 +11,6 @@ import * as fs from 'fs/promises'
import * as path from 'path' import * as path from 'path'
import { fileURLToPath } from 'url' import { fileURLToPath } from 'url'
import { NounType, VerbType } from '../src/types/graphTypes.js' import { NounType, VerbType } from '../src/types/graphTypes.js'
import { resolveDeterministicStamp } from './lib/deterministicStamp.js'
const __dirname = path.dirname(fileURLToPath(import.meta.url)) const __dirname = path.dirname(fileURLToPath(import.meta.url))
@ -374,24 +373,12 @@ async function buildTypeEmbeddings() {
const uint8 = new Uint8Array(buffer) const uint8 = new Uint8Array(buffer)
const base64 = Buffer.from(uint8).toString('base64') const base64 = Buffer.from(uint8).toString('base64')
// Deterministic stamp: derived from the git commit time of this
// generator's inputs, never from wall-clock time — two builds of the
// same source tree must produce byte-identical output.
const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedTypeEmbeddings.ts')
const generatedStamp = resolveDeterministicStamp(
[
path.join(__dirname, 'buildTypeEmbeddings.ts'),
path.join(__dirname, '..', 'src', 'types', 'graphTypes.ts')
],
outputPath
)
// Generate TypeScript file // Generate TypeScript file
const tsContent = `/** const tsContent = `/**
* 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS
* *
* AUTO-GENERATED - DO NOT EDIT * AUTO-GENERATED - DO NOT EDIT
* Generated: ${generatedStamp} * Generated: ${new Date().toISOString()}
* Noun Types: ${nounTypes.length} * Noun Types: ${nounTypes.length}
* Verb Types: ${verbTypes.length} * Verb Types: ${verbTypes.length}
* *
@ -408,7 +395,7 @@ export const TYPE_METADATA = {
verbTypes: ${verbTypes.length}, verbTypes: ${verbTypes.length},
totalTypes: ${totalTypes}, totalTypes: ${totalTypes},
embeddingDimensions: ${embeddingDim}, embeddingDimensions: ${embeddingDim},
generatedAt: "${generatedStamp}", generatedAt: "${new Date().toISOString()}",
sizeBytes: { sizeBytes: {
embeddings: ${buffer.byteLength}, embeddings: ${buffer.byteLength},
base64: ${base64.length} base64: ${base64.length}
@ -507,6 +494,7 @@ prodLog.info(\`🧠 Brainy Type Embeddings loaded: \${TYPE_METADATA.nounTypes} n
` `
// Write the TypeScript file // Write the TypeScript file
const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedTypeEmbeddings.ts')
await fs.writeFile(outputPath, tsContent) await fs.writeFile(outputPath, tsContent)
// Report statistics // Report statistics

View file

@ -1,128 +0,0 @@
#!/usr/bin/env node
/**
* Emit this build's API-contract manifest to docs/api-contract.json.
*
* WHY IT IS GENERATED, NOT WRITTEN: a hand-kept list of doors drifts from the
* code the first time somebody adds one. This reads the surface the build
* actually exposes the prototype's own methods and accessors, the exported
* error classes, the `where` operator sets, the field-addressing vocabulary,
* the health verdicts so a diff between two engines' manifests is a diff
* between two engines, never between two authors.
*
* Requirement marking (required / optional per door) is NOT derivable from the
* surface it is a commitment, recorded with the contract's owner rather than
* here. This manifest carries the surface; the promise lives with the contract.
*
* Usage: node scripts/emit-contract-manifest.mjs [--check]
* --check exits non-zero when the committed manifest is stale.
*/
import { writeFileSync, readFileSync, existsSync } from 'node:fs'
import { join, dirname } from 'node:path'
import { fileURLToPath } from 'node:url'
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..')
const OUT = join(ROOT, 'docs', 'api-contract.json')
const { Brainy } = await import(join(ROOT, 'dist', 'brainy.js'))
const errorsModule = await import(join(ROOT, 'dist', 'errors', 'brainyError.js'))
const versionModule = await import(join(ROOT, 'dist', 'utils', 'version.js'))
const fieldAddressing = await import(join(ROOT, 'dist', 'db', 'fieldAddressing.js'))
/** Every own method and accessor on the class's prototype, minus the private ones. */
function surfaceOf(ctor) {
const doors = []
for (const name of Object.getOwnPropertyNames(ctor.prototype)) {
if (name === 'constructor' || name.startsWith('_')) continue
const descriptor = Object.getOwnPropertyDescriptor(ctor.prototype, name)
if (!descriptor) continue
if (typeof descriptor.value === 'function') {
doors.push({ name, kind: 'method', arity: descriptor.value.length })
} else if (descriptor.get) {
doors.push({ name, kind: 'accessor' })
}
}
return doors.sort((a, b) => a.name.localeCompare(b.name))
}
const errors = Object.entries(errorsModule)
.filter(([name, value]) => typeof value === 'function' && /Error$/.test(name))
.map(([name]) => name)
.sort()
// The operator sets, read from the engine's own refusal message so the
// manifest can never disagree with the validator.
const filterSource = readFileSync(join(ROOT, 'src', 'utils', 'metadataFilter.ts'), 'utf-8')
const acceptedMatch = filterSource.match(/const VALUE_OPERATORS = new Set<string>\(\[([\s\S]*?)\]\)/)
if (!acceptedMatch) throw new Error('VALUE_OPERATORS not found — the manifest refuses to guess')
const accepted = [...acceptedMatch[1].matchAll(/'([^']+)'/g)].map((m) => m[1]).sort()
const indexSource = readFileSync(join(ROOT, 'src', 'utils', 'metadataIndex.ts'), 'utf-8')
const refusedByIndex = ['endsWith', 'length', 'matches', 'startsWith'].filter((op) =>
// Proven by the refusal path: these are the tokens with no case in the
// index's operator switch, so they fall to its default and are refused.
!new RegExp(`case '${op}':`).test(indexSource)
)
const servedOnIndex = accepted.filter((op) => !refusedByIndex.includes(op))
const manifest = {
contractVersion: versionModule.contractVersion(),
engine: '@soulcraftlabs/brainy',
compatibility: {
minor:
'additive — a new optional door, a new served operator, a new error class; every existing implementation still conforms',
major:
'breaking — a door removed, an answer narrowed, an ordering law changed, an optional door promoted to required, or an operator moved from served to refused'
},
doors: surfaceOf(Brainy),
errors,
operators: {
accepted,
servedOnIndexPath: servedOnIndex,
refusedByIndexPath: refusedByIndex,
combinators: ['allOf', 'anyOf', 'not']
},
fieldAddressing: {
systemKeyPrefix: 'system.',
systemEntityScalars: [...(fieldAddressing.SYSTEM_ENTITY_SCALARS ?? [])].sort(),
systemRelationScalars: [...(fieldAddressing.SYSTEM_RELATION_SCALARS ?? [])].sort(),
plumbingFields: [...(fieldAddressing.PLUMBING_FIELDS ?? [])].sort()
},
health: {
verdicts: ['pass', 'warn', 'fail'],
healKinds: ['none', 'repair', 'rebuild'],
servingWithholdingInvariants: [
'index-initialized',
'durable-state-present',
'manifest-residency',
'replay-clean',
'strand-latch'
]
}
}
const rendered = `${JSON.stringify(manifest, null, 2)}\n`
if (process.argv.includes('--check')) {
if (!existsSync(OUT)) {
console.error(`docs/api-contract.json is missing — run: node scripts/emit-contract-manifest.mjs`)
process.exit(1)
}
if (readFileSync(OUT, 'utf-8') !== rendered) {
console.error(
`docs/api-contract.json is STALE — the public surface changed. Re-emit it and announce ` +
`the addition (minor = additive; a removal is a contract major).`
)
process.exit(1)
}
console.log(`docs/api-contract.json is current (${manifest.doors.length} doors, contract ${manifest.contractVersion}).`)
process.exit(0)
}
writeFileSync(OUT, rendered)
console.log(
`Wrote docs/api-contract.json — contract ${manifest.contractVersion}, ` +
`${manifest.doors.length} doors, ${manifest.errors.length} error classes, ` +
`${manifest.operators.accepted.length} operators ` +
`(${manifest.operators.refusedByIndexPath.length} refused by the index path).`
)

View file

@ -1,118 +0,0 @@
/**
* Deterministic generation-stamp resolution for Brainy's build-time code
* generators.
*
* Two builds of the same source tree must produce byte-identical output.
* A wall-clock stamp (`new Date()`) breaks that guarantee, so every
* generator that writes a "Generated:" header or a `generatedAt` field
* into its output must resolve the stamp through this module instead.
*
* Resolution order:
* 1. The newest git commit timestamp among the generator's input files
* (the generator script itself always counts as an input).
* 2. If git metadata is unavailable (for example, building from a
* published npm tarball with no `.git` directory), the stamp already
* recorded in the previously generated output file.
* 3. If neither is available, the fixed epoch string
* `1970-01-01T00:00:00.000Z`.
*
* Every fallback logs a line to stderr deterministic degradation is
* loud, never a silent divergence.
*/
import { execFileSync } from 'child_process'
import * as fs from 'fs'
const EPOCH_STAMP = '1970-01-01T00:00:00.000Z'
const STAMP_PATTERN = /\*\s*Generated:\s*(\S+)/
/**
* Resolve the deterministic stamp for a generator run.
*
* @param inputPaths Absolute paths to every file whose content determines
* the generator's output, including the generator script itself.
* @param previousOutputPath Absolute path to the previously generated
* file, used for the existing-stamp fallback when git is unavailable.
* @returns An ISO-8601 timestamp string that is deterministic for a given
* source tree.
*/
export function resolveDeterministicStamp(
inputPaths: string[],
previousOutputPath: string
): string {
const gitStamp = newestGitCommitTimestamp(inputPaths)
if (gitStamp) {
return gitStamp
}
const existingStamp = readExistingStamp(previousOutputPath)
if (existingStamp) {
process.stderr.write(
`[deterministic-stamp] no git commit history found for generator inputs; ` +
`reusing existing stamp from ${previousOutputPath}: ${existingStamp}\n`
)
return existingStamp
}
process.stderr.write(
`[deterministic-stamp] no git commit history and no previous output at ` +
`${previousOutputPath}; falling back to fixed epoch stamp ${EPOCH_STAMP}\n`
)
return EPOCH_STAMP
}
/**
* Find the newest git commit timestamp among the given input paths.
* Returns null if git is unavailable, the tree is not a git repository,
* or none of the inputs have any commit history yet.
*/
function newestGitCommitTimestamp(inputPaths: string[]): string | null {
let newest: string | null = null
for (const inputPath of inputPaths) {
if (!fs.existsSync(inputPath)) {
continue
}
let out: string
try {
out = execFileSync(
'git',
['log', '-1', '--format=%cI', '--', inputPath],
{ stdio: ['ignore', 'pipe', 'ignore'] }
)
.toString()
.trim()
} catch {
// git missing, not a repository, or no permissions — handled by the
// caller's fallback chain.
continue
}
if (!out) {
// Path exists but has no commit history yet (e.g. newly created,
// uncommitted file).
continue
}
if (!newest || new Date(out).getTime() > new Date(newest).getTime()) {
newest = out
}
}
return newest
}
/**
* Parse the `* Generated: <ISO timestamp>` header out of a previously
* generated file, if one exists.
*/
function readExistingStamp(outputPath: string): string | null {
if (!fs.existsSync(outputPath)) {
return null
}
const content = fs.readFileSync(outputPath, 'utf-8')
const match = content.match(STAMP_PATTERN)
return match ? match[1] : null
}

View file

@ -15,11 +15,11 @@ NC='\033[0m' # No Color
RELEASE_TYPE="${1:-patch}" # patch, minor, or major RELEASE_TYPE="${1:-patch}" # patch, minor, or major
SKIP_TESTS=false SKIP_TESTS=false
DRY_RUN=false DRY_RUN=false
# --source-only is now a no-op: The Source is the one registry, so every # --source-only: the HOME leg only — tag, CI's publish to The Source, and the
# release already ships Source-only — tag, CI's publish to The Source, the # release page; NO storefront (npmjs) publish, NO pair verification, NO docs
# release page, and the docs push, with no separate storefront leg to skip. # push. The pair-gate shape: a prerelease the fleet's other engine devDeps
# The flag is still accepted (for backward-compatible invocations) and just # from our own registry while the pair is proven, never a public artifact.
# prints a notice; it no longer changes behavior. # Refused for a non-prerelease version — a public floor is always a pair.
SOURCE_ONLY=false SOURCE_ONLY=false
for arg in "$@"; do for arg in "$@"; do
@ -109,7 +109,7 @@ else
;; ;;
*) *)
echo -e "${RED}❌ Invalid release type: ${RELEASE_TYPE}${NC}" echo -e "${RED}❌ Invalid release type: ${RELEASE_TYPE}${NC}"
echo "Usage: ./scripts/release.sh [patch|minor|major|<explicit-version>] [--dry-run] [--source-only (no-op; The Source is the one registry)]" echo "Usage: ./scripts/release.sh [patch|minor|major|<explicit-version>] [--dry-run] [--source-only (prereleases only)]"
exit 1 exit 1
;; ;;
esac esac
@ -129,7 +129,11 @@ if [ "$PRERELEASE" = true ]; then
echo -e "${YELLOW}⚠️ Prerelease → npm dist-tag '${NPM_TAG}', GitHub prerelease${NC}" echo -e "${YELLOW}⚠️ Prerelease → npm dist-tag '${NPM_TAG}', GitHub prerelease${NC}"
fi fi
if [ "$SOURCE_ONLY" = true ]; then if [ "$SOURCE_ONLY" = true ]; then
echo -e "${YELLOW}⚠️ The Source is the one registry; --source-only is implied${NC}" if [ "$PRERELEASE" != true ]; then
echo -e "${RED}❌ --source-only is for prereleases only: a non-prerelease version is a public floor and always ships as the byte-identical pair.${NC}"
exit 1
fi
echo -e "${YELLOW}⚠️ --source-only → The Source (home) ONLY: no npmjs publish, no pair verification, no docs push${NC}"
fi fi
echo "" echo ""
@ -154,8 +158,7 @@ else
fi fi
# Create new changelog entry # Create new changelog entry
RELEASE_DATE=$(date +%Y-%m-%d) CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraft/brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) ($(date +%Y-%m-%d))
CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) (${RELEASE_DATE})
${COMMITS} ${COMMITS}
" "
@ -175,19 +178,6 @@ if [ -f "CHANGELOG.md" ]; then
fi fi
echo -e "${GREEN}✅ CHANGELOG updated${NC}\n" echo -e "${GREEN}✅ CHANGELOG updated${NC}\n"
# Step 6b: Update the releases wall entry — mechanical, derived from the
# CHANGELOG entry just composed. The fleet's HQ page reads open-brainy.json
# from the one shared releases repo, soulcraftlabs/releases on The Source —
# this used to be hand-written after every release (David: never again —
# make it a step of the rail, landed in the one shared home; this repo no
# longer hosts its own copy). This step clones/fetches that repo into a
# local cache, prepends the entry, and pushes it directly — a real
# cross-repo push, refusing loudly (never skipping) on any
# clone/validation/commit/push failure.
echo -e "${BLUE}5⃣▸ Updating the releases wall...${NC}"
node scripts/wall-entry.mjs --product open-brainy --version "${NEW_VERSION}" --date "${RELEASE_DATE}" --from-changelog CHANGELOG.md
echo -e "${GREEN}✅ Releases wall updated${NC}\n"
# Step 7: Create release commit # Step 7: Create release commit
echo -e "${BLUE}6⃣ Creating release commit...${NC}" echo -e "${BLUE}6⃣ Creating release commit...${NC}"
git add package.json package-lock.json CHANGELOG.md git add package.json package-lock.json CHANGELOG.md
@ -219,9 +209,9 @@ echo -e "${GREEN}✅ Pushed to origin${NC}\n"
# .forgejo/workflows/publish-source.yml, which builds and publishes on The # .forgejo/workflows/publish-source.yml, which builds and publishes on The
# Source's own runner (datacenter-side: seconds, not the laptop's WAN timing # Source's own runner (datacenter-side: seconds, not the laptop's WAN timing
# out on an 87MB tarball PUT). The laptop holds no home-registry publish # out on an 87MB tarball PUT). The laptop holds no home-registry publish
# credential anymore; it only waits for CI's result before continuing on to # credential anymore; it only waits for CI's result before trusting the
# the release page and the docs push. # home/npmjs pair enough to publish the storefront leg.
SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraftlabs/npm/" SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/"
SOURCE_POLL_INTERVAL_S=15 SOURCE_POLL_INTERVAL_S=15
SOURCE_POLL_MAX_ATTEMPTS=200 # 200 × 15s = 50 minutes — the runner is sequential and a busy day's ci.yml SOURCE_POLL_MAX_ATTEMPTS=200 # 200 × 15s = 50 minutes — the runner is sequential and a busy day's ci.yml
# backlog has twice exceeded the old 20-minute window (8.10.3, 9.0.0); # backlog has twice exceeded the old 20-minute window (8.10.3, 9.0.0);
@ -229,7 +219,7 @@ SOURCE_POLL_MAX_ATTEMPTS=200 # 200 × 15s = 50 minutes — the runner is sequen
echo -e "${BLUE}9⃣ Waiting for CI to publish v${NEW_VERSION} to The Source registry (home)...${NC}" echo -e "${BLUE}9⃣ Waiting for CI to publish v${NEW_VERSION} to The Source registry (home)...${NC}"
SOURCE_LANDED=false SOURCE_LANDED=false
for ((attempt = 1; attempt <= SOURCE_POLL_MAX_ATTEMPTS; attempt++)); do for ((attempt = 1; attempt <= SOURCE_POLL_MAX_ATTEMPTS; attempt++)); do
LANDED_VERSION=$(npm view "@soulcraftlabs/brainy@${NEW_VERSION}" version "--@soulcraftlabs:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "") LANDED_VERSION=$(npm view "@soulcraft/brainy@${NEW_VERSION}" version "--@soulcraft:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "")
if [ "$LANDED_VERSION" = "$NEW_VERSION" ]; then if [ "$LANDED_VERSION" = "$NEW_VERSION" ]; then
SOURCE_LANDED=true SOURCE_LANDED=true
break break
@ -242,16 +232,62 @@ if [ "$SOURCE_LANDED" = true ]; then
echo -e "${GREEN}✅ CI published v${NEW_VERSION} to The Source${NC}\n" echo -e "${GREEN}✅ CI published v${NEW_VERSION} to The Source${NC}\n"
else else
echo -e "${RED}❌ CI's home publish did not land — check the workflow run on The Source; the pair must not diverge.${NC}" echo -e "${RED}❌ CI's home publish did not land — check the workflow run on The Source; the pair must not diverge.${NC}"
echo -e "${RED} v${NEW_VERSION} was tagged and pushed, but @soulcraftlabs/brainy@${NEW_VERSION} never became visible on the${NC}" echo -e "${RED} v${NEW_VERSION} was tagged and pushed, but @soulcraft/brainy@${NEW_VERSION} never became visible on the${NC}"
echo -e "${RED} Source registry after ${SOURCE_POLL_MAX_ATTEMPTS} attempts, ${SOURCE_POLL_INTERVAL_S}s apart. Aborting.${NC}" echo -e "${RED} Source registry after ${SOURCE_POLL_MAX_ATTEMPTS} attempts, ${SOURCE_POLL_INTERVAL_S}s apart. Aborting before npmjs.${NC}"
exit 1 exit 1
fi fi
if [ "$SOURCE_ONLY" = true ]; then
echo -e "${YELLOW}9⃣½ Storefront (npmjs) leg SKIPPED — --source-only: v${NEW_VERSION} lives on The Source under dist-tag '${NPM_TAG}' only${NC}\n"
else
echo -e "${BLUE}9⃣½ Publishing to npmjs (storefront, dist-tag: ${NPM_TAG})...${NC}"
# BYTE-IDENTITY LAW: the storefront republishes CI's EXACT artifact — download
# the tarball The Source serves and publish that file, never a fresh local pack
# (a local rebuild can differ byte-wise, and the fleet verifies the pair by
# shasum across registries).
STOREFRONT_TMP="$(mktemp -d)"
(cd "$STOREFRONT_TMP" && npm pack "@soulcraft/brainy@${NEW_VERSION}" "--@soulcraft:registry=${SOURCE_NPM_REG}" >/dev/null)
SOURCE_TARBALL="$(ls "$STOREFRONT_TMP"/soulcraft-brainy-*.tgz)"
echo -e "${BLUE} home artifact: $(sha256sum "$SOURCE_TARBALL" | cut -d' ' -f1)${NC}"
npm publish "$SOURCE_TARBALL" --tag "$NPM_TAG" "--@soulcraft:registry=https://registry.npmjs.org/"
rm -rf "$STOREFRONT_TMP"
# Brainy is the only PUBLIC @soulcraft package — verify visibility after every publish.
npm access get status @soulcraft/brainy "--@soulcraft:registry=https://registry.npmjs.org/" || true
# Verify the pair is byte-identical by registry-reported shasum — divergence
# here means the storefront leg must be treated as failed, loudly. RETRIED
# with raw curl: npmjs metadata propagates with a lag measured in minutes,
# and a one-shot npm-view probe fired a false DIVERGENCE on 10.0.0 while a
# raw curl of the registry document already confirmed byte-identity. The
# probe now reads the registry JSON directly (no npm cache in the path) and
# gives propagation up to 5 minutes before calling the pair divergent.
NPMJS_VERIFY_ATTEMPTS=20
NPMJS_VERIFY_INTERVAL_S=15 # 20 × 15s = 5 minutes of propagation grace
SOURCE_SHA=$(npm view "@soulcraft/brainy@${NEW_VERSION}" dist.shasum "--@soulcraft:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "source-unavailable")
PAIR_IDENTICAL=false
for ((attempt = 1; attempt <= NPMJS_VERIFY_ATTEMPTS; attempt++)); do
NPMJS_SHA=$(curl -fsSL "https://registry.npmjs.org/@soulcraft%2Fbrainy" 2>/dev/null \
| node -e "let d='';process.stdin.on('data',c=>d+=c).on('end',()=>{try{const v=JSON.parse(d).versions[process.argv[1]];console.log(v?v.dist.shasum:'')}catch{console.log('')}})" "${NEW_VERSION}" \
|| echo "")
if [ -n "$NPMJS_SHA" ] && [ "$SOURCE_SHA" = "$NPMJS_SHA" ]; then
PAIR_IDENTICAL=true
break
fi
echo -e "${YELLOW} … npmjs metadata not settled (attempt ${attempt}/${NPMJS_VERIFY_ATTEMPTS}: '${NPMJS_SHA:-absent}' vs '${SOURCE_SHA}'); retrying in ${NPMJS_VERIFY_INTERVAL_S}s${NC}"
sleep "$NPMJS_VERIFY_INTERVAL_S"
done
if [ "$PAIR_IDENTICAL" = true ]; then
echo -e "${GREEN}✅ Published to npmjs — byte-identical pair (shasum ${NPMJS_SHA})${NC}\n"
else
echo -e "${RED}❌ REGISTRY DIVERGENCE: The Source shasum ${SOURCE_SHA} != npmjs shasum ${NPMJS_SHA} after ${NPMJS_VERIFY_ATTEMPTS} attempts — investigate before announcing${NC}\n"
exit 1
fi
fi
# Step 11: Release object on The Source (presentational — the tag, CHANGELOG, # Step 11: Release object on The Source (presentational — the tag, CHANGELOG,
# and RELEASES.md are the record; this just gives The Source's UI a release page). # and RELEASES.md are the record; this just gives The Source's UI a release page).
echo -e "${BLUE}🔟 Creating release page on The Source...${NC}" echo -e "${BLUE}🔟 Creating release page on The Source...${NC}"
if [ -n "${FORGEJO_RELEASE_TOKEN:-}" ]; then if [ -n "${FORGEJO_RELEASE_TOKEN:-}" ]; then
if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraftlabs/open-brainy/releases" \ if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraft/brainy/releases" \
-H "Authorization: token ${FORGEJO_RELEASE_TOKEN}" -H "Content-Type: application/json" \ -H "Authorization: token ${FORGEJO_RELEASE_TOKEN}" -H "Content-Type: application/json" \
-d "{\"tag_name\":\"v${NEW_VERSION}\",\"name\":\"v${NEW_VERSION}\",\"prerelease\":${PRERELEASE}}" >/dev/null; then -d "{\"tag_name\":\"v${NEW_VERSION}\",\"name\":\"v${NEW_VERSION}\",\"prerelease\":${PRERELEASE}}" >/dev/null; then
echo -e "${GREEN}✅ Release page created on The Source${NC}\n" echo -e "${GREEN}✅ Release page created on The Source${NC}\n"
@ -262,15 +298,29 @@ else
echo -e "${RED}⚠️ FORGEJO_RELEASE_TOKEN unset — no release page created; tag + CHANGELOG remain the record${NC}\n" echo -e "${RED}⚠️ FORGEJO_RELEASE_TOKEN unset — no release page created; tag + CHANGELOG remain the record${NC}\n"
fi fi
# Step 12 RETIRED (2026-08-31, CORTEX-SITE-BRAINY-RENAME round 12, David-ruled): # Step 12: Push public docs to the soulcraft.com docs ingest door
# soulcraft.com/docs carries the paid product's documentation only. This # (VENUE-DOCS-RELEASE-PUSH). Skips with a loud warning when
# engine's documentation home is THIS repository — README and docs/ — and the # DOCS_INGEST_SECRET is unset; fails loudly (without undoing the publish —
# site serves 301s for the slugs this rail used to push. The push script stays # that already happened) when a push errors, so the docs site never
# in the tree for history; the rail no longer calls it. # silently trails npm.
echo -e "${BLUE}Docs step: this engine documents itself in its own repo (site push retired 2026-08-31)${NC}" if [ "$SOURCE_ONLY" = true ]; then
echo -e "${YELLOW}1⃣2⃣ Docs push SKIPPED — --source-only (a home-only prerelease publishes no public docs)${NC}\n"
else
echo -e "${BLUE}1⃣2⃣ Pushing public docs to soulcraft.com/docs...${NC}"
if node scripts/push-docs.js; then
echo -e "${GREEN}✅ Docs push step done${NC}\n"
else
echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n"
fi
fi
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}" echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}"
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo "" echo ""
echo -e "🏠 The Source: ${BLUE}https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v${NEW_VERSION}${NC}" if [ "$SOURCE_ONLY" = true ]; then
echo -e "📦 npmjs: ${YELLOW}not published (--source-only)${NC}"
else
echo -e "📦 npm: ${BLUE}https://www.npmjs.com/package/@soulcraft/brainy/v/${NEW_VERSION}${NC}"
fi
echo -e "🏠 The Source: ${BLUE}https://source.soulcraft.com/soulcraft/brainy/releases/tag/v${NEW_VERSION}${NC}"

View file

@ -1,504 +0,0 @@
#!/usr/bin/env node
/**
* @module scripts/wall-entry
* @description The releases-wall entry, made mechanical. The fleet's HQ page
* reads one public JSON per product from the ONE releases repo on The Source
* (soulcraftlabs/releases, files <product>.json at its root shape
* {product, entries:[{version, date, headline, items, url, thumb?}]}), at
* https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/<product>.json.
* Those entries were hand-written after every release, then briefly written
* into this repo's own releases/<product>.json; this script is the one door
* that composes an entry and lands it in the shared repo, so it is never
* hand-written and never forked across repos again.
*
* Two modes:
*
* 1. Generate + publish (default):
* node wall-entry.mjs --product <p> --version <v> --date <YYYY-MM-DD> \
* --from-changelog <CHANGELOG.md>
* Derives an entry from the CHANGELOG.md entry for <v> (headline = the
* entry's first bullet, items = every bullet, trimmed of its trailing
* commit hash), then:
* - clones (or, if a cached clone already exists, fetches and resets)
* the releases repo into a local cache directory,
* - prepends the entry to <cache>/<p>.json, newest first replacing
* any existing entry for the same version so a re-run is idempotent,
* - validates the file's shape before and after,
* - commits the change as "chore(wall): <p> <v>" and pushes main.
* A failure at any step (clone, validation, commit, push, a
* non-fast-forward remote) exits non-zero naming the cure. Nothing is
* ever skipped the wall either lands correctly or the release fails.
*
* 2. Dry run:
* node wall-entry.mjs --dry-run --product <p> --version <v> \
* --date <YYYY-MM-DD> --from-changelog <CHANGELOG.md>
* Derives the entry exactly as above and prints it, along with the file
* it would be written to, but touches no clone and no remote usable
* from a fresh checkout with no cache and no network.
*
* 3. Validate only (--check):
* node wall-entry.mjs --check --file <path/to/product.json>
* Validates an arbitrary wall file's exact key set (top-level and
* per-entry), field types, and strict-descending semver ordering with
* no duplicates. Read-only; never writes. Exit 0 = clean, exit 1 =
* named violations printed to stderr.
*
* The remote and the local cache directory are each overridable
* (--remote / --cache-dir, or WALL_ENTRY_RELEASES_REMOTE /
* WALL_ENTRY_RELEASES_CACHE_DIR) so tests can point at a throwaway local
* bare repo and a throwaway cache directory never the real remote or the
* real developer cache.
*
* No dependencies beyond the system `git` binary CHANGELOG parsing,
* semver comparison, and JSON shape checking are all hand-rolled below.
*/
import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs'
import { execFileSync } from 'node:child_process'
import { homedir } from 'node:os'
import { dirname, join } from 'node:path'
const DEFAULT_REMOTE = 'git@source.soulcraft.com:soulcraftlabs/releases.git'
/** @returns {string} */
function defaultCacheDir() {
const base = process.env.XDG_CACHE_HOME || join(homedir(), '.cache')
return join(base, 'soulcraft-releases')
}
// Required on every entry; "thumb" is optional (may be absent, or present as
// string | null) — matching the HQ contract's {..., thumb?}.
const ENTRY_REQUIRED_KEYS = ['version', 'date', 'headline', 'items', 'url']
const ENTRY_OPTIONAL_KEYS = ['thumb']
const ENTRY_ALLOWED_KEYS = [...ENTRY_REQUIRED_KEYS, ...ENTRY_OPTIONAL_KEYS]
const FILE_KEYS = ['product', 'entries']
// The public permalink pattern, by product. Every entry MUST carry an https
// permalink: HQ's parser rejects a wall whose entries carry url: null (the
// whole feed became unreadable on 2026-09-02). A product whose forge repo is
// private links its PUBLIC package page on The Source instead of a release
// page that would 404 for HQ's readers.
const RELEASE_URL_PATTERNS = {
'open-brainy': (version) => `https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v${version}`,
'brainy': (version) => `https://source.soulcraft.com/soulcraft/-/packages/npm/@soulcraft%2Fbrainy/${version}`,
}
/**
* Parse argv into a flag map. `--flag value` sets a string; `--flag` alone
* (end of argv, or followed by another `--flag`) sets boolean true.
* @param {string[]} argv
* @returns {Record<string, string | true>}
*/
function parseArgs(argv) {
/** @type {Record<string, string | true>} */
const args = {}
for (let i = 0; i < argv.length; i++) {
const a = argv[i]
if (!a.startsWith('--')) continue
const key = a.slice(2)
const next = argv[i + 1]
if (next === undefined || next.startsWith('--')) {
args[key] = true
} else {
args[key] = next
i++
}
}
return args
}
/**
* Print a loud, named error and exit 1. Every refusal in this script goes
* through here so the failure mode is always the same shape: "wall-entry: <what>".
* @param {string} message
* @returns {never}
*/
function fail(message) {
console.error(`wall-entry: ${message}`)
process.exit(1)
}
/**
* @param {string} version
* @returns {{major: number, minor: number, patch: number, pre: string | null} | null}
*/
function parseSemver(version) {
const m = /^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$/.exec(version)
if (!m) return null
return { major: Number(m[1]), minor: Number(m[2]), patch: Number(m[3]), pre: m[4] ?? null }
}
/**
* @param {string} a
* @param {string} b
* @returns {number} positive if a > b, negative if a < b, 0 if equal.
*/
function compareSemver(a, b) {
const pa = parseSemver(a)
const pb = parseSemver(b)
if (!pa || !pb) throw new Error(`cannot compare non-semver versions "${a}" vs "${b}"`)
if (pa.major !== pb.major) return pa.major - pb.major
if (pa.minor !== pb.minor) return pa.minor - pb.minor
if (pa.patch !== pb.patch) return pa.patch - pb.patch
if (pa.pre === pb.pre) return 0
if (pa.pre === null) return 1 // a release outranks any prerelease of the same core version
if (pb.pre === null) return -1
return pa.pre < pb.pre ? -1 : pa.pre > pb.pre ? 1 : 0
}
/**
* Validate a wall file's full shape: top-level keys ("product", "entries"
* no more, no less), per-entry keys and field types ("thumb" optional), and
* strict-descending semver ordering with no duplicates. Collects every
* violation instead of failing on the first, so a caller reports the whole
* picture in one pass.
* @param {unknown} data
* @returns {string[]} Violation messages; empty means the file is clean.
*/
function validateShape(data) {
/** @type {string[]} */
const errors = []
if (typeof data !== 'object' || data === null || Array.isArray(data)) {
return ['top level: expected a JSON object']
}
const obj = /** @type {Record<string, unknown>} */ (data)
const topKeys = Object.keys(obj)
const missingTop = FILE_KEYS.filter((k) => !(k in obj))
const extraTop = topKeys.filter((k) => !FILE_KEYS.includes(k))
if (missingTop.length) errors.push(`top level: missing key(s) ${missingTop.join(', ')}`)
if (extraTop.length) errors.push(`top level: unexpected key(s) ${extraTop.join(', ')}`)
if (typeof obj.product !== 'string' || obj.product.trim() === '') {
errors.push('top level: "product" must be a non-empty string')
}
if (!Array.isArray(obj.entries)) {
errors.push('top level: "entries" must be an array')
return errors // nothing further to check without an array
}
const entries = /** @type {unknown[]} */ (obj.entries)
entries.forEach((rawEntry, i) => {
const label = `entries[${i}]`
if (typeof rawEntry !== 'object' || rawEntry === null || Array.isArray(rawEntry)) {
errors.push(`${label}: expected an object`)
return
}
const entry = /** @type {Record<string, unknown>} */ (rawEntry)
const keys = Object.keys(entry)
const missing = ENTRY_REQUIRED_KEYS.filter((k) => !(k in entry))
const extra = keys.filter((k) => !ENTRY_ALLOWED_KEYS.includes(k))
if (missing.length) errors.push(`${label}: missing key(s) ${missing.join(', ')}`)
if (extra.length) errors.push(`${label}: unexpected key(s) ${extra.join(', ')}`)
if (typeof entry.version !== 'string' || !parseSemver(entry.version)) {
errors.push(`${label}: "version" must be a semver string (got ${JSON.stringify(entry.version)})`)
}
if (typeof entry.date !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(entry.date) || Number.isNaN(Date.parse(entry.date))) {
errors.push(`${label}: "date" must be a YYYY-MM-DD string (got ${JSON.stringify(entry.date)})`)
}
if (typeof entry.headline !== 'string' || entry.headline.trim() === '') {
errors.push(`${label}: "headline" must be a non-empty string`)
}
if (!Array.isArray(entry.items) || entry.items.length === 0 || entry.items.some((it) => typeof it !== 'string' || it.trim() === '')) {
errors.push(`${label}: "items" must be a non-empty array of non-empty strings`)
}
if (typeof entry.url !== 'string' || !/^https:\/\/\S+$/.test(entry.url)) {
errors.push(`${label}: "url" must be an https permalink — never null; HQ's parser rejects the whole feed`)
}
if ('thumb' in entry && !(entry.thumb === null || typeof entry.thumb === 'string')) {
errors.push(`${label}: "thumb" must be a string or null when present`)
}
})
// Ordering: newest first, strictly descending, no duplicate versions —
// checked only over entries whose version parsed (a bad version is
// already reported above; comparing it too would just be noise).
const versioned = entries
.map((e, i) => ({ i, version: /** @type {any} */ (e)?.version }))
.filter((e) => typeof e.version === 'string' && parseSemver(e.version))
for (let i = 0; i < versioned.length - 1; i++) {
const a = versioned[i]
const b = versioned[i + 1]
const cmp = compareSemver(a.version, b.version)
if (cmp === 0) {
errors.push(`entries[${a.i}] and entries[${b.i}]: duplicate version ${a.version}`)
} else if (cmp < 0) {
errors.push(`entries[${a.i}] (${a.version}) sits above entries[${b.i}] (${b.version}) — not newest-first`)
}
}
return errors
}
/**
* Extract one version's entry body from a standard-version-style CHANGELOG.md
* (headings `### [version](url) (date)`, followed by `- bullet (hash)` lines
* until the next heading or EOF).
* @param {string} changelog
* @param {string} version
* @returns {string[]} Bullet lines, trimmed of their leading "- " and
* trailing " (hash)".
*/
function extractChangelogBullets(changelog, version) {
const lines = changelog.split('\n')
const headingRe = /^### \[([^\]]+)\]\(.*\)\s*\(\d{4}-\d{2}-\d{2}\)\s*$/
let start = -1
for (let i = 0; i < lines.length; i++) {
const m = headingRe.exec(lines[i])
if (m && m[1] === version) {
start = i + 1
break
}
}
if (start === -1) {
fail(
`version ${version} has no CHANGELOG entry yet — run this after the CHANGELOG step composes "### [${version}]", not before`,
)
}
/** @type {string[]} */
const bullets = []
for (let i = start; i < lines.length; i++) {
if (headingRe.test(lines[i])) break // next entry starts
const bulletMatch = /^- (.+?)(?:\s\(([0-9a-f]{6,40})\))?$/.exec(lines[i].trim())
if (lines[i].trim().startsWith('- ') && bulletMatch) {
const text = bulletMatch[1].trim()
if (text) bullets.push(text)
}
}
if (bullets.length === 0) {
fail(`version ${version}'s CHANGELOG entry has no bullets to derive a headline/items from`)
}
return bullets
}
/**
* Derive a wall entry from a CHANGELOG.md.
* @param {{product: string, version: string, date: string, changelogPath: string, url?: string, thumb?: string | null}} opts
* @returns {{version: string, date: string, headline: string, items: string[], url: string, thumb: string | null}}
*/
function deriveEntry({ product, version, date, changelogPath, url, thumb }) {
if (!parseSemver(version)) fail(`--version "${version}" is not a semver string`)
if (!/^\d{4}-\d{2}-\d{2}$/.test(date) || Number.isNaN(Date.parse(date))) {
fail(`--date "${date}" is not a YYYY-MM-DD date`)
}
if (!existsSync(changelogPath)) fail(`--from-changelog "${changelogPath}" does not exist`)
const changelog = readFileSync(changelogPath, 'utf8')
const items = extractChangelogBullets(changelog, version)
const headline = items[0]
const pattern = RELEASE_URL_PATTERNS[product]
if (url === undefined && pattern === undefined) {
throw new Error(`wall-entry: no permalink pattern for product "${product}" — add one to RELEASE_URL_PATTERNS or pass --url; entries never carry url: null`)
}
const resolvedUrl = url !== undefined ? url : pattern(version)
const resolvedThumb = thumb !== undefined ? thumb : null
return { version, date, headline, items, url: resolvedUrl, thumb: resolvedThumb }
}
/**
* Load and shape-validate a wall file.
* @param {string} filePath
* @returns {Record<string, any>}
*/
function loadWallFile(filePath) {
if (!existsSync(filePath)) fail(`"${filePath}" does not exist`)
/** @type {unknown} */
let data
try {
data = JSON.parse(readFileSync(filePath, 'utf8'))
} catch (err) {
fail(`"${filePath}" is not valid JSON: ${/** @type {Error} */ (err).message}`)
}
const errors = validateShape(data)
if (errors.length) {
fail(`"${filePath}" fails shape validation —\n ${errors.join('\n ')}`)
}
return /** @type {Record<string, any>} */ (data)
}
/**
* Run a git command, throwing an Error whose message is git's own stderr
* (trimmed) on failure every caller wraps this to name the cure.
* @param {string[]} args
* @param {string} cwd
* @returns {string} stdout, trimmed.
*/
function git(args, cwd) {
try {
return execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim()
} catch (err) {
const stderr = /** @type {any} */ (err).stderr
const message = (typeof stderr === 'string' && stderr.trim()) || /** @type {Error} */ (err).message
throw new Error(message)
}
}
/**
* Ensure a clean, up-to-date local clone of the releases repo at
* `cacheDir`, checked out on `main` cloning fresh if `cacheDir` has no
* `.git`, otherwise fetching and hard-resetting onto `origin/main` (so a
* stray local commit or edit left by a previous failed run can never leak
* into the next one).
* @param {string} remote
* @param {string} cacheDir
*/
function ensureReleasesClone(remote, cacheDir) {
if (existsSync(join(cacheDir, '.git'))) {
try {
git(['remote', 'set-url', 'origin', remote], cacheDir)
git(['fetch', '--prune', 'origin'], cacheDir)
git(['checkout', 'main'], cacheDir)
git(['reset', '--hard', 'origin/main'], cacheDir)
git(['clean', '-fd'], cacheDir)
} catch (err) {
fail(
`cannot refresh the cached releases checkout at "${cacheDir}" from "${remote}" — ${/** @type {Error} */ (err).message}\n` +
` cure: delete "${cacheDir}" and re-run so it re-clones from scratch, or confirm SSH access with "ssh -T git@source.soulcraft.com"`,
)
}
return
}
mkdirSync(dirname(cacheDir), { recursive: true })
try {
git(['clone', remote, cacheDir], dirname(cacheDir))
} catch (err) {
fail(
`cannot clone "${remote}" — ${/** @type {Error} */ (err).message}\n` +
` cure: confirm SSH access with "ssh -T git@source.soulcraft.com" and that the soulcraftlabs/releases repo exists yet`,
)
}
try {
git(['checkout', 'main'], cacheDir)
} catch (err) {
fail(
`cloned "${remote}" into "${cacheDir}" but could not check out "main" — ${/** @type {Error} */ (err).message}\n` +
` cure: confirm the releases repo's default branch is named "main"`,
)
}
}
/**
* Prepend `entry` to the wall at `<cacheDir>/<product>.json`, replacing any
* existing entry for the same version (idempotent re-runs), validating
* before and after, committing, and pushing or refusing loudly, naming
* the cure, at whichever step fails.
* @param {{version: string, date: string, headline: string, items: string[], url: string, thumb: string | null}} entry
* @param {string} product
* @param {string} remote
* @param {string} cacheDir
*/
function publishEntry(entry, product, remote, cacheDir) {
ensureReleasesClone(remote, cacheDir)
const filePath = join(cacheDir, `${product}.json`)
if (!existsSync(filePath)) {
fail(
`"${filePath}" does not exist in the releases repo — cure: seed "${product}.json" at the repo root first (it must exist before any release rail can prepend to it)`,
)
}
const wall = loadWallFile(filePath)
if (wall.product !== product) {
fail(`"${filePath}" has product "${wall.product}", but --product "${product}" was given — refusing a cross-product write`)
}
const replacing = wall.entries.some((e) => e.version === entry.version)
wall.entries = [entry, ...wall.entries.filter((e) => e.version !== entry.version)]
const postErrors = validateShape(wall)
if (postErrors.length) {
fail(`the entry for ${entry.version} would leave "${filePath}" invalid —\n ${postErrors.join('\n ')}`)
}
writeFileSync(filePath, JSON.stringify(wall, null, 2) + '\n', 'utf8')
const status = git(['status', '--porcelain', '--', `${product}.json`], cacheDir)
if (status === '') {
console.log(`wall-entry: "${product}.json" already carries an identical entry for ${entry.version} — nothing to commit or push`)
return
}
try {
git(['add', `${product}.json`], cacheDir)
git(['commit', '-m', `chore(wall): ${product} ${entry.version}`], cacheDir)
} catch (err) {
fail(`cannot commit the wall entry in "${cacheDir}" — ${/** @type {Error} */ (err).message}\n cure: inspect "${cacheDir}" by hand and re-run once its git state is clean`)
}
try {
git(['push', 'origin', 'main'], cacheDir)
} catch (err) {
fail(
`push to "${remote}" failed (likely a non-fast-forward — another release landed on main first) — ${/** @type {Error} */ (err).message}\n` +
` cure: re-run this release step; it re-fetches and resets onto the latest origin/main before retrying`,
)
}
const sha = git(['rev-parse', 'HEAD'], cacheDir)
console.log(
`wall-entry: ${replacing ? 'replaced' : 'wrote'} v${entry.version} in "${product}.json" (${wall.entries.length} entries, newest first) — pushed ${sha} to ${remote} main`,
)
}
function main() {
const args = parseArgs(process.argv.slice(2))
if (args.check) {
const filePath = /** @type {string | undefined} */ (args.file)
if (!filePath) fail('--check needs --file <path>')
const wall = loadWallFile(/** @type {string} */ (filePath))
console.log(`wall-entry --check: "${filePath}" OK — product "${wall.product}", ${wall.entries.length} entries, newest-first, no duplicates`)
process.exit(0)
}
// Generate mode (default, also covers --dry-run): --product, --version,
// --date, --from-changelog required.
const product = /** @type {string | undefined} */ (args.product)
const version = /** @type {string | undefined} */ (args.version)
const date = /** @type {string | undefined} */ (args.date)
const fromChangelog = /** @type {string | undefined} */ (args['from-changelog'])
const missing = []
if (!product) missing.push('--product')
if (!version) missing.push('--version')
if (!date) missing.push('--date')
if (!fromChangelog) missing.push('--from-changelog')
if (missing.length) {
fail(
`missing required flag(s): ${missing.join(', ')}\n` +
'Usage:\n' +
' wall-entry.mjs --product <p> --version <v> --date <YYYY-MM-DD> --from-changelog <CHANGELOG.md> [--dry-run]\n' +
' wall-entry.mjs --check --file <path/to/product.json>',
)
}
const urlArg = args.url === true ? undefined : /** @type {string | undefined} */ (args.url)
const thumbArg = args.thumb === true ? undefined : /** @type {string | undefined} */ (args.thumb)
const entry = deriveEntry({
product: /** @type {string} */ (product),
version: /** @type {string} */ (version),
date: /** @type {string} */ (date),
changelogPath: /** @type {string} */ (fromChangelog),
url: urlArg,
thumb: thumbArg,
})
const remote = /** @type {string} */ (args.remote ?? process.env.WALL_ENTRY_RELEASES_REMOTE ?? DEFAULT_REMOTE)
const cacheDir = /** @type {string} */ (args['cache-dir'] ?? process.env.WALL_ENTRY_RELEASES_CACHE_DIR ?? defaultCacheDir())
if (args['dry-run']) {
console.log(`wall-entry --dry-run: would write to "${join(cacheDir, `${product}.json`)}" in ${remote} (main), pushed as "chore(wall): ${product} ${version}"`)
console.log(JSON.stringify(entry, null, 2))
process.exit(0)
}
publishEntry(entry, /** @type {string} */ (product), remote, cacheDir)
}
main()

File diff suppressed because it is too large Load diff

View file

@ -872,27 +872,6 @@ export interface StorageAdapter {
*/ */
noteVectorLanded?(id: string): Promise<void> noteVectorLanded?(id: string): Promise<void>
/**
* OPTIONAL narrow ledger hook, the mirror of {@link noteVectorLanded}:
* record that a canonical noun's vector was just REMOVED rewritten from
* a real (non-empty) vector to the "unvectored" empty-array shape. Exists
* for the ONE sanctioned reverse migration this engine supports: the VFS
* root's zero-norm fix (see `VirtualFileSystem.doInitializeRoot()` and
* `Brainy.unvectorNounForRootMigration()`), which rewrites a pre-fix
* store's all-zero placeholder root vector to `[]` and must decrement
* `vectors.all` through this hook so the coverage ledger never drifts.
* NOT a general-purpose "I removed a vector" callback ordinary
* application data has no sanctioned path from vectored back to
* unvectored (`update()` refuses an empty vector as a dimension
* mismatch by design). Callers MUST call this only when the noun held a
* REAL vector immediately before this write (the caller already holds
* that fact for free, from its own pre-write read never an added read).
* A backend without vectored-noun tracking is a no-op via this method's
* absence (feature-detected).
* @param id - The noun whose vector was just removed.
*/
noteVectorUnlanded?(id: string): Promise<void>
/** /**
* Get noun with metadata combined * Get noun with metadata combined
* @returns Combined HNSWNounWithMetadata or null * @returns Combined HNSWNounWithMetadata or null

View file

@ -28,7 +28,7 @@
* speculative `with()` overlay; the canonical storage walk only ever answers * speculative `with()` overlay; the canonical storage walk only ever answers
* "what is live right now." * "what is live right now."
* *
* All are exported from the package root (`@soulcraftlabs/brainy`). * All are exported from the package root (`@soulcraft/brainy`).
*/ */
/** /**

View file

@ -12,11 +12,9 @@
* the verified surface is a small set of rollup invariants (entity/ * the verified surface is a small set of rollup invariants (entity/
* relationship counts) plus `sourceGeneration`. * relationship counts) plus `sourceGeneration`.
* *
* `sourceGeneration` is the COMMITTED generation of the source-of-truth log * `sourceGeneration` is the generation of the source-of-truth log this
* this projection reflects never the allocated counter, which names a * projection reflects open-time coherence becomes a COMPARISON (stamp vs
* generation that may never commit (see {@link StampVerdict.torn}) so * log head), not a walk:
* open-time coherence becomes a COMPARISON (stamp vs committed head), not a
* walk:
* *
* - equal + invariants hold coherent, serve. * - equal + invariants hold coherent, serve.
* - behind the projection missed the tail (crash between commit and stamp); * - behind the projection missed the tail (crash between commit and stamp);
@ -26,9 +24,6 @@
* - invariants FAIL at equal generation genuine incoherence: loud, and the * - invariants FAIL at equal generation genuine incoherence: loud, and the
* repair ritual (`repairIndex()`, whose recount rebuilds the rollups from a * repair ritual (`repairIndex()`, whose recount rebuilds the rollups from a
* canonical walk) heals it. * canonical walk) heals it.
* - AHEAD a torn generation-log tail: the stamp's fsync outlived the log
* tail's. TERMINAL, never a wait the generation the stamp names does not
* exist to arrive.
* *
* Stamps are JSON on purpose every incident gets debugged by reading a * Stamps are JSON on purpose every incident gets debugged by reading a
* stamp in a terminal. * stamp in a terminal.
@ -75,12 +70,6 @@ export type StampVerdict =
| { state: 'coherent' } | { state: 'coherent' }
| { state: 'absent' } // legacy store — first stamp writes at the next flush | { state: 'absent' } // legacy store — first stamp writes at the next flush
| { state: 'behind'; stampSource: number; head: number } | { state: 'behind'; stampSource: number; head: number }
/**
* TORN GENERATION-LOG TAIL: the stamp witnesses a source generation the
* store's committed watermark can no longer show. TERMINAL there is no
* generation to wait for, so the open demotes (or refuses) and never spins.
*/
| { state: 'torn'; stampSource: number; head: number }
| { state: 'incoherent'; failures: string[] } | { state: 'incoherent'; failures: string[] }
| { state: 'unverifiable'; reason: string } // a FAULT reading the stamp — never conflated with absence | { state: 'unverifiable'; reason: string } // a FAULT reading the stamp — never conflated with absence
@ -129,15 +118,12 @@ export function verifyFamilyStamp(
): StampVerdict { ): StampVerdict {
if (stamp === null) return { state: 'absent' } if (stamp === null) return { state: 'absent' }
if (stamp.sourceGeneration > head) { if (stamp.sourceGeneration > head) {
// A stamp AHEAD of committed truth witnesses a generation the store can no // A stamp AHEAD of the log claims state that never committed — the
// longer show: the stamp's fsync survived a crash that the log tail did // projection was stamped against truth that a crash rolled back.
// not. This is the TORN GENERATION-LOG TAIL — its own class, never folded return {
// in with `incoherent` (a count that drifted at a generation both sides state: 'incoherent',
// agree on), because the two have opposite cures: incoherence is recounted, failures: [`sourceGeneration ${stamp.sourceGeneration} is ahead of the log head ${head}`]
// a tear is DEMOTED. It is also terminal by construction — there is no }
// generation the open can wait for, because the one the stamp names is
// gone.
return { state: 'torn', stampSource: stamp.sourceGeneration, head }
} }
if (stamp.sourceGeneration < head) { if (stamp.sourceGeneration < head) {
return { state: 'behind', stampSource: stamp.sourceGeneration, head } return { state: 'behind', stampSource: stamp.sourceGeneration, head }

View file

@ -147,60 +147,6 @@ export class GenerationSegmentStore {
return this.coveringSegment(gen) !== null return this.coveringSegment(gen) !== null
} }
/**
* @description True when `meta` declares more generations than it holds
* frames a segment sealed by a writer that folded across a hole. The
* manifest records `frames` at fold time, so this is an O(1) comparison
* against the declared span and needs no I/O.
*/
private isSparse(meta: SegmentMeta): boolean {
return meta.lastGeneration - meta.firstGeneration + 1 !== meta.frames
}
/**
* @description The generations this tier ACTUALLY holds, as coalesced
* ascending intervals not what the segments declare.
*
* Dense segments (every one a current writer produces) contribute their
* declared range with no I/O. A SPARSE segment one sealed before the
* density law was enforced, whose declared range spans generations it has
* no frame for has its real generation list read from its sidecar and
* contributed instead, with the discrepancy narrated once.
*
* This is what keeps a store that already carries the damage from wedging.
* `open()` seeds `committedRanges` from these intervals, so a hole is never
* re-admitted as a committed generation, and the auto-compaction pass that
* used to fail on every run with "packed history is damaged" simply never
* asks for the missing frame.
*
* @returns Ascending, non-overlapping `[first, last]` intervals.
*/
async actualRanges(): Promise<Array<[number, number]>> {
const out: Array<[number, number]> = []
for (const meta of this.manifest.segments) {
if (!this.isSparse(meta)) {
out.push([meta.firstGeneration, meta.lastGeneration])
continue
}
const missing = meta.lastGeneration - meta.firstGeneration + 1 - meta.frames
prodLog.warn(
`[GenerationSegments] sealed segment ${meta.file} declares generations ` +
`${meta.firstGeneration}..${meta.lastGeneration} but holds only ${meta.frames} ` +
`frame(s) — ${missing} generation(s) in that span were never folded into it. ` +
`Serving the frames it actually holds; the declared span is not treated as ` +
`committed history. (Written by a pre-density-law writer that folded across a ` +
`gap; the segment itself is intact and no record is lost.)`
)
const idx = await this.sidecarFor(meta)
for (const [gen] of idx.generations) {
const last = out[out.length - 1]
if (last !== undefined && gen === last[1] + 1) last[1] = gen
else out.push([gen, gen])
}
}
return out
}
/** /**
* Fold consecutive generations into ONE new sealed segment + sidecar and * Fold consecutive generations into ONE new sealed segment + sidecar and
* append it to the manifest atomically. Caller guarantees: `gens` is * append it to the manifest atomically. Caller guarantees: `gens` is
@ -218,38 +164,6 @@ export class GenerationSegmentStore {
throw new Error('[GenerationSegments] fold() input must be strictly ascending') throw new Error('[GenerationSegments] fold() input must be strictly ascending')
} }
} }
// THE DENSITY LAW, MADE MECHANICAL.
//
// A sealed segment declares a CONTIGUOUS range [firstGeneration,
// lastGeneration] and every reader treats that range as containment:
// `coveringSegment` is an interval test, `hasGeneration` returns true for
// anything inside it, and `open()` seeds committedRanges from it. So a
// segment folded from a SPARSE input silently claims generations it does
// not hold, and the first read of one of those holes throws
// "inside sealed segment ... but has no frame — packed history is damaged".
//
// That is exactly how the damage was produced. `repackHistory` skipped
// generations mid-batch — ones absent from committedRanges, ones still in
// the pending buffer, ones whose tx.json would not read — and handed the
// survivors here, where the range was computed from the first and last of
// them. Worse, the mis-declared range was then merged back into
// committedRanges at the next open, which is what turned a quiet hole into
// a repeating auto-compaction failure on every subsequent run.
//
// Callers now split at discontinuities; this refusal is what keeps any
// future caller from reintroducing the class. A refusal here loses
// nothing — the generations stay in the live tier, readable, and the next
// pass folds them correctly.
for (let i = 1; i < gens.length; i++) {
if (gens[i].generation !== gens[i - 1].generation + 1) {
throw new Error(
`[GenerationSegments] fold() input is not contiguous: ${gens[i - 1].generation}` +
`${gens[i].generation} skips ${gens[i].generation - gens[i - 1].generation - 1} ` +
`generation(s). A sealed segment declares a dense range, so folding a sparse ` +
`batch would claim generations it does not hold. Split the batch at the gap.`
)
}
}
const last = this.manifest.segments[this.manifest.segments.length - 1] const last = this.manifest.segments[this.manifest.segments.length - 1]
if (last && gens[0].generation <= last.lastGeneration) { if (last && gens[0].generation <= last.lastGeneration) {
throw new Error( throw new Error(
@ -450,37 +364,12 @@ export class GenerationSegmentStore {
return this.decodeFrame(payload) return this.decodeFrame(payload)
} }
} }
// Inside the covering range but with no frame. Two very different causes, // In the covering range but not present: the packed tier is dense by
// and conflating them is what made this class wedge every maintenance pass // construction (fold packs every generation it is handed, including
// on the affected stores. // record-less ones) — absence inside a sealed range is damage.
//
// (1) A SPARSE SEGMENT — the manifest's own `frames` count is smaller than
// the span it declares. That segment was sealed by a writer that
// folded across a hole (the class this file's density law now bars).
// The segment is INTACT and nothing is lost; it simply never held this
// generation. Answering "not packed" is the honest answer, and it lets
// the caller's two-tier read decide what a genuinely absent generation
// means, instead of every compaction pass dying on a repeating throw.
// `actualRanges()` keeps such holes out of committedRanges at open, so
// in a healed store nobody asks this question in the first place.
//
// (2) A DENSE SEGMENT missing a frame it says it has — the manifest and
// the sidecar disagree about a segment that claims to be complete.
// That IS damage, and it stays loud.
if (this.isSparse(meta)) {
prodLog.warn(
`[GenerationSegments] generation ${gen} falls inside sealed segment ${meta.file}'s ` +
`declared range ${meta.firstGeneration}..${meta.lastGeneration}, but that segment ` +
`holds ${meta.frames} frame(s) for a ${meta.lastGeneration - meta.firstGeneration + 1}` +
`-generation span — it was sealed across a gap and never held this generation. ` +
`Reporting it as unpacked rather than as damage; no record is lost.`
)
return null
}
throw new Error( throw new Error(
`[GenerationSegments] generation ${gen} is inside sealed segment ${meta.file}'s declared ` + `[GenerationSegments] generation ${gen} is inside sealed segment ${meta.file}'s declared ` +
`range but has no frame, and that segment declares a complete ${meta.frames}-frame ` + `range but has no frame — packed history is damaged`
`span — the manifest and the sidecar disagree; packed history is damaged`
) )
} }

View file

@ -96,35 +96,6 @@ export const FOLD_CHECKPOINT_PATH = '_system/fold-checkpoint.json'
/** Storage-root-relative prefix of the per-generation record directories. */ /** Storage-root-relative prefix of the per-generation record directories. */
export const GENERATIONS_PREFIX = '_generations' export const GENERATIONS_PREFIX = '_generations'
/**
* @description Split an ascending list of fold candidates into maximal
* CONTIGUOUS runs `[7,8,9,12,13]` becomes `[[7,8,9],[12,13]]`.
*
* A sealed segment declares one dense range `[firstGeneration,
* lastGeneration]`, and every reader treats that range as containment. So a
* batch with a hole in it must never become one segment: it would claim a
* generation it does not hold, and the first read of that hole reports the
* packed history as damaged. One run, one segment the ranges then describe
* exactly what the segments contain.
*
* @param gens - Fold candidates, strictly ascending by generation.
* @returns One array per contiguous run, in ascending order. Empty in, empty out.
*/
export function contiguousRuns(gens: FoldGeneration[]): FoldGeneration[][] {
const runs: FoldGeneration[][] = []
let run: FoldGeneration[] = []
for (const g of gens) {
const prev = run[run.length - 1]
if (prev !== undefined && g.generation !== prev.generation + 1) {
runs.push(run)
run = []
}
run.push(g)
}
if (run.length > 0) runs.push(run)
return runs
}
/** /**
* @description Phases of the {@link GenerationStore.commitTransaction} commit * @description Phases of the {@link GenerationStore.commitTransaction} commit
* protocol at which a test-only fault injector can simulate a process crash. * protocol at which a test-only fault injector can simulate a process crash.
@ -566,29 +537,12 @@ export class GenerationStore {
this.horizonGen = finiteGen(manifest?.horizon, 'manifest horizon') this.horizonGen = finiteGen(manifest?.horizon, 'manifest horizon')
this.counter = Math.max(finiteGen(counterFile?.generation, 'generation counter'), this.committed) this.counter = Math.max(finiteGen(counterFile?.generation, 'generation counter'), this.committed)
// Discover existing generation record directories — BY DIRECTORY NAME. // Discover existing generation record directories.
// This used to call listRawObjects(), which recurses the whole const recordPaths = await this.storage.listRawObjects(GENERATIONS_PREFIX)
// `_generations/` tree and returns every file in every generation, to
// extract a set of integers the top-level directory names already spell.
// MEASURED on a real store with an 11 GB generation history: the phase
// this sits in cost 55,538 ms of a WARM REOPEN after a clean close, with
// no fold to blame — this walk is what it was doing. An adapter without
// the one-level door falls back to the recursive listing, unchanged.
const seenGens = new Set<number>() const seenGens = new Set<number>()
const oneLevel = ( for (const p of recordPaths) {
this.storage as { listRawPrefixes?: (prefix: string) => Promise<string[]> } const gen = parseGenerationFromPath(p)
).listRawPrefixes if (gen !== null) seenGens.add(gen)
if (typeof oneLevel === 'function') {
for (const name of await oneLevel.call(this.storage, GENERATIONS_PREFIX)) {
const gen = Number(name)
if (Number.isSafeInteger(gen) && gen >= 0) seenGens.add(gen)
}
} else {
const recordPaths = await this.storage.listRawObjects(GENERATIONS_PREFIX)
for (const p of recordPaths) {
const gen = parseGenerationFromPath(p)
if (gen !== null) seenGens.add(gen)
}
} }
let rolledBack = 0 let rolledBack = 0
@ -698,56 +652,21 @@ export class GenerationStore {
: 'WHOLE-LOG fold' : 'WHOLE-LOG fold'
: 'above-manifest replay' : 'above-manifest replay'
let replayed = 0 let replayed = 0
const foldStartedAt = Date.now()
const replayFact = async (fact: CommitFact): Promise<void> => { const replayFact = async (fact: CommitFact): Promise<void> => {
for (const op of fact.ops) { for (const op of fact.ops) {
let image: { metadata: unknown | null; vector: unknown | null } const image =
if (op.record === null) { op.record === null
// A genuine tombstone (both legs absent) — the fold removes ? { metadata: null, vector: null }
// both legs, exactly like `writeNounRaw`/`writeVerbRaw`'s raw : { metadata: op.record.metadata, vector: op.record.vector }
// exact-restore contract.
image = { metadata: null, vector: null }
} else if (
op.record.metadata !== null &&
(op.record.vector === null || op.record.vector === undefined)
) {
// PRESERVE-IF-ABSENT (population law, ADR-008 G1 — the fold's
// half): a metadata-only after-image must never DELETE an
// existing vector leg through the fold. `writeNounRaw`/
// `writeVerbRaw` are exact-restore primitives — a `vector:
// null` there means "delete", which is exactly right for
// `rollBackUncommittedGeneration`'s before-image restore (a
// transaction abort legitimately un-writes a vector the failed
// transaction added). It is NOT right here: this fold replays
// AFTER-IMAGES, and re-applying an already-intact record must
// be byte-safe (this module's own invariant, see the log-authority
// comment above) — silently erasing a landed vector because one
// replayed fact's vector leg came back null is the exact defect
// that left metadata-counted, never-enumerated rows in a
// production store (confirmed root cause: the enumeration walk
// used to key on the vector leg, so a preserved-but-then-deleted
// vector made the row invisible while the ledger still counted
// it by metadata). A genuine "unvector" has its own sanctioned,
// ledger-correct path (`Brainy.unvectorNounForRootMigration`) —
// never this raw primitive, and never the fold.
const current =
op.kind === 'verb'
? await this.storage.readVerbRaw(op.id)
: await this.storage.readNounRaw(op.id)
image = { metadata: op.record.metadata, vector: current.vector ?? null }
} else {
image = { metadata: op.record.metadata, vector: op.record.vector }
}
if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image) if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image)
else await this.storage.writeNounRaw(op.id, image) else await this.storage.writeNounRaw(op.id, image)
this.noteCheckpointDirty(op.kind, op.id) this.noteCheckpointDirty(op.kind, op.id)
} }
replayed++ replayed++
if (replayed % 1000 === 0) { if (replayed % 1000 === 0) {
prodLog.narrate( prodLog.warn(
`[GenerationStore] recovery fold in progress — ${replayed} fact(s) folded ` + `[GenerationStore] recovery fold in progress — ${replayed} fact(s) folded ` +
`in ${Date.now() - foldStartedAt}ms (at generation ${fact.generation}); ` + `(at generation ${fact.generation}); do not restart, the fold is finite`
`do not restart, the fold is finite`
) )
} }
if (fact.generation > this.committed) { if (fact.generation > this.committed) {
@ -762,7 +681,7 @@ export class GenerationStore {
} }
} }
if (uncleanOpen) { if (uncleanOpen) {
prodLog.narrate( prodLog.warn(
`[GenerationStore] log-authority recovery: ${foldKind} beginning ` + `[GenerationStore] log-authority recovery: ${foldKind} beginning ` +
`(unclean shutdown detected) — streaming replay, bounded memory, ` + `(unclean shutdown detected) — streaming replay, bounded memory, ` +
`progress every 1000 facts. Do not restart the process; a restart ` + `progress every 1000 facts. Do not restart the process; a restart ` +
@ -785,10 +704,9 @@ export class GenerationStore {
} }
await this.storage.writeRawObject(MANIFEST_PATH, manifest) await this.storage.writeRawObject(MANIFEST_PATH, manifest)
await this.storage.syncRawObjects([MANIFEST_PATH]) await this.storage.syncRawObjects([MANIFEST_PATH])
prodLog.narrate( prodLog.warn(
`[GenerationStore] log-authority recovery replayed ${replayed} fact(s) into ` + `[GenerationStore] log-authority recovery replayed ${replayed} fact(s) into ` +
`canonical in ${Date.now() - foldStartedAt}ms (${foldKind}; committed at ` + `canonical (${foldKind}; committed at ${this.committed}) — an acked write is never lost`
`${this.committed}) — an acked write is never lost`
) )
} }
// A recovery fold re-applied (and the barrier below re-syncs) every // A recovery fold re-applied (and the barrier below re-syncs) every
@ -813,15 +731,9 @@ export class GenerationStore {
if (storageSupportsFactLog(this.storage)) { if (storageSupportsFactLog(this.storage)) {
this.segments = new GenerationSegmentStore(this.storage) this.segments = new GenerationSegmentStore(this.storage)
await this.segments.open() await this.segments.open()
// ACTUAL ranges, not declared ones. A segment sealed by a pre-density-law const packedRanges = this.segments
// writer can declare a span wider than the frames it holds; seeding .segments()
// committedRanges from the declared span re-admits those holes as .map((s): [number, number] => [s.firstGeneration, Math.min(s.lastGeneration, this.committed)])
// committed generations, and every later maintenance pass then asks for a
// frame that was never written. `actualRanges()` reads the real
// generation list from the sidecar for exactly those segments (and does
// no I/O for the dense ones, which is all of them on a healthy store).
const packedRanges = (await this.segments.actualRanges())
.map((r): [number, number] => [r[0], Math.min(r[1], this.committed)])
.filter(([lo, hi]) => lo <= hi) .filter(([lo, hi]) => lo <= hi)
if (packedRanges.length > 0) { if (packedRanges.length > 0) {
// Merge packed (older) + live (newer) interval sets — both ascending; // Merge packed (older) + live (newer) interval sets — both ascending;
@ -3156,26 +3068,13 @@ export class GenerationStore {
foldInput.push({ generation: gen, timestamp: delta.timestamp, delta, records }) foldInput.push({ generation: gen, timestamp: delta.timestamp, delta, records })
} }
if (foldInput.length === 0) continue if (foldInput.length === 0) continue
// SPLIT AT DISCONTINUITIES. `eligible` is NOT contiguous — three await segments.fold(foldInput)
// filters above punch holes in it: a generation missing from segmentsCreated++
// committedRanges never appears, one still in the pending buffer is // Segment + manifest durable → the live copies retire.
// skipped, and one whose tx.json will not read is skipped. A sealed for (const g of foldInput) {
// segment declares a DENSE range, so folding across such a hole makes await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`)
// the segment claim a generation it does not hold; the next open
// merges that mis-declared range into committedRanges, and every
// subsequent auto-compaction pass then asks for the missing frame and
// fails with "packed history is damaged". Fold each contiguous RUN as
// its own segment instead — same bytes, honest ranges.
for (const run of contiguousRuns(foldInput)) {
if (deadline !== undefined && Date.now() >= deadline) break
await segments.fold(run)
segmentsCreated++
// Segment + manifest durable → the live copies retire.
for (const g of run) {
await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`)
}
folded += run.length
} }
folded += foldInput.length
} }
if (folded > 0) { if (folded > 0) {
prodLog.info( prodLog.info(

View file

@ -450,21 +450,6 @@ export interface GenerationStorage {
deleteRawObject(path: string): Promise<void> deleteRawObject(path: string): Promise<void>
/** List raw object paths under a prefix (normalized, `.gz`-stripped). */ /** List raw object paths under a prefix (normalized, `.gz`-stripped). */
listRawObjects(prefix: string): Promise<string[]> listRawObjects(prefix: string): Promise<string[]>
/**
* OPTIONAL: the IMMEDIATE child directory names under a prefix one level,
* no recursion, no file paths.
*
* Why it exists: discovering which generations are on disk needs only the
* top-level directory NAMES under `_generations/`, but the only door for it
* was `listRawObjects`, which recurses the whole tree and returns every file
* in every generation. On a store with a long history that is a full walk of
* the entire generation log, paid on EVERY open, to learn a set of integers
* the directory names already spell out.
*
* An adapter without this door keeps working the caller falls back to the
* recursive listing.
*/
listRawPrefixes?(prefix: string): Promise<string[]>
/** Remove every object under a prefix (and the directory itself on disk). */ /** Remove every object under a prefix (and the directory itself on disk). */
removeRawPrefix(prefix: string): Promise<void> removeRawPrefix(prefix: string): Promise<void>
/** Durability barrier: fsync the given object paths (no-op in memory). */ /** Durability barrier: fsync the given object paths (no-op in memory). */

View file

@ -128,7 +128,7 @@ async function loadBunAssets(): Promise<ModelAssets> {
} }
// Strategy 2: node_modules path relative to CWD (for installed packages) // Strategy 2: node_modules path relative to CWD (for installed packages)
const nmPath = './node_modules/@soulcraftlabs/brainy/assets/models/all-MiniLM-L6-v2' const nmPath = './node_modules/@soulcraft/brainy/assets/models/all-MiniLM-L6-v2'
pathsToTry.push([ pathsToTry.push([
`${nmPath}/model.safetensors`, `${nmPath}/model.safetensors`,
`${nmPath}/tokenizer.json`, `${nmPath}/tokenizer.json`,
@ -168,9 +168,9 @@ async function loadBunAssets(): Promise<ModelAssets> {
// If all strategies fail, provide helpful error message // If all strategies fail, provide helpful error message
throw new Error( throw new Error(
'Could not load model assets. For bun --compile, ensure model files are accessible:\n' + 'Could not load model assets. For bun --compile, ensure model files are accessible:\n' +
' Option 1: Keep node_modules/@soulcraftlabs/brainy/assets/ alongside your binary\n' + ' Option 1: Keep node_modules/@soulcraft/brainy/assets/ alongside your binary\n' +
' Option 2: Copy assets/ folder to your working directory\n' + ' Option 2: Copy assets/ folder to your working directory\n' +
' Option 3: Use --asset flag: bun build --compile --asset="./node_modules/@soulcraftlabs/brainy/assets/**/*"' ' Option 3: Use --asset flag: bun build --compile --asset="./node_modules/@soulcraft/brainy/assets/**/*"'
) )
} }
@ -190,7 +190,7 @@ async function loadNodeAssets(): Promise<ModelAssets> {
if (!fs.existsSync(assetsDir)) { if (!fs.existsSync(assetsDir)) {
throw new Error( throw new Error(
`Model assets not found: ${assetsDir}\n` + `Model assets not found: ${assetsDir}\n` +
`Ensure @soulcraftlabs/brainy is installed correctly.` `Ensure @soulcraft/brainy is installed correctly.`
) )
} }

View file

@ -14,7 +14,7 @@
* - {@link RelationNotFoundError} a referenced relationship (verb) does * - {@link RelationNotFoundError} a referenced relationship (verb) does
* not exist. * not exist.
* *
* Both are exported from the package root (`@soulcraftlabs/brainy`). * Both are exported from the package root (`@soulcraft/brainy`).
*/ */
/** /**

View file

@ -1052,17 +1052,6 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
*/ */
private startAutoFlush(): void { private startAutoFlush(): void {
this.flushTimer = setInterval(async () => { this.flushTimer = setInterval(async () => {
// NO PERIODIC WORK WITHOUT A CAUSE. Ask first, in two O(1) reads: an
// index nobody has written to since the last flush has nothing to
// write, and calling into the trees (and their logging) on a cadence
// over a quiet store is exactly the idle cost this law exists to
// remove.
if (
!this.lsmTreeVerbsBySource.hasPendingWrites() &&
!this.lsmTreeVerbsByTarget.hasPendingWrites()
) {
return
}
await this.flush() await this.flush()
}, this.config.flushInterval) }, this.config.flushInterval)
// Background maintenance must never keep the host process alive — // Background maintenance must never keep the host process alive —

View file

@ -687,17 +687,6 @@ export class LSMTree {
} }
} }
/**
* @description Whether this tree holds anything a flush would write
* the MemTable is non-empty. Synchronous and O(1), so a background cadence
* can ask before it does anything at all: the engine does no periodic work
* without a cause.
* @returns true when a flush would write; false when it would be a no-op.
*/
hasPendingWrites(): boolean {
return !this.memTable.isEmpty()
}
async close(): Promise<void> { async close(): Promise<void> {
this.stopCompactionTimer() this.stopCompactionTimer()

View file

@ -10,7 +10,7 @@ import {
Vector, Vector,
VectorDocument VectorDocument
} from '../coreTypes.js' } from '../coreTypes.js'
import { euclideanDistance, calculateDistancesBatch, isZeroNormVector } from '../utils/index.js' import { euclideanDistance, calculateDistancesBatch } from '../utils/index.js'
import type { BaseStorage } from '../storage/baseStorage.js' import type { BaseStorage } from '../storage/baseStorage.js'
import { getGlobalCache, UnifiedCache } from '../utils/unifiedCache.js' import { getGlobalCache, UnifiedCache } from '../utils/unifiedCache.js'
import { prodLog } from '../utils/logger.js' import { prodLog } from '../utils/logger.js'
@ -64,34 +64,6 @@ export class HnswFlushError extends Error {
} }
} }
/**
* @description Thrown by {@link JsHnswVectorIndex.addItem} / {@link
* JsHnswVectorIndex.updateItem} when handed a length-0 vector. A length-0
* vector is the sanctioned "unvectored" shape for a canonical noun record
* (class-J: a VFS-system row, a deferred embed not yet landed, or any other
* legitimately-vector-less row) but it is NEVER a legal INDEX insert. The
* index itself has no concept of "unvectored"; deciding that a row is
* unvectored and therefore skippable is the FILL/REBUILD/LOAD consumer's job
* (see {@link JsHnswVectorIndex.rebuild}), done BEFORE ever calling addItem.
* A length-0 vector reaching this point is a caller bug: silently accepting
* it would pin `this.dimension = 0` on an empty index (poisoning every real
* insert thereafter with a dimension mismatch) or store a vector-less node
* that a distance calculation can never safely compare against. Loud errors,
* never quiet losses this throws instead of either.
*/
export class EmptyVectorIndexError extends Error {
constructor(public readonly id: string, operation: 'addItem' | 'updateItem') {
super(
`${operation}(${id}): refusing to index a length-0 vector — a length-0 vector is the ` +
`sanctioned "unvectored" shape for a canonical row, but it is never a legal index ` +
`insert. Callers that fill/rebuild/load the index must skip vector.length === 0 rows ` +
`themselves (unvectored = nothing to index, not an error at that layer); reaching ` +
`here with one is a caller bug.`
)
this.name = 'EmptyVectorIndexError'
}
}
/** /**
* Implements {@link VectorIndexProvider}: the vector-index surface Brainy calls * Implements {@link VectorIndexProvider}: the vector-index surface Brainy calls
* on whatever the `'vector'` factory returns (its own `JsHnswVectorIndex`, or a native * on whatever the `'vector'` factory returns (its own `JsHnswVectorIndex`, or a native
@ -608,15 +580,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
throw new Error('Vector is undefined or null') throw new Error('Vector is undefined or null')
} }
// THE INDEX REFUSES A LENGTH-0 VECTOR (see EmptyVectorIndexError's JSDoc):
// an empty vector is the sanctioned "unvectored" shape at the canonical
// layer, never a legal index member. Refusing here — loudly, before the
// dimension pin below — means no future fill/rebuild/load path can ever
// poison `this.dimension` to 0 or park a vector-less node in the graph.
if (vector.length === 0) {
throw new EmptyVectorIndexError(id, 'addItem')
}
// Set dimension on first insert // Set dimension on first insert
if (this.dimension === null) { if (this.dimension === null) {
this.dimension = vector.length this.dimension = vector.length
@ -991,13 +954,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
return return
} }
// Same refusal as addItem (see EmptyVectorIndexError's JSDoc) — an
// in-place relink must never rewrite an already-indexed node down to the
// unvectored shape or poison the pinned dimension.
if (vector.length === 0) {
throw new EmptyVectorIndexError(id, 'updateItem')
}
if (this.dimension === null) { if (this.dimension === null) {
this.dimension = vector.length this.dimension = vector.length
} else if (vector.length !== this.dimension) { } else if (vector.length !== this.dimension) {
@ -1599,15 +1555,7 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
} }
const loaded = await this.storage.getNounVector(noun.id) const loaded = await this.storage.getNounVector(noun.id)
// `loaded` is a length-0 array (not null/undefined) for a canonical row if (!loaded) {
// that is legitimately unvectored — `![]` is FALSE (an empty array is
// truthy), so the bare `!loaded` check below would silently accept it
// as "found" and hand a dimension-0 vector to a distance calculation.
// A node only reaches this lazy-load path because it is a MEMBER of
// the live index (rebuild() now refuses to admit unvectored rows — see
// its JSDoc), so an empty vector here is never legitimate: treat it
// exactly like "not found", loudly.
if (!loaded || loaded.length === 0) {
throw new Error(`Vector not found for noun ${noun.id}`) throw new Error(`Vector not found for noun ${noun.id}`)
} }
@ -1817,56 +1765,9 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
totalCount = result.totalCount || result.items.length totalCount = result.totalCount || result.items.length
// UNVECTORED ROWS ARE NOT AN INDEX MEMBER (the class-J law): a canonical
// noun whose vector leg is `[]` (a VFS-root-style system row, a
// deferred embed not yet landed, or a best-effort fallback for an
// unreadable vector leg) is a normal, enumerable, countable row — it
// is simply not indexed. `storage.getVectorIndexData()` derives its
// {level, connections} answer straight from the noun's OWN record, so
// it returns non-null for every existing noun regardless of whether
// that noun ever actually reached `addItem()` — it cannot be used to
// decide indexability. `nounData.vector.length === 0` is the one
// truthful signal (mirrors the `noun.vector.length > 0` guards in
// {@link getVectorSafe} / {@link getVectorSync}): skip here, counted
// once in a summary line, never per-row spam.
let skippedUnvectored = 0
// Process all nouns at once // Process all nouns at once
for (const nounData of result.items) { for (const nounData of result.items) {
try { try {
if (!Array.isArray(nounData.vector) || nounData.vector.length === 0) {
skippedUnvectored++
continue
}
// THE ZERO-NORM LAW — bulk-rebuild leg: a persisted zero-norm
// vector (a pre-10.4.2 row the canonical write has not yet
// normalized) must never enter the index either, mirroring the
// belt AddToVectorIndexOperation enforces on the live write path.
// Only the canonical vector is authoritative here — persisted
// HNSW graph metadata (level/connections) can outlive an unvector.
if (isZeroNormVector(nounData.vector)) {
prodLog.warn(
`[HNSW] rebuild(): skipping entity ${nounData.id} — persisted vector is ` +
`zero-norm (a zero-norm vector is not a vector and never crosses an ` +
`engine boundary)`
)
continue
}
// Restore the pinned dimension from the first real vector this
// rebuild loads. `addItem`/`updateItem` only pin `this.dimension`
// on a LIVE insert — a fresh rebuild from storage never goes
// through either, so without this the pin stays `null` across a
// restart. A `null` pin means the very next insert (correct OR
// wrong length) silently BECOMES the new pin instead of being
// checked against the store's real dimension — the wrong-length
// case then fails much later and less clearly, inside a distance
// calculation against an already-loaded node, instead of here,
// immediately, with a named expected-vs-got mismatch.
if (this.dimension === null) {
this.dimension = nounData.vector.length
}
// Load HNSW graph data for this entity // Load HNSW graph data for this entity
const hnswData = await this.storage.getVectorIndexData(nounData.id) const hnswData = await this.storage.getVectorIndexData(nounData.id)
@ -1914,10 +1815,7 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
options.onProgress(loadedCount, totalCount) options.onProgress(loadedCount, totalCount)
} }
prodLog.info( prodLog.info(`HNSW: Loaded ${loadedCount.toLocaleString()} nodes (${storageType})`)
`HNSW: Loaded ${loadedCount.toLocaleString()} nodes (${storageType})` +
(skippedUnvectored > 0 ? `${skippedUnvectored.toLocaleString()} unvectored row(s) skipped` : '')
)
} }
// Step 5: CRITICAL - Recover entry point if missing) // Step 5: CRITICAL - Recover entry point if missing)

View file

@ -184,7 +184,6 @@ export {
// Export version utilities // Export version utilities
export { getBrainyVersion } from './utils/version.js' export { getBrainyVersion } from './utils/version.js'
export { contractVersion, BRAINY_CONTRACT_VERSION } from './utils/version.js'
// Export plugin system // Export plugin system
export type { BrainyPlugin, BrainyPluginContext, StorageAdapterFactory } from './plugin.js' export type { BrainyPlugin, BrainyPluginContext, StorageAdapterFactory } from './plugin.js'

View file

@ -9,7 +9,7 @@
* *
* @example Enable integrations (recommended) * @example Enable integrations (recommended)
* ```typescript * ```typescript
* import { Brainy } from '@soulcraftlabs/brainy' * import { Brainy } from '@soulcraft/brainy'
* *
* const brain = new Brainy({ integrations: true }) * const brain = new Brainy({ integrations: true })
* await brain.init() * await brain.init()

View file

@ -41,7 +41,7 @@ The `BrainyMCPService` has been refactored to separate the core functionality fr
### In Any Environment (Browser, Node.js, Server) ### In Any Environment (Browser, Node.js, Server)
```typescript ```typescript
import { Brainy, BrainyMCPAdapter, MCPAugmentationToolset } from '@soulcraftlabs/brainy' import { Brainy, BrainyMCPAdapter, MCPAugmentationToolset } from '@soulcraft/brainy'
// Create a Brainy instance // Create a Brainy instance
const brainyData = new Brainy() const brainyData = new Brainy()
@ -81,7 +81,7 @@ const toolResponse = await toolset.handleRequest({
### In Browser Environment (Core Functionality Only) ### In Browser Environment (Core Functionality Only)
```typescript ```typescript
import { Brainy, BrainyMCPService } from '@soulcraftlabs/brainy' import { Brainy, BrainyMCPService } from '@soulcraft/brainy'
// Create a Brainy instance // Create a Brainy instance
const brainyData = new Brainy() const brainyData = new Brainy()

View file

@ -2,7 +2,7 @@
* 🧠 BRAINY EMBEDDED PATTERNS * 🧠 BRAINY EMBEDDED PATTERNS
* *
* AUTO-GENERATED - DO NOT EDIT * AUTO-GENERATED - DO NOT EDIT
* Generated: 2025-09-29T10:10:00-07:00 * Generated: 2026-07-02T21:43:26.976Z
* Patterns: 220 * Patterns: 220
* Coverage: 94-98% of all queries * Coverage: 94-98% of all queries
* *

View file

@ -2,7 +2,7 @@
* 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS
* *
* AUTO-GENERATED - DO NOT EDIT * AUTO-GENERATED - DO NOT EDIT
* Generated: 2026-06-29T10:04:19-07:00 * Generated: 2026-02-09T16:59:48.867Z
* Noun Types: 42 * Noun Types: 42
* Verb Types: 127 * Verb Types: 127
* *
@ -19,7 +19,7 @@ export const TYPE_METADATA = {
verbTypes: 127, verbTypes: 127,
totalTypes: 169, totalTypes: 169,
embeddingDimensions: 384, embeddingDimensions: 384,
generatedAt: "2026-06-29T10:04:19-07:00", generatedAt: "2026-02-09T16:59:48.867Z",
sizeBytes: { sizeBytes: {
embeddings: 259584, embeddings: 259584,
base64: 346112 base64: 346112

View file

@ -22,7 +22,7 @@ import type { GraphIndexStats } from './graph/graphAdjacencyIndex.js'
// Re-export the provider contracts that already live closer to their // Re-export the provider contracts that already live closer to their
// implementations so a plugin author (Cor) can import the *entire* // implementations so a plugin author (Cor) can import the *entire*
// provider surface from one stable entrypoint: `@soulcraftlabs/brainy/plugin`. // provider surface from one stable entrypoint: `@soulcraft/brainy/plugin`.
export type { ColumnStoreProvider } from './indexes/columnStore/types.js' export type { ColumnStoreProvider } from './indexes/columnStore/types.js'
export type { export type {
AggregationProvider, AggregationProvider,
@ -41,7 +41,7 @@ export interface BrainyPlugin {
name: string name: string
/** /**
* Optional semver range of `@soulcraftlabs/brainy` this plugin supports * Optional semver range of `@soulcraft/brainy` this plugin supports
* (e.g. `'>=8.0.0 <9.0.0'` or `'^8.0.0'`). When set and the running brainy is * (e.g. `'>=8.0.0 <9.0.0'` or `'^8.0.0'`). When set and the running brainy is
* OUTSIDE the range, `init()` THROWS rather than silently falling back to the * OUTSIDE the range, `init()` THROWS rather than silently falling back to the
* default JS engine. This is the version-coupling guard for the native * default JS engine. This is the version-coupling guard for the native

View file

@ -1066,18 +1066,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
protected allCountsSuspect = false protected allCountsSuspect = false
/** One narration per session for the suspect transition (never per delete). */ /** One narration per session for the suspect transition (never per delete). */
private allCountsSuspectNarrated = false private allCountsSuspectNarrated = false
/**
* Which rule produced the ALL scalars currently in memory. `'identity-record'`
* means one counted entity per metadata content leg the honest rule: a
* bare id-directory (a ghost or scar left by a partial-delete defect, no
* content leg) counts zero. Set by the one-time derivation and by the
* sanctioned recount, alongside `allCountsSuspect = false`; left `undefined`
* when a loaded counts.json carries the ALL scalars but no stamp the
* legacy container-rule derivation, which forces `allCountsSuspect = true`
* at load instead. A filesystem concern: `MemoryStorage` has no counts.json
* and never sets this.
*/
protected allCountsDerivedBy?: 'identity-record'
protected entityCounts: Map<string, number> = new Map() // type -> count protected entityCounts: Map<string, number> = new Map() // type -> count
protected verbCounts: Map<string, number> = new Map() // verb type -> count protected verbCounts: Map<string, number> = new Map() // verb type -> count
protected countCache: Map<string, { count: number; timestamp: number }> = new Map() protected countCache: Map<string, { count: number; timestamp: number }> = new Map()
@ -1164,24 +1152,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
}) })
} }
/**
* OPTIONAL narrow ledger hook (see {@link StorageAdapter.noteVectorUnlanded}):
* the mirror of {@link noteVectorLanded} record a noun's vector was just
* REMOVED (rewritten to the unvectored `[]` shape). Never below zero: a
* caller that (incorrectly) fires this for a noun already unvectored would
* otherwise drive the ledger negative clamped defensively, matching the
* delete path's `if (this.totalVectoredNounCount > 0)` guard.
* @param id - The noun whose vector was just removed (retained for a
* future narration seam; the count itself needs no id-keyed state).
*/
async noteVectorUnlanded(id: string): Promise<void> {
void id
if (this.totalVectoredNounCount > 0) this.totalVectoredNounCount--
this.scheduleCountPersist().catch(() => {
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
})
}
/** /**
* Increment count for entity type - O(1) operation. * Increment count for entity type - O(1) operation.
* Concurrency is handled by the process-global mutex * Concurrency is handled by the process-global mutex

View file

@ -14,13 +14,10 @@ import {
StorageBatchConfig, StorageBatchConfig,
SYSTEM_DIR, SYSTEM_DIR,
STATISTICS_KEY, STATISTICS_KEY,
WriterLockInfo, WriterLockInfo
WriterCloseRecord
} from '../baseStorage.js' } from '../baseStorage.js'
import { getBrainyVersion } from '../../utils/index.js' import { getBrainyVersion } from '../../utils/index.js'
import { isAbsentError } from '../../utils/errorClassification.js' import { isAbsentError } from '../../utils/errorClassification.js'
import { prodLog } from '../../utils/logger.js'
import { isZeroNormVector } from '../../utils/distance.js'
import { import {
TornRecordError, TornRecordError,
isUnparseablePayloadError, isUnparseablePayloadError,
@ -100,30 +97,7 @@ export class FileSystemStorage extends BaseStorage {
// timer rewrites the lock every 10s so stale-lock detection can tell a dead // timer rewrites the lock every 10s so stale-lock detection can tell a dead
// writer from a slow one. The constant name matches the file path used. // writer from a slow one. The constant name matches the file path used.
private static readonly WRITER_LOCK_FILE = '_writer.lock' private static readonly WRITER_LOCK_FILE = '_writer.lock'
/** private static readonly WRITER_HEARTBEAT_MS = 10_000
* The clean-close record at `locks/_writer.close` (see
* {@link WriterCloseRecord}). Written when the lock is released, consumed by
* the next claim, so an open can distinguish "the previous writer left" from
* "the previous writer died" without inferring either from a pid.
*/
private static readonly WRITER_CLOSE_FILE = '_writer.close'
/**
* How often the lock file's `lastHeartbeat` is rewritten.
*
* THIS IS OBSERVABILITY ONLY, and the cadence follows from that. Staleness
* is decided by PID LIVENESS alone (see isWriterLockStale) and the fence
* compares pid + hostname no decision anywhere reads this timestamp. It
* exists so an operator inspecting a lock file, or reading the
* BRAINY_WRITER_LOCKED error, can judge liveness themselves.
*
* At 10s it was a lock-file WRITE every ten seconds per brain, forever: 2.1
* writes/s across a production process holding 21 idle brains, for a
* human-readable timestamp nothing computes with. At 60s an operator still
* sees a heartbeat inside the minute, at a sixth of the cost. With the
* clean-close record now recording orderly releases explicitly, the
* heartbeat carries even less weight than it did.
*/
private static readonly WRITER_HEARTBEAT_MS = 60_000
private static readonly WRITER_STALE_THRESHOLD_MS = 60_000 private static readonly WRITER_STALE_THRESHOLD_MS = 60_000
private writerLockHeartbeat?: NodeJS.Timeout private writerLockHeartbeat?: NodeJS.Timeout
private writerLockInfo?: WriterLockInfo private writerLockInfo?: WriterLockInfo
@ -136,13 +110,6 @@ export class FileSystemStorage extends BaseStorage {
*/ */
private writerHeartbeatInFlight?: Promise<void> private writerHeartbeatInFlight?: Promise<void>
/**
* The in-flight background count-ledger derivation, if one was needed at
* open. See {@link scheduleCountLedgerDerivation} awaited only by
* {@link whenCountLedgerSettled}, never by a read.
*/
private countLedgerDerivation?: Promise<void>
// Flush-request RPC state. The writer polls `locks/_flush_requests/` for // Flush-request RPC state. The writer polls `locks/_flush_requests/` for
// new `.req` files and emits `.ack` files in `locks/_flush_responses/` after // new `.req` files and emits `.ack` files in `locks/_flush_responses/` after
// flushing. Inspectors call `requestFlushOverFilesystem` to drop a request // flushing. Inspectors call `requestFlushOverFilesystem` to drop a request
@ -151,16 +118,9 @@ export class FileSystemStorage extends BaseStorage {
private static readonly FLUSH_REQUEST_DIR = '_flush_requests' private static readonly FLUSH_REQUEST_DIR = '_flush_requests'
private static readonly FLUSH_RESPONSE_DIR = '_flush_responses' private static readonly FLUSH_RESPONSE_DIR = '_flush_responses'
private static readonly FLUSH_WATCH_INTERVAL_MS = 500 private static readonly FLUSH_WATCH_INTERVAL_MS = 500
/**
* The safety sweep behind the fs.watch: catches events an exotic filesystem
* dropped, and runs the stale-request GC. See startFlushRequestWatcher.
*/
private static readonly FLUSH_SAFETY_SWEEP_MS = 30_000
private static readonly FLUSH_POLL_INTERVAL_MS = 100 private static readonly FLUSH_POLL_INTERVAL_MS = 100
private static readonly FLUSH_REQUEST_TTL_MS = 60_000 private static readonly FLUSH_REQUEST_TTL_MS = 60_000
private flushWatcherInterval?: NodeJS.Timeout private flushWatcherInterval?: NodeJS.Timeout
/** The inotify-backed watch on the request directory, when the FS supports one. */
private flushWatcher?: import('node:fs').FSWatcher
private flushWatcherInFlight = false private flushWatcherInFlight = false
private flushWatcherOnRequest?: () => Promise<void> private flushWatcherOnRequest?: () => Promise<void>
@ -642,20 +602,6 @@ export class FileSystemStorage extends BaseStorage {
* automatically. Returns the pruned container ids so the caller can recompute * automatically. Returns the pruned container ids so the caller can recompute
* counts. * counts.
*/ */
/**
* @description Whether an id directory's file legs include the metadata
* CONTENT leg (`metadata.json` or its `.json.gz` variant) the single
* test that decides whether an `entities/<kind>/<shard>/<id>/` container is
* a live entity or a ghost/scar orphan left by the pre-8.3.1 partial-delete
* defect (see {@link pruneOrphanedEntities}). Shared by the orphan prune
* and {@link scanCanonicalEntities} so the two agree by construction one
* counted entity per identity record, never per bare container.
* @param legs - File names in one `entities/<kind>/<shard>/<id>/` directory.
*/
private hasMetadataContentLeg(legs: string[]): boolean {
return legs.some((f) => f.startsWith('metadata.json'))
}
public async pruneOrphanedEntities(): Promise<{ nouns: string[]; verbs: string[] }> { public async pruneOrphanedEntities(): Promise<{ nouns: string[]; verbs: string[] }> {
await this.ensureInitialized() await this.ensureInitialized()
const pruned: { nouns: string[]; verbs: string[] } = { nouns: [], verbs: [] } const pruned: { nouns: string[]; verbs: string[] } = { nouns: [], verbs: [] }
@ -695,7 +641,7 @@ export class FileSystemStorage extends BaseStorage {
} }
// A live entity has its metadata content leg. No content leg → a // A live entity has its metadata content leg. No content leg → a
// vector-only ghost or an empty scar → prune the whole container. // vector-only ghost or an empty scar → prune the whole container.
if (this.hasMetadataContentLeg(legs)) continue if (legs.some((f) => f.startsWith('metadata.json'))) continue
await fs.promises.rm(idAbs, { recursive: true, force: true }) await fs.promises.rm(idAbs, { recursive: true, force: true })
pruned[kind].push(entry.name) pruned[kind].push(entry.name)
console.warn( console.warn(
@ -709,30 +655,6 @@ export class FileSystemStorage extends BaseStorage {
return pruned return pruned
} }
/**
* @description The IMMEDIATE child directory names under a prefix ONE
* `readdir`, no recursion, no file paths. See the seam's JSDoc
* (`src/db/types.ts`) for what this replaced: discovering the generations on
* disk walked the entire generation log on every open, reading out every
* file in every generation, to learn the set of integers the top-level
* directory names already spell.
* @param prefix - Storage-root-relative directory prefix.
* @returns The child directory names (not paths); empty when the prefix does
* not exist.
*/
public override async listRawPrefixes(prefix: string): Promise<string[]> {
await this.ensureInitialized()
const fullPath = path.join(this.rootDir, prefix)
try {
const entries = await fs.promises.readdir(fullPath, { withFileTypes: true })
return entries.filter((e: { isDirectory: () => boolean }) => e.isDirectory())
.map((e: { name: string }) => e.name)
} catch (error: any) {
if (error?.code === 'ENOENT') return []
throw error
}
}
/** /**
* Primitive operation: List objects under path prefix * Primitive operation: List objects under path prefix
* All metadata operations use this internally via base class routing * All metadata operations use this internally via base class routing
@ -1943,41 +1865,18 @@ export class FileSystemStorage extends BaseStorage {
} }
} }
// THE CLEAN-CLOSE RECORD IS READ BEFORE ANY VERDICT (see
// WriterCloseRecord). A lock file whose release was RECORDED is
// bookkeeping left by an orderly shutdown, not evidence of anything —
// and that is true whether the previous holder was another process or
// an earlier instance in THIS one. A production restart reported
// "Re-acquiring writer lock ... this is a bug" immediately after a clean
// close, sending an operator hunting for a leak that did not exist.
const closeRecord = existing ? await this.readWriterCloseRecord() : null
const releasedCleanly =
existing !== null &&
closeRecord !== null &&
this.closeRecordVouchesFor(closeRecord, existing)
if (existing) { if (existing) {
// Same-process re-open: a second Brainy instance in this Node process // Same-process re-open: a second Brainy instance in this Node process
// (e.g. test "simulate server restart" patterns, or a consumer that // (e.g. test "simulate server restart" patterns, or a consumer that
// explicitly re-instantiates without closing first). This isn't the // explicitly re-instantiates without closing first). This isn't the
// dangerous cross-process case the lock exists to prevent — the two // dangerous cross-process case the lock exists to prevent — the two
// instances share a memory space and can't silently diverge from each // instances share a memory space and can't silently diverge from each
// other beyond what their callers already see. Warn and take over — // other beyond what their callers already see. Warn and take over.
// unless the record proves the previous instance already let go, in
// which case there is nothing to warn about.
if (existing.pid === myPid && existing.hostname === hostname && !options?.force) { if (existing.pid === myPid && existing.hostname === hostname && !options?.force) {
if (releasedCleanly) { console.warn(
console.warn( `[brainy] Re-acquiring writer lock for ${this.rootDir} held by the same process (PID ${existing.pid}). ` +
`[brainy] Clearing the leftover writer lock for ${this.rootDir} — an earlier ` + `If you intended to keep the previous Brainy instance alive, this is a bug — close it first.`
`instance in this process (PID ${existing.pid}) RELEASED it cleanly at ` + )
`${closeRecord!.closedAt} but could not remove the file. Nothing to recover.`
)
} else {
console.warn(
`[brainy] Re-acquiring writer lock for ${this.rootDir} held by the same process (PID ${existing.pid}). ` +
`If you intended to keep the previous Brainy instance alive, this is a bug — close it first.`
)
}
const info: WriterLockInfo = { const info: WriterLockInfo = {
pid: myPid, pid: myPid,
hostname, hostname,
@ -1987,18 +1886,11 @@ export class FileSystemStorage extends BaseStorage {
rootDir: this.rootDir rootDir: this.rootDir
} }
await this.writeFileAtomic(lockFile, JSON.stringify(info, null, 2)) await this.writeFileAtomic(lockFile, JSON.stringify(info, null, 2))
await this.clearWriterCloseRecord()
this.installWriterLock(info) this.installWriterLock(info)
return info return info
} }
// A cleanly-released lock is stale by RECORD, not by inference. Only const stale = !options?.force && (await this.isWriterLockStale(existing))
// when no record vouches for this lock do we fall back to pid
// liveness, and then we say THAT honestly too: an unrecorded lock
// means the writer did not complete its close, so the store was not
// closed cleanly and this open pays recovery.
const stale =
releasedCleanly || (!options?.force && (await this.isWriterLockStale(existing)))
if (!options?.force && !stale) { if (!options?.force && !stale) {
// Consumer-facing error contract: callers detect this case via // Consumer-facing error contract: callers detect this case via
// err.code and read the holder's details from err.lockInfo. // err.code and read the holder's details from err.lockInfo.
@ -2009,16 +1901,8 @@ export class FileSystemStorage extends BaseStorage {
options?.force options?.force
? `[brainy] Force-overwriting writer lock for ${this.rootDir} ` + ? `[brainy] Force-overwriting writer lock for ${this.rootDir} ` +
`(was held by PID ${existing.pid} on ${existing.hostname}).` `(was held by PID ${existing.pid} on ${existing.hostname}).`
: releasedCleanly : `[brainy] Overwriting stale writer lock for ${this.rootDir} ` +
? `[brainy] Clearing the leftover writer lock for ${this.rootDir}` + `(PID ${existing.pid} on ${existing.hostname} appears dead).`
`PID ${existing.pid} on ${existing.hostname} RELEASED it cleanly at ` +
`${closeRecord!.closedAt} but could not remove the file. ` +
`Nothing to recover.`
: `[brainy] Overwriting stale writer lock for ${this.rootDir} ` +
`(PID ${existing.pid} on ${existing.hostname} is gone and left NO ` +
`clean-close record — that writer did not finish closing, so this ` +
`store was not closed cleanly; open will run crash recovery and ` +
`report its wall).`
) )
// Takeover: verify the file still holds the lock we judged (a live // Takeover: verify the file still holds the lock we judged (a live
// successor may have claimed meanwhile), then remove it and fall // successor may have claimed meanwhile), then remove it and fall
@ -2072,12 +1956,6 @@ export class FileSystemStorage extends BaseStorage {
await fs.promises.unlink(claimTmp).catch(() => {}) await fs.promises.unlink(claimTmp).catch(() => {})
} }
// CONSUME the previous writer's clean-close record. It described the
// lock generation that just ended; leaving it in place would let it
// vouch for OUR lock if this process later dies without closing —
// turning a real crash into a "closed cleanly" verdict. One unlink.
await this.clearWriterCloseRecord()
this.installWriterLock(info) this.installWriterLock(info)
return info return info
} }
@ -2201,27 +2079,13 @@ export class FileSystemStorage extends BaseStorage {
return return
} }
const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE) const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE)
const released = this.writerLockInfo
try { try {
// Only delete if we still own it — avoid clobbering a successor that // Only delete if we still own it — avoid clobbering a successor that
// claimed the lock via force-override. // claimed the lock via force-override.
const current = await this.readWriterLock() const current = await this.readWriterLock()
const ours = if (current && current.pid === this.writerLockInfo.pid && current.hostname === this.writerLockInfo.hostname) {
current === null ||
(current.pid === released.pid && current.hostname === released.hostname)
if (current && ours) {
await fs.promises.unlink(lockFile) await fs.promises.unlink(lockFile)
} }
// THE CLEAN-CLOSE RECORD (see WriterCloseRecord). Written whenever this
// instance gives up a lock nobody else has taken — the unlink above
// having succeeded OR the file already being gone. The next open reads
// it instead of guessing from pid liveness: a recorded release is an
// orderly shutdown, an absent record is a writer that never finished
// closing. Not written when a successor holds the lock: our release is
// then a no-op and a record would slander their live lock.
if (ours) {
await this.writeWriterCloseRecord(released)
}
} catch (err: any) { } catch (err: any) {
if (err.code !== 'ENOENT') { if (err.code !== 'ENOENT') {
console.warn('[brainy] Failed to release writer lock file:', err) console.warn('[brainy] Failed to release writer lock file:', err)
@ -2231,97 +2095,6 @@ export class FileSystemStorage extends BaseStorage {
} }
} }
/**
* @description Read the clean-close record at `locks/_writer.close`, or
* `null` when it is absent or unparseable. A torn record is treated as
* absent the conservative direction, since an unreadable record can
* vouch for nothing.
* @returns The record, or null.
*/
public async readWriterCloseRecord(): Promise<WriterCloseRecord | null> {
await this.ensureInitialized()
const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE)
try {
const raw = await fs.promises.readFile(recordFile, 'utf-8')
const parsed = JSON.parse(raw) as WriterCloseRecord
if (
typeof parsed?.pid !== 'number' ||
typeof parsed?.hostname !== 'string' ||
typeof parsed?.startedAt !== 'string' ||
typeof parsed?.closedAt !== 'string'
) {
return null
}
return parsed
} catch (err: any) {
if (err.code === 'ENOENT') return null
return null
}
}
/**
* @description Whether a clean-close record describes the very lock
* generation `lock` represents. The match is pid + hostname + `startedAt`:
* `startedAt` is the lock generation's identity, so a record can never
* vouch for a LATER lock taken by the same pid on the same host (the
* same-process re-open path mints a fresh `startedAt`).
* @param record - The clean-close record read from disk.
* @param lock - The lock file's contents.
*/
private closeRecordVouchesFor(record: WriterCloseRecord, lock: WriterLockInfo): boolean {
return (
record.pid === lock.pid &&
record.hostname === lock.hostname &&
record.startedAt === lock.startedAt
)
}
/**
* @description Write the clean-close record for a lock this instance just
* released. Atomic (temp + rename) so a concurrent opener never reads half
* a record. A failure here costs the next open nothing but the honest
* fallback (pid liveness), so it warns rather than failing the close.
* @param released - The lock info this instance held.
*/
private async writeWriterCloseRecord(released: WriterLockInfo): Promise<void> {
const record: WriterCloseRecord = {
pid: released.pid,
hostname: released.hostname,
startedAt: released.startedAt,
closedAt: new Date().toISOString(),
version: released.version
}
const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE)
try {
await this.writeFileAtomic(recordFile, JSON.stringify(record, null, 2))
} catch (err) {
// ENOENT = the lock directory is gone, i.e. the whole store was removed
// under us. There is no next open to inform.
if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return
console.warn(
`[brainy] Failed to write the writer clean-close record for ${this.rootDir}` +
`the next open will fall back to pid liveness and may report this orderly ` +
`shutdown as a crash:`,
err
)
}
}
/**
* @description Remove the clean-close record. Called by every successful
* lock claim so a record never outlives the lock generation it describes.
*/
private async clearWriterCloseRecord(): Promise<void> {
const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE)
try {
await fs.promises.unlink(recordFile)
} catch (err: any) {
if (err.code !== 'ENOENT') {
console.warn('[brainy] Failed to clear the writer clean-close record:', err)
}
}
}
public override async readWriterLock(): Promise<WriterLockInfo | null> { public override async readWriterLock(): Promise<WriterLockInfo | null> {
await this.ensureInitialized() await this.ensureInitialized()
const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE) const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE)
@ -2408,115 +2181,36 @@ export class FileSystemStorage extends BaseStorage {
/** /**
* Start watching for cross-process flush requests. Called by Brainy.init() * Start watching for cross-process flush requests. Called by Brainy.init()
* in writer mode. Each new `.req` file in `locks/_flush_requests/` triggers * in writer mode. Polls `locks/_flush_requests/` every
* the supplied callback (`brain.flush()`), after which an `.ack` is written * FLUSH_WATCH_INTERVAL_MS each new `.req` file triggers the supplied
* to `locks/_flush_responses/` with the same request ID. Stale `.req` files * callback (`brain.flush()`), after which an `.ack` is written to
* (>FLUSH_REQUEST_TTL_MS) are garbage-collected on each sweep. * `locks/_flush_responses/` with the same request ID. Stale `.req` files
* * (>FLUSH_REQUEST_TTL_MS) are garbage-collected on every tick.
* THE WATCH IS EVENT-DRIVEN, NOT A POLL. It used to `readdir` the request
* directory every 500 ms, per brain, for the entire life of every writer
* armed on every non-reader brain whether or not any inspector process
* existed. MEASURED on a production process holding 21 brains: 42 directory
* reads per second on a completely idle service, plus a stale-request GC
* pass on every one of them. The engine does no periodic work without a
* cause, and a request that has not been made is not a cause.
*
* `fs.watch` (inotify on Linux) delivers the arrival itself, so a request is
* seen SOONER than the old poll saw it. Two honest concessions ride with it:
* - a slow SAFETY SWEEP (FLUSH_SAFETY_SWEEP_MS) still runs, because
* `fs.watch` can miss events on network and fuse filesystems and because
* the stale-request GC needs some tick of its own. At 30s that is 0.7
* reads/s across 21 brains where the poll cost 42.
* - a filesystem that cannot watch at all falls back to the ORIGINAL
* 500 ms poll, narrated once, because correctness outranks idle cost:
* an inspector whose request is never seen waits forever.
*/ */
public override startFlushRequestWatcher(onRequest: () => Promise<void>): void { public override startFlushRequestWatcher(onRequest: () => Promise<void>): void {
// Already watching — or already ARMING. The arm is asynchronous (the if (this.flushWatcherInterval) return // already watching
// request directory is created before it can be watched), so neither the
// watcher nor the interval exists yet during that window; the callback is
// the flag that covers it. Without this a second call in the window would
// leave two watchers and two sweeps running for the life of the store.
if (this.flushWatcherInterval || this.flushWatcher || this.flushWatcherOnRequest) return
this.flushWatcherOnRequest = onRequest this.flushWatcherOnRequest = onRequest
const reqDir = path.join(this.lockDir, FileSystemStorage.FLUSH_REQUEST_DIR) const reqDir = path.join(this.lockDir, FileSystemStorage.FLUSH_REQUEST_DIR)
const ackDir = path.join(this.lockDir, FileSystemStorage.FLUSH_RESPONSE_DIR) const ackDir = path.join(this.lockDir, FileSystemStorage.FLUSH_RESPONSE_DIR)
const sweep = (): void => { // Ensure both dirs exist up front so the first .req drop doesn't race with mkdir.
if (this.flushWatcherInFlight) return // skip overlapping sweep this.ensureDirectoryExists(reqDir).catch(() => {})
this.ensureDirectoryExists(ackDir).catch(() => {})
this.flushWatcherInterval = setInterval(() => {
if (this.flushWatcherInFlight) return // skip overlapping tick
this.flushWatcherInFlight = true this.flushWatcherInFlight = true
this.processFlushRequests(reqDir, ackDir).finally(() => { this.processFlushRequests(reqDir, ackDir).finally(() => {
this.flushWatcherInFlight = false this.flushWatcherInFlight = false
}) })
} }, FileSystemStorage.FLUSH_WATCH_INTERVAL_MS)
// Ensure both dirs exist up front so the first .req drop doesn't race with
// mkdir — and so there is a directory to watch.
void this.ensureDirectoryExists(reqDir)
.then(() => this.ensureDirectoryExists(ackDir))
.then(() => {
if (this.flushWatcherOnRequest !== onRequest) return // stopped meanwhile
try {
const watcher = fs.watch(reqDir, () => sweep())
this.flushWatcher = watcher
watcher.on('error', (err: Error) => {
// A watch that dies mid-life must not leave the door deaf.
console.warn(
`[brainy] Flush-request watch failed (${err.message}) — falling back to polling.`
)
this.flushWatcher?.close()
this.flushWatcher = undefined
// The SAFETY sweep must go first. It is already armed at 30s, and
// startFlushRequestPolling() declines to arm over an existing
// interval — so leaving it would quietly leave this store answering
// flush requests on a 30s cadence instead of the 500ms one the door
// promises. A degrade nobody asked for is still a degrade.
if (this.flushWatcherInterval) {
clearInterval(this.flushWatcherInterval)
this.flushWatcherInterval = undefined
}
this.startFlushRequestPolling(sweep)
})
if (typeof watcher.unref === 'function') watcher.unref()
// The safety sweep: missed events on exotic filesystems, and the
// stale-request GC.
this.flushWatcherInterval = setInterval(sweep, FileSystemStorage.FLUSH_SAFETY_SWEEP_MS)
if (typeof this.flushWatcherInterval.unref === 'function') {
this.flushWatcherInterval.unref()
}
// One sweep now: a request may have been dropped before the watch armed.
sweep()
} catch (err) {
console.warn(
`[brainy] Flush-request directory cannot be watched on this filesystem ` +
`(${(err as Error).message}) — polling every ` +
`${FileSystemStorage.FLUSH_WATCH_INTERVAL_MS}ms instead.`
)
this.startFlushRequestPolling(sweep)
}
})
.catch(() => {
// The request directory could not be created; nothing to watch. A
// cross-process flush request cannot be made either, so there is
// nothing to miss.
})
}
/** The original 500 ms poll — the fallback when a directory cannot be watched. */
private startFlushRequestPolling(sweep: () => void): void {
if (this.flushWatcherInterval) return
this.flushWatcherInterval = setInterval(sweep, FileSystemStorage.FLUSH_WATCH_INTERVAL_MS)
if (typeof this.flushWatcherInterval.unref === 'function') { if (typeof this.flushWatcherInterval.unref === 'function') {
this.flushWatcherInterval.unref() this.flushWatcherInterval.unref()
} }
} }
public override stopFlushRequestWatcher(): void { public override stopFlushRequestWatcher(): void {
if (this.flushWatcher) {
this.flushWatcher.close()
this.flushWatcher = undefined
}
if (this.flushWatcherInterval) { if (this.flushWatcherInterval) {
clearInterval(this.flushWatcherInterval) clearInterval(this.flushWatcherInterval)
this.flushWatcherInterval = undefined this.flushWatcherInterval = undefined
@ -2896,46 +2590,19 @@ export class FileSystemStorage extends BaseStorage {
) { ) {
this.totalNounCountAll = counts.totalNounCountAll this.totalNounCountAll = counts.totalNounCountAll
this.totalVerbCountAll = counts.totalVerbCountAll this.totalVerbCountAll = counts.totalVerbCountAll
if (counts.allCountsDerivedBy === 'identity-record') { this.allCountsSuspect = counts.allCountsSuspect === true
// Derived (or recounted) under the honest rule — one counted
// entity per metadata content leg. Trust the persisted suspect
// flag as-is; an unprovable delete since may still have set it.
this.allCountsDerivedBy = 'identity-record'
this.allCountsSuspect = counts.allCountsSuspect === true
} else {
// The ALL scalars exist but predate the identity-record stamp —
// they were derived under the legacy rule that counted one
// entity per id DIRECTORY, so orphaned ghost/scar containers (a
// pre-8.3.1 partial-delete defect — see pruneOrphanedEntities())
// were counted as entities too. O(1) field read, NEVER a walk
// here: force suspect and name it loudly. A sanctioned recount
// (repairIndex) restores exact denominators and clears this.
this.allCountsDerivedBy = undefined
this.allCountsSuspect = true
needsPersist = true
prodLog.narrate(
'[FileSystemStorage] canonical count ledger was derived under the legacy ' +
'container rule — it counts one entity per id DIRECTORY, so every ghost/scar ' +
'container inflates it. Marked suspect, and an honest recount is scheduled to ' +
'run in the background after this open; until it lands, do not subtract ' +
'against these ALL scalars.'
)
// A suspect ledger used to stay wrong for the life of the store,
// waiting for an operator to run repairIndex. A downstream index
// heal took its "remaining" figure from these inflated
// denominators and reported work that did not exist. The ledger
// now HEALS ITSELF — in the background, because a denominator is
// a derived scalar and no read is ever served from it.
this.scheduleCountLedgerDerivation('legacy container-rule ledger')
}
} else { } else {
// No ALL scalars at all. There is nothing to serve in the meantime — const nouns = await this.scanCanonicalEntities('nouns')
// a zero would read as an empty store — so the scalars stay unknown const verbs = await this.scanCanonicalEntities('verbs')
// and SUSPECT until the background derivation lands. The open does this.totalNounCountAll = nouns.count
// not wait for it: an id-tree walk is O(ids) and this file has been this.totalVerbCountAll = verbs.count
// the whole reason a 24k-id store opened in silence. this.allCountsSuspect = false
this.allCountsSuspect = true console.warn(
this.scheduleCountLedgerDerivation('counts.json predates the ALL-visibility ledger') `[FileSystemStorage] counts.json predates the ALL-visibility count ledger — ` +
`derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` +
`every tier) and persisted; no further scan.`
)
needsPersist = true
} }
// The vectored-noun scalar (shipped after the ALL scalars above — a // The vectored-noun scalar (shipped after the ALL scalars above — a
@ -2948,12 +2615,14 @@ export class FileSystemStorage extends BaseStorage {
if (typeof counts.totalVectoredNounCount === 'number') { if (typeof counts.totalVectoredNounCount === 'number') {
this.totalVectoredNounCount = counts.totalVectoredNounCount this.totalVectoredNounCount = counts.totalVectoredNounCount
} else { } else {
// O(nouns) CONTENT reads — the most expensive derivation of the const vectored = await this.scanVectoredNounCount()
// three, and the one most likely to have been the silent minutes at this.totalVectoredNounCount = vectored
// the front of a large store's open. Background, suspect until it console.warn(
// lands, same as the ALL scalars. `[FileSystemStorage] counts.json predates the vectored-noun count ledger — ` +
this.allCountsSuspect = true `derived once by reading every noun's vectors.json (${vectored} vectored) and ` +
this.scheduleCountLedgerDerivation('counts.json predates the vectored-noun ledger') `persisted; no further scan.`
)
needsPersist = true
} }
if (needsPersist) { if (needsPersist) {
await this.persistCounts() await this.persistCounts()
@ -2982,22 +2651,6 @@ export class FileSystemStorage extends BaseStorage {
* Initialize counts by scanning disk (only done once) * Initialize counts by scanning disk (only done once)
*/ */
private async initializeCountsFromDisk(): Promise<void> { private async initializeCountsFromDisk(): Promise<void> {
const startedAt = Date.now()
// THIS ONE CANNOT LEAVE THE FOREGROUND, and the reason is worth stating:
// it derives `totalNounCount` / `totalVerbCount`, the scalars
// `getNounCount()` and `getVerbCount()` RETURN. Backgrounding it would
// make a populated store answer "0 entities" until the walk landed — a
// wrong answer, not a slow one, and the serving law grades a failure by
// whether an answer could be wrong. The ALL-visibility denominators, which
// no read is served from, DO run in the background (see
// scheduleCountLedgerDerivation). What this walk owes the operator instead
// is narration: it announces itself, and reports its wall.
prodLog.narrate(
`[FileSystemStorage] no usable counts.json — deriving the entity counters from ` +
`the canonical id tree now. This is O(ids) listings plus one vectors.json read ` +
`per noun, and it BLOCKS the open because getNounCount()/getVerbCount() are ` +
`served from it. It runs once; the result is persisted.`
)
try { try {
// Count the CANONICAL 8.0 layout (`entities/<kind>/<shard>/<id>/…`) — // Count the CANONICAL 8.0 layout (`entities/<kind>/<shard>/<id>/…`) —
// the tree saveNoun/getNouns actually read and write. The previous scan // the tree saveNoun/getNouns actually read and write. The previous scan
@ -3014,7 +2667,6 @@ export class FileSystemStorage extends BaseStorage {
this.totalNounCountAll = nouns.count this.totalNounCountAll = nouns.count
this.totalVerbCountAll = verbs.count this.totalVerbCountAll = verbs.count
this.allCountsSuspect = false this.allCountsSuspect = false
this.allCountsDerivedBy = 'identity-record'
// Vectored-noun scalar: presence needs each noun's vectors.json CONTENT // Vectored-noun scalar: presence needs each noun's vectors.json CONTENT
// (a deferred-embed noun's file exists but holds an empty vector until // (a deferred-embed noun's file exists but holds an empty vector until
// its embed lands), so this is a full O(nouns) content scan — see // its embed lands), so this is a full O(nouns) content scan — see
@ -3045,11 +2697,6 @@ export class FileSystemStorage extends BaseStorage {
} }
await this.persistCounts() await this.persistCounts()
prodLog.narrate(
`[FileSystemStorage] counter derivation from the canonical id tree finished in ` +
`${Date.now() - startedAt}ms: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs, ` +
`${this.totalVectoredNounCount} vectored nouns — persisted, stamped identity-record.`
)
} catch (error) { } catch (error) {
console.error('Error initializing counts from disk:', error) console.error('Error initializing counts from disk:', error)
} }
@ -3057,132 +2704,11 @@ export class FileSystemStorage extends BaseStorage {
/** /**
* Walk the canonical `entities/<kind>/<2-hex-shard>/<id>/` tree, counting * Walk the canonical `entities/<kind>/<2-hex-shard>/<id>/` tree, counting
* one entity per id directory that holds the metadata CONTENT leg * one entity per id directory (the layout `getNounVectorPath`/`getNouns`
* (`metadata.json` or its `.json.gz` variant see * use). Returns up to 100 sampled entity directories (absolute paths)
* {@link hasMetadataContentLeg}). A bare container a ghost (a stale * nouns feed the type-distribution estimate above. An absent tree (fresh
* `vectors.json` left with no metadata leg) or a scar (an empty directory), * store) counts zero.
* both artifacts of the pre-8.3.1 partial-delete defect counts ZERO: the
* identity record IS the population (ADR-008 G1), never the directory.
* This is the ONE-TIME legacy derivation walk (see callers); a prior
* version of this scan counted every id directory regardless of content,
* over-counting any store carrying orphaned containers see
* `allCountsDerivedBy` for how a counts.json derived under that old rule is
* marked suspect on load. Returns up to 100 sampled *counted* entity
* directories (absolute paths) nouns feed the type-distribution estimate
* above. An absent tree (fresh store) counts zero.
*/ */
/**
* @description Derive the ALL-visibility count ledger honestly one entity
* per IDENTITY RECORD, never per id directory IN THE BACKGROUND, once,
* and persist the result stamped `identity-record`.
*
* Why background: these scalars are DENOMINATORS. No read is served from
* them, so deriving them cannot be allowed to hold an open hostage a
* store with 24,898 ids spent minutes of a production restart inside walks
* exactly like these, in silence, before serving anything. Why at all: a
* ledger derived under the old container rule stayed wrong for the life of
* the store, and a downstream index heal subtracted against it and reported
* remaining work that did not exist (measured on a real store: 14,231
* derived against 14,056 identity records precisely the store's 25 noun
* scar directories; verbs 72,729 against 72,679, its 50 verb scars).
*
* Idempotent: a second call while one is in flight joins the first.
* @param reason - What made the ledger untrustworthy, quoted in narration.
* @returns Nothing; observe completion with {@link whenCountLedgerSettled}.
*/
private scheduleCountLedgerDerivation(reason: string): void {
if (this.countLedgerDerivation) return
this.countLedgerDerivation = (async () => {
const startedAt = Date.now()
prodLog.narrate(
`[FileSystemStorage] count-ledger derivation started in the background ` +
`(${reason}) — counting identity records, not id directories; the open does ` +
`not wait for it and no read is served from these scalars.`
)
try {
const beforeNouns = this.totalNounCountAll
const beforeVerbs = this.totalVerbCountAll
const beforeVectored = this.totalVectoredNounCount
// A walk that RACED A WRITE cannot prove its number: a row that landed
// mid-walk may or may not have been in the shard the walk had already
// passed. Rather than persist a figure that might be off by one and
// stamp it "exact", the walk is repeated once on a quiet store, and if
// the store is never quiet the ledger stays SUSPECT and says so. One
// retry, never a spin.
let attempt = 0
let derived: { nouns: number; verbs: number; vectored: number } | null = null
while (attempt < 2 && derived === null) {
attempt++
const activityBefore = this.ledgerActivityStamp()
const nouns = await this.scanCanonicalEntities('nouns')
const verbs = await this.scanCanonicalEntities('verbs')
const vectored = await this.scanVectoredNounCount()
if (this.ledgerActivityStamp() === activityBefore) {
derived = { nouns: nouns.count, verbs: verbs.count, vectored }
}
}
if (derived === null) {
this.allCountsSuspect = true
prodLog.narrate(
`[FileSystemStorage] count-ledger derivation could not finish on a quiet store ` +
`after ${attempt} attempts (${Date.now() - startedAt}ms) — writes landed during ` +
`every walk. The ALL-visibility scalars stay SUSPECT and must not be subtracted ` +
`against; brain.repairIndex() derives them under a recount barrier.`
)
return
}
this.totalNounCountAll = derived.nouns
this.totalVerbCountAll = derived.verbs
this.totalVectoredNounCount = derived.vectored
this.allCountsDerivedBy = 'identity-record'
this.allCountsSuspect = false
await this.persistCounts()
prodLog.narrate(
`[FileSystemStorage] count-ledger derivation finished in ${Date.now() - startedAt}ms: ` +
`${derived.nouns} nouns / ${derived.verbs} verbs / ${derived.vectored} vectored nouns` +
(beforeNouns !== derived.nouns ||
beforeVerbs !== derived.verbs ||
beforeVectored !== derived.vectored
? ` (corrected from ${beforeNouns} / ${beforeVerbs} / ${beforeVectored} — the ` +
`difference is ghost and scar containers the old rule counted as entities)`
: ' (unchanged)') +
` — persisted, stamped identity-record, no longer suspect.`
)
} catch (error) {
// The ledger stays suspect and the next open retries. Loud: a
// denominator nobody can derive is a fact an operator must have.
this.allCountsSuspect = true
prodLog.error(
`[FileSystemStorage] count-ledger derivation FAILED after ` +
`${Date.now() - startedAt}ms — the ALL-visibility scalars remain SUSPECT ` +
`and must not be subtracted against; the next open retries:`,
error
)
}
})()
}
/**
* @description A cheap witness that the ledger changed while a walk was
* running. Every landed write moves one of these live counters, so an
* unchanged stamp across a walk means no write landed during it.
* @returns A value that differs whenever the live ALL scalars have moved.
*/
private ledgerActivityStamp(): string {
return `${this.totalNounCountAll}:${this.totalVerbCountAll}:${this.totalVectoredNounCount}`
}
/**
* @description Resolve once any background count-ledger derivation has
* settled (succeeded or failed). Resolves immediately when none was needed.
* Exists so tests and operators can observe the ledger's honest value rather
* than race it; nothing in the read path waits on this.
* @returns A promise that settles with the derivation.
*/
public async whenCountLedgerSettled(): Promise<void> {
await this.countLedgerDerivation
}
private async scanCanonicalEntities( private async scanCanonicalEntities(
kind: 'nouns' | 'verbs' kind: 'nouns' | 'verbs'
): Promise<{ count: number; sampleDirs: string[] }> { ): Promise<{ count: number; sampleDirs: string[] }> {
@ -3198,21 +2724,9 @@ export class FileSystemStorage extends BaseStorage {
const ids = await fs.promises.readdir(shardPath, { withFileTypes: true }) const ids = await fs.promises.readdir(shardPath, { withFileTypes: true })
for (const entry of ids) { for (const entry of ids) {
if (!entry.isDirectory()) continue if (!entry.isDirectory()) continue
const idAbs = path.join(shardPath, entry.name)
let legs: string[]
try {
legs = await fs.promises.readdir(idAbs)
} catch (error: any) {
if (error?.code === 'ENOENT') continue
throw error
}
// No metadata content leg → a ghost or scar container → not an
// entity. Same test pruneOrphanedEntities() uses, so the two agree
// by construction.
if (!this.hasMetadataContentLeg(legs)) continue
count++ count++
if (sampleDirs.length < SAMPLE_MAX) { if (sampleDirs.length < SAMPLE_MAX) {
sampleDirs.push(idAbs) sampleDirs.push(path.join(shardPath, entry.name))
} }
} }
} }
@ -3267,20 +2781,14 @@ export class FileSystemStorage extends BaseStorage {
} }
/** /**
* Count canonical nouns holding a REAL (non-empty, non-zero-norm) vector * Count canonical nouns holding a REAL (non-empty) vector the vectored-
* the vectored-noun ledger scalar. UNLIKE {@link scanCanonicalEntities}, * noun ledger scalar. UNLIKE {@link scanCanonicalEntities}, presence
* presence cannot be decided from the id-directory listing alone: a * cannot be decided from the id-directory listing alone: a deferred-embed
* deferred-embed noun's `vectors.json` EXISTS (written at `add()` time * noun's `vectors.json` EXISTS (written at `add()` time with `vector: []`)
* with `vector: []`) until its embed LANDS, so this walk reads every * until its embed LANDS, so this walk reads every noun's `vectors.json`
* noun's `vectors.json` CONTENT O(nouns) reads, not O(ids) listing. * CONTENT O(nouns) reads, not O(ids) listing. Used ONLY for a one-time
* ZERO-NORM LAW: a real all-zero vector is not a vector it never counts * legacy-counts.json derivation or a lost/corrupted counts.json recovery;
* here either (Brainy's write paths normalize an explicit zero-norm * the result is persisted so this scan never repeats.
* vector to `[]` at write time, but a store created before that fix may
* still carry legacy all-zero rows on disk; this derivation must agree
* with the live ledger's definition of "vectored" regardless of when the
* row was written). Used ONLY for a one-time legacy-counts.json derivation
* or a lost/corrupted counts.json recovery; the result is persisted so
* this scan never repeats.
*/ */
private async scanVectoredNounCount(): Promise<number> { private async scanVectoredNounCount(): Promise<number> {
const base = path.join(this.rootDir, 'entities', 'nouns') const base = path.join(this.rootDir, 'entities', 'nouns')
@ -3294,12 +2802,7 @@ export class FileSystemStorage extends BaseStorage {
for (const entry of ids) { for (const entry of ids) {
if (!entry.isDirectory()) continue if (!entry.isDirectory()) continue
const record = await this.readEntityVectorRaw(path.join(shardPath, entry.name)) const record = await this.readEntityVectorRaw(path.join(shardPath, entry.name))
if ( if (record && Array.isArray(record.vector) && record.vector.length > 0) {
record &&
Array.isArray(record.vector) &&
record.vector.length > 0 &&
!isZeroNormVector(record.vector)
) {
vectored++ vectored++
} }
} }
@ -3331,25 +2834,13 @@ export class FileSystemStorage extends BaseStorage {
// scanVectoredNounCount()'s JSDoc). // scanVectoredNounCount()'s JSDoc).
totalVectoredNounCount: this.totalVectoredNounCount, totalVectoredNounCount: this.totalVectoredNounCount,
allCountsSuspect: this.allCountsSuspect, allCountsSuspect: this.allCountsSuspect,
// Derivation-rule stamp for the ALL scalars above — 'identity-record'
// when they were counted one-per-metadata-content-leg (the honest
// rule); omitted (JSON.stringify drops `undefined`) when the current
// in-memory scalars came from a legacy container-rule counts.json
// that hasn't been through a sanctioned recount yet, so a future load
// keeps naming them suspect rather than trusting an unproven value.
allCountsDerivedBy: this.allCountsDerivedBy,
lastUpdated: new Date().toISOString() lastUpdated: new Date().toISOString()
} }
// ATOMIC (temp + rename), never a plain writeFile. A direct write await fs.promises.writeFile(
// truncates the file first, so every persist opened a window — measured this.countsFilePath,
// at roughly 750ms after a flush or close on a real store — in which a JSON.stringify(counts, null, 2)
// concurrent reader saw counts.json EMPTY. An empty file is unparseable, )
// and an unparseable ledger sends the next open down the full-rescan
// path: the cheapest file in the store was costing the most expensive
// recovery. The rename is atomic, so a reader sees the old ledger or the
// new one, never neither.
await this.writeFileAtomic(this.countsFilePath, JSON.stringify(counts, null, 2))
} catch (error) { } catch (error) {
console.error('Error persisting counts:', error) console.error('Error persisting counts:', error)
} }

View file

@ -125,36 +125,6 @@ export interface WriterLockInfo {
rootDir?: string // Convenience for log lines / error messages rootDir?: string // Convenience for log lines / error messages
} }
/**
* THE CLEAN-CLOSE RECORD. Written by `releaseWriterLock()` at the instant it
* gives up the writer lock, naming the lock identity it released. The next
* `acquireWriterLock()` reads it and can then say from a RECORD, not from a
* guess whether the previous writer left on purpose.
*
* Why a record and not PID liveness: "the recorded PID is no longer alive" is
* true of every orderly restart AND of every crash, so the two were reported
* identically ("appears dead") and neither could be trusted. Worse, the same
* inference fails the other way when the operating system RECYCLES the pid
* a live unrelated process makes a long-dead writer's lock look held, and the
* store refuses to open naming a pid that was never Brainy. A record settles
* both: matched the previous writer closed cleanly, nothing to recover;
* absent say so, and name what recovery the open will now run.
*
* Lifecycle: written at release, consumed (deleted) by the next successful
* lock claim a record must never outlive the lock generation it describes,
* or it would vouch for a later crash.
*/
export interface WriterCloseRecord {
pid: number
hostname: string
/** `startedAt` of the lock this close released — the identity match key. */
startedAt: string
/** ISO timestamp at which the lock was released. */
closedAt: string
/** Brainy version that performed the close. */
version: string
}
/** /**
* FNV-1a hash returning a 2-char hex bucket (00-ff). * FNV-1a hash returning a 2-char hex bucket (00-ff).
* Distributes system keys across 256 sub-prefixes to avoid * Distributes system keys across 256 sub-prefixes to avoid
@ -233,40 +203,6 @@ function idFromVectorPath(path: string): string {
return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix
} }
/**
* @description Extract the entity id embedded in a metadata path
* (`entities/{nouns|verbs}/{shard}/{id}/metadata.json`) the IDENTITY-RECORD
* mirror of {@link idFromVectorPath}. The cursored noun/verb walks key their
* population on this file (ADR-008 G1: the metadata record IS the population;
* the vector leg is optional), so walk ordering and cursor resume derive the
* id from THIS path, never the vector path a row with metadata and no
* vector file must still be listed, ordered, and resumable.
* @param path - A metadata path (full or prefix-relative; must end with `/metadata.json`).
* @returns The entity id (the path segment immediately before `/metadata.json`).
*/
function idFromMetadataPath(path: string): string {
const withoutSuffix = path.replace(/\/metadata\.json$/, '')
const lastSlash = withoutSuffix.lastIndexOf('/')
return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix
}
/**
* @description The sanctioned UNVECTORED shape for a noun hydrated during
* enumeration when its identity record (metadata.json) exists but its vector
* leg (vectors.json) does not a fold-born metadata-only after-image, or any
* row genuinely without a vector yet. Mirrors the shape
* `unvectorNounForRootMigration` (src/brainy.ts) writes for the sanctioned
* unvector path (`{ vector: [], connections: new Map(), level: 0 }`), so a
* walk-yielded unvectored row is byte-shape-identical to one produced by that
* migration. Callers already handle `vector: []` as first-class
* (validateAddParams exempts it; index gates key on `length > 0`).
* @param id - The noun id.
* @returns A structurally-valid, vector-empty `HNSWNoun`.
*/
function unvectoredNoun(id: string): HNSWNoun {
return { id, vector: [], connections: new Map<number, Set<string>>(), level: 0 }
}
/** /**
* Get ID-first path for verb metadata * Get ID-first path for verb metadata
* No type parameter needed - direct O(1) lookup by ID * No type parameter needed - direct O(1) lookup by ID
@ -1437,29 +1373,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
return this.listObjectsUnderPath(prefix) return this.listObjectsUnderPath(prefix)
} }
/**
* @description The IMMEDIATE child directory names under a prefix one
* level, no recursion. See the seam's JSDoc (`db/types.ts`) for why a
* separate door exists. This default derives them from the recursive
* listing, so it is never WRONG, only never faster; the filesystem adapter
* overrides it with a single directory read.
* @param prefix - Storage-root-relative directory prefix.
* @returns The child directory names (not paths), in listing order.
*/
public async listRawPrefixes(prefix: string): Promise<string[]> {
await this.ensureInitialized()
const paths = await this.listObjectsUnderPath(prefix)
const normalizedPrefix = prefix.endsWith('/') ? prefix : `${prefix}/`
const names = new Set<string>()
for (const p of paths) {
const rest = p.startsWith(normalizedPrefix) ? p.slice(normalizedPrefix.length) : null
if (rest === null) continue
const slash = rest.search(/[/\\]/)
if (slash > 0) names.add(rest.slice(0, slash))
}
return [...names]
}
/** /**
* Remove every object under a storage-root-relative prefix. The filesystem * Remove every object under a storage-root-relative prefix. The filesystem
* adapter overrides this with a recursive directory removal; this default * adapter overrides this with a recursive directory removal; this default
@ -1540,18 +1453,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
* rollups are derived state with their own rebuild paths * rollups are derived state with their own rebuild paths
* (`rebuildTypeCounts()` / `rebuildSubtypeCounts()`). * (`rebuildTypeCounts()` / `rebuildSubtypeCounts()`).
* *
* EXACT-RESTORE PRIMITIVE `vector: null` DELETES the vector leg, on
* purpose: `GenerationStore.rollBackUncommittedGeneration()` depends on
* this to legitimately un-write a vector a failed transaction added. This
* is deliberately NOT "preserve if absent" a caller replaying an
* AFTER-IMAGE (the recovery fold, `GenerationStore`'s `replayFact`) must
* apply preserve-if-absent itself BEFORE calling this, by reading the
* current vector and carrying it forward when the after-image's own
* vector leg is null/undefined but its metadata is not (see `replayFact`
* for the implementation and full rationale). A caller that genuinely
* wants to unvector a row uses the sanctioned, ledger-correct path
* (`Brainy.unvectorNounForRootMigration`) never this primitive.
*
* @param id - The entity id. * @param id - The entity id.
* @param record - Raw stored objects as returned by {@link BaseStorage.readNounRaw}. * @param record - Raw stored objects as returned by {@link BaseStorage.readNounRaw}.
*/ */
@ -1588,9 +1489,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
/** /**
* Restore a relationship's raw stored objects byte-for-byte (verb-side * Restore a relationship's raw stored objects byte-for-byte (verb-side
* mirror of {@link BaseStorage.writeNounRaw}; same bookkeeping caveats, * mirror of {@link BaseStorage.writeNounRaw}; same bookkeeping caveats).
* same EXACT-RESTORE contract `vector: null` deletes, on purpose; the
* fold's preserve-if-absent logic lives at its call site, not here).
* *
* @param id - The relationship id. * @param id - The relationship id.
* @param record - Raw stored objects as returned by {@link BaseStorage.readVerbRaw}. * @param record - Raw stored objects as returned by {@link BaseStorage.readVerbRaw}.
@ -2284,18 +2183,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// Stable within-shard order (by noun id) so offset windows and cursor resume // Stable within-shard order (by noun id) so offset windows and cursor resume
// are deterministic; ids come from the path so skipped nouns are never read. // are deterministic; ids come from the path so skipped nouns are never read.
//
// IDENTITY-KEYED WALK (population law, ADR-008 G1): the metadata record
// (not the vector) IS the population — a noun with metadata and no vector
// file (a fold-born after-image, see writeNounRaw's preserve-if-absent
// contract) must still enumerate. Keying on metadata.json here means the
// ledger recount (rebuildTypeCounts' `allNouns`, also metadata.json-keyed)
// and this walk agree on population by construction. Ordering is
// unaffected for a healthy store: every vectored noun has both legs, so
// the id set and sort order are identical to the old vectors.json keying.
const entries = nounFiles const entries = nounFiles
.filter((p) => p.includes('/metadata.json')) .filter((p) => p.includes('/vectors.json'))
.map((p) => ({ path: p, id: idFromMetadataPath(p) })) .map((p) => ({ path: p, id: idFromVectorPath(p) }))
.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)) .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
// Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor
@ -2320,24 +2210,13 @@ export abstract class BaseStorage extends BaseStorageAdapter {
) { ) {
const batch = toHydrate.slice(i, i + BaseStorage.HYDRATE_CONCURRENCY) const batch = toHydrate.slice(i, i + BaseStorage.HYDRATE_CONCURRENCY)
const hydrated = await Promise.all( const hydrated = await Promise.all(
batch.map(async ({ path: metadataPath, id }) => { batch.map(async ({ path: nounPath }) => {
try { try {
const metadata = await this.readCanonicalObject(metadataPath) const noun = await this.readCanonicalObject(nounPath)
if (!noun) return null
const deserialized = this.deserializeNoun(noun)
const metadata = await this.getNounMetadata(deserialized.id)
if (!metadata) return null if (!metadata) return null
// The vector leg is OPTIONAL (population law): a metadata-only
// row hydrates with the sanctioned unvectored shape rather than
// being dropped from the walk. A fault reading the vector leg
// is treated the same as absence — best-effort, matching the
// canonical recount's tolerance for an unreadable vectors.json
// (rebuildTypeCounts) — a vector-leg problem never hides an
// otherwise-good identity record.
let deserialized: HNSWNoun
try {
const vectorRecord = await this.readCanonicalObject(getNounVectorPath(id))
deserialized = vectorRecord ? this.deserializeNoun(vectorRecord) : unvectoredNoun(id)
} catch {
deserialized = unvectoredNoun(id)
}
return { deserialized, metadata } return { deserialized, metadata }
} catch (error) { } catch (error) {
// A TORN record must surface typed — a paginated read that // A TORN record must surface typed — a paginated read that
@ -2347,9 +2226,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// walk's job is to HEAL PAST it — skip the victim, serve the rest. // walk's job is to HEAL PAST it — skip the victim, serve the rest.
// Identity point-reads (get-by-id) still throw typed upstream. // Identity point-reads (get-by-id) still throw typed upstream.
if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ }
// Skip nouns whose IDENTITY record fails to load (the metadata // Skip nouns that fail to load
// read above) — that is the one leg this walk cannot proceed
// without.
return null return null
} }
}) })
@ -2470,14 +2347,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
const shardDir = `entities/nouns/${shardHex}` const shardDir = `entities/nouns/${shardHex}`
try { try {
const nounFiles = await this.listCanonicalObjects(shardDir) const nounFiles = await this.listCanonicalObjects(shardDir)
// IDENTITY-KEYED WALK (population law, ADR-008 G1) — see the matching
// comment in getNounsWithPagination: metadata.json is the population;
// the vector leg is optional, so a metadata-only row must still be
// listed (and here, for the unfiltered case, needs ZERO reads either
// way — the id comes straight from the path).
const entries = nounFiles const entries = nounFiles
.filter((p) => p.includes('/metadata.json')) .filter((p) => p.includes('/vectors.json'))
.map((p) => idFromMetadataPath(p)) .map((p) => idFromVectorPath(p))
.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)) .sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))
const toWalk = const toWalk =
cursor && shard === cursor.shard ? entries.filter((id) => id > cursor.id) : entries cursor && shard === cursor.shard ? entries.filter((id) => id > cursor.id) : entries
@ -2688,79 +2560,23 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// Stable within-shard order (by verb id) so offset windows and cursor resume // Stable within-shard order (by verb id) so offset windows and cursor resume
// are deterministic and consistent across calls. Ids come from the path, so // are deterministic and consistent across calls. Ids come from the path, so
// verbs skipped by the cursor are never read. // verbs skipped by the cursor are never read.
//
// IDENTITY-KEYED WALK (population law, ADR-008 G1) — the noun mirror of
// this comment in getNounsWithPagination applies here too: metadata.json
// is the population; keying on it here means this walk and the ledger
// recount (rebuildTypeCounts' `allVerbs`, already metadata.json-keyed)
// agree on population by construction. Unchanged for a healthy store —
// `relate()` always writes both legs of a verb in the same commit, so
// the id set and order match the old vectors.json keying exactly; this
// only additionally surfaces a fold-born metadata-only row (see
// writeVerbRaw's preserve-if-absent contract).
const entries = verbFiles const entries = verbFiles
.filter((p) => p.includes('/metadata.json')) .filter((p) => p.includes('/vectors.json'))
.map((p) => ({ path: p, id: idFromMetadataPath(p) })) .map((p) => ({ path: p, id: idFromVectorPath(p) }))
.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)) .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
for (const { path: metadataPath, id: verbId } of entries) { for (const { path: verbPath, id: verbId } of entries) {
if (collected.length >= peekCount) break if (collected.length >= peekCount) break
// Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor id // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor id
// (later shards are processed in full). No read for skipped verbs. // (later shards are processed in full). No read for skipped verbs.
if (cursor && shard === cursor.shard && verbId <= cursor.id) continue if (cursor && shard === cursor.shard && verbId <= cursor.id) continue
try { try {
// Identity leg first — required. A verb this walk cannot read const rawVerb = await this.readCanonicalObject(verbPath)
// metadata for cannot be hydrated at all (same as before). if (!rawVerb) continue
const metadata = await this.readCanonicalObject(metadataPath)
if (!metadata) continue
// The vector leg is the verb's STRUCTURAL core (verb/sourceId/ // Deserialize connections Map from JSON storage format
// targetId live there — see coreTypes.ts HNSWVerb), unlike a const verb = this.deserializeVerb(rawVerb)
// noun's vector, which is pure embedding data. `relate()` always
// writes both legs atomically and verbs have no deferred-embed
// path, so a healthy store's verbs always have both. A vector-leg
// absence here can only be a fold-born after-image (see
// writeVerbRaw's preserve-if-absent contract) — and unlike a
// noun, this walk cannot safely FABRICATE sourceId/targetId to
// synthesize a structurally-valid verb (an empty-string endpoint
// would silently create a phantom edge — worse than omission).
// If the metadata record happens to carry its own sourceId/
// targetId (never true for current production writes, but not
// disallowed — e.g. a future schema or a repair tool could
// populate them), reconstruct from those; otherwise this row is
// loudly skipped — counted by the ledger, but not returned as an
// item, until a repair can supply the missing endpoints.
const rawVerb = await this.readCanonicalObject(getVerbVectorPath(verbId))
let verb: HNSWVerb
if (rawVerb) {
verb = this.deserializeVerb(rawVerb)
} else {
const metaSourceId = (metadata as Record<string, unknown>).sourceId
const metaTargetId = (metadata as Record<string, unknown>).targetId
const metaVerbType = (metadata as Record<string, unknown>).verb
if (
typeof metaSourceId === 'string' && metaSourceId.length > 0 &&
typeof metaTargetId === 'string' && metaTargetId.length > 0 &&
typeof metaVerbType === 'string' && metaVerbType.length > 0
) {
verb = {
id: verbId,
vector: [],
connections: new Map<number, Set<string>>(),
verb: metaVerbType as VerbType,
sourceId: metaSourceId,
targetId: metaTargetId
}
} else {
prodLog.error(
`[BaseStorage] getVerbsWithPagination: verb ${verbId} has a metadata ` +
`record but no vector leg and no recoverable sourceId/targetId — ` +
`skipping (counted by the ledger, not yielded; needs repair).`
)
continue
}
}
// Apply type filter // Apply type filter
if (filterVerbTypes && !filterVerbTypes.has(verb.verb)) { if (filterVerbTypes && !filterVerbTypes.has(verb.verb)) {
@ -2777,6 +2593,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
continue continue
} }
// Load metadata
const metadata = await this.getVerbMetadata(verb.id)
// Apply subtype filter (requires metadata — checked AFTER load) // Apply subtype filter (requires metadata — checked AFTER load)
if (filterSubtypes) { if (filterSubtypes) {
const subtype = metadata?.subtype as string | undefined const subtype = metadata?.subtype as string | undefined
@ -4873,10 +4692,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
this.totalVerbCountAll = allVerbs this.totalVerbCountAll = allVerbs
this.totalVectoredNounCount = allVectoredNouns this.totalVectoredNounCount = allVectoredNouns
this.allCountsSuspect = false this.allCountsSuspect = false
// This walk counts one entity per metadata.json record (never per bare
// container) — the identity-record rule. Stamp it so a future load
// trusts these scalars instead of naming them suspect at open.
this.allCountsDerivedBy = 'identity-record'
this.countCache.clear() this.countCache.clear()
await this.persistCounts() await this.persistCounts()

View file

@ -13,8 +13,6 @@ import type { VectorIndexProvider, GraphIndexProvider } from '../../plugin.js'
import type { MetadataIndexManager } from '../../utils/metadataIndex.js' import type { MetadataIndexManager } from '../../utils/metadataIndex.js'
import type { GraphVerb } from '../../coreTypes.js' import type { GraphVerb } from '../../coreTypes.js'
import type { Operation, RollbackAction } from '../types.js' import type { Operation, RollbackAction } from '../types.js'
import { isZeroNormVector } from '../../utils/distance.js'
import { prodLog } from '../../utils/logger.js'
/** /**
* Backend identity stamped into an operation's emitted `name` string (e.g. * Backend identity stamped into an operation's emitted `name` string (e.g.
@ -90,30 +88,6 @@ export class AddToVectorIndexOperation implements Operation {
} }
async execute(): Promise<RollbackAction> { async execute(): Promise<RollbackAction> {
// THE ZERO-NORM LAW (the live provider-write seam's belt): a zero-norm
// vector is not a vector — it never crosses an engine boundary. This
// engine's own cosine distance treats an all-zero vector safely (a
// zero-norm operand always scores MAXIMUM distance, see
// {@link isZeroNormVector}'s JSDoc), but a downstream engine serving
// squared-euclidean distance cannot tell it apart from a legitimate
// origin point — a false attractor that silently darkens real results.
// The canonical write already landed (SaveNoun/SaveNounMetadata
// operations are staged ahead of this one in every caller) — only the
// INDEX INSERT is refused here, loudly, never a throw. A length-0
// vector is the unrelated "unvectored" shape and is skipped silently
// (the same contract callers already rely on for deferred embeds).
if (this.vector.length === 0) {
return async () => {}
}
if (isZeroNormVector(this.vector)) {
prodLog.warn(
`[vector-index] refusing to index a zero-norm vector for entity ${this.id}` +
`a zero-norm vector is not a vector and never crosses an engine boundary ` +
`(the canonical write is unaffected; only the vector-index insert is skipped)`
)
return async () => {}
}
// Check if item already exists (for rollback decision) // Check if item already exists (for rollback decision)
const existed = await this.itemExists(this.id) const existed = await this.itemExists(this.id)
@ -289,52 +263,14 @@ export class ReplaceInVectorIndexOperation implements Operation {
// One commit generation for the whole replace (both branches + rollback). // One commit generation for the whole replace (both branches + rollback).
const generation = this.generationFn?.() const generation = this.generationFn?.()
// THE ZERO-NORM LAW (see AddToVectorIndexOperation's matching JSDoc): a
// real all-zero replacement vector must never land in the index — refuse
// loudly, canonical write unaffected. The row must not be left stale
// either: if it was genuinely indexed under `oldVector`, remove it
// rather than pretend the old vector still describes the row. A
// length-0 `newVector` (the unrelated "unvectored" shape) is handled the
// same way, silently — no caller today reaches this with an empty
// replacement (update() rejects a dimension-mismatched empty vector),
// but the seam stays consistent in case one ever legitimately does.
if (isZeroNormVector(this.newVector) || this.newVector.length === 0) {
const wasIndexed = this.oldVector.length > 0 && !isZeroNormVector(this.oldVector)
if (isZeroNormVector(this.newVector)) {
prodLog.warn(
`[vector-index] refusing to replace with a zero-norm vector for entity ${this.id}` +
`a zero-norm vector is not a vector and never crosses an engine boundary ` +
`(the canonical write is unaffected; the row is removed from the vector index instead)`
)
}
if (wasIndexed) {
await this.index.removeItem(this.id, generation)
}
return async () => {
// Restore the declared before-state.
if (wasIndexed) {
await this.index.addItem({ id: this.id, vector: this.oldVector }, generation)
}
}
}
if (typeof index.updateItem === 'function') { if (typeof index.updateItem === 'function') {
// Atomic path: one in-place call, the row never leaves the index. // Atomic path: one in-place call, the row never leaves the index.
await index.updateItem({ id: this.id, vector: this.newVector }, generation) await index.updateItem({ id: this.id, vector: this.newVector }, generation)
return async () => { return async () => {
// Restore the declared before-state in place (see class JSDoc for // Restore the declared before-state in place (see class JSDoc for
// the item-did-not-exist posture). A length-0 oldVector means the row // the item-did-not-exist posture).
// was never actually indexed before this op ran (a length-0 vector is await index.updateItem!({ id: this.id, vector: this.oldVector }, generation)
// never a legal index member — see EmptyVectorIndexError) — there is
// no in-place "restore to empty" for the provider to perform, so
// rollback removes the row instead, leaving the same "not indexed"
// state the row was in before execute().
if (this.oldVector.length > 0) {
await index.updateItem!({ id: this.id, vector: this.oldVector }, generation)
} else {
await this.index.removeItem(this.id, generation)
}
} }
} }
@ -345,14 +281,9 @@ export class ReplaceInVectorIndexOperation implements Operation {
return async () => { return async () => {
// updateItem-style restore via the same adjacent pair, back to the // updateItem-style restore via the same adjacent pair, back to the
// declared before-state. Same length-0 carve-out as the updateItem // declared before-state.
// path above: an empty oldVector was never a legal index member, so
// rollback just leaves the row removed rather than attempting an
// illegal empty re-add.
await this.index.removeItem(this.id, generation) await this.index.removeItem(this.id, generation)
if (this.oldVector.length > 0) { await this.index.addItem({ id: this.id, vector: this.oldVector }, generation)
await this.index.addItem({ id: this.id, vector: this.oldVector }, generation)
}
} }
} }
} }

View file

@ -215,7 +215,7 @@ export interface ScoreExplanation {
* *
* @example * @example
* ```ts * ```ts
* declare module '@soulcraftlabs/brainy' { * declare module '@soulcraft/brainy' {
* interface SubtypeRegistry { * interface SubtypeRegistry {
* // For NounType.Person, subtype 'employee': * // For NounType.Person, subtype 'employee':
* 'person:employee': { employeeId: string; department: string } * 'person:employee': { employeeId: string; department: string }
@ -1217,13 +1217,6 @@ export interface RepairFamilyReport {
skipped?: string skipped?: string
/** Why the outcome is what it is when neither `detail` nor `skipped` says it. */ /** Why the outcome is what it is when neither `detail` nor `skipped` says it. */
reason?: string reason?: string
/**
* The phase's own wall, in milliseconds. A repair on a production store ran
* for over thirty minutes without a single line of output; an operator had
* to read `top` to know it was alive. A receipt that cannot say WHERE the
* time went is not a receipt every row carries its own.
*/
durationMs?: number
} }
/** The full receipt returned by repairIndex(). */ /** The full receipt returned by repairIndex(). */

View file

@ -65,7 +65,7 @@
* | `_rev` | system-managed revision counter pass `ifRev` to `update()` for CAS | * | `_rev` | system-managed revision counter pass `ifRev` to `update()` for CAS |
* *
* @example * @example
* import { RESERVED_ENTITY_FIELDS } from '@soulcraftlabs/brainy' * import { RESERVED_ENTITY_FIELDS } from '@soulcraft/brainy'
* const isReserved = (key: string) => * const isReserved = (key: string) =>
* (RESERVED_ENTITY_FIELDS as readonly string[]).includes(key) * (RESERVED_ENTITY_FIELDS as readonly string[]).includes(key)
*/ */

View file

@ -6,7 +6,7 @@
* *
* @example * @example
* ```typescript * ```typescript
* import { BrainyTypes } from '@soulcraftlabs/brainy' * import { BrainyTypes } from '@soulcraft/brainy'
* *
* // Get all available types * // Get all available types
* const nounTypes = BrainyTypes.nouns // ['Person', 'Organization', ...] * const nounTypes = BrainyTypes.nouns // ['Person', 'Organization', ...]

View file

@ -65,29 +65,6 @@ export const cosineDistance: DistanceFunction = (a: Vector, b: Vector): number =
return 1 - similarity return 1 - similarity
} }
/**
* True when `vector` is a REAL (non-empty) all-zero vector the "false
* attractor" shape this engine's own cosine distance treats safely (a
* zero-norm operand always scores the MAXIMUM distance, see
* {@link cosineDistance}) but a downstream engine serving squared-euclidean
* distance cannot distinguish from a legitimate origin point. THE LAW: a
* zero-norm vector is not a vector it never crosses an engine boundary
* (never handed to a vector-index provider as a searchable item).
*
* A length-0 vector is the UNRELATED "unvectored, not yet embedded" shape
* (the deferred-embed stub, a permanently-vectorless system row) and is
* deliberately NOT zero-norm here callers checking for "nothing to index"
* should test `vector.length === 0` separately; this only flags the
* dangerous non-empty all-zero case.
*/
export function isZeroNormVector(vector: readonly number[]): boolean {
if (vector.length === 0) return false
for (let i = 0; i < vector.length; i++) {
if (vector[i] !== 0) return false
}
return true
}
/** /**
* Calculates the Manhattan (L1) distance between two vectors. * Calculates the Manhattan (L1) distance between two vectors.
* Lower values indicate higher similarity. * Lower values indicate higher similarity.

View file

@ -153,83 +153,3 @@ export function assessProviderHealth(provider: unknown): ProviderHealthAssessmen
reasons: readiness === 'not-ready' ? ['isReady() returned false'] : [] reasons: readiness === 'not-ready' ? ['isReady() returned false'] : []
} }
} }
/**
* @description A provider's self-report that it is REBUILDING ITS OWN index
* right now. Returned by the optional `rebuildInProgress()` hook.
*
* The distinction this exists to make: a provider reporting `serving: false`
* because it is BROKEN and a provider reporting `serving: false` because it is
* BUSY BUILDING ITSELF look identical through `healthReport()` alone, and
* brainy treated both the same way it called `rebuild()` and waited for it,
* on the foreground of `init()`. A production store whose metadata provider
* had to rebuild paid 641 SECONDS of that wait before `init()` returned, with
* every other family idle behind it.
*
* A provider that reports progress here owns its own rebuild: brainy neither
* starts one nor waits for it, `init()` returns, the other families serve, and
* THAT family's doors refuse by name carrying this progress until the
* provider reports itself serving.
*
* Every field but `phase` is optional and every field is a MEASUREMENT: a
* provider reports only what it actually tracks, never an estimate dressed as
* a fact.
*/
export interface ProviderRebuildProgress {
/** The provider's own name for what it is doing. Quoted verbatim in refusals. */
phase: string
/** Units completed so far, if the provider counts them. */
done?: number
/** Units expected in total, if the provider knows it. */
total?: number
/** Epoch millis when this rebuild started, if the provider tracks it. */
startedAt?: number
}
/** A provider that can report a rebuild it is running itself. */
interface MaybeRebuildingProvider {
rebuildInProgress?: () => ProviderRebuildProgress | null
}
/**
* @description Ask a provider whether it is rebuilding itself right now.
* Synchronous, O(1), feature-detected: a provider without the hook reports
* nothing and is treated exactly as before.
* @param provider - Any index provider, or `null`/`undefined`.
* @returns The provider's progress, or `null` when it is not rebuilding (or
* does not implement the hook).
*/
export function assessProviderRebuild(provider: unknown): ProviderRebuildProgress | null {
const p = provider as MaybeRebuildingProvider | null | undefined
if (p == null || typeof p.rebuildInProgress !== 'function') return null
try {
const progress = p.rebuildInProgress()
if (!progress || typeof progress.phase !== 'string' || progress.phase.length === 0) {
return null
}
return progress
} catch {
// A throwing hook says nothing trustworthy about a rebuild; fall through to
// the ordinary health verdict rather than inventing one.
return null
}
}
/**
* @description Render a rebuild progress report as one operator-facing clause,
* for a refusal message. Includes only what the provider actually measured.
* @param progress - The provider's report.
* @returns A clause such as `rebuilding ("metadata shadow build", 4,096/14,056, 12s elapsed)`.
*/
export function describeRebuildProgress(progress: ProviderRebuildProgress): string {
const parts: string[] = [`"${progress.phase}"`]
if (typeof progress.done === 'number' && typeof progress.total === 'number') {
parts.push(`${progress.done.toLocaleString()}/${progress.total.toLocaleString()}`)
} else if (typeof progress.done === 'number') {
parts.push(`${progress.done.toLocaleString()} done`)
}
if (typeof progress.startedAt === 'number') {
parts.push(`${Math.round((Date.now() - progress.startedAt) / 1000)}s elapsed`)
}
return `rebuilding (${parts.join(', ')})`
}

View file

@ -266,26 +266,6 @@ export const prodLog = {
console.error(message, ...args) console.error(message, ...args)
}, },
/**
* THE NARRATION CHANNEL always visible, exactly like `error`.
*
* `warn`/`info`/`log` below are clamped to ERROR in any environment that
* looks like production (see isProductionEnvironment), which is the right
* default for chatter and the wrong one for the two things an operator is
* entitled to hear from a database no matter what: WHY IT IS SLOW and WHAT
* IT IS DOING ABOUT IT. A production service opening a 16 GB store spent
* three minutes emitting nothing at all the phase timings that would have
* named the slow phase were written to `warn` and thrown away by the log
* level. Progress and cost narration goes here; it is never a per-record
* line, always a phase, a wall, or a bounded-cadence heartbeat.
*
* `silent: true` still silences it that is the consumer's explicit
* request, not a cost default.
*/
narrate: (message?: any, ...args: any[]) => {
console.warn(message, ...args)
},
// These are suppressed in production unless BRAINY_LOG_LEVEL is set // These are suppressed in production unless BRAINY_LOG_LEVEL is set
warn: (message?: any, ...args: any[]) => smartConsole.warn(message, ...args), warn: (message?: any, ...args: any[]) => smartConsole.warn(message, ...args),
info: (message?: any, ...args: any[]) => smartConsole.info(message, ...args), info: (message?: any, ...args: any[]) => smartConsole.info(message, ...args),

Some files were not shown because too many files have changed in this diff Show more