Compare commits
41 commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4f1e27c9a0 | |||
| 297a3d7657 | |||
| 7ab670b525 | |||
| f097cbf6f2 | |||
| e64e2bc175 | |||
| 655aa13ea7 | |||
| 39c71ecdac | |||
| 0759c03a82 | |||
| 9a888c37e9 | |||
|
|
298cb6daca | ||
| b8475cc86a | |||
| ff39941b0a | |||
| d49148e140 | |||
| 42e2da259b | |||
| 5ebd3b4061 | |||
| a8c724a202 | |||
| 61a469270e | |||
| 02c6163637 | |||
| 2cf3801007 | |||
| 5c22f9500c | |||
| 16d2e1a97e | |||
| fb1da1c56d | |||
| 417ddb5143 | |||
| 9dd399216b | |||
| e4c27fbca8 | |||
| 5a091ccad9 | |||
| 4a67aa0fb9 | |||
| c1f0972395 | |||
| 48802ba385 | |||
| 50676c02f4 | |||
| 131daa08cd | |||
| f5a6cb3f61 | |||
| 3fffd9c6e6 | |||
| f4e2d34b4e | |||
| afe08a1ff9 | |||
| e652162c1f | |||
| 38c3397b60 | |||
| 384f4b6b9c | |||
| a58372f03f | |||
| a99b1e83c4 | |||
| 9f248b2495 |
109 changed files with 6072 additions and 602 deletions
|
|
@ -2,7 +2,7 @@
|
||||||
|
|
||||||
## What Is Brainy
|
## What Is Brainy
|
||||||
|
|
||||||
@soulcraft/brainy (v7.17.0) is a Universal Knowledge Protocol -- a Triple Intelligence database combining vector search, graph traversal, and metadata filtering in a single library. Published to npm as a public MIT-licensed package.
|
@soulcraftlabs/brainy (v7.17.0) is a Universal Knowledge Protocol -- a Triple Intelligence database combining vector search, graph traversal, and metadata filtering in a single library. Published to npm as a public MIT-licensed package.
|
||||||
|
|
||||||
## Core Architecture
|
## Core Architecture
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -32,7 +32,7 @@ jobs:
|
||||||
run: |
|
run: |
|
||||||
set -eo pipefail
|
set -eo pipefail
|
||||||
|
|
||||||
SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/"
|
SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraftlabs/npm/"
|
||||||
VERSION="$(node -p "require('./package.json').version")"
|
VERSION="$(node -p "require('./package.json').version")"
|
||||||
# The dist-tag follows the version: a prerelease (any hyphen —
|
# The dist-tag follows the version: a prerelease (any hyphen —
|
||||||
# 10.4.0-rc.1) publishes under 'rc' and must NEVER move 'latest' —
|
# 10.4.0-rc.1) publishes under 'rc' and must NEVER move 'latest' —
|
||||||
|
|
@ -43,13 +43,13 @@ jobs:
|
||||||
case "$VERSION" in
|
case "$VERSION" in
|
||||||
*-*) NPM_TAG="rc" ;;
|
*-*) NPM_TAG="rc" ;;
|
||||||
esac
|
esac
|
||||||
echo "Publishing @soulcraft/brainy@${VERSION} to The Source registry (dist-tag: ${NPM_TAG})..."
|
echo "Publishing @soulcraftlabs/brainy@${VERSION} to The Source registry (dist-tag: ${NPM_TAG})..."
|
||||||
|
|
||||||
TMPRC="$(mktemp)"
|
TMPRC="$(mktemp)"
|
||||||
chmod 600 "$TMPRC"
|
chmod 600 "$TMPRC"
|
||||||
{
|
{
|
||||||
echo "@soulcraft:registry=${SOURCE_NPM_REG}"
|
echo "@soulcraftlabs:registry=${SOURCE_NPM_REG}"
|
||||||
echo "//source.soulcraft.com/api/packages/soulcraft/npm/:_authToken=${FORGE_NPM_TOKEN}"
|
echo "//source.soulcraft.com/api/packages/soulcraftlabs/npm/:_authToken=${FORGE_NPM_TOKEN}"
|
||||||
} > "$TMPRC"
|
} > "$TMPRC"
|
||||||
|
|
||||||
# The release script bumps package.json's version before it tags, so
|
# The release script bumps package.json's version before it tags, so
|
||||||
|
|
@ -64,7 +64,7 @@ jobs:
|
||||||
# exit code: a benign duplicate publish (a prior run, or a mirror, already
|
# exit code: a benign duplicate publish (a prior run, or a mirror, already
|
||||||
# landed this exact version) reports failure even though the registry
|
# landed this exact version) reports failure even though the registry
|
||||||
# already holds the right content.
|
# already holds the right content.
|
||||||
LANDED_VERSION="$(npm view "@soulcraft/brainy@${VERSION}" version --userconfig "$TMPRC" 2>/dev/null || echo "")"
|
LANDED_VERSION="$(npm view "@soulcraftlabs/brainy@${VERSION}" version --userconfig "$TMPRC" 2>/dev/null || echo "")"
|
||||||
rm -f "$TMPRC"
|
rm -f "$TMPRC"
|
||||||
|
|
||||||
if [ "$LANDED_VERSION" != "$VERSION" ]; then
|
if [ "$LANDED_VERSION" != "$VERSION" ]; then
|
||||||
|
|
@ -73,7 +73,7 @@ jobs:
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ "$PUBLISH_OK" = true ]; then
|
if [ "$PUBLISH_OK" = true ]; then
|
||||||
echo "Published and verified @soulcraft/brainy@${VERSION} on The Source registry."
|
echo "Published and verified @soulcraftlabs/brainy@${VERSION} on The Source registry."
|
||||||
else
|
else
|
||||||
echo "::warning::npm publish reported failure, but readback confirms @soulcraft/brainy@${VERSION} is already live on The Source (a prior run or mirror landed it) — treating this run as successful, since the registry content is correct. Any OTHER failure mode would have failed the readback check above instead."
|
echo "::warning::npm publish reported failure, but readback confirms @soulcraftlabs/brainy@${VERSION} is already live on The Source (a prior run or mirror landed it) — treating this run as successful, since the registry content is correct. Any OTHER failure mode would have failed the readback check above instead."
|
||||||
fi
|
fi
|
||||||
|
|
|
||||||
72
CHANGELOG.md
72
CHANGELOG.md
|
|
@ -2,12 +2,48 @@
|
||||||
|
|
||||||
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
||||||
|
|
||||||
### [10.4.2](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.2-rc.1...v10.4.2) (2026-08-27)
|
### [10.4.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.3...v10.4.4) (2026-08-28)
|
||||||
|
|
||||||
|
- fix(vfs): the old-root sweep narrates only when it has something to say (d49148e1)
|
||||||
|
- fix(tests): the health-gate pin follows the verdict, and the VFS suite uses its own store (42e2da25)
|
||||||
|
- Merge branch 'next/open-lazy-open-and-counts' (5ebd3b40)
|
||||||
|
- docs: the contract manifest stands alone; public docs describe this engine only (a8c724a2)
|
||||||
|
- docs(releases): 10.4.4 consumer notes — correctness and observability, with the performance line stated exactly (61a46927)
|
||||||
|
- docs: measurements in public history carry numbers, not provenance (02c61636)
|
||||||
|
- feat(open): name the two steps that hold the vfs-bootstrap phase (2cf38010)
|
||||||
|
- fix(storage): a dead flush watch falls back to the 500ms poll, not the 30s sweep (5c22f950)
|
||||||
|
- fix(storage): the flush watcher cannot arm twice in its async window (16d2e1a9)
|
||||||
|
- perf(idle): the flush-request watch is event-driven; the heartbeat is observability (fb1da1c5)
|
||||||
|
- perf(open): answer "are there any entities?" with one directory read (417ddb51)
|
||||||
|
- perf(generations): discover generations by directory name, not by walking the log (9dd39921)
|
||||||
|
- fix(flush): clear() and repairIndex() set the dirty witness themselves (e4c27fbc)
|
||||||
|
- feat(open): the open names the STEP that cost the time, not just the phase (5a091cca)
|
||||||
|
- perf(vfs): the old-root sweep runs once per store, not once per open (4a67aa0f)
|
||||||
|
- chore: keep the generated neural stamps at main's values (c1f09723)
|
||||||
|
- feat(contract): declare contract 1, serve three operators, refuse four by name (48802ba3)
|
||||||
|
- fix(open): a provider rebuilding itself is a third state, not a CRITICAL (50676c02)
|
||||||
|
- feat(open): open never waits for a provider that is rebuilding itself (131daa08)
|
||||||
|
- perf(flush): an idle brain does no work — no periodic flush without a write (f5a6cb3f)
|
||||||
|
- feat(repair): repairIndex narrates every phase and its receipt carries the walls (3fffd9c6)
|
||||||
|
- fix(storage): a suspect count ledger heals itself, and counts.json is written atomically (f4e2d34b)
|
||||||
|
- feat(open): the open narrates itself, on a channel production cannot clamp (afe08a1f)
|
||||||
|
- fix(storage): a clean close is recorded, and the writer lock is always given up (e652162c)
|
||||||
|
- docs: repository links point at soulcraftlabs/open-brainy — the soulcraft/brainy path becomes the native engine's repo tonight (38c3397b)
|
||||||
|
|
||||||
|
|
||||||
|
### [10.4.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.2...v10.4.3) (2026-08-27)
|
||||||
|
|
||||||
|
- Merge branch 'next/open-brainy-rename' (a58372f0)
|
||||||
|
- chore: rename to @soulcraftlabs/brainy for Open Brainy on The Source (a99b1e83)
|
||||||
|
- docs(releases): 10.4.3 — Open Brainy's first release under the new name, same engine as 10.4.2; The Source is the one registry (9f248b24)
|
||||||
|
|
||||||
|
|
||||||
|
### [10.4.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.2-rc.1...v10.4.2) (2026-08-27)
|
||||||
|
|
||||||
- docs(releases): 10.4.1 and 10.4.2 consumer notes; 10.4.2 is the last MIT release under this name, Open Brainy continues at @soulcraftlabs/brainy (a082e0ef)
|
- docs(releases): 10.4.1 and 10.4.2 consumer notes; 10.4.2 is the last MIT release under this name, Open Brainy continues at @soulcraftlabs/brainy (a082e0ef)
|
||||||
|
|
||||||
|
|
||||||
### [10.4.2-rc.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.1...v10.4.2-rc.1) (2026-08-27)
|
### [10.4.2-rc.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.1...v10.4.2-rc.1) (2026-08-27)
|
||||||
|
|
||||||
- Merge branch 'next/zero-norm-unvector-door' (9b84ef5b)
|
- Merge branch 'next/zero-norm-unvector-door' (9b84ef5b)
|
||||||
- fix(vectors): a zero-norm vector is not a vector, canonical side included, plus the sanctioned unvector door (0de76659)
|
- fix(vectors): a zero-norm vector is not a vector, canonical side included, plus the sanctioned unvector door (0de76659)
|
||||||
|
|
@ -24,23 +60,23 @@ All notable changes to this project will be documented in this file. See [standa
|
||||||
- chore(release): 10.4.1-rc.1 (7870dc40)
|
- chore(release): 10.4.1-rc.1 (7870dc40)
|
||||||
|
|
||||||
|
|
||||||
### [10.4.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0...v10.4.1) (2026-08-26)
|
### [10.4.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0...v10.4.1) (2026-08-26)
|
||||||
|
|
||||||
- fix(reads): the read gate is per-family; a write carrying unchanged data never re-embeds (c039411e)
|
- fix(reads): the read gate is per-family; a write carrying unchanged data never re-embeds (c039411e)
|
||||||
- docs(guide): the docs pipeline publishes through the ingest API — the separate deploy step is retired (21e506e8)
|
- docs(guide): the docs pipeline publishes through the ingest API — the separate deploy step is retired (21e506e8)
|
||||||
|
|
||||||
|
|
||||||
### [10.4.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.4...v10.4.0) (2026-08-26)
|
### [10.4.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.4...v10.4.0) (2026-08-26)
|
||||||
|
|
||||||
- docs(releases): the 10.4.0 entry catches up to the late trains — repair routing, the vector ledger and open-gate leg, the loud config guard, the JSON-safe crossing (834149ed)
|
- docs(releases): the 10.4.0 entry catches up to the late trains — repair routing, the vector ledger and open-gate leg, the loud config guard, the JSON-safe crossing (834149ed)
|
||||||
|
|
||||||
|
|
||||||
### [10.4.0-rc.4](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.3...v10.4.0-rc.4) (2026-08-25)
|
### [10.4.0-rc.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.3...v10.4.0-rc.4) (2026-08-25)
|
||||||
|
|
||||||
- feat(vector): the vectored-noun scalar joins the count ledger; the open gate closes the vector leg (9730835b)
|
- feat(vector): the vectored-noun scalar joins the count ledger; the open gate closes the vector leg (9730835b)
|
||||||
|
|
||||||
|
|
||||||
### [10.4.0-rc.3](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.2...v10.4.0-rc.3) (2026-08-25)
|
### [10.4.0-rc.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.2...v10.4.0-rc.3) (2026-08-25)
|
||||||
|
|
||||||
- fix(update-seam): the metadata crossing never carries BigInt endpoint ints (f4780c8e)
|
- fix(update-seam): the metadata crossing never carries BigInt endpoint ints (f4780c8e)
|
||||||
- Merge branch 'worktree-agent-ad3aff0dffd17a6eb' (f14da34b)
|
- Merge branch 'worktree-agent-ad3aff0dffd17a6eb' (f14da34b)
|
||||||
|
|
@ -49,7 +85,7 @@ All notable changes to this project will be documented in this file. See [standa
|
||||||
- feat(open-path): init never gates on the embedding model; open goes concurrent; slow opens narrate (96624f40)
|
- feat(open-path): init never gates on the embedding model; open goes concurrent; slow opens narrate (96624f40)
|
||||||
|
|
||||||
|
|
||||||
### [10.4.0-rc.2](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.1...v10.4.0-rc.2) (2026-08-25)
|
### [10.4.0-rc.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.1...v10.4.0-rc.2) (2026-08-25)
|
||||||
|
|
||||||
- test(readiness): the report helper's clock freezes — two independently-built reports compared across a millisecond tick made the plant lane red (39b916a3)
|
- test(readiness): the report helper's clock freezes — two independently-built reports compared across a millisecond tick made the plant lane red (39b916a3)
|
||||||
- feat(repair): a heal:'repair' verdict routes to the provider's own incremental repair() (553e0d97)
|
- feat(repair): a heal:'repair' verdict routes to the provider's own incremental repair() (553e0d97)
|
||||||
|
|
@ -60,7 +96,7 @@ All notable changes to this project will be documented in this file. See [standa
|
||||||
- feat(health): the gate reads the named report — reads refuse loudly, never rebuild; open serves before it returns; the ceremony door (f8f64780)
|
- feat(health): the gate reads the named report — reads refuse loudly, never rebuild; open serves before it returns; the ceremony door (f8f64780)
|
||||||
|
|
||||||
|
|
||||||
### [10.4.0-rc.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.3.1...v10.4.0-rc.1) (2026-08-24)
|
### [10.4.0-rc.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.3.1...v10.4.0-rc.1) (2026-08-24)
|
||||||
|
|
||||||
- ci(publish): the home dist-tag follows the version — a prerelease publishes under 'rc' and never moves 'latest' (a1376e4a)
|
- ci(publish): the home dist-tag follows the version — a prerelease publishes under 'rc' and never moves 'latest' (a1376e4a)
|
||||||
- chore(release): --source-only — a home-only prerelease mode (The Source, never the storefront) (dcbad176)
|
- chore(release): --source-only — a home-only prerelease mode (The Source, never the storefront) (dcbad176)
|
||||||
|
|
@ -73,13 +109,13 @@ All notable changes to this project will be documented in this file. See [standa
|
||||||
- ci(gate): the machine-health preflight and the truncation verdict guard (1e046aa1)
|
- ci(gate): the machine-health preflight and the truncation verdict guard (1e046aa1)
|
||||||
|
|
||||||
|
|
||||||
### [10.3.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.3.0...v10.3.1) (2026-08-18)
|
### [10.3.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.3.0...v10.3.1) (2026-08-18)
|
||||||
|
|
||||||
- docs(releases): the 10.3.1 consumer entry — the fold that behaves (900cc895)
|
- docs(releases): the 10.3.1 consumer entry — the fold that behaves (900cc895)
|
||||||
- fix(recovery): the fold streams and narrates; the checkpoint chain arms at the flip (ed7d1db9)
|
- fix(recovery): the fold streams and narrates; the checkpoint chain arms at the flip (ed7d1db9)
|
||||||
|
|
||||||
|
|
||||||
### [10.3.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.2.0...v10.3.0) (2026-08-18)
|
### [10.3.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.2.0...v10.3.0) (2026-08-18)
|
||||||
|
|
||||||
- docs(releases): the 10.3.0 consumer entry — the trust-and-provenance release (97d75649)
|
- docs(releases): the 10.3.0 consumer entry — the trust-and-provenance release (97d75649)
|
||||||
- fix(locks): the fence keys ownership on pid+hostname — a same-process re-open never fences its predecessor (0991cf28)
|
- fix(locks): the fence keys ownership on pid+hostname — a same-process re-open never fences its predecessor (0991cf28)
|
||||||
|
|
@ -88,14 +124,14 @@ All notable changes to this project will be documented in this file. See [standa
|
||||||
- feat(log): system commits carry their origin; the attested per-id reconcile door (9ac9e706)
|
- feat(log): system commits carry their origin; the attested per-id reconcile door (9ac9e706)
|
||||||
|
|
||||||
|
|
||||||
### [10.2.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.1.0...v10.2.0) (2026-08-17)
|
### [10.2.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.1.0...v10.2.0) (2026-08-17)
|
||||||
|
|
||||||
- docs(releases): the 10.2.0 consumer entry — adoption completes in one call (97538e1f)
|
- docs(releases): the 10.2.0 consumer entry — adoption completes in one call (97538e1f)
|
||||||
- ci: the correctness plant runs integration + conformance on every push — a release never waits on a second machine (b17fdc8e)
|
- ci: the correctness plant runs integration + conformance on every push — a release never waits on a second machine (b17fdc8e)
|
||||||
- fix(adoption): the baseline backfill runs to completion — one call adopts a pre-log baseline of any size (a5a18838)
|
- fix(adoption): the baseline backfill runs to completion — one call adopts a pre-log baseline of any size (a5a18838)
|
||||||
|
|
||||||
|
|
||||||
### [10.1.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.0.0...v10.1.0) (2026-08-13)
|
### [10.1.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.0.0...v10.1.0) (2026-08-13)
|
||||||
|
|
||||||
- docs(releases): the 10.1.0 consumer entry — bounded recovery, restore founding, the two write-path cures (7d3c8696)
|
- docs(releases): the 10.1.0 consumer entry — bounded recovery, restore founding, the two write-path cures (7d3c8696)
|
||||||
- fix(restore): a restore is an unclean event — the swap runs quiesced and the snapshot's durability stamps never survive it (9ca80667)
|
- fix(restore): a restore is an unclean event — the swap runs quiesced and the snapshot's durability stamps never survive it (9ca80667)
|
||||||
|
|
@ -104,7 +140,7 @@ All notable changes to this project will be documented in this file. See [standa
|
||||||
- feat(query): the sparse-store cut — where on a never-carried field serves operator truth, never a refusal (7b67db4d)
|
- feat(query): the sparse-store cut — where on a never-carried field serves operator truth, never a refusal (7b67db4d)
|
||||||
|
|
||||||
|
|
||||||
### [10.0.0](https://source.soulcraft.com/soulcraft/brainy/compare/v9.0.0...v10.0.0) (2026-08-12)
|
### [10.0.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v9.0.0...v10.0.0) (2026-08-12)
|
||||||
|
|
||||||
- fix(adoption): the baseline backfill cures hydration-law drift — existing brains reach the crash-safe default with zero operator steps (25f0dd96)
|
- fix(adoption): the baseline backfill cures hydration-law drift — existing brains reach the crash-safe default with zero operator steps (25f0dd96)
|
||||||
- fix(adoption): the reserved-root mint exemption — int 0 is legitimate for exactly one id (2abe8b38)
|
- fix(adoption): the reserved-root mint exemption — int 0 is legitimate for exactly one id (2abe8b38)
|
||||||
|
|
@ -136,7 +172,7 @@ All notable changes to this project will be documented in this file. See [standa
|
||||||
- test: version-coupling pins go major-agnostic — the 8.x literals broke at the 9.0.0 bump while the coupling law itself behaved correctly (8a6807e8)
|
- test: version-coupling pins go major-agnostic — the 8.x literals broke at the 9.0.0 bump while the coupling law itself behaved correctly (8a6807e8)
|
||||||
|
|
||||||
|
|
||||||
### [9.0.0](https://source.soulcraft.com/soulcraft/brainy/compare/v8.11.0...v9.0.0) (2026-08-04)
|
### [9.0.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.11.0...v9.0.0) (2026-08-04)
|
||||||
|
|
||||||
- docs: 9.0 namespace-migration guide — the simple story + the mechanical sweep checklist, published for humans and tooling alike (61ab9db2)
|
- docs: 9.0 namespace-migration guide — the simple story + the mechanical sweep checklist, published for humans and tooling alike (61ab9db2)
|
||||||
- fix(release): storefront leg republishes CI's exact forge artifact — byte-identity by construction, verified by cross-registry shasum before the ceremony reports success (d89df2ed)
|
- fix(release): storefront leg republishes CI's exact forge artifact — byte-identity by construction, verified by cross-registry shasum before the ceremony reports success (d89df2ed)
|
||||||
|
|
@ -171,7 +207,7 @@ All notable changes to this project will be documented in this file. See [standa
|
||||||
- feat: scanFacts liveness contract — first batch or loud failure within a documented bound (f8e6da2b)
|
- feat: scanFacts liveness contract — first batch or loud failure within a documented bound (f8e6da2b)
|
||||||
|
|
||||||
|
|
||||||
### [8.11.0](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.1...v8.11.0) (2026-07-27)
|
### [8.11.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.1...v8.11.0) (2026-07-27)
|
||||||
|
|
||||||
- docs: the last two archived-host links point home (91ef1c8b)
|
- docs: the last two archived-host links point home (91ef1c8b)
|
||||||
- feat: includeHidden — export carries every visibility tier for migration-grade canon completeness (63c1eeb9)
|
- feat: includeHidden — export carries every visibility tier for migration-grade canon completeness (63c1eeb9)
|
||||||
|
|
@ -180,19 +216,19 @@ All notable changes to this project will be documented in this file. See [standa
|
||||||
- ci: run the pipeline on the forge (999d0ebb)
|
- ci: run the pipeline on the forge (999d0ebb)
|
||||||
|
|
||||||
|
|
||||||
### [8.10.3](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.2...v8.10.3) (2026-08-03)
|
### [8.10.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.2...v8.10.3) (2026-08-03)
|
||||||
|
|
||||||
- docs: dedupe the 8.10.2 release-notes entry the cherry doubled onto the branch (8c956608)
|
- docs: dedupe the 8.10.2 release-notes entry the cherry doubled onto the branch (8c956608)
|
||||||
- fix: user metadata named 'level' is a real field everywhere — the engine-internal node layer no longer shadows it in sort/filter/aggregation, and the indexing views stop stamping a phantom 0 into its column; index epoch 2 rebuilds existing brains at first open (958a0859)
|
- fix: user metadata named 'level' is a real field everywhere — the engine-internal node layer no longer shadows it in sort/filter/aggregation, and the indexing views stop stamping a phantom 0 into its column; index epoch 2 rebuilds existing brains at first open (958a0859)
|
||||||
|
|
||||||
|
|
||||||
### [8.10.2](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.1...v8.10.2) (2026-07-29)
|
### [8.10.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.1...v8.10.2) (2026-07-29)
|
||||||
|
|
||||||
- docs: 8.10.2 consumer release notes — update() write granularity, PathResolver idle-log fix, graph-lsm key recognition (a0123b5b)
|
- docs: 8.10.2 consumer release notes — update() write granularity, PathResolver idle-log fix, graph-lsm key recognition (a0123b5b)
|
||||||
- fix: metadata-only update() never rewrites the noun record — the unconditional whole-vector save turned per-entity stat touches into full rewrites+fsync, amplifying read-heavy sweeps into disk saturation on a production deployment (5b65eb82)
|
- fix: metadata-only update() never rewrites the noun record — the unconditional whole-vector save turned per-entity stat touches into full rewrites+fsync, amplifying read-heavy sweeps into disk saturation on a production deployment (5b65eb82)
|
||||||
|
|
||||||
|
|
||||||
### [8.10.1](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.0...v8.10.1) (2026-07-24)
|
### [8.10.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.0...v8.10.1) (2026-07-24)
|
||||||
|
|
||||||
- refactor: remove the orphaned transaction-result type left behind by the dead-path removal (edf123a5)
|
- refactor: remove the orphaned transaction-result type left behind by the dead-path removal (edf123a5)
|
||||||
- fix: warm() metadata surface routes through the active provider (warm hook added to the metadata contract); add maintenanceDebt() observability surface (5b2cbf74)
|
- fix: warm() metadata surface routes through the active provider (warm hook added to the metadata contract); add maintenanceDebt() observability surface (5b2cbf74)
|
||||||
|
|
|
||||||
|
|
@ -12,13 +12,13 @@ Handoff file: `/home/dpsifr/.strategy/PLATFORM-HANDOFF.md`
|
||||||
|
|
||||||
**Brainy's current open actions:** None. MIT open-source — no platform-specific actions.
|
**Brainy's current open actions:** None. MIT open-source — no platform-specific actions.
|
||||||
|
|
||||||
**Current version:** run `npm view @soulcraft/brainy version` (never trust a hardcoded number here — this line went stale for months); consumer-facing changes tracked in `RELEASES.md`
|
**Current version:** run `npm view @soulcraftlabs/brainy version --registry https://source.soulcraft.com/api/packages/soulcraftlabs/npm/` (never trust a hardcoded number here — this line went stale for months); consumer-facing changes tracked in `RELEASES.md`
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Project Overview
|
## Project Overview
|
||||||
|
|
||||||
Brainy is a Universal Knowledge Protocol -- a Triple Intelligence database that combines vector similarity search, graph traversal, and metadata filtering into a single TypeScript library. Published as `@soulcraft/brainy` on npm under the MIT license.
|
Brainy is a Universal Knowledge Protocol -- a Triple Intelligence database that combines vector similarity search, graph traversal, and metadata filtering into a single TypeScript library. Published as `@soulcraftlabs/brainy` on The Source (source.soulcraft.com registry) under the MIT license.
|
||||||
|
|
||||||
## Getting Started
|
## Getting Started
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ may find elsewhere in the repo's history.
|
||||||
|
|
||||||
## Where the project lives
|
## Where the project lives
|
||||||
|
|
||||||
The source of truth is a self-hosted forge: **source.soulcraft.com/soulcraft/brainy**.
|
The source of truth is a self-hosted forge: **source.soulcraft.com/soulcraftlabs/open-brainy**.
|
||||||
It's anonymously readable and cloneable — no account needed to browse, clone,
|
It's anonymously readable and cloneable — no account needed to browse, clone,
|
||||||
or build.
|
or build.
|
||||||
|
|
||||||
|
|
@ -31,7 +31,7 @@ fine) to talk through the approach saves everyone rework.
|
||||||
## Development setup
|
## Development setup
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone https://source.soulcraft.com/soulcraft/brainy.git
|
git clone https://source.soulcraft.com/soulcraftlabs/open-brainy.git
|
||||||
cd brainy
|
cd brainy
|
||||||
npm install
|
npm install
|
||||||
npm run build
|
npm run build
|
||||||
|
|
@ -57,6 +57,17 @@ see `package.json` for `test:integration`, `test:coverage`, and friends.
|
||||||
description states a number, cite the benchmark that produced it (see
|
description states a number, cite the benchmark that produced it (see
|
||||||
[docs/performance-envelopes.md](docs/performance-envelopes.md) for the
|
[docs/performance-envelopes.md](docs/performance-envelopes.md) for the
|
||||||
pattern). Don't state an estimate as if it were measured.
|
pattern). Don't state an estimate as if it were measured.
|
||||||
|
- **Measurements carry numbers, not provenance.** Public commit messages and
|
||||||
|
docs give the SHAPE a number was taken at and never where it was taken: no
|
||||||
|
hostnames, no store or deployment identities, no operational anecdotes about
|
||||||
|
someone's running system. "A 14,056-noun / 72,679-verb production-shaped
|
||||||
|
store, measured solo under an exclusive lock" tells a reader everything the
|
||||||
|
number depends on; the machine it ran on and whose data it was tell them
|
||||||
|
nothing except where somebody's infrastructure lives.
|
||||||
|
- **Documents that answer or reference a confidential specification never enter
|
||||||
|
this repository, even summarized.** The public docs describe THIS engine and
|
||||||
|
the published contract, and nothing else — a summary of a private document is
|
||||||
|
still that document's contents.
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
|
|
|
||||||
18
README.md
18
README.md
|
|
@ -1,5 +1,5 @@
|
||||||
<p align="center">
|
<p align="center">
|
||||||
<img src="https://source.soulcraft.com/soulcraft/brainy/raw/branch/main/brainy.png" alt="Brainy" width="180">
|
<img src="https://source.soulcraft.com/soulcraftlabs/open-brainy/raw/branch/main/brainy.png" alt="Brainy" width="180">
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
<h1 align="center">Brainy</h1>
|
<h1 align="center">Brainy</h1>
|
||||||
|
|
@ -11,9 +11,9 @@
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
<p align="center">
|
<p align="center">
|
||||||
<a href="https://www.npmjs.com/package/@soulcraft/brainy"><img src="https://img.shields.io/npm/v/@soulcraft/brainy.svg" alt="npm version"></a>
|
<a href="https://source.soulcraft.com/soulcraftlabs/-/packages/npm/brainy"><img src="https://img.shields.io/badge/package-The%20Source-2c3e50.svg" alt="Package on The Source"></a>
|
||||||
<a href="https://www.npmjs.com/package/@soulcraft/brainy"><img src="https://img.shields.io/npm/dm/@soulcraft/brainy.svg" alt="npm downloads"></a>
|
<a href="https://source.soulcraft.com/soulcraftlabs/open-brainy"><img src="https://img.shields.io/badge/repo-open--brainy-2c3e50.svg" alt="Repository"></a>
|
||||||
<a href="https://source.soulcraft.com/soulcraft/brainy/actions"><img src="https://source.soulcraft.com/soulcraft/brainy/actions/workflows/ci.yml/badge.svg?branch=main" alt="CI"></a>
|
<a href="https://source.soulcraft.com/soulcraftlabs/open-brainy/actions"><img src="https://source.soulcraft.com/soulcraftlabs/open-brainy/actions/workflows/ci.yml/badge.svg?branch=main" alt="CI"></a>
|
||||||
<a href="https://soulcraft.com/docs"><img src="https://img.shields.io/badge/docs-soulcraft.com-blue.svg" alt="Documentation"></a>
|
<a href="https://soulcraft.com/docs"><img src="https://img.shields.io/badge/docs-soulcraft.com-blue.svg" alt="Documentation"></a>
|
||||||
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="MIT License"></a>
|
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="MIT License"></a>
|
||||||
<a href="https://www.typescriptlang.org/"><img src="https://img.shields.io/badge/%3C%2F%3E-TypeScript-%230074c1.svg" alt="TypeScript"></a>
|
<a href="https://www.typescriptlang.org/"><img src="https://img.shields.io/badge/%3C%2F%3E-TypeScript-%230074c1.svg" alt="TypeScript"></a>
|
||||||
|
|
@ -30,6 +30,8 @@
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
**Open Brainy** is the MIT engine — the open API, client library, types, and protocol; an openly specified canonical on-disk format; and this TypeScript reference engine, scoped as a single-node engine for stores up to roughly one million rows. `@soulcraft/brainy` 10.4.2 was the last release under the old package name — the name passes to the native engine, **Brainy**, at 11.0.0: the same API over the same open format at production scale, and it requires a license.
|
||||||
|
|
||||||
Built because we were tired of stitching a vector store to a graph database to a document store — and spending weeks on plumbing before writing a line of business logic. Brainy indexes every fact **three ways at once** and lets one call query them together:
|
Built because we were tired of stitching a vector store to a graph database to a document store — and spending weeks on plumbing before writing a line of business logic. Brainy indexes every fact **three ways at once** and lets one call query them together:
|
||||||
|
|
||||||
| You write | Brainy indexes it as | You query it with |
|
| You write | Brainy indexes it as | You query it with |
|
||||||
|
|
@ -45,12 +47,14 @@ It runs **inside your process** — no server, no Docker, nothing to operate —
|
||||||
## Quick start
|
## Quick start
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
bun add @soulcraft/brainy # Bun ≥ 1.1 — recommended
|
bun add @soulcraftlabs/brainy # Bun ≥ 1.1 — recommended
|
||||||
npm install @soulcraft/brainy # Node.js ≥ 22
|
npm install @soulcraftlabs/brainy # Node.js ≥ 22
|
||||||
```
|
```
|
||||||
|
|
||||||
|
> **Registry**: add `@soulcraftlabs:registry=https://source.soulcraft.com/api/packages/soulcraftlabs/npm/` to your `.npmrc` (anonymous read).
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
|
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy() // in-memory; one line swaps to disk
|
const brain = new Brainy() // in-memory; one line swaps to disk
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
|
||||||
136
RELEASES.md
136
RELEASES.md
|
|
@ -1,7 +1,7 @@
|
||||||
# @soulcraft/brainy — Release Notes for Consumers
|
# @soulcraft/brainy — Release Notes for Consumers
|
||||||
|
|
||||||
This file is the **quick reference for downstream sessions** tracking Brainy changes.
|
This file is the **quick reference for downstream sessions** tracking Brainy changes.
|
||||||
Full auto-generated changelog: `CHANGELOG.md` · Releases: https://source.soulcraft.com/soulcraft/brainy/releases
|
Full auto-generated changelog: `CHANGELOG.md` · Releases: https://source.soulcraft.com/soulcraftlabs/open-brainy/releases
|
||||||
|
|
||||||
**How to use:** Brainy is the underlying data engine for downstream applications. Read this when:
|
**How to use:** Brainy is the underlying data engine for downstream applications. Read this when:
|
||||||
- Upgrading `@soulcraft/brainy` in your application
|
- Upgrading `@soulcraft/brainy` in your application
|
||||||
|
|
@ -31,7 +31,139 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## v10.4.2 — 2026-08-28 (a zero-norm vector is not a vector)
|
## v10.4.4 — 2026-08-28
|
||||||
|
|
||||||
|
**A correctness and observability release.** The headline is not speed: it is that a
|
||||||
|
restart now tells you the truth about itself, a store stops lying about how much it
|
||||||
|
holds, and the engine stops doing work nobody asked for. There is a performance
|
||||||
|
improvement and it is modest; it is stated exactly below rather than rounded up.
|
||||||
|
|
||||||
|
### The dark restart — fixed at the root
|
||||||
|
|
||||||
|
A service could stop cleanly, exit 0, having awaited `close()` on every store it held,
|
||||||
|
and its next boot would announce `Overwriting stale writer lock … appears dead` for
|
||||||
|
every one of them. Nothing had crashed. Two deployments hit this; the same defect also
|
||||||
|
made those boots pay a crash-recovery fold they did not owe.
|
||||||
|
|
||||||
|
The cause was not the lock. `close()` released it correctly — when it got there. A
|
||||||
|
failure part-way through close skipped both the release AND the clean-shutdown marker,
|
||||||
|
and "the recorded pid is gone" reads identically for an orderly restart and a crash.
|
||||||
|
|
||||||
|
- `close()` is now two parts and the second is unconditional: the flush-request watcher,
|
||||||
|
the **writer lock**, the VFS timers and the terminal `closed` flag are released whether
|
||||||
|
the durable steps succeeded or not. The original failure is narrated with what it costs
|
||||||
|
the next open, then rethrown.
|
||||||
|
- Releasing the lock writes a **clean-close record** naming the lock generation it gave
|
||||||
|
up. The next open reads that record instead of guessing: recorded → nothing to recover;
|
||||||
|
absent → it says so, and names the recovery it is about to run. This also ends two
|
||||||
|
long-standing false alarms — a recycled pid locking a store out of its own reopen, and
|
||||||
|
`Re-acquiring writer lock … this is a bug` after a perfectly clean close.
|
||||||
|
- The signal path stopped failing in a batch. One store's failing flush used to strand
|
||||||
|
every remaining store's lock and markers — at exit code 0. Now: per-store isolation, the
|
||||||
|
generation store's close (the marker) is part of shutdown, the lock goes in a `finally`,
|
||||||
|
and the handler no longer calls `process.exit()` when the host application has its own
|
||||||
|
signal handler, a race that truncated the host's own shutdown mid-flight.
|
||||||
|
|
||||||
|
### The count ledger stops lying, and `counts.json` is written atomically
|
||||||
|
|
||||||
|
The all-tier scalars are the denominator a coverage check subtracts against. A ledger
|
||||||
|
derived under the old rule — one entity per id DIRECTORY — counted ghost and scar
|
||||||
|
containers as rows, and was only FLAGGED suspect: it went on serving wrong numbers for
|
||||||
|
the life of the store. Two copies of one archive could disagree, and a downstream index
|
||||||
|
heal reported remaining work that did not exist.
|
||||||
|
|
||||||
|
- Such a ledger now derives itself honestly **in the background** after the open, counting
|
||||||
|
identity records, and persists the correction stamped. Nothing waits for it, because no
|
||||||
|
read is served from a denominator.
|
||||||
|
- A derivation that raced a write refuses to stamp its number: one retry on a quiet store,
|
||||||
|
then the ledger stays SUSPECT and names `repairIndex()` as the door that recounts under
|
||||||
|
a barrier.
|
||||||
|
- `counts.json` is written temp+rename. A truncating write left a window in which a
|
||||||
|
concurrent reader saw the file EMPTY — and an unparseable ledger sends the next open
|
||||||
|
down the full-rescan path, so the cheapest file in the store was buying the most
|
||||||
|
expensive recovery.
|
||||||
|
|
||||||
|
### An open and a repair narrate themselves — on a channel a log level cannot silence
|
||||||
|
|
||||||
|
A store could open for three minutes and print nothing at all. The phase timings existed;
|
||||||
|
they were written to a channel that every production-looking environment clamps away.
|
||||||
|
|
||||||
|
- Narration moved to an always-visible channel. An open now heartbeats the phase it is in,
|
||||||
|
names each phase as it ends with what it was paying for, and names the expensive STEP
|
||||||
|
inside a phase. `repairIndex()` does the same and its receipt carries a per-family
|
||||||
|
`durationMs` — a repair that ran for half an hour with no output could only be watched
|
||||||
|
through `top`.
|
||||||
|
- A brain nobody has written to now does nothing: a flush over a clean store is a no-op
|
||||||
|
and says nothing, the graph index's auto-flush asks before it acts, and the
|
||||||
|
cross-process flush-request watch is **event-driven** (`fs.watch`) instead of polling a
|
||||||
|
directory every 500 ms per store forever, with a slow safety sweep behind it and a
|
||||||
|
narrated fall back to polling where a filesystem cannot be watched.
|
||||||
|
- A provider that is REBUILDING ITSELF is no longer confused with a broken one. `init()`
|
||||||
|
does not wait for it, every other family serves, and that family's doors refuse **by
|
||||||
|
name, carrying the provider's own progress**, saying plainly that they open by
|
||||||
|
themselves and no action is needed. Health narration dedupes by content, so an unchanged
|
||||||
|
verdict is silent however a provider's generation counter moves.
|
||||||
|
|
||||||
|
### For operators — one behaviour change
|
||||||
|
|
||||||
|
**Four `where` operators that previously returned an empty page now raise
|
||||||
|
`INVALID_QUERY`:** `startsWith`, `endsWith`, `matches` and `length`. An equality/range
|
||||||
|
posting index cannot evaluate a substring, a pattern or an array length without reading
|
||||||
|
every row, and it now refuses by name instead of answering with an empty result that
|
||||||
|
looks like an answer.
|
||||||
|
|
||||||
|
**Three that previously returned an empty page are now SERVED:** `hasAll`, `noneOf` and
|
||||||
|
`excludes`. All 25 accepted operator tokens now agree between this engine and its
|
||||||
|
accelerated counterpart.
|
||||||
|
|
||||||
|
### Performance — stated exactly
|
||||||
|
|
||||||
|
Measured on a 14,056-noun / 72,679-verb production-shaped store, both builds solo under
|
||||||
|
an exclusive lock:
|
||||||
|
|
||||||
|
- **Warm reopen after a clean close: 85.7 s → 77.0 s (−10.2%).** The whole of that gain is
|
||||||
|
one fix — generation discovery reads directory NAMES instead of recursively walking the
|
||||||
|
entire generation log (−9.2 s, and it scales with history rather than row count). The
|
||||||
|
VFS phase is **unchanged**.
|
||||||
|
- **Cold open: −31.4 s** (518.1 s → 486.7 s), of which the count-ledger derivation moving
|
||||||
|
off the critical path accounts for storage-init dropping 5,941 ms → 25 ms.
|
||||||
|
- **A dominant ~38 s remains, diagnosed and NOT fixed.** It is not the VFS — the VFS's own
|
||||||
|
init is under 2 s of that phase. It is the log-authority adoption and/or the
|
||||||
|
pending-embed log recovery, both now instrumented so the next measurement names the
|
||||||
|
culprit outright.
|
||||||
|
|
||||||
|
Continuing work, named so nobody has to rediscover it: that ~38 s term; making the
|
||||||
|
generation store's committed-range set lazy; the hydration path that substitutes
|
||||||
|
`Date.now()` for an unreadable stored timestamp (inventing data); and a VFS path-prefix
|
||||||
|
filter built with a `$startsWith` spelling no operator set accepts, so
|
||||||
|
`searchFiles({ path })` throws today.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## v10.4.3 — 2026-08-27 (Open Brainy's first release)
|
||||||
|
|
||||||
|
**`@soulcraftlabs/brainy` 10.4.3 is the same engine as `@soulcraft/brainy` 10.4.2, byte for
|
||||||
|
byte — only the name, the registry, and the pointers changed.** Install:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm install @soulcraftlabs/brainy
|
||||||
|
```
|
||||||
|
|
||||||
|
with the registry line in your `.npmrc` (anonymous read):
|
||||||
|
|
||||||
|
```
|
||||||
|
@soulcraftlabs:registry=https://source.soulcraft.com/api/packages/soulcraftlabs/npm/
|
||||||
|
```
|
||||||
|
|
||||||
|
- **The Source is the one registry.** Open Brainy publishes to source.soulcraft.com only; the
|
||||||
|
npmjs republish step is retired from the release rail. Existing npmjs versions of
|
||||||
|
`@soulcraft/brainy` stay as they are and receive no new versions.
|
||||||
|
- **The repository moved** to `soulcraftlabs/open-brainy` on The Source; the old path redirects.
|
||||||
|
- **No engine change.** Everything in the 10.4.2 notes applies unchanged; adoption is one
|
||||||
|
install-line change (`@soulcraft/brainy` → `@soulcraftlabs/brainy`), which downstream
|
||||||
|
applications make together with their native-engine bump.
|
||||||
|
|
||||||
|
## v10.4.2 — 2026-08-27 (a zero-norm vector is not a vector)
|
||||||
|
|
||||||
**This is the last release of the MIT engine under the `@soulcraft/brainy` name.**
|
**This is the last release of the MIT engine under the `@soulcraft/brainy` name.**
|
||||||
The MIT package continues as **Open Brainy** — `@soulcraftlabs/brainy`: the open API,
|
The MIT package continues as **Open Brainy** — `@soulcraftlabs/brainy`: the open API,
|
||||||
|
|
|
||||||
|
|
@ -30,7 +30,7 @@ commit to backporting fixes to unsupported lines.
|
||||||
|
|
||||||
## Scope
|
## Scope
|
||||||
|
|
||||||
This policy covers the `@soulcraft/brainy` package itself — the code in
|
This policy covers the `@soulcraftlabs/brainy` package itself — the code in
|
||||||
this repository. If you're evaluating a deployment that also uses
|
this repository. If you're evaluating a deployment that also uses
|
||||||
`@soulcraft/cor`, report issues in that package the same way, to the same
|
`@soulcraft/cor`, report issues in that package the same way, to the same
|
||||||
address; we'll route internally.
|
address; we'll route internally.
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@
|
||||||
/**
|
/**
|
||||||
* Modern TypeScript CLI Runner
|
* Modern TypeScript CLI Runner
|
||||||
*
|
*
|
||||||
* This is the entry point after npm install @soulcraft/brainy
|
* This is the entry point after npm install @soulcraftlabs/brainy
|
||||||
* It runs the compiled TypeScript CLI code
|
* It runs the compiled TypeScript CLI code
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
|
|
||||||
2
bun.lock
2
bun.lock
|
|
@ -3,7 +3,7 @@
|
||||||
"configVersion": 0,
|
"configVersion": 0,
|
||||||
"workspaces": {
|
"workspaces": {
|
||||||
"": {
|
"": {
|
||||||
"name": "@soulcraft/brainy",
|
"name": "@soulcraftlabs/brainy",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@aws-sdk/client-s3": "^3.540.0",
|
"@aws-sdk/client-s3": "^3.540.0",
|
||||||
"@azure/identity": "^4.0.0",
|
"@azure/identity": "^4.0.0",
|
||||||
|
|
|
||||||
|
|
@ -25,13 +25,13 @@
|
||||||
|
|
||||||
### Prerequisites
|
### Prerequisites
|
||||||
```bash
|
```bash
|
||||||
npm install @soulcraft/brainy
|
npm install @soulcraftlabs/brainy
|
||||||
```
|
```
|
||||||
|
|
||||||
### Your First Neural Database
|
### Your First Neural Database
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy, NounType } from '@soulcraft/brainy'
|
import { Brainy, NounType } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
// Step 1: Create and initialize Brainy
|
// Step 1: Create and initialize Brainy
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
|
|
@ -143,7 +143,7 @@ Once you're comfortable with basic operations, move to **Level 2** to learn abou
|
||||||
### Building a Knowledge Graph
|
### Building a Knowledge Graph
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
|
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy({ storage: { type: 'memory' } })
|
const brain = new Brainy({ storage: { type: 'memory' } })
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -314,7 +314,7 @@ Ready for AI-powered search and clustering? Move to **Level 3**.
|
||||||
### Triple Intelligence in Action
|
### Triple Intelligence in Action
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
|
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy({ storage: { type: 'memory' } })
|
const brain = new Brainy({ storage: { type: 'memory' } })
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -529,7 +529,7 @@ Want to treat files as intelligent entities? Learn the **Virtual Filesystem** in
|
||||||
### Files as Intelligent Entities
|
### Files as Intelligent Entities
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
|
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy({ storage: { type: 'memory' } })
|
const brain = new Brainy({ storage: { type: 'memory' } })
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -832,7 +832,7 @@ Ready for production deployment? Level 5 covers **planet-scale architecture**.
|
||||||
### Production-Ready Deployment
|
### Production-Ready Deployment
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy, NounType } from '@soulcraft/brainy'
|
import { Brainy, NounType } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
// 1. PRODUCTION STORAGE - Filesystem with off-site snapshots
|
// 1. PRODUCTION STORAGE - Filesystem with off-site snapshots
|
||||||
console.log('Initializing production storage...\n')
|
console.log('Initializing production storage...\n')
|
||||||
|
|
|
||||||
|
|
@ -1217,7 +1217,7 @@ where: {
|
||||||
await brain.find({ type: 'Document' })
|
await brain.find({ type: 'Document' })
|
||||||
|
|
||||||
// ✅ Correct: Use NounType enum
|
// ✅ Correct: Use NounType enum
|
||||||
import { NounType } from '@soulcraft/brainy'
|
import { NounType } from '@soulcraftlabs/brainy'
|
||||||
await brain.find({ type: NounType.Document })
|
await brain.find({ type: NounType.Document })
|
||||||
|
|
||||||
// ❌ Error: Operator not recognized
|
// ❌ Error: Operator not recognized
|
||||||
|
|
|
||||||
|
|
@ -153,13 +153,13 @@ brainy-data/
|
||||||
### Step 1: Update Brainy Package
|
### Step 1: Update Brainy Package
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
npm install @soulcraft/brainy@latest
|
npm install @soulcraftlabs/brainy@latest
|
||||||
```
|
```
|
||||||
|
|
||||||
**Check your version:**
|
**Check your version:**
|
||||||
```bash
|
```bash
|
||||||
npm list @soulcraft/brainy
|
npm list @soulcraftlabs/brainy
|
||||||
# Should show: @soulcraft/brainy@4.0.0
|
# Should show: @soulcraftlabs/brainy@4.0.0
|
||||||
```
|
```
|
||||||
|
|
||||||
### Step 2: No Code Changes Required! ✅
|
### Step 2: No Code Changes Required! ✅
|
||||||
|
|
@ -374,7 +374,7 @@ If you encounter issues, you can rollback:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Reinstall v3
|
# Reinstall v3
|
||||||
npm install @soulcraft/brainy@^3.50.0
|
npm install @soulcraftlabs/brainy@^3.50.0
|
||||||
|
|
||||||
# Restart application
|
# Restart application
|
||||||
```
|
```
|
||||||
|
|
@ -389,7 +389,7 @@ rm -rf ./data
|
||||||
cp -r ./data-backup ./data
|
cp -r ./data-backup ./data
|
||||||
|
|
||||||
# Reinstall v3
|
# Reinstall v3
|
||||||
npm install @soulcraft/brainy@^3.50.0
|
npm install @soulcraftlabs/brainy@^3.50.0
|
||||||
```
|
```
|
||||||
|
|
||||||
## Common Migration Scenarios
|
## Common Migration Scenarios
|
||||||
|
|
@ -539,7 +539,7 @@ console.log('Storage type:', status.type)
|
||||||
|
|
||||||
**Migration Checklist:**
|
**Migration Checklist:**
|
||||||
- ✅ Backup data
|
- ✅ Backup data
|
||||||
- ✅ Update npm package (`npm install @soulcraft/brainy@latest`)
|
- ✅ Update npm package (`npm install @soulcraftlabs/brainy@latest`)
|
||||||
- ✅ Restart application (automatic migration)
|
- ✅ Restart application (automatic migration)
|
||||||
- ✅ Verify data integrity
|
- ✅ Verify data integrity
|
||||||
- ✅ Enable lifecycle policies
|
- ✅ Enable lifecycle policies
|
||||||
|
|
|
||||||
|
|
@ -46,7 +46,7 @@ If no plugin provides a given key, brainy uses its built-in JavaScript implement
|
||||||
### 1. Implement the `BrainyPlugin` interface
|
### 1. Implement the `BrainyPlugin` interface
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import type { BrainyPlugin, BrainyPluginContext } from '@soulcraft/brainy/plugin'
|
import type { BrainyPlugin, BrainyPluginContext } from '@soulcraftlabs/brainy/plugin'
|
||||||
|
|
||||||
const myPlugin: BrainyPlugin = {
|
const myPlugin: BrainyPlugin = {
|
||||||
name: 'my-brainy-plugin', // Must be unique (typically your npm package name)
|
name: 'my-brainy-plugin', // Must be unique (typically your npm package name)
|
||||||
|
|
@ -90,7 +90,7 @@ await brain.init()
|
||||||
**Programmatic registration:** For plugins not installed as npm packages, use `brain.use()`:
|
**Programmatic registration:** For plugins not installed as npm packages, use `brain.use()`:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
import myPlugin from './my-plugin.js'
|
import myPlugin from './my-plugin.js'
|
||||||
|
|
||||||
const brain = new Brainy()
|
const brain = new Brainy()
|
||||||
|
|
@ -272,10 +272,10 @@ When provided by an optional native acceleration plugin (such as `@soulcraft/cor
|
||||||
#### `cache`
|
#### `cache`
|
||||||
**Type:** `UnifiedCache`
|
**Type:** `UnifiedCache`
|
||||||
|
|
||||||
Replaces the global `UnifiedCache` singleton used for VFS path resolution, semantic caching, and vector index caching. Must implement the `UnifiedCache` interface (available from `@soulcraft/brainy/internals`).
|
Replaces the global `UnifiedCache` singleton used for VFS path resolution, semantic caching, and vector index caching. Must implement the `UnifiedCache` interface (available from `@soulcraftlabs/brainy/internals`).
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import type { UnifiedCache } from '@soulcraft/brainy/internals'
|
import type { UnifiedCache } from '@soulcraftlabs/brainy/internals'
|
||||||
|
|
||||||
context.registerProvider('cache', myNativeCache)
|
context.registerProvider('cache', myNativeCache)
|
||||||
```
|
```
|
||||||
|
|
@ -325,8 +325,8 @@ Plugins can register custom storage backends that users reference by name.
|
||||||
### Implementing a Storage Adapter
|
### Implementing a Storage Adapter
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import type { StorageAdapterFactory } from '@soulcraft/brainy/plugin'
|
import type { StorageAdapterFactory } from '@soulcraftlabs/brainy/plugin'
|
||||||
import type { StorageAdapter } from '@soulcraft/brainy'
|
import type { StorageAdapter } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
class MyStorageAdapter implements StorageAdapter {
|
class MyStorageAdapter implements StorageAdapter {
|
||||||
async init(): Promise<void> { /* ... */ }
|
async init(): Promise<void> { /* ... */ }
|
||||||
|
|
@ -360,9 +360,9 @@ Brainy provides three entry points for plugin developers:
|
||||||
|
|
||||||
| Import Path | Contents | Stability |
|
| Import Path | Contents | Stability |
|
||||||
|-------------|----------|-----------|
|
|-------------|----------|-----------|
|
||||||
| `@soulcraft/brainy` | Public API, types, StorageAdapter | Stable (semver) |
|
| `@soulcraftlabs/brainy` | Public API, types, StorageAdapter | Stable (semver) |
|
||||||
| `@soulcraft/brainy/plugin` | BrainyPlugin, BrainyPluginContext, StorageAdapterFactory | Stable (semver) |
|
| `@soulcraftlabs/brainy/plugin` | BrainyPlugin, BrainyPluginContext, StorageAdapterFactory | Stable (semver) |
|
||||||
| `@soulcraft/brainy/internals` | UnifiedCache, EntityIdMapper, logger utilities | Internal (may change between minor versions) |
|
| `@soulcraftlabs/brainy/internals` | UnifiedCache, EntityIdMapper, logger utilities | Internal (may change between minor versions) |
|
||||||
|
|
||||||
## Diagnostics
|
## Diagnostics
|
||||||
|
|
||||||
|
|
@ -440,7 +440,7 @@ A minimal but useful plugin that provides SIMD-accelerated distance calculations
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// simd-distance-plugin/src/plugin.ts
|
// simd-distance-plugin/src/plugin.ts
|
||||||
import type { BrainyPlugin, BrainyPluginContext } from '@soulcraft/brainy/plugin'
|
import type { BrainyPlugin, BrainyPluginContext } from '@soulcraftlabs/brainy/plugin'
|
||||||
|
|
||||||
// Hypothetical native module
|
// Hypothetical native module
|
||||||
import { simdCosineDistance } from './native.js'
|
import { simdCosineDistance } from './native.js'
|
||||||
|
|
@ -470,7 +470,7 @@ export default simdDistancePlugin
|
||||||
"main": "./dist/plugin.js",
|
"main": "./dist/plugin.js",
|
||||||
"types": "./dist/plugin.d.ts",
|
"types": "./dist/plugin.d.ts",
|
||||||
"peerDependencies": {
|
"peerDependencies": {
|
||||||
"@soulcraft/brainy": ">=7.0.0"
|
"@soulcraftlabs/brainy": ">=7.0.0"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
@ -478,7 +478,7 @@ export default simdDistancePlugin
|
||||||
Usage:
|
Usage:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy({ plugins: ['brainy-simd-distance'] })
|
const brain = new Brainy({ plugins: ['brainy-simd-distance'] })
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
|
||||||
|
|
@ -54,7 +54,7 @@ After 40 API calls:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// server.ts
|
// server.ts
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
// SINGLETON INSTANCE
|
// SINGLETON INSTANCE
|
||||||
let brainInstance: Brainy | null = null
|
let brainInstance: Brainy | null = null
|
||||||
|
|
@ -174,7 +174,7 @@ process.on('SIGTERM', async () => {
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// server.ts - Clean Bun implementation
|
// server.ts - Clean Bun implementation
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
let brain: Brainy | null = null
|
let brain: Brainy | null = null
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@
|
||||||
## Quick Start
|
## Quick Start
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
|
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy()
|
const brain = new Brainy()
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
|
||||||
|
|
@ -99,7 +99,7 @@ Examples:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# 1. Deprecate wrong version on npm
|
# 1. Deprecate wrong version on npm
|
||||||
npm deprecate @soulcraft/brainy@X.X.X "Incorrect version - use Y.Y.Y"
|
npm deprecate @soulcraftlabs/brainy@X.X.X "Incorrect version - use Y.Y.Y"
|
||||||
|
|
||||||
# 2. Fix version in package.json
|
# 2. Fix version in package.json
|
||||||
# 3. Republish correct version
|
# 3. Republish correct version
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,7 @@
|
||||||
|
|
||||||
### In-Memory
|
### In-Memory
|
||||||
```typescript
|
```typescript
|
||||||
import Brainy from '@soulcraft/brainy'
|
import Brainy from '@soulcraftlabs/brainy'
|
||||||
const brain = new Brainy({ storage: { type: 'memory' } })
|
const brain = new Brainy({ storage: { type: 'memory' } })
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -43,7 +43,7 @@ The native vector provider (via the optional `@soulcraft/cor` package) extends t
|
||||||
|
|
||||||
Numbers below are **measured** by `tests/benchmarks/find-composition-scale.js` (a single
|
Numbers below are **measured** by `tests/benchmarks/find-composition-scale.js` (a single
|
||||||
Node 22 process, in-memory storage, 384-dim vectors, `balanced` recall). They are the
|
Node 22 process, in-memory storage, 384-dim vectors, `balanced` recall). They are the
|
||||||
open-core (pure-TypeScript) path — what you get from `@soulcraft/brainy` with no native
|
open-core (pure-TypeScript) path — what you get from `@soulcraftlabs/brainy` with no native
|
||||||
provider installed. Run it yourself: `node --max-old-space-size=8192 tests/benchmarks/find-composition-scale.js 100000`.
|
provider installed. Run it yourself: `node --max-old-space-size=8192 tests/benchmarks/find-composition-scale.js 100000`.
|
||||||
|
|
||||||
`find()` query latency, p50 / p95 (200 queries each):
|
`find()` query latency, p50 / p95 (200 queries each):
|
||||||
|
|
|
||||||
1544
docs/api-contract.json
Normal file
1544
docs/api-contract.json
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -24,7 +24,7 @@ next:
|
||||||
## Quick Start
|
## Quick Start
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
|
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy() // Zero config!
|
const brain = new Brainy() // Zero config!
|
||||||
await brain.init() // VFS auto-initialized!
|
await brain.init() // VFS auto-initialized!
|
||||||
|
|
@ -1010,7 +1010,7 @@ await db.release() // unpin + free cached materialization
|
||||||
|
|
||||||
### Db API errors
|
### Db API errors
|
||||||
|
|
||||||
All exported from `@soulcraft/brainy`:
|
All exported from `@soulcraftlabs/brainy`:
|
||||||
|
|
||||||
| Error | Thrown by | Meaning |
|
| Error | Thrown by | Meaning |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
|
|
@ -1918,11 +1918,11 @@ isn't serving throws instead of rebuilding mid-query:
|
||||||
| `MetadataIndexNotReadyError` | `find({ where })` | Metadata/field index isn't serving |
|
| `MetadataIndexNotReadyError` | `find({ where })` | Metadata/field index isn't serving |
|
||||||
| `VectorIndexNotReadyError` | `find({ query })`, `similar()` | Vector index isn't serving |
|
| `VectorIndexNotReadyError` | `find({ query })`, `similar()` | Vector index isn't serving |
|
||||||
|
|
||||||
All three are exported from `@soulcraft/brainy`. Catch them to distinguish
|
All three are exported from `@soulcraftlabs/brainy`. Catch them to distinguish
|
||||||
"index not ready" from a genuine empty result:
|
"index not ready" from a genuine empty result:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { MetadataIndexNotReadyError } from '@soulcraft/brainy'
|
import { MetadataIndexNotReadyError } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const rows = await brain.find({ where: { status: 'active' } })
|
const rows = await brain.find({ where: { status: 'active' } })
|
||||||
|
|
@ -2208,7 +2208,7 @@ For the full taxonomy with all 169 types and their descriptions, see:
|
||||||
- **📖 Documentation:** [Full Documentation](../)
|
- **📖 Documentation:** [Full Documentation](../)
|
||||||
- **🐛 Issues:** [GitHub Issues](https://github.com/soulcraftlabs/brainy/issues)
|
- **🐛 Issues:** [GitHub Issues](https://github.com/soulcraftlabs/brainy/issues)
|
||||||
- **💬 Discussions:** [GitHub Discussions](https://github.com/soulcraftlabs/brainy/discussions)
|
- **💬 Discussions:** [GitHub Discussions](https://github.com/soulcraftlabs/brainy/discussions)
|
||||||
- **📦 NPM:** [@soulcraft/brainy](https://www.npmjs.com/package/@soulcraft/brainy)
|
- **📦 NPM:** [@soulcraftlabs/brainy](https://www.npmjs.com/package/@soulcraftlabs/brainy)
|
||||||
- **⭐ GitHub:** [Star us](https://github.com/soulcraftlabs/brainy)
|
- **⭐ GitHub:** [Star us](https://github.com/soulcraftlabs/brainy)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
|
||||||
|
|
@ -268,7 +268,7 @@ locks/_flush_responses/ # writer answers with <uuid>.ack
|
||||||
| **Counts/statistics** | Per-type and per-subtype maps | `_system/{type,subtype,verb-subtype}-statistics.json.gz`, `counts.json` | Recomputable by scanning entities (`brainy inspect repair`) |
|
| **Counts/statistics** | Per-type and per-subtype maps | `_system/{type,subtype,verb-subtype}-statistics.json.gz`, `counts.json` | Recomputable by scanning entities (`brainy inspect repair`) |
|
||||||
|
|
||||||
A pluggable index provider (the 8.0 plugin contract in
|
A pluggable index provider (the 8.0 plugin contract in
|
||||||
`@soulcraft/brainy/plugin`) may replace any of the JS implementations; the
|
`@soulcraftlabs/brainy/plugin`) may replace any of the JS implementations; the
|
||||||
persisted formats above are contract-bound so JS and native implementations
|
persisted formats above are contract-bound so JS and native implementations
|
||||||
can interleave on the same directory.
|
can interleave on the same directory.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -126,7 +126,7 @@ class TypeAwareMetadataIndex {
|
||||||
**The Design**: Specify types clearly in your API calls:
|
**The Design**: Specify types clearly in your API calls:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
|
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
// Add entity with explicit type
|
// Add entity with explicit type
|
||||||
await brain.add({
|
await brain.add({
|
||||||
|
|
@ -231,7 +231,7 @@ class OrgEnrichmentAugmentation {
|
||||||
**Brainy's Approach**: Extract **typed** concepts:
|
**Brainy's Approach**: Extract **typed** concepts:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { NaturalLanguageProcessor } from '@soulcraft/brainy'
|
import { NaturalLanguageProcessor } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const nlp = new NaturalLanguageProcessor()
|
const nlp = new NaturalLanguageProcessor()
|
||||||
const concepts = await nlp.extractConcepts("Alice works at Google in San Francisco")
|
const concepts = await nlp.extractConcepts("Alice works at Google in San Francisco")
|
||||||
|
|
@ -382,7 +382,7 @@ import {
|
||||||
getVerbTypes,
|
getVerbTypes,
|
||||||
BrainyTypes,
|
BrainyTypes,
|
||||||
suggestType
|
suggestType
|
||||||
} from '@soulcraft/brainy'
|
} from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
// Get all available noun types
|
// Get all available noun types
|
||||||
const nounTypes = getNounTypes()
|
const nounTypes = getNounTypes()
|
||||||
|
|
|
||||||
|
|
@ -127,7 +127,7 @@ For reference, a clean migration path:
|
||||||
`isMultiProcessSafe` type-guard. Keep `hasStorageMethod` for
|
`isMultiProcessSafe` type-guard. Keep `hasStorageMethod` for
|
||||||
build/install artifact protection.
|
build/install artifact protection.
|
||||||
5. Document the new contract in `concepts/storage-adapters.md`.
|
5. Document the new contract in `concepts/storage-adapters.md`.
|
||||||
6. Major-version-bump the `@soulcraft/brainy` peerDep range expected by
|
6. Major-version-bump the `@soulcraftlabs/brainy` peerDep range expected by
|
||||||
plugins.
|
plugins.
|
||||||
|
|
||||||
Estimated work: ~half a day of code, ~2 hours of doc/example updates,
|
Estimated work: ~half a day of code, ~2 hours of doc/example updates,
|
||||||
|
|
|
||||||
|
|
@ -20,7 +20,7 @@ next:
|
||||||
Every example on this page is written against the real Brainy 8.0 API. The setup is always the same:
|
Every example on this page is written against the real Brainy 8.0 API. The setup is always the same:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
|
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy()
|
const brain = new Brainy()
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -40,7 +40,7 @@ Brainy's **Noun-Verb Taxonomy** achieves broad coverage of human knowledge throu
|
||||||
- **Multi-hop Graph Traversals = Relationship Complexity**
|
- **Multi-hop Graph Traversals = Relationship Complexity**
|
||||||
- **Result: Model data across virtually any industry**
|
- **Result: Model data across virtually any industry**
|
||||||
|
|
||||||
Every piece of information can be represented as entities (nouns) connected by relationships (verbs) carrying properties (metadata). The standardized type system from `@soulcraft/brainy` (`NounType`, `VerbType`) gives those nouns and verbs a stable, shared name.
|
Every piece of information can be represented as entities (nouns) connected by relationships (verbs) carrying properties (metadata). The standardized type system from `@soulcraftlabs/brainy` (`NounType`, `VerbType`) gives those nouns and verbs a stable, shared name.
|
||||||
|
|
||||||
## The Power of Standardization: Universal Interoperability
|
## The Power of Standardization: Universal Interoperability
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -35,7 +35,7 @@ constructor and `init()`.
|
||||||
## Instant Start
|
## Instant Start
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
// That's it. No config needed.
|
// That's it. No config needed.
|
||||||
const brain = new Brainy()
|
const brain = new Brainy()
|
||||||
|
|
|
||||||
|
|
@ -167,7 +167,7 @@ await brain.find({ orderBy: 'createdAt' })
|
||||||
`UnresolvableFieldError` is exported from the package root:
|
`UnresolvableFieldError` is exported from the package root:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { UnresolvableFieldError } from '@soulcraft/brainy'
|
import { UnresolvableFieldError } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await brain.find({ orderBy: 'createdAt' })
|
await brain.find({ orderBy: 'createdAt' })
|
||||||
|
|
|
||||||
|
|
@ -95,7 +95,7 @@ catchable error naming the reason:
|
||||||
| `MetadataIndexNotReadyError` | `find({ where })` | The metadata/field index isn't serving — a filtered read would otherwise return `[]` indistinguishable from "no matches" |
|
| `MetadataIndexNotReadyError` | `find({ where })` | The metadata/field index isn't serving — a filtered read would otherwise return `[]` indistinguishable from "no matches" |
|
||||||
| `VectorIndexNotReadyError` | `find({ query })`, `similar()` | The vector index isn't serving — a semantic search would otherwise return `[]` indistinguishable from "nothing similar" |
|
| `VectorIndexNotReadyError` | `find({ query })`, `similar()` | The vector index isn't serving — a semantic search would otherwise return `[]` indistinguishable from "nothing similar" |
|
||||||
|
|
||||||
All three are exported from `@soulcraft/brainy`. Catch them where your application
|
All three are exported from `@soulcraftlabs/brainy`. Catch them where your application
|
||||||
needs to distinguish "this index isn't ready yet" from "there's genuinely nothing
|
needs to distinguish "this index isn't ready yet" from "there's genuinely nothing
|
||||||
here" — a health dashboard, a retry policy, an operator alert. The fix is always
|
here" — a health dashboard, a retry policy, an operator alert. The fix is always
|
||||||
the same: reconcile the index, either by reopening the brain (which brings every
|
the same: reconcile the index, either by reopening the brain (which brings every
|
||||||
|
|
|
||||||
|
|
@ -61,7 +61,7 @@ The only required override is the capability flag. Returning `true` from
|
||||||
to call `acquireWriterLock()` at init.
|
to call `acquireWriterLock()` at init.
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { FileSystemStorage } from '@soulcraft/brainy'
|
import { FileSystemStorage } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
export class MmapFileSystemStorage extends FileSystemStorage {
|
export class MmapFileSystemStorage extends FileSystemStorage {
|
||||||
public supportsMultiProcessLocking(): boolean {
|
public supportsMultiProcessLocking(): boolean {
|
||||||
|
|
@ -79,7 +79,7 @@ If your storage is **not filesystem-backed** (a custom
|
||||||
network backend), extend `BaseStorage` directly:
|
network backend), extend `BaseStorage` directly:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { BaseStorage } from '@soulcraft/brainy'
|
import { BaseStorage } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
export class MyCloudStorage extends BaseStorage {
|
export class MyCloudStorage extends BaseStorage {
|
||||||
// BaseStorage's default no-op implementations of the multi-process
|
// BaseStorage's default no-op implementations of the multi-process
|
||||||
|
|
@ -101,7 +101,7 @@ The defensive check at every new-storage-method call site (`brainy.ts`,
|
||||||
`hasStorageMethod(name)`) does **not** exist to handle "plugin bundles a
|
`hasStorageMethod(name)`) does **not** exist to handle "plugin bundles a
|
||||||
stale BaseStorage." Plugins ship a dist that preserves the dynamic ESM
|
stale BaseStorage." Plugins ship a dist that preserves the dynamic ESM
|
||||||
import (verify in your plugin's `dist/`: `import { FileSystemStorage } from
|
import (verify in your plugin's `dist/`: `import { FileSystemStorage } from
|
||||||
'@soulcraft/brainy'` is not rewritten to a vendored copy). The prototype
|
'@soulcraftlabs/brainy'` is not rewritten to a vendored copy). The prototype
|
||||||
chain at runtime resolves to whatever Brainy version your consumer has
|
chain at runtime resolves to whatever Brainy version your consumer has
|
||||||
installed.
|
installed.
|
||||||
|
|
||||||
|
|
@ -109,8 +109,8 @@ installed.
|
||||||
the prototype chain at the consumer-app level:
|
the prototype chain at the consumer-app level:
|
||||||
|
|
||||||
- **Stale `node_modules`** — a lingering install from before the consumer
|
- **Stale `node_modules`** — a lingering install from before the consumer
|
||||||
upgraded Brainy. The package.json says `@soulcraft/brainy@7.22.0` but
|
upgraded Brainy. The package.json says `@soulcraftlabs/brainy@7.22.0` but
|
||||||
`node_modules/@soulcraft/brainy` is still 7.20.x.
|
`node_modules/@soulcraftlabs/brainy` is still 7.20.x.
|
||||||
- **Lockfile drift** — `bun.lockb` / `package-lock.json` pins a brainy
|
- **Lockfile drift** — `bun.lockb` / `package-lock.json` pins a brainy
|
||||||
version older than the package.json range, and `bun install` honors the
|
version older than the package.json range, and `bun install` honors the
|
||||||
lockfile.
|
lockfile.
|
||||||
|
|
@ -131,7 +131,7 @@ and the warning names the adapter class plus a remediation hint:
|
||||||
methods on its prototype chain. Writer locking and the flush-request RPC are
|
methods on its prototype chain. Writer locking and the flush-request RPC are
|
||||||
disabled for this directory. Likely fix: clean install (`rm -rf node_modules
|
disabled for this directory. Likely fix: clean install (`rm -rf node_modules
|
||||||
bun.lockb && bun install`) or rebuild your container image to refresh
|
bun.lockb && bun install`) or rebuild your container image to refresh
|
||||||
`@soulcraft/brainy` to ≥7.21. See docs/concepts/storage-adapters.md.
|
`@soulcraftlabs/brainy` to ≥7.21. See docs/concepts/storage-adapters.md.
|
||||||
```
|
```
|
||||||
|
|
||||||
## Authoring a new storage adapter — minimum checklist
|
## Authoring a new storage adapter — minimum checklist
|
||||||
|
|
@ -168,7 +168,7 @@ bun.lockb && bun install`) or rebuild your container image to refresh
|
||||||
install time — fix install, not your plugin.
|
install time — fix install, not your plugin.
|
||||||
|
|
||||||
6. **Pin your peer dep generously.** `"peerDependencies": {
|
6. **Pin your peer dep generously.** `"peerDependencies": {
|
||||||
"@soulcraft/brainy": "^7.21.0" }` accepts any compatible 7.x. Don't pin
|
"@soulcraftlabs/brainy": "^7.21.0" }` accepts any compatible 7.x. Don't pin
|
||||||
to an exact patch unless you're tracking a known regression.
|
to an exact patch unless you're tracking a known regression.
|
||||||
|
|
||||||
## Future direction
|
## Future direction
|
||||||
|
|
@ -185,5 +185,5 @@ follow-up; consumers don't need to anticipate the change.
|
||||||
heartbeat semantics, what the lock protects.
|
heartbeat semantics, what the lock protects.
|
||||||
- [`guides/inspection`](../guides/inspection.md) — `brainy inspect` and the
|
- [`guides/inspection`](../guides/inspection.md) — `brainy inspect` and the
|
||||||
read-only mode.
|
read-only mode.
|
||||||
- `node_modules/@soulcraft/brainy/dist/storage/baseStorage.d.ts` — the
|
- `node_modules/@soulcraftlabs/brainy/dist/storage/baseStorage.d.ts` — the
|
||||||
authoritative type signatures for every method this page references.
|
authoritative type signatures for every method this page references.
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,7 @@ they share a single scan.
|
||||||
## Quick Start
|
## Quick Start
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy, NounType } from '@soulcraft/brainy'
|
import { Brainy, NounType } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy()
|
const brain = new Brainy()
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@ Brainy is **framework-friendly** - designed to drop into the server side of any
|
||||||
|
|
||||||
Brainy embeds an HNSW vector index, a graph engine, and a filesystem-backed persistence layer. These belong on the server:
|
Brainy embeds an HNSW vector index, a graph engine, and a filesystem-backed persistence layer. These belong on the server:
|
||||||
|
|
||||||
- **Zero configuration**: Just `import { Brainy } from '@soulcraft/brainy'`
|
- **Zero configuration**: Just `import { Brainy } from '@soulcraftlabs/brainy'`
|
||||||
- **Auto storage detection**: `new Brainy()` auto-selects filesystem persistence on Node
|
- **Auto storage detection**: `new Brainy()` auto-selects filesystem persistence on Node
|
||||||
- **Cleaner code**: No browser polyfills, no conditional client/server imports
|
- **Cleaner code**: No browser polyfills, no conditional client/server imports
|
||||||
- **Better DX**: One instance shared across your server routes
|
- **Better DX**: One instance shared across your server routes
|
||||||
|
|
@ -18,13 +18,13 @@ Brainy embeds an HNSW vector index, a graph engine, and a filesystem-backed pers
|
||||||
### Install Brainy
|
### Install Brainy
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
npm install @soulcraft/brainy
|
npm install @soulcraftlabs/brainy
|
||||||
```
|
```
|
||||||
|
|
||||||
### Basic Integration
|
### Basic Integration
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
// Run on the server (API route, server component, backend service)
|
// Run on the server (API route, server component, backend service)
|
||||||
// new Brainy() auto-detects filesystem persistence on Node
|
// new Brainy() auto-detects filesystem persistence on Node
|
||||||
|
|
@ -105,7 +105,7 @@ On the server, create one Brainy instance and reuse it across requests. This mod
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// lib/brain.server.js
|
// lib/brain.server.js
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
let brainPromise
|
let brainPromise
|
||||||
|
|
||||||
|
|
@ -163,7 +163,7 @@ On the server, create one Brainy instance and reuse it across requests:
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// server/brain.js (server-only module)
|
// server/brain.js (server-only module)
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
let brainPromise
|
let brainPromise
|
||||||
|
|
||||||
|
|
@ -248,7 +248,7 @@ The matching backend endpoint uses Brainy directly (Node/Bun):
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// server: api/search
|
// server: api/search
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy() // auto-detects filesystem persistence on Node
|
const brain = new Brainy() // auto-detects filesystem persistence on Node
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -266,7 +266,7 @@ In Next.js, Brainy lives in server code only: API routes, server components, or
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// lib/brain.server.js (imported only by server code)
|
// lib/brain.server.js (imported only by server code)
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
let brainPromise
|
let brainPromise
|
||||||
|
|
||||||
|
|
@ -318,7 +318,7 @@ Brainy runs in a server-only module (`*.server.js`); the component fetches resul
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// src/lib/server/brain.js (server-only — note the .server suffix)
|
// src/lib/server/brain.js (server-only — note the .server suffix)
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
let brainPromise
|
let brainPromise
|
||||||
|
|
||||||
|
|
@ -432,7 +432,7 @@ import { defineConfig } from 'vite'
|
||||||
|
|
||||||
export default defineConfig({
|
export default defineConfig({
|
||||||
ssr: {
|
ssr: {
|
||||||
external: ['@soulcraft/brainy']
|
external: ['@soulcraftlabs/brainy']
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
@ -440,7 +440,7 @@ export default defineConfig({
|
||||||
```javascript
|
```javascript
|
||||||
// rollup.config.js (server bundle)
|
// rollup.config.js (server bundle)
|
||||||
export default {
|
export default {
|
||||||
external: ['@soulcraft/brainy', 'node:fs', 'node:path', 'node:crypto']
|
external: ['@soulcraftlabs/brainy', 'node:fs', 'node:path', 'node:crypto']
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -466,7 +466,7 @@ export async function load({ url }) {
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// For build-time usage (runs in Node during the build)
|
// For build-time usage (runs in Node during the build)
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
export async function generateStaticProps() {
|
export async function generateStaticProps() {
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
|
|
@ -513,7 +513,7 @@ export async function generateStaticProps() {
|
||||||
|
|
||||||
### Issue: Large client bundle size
|
### Issue: Large client bundle size
|
||||||
**Cause**: A client module is pulling in Brainy.
|
**Cause**: A client module is pulling in Brainy.
|
||||||
**Solution**: Move the `import { Brainy } from '@soulcraft/brainy'` into a server-only module so it never reaches the browser bundle.
|
**Solution**: Move the `import { Brainy } from '@soulcraftlabs/brainy'` into a server-only module so it never reaches the browser bundle.
|
||||||
|
|
||||||
### Issue: SSR hydration mismatch
|
### Issue: SSR hydration mismatch
|
||||||
**Solution**: Run the search on the server (loader / server action / API route) and pass the results down as props, so server and client render the same markup.
|
**Solution**: Run the search on the server (loader / server action / API route) and pass the results down as props, so server and client render the same markup.
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,7 @@ Brainy's import is **ONE magical method** that understands EVERYTHING:
|
||||||
## The Ultimate Simplicity
|
## The Ultimate Simplicity
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy()
|
const brain = new Brainy()
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,7 @@ Brainy provides real-time progress tracking for **all 7 supported file formats**
|
||||||
### Basic Progress Tracking
|
### Basic Progress Tracking
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
import * as fs from 'fs'
|
import * as fs from 'fs'
|
||||||
|
|
||||||
const brain = await Brainy.create()
|
const brain = await Brainy.create()
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@
|
||||||
## Basic Import
|
## Basic Import
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy()
|
const brain = new Brainy()
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -187,7 +187,7 @@ await brain.import(file, {
|
||||||
## Complete Example
|
## Complete Example
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
import * as fs from 'fs'
|
import * as fs from 'fs'
|
||||||
|
|
||||||
async function importCatalog() {
|
async function importCatalog() {
|
||||||
|
|
|
||||||
|
|
@ -108,7 +108,7 @@ check fails — useful for piping into monitoring or CI.
|
||||||
## Programmatic inspection
|
## Programmatic inspection
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const reader = await Brainy.openReadOnly({
|
const reader = await Brainy.openReadOnly({
|
||||||
storage: { type: 'filesystem', path: '/data/brain' }
|
storage: { type: 'filesystem', path: '/data/brain' }
|
||||||
|
|
|
||||||
|
|
@ -21,21 +21,21 @@ next:
|
||||||
## Install
|
## Install
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
npm install @soulcraft/brainy
|
npm install @soulcraftlabs/brainy
|
||||||
```
|
```
|
||||||
|
|
||||||
Or with your preferred package manager:
|
Or with your preferred package manager:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
bun add @soulcraft/brainy
|
bun add @soulcraftlabs/brainy
|
||||||
yarn add @soulcraft/brainy
|
yarn add @soulcraftlabs/brainy
|
||||||
pnpm add @soulcraft/brainy
|
pnpm add @soulcraftlabs/brainy
|
||||||
```
|
```
|
||||||
|
|
||||||
## Verify
|
## Verify
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy()
|
const brain = new Brainy()
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -52,7 +52,7 @@ npm install @soulcraft/cor
|
||||||
```
|
```
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy({ plugins: ['@soulcraft/cor'] })
|
const brain = new Brainy({ plugins: ['@soulcraft/cor'] })
|
||||||
await brain.init() // native providers registered during init
|
await brain.init() // native providers registered during init
|
||||||
|
|
@ -71,7 +71,7 @@ remains available on npm if you need it.
|
||||||
Brainy ships with full TypeScript types. No `@types/` package needed:
|
Brainy ships with full TypeScript types. No `@types/` package needed:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
|
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy()
|
const brain = new Brainy()
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
|
||||||
|
|
@ -66,7 +66,7 @@ const results = await brain.search("query")
|
||||||
**New diagnostics for capacity planning and performance tuning.**
|
**New diagnostics for capacity planning and performance tuning.**
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy()
|
const brain = new Brainy()
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -112,7 +112,7 @@ Recommendations: ${stats.recommendations.join(', ')}
|
||||||
### Step 1: Update Package
|
### Step 1: Update Package
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
npm install @soulcraft/brainy@latest
|
npm install @soulcraftlabs/brainy@latest
|
||||||
```
|
```
|
||||||
|
|
||||||
### Step 2: Restart Your Application
|
### Step 2: Restart Your Application
|
||||||
|
|
@ -134,7 +134,7 @@ npm run start
|
||||||
### Check Adaptive Sizing is Working
|
### Check Adaptive Sizing is Working
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy()
|
const brain = new Brainy()
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -218,7 +218,7 @@ For debugging or compatibility testing:
|
||||||
If you need to rollback to v3.35.0:
|
If you need to rollback to v3.35.0:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
npm install @soulcraft/brainy@3.35.0
|
npm install @soulcraftlabs/brainy@3.35.0
|
||||||
```
|
```
|
||||||
|
|
||||||
**Note:** We don't anticipate any issues, but rollback is straightforward if needed.
|
**Note:** We don't anticipate any issues, but rollback is straightforward if needed.
|
||||||
|
|
@ -367,7 +367,7 @@ if (stats.fairness.fairnessViolation) {
|
||||||
|
|
||||||
## Next Steps
|
## Next Steps
|
||||||
|
|
||||||
1. ✅ **Upgrade:** `npm install @soulcraft/brainy@latest`
|
1. ✅ **Upgrade:** `npm install @soulcraftlabs/brainy@latest`
|
||||||
2. 📊 **Monitor:** Use `getCacheStats()` to verify performance improvements
|
2. 📊 **Monitor:** Use `getCacheStats()` to verify performance improvements
|
||||||
3. 🎯 **Tune:** Adjust based on recommendations (if needed)
|
3. 🎯 **Tune:** Adjust based on recommendations (if needed)
|
||||||
4. 📖 **Read:** [Operations Guide](../operations/capacity-planning.md) for capacity planning
|
4. 📖 **Read:** [Operations Guide](../operations/capacity-planning.md) for capacity planning
|
||||||
|
|
|
||||||
|
|
@ -37,7 +37,7 @@ This single WASM file contains everything needed for sentence embeddings.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Bun as a runtime — supported and recommended
|
# Bun as a runtime — supported and recommended
|
||||||
bun add @soulcraft/brainy
|
bun add @soulcraftlabs/brainy
|
||||||
bun run server.ts
|
bun run server.ts
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -80,7 +80,7 @@ If you read raw stored records (fact-log scanners, export tooling), use
|
||||||
the exported shape-aware splitters — they handle both record eras:
|
the exported shape-aware splitters — they handle both record eras:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { splitNounMetadataRecord } from '@soulcraft/brainy'
|
import { splitNounMetadataRecord } from '@soulcraftlabs/brainy'
|
||||||
const { reserved, custom } = splitNounMetadataRecord(rawRecord)
|
const { reserved, custom } = splitNounMetadataRecord(rawRecord)
|
||||||
// reserved = engine fields · custom = the user's bag, ANY names
|
// reserved = engine fields · custom = the user's bag, ANY names
|
||||||
```
|
```
|
||||||
|
|
@ -88,7 +88,7 @@ const { reserved, custom } = splitNounMetadataRecord(rawRecord)
|
||||||
Feature detection (never version-sniff):
|
Feature detection (never version-sniff):
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import * as brainy from '@soulcraft/brainy'
|
import * as brainy from '@soulcraftlabs/brainy'
|
||||||
const lawActive = 'FIELD_ADDRESSING_CAPABILITY' in brainy // 'field-addressing/v1'
|
const lawActive = 'FIELD_ADDRESSING_CAPABILITY' in brainy // 'field-addressing/v1'
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,7 @@ Complete guide to integrating Brainy with Next.js applications, covering App Rou
|
||||||
```bash
|
```bash
|
||||||
npx create-next-app@latest my-brainy-app
|
npx create-next-app@latest my-brainy-app
|
||||||
cd my-brainy-app
|
cd my-brainy-app
|
||||||
npm install @soulcraft/brainy
|
npm install @soulcraftlabs/brainy
|
||||||
```
|
```
|
||||||
|
|
||||||
### Basic Setup
|
### Basic Setup
|
||||||
|
|
@ -18,7 +18,7 @@ npm install @soulcraft/brainy
|
||||||
// app/components/BrainyProvider.jsx
|
// app/components/BrainyProvider.jsx
|
||||||
'use client'
|
'use client'
|
||||||
import { createContext, useContext, useEffect, useState } from 'react'
|
import { createContext, useContext, useEffect, useState } from 'react'
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const BrainyContext = createContext()
|
const BrainyContext = createContext()
|
||||||
|
|
||||||
|
|
@ -271,7 +271,7 @@ export default function SearchPage() {
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// app/api/search/route.js (App Router)
|
// app/api/search/route.js (App Router)
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
let brain = null
|
let brain = null
|
||||||
|
|
||||||
|
|
@ -332,7 +332,7 @@ export async function GET() {
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// pages/api/search.js (Pages Router)
|
// pages/api/search.js (Pages Router)
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
let brain = null
|
let brain = null
|
||||||
|
|
||||||
|
|
@ -374,7 +374,7 @@ export default async function handler(req, res) {
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// app/api/data/route.js
|
// app/api/data/route.js
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
let brain = null
|
let brain = null
|
||||||
|
|
||||||
|
|
@ -418,7 +418,7 @@ export async function POST(request) {
|
||||||
```jsx
|
```jsx
|
||||||
// app/actions/brainy.js
|
// app/actions/brainy.js
|
||||||
'use server'
|
'use server'
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
let brain = null
|
let brain = null
|
||||||
|
|
||||||
|
|
@ -630,7 +630,7 @@ CMD ["npm", "start"]
|
||||||
/** @type {import('next').NextConfig} */
|
/** @type {import('next').NextConfig} */
|
||||||
const nextConfig = {
|
const nextConfig = {
|
||||||
experimental: {
|
experimental: {
|
||||||
serverComponentsExternalPackages: ['@soulcraft/brainy']
|
serverComponentsExternalPackages: ['@soulcraftlabs/brainy']
|
||||||
},
|
},
|
||||||
webpack: (config, { isServer }) => {
|
webpack: (config, { isServer }) => {
|
||||||
if (!isServer) {
|
if (!isServer) {
|
||||||
|
|
@ -797,7 +797,7 @@ export function rateLimit(req, limit = 100, window = 60000) {
|
||||||
// app/contexts/BrainyContext.jsx
|
// app/contexts/BrainyContext.jsx
|
||||||
'use client'
|
'use client'
|
||||||
import { createContext, useContext, useReducer, useEffect } from 'react'
|
import { createContext, useContext, useReducer, useEffect } from 'react'
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const BrainyContext = createContext()
|
const BrainyContext = createContext()
|
||||||
|
|
||||||
|
|
@ -873,7 +873,7 @@ import { BrainyProvider } from '../app/components/BrainyProvider'
|
||||||
import { Search } from '../app/components/Search'
|
import { Search } from '../app/components/Search'
|
||||||
|
|
||||||
// Mock Brainy
|
// Mock Brainy
|
||||||
jest.mock('@soulcraft/brainy', () => ({
|
jest.mock('@soulcraftlabs/brainy', () => ({
|
||||||
Brainy: jest.fn().mockImplementation(() => ({
|
Brainy: jest.fn().mockImplementation(() => ({
|
||||||
init: jest.fn().mockResolvedValue(undefined),
|
init: jest.fn().mockResolvedValue(undefined),
|
||||||
find: jest.fn().mockResolvedValue([
|
find: jest.fn().mockResolvedValue([
|
||||||
|
|
|
||||||
|
|
@ -32,7 +32,7 @@ Brainy 7.31.0 adds a per-entity revision counter so multiple writers can coordin
|
||||||
Every distributed-job scheduler eventually wants this exact loop:
|
Every distributed-job scheduler eventually wants this exact loop:
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
import { Brainy, RevisionConflictError } from '@soulcraft/brainy'
|
import { Brainy, RevisionConflictError } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const LOCK_ID = '...uuid for this job slot...'
|
const LOCK_ID = '...uuid for this job slot...'
|
||||||
|
|
||||||
|
|
@ -137,7 +137,7 @@ await brain.addIfMissing({ // ← not a real API
|
||||||
It's race-prone as a plain read-then-write: two concurrent imports both see "not found," both insert, you get duplicates. Without a unique-index primitive (which Brainy doesn't have today), close the race with whole-store CAS — read at a pinned generation, then commit only if nothing moved:
|
It's race-prone as a plain read-then-write: two concurrent imports both see "not found," both insert, you get duplicates. Without a unique-index primitive (which Brainy doesn't have today), close the race with whole-store CAS — read at a pinned generation, then commit only if nothing moved:
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
import { GenerationConflictError } from '@soulcraft/brainy'
|
import { GenerationConflictError } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
async function addIfMissingByEmail(email: string, data: string) {
|
async function addIfMissingByEmail(email: string, data: string) {
|
||||||
for (let attempt = 0; attempt < 5; attempt++) {
|
for (let attempt = 0; attempt < 5; attempt++) {
|
||||||
|
|
|
||||||
|
|
@ -18,13 +18,13 @@ Get Brainy running in under a minute.
|
||||||
## 1. Install
|
## 1. Install
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
npm install @soulcraft/brainy
|
npm install @soulcraftlabs/brainy
|
||||||
```
|
```
|
||||||
|
|
||||||
## 2. Initialize
|
## 2. Initialize
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
|
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy()
|
const brain = new Brainy()
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -67,7 +67,7 @@ await brain.relate({
|
||||||
## 5. Query with Triple Intelligence
|
## 5. Query with Triple Intelligence
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import type { Result } from '@soulcraft/brainy'
|
import type { Result } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
// All three search paradigms in one call
|
// All three search paradigms in one call
|
||||||
const results: Result[] = await brain.find({
|
const results: Result[] = await brain.find({
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,7 @@
|
||||||
### One Interface for Everything
|
### One Interface for Everything
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = await Brainy.create()
|
const brain = await Brainy.create()
|
||||||
|
|
||||||
|
|
@ -78,7 +78,7 @@ interface ImportProgress {
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { useState } from 'react'
|
import { useState } from 'react'
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
function UniversalImportProgress({ file }: { file: File }) {
|
function UniversalImportProgress({ file }: { file: File }) {
|
||||||
const [progress, setProgress] = useState({
|
const [progress, setProgress] = useState({
|
||||||
|
|
@ -177,7 +177,7 @@ function UniversalImportProgress({ file }: { file: File }) {
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import ora from 'ora'
|
import ora from 'ora'
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
async function importWithProgress(filePath: string) {
|
async function importWithProgress(filePath: string) {
|
||||||
const spinner = ora('Starting import...').start()
|
const spinner = ora('Starting import...').start()
|
||||||
|
|
|
||||||
|
|
@ -28,7 +28,7 @@ on-disk layout (memory's "disk" is a JS Map).
|
||||||
## Quick start
|
## Quick start
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
// Filesystem (recommended for any persistent workload):
|
// Filesystem (recommended for any persistent workload):
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
|
|
@ -134,7 +134,7 @@ config; the `type` is optional.
|
||||||
If you want to skip the factory:
|
If you want to skip the factory:
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
import { FileSystemStorage, MemoryStorage } from '@soulcraft/brainy'
|
import { FileSystemStorage, MemoryStorage } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const fsStorage = new FileSystemStorage('./brainy-data')
|
const fsStorage = new FileSystemStorage('./brainy-data')
|
||||||
const memStorage = new MemoryStorage()
|
const memStorage = new MemoryStorage()
|
||||||
|
|
|
||||||
|
|
@ -34,7 +34,7 @@ Three layers solve this:
|
||||||
### Write
|
### Write
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy, NounType } from '@soulcraft/brainy'
|
import { Brainy, NounType } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy()
|
const brain = new Brainy()
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -240,7 +240,7 @@ await brain.migrateField({
|
||||||
A realistic adoption sequence for a brain that started without these primitives:
|
A realistic adoption sequence for a brain that started without these primitives:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy, NounType } from '@soulcraft/brainy'
|
import { Brainy, NounType } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy({ storage: { type: 'filesystem', path: './brain-data' } })
|
const brain = new Brainy({ storage: { type: 'filesystem', path: './brain-data' } })
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
|
||||||
|
|
@ -25,7 +25,7 @@ content — and how 8.0 recovers it for you.
|
||||||
|
|
||||||
## TL;DR
|
## TL;DR
|
||||||
|
|
||||||
- **Just upgrade to `@soulcraft/brainy@8.0.12` (or later) and open the store.**
|
- **Just upgrade to `@soulcraftlabs/brainy@8.0.12` (or later) and open the store.**
|
||||||
If a previous upgrade left VFS content stranded, 8.0.12 **heals it on open**,
|
If a previous upgrade left VFS content stranded, 8.0.12 **heals it on open**,
|
||||||
with no operator action.
|
with no operator action.
|
||||||
- Want to force or script it? Call **`await brain.vfs.adoptOrphanedBlobs()`**.
|
- Want to force or script it? Call **`await brain.vfs.adoptOrphanedBlobs()`**.
|
||||||
|
|
@ -90,7 +90,7 @@ So the operator action for a stranded store is simply: **upgrade to 8.0.12 and
|
||||||
open it.**
|
open it.**
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
// Opening the store is all that is required — recovery runs during init().
|
// Opening the store is all that is required — recovery runs during init().
|
||||||
const brain = new Brainy({ storage: { type: 'filesystem', path: '/data/my-store' } })
|
const brain = new Brainy({ storage: { type: 'filesystem', path: '/data/my-store' } })
|
||||||
|
|
@ -182,5 +182,5 @@ and opening each store is sufficient.
|
||||||
The recovery is copy-only, so no rollback of the recovery itself is ever needed.
|
The recovery is copy-only, so no rollback of the recovery itself is ever needed.
|
||||||
If you need to roll back the **whole** 7→8 upgrade, restore the directory from
|
If you need to roll back the **whole** 7→8 upgrade, restore the directory from
|
||||||
your pre-upgrade backup (retained automatically while recovery is incomplete, or
|
your pre-upgrade backup (retained automatically while recovery is incomplete, or
|
||||||
your own snapshot) and pin `@soulcraft/brainy@7.x`. 8.0 does not keep the old
|
your own snapshot) and pin `@soulcraftlabs/brainy@7.x`. 8.0 does not keep the old
|
||||||
branch layout in place, so a directory-level restore is the rollback path.
|
branch layout in place, so a directory-level restore is the rollback path.
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,7 @@ Complete guide to integrating Brainy with Vue.js applications, covering Vue 3, N
|
||||||
npm create vue@latest my-brainy-app
|
npm create vue@latest my-brainy-app
|
||||||
cd my-brainy-app
|
cd my-brainy-app
|
||||||
npm install
|
npm install
|
||||||
npm install @soulcraft/brainy
|
npm install @soulcraftlabs/brainy
|
||||||
```
|
```
|
||||||
|
|
||||||
### Basic Setup
|
### Basic Setup
|
||||||
|
|
@ -574,7 +574,7 @@ Nuxt's server engine (Nitro) is the natural home for Brainy: it runs on Node/Bun
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// server/utils/brain.js (server-only — Nitro never bundles this into the client)
|
// server/utils/brain.js (server-only — Nitro never bundles this into the client)
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
let brainPromise
|
let brainPromise
|
||||||
|
|
||||||
|
|
@ -1201,7 +1201,7 @@ import vue from '@vitejs/plugin-vue'
|
||||||
export default defineConfig({
|
export default defineConfig({
|
||||||
plugins: [vue()],
|
plugins: [vue()],
|
||||||
ssr: {
|
ssr: {
|
||||||
external: ['@soulcraft/brainy']
|
external: ['@soulcraftlabs/brainy']
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
|
||||||
|
|
@ -24,7 +24,7 @@ Brainy's neural extraction system uses a **4-signal ensemble architecture** to c
|
||||||
### Method 1: Brain Instance (Recommended)
|
### Method 1: Brain Instance (Recommended)
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy, NounType } from '@soulcraft/brainy'
|
import { Brainy, NounType } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy()
|
const brain = new Brainy()
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -62,9 +62,9 @@ const people = await brain.extractEntities('...', {
|
||||||
import {
|
import {
|
||||||
SmartExtractor,
|
SmartExtractor,
|
||||||
SmartRelationshipExtractor
|
SmartRelationshipExtractor
|
||||||
} from '@soulcraft/brainy'
|
} from '@soulcraftlabs/brainy'
|
||||||
// Or use subpath imports:
|
// Or use subpath imports:
|
||||||
import { SmartExtractor } from '@soulcraft/brainy/neural/SmartExtractor'
|
import { SmartExtractor } from '@soulcraftlabs/brainy/neural/SmartExtractor'
|
||||||
|
|
||||||
const brain = new Brainy()
|
const brain = new Brainy()
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -176,7 +176,7 @@ const withVectors = await brain.extractEntities(text, {
|
||||||
**Direct entity type classifier.** Use when you have pre-detected candidates or need custom configuration.
|
**Direct entity type classifier.** Use when you have pre-detected candidates or need custom configuration.
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { SmartExtractor, FormatContext } from '@soulcraft/brainy'
|
import { SmartExtractor, FormatContext } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const extractor = new SmartExtractor(brain, {
|
const extractor = new SmartExtractor(brain, {
|
||||||
minConfidence: 0.7, // Threshold
|
minConfidence: 0.7, // Threshold
|
||||||
|
|
@ -229,7 +229,7 @@ interface ExtractionResult {
|
||||||
**Relationship type classifier.** Determines verb/relationship types between entities.
|
**Relationship type classifier.** Determines verb/relationship types between entities.
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { SmartRelationshipExtractor } from '@soulcraft/brainy'
|
import { SmartRelationshipExtractor } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const relExtractor = new SmartRelationshipExtractor(brain, {
|
const relExtractor = new SmartRelationshipExtractor(brain, {
|
||||||
minConfidence: 0.6,
|
minConfidence: 0.6,
|
||||||
|
|
@ -286,7 +286,7 @@ const rel = await relExtractor.infer(
|
||||||
**Full extraction orchestrator.** Handles candidate detection, classification, and deduplication.
|
**Full extraction orchestrator.** Handles candidate detection, classification, and deduplication.
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { NeuralEntityExtractor } from '@soulcraft/brainy'
|
import { NeuralEntityExtractor } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const extractor = new NeuralEntityExtractor(brain)
|
const extractor = new NeuralEntityExtractor(brain)
|
||||||
|
|
||||||
|
|
@ -607,7 +607,7 @@ const locations = entities.filter(e => e.type === NounType.Location)
|
||||||
### Example 2: Excel Data Classification
|
### Example 2: Excel Data Classification
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { SmartExtractor } from '@soulcraft/brainy'
|
import { SmartExtractor } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const extractor = new SmartExtractor(brain)
|
const extractor = new SmartExtractor(brain)
|
||||||
|
|
||||||
|
|
@ -629,7 +629,7 @@ for (let i = 0; i < cells.length; i++) {
|
||||||
### Example 3: Relationship Extraction
|
### Example 3: Relationship Extraction
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { SmartRelationshipExtractor } from '@soulcraft/brainy'
|
import { SmartRelationshipExtractor } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const relExtractor = new SmartRelationshipExtractor(brain)
|
const relExtractor = new SmartRelationshipExtractor(brain)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -204,8 +204,8 @@ await brain.add({ data: { name: 'Entity' }, type: NounType.Thing })
|
||||||
### Basic Add Operation
|
### Basic Add Operation
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
import { NounType } from '@soulcraft/brainy/types'
|
import { NounType } from '@soulcraftlabs/brainy/types'
|
||||||
|
|
||||||
const brain = new Brainy()
|
const brain = new Brainy()
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -428,7 +428,7 @@ await brain.relate({ ... }) // a crash here leaves the entity unlinked
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { describe, it, expect } from 'vitest'
|
import { describe, it, expect } from 'vitest'
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
describe('Transaction Tests', () => {
|
describe('Transaction Tests', () => {
|
||||||
it('should rollback on failure', async () => {
|
it('should rollback on failure', async () => {
|
||||||
|
|
|
||||||
|
|
@ -23,7 +23,7 @@ The Universal Display Augmentation is a powerful AI-powered system that automati
|
||||||
### Basic Usage
|
### Basic Usage
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brainy = new Brainy()
|
const brainy = new Brainy()
|
||||||
await brainy.init()
|
await brainy.init()
|
||||||
|
|
|
||||||
|
|
@ -71,9 +71,9 @@ Let's build a projection that organizes files by priority (high, medium, low):
|
||||||
### Step 1: Create the Strategy Class
|
### Step 1: Create the Strategy Class
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { BaseProjectionStrategy } from '@soulcraft/brainy/vfs/semantic'
|
import { BaseProjectionStrategy } from '@soulcraftlabs/brainy/vfs/semantic'
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
import { VirtualFileSystem, VFSEntity } from '@soulcraft/brainy/vfs'
|
import { VirtualFileSystem, VFSEntity } from '@soulcraftlabs/brainy/vfs'
|
||||||
|
|
||||||
export class PriorityProjection extends BaseProjectionStrategy {
|
export class PriorityProjection extends BaseProjectionStrategy {
|
||||||
readonly name = 'priority'
|
readonly name = 'priority'
|
||||||
|
|
@ -141,7 +141,7 @@ export class PriorityProjection extends BaseProjectionStrategy {
|
||||||
### Step 2: Register the Strategy
|
### Step 2: Register the Strategy
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
import { PriorityProjection } from './PriorityProjection'
|
import { PriorityProjection } from './PriorityProjection'
|
||||||
|
|
||||||
const brain = new Brainy()
|
const brain = new Brainy()
|
||||||
|
|
@ -537,7 +537,7 @@ Use the projection's resolve cache:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { describe, it, expect, beforeAll } from 'vitest'
|
import { describe, it, expect, beforeAll } from 'vitest'
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
import { PriorityProjection } from './PriorityProjection'
|
import { PriorityProjection } from './PriorityProjection'
|
||||||
|
|
||||||
describe('PriorityProjection', () => {
|
describe('PriorityProjection', () => {
|
||||||
|
|
@ -714,7 +714,7 @@ async resolve(brain, vfs, value: string) {
|
||||||
3. Use appropriate limits: Don't fetch more than needed
|
3. Use appropriate limits: Don't fetch more than needed
|
||||||
|
|
||||||
### Type errors
|
### Type errors
|
||||||
1. Import correct types: `import { Brainy, VirtualFileSystem } from '@soulcraft/brainy'`
|
1. Import correct types: `import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy'`
|
||||||
2. Use `as VFSEntity` when mapping results
|
2. Use `as VFSEntity` when mapping results
|
||||||
3. Check BaseProjectionStrategy import
|
3. Check BaseProjectionStrategy import
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -14,11 +14,11 @@ A file explorer that:
|
||||||
## ⚡ Step 1: Basic Setup (1 minute)
|
## ⚡ Step 1: Basic Setup (1 minute)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
npm install @soulcraft/brainy
|
npm install @soulcraftlabs/brainy
|
||||||
```
|
```
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
// ✅ CORRECT: Use filesystem storage for production
|
// ✅ CORRECT: Use filesystem storage for production
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
|
|
@ -115,7 +115,7 @@ Here's a complete React component using the correct patterns:
|
||||||
|
|
||||||
```tsx
|
```tsx
|
||||||
import React, { useState, useEffect } from 'react'
|
import React, { useState, useEffect } from 'react'
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
export function FileExplorer() {
|
export function FileExplorer() {
|
||||||
const [brain, setBrain] = useState(null)
|
const [brain, setBrain] = useState(null)
|
||||||
|
|
@ -288,8 +288,8 @@ Your file explorer is now working! Here's what to explore next:
|
||||||
### "Module not found" errors
|
### "Module not found" errors
|
||||||
```bash
|
```bash
|
||||||
# Make sure you're using the right import
|
# Make sure you're using the right import
|
||||||
npm ls @soulcraft/brainy # Check version
|
npm ls @soulcraftlabs/brainy # Check version
|
||||||
npm install @soulcraft/brainy@latest # Update if needed
|
npm install @soulcraftlabs/brainy@latest # Update if needed
|
||||||
```
|
```
|
||||||
|
|
||||||
### "VFS not initialized" errors
|
### "VFS not initialized" errors
|
||||||
|
|
|
||||||
|
|
@ -24,7 +24,7 @@ Brainy VFS is a revolutionary virtual filesystem that runs on top of Brainy's ne
|
||||||
## Quick Start
|
## Quick Start
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
import { VirtualFileSystem } from '@soulcraft/brainy/vfs'
|
import { VirtualFileSystem } from '@soulcraftlabs/brainy/vfs'
|
||||||
|
|
||||||
// Initialize the VFS
|
// Initialize the VFS
|
||||||
const vfs = new VirtualFileSystem({
|
const vfs = new VirtualFileSystem({
|
||||||
|
|
@ -381,7 +381,7 @@ Brainy VFS fully leverages Brainy's revolutionary Triple Intelligence system:
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
npm install @soulcraft/brainy
|
npm install @soulcraftlabs/brainy
|
||||||
```
|
```
|
||||||
|
|
||||||
## Requirements
|
## Requirements
|
||||||
|
|
|
||||||
|
|
@ -135,7 +135,7 @@ Mount VFS as a native filesystem on Linux/Mac/Windows.
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Planned (research phase)
|
// Planned (research phase)
|
||||||
import { mountVFS } from '@soulcraft/brainy/vfs/fuse'
|
import { mountVFS } from '@soulcraftlabs/brainy/vfs/fuse'
|
||||||
|
|
||||||
await mountVFS(vfs, {
|
await mountVFS(vfs, {
|
||||||
mountPoint: '/mnt/brainy',
|
mountPoint: '/mnt/brainy',
|
||||||
|
|
@ -160,7 +160,7 @@ These features would benefit from community contributions. If you're interested
|
||||||
### Express.js Static Middleware
|
### Express.js Static Middleware
|
||||||
```typescript
|
```typescript
|
||||||
// Wanted: Community contribution
|
// Wanted: Community contribution
|
||||||
import { createStaticMiddleware } from '@soulcraft/brainy/vfs/express'
|
import { createStaticMiddleware } from '@soulcraftlabs/brainy/vfs/express'
|
||||||
|
|
||||||
app.use('/files', createStaticMiddleware(vfs, {
|
app.use('/files', createStaticMiddleware(vfs, {
|
||||||
index: ['index.html', 'index.md'],
|
index: ['index.html', 'index.md'],
|
||||||
|
|
@ -172,7 +172,7 @@ app.use('/files', createStaticMiddleware(vfs, {
|
||||||
### VSCode Extension
|
### VSCode Extension
|
||||||
```typescript
|
```typescript
|
||||||
// Wanted: Community contribution
|
// Wanted: Community contribution
|
||||||
import { VFSProvider } from '@soulcraft/brainy/vfs/vscode'
|
import { VFSProvider } from '@soulcraftlabs/brainy/vfs/vscode'
|
||||||
|
|
||||||
const provider = new VFSProvider(vfs)
|
const provider = new VFSProvider(vfs)
|
||||||
vscode.workspace.registerFileSystemProvider('brainy', provider)
|
vscode.workspace.registerFileSystemProvider('brainy', provider)
|
||||||
|
|
|
||||||
|
|
@ -327,7 +327,7 @@ console.log(id1 === id2 && id2 === id3) // true
|
||||||
Create your own semantic dimensions:
|
Create your own semantic dimensions:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { BaseProjectionStrategy } from '@soulcraft/brainy/vfs/semantic'
|
import { BaseProjectionStrategy } from '@soulcraftlabs/brainy/vfs/semantic'
|
||||||
|
|
||||||
class PriorityProjection extends BaseProjectionStrategy {
|
class PriorityProjection extends BaseProjectionStrategy {
|
||||||
readonly name = 'priority'
|
readonly name = 'priority'
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ Brainy's Virtual Filesystem (VFS) provides a POSIX-like filesystem interface tha
|
||||||
## Quick Start
|
## Quick Start
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
// Initialize Brainy
|
// Initialize Brainy
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
|
|
@ -598,7 +598,7 @@ const user = await store.findById('users', 'user123')
|
||||||
VFS uses standard POSIX-style errors:
|
VFS uses standard POSIX-style errors:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { VFSError, VFSErrorCode } from '@soulcraft/brainy'
|
import { VFSError, VFSErrorCode } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await vfs.readFile('/nonexistent.txt')
|
await vfs.readFile('/nonexistent.txt')
|
||||||
|
|
|
||||||
|
|
@ -280,7 +280,7 @@ GitBridge provides Git import/export capabilities:
|
||||||
#### GitBridge Usage
|
#### GitBridge Usage
|
||||||
```javascript
|
```javascript
|
||||||
// Import and instantiate GitBridge
|
// Import and instantiate GitBridge
|
||||||
import { GitBridge } from '@soulcraft/brainy'
|
import { GitBridge } from '@soulcraftlabs/brainy'
|
||||||
const gitBridge = new GitBridge(vfs, brain)
|
const gitBridge = new GitBridge(vfs, brain)
|
||||||
|
|
||||||
// Export VFS to Git repository structure
|
// Export VFS to Git repository structure
|
||||||
|
|
@ -452,7 +452,7 @@ This ordering prevents race conditions where file writes might fail because pare
|
||||||
## Complete Example
|
## Complete Example
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
async function vfsExample() {
|
async function vfsExample() {
|
||||||
// Initialize
|
// Initialize
|
||||||
|
|
|
||||||
|
|
@ -196,5 +196,5 @@ await brain.relate({
|
||||||
Always import and use the type enums:
|
Always import and use the type enums:
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
import { NounType, VerbType } from '@soulcraft/brainy'
|
import { NounType, VerbType } from '@soulcraftlabs/brainy'
|
||||||
```
|
```
|
||||||
|
|
@ -5,7 +5,7 @@
|
||||||
The Brainy VFS is automatically initialized during `brain.init()`. No separate initialization needed!
|
The Brainy VFS is automatically initialized during `brain.init()`. No separate initialization needed!
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
// Create and initialize Brainy
|
// Create and initialize Brainy
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
|
|
@ -71,7 +71,7 @@ VFS stores files as entities and relationships in the same graph as everything e
|
||||||
## Complete Example
|
## Complete Example
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
async function useVFS() {
|
async function useVFS() {
|
||||||
// Initialize Brainy
|
// Initialize Brainy
|
||||||
|
|
@ -100,7 +100,7 @@ useVFS().catch(console.error)
|
||||||
## TypeScript Usage
|
## TypeScript Usage
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy, VirtualFileSystem } from '@soulcraft/brainy'
|
import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
class FileManager {
|
class FileManager {
|
||||||
private brain: Brainy
|
private brain: Brainy
|
||||||
|
|
|
||||||
|
|
@ -37,7 +37,7 @@ Brainy VFS provides safe, tree-aware methods that prevent these issues:
|
||||||
### Method 1: Use `getDirectChildren()` (Recommended)
|
### Method 1: Use `getDirectChildren()` (Recommended)
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy, VirtualFileSystem } from '@soulcraft/brainy'
|
import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy()
|
const brain = new Brainy()
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -97,7 +97,7 @@ Here's a complete example using React:
|
||||||
|
|
||||||
```tsx
|
```tsx
|
||||||
import React, { useState, useEffect } from 'react'
|
import React, { useState, useEffect } from 'react'
|
||||||
import { VirtualFileSystem } from '@soulcraft/brainy'
|
import { VirtualFileSystem } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
interface FileNode {
|
interface FileNode {
|
||||||
name: string
|
name: string
|
||||||
|
|
@ -177,7 +177,7 @@ function TreeView({ node, onToggle, expanded }) {
|
||||||
If you must build trees manually from flat lists, use the `VFSTreeUtils`:
|
If you must build trees manually from flat lists, use the `VFSTreeUtils`:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { VFSTreeUtils } from '@soulcraft/brainy/vfs'
|
import { VFSTreeUtils } from '@soulcraftlabs/brainy/vfs'
|
||||||
|
|
||||||
// Get all entities somehow
|
// Get all entities somehow
|
||||||
const allEntities = await vfs.getDescendants('/root')
|
const allEntities = await vfs.getDescendants('/root')
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@
|
||||||
* the Bluesky firehose with Brainy's distributed architecture
|
* the Bluesky firehose with Brainy's distributed architecture
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
import { WebSocket } from 'ws'
|
import { WebSocket } from 'ws'
|
||||||
|
|
||||||
// =====================================================
|
// =====================================================
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,7 @@
|
||||||
* ts-node examples/monitor-cache-performance.ts
|
* ts-node examples/monitor-cache-performance.ts
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { Brainy, NounType } from '@soulcraft/brainy'
|
import { Brainy, NounType } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
// ANSI color codes for pretty output
|
// ANSI color codes for pretty output
|
||||||
const colors = {
|
const colors = {
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ Connect Brainy to spreadsheets, BI tools, and external systems with zero configu
|
||||||
## Quick Start
|
## Quick Start
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy({ integrations: true })
|
const brain = new Brainy({ integrations: true })
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -178,7 +178,7 @@ Webhooks include `X-Brainy-Signature` header with HMAC-SHA256 signature.
|
||||||
### Minimal (in-memory):
|
### Minimal (in-memory):
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy({ integrations: true })
|
const brain = new Brainy({ integrations: true })
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -194,7 +194,7 @@ console.log(brain.hub.getInstructions())
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import express from 'express'
|
import express from 'express'
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const app = express()
|
const app = express()
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
|
|
@ -232,7 +232,7 @@ app.listen(3000, () => {
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Hono } from 'hono'
|
import { Hono } from 'hono'
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const app = new Hono()
|
const app = new Hono()
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -99,7 +99,7 @@ Add the `BRAINY_URL` script property in Apps Script settings.
|
||||||
The simplest way to enable all integrations:
|
The simplest way to enable all integrations:
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const brain = new Brainy({ integrations: true })
|
const brain = new Brainy({ integrations: true })
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -112,7 +112,7 @@ With Express:
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
import express from 'express'
|
import express from 'express'
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
const app = express()
|
const app = express()
|
||||||
const brain = new Brainy({ integrations: true })
|
const brain = new Brainy({ integrations: true })
|
||||||
|
|
|
||||||
8
package-lock.json
generated
8
package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
||||||
{
|
{
|
||||||
"name": "@soulcraft/brainy",
|
"name": "@soulcraftlabs/brainy",
|
||||||
"version": "10.4.2",
|
"version": "10.4.4",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "@soulcraft/brainy",
|
"name": "@soulcraftlabs/brainy",
|
||||||
"version": "10.4.2",
|
"version": "10.4.4",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@msgpack/msgpack": "^3.1.2",
|
"@msgpack/msgpack": "^3.1.2",
|
||||||
|
|
|
||||||
14
package.json
14
package.json
|
|
@ -1,6 +1,7 @@
|
||||||
{
|
{
|
||||||
"name": "@soulcraft/brainy",
|
"name": "@soulcraftlabs/brainy",
|
||||||
"version": "10.4.2",
|
"version": "10.4.4",
|
||||||
|
"brainyContract": 1,
|
||||||
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
|
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"module": "dist/index.js",
|
"module": "dist/index.js",
|
||||||
|
|
@ -126,15 +127,16 @@
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"private": false,
|
"private": false,
|
||||||
"publishConfig": {
|
"publishConfig": {
|
||||||
"access": "public"
|
"access": "public",
|
||||||
|
"registry": "https://source.soulcraft.com/api/packages/soulcraftlabs/npm/"
|
||||||
},
|
},
|
||||||
"homepage": "https://source.soulcraft.com/soulcraft/brainy",
|
"homepage": "https://source.soulcraft.com/soulcraftlabs/open-brainy",
|
||||||
"bugs": {
|
"bugs": {
|
||||||
"url": "https://source.soulcraft.com/soulcraft/brainy/issues"
|
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/issues"
|
||||||
},
|
},
|
||||||
"repository": {
|
"repository": {
|
||||||
"type": "git",
|
"type": "git",
|
||||||
"url": "git+https://source.soulcraft.com/soulcraft/brainy.git"
|
"url": "git+https://source.soulcraft.com/soulcraftlabs/open-brainy.git"
|
||||||
},
|
},
|
||||||
"files": [
|
"files": [
|
||||||
"dist/**/*.js",
|
"dist/**/*.js",
|
||||||
|
|
|
||||||
76
releases/brainy.json
Normal file
76
releases/brainy.json
Normal file
|
|
@ -0,0 +1,76 @@
|
||||||
|
{
|
||||||
|
"product": "brainy",
|
||||||
|
"entries": [
|
||||||
|
{
|
||||||
|
"version": "11.0.5",
|
||||||
|
"date": "2026-09-02",
|
||||||
|
"headline": "Graph-first finds in production, and opens that stop rescanning history",
|
||||||
|
"items": [
|
||||||
|
"find({ connected, where }) now walks the neighbours first and filters only those rows through a native door — correct at every page and O(neighbours), never the whole store.",
|
||||||
|
"related() with a list of verb types returns every requested kind (a fast path had silently kept only the first).",
|
||||||
|
"Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds."
|
||||||
|
],
|
||||||
|
"url": null,
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "11.0.4",
|
||||||
|
"date": "2026-09-01",
|
||||||
|
"headline": "Closes in milliseconds, index rebuilds without the disk-sync storm",
|
||||||
|
"items": [
|
||||||
|
"close() no longer pays deferred compaction or waits out an in-flight rebuild — measured 8 ms against the 4-minute closes it replaces; deferred work resumes at the next open, in the background.",
|
||||||
|
"The metadata index's rebuild syncs to disk per shard instead of per row, and the durability point moved to the publish step — the same guarantee, a fraction of the disk traffic.",
|
||||||
|
"A new native filter door evaluates queries over exactly the candidate rows a graph walk found, never the whole store."
|
||||||
|
],
|
||||||
|
"url": null,
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "11.0.3",
|
||||||
|
"date": "2026-09-01",
|
||||||
|
"headline": "The embedding upgrade ceremony runs on every brain",
|
||||||
|
"items": [
|
||||||
|
"A brain opened through the standard plugin now carries its embedding-model identity, so the full-precision upgrade ceremony can run on it.",
|
||||||
|
"A one-fix release; nothing else changed."
|
||||||
|
],
|
||||||
|
"url": null,
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "11.0.2",
|
||||||
|
"date": "2026-08-31",
|
||||||
|
"headline": "One embedding quality everywhere, 3–4× faster imports",
|
||||||
|
"items": [
|
||||||
|
"Every runtime embeds with the same full-precision model — search quality no longer depends on where you run.",
|
||||||
|
"Bulk embedding measured 3.1–4.2× faster, and an online re-embed ceremony upgrades existing stores without downtime.",
|
||||||
|
"The engine's change feed is documented, with the SSE/WebSocket fan-out pattern for realtime surfaces."
|
||||||
|
],
|
||||||
|
"url": null,
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "11.0.1",
|
||||||
|
"date": "2026-08-31",
|
||||||
|
"headline": "Deletes inside transactions are safe",
|
||||||
|
"items": [
|
||||||
|
"Deleting relations inside a transact() no longer corrupts index bookkeeping.",
|
||||||
|
"A store that deletes its last relation keeps serving instead of refusing."
|
||||||
|
],
|
||||||
|
"url": null,
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "11.0.0",
|
||||||
|
"date": "2026-08-28",
|
||||||
|
"headline": "One install, one engine — Brainy",
|
||||||
|
"items": [
|
||||||
|
"The former two-package pair is one package: the native engine under the familiar API. One import is the whole install.",
|
||||||
|
"A missing native build refuses loudly with its cures named; nothing falls back silently.",
|
||||||
|
"Stores open in place — no migration."
|
||||||
|
],
|
||||||
|
"url": null,
|
||||||
|
"thumb": null
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"history": "The version line continues from the 4.3.x native-engine releases; their record lives in the product repository's CHANGELOG.md."
|
||||||
|
}
|
||||||
110
releases/open-brainy.json
Normal file
110
releases/open-brainy.json
Normal file
|
|
@ -0,0 +1,110 @@
|
||||||
|
{
|
||||||
|
"product": "open-brainy",
|
||||||
|
"entries": [
|
||||||
|
{
|
||||||
|
"version": "10.4.9",
|
||||||
|
"date": "2026-09-02",
|
||||||
|
"headline": "Graph-first finds, honest verb arrays, and opens that stop rescanning history",
|
||||||
|
"items": [
|
||||||
|
"find({ connected, where }) now walks the neighbours first and filters only those rows — correct at every page, and O(neighbours) instead of O(store).",
|
||||||
|
"related() with a list of verb types (or sources, or targets) returns every requested kind — four fast paths silently kept only the first.",
|
||||||
|
"Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds."
|
||||||
|
],
|
||||||
|
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.9",
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "10.4.7",
|
||||||
|
"date": "2026-09-01",
|
||||||
|
"headline": "Count ledgers can no longer race themselves",
|
||||||
|
"items": [
|
||||||
|
"Concurrent count flushes coalesce into one writer with a trailing pass — parallel flushes can no longer corrupt a store's count ledger.",
|
||||||
|
"Atomic writes carry a per-process sequence, so two processes' temp files can never collide."
|
||||||
|
],
|
||||||
|
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.7",
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "10.4.6",
|
||||||
|
"date": "2026-08-31",
|
||||||
|
"headline": "Transactions cross the index seam safely",
|
||||||
|
"items": [
|
||||||
|
"Deleting relations inside a transact() no longer fails against the metadata index — operations take a JSON-safe view at the moment they execute.",
|
||||||
|
"Fixes a class of transaction failures on stores with integer-mapped relation endpoints."
|
||||||
|
],
|
||||||
|
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.6",
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "10.4.5",
|
||||||
|
"date": "2026-08-31",
|
||||||
|
"headline": "Recovery tells the truth, docs live at home",
|
||||||
|
"items": [
|
||||||
|
"A torn generation-log tail is a terminal verdict with a named cure — never an endless wait at open.",
|
||||||
|
"A sealed segment declares only the generations it actually holds.",
|
||||||
|
"The engine's documentation now publishes from its own repository."
|
||||||
|
],
|
||||||
|
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.5",
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "10.4.4",
|
||||||
|
"date": "2026-08-28",
|
||||||
|
"headline": "Faster opens, quieter idle",
|
||||||
|
"items": [
|
||||||
|
"Opening a store discovers generations from directory names instead of walking the log, and answers \"any entities?\" with one directory read.",
|
||||||
|
"The flush-request watch is event-driven; idle stores stop paying a polling heartbeat.",
|
||||||
|
"A slow open now names the exact step it is in, so operators see what is being paid and why."
|
||||||
|
],
|
||||||
|
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.4",
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "10.4.3",
|
||||||
|
"date": "2026-08-27",
|
||||||
|
"headline": "Open Brainy, under its own name",
|
||||||
|
"items": [
|
||||||
|
"The same engine as 10.4.2, now published as @soulcraftlabs/brainy — the MIT reference engine, on The Source.",
|
||||||
|
"No code changes; your imports change once and everything else stays put."
|
||||||
|
],
|
||||||
|
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.3",
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "10.4.2",
|
||||||
|
"date": "2026-08-27",
|
||||||
|
"headline": "Vectors that lie are refused, counts that drift are caught",
|
||||||
|
"items": [
|
||||||
|
"A zero-norm vector is not a vector: the index refuses them, rebuilds skip them, and a sanctioned unvector door removes them cleanly.",
|
||||||
|
"The canonical count ledger derives from identity records and marks legacy-derived ledgers suspect at load.",
|
||||||
|
"Plugin activation failures keep their original error as cause, so the real frame reaches your logs."
|
||||||
|
],
|
||||||
|
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.2",
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "10.4.1",
|
||||||
|
"date": "2026-08-26",
|
||||||
|
"headline": "Writes that change nothing cost nothing",
|
||||||
|
"items": [
|
||||||
|
"The read gate is per index family, and a write carrying unchanged data never re-embeds.",
|
||||||
|
"The vectored-row count joins the ledger, so vector coverage is a number you can read, not a guess."
|
||||||
|
],
|
||||||
|
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.1",
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "10.4.0",
|
||||||
|
"date": "2026-08-26",
|
||||||
|
"headline": "Repair routing, the vector ledger, and honest empties",
|
||||||
|
"items": [
|
||||||
|
"Repairs route to the index that owns the damage, and the open gate closes the vector leg until coverage is proven.",
|
||||||
|
"An empty string is real data, not a missing field.",
|
||||||
|
"The metadata crossing never carries raw integer relation endpoints — a whole class of serialization faults closed."
|
||||||
|
],
|
||||||
|
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.0",
|
||||||
|
"thumb": null
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"history": "Earlier releases are recorded in CHANGELOG.md in this repository."
|
||||||
|
}
|
||||||
128
scripts/emit-contract-manifest.mjs
Normal file
128
scripts/emit-contract-manifest.mjs
Normal file
|
|
@ -0,0 +1,128 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* Emit this build's API-contract manifest to docs/api-contract.json.
|
||||||
|
*
|
||||||
|
* WHY IT IS GENERATED, NOT WRITTEN: a hand-kept list of doors drifts from the
|
||||||
|
* code the first time somebody adds one. This reads the surface the build
|
||||||
|
* actually exposes — the prototype's own methods and accessors, the exported
|
||||||
|
* error classes, the `where` operator sets, the field-addressing vocabulary,
|
||||||
|
* the health verdicts — so a diff between two engines' manifests is a diff
|
||||||
|
* between two engines, never between two authors.
|
||||||
|
*
|
||||||
|
* Requirement marking (required / optional per door) is NOT derivable from the
|
||||||
|
* surface — it is a commitment, recorded with the contract's owner rather than
|
||||||
|
* here. This manifest carries the surface; the promise lives with the contract.
|
||||||
|
*
|
||||||
|
* Usage: node scripts/emit-contract-manifest.mjs [--check]
|
||||||
|
* --check exits non-zero when the committed manifest is stale.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { writeFileSync, readFileSync, existsSync } from 'node:fs'
|
||||||
|
import { join, dirname } from 'node:path'
|
||||||
|
import { fileURLToPath } from 'node:url'
|
||||||
|
|
||||||
|
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..')
|
||||||
|
const OUT = join(ROOT, 'docs', 'api-contract.json')
|
||||||
|
|
||||||
|
const { Brainy } = await import(join(ROOT, 'dist', 'brainy.js'))
|
||||||
|
const errorsModule = await import(join(ROOT, 'dist', 'errors', 'brainyError.js'))
|
||||||
|
const versionModule = await import(join(ROOT, 'dist', 'utils', 'version.js'))
|
||||||
|
const fieldAddressing = await import(join(ROOT, 'dist', 'db', 'fieldAddressing.js'))
|
||||||
|
|
||||||
|
/** Every own method and accessor on the class's prototype, minus the private ones. */
|
||||||
|
function surfaceOf(ctor) {
|
||||||
|
const doors = []
|
||||||
|
for (const name of Object.getOwnPropertyNames(ctor.prototype)) {
|
||||||
|
if (name === 'constructor' || name.startsWith('_')) continue
|
||||||
|
const descriptor = Object.getOwnPropertyDescriptor(ctor.prototype, name)
|
||||||
|
if (!descriptor) continue
|
||||||
|
if (typeof descriptor.value === 'function') {
|
||||||
|
doors.push({ name, kind: 'method', arity: descriptor.value.length })
|
||||||
|
} else if (descriptor.get) {
|
||||||
|
doors.push({ name, kind: 'accessor' })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return doors.sort((a, b) => a.name.localeCompare(b.name))
|
||||||
|
}
|
||||||
|
|
||||||
|
const errors = Object.entries(errorsModule)
|
||||||
|
.filter(([name, value]) => typeof value === 'function' && /Error$/.test(name))
|
||||||
|
.map(([name]) => name)
|
||||||
|
.sort()
|
||||||
|
|
||||||
|
// The operator sets, read from the engine's own refusal message so the
|
||||||
|
// manifest can never disagree with the validator.
|
||||||
|
const filterSource = readFileSync(join(ROOT, 'src', 'utils', 'metadataFilter.ts'), 'utf-8')
|
||||||
|
const acceptedMatch = filterSource.match(/const VALUE_OPERATORS = new Set<string>\(\[([\s\S]*?)\]\)/)
|
||||||
|
if (!acceptedMatch) throw new Error('VALUE_OPERATORS not found — the manifest refuses to guess')
|
||||||
|
const accepted = [...acceptedMatch[1].matchAll(/'([^']+)'/g)].map((m) => m[1]).sort()
|
||||||
|
|
||||||
|
const indexSource = readFileSync(join(ROOT, 'src', 'utils', 'metadataIndex.ts'), 'utf-8')
|
||||||
|
const refusedByIndex = ['endsWith', 'length', 'matches', 'startsWith'].filter((op) =>
|
||||||
|
// Proven by the refusal path: these are the tokens with no case in the
|
||||||
|
// index's operator switch, so they fall to its default and are refused.
|
||||||
|
!new RegExp(`case '${op}':`).test(indexSource)
|
||||||
|
)
|
||||||
|
const servedOnIndex = accepted.filter((op) => !refusedByIndex.includes(op))
|
||||||
|
|
||||||
|
const manifest = {
|
||||||
|
contractVersion: versionModule.contractVersion(),
|
||||||
|
engine: '@soulcraftlabs/brainy',
|
||||||
|
compatibility: {
|
||||||
|
minor:
|
||||||
|
'additive — a new optional door, a new served operator, a new error class; every existing implementation still conforms',
|
||||||
|
major:
|
||||||
|
'breaking — a door removed, an answer narrowed, an ordering law changed, an optional door promoted to required, or an operator moved from served to refused'
|
||||||
|
},
|
||||||
|
doors: surfaceOf(Brainy),
|
||||||
|
errors,
|
||||||
|
operators: {
|
||||||
|
accepted,
|
||||||
|
servedOnIndexPath: servedOnIndex,
|
||||||
|
refusedByIndexPath: refusedByIndex,
|
||||||
|
combinators: ['allOf', 'anyOf', 'not']
|
||||||
|
},
|
||||||
|
fieldAddressing: {
|
||||||
|
systemKeyPrefix: 'system.',
|
||||||
|
systemEntityScalars: [...(fieldAddressing.SYSTEM_ENTITY_SCALARS ?? [])].sort(),
|
||||||
|
systemRelationScalars: [...(fieldAddressing.SYSTEM_RELATION_SCALARS ?? [])].sort(),
|
||||||
|
plumbingFields: [...(fieldAddressing.PLUMBING_FIELDS ?? [])].sort()
|
||||||
|
},
|
||||||
|
health: {
|
||||||
|
verdicts: ['pass', 'warn', 'fail'],
|
||||||
|
healKinds: ['none', 'repair', 'rebuild'],
|
||||||
|
servingWithholdingInvariants: [
|
||||||
|
'index-initialized',
|
||||||
|
'durable-state-present',
|
||||||
|
'manifest-residency',
|
||||||
|
'replay-clean',
|
||||||
|
'strand-latch'
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const rendered = `${JSON.stringify(manifest, null, 2)}\n`
|
||||||
|
|
||||||
|
if (process.argv.includes('--check')) {
|
||||||
|
if (!existsSync(OUT)) {
|
||||||
|
console.error(`docs/api-contract.json is missing — run: node scripts/emit-contract-manifest.mjs`)
|
||||||
|
process.exit(1)
|
||||||
|
}
|
||||||
|
if (readFileSync(OUT, 'utf-8') !== rendered) {
|
||||||
|
console.error(
|
||||||
|
`docs/api-contract.json is STALE — the public surface changed. Re-emit it and announce ` +
|
||||||
|
`the addition (minor = additive; a removal is a contract major).`
|
||||||
|
)
|
||||||
|
process.exit(1)
|
||||||
|
}
|
||||||
|
console.log(`docs/api-contract.json is current (${manifest.doors.length} doors, contract ${manifest.contractVersion}).`)
|
||||||
|
process.exit(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
writeFileSync(OUT, rendered)
|
||||||
|
console.log(
|
||||||
|
`Wrote docs/api-contract.json — contract ${manifest.contractVersion}, ` +
|
||||||
|
`${manifest.doors.length} doors, ${manifest.errors.length} error classes, ` +
|
||||||
|
`${manifest.operators.accepted.length} operators ` +
|
||||||
|
`(${manifest.operators.refusedByIndexPath.length} refused by the index path).`
|
||||||
|
)
|
||||||
|
|
@ -15,11 +15,11 @@ NC='\033[0m' # No Color
|
||||||
RELEASE_TYPE="${1:-patch}" # patch, minor, or major
|
RELEASE_TYPE="${1:-patch}" # patch, minor, or major
|
||||||
SKIP_TESTS=false
|
SKIP_TESTS=false
|
||||||
DRY_RUN=false
|
DRY_RUN=false
|
||||||
# --source-only: the HOME leg only — tag, CI's publish to The Source, and the
|
# --source-only is now a no-op: The Source is the one registry, so every
|
||||||
# release page; NO storefront (npmjs) publish, NO pair verification, NO docs
|
# release already ships Source-only — tag, CI's publish to The Source, the
|
||||||
# push. The pair-gate shape: a prerelease the fleet's other engine devDeps
|
# release page, and the docs push, with no separate storefront leg to skip.
|
||||||
# from our own registry while the pair is proven, never a public artifact.
|
# The flag is still accepted (for backward-compatible invocations) and just
|
||||||
# Refused for a non-prerelease version — a public floor is always a pair.
|
# prints a notice; it no longer changes behavior.
|
||||||
SOURCE_ONLY=false
|
SOURCE_ONLY=false
|
||||||
|
|
||||||
for arg in "$@"; do
|
for arg in "$@"; do
|
||||||
|
|
@ -109,7 +109,7 @@ else
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
echo -e "${RED}❌ Invalid release type: ${RELEASE_TYPE}${NC}"
|
echo -e "${RED}❌ Invalid release type: ${RELEASE_TYPE}${NC}"
|
||||||
echo "Usage: ./scripts/release.sh [patch|minor|major|<explicit-version>] [--dry-run] [--source-only (prereleases only)]"
|
echo "Usage: ./scripts/release.sh [patch|minor|major|<explicit-version>] [--dry-run] [--source-only (no-op; The Source is the one registry)]"
|
||||||
exit 1
|
exit 1
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
@ -129,11 +129,7 @@ if [ "$PRERELEASE" = true ]; then
|
||||||
echo -e "${YELLOW}⚠️ Prerelease → npm dist-tag '${NPM_TAG}', GitHub prerelease${NC}"
|
echo -e "${YELLOW}⚠️ Prerelease → npm dist-tag '${NPM_TAG}', GitHub prerelease${NC}"
|
||||||
fi
|
fi
|
||||||
if [ "$SOURCE_ONLY" = true ]; then
|
if [ "$SOURCE_ONLY" = true ]; then
|
||||||
if [ "$PRERELEASE" != true ]; then
|
echo -e "${YELLOW}⚠️ The Source is the one registry; --source-only is implied${NC}"
|
||||||
echo -e "${RED}❌ --source-only is for prereleases only: a non-prerelease version is a public floor and always ships as the byte-identical pair.${NC}"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
echo -e "${YELLOW}⚠️ --source-only → The Source (home) ONLY: no npmjs publish, no pair verification, no docs push${NC}"
|
|
||||||
fi
|
fi
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
|
|
@ -158,7 +154,7 @@ else
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Create new changelog entry
|
# Create new changelog entry
|
||||||
CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraft/brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) ($(date +%Y-%m-%d))
|
CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) ($(date +%Y-%m-%d))
|
||||||
|
|
||||||
${COMMITS}
|
${COMMITS}
|
||||||
"
|
"
|
||||||
|
|
@ -209,9 +205,9 @@ echo -e "${GREEN}✅ Pushed to origin${NC}\n"
|
||||||
# .forgejo/workflows/publish-source.yml, which builds and publishes on The
|
# .forgejo/workflows/publish-source.yml, which builds and publishes on The
|
||||||
# Source's own runner (datacenter-side: seconds, not the laptop's WAN timing
|
# Source's own runner (datacenter-side: seconds, not the laptop's WAN timing
|
||||||
# out on an 87MB tarball PUT). The laptop holds no home-registry publish
|
# out on an 87MB tarball PUT). The laptop holds no home-registry publish
|
||||||
# credential anymore; it only waits for CI's result before trusting the
|
# credential anymore; it only waits for CI's result before continuing on to
|
||||||
# home/npmjs pair enough to publish the storefront leg.
|
# the release page and the docs push.
|
||||||
SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/"
|
SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraftlabs/npm/"
|
||||||
SOURCE_POLL_INTERVAL_S=15
|
SOURCE_POLL_INTERVAL_S=15
|
||||||
SOURCE_POLL_MAX_ATTEMPTS=200 # 200 × 15s = 50 minutes — the runner is sequential and a busy day's ci.yml
|
SOURCE_POLL_MAX_ATTEMPTS=200 # 200 × 15s = 50 minutes — the runner is sequential and a busy day's ci.yml
|
||||||
# backlog has twice exceeded the old 20-minute window (8.10.3, 9.0.0);
|
# backlog has twice exceeded the old 20-minute window (8.10.3, 9.0.0);
|
||||||
|
|
@ -219,7 +215,7 @@ SOURCE_POLL_MAX_ATTEMPTS=200 # 200 × 15s = 50 minutes — the runner is sequen
|
||||||
echo -e "${BLUE}9️⃣ Waiting for CI to publish v${NEW_VERSION} to The Source registry (home)...${NC}"
|
echo -e "${BLUE}9️⃣ Waiting for CI to publish v${NEW_VERSION} to The Source registry (home)...${NC}"
|
||||||
SOURCE_LANDED=false
|
SOURCE_LANDED=false
|
||||||
for ((attempt = 1; attempt <= SOURCE_POLL_MAX_ATTEMPTS; attempt++)); do
|
for ((attempt = 1; attempt <= SOURCE_POLL_MAX_ATTEMPTS; attempt++)); do
|
||||||
LANDED_VERSION=$(npm view "@soulcraft/brainy@${NEW_VERSION}" version "--@soulcraft:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "")
|
LANDED_VERSION=$(npm view "@soulcraftlabs/brainy@${NEW_VERSION}" version "--@soulcraftlabs:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "")
|
||||||
if [ "$LANDED_VERSION" = "$NEW_VERSION" ]; then
|
if [ "$LANDED_VERSION" = "$NEW_VERSION" ]; then
|
||||||
SOURCE_LANDED=true
|
SOURCE_LANDED=true
|
||||||
break
|
break
|
||||||
|
|
@ -232,62 +228,16 @@ if [ "$SOURCE_LANDED" = true ]; then
|
||||||
echo -e "${GREEN}✅ CI published v${NEW_VERSION} to The Source${NC}\n"
|
echo -e "${GREEN}✅ CI published v${NEW_VERSION} to The Source${NC}\n"
|
||||||
else
|
else
|
||||||
echo -e "${RED}❌ CI's home publish did not land — check the workflow run on The Source; the pair must not diverge.${NC}"
|
echo -e "${RED}❌ CI's home publish did not land — check the workflow run on The Source; the pair must not diverge.${NC}"
|
||||||
echo -e "${RED} v${NEW_VERSION} was tagged and pushed, but @soulcraft/brainy@${NEW_VERSION} never became visible on the${NC}"
|
echo -e "${RED} v${NEW_VERSION} was tagged and pushed, but @soulcraftlabs/brainy@${NEW_VERSION} never became visible on the${NC}"
|
||||||
echo -e "${RED} Source registry after ${SOURCE_POLL_MAX_ATTEMPTS} attempts, ${SOURCE_POLL_INTERVAL_S}s apart. Aborting before npmjs.${NC}"
|
echo -e "${RED} Source registry after ${SOURCE_POLL_MAX_ATTEMPTS} attempts, ${SOURCE_POLL_INTERVAL_S}s apart. Aborting.${NC}"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ "$SOURCE_ONLY" = true ]; then
|
|
||||||
echo -e "${YELLOW}9️⃣½ Storefront (npmjs) leg SKIPPED — --source-only: v${NEW_VERSION} lives on The Source under dist-tag '${NPM_TAG}' only${NC}\n"
|
|
||||||
else
|
|
||||||
echo -e "${BLUE}9️⃣½ Publishing to npmjs (storefront, dist-tag: ${NPM_TAG})...${NC}"
|
|
||||||
# BYTE-IDENTITY LAW: the storefront republishes CI's EXACT artifact — download
|
|
||||||
# the tarball The Source serves and publish that file, never a fresh local pack
|
|
||||||
# (a local rebuild can differ byte-wise, and the fleet verifies the pair by
|
|
||||||
# shasum across registries).
|
|
||||||
STOREFRONT_TMP="$(mktemp -d)"
|
|
||||||
(cd "$STOREFRONT_TMP" && npm pack "@soulcraft/brainy@${NEW_VERSION}" "--@soulcraft:registry=${SOURCE_NPM_REG}" >/dev/null)
|
|
||||||
SOURCE_TARBALL="$(ls "$STOREFRONT_TMP"/soulcraft-brainy-*.tgz)"
|
|
||||||
echo -e "${BLUE} home artifact: $(sha256sum "$SOURCE_TARBALL" | cut -d' ' -f1)${NC}"
|
|
||||||
npm publish "$SOURCE_TARBALL" --tag "$NPM_TAG" "--@soulcraft:registry=https://registry.npmjs.org/"
|
|
||||||
rm -rf "$STOREFRONT_TMP"
|
|
||||||
# Brainy is the only PUBLIC @soulcraft package — verify visibility after every publish.
|
|
||||||
npm access get status @soulcraft/brainy "--@soulcraft:registry=https://registry.npmjs.org/" || true
|
|
||||||
# Verify the pair is byte-identical by registry-reported shasum — divergence
|
|
||||||
# here means the storefront leg must be treated as failed, loudly. RETRIED
|
|
||||||
# with raw curl: npmjs metadata propagates with a lag measured in minutes,
|
|
||||||
# and a one-shot npm-view probe fired a false DIVERGENCE on 10.0.0 while a
|
|
||||||
# raw curl of the registry document already confirmed byte-identity. The
|
|
||||||
# probe now reads the registry JSON directly (no npm cache in the path) and
|
|
||||||
# gives propagation up to 5 minutes before calling the pair divergent.
|
|
||||||
NPMJS_VERIFY_ATTEMPTS=20
|
|
||||||
NPMJS_VERIFY_INTERVAL_S=15 # 20 × 15s = 5 minutes of propagation grace
|
|
||||||
SOURCE_SHA=$(npm view "@soulcraft/brainy@${NEW_VERSION}" dist.shasum "--@soulcraft:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "source-unavailable")
|
|
||||||
PAIR_IDENTICAL=false
|
|
||||||
for ((attempt = 1; attempt <= NPMJS_VERIFY_ATTEMPTS; attempt++)); do
|
|
||||||
NPMJS_SHA=$(curl -fsSL "https://registry.npmjs.org/@soulcraft%2Fbrainy" 2>/dev/null \
|
|
||||||
| node -e "let d='';process.stdin.on('data',c=>d+=c).on('end',()=>{try{const v=JSON.parse(d).versions[process.argv[1]];console.log(v?v.dist.shasum:'')}catch{console.log('')}})" "${NEW_VERSION}" \
|
|
||||||
|| echo "")
|
|
||||||
if [ -n "$NPMJS_SHA" ] && [ "$SOURCE_SHA" = "$NPMJS_SHA" ]; then
|
|
||||||
PAIR_IDENTICAL=true
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
echo -e "${YELLOW} … npmjs metadata not settled (attempt ${attempt}/${NPMJS_VERIFY_ATTEMPTS}: '${NPMJS_SHA:-absent}' vs '${SOURCE_SHA}'); retrying in ${NPMJS_VERIFY_INTERVAL_S}s${NC}"
|
|
||||||
sleep "$NPMJS_VERIFY_INTERVAL_S"
|
|
||||||
done
|
|
||||||
if [ "$PAIR_IDENTICAL" = true ]; then
|
|
||||||
echo -e "${GREEN}✅ Published to npmjs — byte-identical pair (shasum ${NPMJS_SHA})${NC}\n"
|
|
||||||
else
|
|
||||||
echo -e "${RED}❌ REGISTRY DIVERGENCE: The Source shasum ${SOURCE_SHA} != npmjs shasum ${NPMJS_SHA} after ${NPMJS_VERIFY_ATTEMPTS} attempts — investigate before announcing${NC}\n"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Step 11: Release object on The Source (presentational — the tag, CHANGELOG,
|
# Step 11: Release object on The Source (presentational — the tag, CHANGELOG,
|
||||||
# and RELEASES.md are the record; this just gives The Source's UI a release page).
|
# and RELEASES.md are the record; this just gives The Source's UI a release page).
|
||||||
echo -e "${BLUE}🔟 Creating release page on The Source...${NC}"
|
echo -e "${BLUE}🔟 Creating release page on The Source...${NC}"
|
||||||
if [ -n "${FORGEJO_RELEASE_TOKEN:-}" ]; then
|
if [ -n "${FORGEJO_RELEASE_TOKEN:-}" ]; then
|
||||||
if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraft/brainy/releases" \
|
if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraftlabs/open-brainy/releases" \
|
||||||
-H "Authorization: token ${FORGEJO_RELEASE_TOKEN}" -H "Content-Type: application/json" \
|
-H "Authorization: token ${FORGEJO_RELEASE_TOKEN}" -H "Content-Type: application/json" \
|
||||||
-d "{\"tag_name\":\"v${NEW_VERSION}\",\"name\":\"v${NEW_VERSION}\",\"prerelease\":${PRERELEASE}}" >/dev/null; then
|
-d "{\"tag_name\":\"v${NEW_VERSION}\",\"name\":\"v${NEW_VERSION}\",\"prerelease\":${PRERELEASE}}" >/dev/null; then
|
||||||
echo -e "${GREEN}✅ Release page created on The Source${NC}\n"
|
echo -e "${GREEN}✅ Release page created on The Source${NC}\n"
|
||||||
|
|
@ -298,29 +248,15 @@ else
|
||||||
echo -e "${RED}⚠️ FORGEJO_RELEASE_TOKEN unset — no release page created; tag + CHANGELOG remain the record${NC}\n"
|
echo -e "${RED}⚠️ FORGEJO_RELEASE_TOKEN unset — no release page created; tag + CHANGELOG remain the record${NC}\n"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Step 12: Push public docs to the soulcraft.com docs ingest door
|
# Step 12 RETIRED (2026-08-31, CORTEX-SITE-BRAINY-RENAME round 12, David-ruled):
|
||||||
# (VENUE-DOCS-RELEASE-PUSH). Skips with a loud warning when
|
# soulcraft.com/docs carries the paid product's documentation only. This
|
||||||
# DOCS_INGEST_SECRET is unset; fails loudly (without undoing the publish —
|
# engine's documentation home is THIS repository — README and docs/ — and the
|
||||||
# that already happened) when a push errors, so the docs site never
|
# site serves 301s for the slugs this rail used to push. The push script stays
|
||||||
# silently trails npm.
|
# in the tree for history; the rail no longer calls it.
|
||||||
if [ "$SOURCE_ONLY" = true ]; then
|
echo -e "${BLUE}Docs step: this engine documents itself in its own repo (site push retired 2026-08-31)${NC}"
|
||||||
echo -e "${YELLOW}1️⃣2️⃣ Docs push SKIPPED — --source-only (a home-only prerelease publishes no public docs)${NC}\n"
|
|
||||||
else
|
|
||||||
echo -e "${BLUE}1️⃣2️⃣ Pushing public docs to soulcraft.com/docs...${NC}"
|
|
||||||
if node scripts/push-docs.js; then
|
|
||||||
echo -e "${GREEN}✅ Docs push step done${NC}\n"
|
|
||||||
else
|
|
||||||
echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||||
echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}"
|
echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}"
|
||||||
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||||
echo ""
|
echo ""
|
||||||
if [ "$SOURCE_ONLY" = true ]; then
|
echo -e "🏠 The Source: ${BLUE}https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v${NEW_VERSION}${NC}"
|
||||||
echo -e "📦 npmjs: ${YELLOW}not published (--source-only)${NC}"
|
|
||||||
else
|
|
||||||
echo -e "📦 npm: ${BLUE}https://www.npmjs.com/package/@soulcraft/brainy/v/${NEW_VERSION}${NC}"
|
|
||||||
fi
|
|
||||||
echo -e "🏠 The Source: ${BLUE}https://source.soulcraft.com/soulcraft/brainy/releases/tag/v${NEW_VERSION}${NC}"
|
|
||||||
|
|
|
||||||
944
src/brainy.ts
944
src/brainy.ts
File diff suppressed because it is too large
Load diff
|
|
@ -28,7 +28,7 @@
|
||||||
* speculative `with()` overlay; the canonical storage walk only ever answers
|
* speculative `with()` overlay; the canonical storage walk only ever answers
|
||||||
* "what is live right now."
|
* "what is live right now."
|
||||||
*
|
*
|
||||||
* All are exported from the package root (`@soulcraft/brainy`).
|
* All are exported from the package root (`@soulcraftlabs/brainy`).
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -12,9 +12,11 @@
|
||||||
* the verified surface is a small set of rollup invariants (entity/
|
* the verified surface is a small set of rollup invariants (entity/
|
||||||
* relationship counts) plus `sourceGeneration`.
|
* relationship counts) plus `sourceGeneration`.
|
||||||
*
|
*
|
||||||
* `sourceGeneration` is the generation of the source-of-truth log this
|
* `sourceGeneration` is the COMMITTED generation of the source-of-truth log
|
||||||
* projection reflects — open-time coherence becomes a COMPARISON (stamp vs
|
* this projection reflects — never the allocated counter, which names a
|
||||||
* log head), not a walk:
|
* generation that may never commit (see {@link StampVerdict.torn}) — so
|
||||||
|
* open-time coherence becomes a COMPARISON (stamp vs committed head), not a
|
||||||
|
* walk:
|
||||||
*
|
*
|
||||||
* - equal + invariants hold → coherent, serve.
|
* - equal + invariants hold → coherent, serve.
|
||||||
* - behind → the projection missed the tail (crash between commit and stamp);
|
* - behind → the projection missed the tail (crash between commit and stamp);
|
||||||
|
|
@ -24,6 +26,9 @@
|
||||||
* - invariants FAIL at equal generation → genuine incoherence: loud, and the
|
* - invariants FAIL at equal generation → genuine incoherence: loud, and the
|
||||||
* repair ritual (`repairIndex()`, whose recount rebuilds the rollups from a
|
* repair ritual (`repairIndex()`, whose recount rebuilds the rollups from a
|
||||||
* canonical walk) heals it.
|
* canonical walk) heals it.
|
||||||
|
* - AHEAD → a torn generation-log tail: the stamp's fsync outlived the log
|
||||||
|
* tail's. TERMINAL, never a wait — the generation the stamp names does not
|
||||||
|
* exist to arrive.
|
||||||
*
|
*
|
||||||
* Stamps are JSON on purpose — every incident gets debugged by reading a
|
* Stamps are JSON on purpose — every incident gets debugged by reading a
|
||||||
* stamp in a terminal.
|
* stamp in a terminal.
|
||||||
|
|
@ -70,6 +75,12 @@ export type StampVerdict =
|
||||||
| { state: 'coherent' }
|
| { state: 'coherent' }
|
||||||
| { state: 'absent' } // legacy store — first stamp writes at the next flush
|
| { state: 'absent' } // legacy store — first stamp writes at the next flush
|
||||||
| { state: 'behind'; stampSource: number; head: number }
|
| { state: 'behind'; stampSource: number; head: number }
|
||||||
|
/**
|
||||||
|
* TORN GENERATION-LOG TAIL: the stamp witnesses a source generation the
|
||||||
|
* store's committed watermark can no longer show. TERMINAL — there is no
|
||||||
|
* generation to wait for, so the open demotes (or refuses) and never spins.
|
||||||
|
*/
|
||||||
|
| { state: 'torn'; stampSource: number; head: number }
|
||||||
| { state: 'incoherent'; failures: string[] }
|
| { state: 'incoherent'; failures: string[] }
|
||||||
| { state: 'unverifiable'; reason: string } // a FAULT reading the stamp — never conflated with absence
|
| { state: 'unverifiable'; reason: string } // a FAULT reading the stamp — never conflated with absence
|
||||||
|
|
||||||
|
|
@ -118,12 +129,15 @@ export function verifyFamilyStamp(
|
||||||
): StampVerdict {
|
): StampVerdict {
|
||||||
if (stamp === null) return { state: 'absent' }
|
if (stamp === null) return { state: 'absent' }
|
||||||
if (stamp.sourceGeneration > head) {
|
if (stamp.sourceGeneration > head) {
|
||||||
// A stamp AHEAD of the log claims state that never committed — the
|
// A stamp AHEAD of committed truth witnesses a generation the store can no
|
||||||
// projection was stamped against truth that a crash rolled back.
|
// longer show: the stamp's fsync survived a crash that the log tail did
|
||||||
return {
|
// not. This is the TORN GENERATION-LOG TAIL — its own class, never folded
|
||||||
state: 'incoherent',
|
// in with `incoherent` (a count that drifted at a generation both sides
|
||||||
failures: [`sourceGeneration ${stamp.sourceGeneration} is ahead of the log head ${head}`]
|
// agree on), because the two have opposite cures: incoherence is recounted,
|
||||||
}
|
// a tear is DEMOTED. It is also terminal by construction — there is no
|
||||||
|
// generation the open can wait for, because the one the stamp names is
|
||||||
|
// gone.
|
||||||
|
return { state: 'torn', stampSource: stamp.sourceGeneration, head }
|
||||||
}
|
}
|
||||||
if (stamp.sourceGeneration < head) {
|
if (stamp.sourceGeneration < head) {
|
||||||
return { state: 'behind', stampSource: stamp.sourceGeneration, head }
|
return { state: 'behind', stampSource: stamp.sourceGeneration, head }
|
||||||
|
|
|
||||||
|
|
@ -147,6 +147,60 @@ export class GenerationSegmentStore {
|
||||||
return this.coveringSegment(gen) !== null
|
return this.coveringSegment(gen) !== null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description True when `meta` declares more generations than it holds
|
||||||
|
* frames — a segment sealed by a writer that folded across a hole. The
|
||||||
|
* manifest records `frames` at fold time, so this is an O(1) comparison
|
||||||
|
* against the declared span and needs no I/O.
|
||||||
|
*/
|
||||||
|
private isSparse(meta: SegmentMeta): boolean {
|
||||||
|
return meta.lastGeneration - meta.firstGeneration + 1 !== meta.frames
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description The generations this tier ACTUALLY holds, as coalesced
|
||||||
|
* ascending intervals — not what the segments declare.
|
||||||
|
*
|
||||||
|
* Dense segments (every one a current writer produces) contribute their
|
||||||
|
* declared range with no I/O. A SPARSE segment — one sealed before the
|
||||||
|
* density law was enforced, whose declared range spans generations it has
|
||||||
|
* no frame for — has its real generation list read from its sidecar and
|
||||||
|
* contributed instead, with the discrepancy narrated once.
|
||||||
|
*
|
||||||
|
* This is what keeps a store that already carries the damage from wedging.
|
||||||
|
* `open()` seeds `committedRanges` from these intervals, so a hole is never
|
||||||
|
* re-admitted as a committed generation, and the auto-compaction pass that
|
||||||
|
* used to fail on every run with "packed history is damaged" simply never
|
||||||
|
* asks for the missing frame.
|
||||||
|
*
|
||||||
|
* @returns Ascending, non-overlapping `[first, last]` intervals.
|
||||||
|
*/
|
||||||
|
async actualRanges(): Promise<Array<[number, number]>> {
|
||||||
|
const out: Array<[number, number]> = []
|
||||||
|
for (const meta of this.manifest.segments) {
|
||||||
|
if (!this.isSparse(meta)) {
|
||||||
|
out.push([meta.firstGeneration, meta.lastGeneration])
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
const missing = meta.lastGeneration - meta.firstGeneration + 1 - meta.frames
|
||||||
|
prodLog.warn(
|
||||||
|
`[GenerationSegments] sealed segment ${meta.file} declares generations ` +
|
||||||
|
`${meta.firstGeneration}..${meta.lastGeneration} but holds only ${meta.frames} ` +
|
||||||
|
`frame(s) — ${missing} generation(s) in that span were never folded into it. ` +
|
||||||
|
`Serving the frames it actually holds; the declared span is not treated as ` +
|
||||||
|
`committed history. (Written by a pre-density-law writer that folded across a ` +
|
||||||
|
`gap; the segment itself is intact and no record is lost.)`
|
||||||
|
)
|
||||||
|
const idx = await this.sidecarFor(meta)
|
||||||
|
for (const [gen] of idx.generations) {
|
||||||
|
const last = out[out.length - 1]
|
||||||
|
if (last !== undefined && gen === last[1] + 1) last[1] = gen
|
||||||
|
else out.push([gen, gen])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Fold consecutive generations into ONE new sealed segment + sidecar and
|
* Fold consecutive generations into ONE new sealed segment + sidecar and
|
||||||
* append it to the manifest atomically. Caller guarantees: `gens` is
|
* append it to the manifest atomically. Caller guarantees: `gens` is
|
||||||
|
|
@ -164,6 +218,38 @@ export class GenerationSegmentStore {
|
||||||
throw new Error('[GenerationSegments] fold() input must be strictly ascending')
|
throw new Error('[GenerationSegments] fold() input must be strictly ascending')
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// THE DENSITY LAW, MADE MECHANICAL.
|
||||||
|
//
|
||||||
|
// A sealed segment declares a CONTIGUOUS range [firstGeneration,
|
||||||
|
// lastGeneration] and every reader treats that range as containment:
|
||||||
|
// `coveringSegment` is an interval test, `hasGeneration` returns true for
|
||||||
|
// anything inside it, and `open()` seeds committedRanges from it. So a
|
||||||
|
// segment folded from a SPARSE input silently claims generations it does
|
||||||
|
// not hold, and the first read of one of those holes throws
|
||||||
|
// "inside sealed segment ... but has no frame — packed history is damaged".
|
||||||
|
//
|
||||||
|
// That is exactly how the damage was produced. `repackHistory` skipped
|
||||||
|
// generations mid-batch — ones absent from committedRanges, ones still in
|
||||||
|
// the pending buffer, ones whose tx.json would not read — and handed the
|
||||||
|
// survivors here, where the range was computed from the first and last of
|
||||||
|
// them. Worse, the mis-declared range was then merged back into
|
||||||
|
// committedRanges at the next open, which is what turned a quiet hole into
|
||||||
|
// a repeating auto-compaction failure on every subsequent run.
|
||||||
|
//
|
||||||
|
// Callers now split at discontinuities; this refusal is what keeps any
|
||||||
|
// future caller from reintroducing the class. A refusal here loses
|
||||||
|
// nothing — the generations stay in the live tier, readable, and the next
|
||||||
|
// pass folds them correctly.
|
||||||
|
for (let i = 1; i < gens.length; i++) {
|
||||||
|
if (gens[i].generation !== gens[i - 1].generation + 1) {
|
||||||
|
throw new Error(
|
||||||
|
`[GenerationSegments] fold() input is not contiguous: ${gens[i - 1].generation} → ` +
|
||||||
|
`${gens[i].generation} skips ${gens[i].generation - gens[i - 1].generation - 1} ` +
|
||||||
|
`generation(s). A sealed segment declares a dense range, so folding a sparse ` +
|
||||||
|
`batch would claim generations it does not hold. Split the batch at the gap.`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
const last = this.manifest.segments[this.manifest.segments.length - 1]
|
const last = this.manifest.segments[this.manifest.segments.length - 1]
|
||||||
if (last && gens[0].generation <= last.lastGeneration) {
|
if (last && gens[0].generation <= last.lastGeneration) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
|
|
@ -364,12 +450,37 @@ export class GenerationSegmentStore {
|
||||||
return this.decodeFrame(payload)
|
return this.decodeFrame(payload)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// In the covering range but not present: the packed tier is dense by
|
// Inside the covering range but with no frame. Two very different causes,
|
||||||
// construction (fold packs every generation it is handed, including
|
// and conflating them is what made this class wedge every maintenance pass
|
||||||
// record-less ones) — absence inside a sealed range is damage.
|
// on the affected stores.
|
||||||
|
//
|
||||||
|
// (1) A SPARSE SEGMENT — the manifest's own `frames` count is smaller than
|
||||||
|
// the span it declares. That segment was sealed by a writer that
|
||||||
|
// folded across a hole (the class this file's density law now bars).
|
||||||
|
// The segment is INTACT and nothing is lost; it simply never held this
|
||||||
|
// generation. Answering "not packed" is the honest answer, and it lets
|
||||||
|
// the caller's two-tier read decide what a genuinely absent generation
|
||||||
|
// means, instead of every compaction pass dying on a repeating throw.
|
||||||
|
// `actualRanges()` keeps such holes out of committedRanges at open, so
|
||||||
|
// in a healed store nobody asks this question in the first place.
|
||||||
|
//
|
||||||
|
// (2) A DENSE SEGMENT missing a frame it says it has — the manifest and
|
||||||
|
// the sidecar disagree about a segment that claims to be complete.
|
||||||
|
// That IS damage, and it stays loud.
|
||||||
|
if (this.isSparse(meta)) {
|
||||||
|
prodLog.warn(
|
||||||
|
`[GenerationSegments] generation ${gen} falls inside sealed segment ${meta.file}'s ` +
|
||||||
|
`declared range ${meta.firstGeneration}..${meta.lastGeneration}, but that segment ` +
|
||||||
|
`holds ${meta.frames} frame(s) for a ${meta.lastGeneration - meta.firstGeneration + 1}` +
|
||||||
|
`-generation span — it was sealed across a gap and never held this generation. ` +
|
||||||
|
`Reporting it as unpacked rather than as damage; no record is lost.`
|
||||||
|
)
|
||||||
|
return null
|
||||||
|
}
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`[GenerationSegments] generation ${gen} is inside sealed segment ${meta.file}'s declared ` +
|
`[GenerationSegments] generation ${gen} is inside sealed segment ${meta.file}'s declared ` +
|
||||||
`range but has no frame — packed history is damaged`
|
`range but has no frame, and that segment declares a complete ${meta.frames}-frame ` +
|
||||||
|
`span — the manifest and the sidecar disagree; packed history is damaged`
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -96,6 +96,35 @@ export const FOLD_CHECKPOINT_PATH = '_system/fold-checkpoint.json'
|
||||||
/** Storage-root-relative prefix of the per-generation record directories. */
|
/** Storage-root-relative prefix of the per-generation record directories. */
|
||||||
export const GENERATIONS_PREFIX = '_generations'
|
export const GENERATIONS_PREFIX = '_generations'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Split an ascending list of fold candidates into maximal
|
||||||
|
* CONTIGUOUS runs — `[7,8,9,12,13]` becomes `[[7,8,9],[12,13]]`.
|
||||||
|
*
|
||||||
|
* A sealed segment declares one dense range `[firstGeneration,
|
||||||
|
* lastGeneration]`, and every reader treats that range as containment. So a
|
||||||
|
* batch with a hole in it must never become one segment: it would claim a
|
||||||
|
* generation it does not hold, and the first read of that hole reports the
|
||||||
|
* packed history as damaged. One run, one segment — the ranges then describe
|
||||||
|
* exactly what the segments contain.
|
||||||
|
*
|
||||||
|
* @param gens - Fold candidates, strictly ascending by generation.
|
||||||
|
* @returns One array per contiguous run, in ascending order. Empty in, empty out.
|
||||||
|
*/
|
||||||
|
export function contiguousRuns(gens: FoldGeneration[]): FoldGeneration[][] {
|
||||||
|
const runs: FoldGeneration[][] = []
|
||||||
|
let run: FoldGeneration[] = []
|
||||||
|
for (const g of gens) {
|
||||||
|
const prev = run[run.length - 1]
|
||||||
|
if (prev !== undefined && g.generation !== prev.generation + 1) {
|
||||||
|
runs.push(run)
|
||||||
|
run = []
|
||||||
|
}
|
||||||
|
run.push(g)
|
||||||
|
}
|
||||||
|
if (run.length > 0) runs.push(run)
|
||||||
|
return runs
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @description Phases of the {@link GenerationStore.commitTransaction} commit
|
* @description Phases of the {@link GenerationStore.commitTransaction} commit
|
||||||
* protocol at which a test-only fault injector can simulate a process crash.
|
* protocol at which a test-only fault injector can simulate a process crash.
|
||||||
|
|
@ -537,12 +566,29 @@ export class GenerationStore {
|
||||||
this.horizonGen = finiteGen(manifest?.horizon, 'manifest horizon')
|
this.horizonGen = finiteGen(manifest?.horizon, 'manifest horizon')
|
||||||
this.counter = Math.max(finiteGen(counterFile?.generation, 'generation counter'), this.committed)
|
this.counter = Math.max(finiteGen(counterFile?.generation, 'generation counter'), this.committed)
|
||||||
|
|
||||||
// Discover existing generation record directories.
|
// Discover existing generation record directories — BY DIRECTORY NAME.
|
||||||
const recordPaths = await this.storage.listRawObjects(GENERATIONS_PREFIX)
|
// This used to call listRawObjects(), which recurses the whole
|
||||||
|
// `_generations/` tree and returns every file in every generation, to
|
||||||
|
// extract a set of integers the top-level directory names already spell.
|
||||||
|
// MEASURED on a real store with an 11 GB generation history: the phase
|
||||||
|
// this sits in cost 55,538 ms of a WARM REOPEN after a clean close, with
|
||||||
|
// no fold to blame — this walk is what it was doing. An adapter without
|
||||||
|
// the one-level door falls back to the recursive listing, unchanged.
|
||||||
const seenGens = new Set<number>()
|
const seenGens = new Set<number>()
|
||||||
for (const p of recordPaths) {
|
const oneLevel = (
|
||||||
const gen = parseGenerationFromPath(p)
|
this.storage as { listRawPrefixes?: (prefix: string) => Promise<string[]> }
|
||||||
if (gen !== null) seenGens.add(gen)
|
).listRawPrefixes
|
||||||
|
if (typeof oneLevel === 'function') {
|
||||||
|
for (const name of await oneLevel.call(this.storage, GENERATIONS_PREFIX)) {
|
||||||
|
const gen = Number(name)
|
||||||
|
if (Number.isSafeInteger(gen) && gen >= 0) seenGens.add(gen)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
const recordPaths = await this.storage.listRawObjects(GENERATIONS_PREFIX)
|
||||||
|
for (const p of recordPaths) {
|
||||||
|
const gen = parseGenerationFromPath(p)
|
||||||
|
if (gen !== null) seenGens.add(gen)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let rolledBack = 0
|
let rolledBack = 0
|
||||||
|
|
@ -652,6 +698,7 @@ export class GenerationStore {
|
||||||
: 'WHOLE-LOG fold'
|
: 'WHOLE-LOG fold'
|
||||||
: 'above-manifest replay'
|
: 'above-manifest replay'
|
||||||
let replayed = 0
|
let replayed = 0
|
||||||
|
const foldStartedAt = Date.now()
|
||||||
const replayFact = async (fact: CommitFact): Promise<void> => {
|
const replayFact = async (fact: CommitFact): Promise<void> => {
|
||||||
for (const op of fact.ops) {
|
for (const op of fact.ops) {
|
||||||
let image: { metadata: unknown | null; vector: unknown | null }
|
let image: { metadata: unknown | null; vector: unknown | null }
|
||||||
|
|
@ -697,9 +744,10 @@ export class GenerationStore {
|
||||||
}
|
}
|
||||||
replayed++
|
replayed++
|
||||||
if (replayed % 1000 === 0) {
|
if (replayed % 1000 === 0) {
|
||||||
prodLog.warn(
|
prodLog.narrate(
|
||||||
`[GenerationStore] recovery fold in progress — ${replayed} fact(s) folded ` +
|
`[GenerationStore] recovery fold in progress — ${replayed} fact(s) folded ` +
|
||||||
`(at generation ${fact.generation}); do not restart, the fold is finite`
|
`in ${Date.now() - foldStartedAt}ms (at generation ${fact.generation}); ` +
|
||||||
|
`do not restart, the fold is finite`
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
if (fact.generation > this.committed) {
|
if (fact.generation > this.committed) {
|
||||||
|
|
@ -714,7 +762,7 @@ export class GenerationStore {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (uncleanOpen) {
|
if (uncleanOpen) {
|
||||||
prodLog.warn(
|
prodLog.narrate(
|
||||||
`[GenerationStore] log-authority recovery: ${foldKind} beginning ` +
|
`[GenerationStore] log-authority recovery: ${foldKind} beginning ` +
|
||||||
`(unclean shutdown detected) — streaming replay, bounded memory, ` +
|
`(unclean shutdown detected) — streaming replay, bounded memory, ` +
|
||||||
`progress every 1000 facts. Do not restart the process; a restart ` +
|
`progress every 1000 facts. Do not restart the process; a restart ` +
|
||||||
|
|
@ -737,9 +785,10 @@ export class GenerationStore {
|
||||||
}
|
}
|
||||||
await this.storage.writeRawObject(MANIFEST_PATH, manifest)
|
await this.storage.writeRawObject(MANIFEST_PATH, manifest)
|
||||||
await this.storage.syncRawObjects([MANIFEST_PATH])
|
await this.storage.syncRawObjects([MANIFEST_PATH])
|
||||||
prodLog.warn(
|
prodLog.narrate(
|
||||||
`[GenerationStore] log-authority recovery replayed ${replayed} fact(s) into ` +
|
`[GenerationStore] log-authority recovery replayed ${replayed} fact(s) into ` +
|
||||||
`canonical (${foldKind}; committed at ${this.committed}) — an acked write is never lost`
|
`canonical in ${Date.now() - foldStartedAt}ms (${foldKind}; committed at ` +
|
||||||
|
`${this.committed}) — an acked write is never lost`
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
// A recovery fold re-applied (and the barrier below re-syncs) every
|
// A recovery fold re-applied (and the barrier below re-syncs) every
|
||||||
|
|
@ -764,9 +813,15 @@ export class GenerationStore {
|
||||||
if (storageSupportsFactLog(this.storage)) {
|
if (storageSupportsFactLog(this.storage)) {
|
||||||
this.segments = new GenerationSegmentStore(this.storage)
|
this.segments = new GenerationSegmentStore(this.storage)
|
||||||
await this.segments.open()
|
await this.segments.open()
|
||||||
const packedRanges = this.segments
|
// ACTUAL ranges, not declared ones. A segment sealed by a pre-density-law
|
||||||
.segments()
|
// writer can declare a span wider than the frames it holds; seeding
|
||||||
.map((s): [number, number] => [s.firstGeneration, Math.min(s.lastGeneration, this.committed)])
|
// committedRanges from the declared span re-admits those holes as
|
||||||
|
// committed generations, and every later maintenance pass then asks for a
|
||||||
|
// frame that was never written. `actualRanges()` reads the real
|
||||||
|
// generation list from the sidecar for exactly those segments (and does
|
||||||
|
// no I/O for the dense ones, which is all of them on a healthy store).
|
||||||
|
const packedRanges = (await this.segments.actualRanges())
|
||||||
|
.map((r): [number, number] => [r[0], Math.min(r[1], this.committed)])
|
||||||
.filter(([lo, hi]) => lo <= hi)
|
.filter(([lo, hi]) => lo <= hi)
|
||||||
if (packedRanges.length > 0) {
|
if (packedRanges.length > 0) {
|
||||||
// Merge packed (older) + live (newer) interval sets — both ascending;
|
// Merge packed (older) + live (newer) interval sets — both ascending;
|
||||||
|
|
@ -3101,13 +3156,26 @@ export class GenerationStore {
|
||||||
foldInput.push({ generation: gen, timestamp: delta.timestamp, delta, records })
|
foldInput.push({ generation: gen, timestamp: delta.timestamp, delta, records })
|
||||||
}
|
}
|
||||||
if (foldInput.length === 0) continue
|
if (foldInput.length === 0) continue
|
||||||
await segments.fold(foldInput)
|
// SPLIT AT DISCONTINUITIES. `eligible` is NOT contiguous — three
|
||||||
segmentsCreated++
|
// filters above punch holes in it: a generation missing from
|
||||||
// Segment + manifest durable → the live copies retire.
|
// committedRanges never appears, one still in the pending buffer is
|
||||||
for (const g of foldInput) {
|
// skipped, and one whose tx.json will not read is skipped. A sealed
|
||||||
await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`)
|
// segment declares a DENSE range, so folding across such a hole makes
|
||||||
|
// the segment claim a generation it does not hold; the next open
|
||||||
|
// merges that mis-declared range into committedRanges, and every
|
||||||
|
// subsequent auto-compaction pass then asks for the missing frame and
|
||||||
|
// fails with "packed history is damaged". Fold each contiguous RUN as
|
||||||
|
// its own segment instead — same bytes, honest ranges.
|
||||||
|
for (const run of contiguousRuns(foldInput)) {
|
||||||
|
if (deadline !== undefined && Date.now() >= deadline) break
|
||||||
|
await segments.fold(run)
|
||||||
|
segmentsCreated++
|
||||||
|
// Segment + manifest durable → the live copies retire.
|
||||||
|
for (const g of run) {
|
||||||
|
await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`)
|
||||||
|
}
|
||||||
|
folded += run.length
|
||||||
}
|
}
|
||||||
folded += foldInput.length
|
|
||||||
}
|
}
|
||||||
if (folded > 0) {
|
if (folded > 0) {
|
||||||
prodLog.info(
|
prodLog.info(
|
||||||
|
|
|
||||||
|
|
@ -450,6 +450,21 @@ export interface GenerationStorage {
|
||||||
deleteRawObject(path: string): Promise<void>
|
deleteRawObject(path: string): Promise<void>
|
||||||
/** List raw object paths under a prefix (normalized, `.gz`-stripped). */
|
/** List raw object paths under a prefix (normalized, `.gz`-stripped). */
|
||||||
listRawObjects(prefix: string): Promise<string[]>
|
listRawObjects(prefix: string): Promise<string[]>
|
||||||
|
/**
|
||||||
|
* OPTIONAL: the IMMEDIATE child directory names under a prefix — one level,
|
||||||
|
* no recursion, no file paths.
|
||||||
|
*
|
||||||
|
* Why it exists: discovering which generations are on disk needs only the
|
||||||
|
* top-level directory NAMES under `_generations/`, but the only door for it
|
||||||
|
* was `listRawObjects`, which recurses the whole tree and returns every file
|
||||||
|
* in every generation. On a store with a long history that is a full walk of
|
||||||
|
* the entire generation log, paid on EVERY open, to learn a set of integers
|
||||||
|
* the directory names already spell out.
|
||||||
|
*
|
||||||
|
* An adapter without this door keeps working — the caller falls back to the
|
||||||
|
* recursive listing.
|
||||||
|
*/
|
||||||
|
listRawPrefixes?(prefix: string): Promise<string[]>
|
||||||
/** Remove every object under a prefix (and the directory itself on disk). */
|
/** Remove every object under a prefix (and the directory itself on disk). */
|
||||||
removeRawPrefix(prefix: string): Promise<void>
|
removeRawPrefix(prefix: string): Promise<void>
|
||||||
/** Durability barrier: fsync the given object paths (no-op in memory). */
|
/** Durability barrier: fsync the given object paths (no-op in memory). */
|
||||||
|
|
|
||||||
|
|
@ -128,7 +128,7 @@ async function loadBunAssets(): Promise<ModelAssets> {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Strategy 2: node_modules path relative to CWD (for installed packages)
|
// Strategy 2: node_modules path relative to CWD (for installed packages)
|
||||||
const nmPath = './node_modules/@soulcraft/brainy/assets/models/all-MiniLM-L6-v2'
|
const nmPath = './node_modules/@soulcraftlabs/brainy/assets/models/all-MiniLM-L6-v2'
|
||||||
pathsToTry.push([
|
pathsToTry.push([
|
||||||
`${nmPath}/model.safetensors`,
|
`${nmPath}/model.safetensors`,
|
||||||
`${nmPath}/tokenizer.json`,
|
`${nmPath}/tokenizer.json`,
|
||||||
|
|
@ -168,9 +168,9 @@ async function loadBunAssets(): Promise<ModelAssets> {
|
||||||
// If all strategies fail, provide helpful error message
|
// If all strategies fail, provide helpful error message
|
||||||
throw new Error(
|
throw new Error(
|
||||||
'Could not load model assets. For bun --compile, ensure model files are accessible:\n' +
|
'Could not load model assets. For bun --compile, ensure model files are accessible:\n' +
|
||||||
' Option 1: Keep node_modules/@soulcraft/brainy/assets/ alongside your binary\n' +
|
' Option 1: Keep node_modules/@soulcraftlabs/brainy/assets/ alongside your binary\n' +
|
||||||
' Option 2: Copy assets/ folder to your working directory\n' +
|
' Option 2: Copy assets/ folder to your working directory\n' +
|
||||||
' Option 3: Use --asset flag: bun build --compile --asset="./node_modules/@soulcraft/brainy/assets/**/*"'
|
' Option 3: Use --asset flag: bun build --compile --asset="./node_modules/@soulcraftlabs/brainy/assets/**/*"'
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -190,7 +190,7 @@ async function loadNodeAssets(): Promise<ModelAssets> {
|
||||||
if (!fs.existsSync(assetsDir)) {
|
if (!fs.existsSync(assetsDir)) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`Model assets not found: ${assetsDir}\n` +
|
`Model assets not found: ${assetsDir}\n` +
|
||||||
`Ensure @soulcraft/brainy is installed correctly.`
|
`Ensure @soulcraftlabs/brainy is installed correctly.`
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,7 @@
|
||||||
* - {@link RelationNotFoundError} — a referenced relationship (verb) does
|
* - {@link RelationNotFoundError} — a referenced relationship (verb) does
|
||||||
* not exist.
|
* not exist.
|
||||||
*
|
*
|
||||||
* Both are exported from the package root (`@soulcraft/brainy`).
|
* Both are exported from the package root (`@soulcraftlabs/brainy`).
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -1052,6 +1052,17 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
|
||||||
*/
|
*/
|
||||||
private startAutoFlush(): void {
|
private startAutoFlush(): void {
|
||||||
this.flushTimer = setInterval(async () => {
|
this.flushTimer = setInterval(async () => {
|
||||||
|
// NO PERIODIC WORK WITHOUT A CAUSE. Ask first, in two O(1) reads: an
|
||||||
|
// index nobody has written to since the last flush has nothing to
|
||||||
|
// write, and calling into the trees (and their logging) on a cadence
|
||||||
|
// over a quiet store is exactly the idle cost this law exists to
|
||||||
|
// remove.
|
||||||
|
if (
|
||||||
|
!this.lsmTreeVerbsBySource.hasPendingWrites() &&
|
||||||
|
!this.lsmTreeVerbsByTarget.hasPendingWrites()
|
||||||
|
) {
|
||||||
|
return
|
||||||
|
}
|
||||||
await this.flush()
|
await this.flush()
|
||||||
}, this.config.flushInterval)
|
}, this.config.flushInterval)
|
||||||
// Background maintenance must never keep the host process alive —
|
// Background maintenance must never keep the host process alive —
|
||||||
|
|
|
||||||
|
|
@ -687,6 +687,17 @@ export class LSMTree {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Whether this tree holds anything a flush would write —
|
||||||
|
* the MemTable is non-empty. Synchronous and O(1), so a background cadence
|
||||||
|
* can ask before it does anything at all: the engine does no periodic work
|
||||||
|
* without a cause.
|
||||||
|
* @returns true when a flush would write; false when it would be a no-op.
|
||||||
|
*/
|
||||||
|
hasPendingWrites(): boolean {
|
||||||
|
return !this.memTable.isEmpty()
|
||||||
|
}
|
||||||
|
|
||||||
async close(): Promise<void> {
|
async close(): Promise<void> {
|
||||||
this.stopCompactionTimer()
|
this.stopCompactionTimer()
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -184,6 +184,7 @@ export {
|
||||||
|
|
||||||
// Export version utilities
|
// Export version utilities
|
||||||
export { getBrainyVersion } from './utils/version.js'
|
export { getBrainyVersion } from './utils/version.js'
|
||||||
|
export { contractVersion, BRAINY_CONTRACT_VERSION } from './utils/version.js'
|
||||||
|
|
||||||
// Export plugin system
|
// Export plugin system
|
||||||
export type { BrainyPlugin, BrainyPluginContext, StorageAdapterFactory } from './plugin.js'
|
export type { BrainyPlugin, BrainyPluginContext, StorageAdapterFactory } from './plugin.js'
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,7 @@
|
||||||
*
|
*
|
||||||
* @example Enable integrations (recommended)
|
* @example Enable integrations (recommended)
|
||||||
* ```typescript
|
* ```typescript
|
||||||
* import { Brainy } from '@soulcraft/brainy'
|
* import { Brainy } from '@soulcraftlabs/brainy'
|
||||||
*
|
*
|
||||||
* const brain = new Brainy({ integrations: true })
|
* const brain = new Brainy({ integrations: true })
|
||||||
* await brain.init()
|
* await brain.init()
|
||||||
|
|
|
||||||
|
|
@ -41,7 +41,7 @@ The `BrainyMCPService` has been refactored to separate the core functionality fr
|
||||||
### In Any Environment (Browser, Node.js, Server)
|
### In Any Environment (Browser, Node.js, Server)
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy, BrainyMCPAdapter, MCPAugmentationToolset } from '@soulcraft/brainy'
|
import { Brainy, BrainyMCPAdapter, MCPAugmentationToolset } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
// Create a Brainy instance
|
// Create a Brainy instance
|
||||||
const brainyData = new Brainy()
|
const brainyData = new Brainy()
|
||||||
|
|
@ -81,7 +81,7 @@ const toolResponse = await toolset.handleRequest({
|
||||||
### In Browser Environment (Core Functionality Only)
|
### In Browser Environment (Core Functionality Only)
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy, BrainyMCPService } from '@soulcraft/brainy'
|
import { Brainy, BrainyMCPService } from '@soulcraftlabs/brainy'
|
||||||
|
|
||||||
// Create a Brainy instance
|
// Create a Brainy instance
|
||||||
const brainyData = new Brainy()
|
const brainyData = new Brainy()
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,7 @@ import type { GraphIndexStats } from './graph/graphAdjacencyIndex.js'
|
||||||
|
|
||||||
// Re-export the provider contracts that already live closer to their
|
// Re-export the provider contracts that already live closer to their
|
||||||
// implementations so a plugin author (Cor) can import the *entire*
|
// implementations so a plugin author (Cor) can import the *entire*
|
||||||
// provider surface from one stable entrypoint: `@soulcraft/brainy/plugin`.
|
// provider surface from one stable entrypoint: `@soulcraftlabs/brainy/plugin`.
|
||||||
export type { ColumnStoreProvider } from './indexes/columnStore/types.js'
|
export type { ColumnStoreProvider } from './indexes/columnStore/types.js'
|
||||||
export type {
|
export type {
|
||||||
AggregationProvider,
|
AggregationProvider,
|
||||||
|
|
@ -41,7 +41,7 @@ export interface BrainyPlugin {
|
||||||
name: string
|
name: string
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Optional semver range of `@soulcraft/brainy` this plugin supports
|
* Optional semver range of `@soulcraftlabs/brainy` this plugin supports
|
||||||
* (e.g. `'>=8.0.0 <9.0.0'` or `'^8.0.0'`). When set and the running brainy is
|
* (e.g. `'>=8.0.0 <9.0.0'` or `'^8.0.0'`). When set and the running brainy is
|
||||||
* OUTSIDE the range, `init()` THROWS rather than silently falling back to the
|
* OUTSIDE the range, `init()` THROWS rather than silently falling back to the
|
||||||
* default JS engine. This is the version-coupling guard for the native
|
* default JS engine. This is the version-coupling guard for the native
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,8 @@ import {
|
||||||
StorageBatchConfig,
|
StorageBatchConfig,
|
||||||
SYSTEM_DIR,
|
SYSTEM_DIR,
|
||||||
STATISTICS_KEY,
|
STATISTICS_KEY,
|
||||||
WriterLockInfo
|
WriterLockInfo,
|
||||||
|
WriterCloseRecord
|
||||||
} from '../baseStorage.js'
|
} from '../baseStorage.js'
|
||||||
import { getBrainyVersion } from '../../utils/index.js'
|
import { getBrainyVersion } from '../../utils/index.js'
|
||||||
import { isAbsentError } from '../../utils/errorClassification.js'
|
import { isAbsentError } from '../../utils/errorClassification.js'
|
||||||
|
|
@ -99,7 +100,30 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
// timer rewrites the lock every 10s so stale-lock detection can tell a dead
|
// timer rewrites the lock every 10s so stale-lock detection can tell a dead
|
||||||
// writer from a slow one. The constant name matches the file path used.
|
// writer from a slow one. The constant name matches the file path used.
|
||||||
private static readonly WRITER_LOCK_FILE = '_writer.lock'
|
private static readonly WRITER_LOCK_FILE = '_writer.lock'
|
||||||
private static readonly WRITER_HEARTBEAT_MS = 10_000
|
/**
|
||||||
|
* The clean-close record at `locks/_writer.close` (see
|
||||||
|
* {@link WriterCloseRecord}). Written when the lock is released, consumed by
|
||||||
|
* the next claim, so an open can distinguish "the previous writer left" from
|
||||||
|
* "the previous writer died" without inferring either from a pid.
|
||||||
|
*/
|
||||||
|
private static readonly WRITER_CLOSE_FILE = '_writer.close'
|
||||||
|
/**
|
||||||
|
* How often the lock file's `lastHeartbeat` is rewritten.
|
||||||
|
*
|
||||||
|
* THIS IS OBSERVABILITY ONLY, and the cadence follows from that. Staleness
|
||||||
|
* is decided by PID LIVENESS alone (see isWriterLockStale) and the fence
|
||||||
|
* compares pid + hostname — no decision anywhere reads this timestamp. It
|
||||||
|
* exists so an operator inspecting a lock file, or reading the
|
||||||
|
* BRAINY_WRITER_LOCKED error, can judge liveness themselves.
|
||||||
|
*
|
||||||
|
* At 10s it was a lock-file WRITE every ten seconds per brain, forever: 2.1
|
||||||
|
* writes/s across a production process holding 21 idle brains, for a
|
||||||
|
* human-readable timestamp nothing computes with. At 60s an operator still
|
||||||
|
* sees a heartbeat inside the minute, at a sixth of the cost. With the
|
||||||
|
* clean-close record now recording orderly releases explicitly, the
|
||||||
|
* heartbeat carries even less weight than it did.
|
||||||
|
*/
|
||||||
|
private static readonly WRITER_HEARTBEAT_MS = 60_000
|
||||||
private static readonly WRITER_STALE_THRESHOLD_MS = 60_000
|
private static readonly WRITER_STALE_THRESHOLD_MS = 60_000
|
||||||
private writerLockHeartbeat?: NodeJS.Timeout
|
private writerLockHeartbeat?: NodeJS.Timeout
|
||||||
private writerLockInfo?: WriterLockInfo
|
private writerLockInfo?: WriterLockInfo
|
||||||
|
|
@ -112,6 +136,13 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
*/
|
*/
|
||||||
private writerHeartbeatInFlight?: Promise<void>
|
private writerHeartbeatInFlight?: Promise<void>
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The in-flight background count-ledger derivation, if one was needed at
|
||||||
|
* open. See {@link scheduleCountLedgerDerivation} — awaited only by
|
||||||
|
* {@link whenCountLedgerSettled}, never by a read.
|
||||||
|
*/
|
||||||
|
private countLedgerDerivation?: Promise<void>
|
||||||
|
|
||||||
// Flush-request RPC state. The writer polls `locks/_flush_requests/` for
|
// Flush-request RPC state. The writer polls `locks/_flush_requests/` for
|
||||||
// new `.req` files and emits `.ack` files in `locks/_flush_responses/` after
|
// new `.req` files and emits `.ack` files in `locks/_flush_responses/` after
|
||||||
// flushing. Inspectors call `requestFlushOverFilesystem` to drop a request
|
// flushing. Inspectors call `requestFlushOverFilesystem` to drop a request
|
||||||
|
|
@ -120,9 +151,16 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
private static readonly FLUSH_REQUEST_DIR = '_flush_requests'
|
private static readonly FLUSH_REQUEST_DIR = '_flush_requests'
|
||||||
private static readonly FLUSH_RESPONSE_DIR = '_flush_responses'
|
private static readonly FLUSH_RESPONSE_DIR = '_flush_responses'
|
||||||
private static readonly FLUSH_WATCH_INTERVAL_MS = 500
|
private static readonly FLUSH_WATCH_INTERVAL_MS = 500
|
||||||
|
/**
|
||||||
|
* The safety sweep behind the fs.watch: catches events an exotic filesystem
|
||||||
|
* dropped, and runs the stale-request GC. See startFlushRequestWatcher.
|
||||||
|
*/
|
||||||
|
private static readonly FLUSH_SAFETY_SWEEP_MS = 30_000
|
||||||
private static readonly FLUSH_POLL_INTERVAL_MS = 100
|
private static readonly FLUSH_POLL_INTERVAL_MS = 100
|
||||||
private static readonly FLUSH_REQUEST_TTL_MS = 60_000
|
private static readonly FLUSH_REQUEST_TTL_MS = 60_000
|
||||||
private flushWatcherInterval?: NodeJS.Timeout
|
private flushWatcherInterval?: NodeJS.Timeout
|
||||||
|
/** The inotify-backed watch on the request directory, when the FS supports one. */
|
||||||
|
private flushWatcher?: import('node:fs').FSWatcher
|
||||||
private flushWatcherInFlight = false
|
private flushWatcherInFlight = false
|
||||||
private flushWatcherOnRequest?: () => Promise<void>
|
private flushWatcherOnRequest?: () => Promise<void>
|
||||||
|
|
||||||
|
|
@ -671,6 +709,30 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
return pruned
|
return pruned
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description The IMMEDIATE child directory names under a prefix — ONE
|
||||||
|
* `readdir`, no recursion, no file paths. See the seam's JSDoc
|
||||||
|
* (`src/db/types.ts`) for what this replaced: discovering the generations on
|
||||||
|
* disk walked the entire generation log on every open, reading out every
|
||||||
|
* file in every generation, to learn the set of integers the top-level
|
||||||
|
* directory names already spell.
|
||||||
|
* @param prefix - Storage-root-relative directory prefix.
|
||||||
|
* @returns The child directory names (not paths); empty when the prefix does
|
||||||
|
* not exist.
|
||||||
|
*/
|
||||||
|
public override async listRawPrefixes(prefix: string): Promise<string[]> {
|
||||||
|
await this.ensureInitialized()
|
||||||
|
const fullPath = path.join(this.rootDir, prefix)
|
||||||
|
try {
|
||||||
|
const entries = await fs.promises.readdir(fullPath, { withFileTypes: true })
|
||||||
|
return entries.filter((e: { isDirectory: () => boolean }) => e.isDirectory())
|
||||||
|
.map((e: { name: string }) => e.name)
|
||||||
|
} catch (error: any) {
|
||||||
|
if (error?.code === 'ENOENT') return []
|
||||||
|
throw error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Primitive operation: List objects under path prefix
|
* Primitive operation: List objects under path prefix
|
||||||
* All metadata operations use this internally via base class routing
|
* All metadata operations use this internally via base class routing
|
||||||
|
|
@ -1881,18 +1943,41 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// THE CLEAN-CLOSE RECORD IS READ BEFORE ANY VERDICT (see
|
||||||
|
// WriterCloseRecord). A lock file whose release was RECORDED is
|
||||||
|
// bookkeeping left by an orderly shutdown, not evidence of anything —
|
||||||
|
// and that is true whether the previous holder was another process or
|
||||||
|
// an earlier instance in THIS one. A production restart reported
|
||||||
|
// "Re-acquiring writer lock ... this is a bug" immediately after a clean
|
||||||
|
// close, sending an operator hunting for a leak that did not exist.
|
||||||
|
const closeRecord = existing ? await this.readWriterCloseRecord() : null
|
||||||
|
const releasedCleanly =
|
||||||
|
existing !== null &&
|
||||||
|
closeRecord !== null &&
|
||||||
|
this.closeRecordVouchesFor(closeRecord, existing)
|
||||||
|
|
||||||
if (existing) {
|
if (existing) {
|
||||||
// Same-process re-open: a second Brainy instance in this Node process
|
// Same-process re-open: a second Brainy instance in this Node process
|
||||||
// (e.g. test "simulate server restart" patterns, or a consumer that
|
// (e.g. test "simulate server restart" patterns, or a consumer that
|
||||||
// explicitly re-instantiates without closing first). This isn't the
|
// explicitly re-instantiates without closing first). This isn't the
|
||||||
// dangerous cross-process case the lock exists to prevent — the two
|
// dangerous cross-process case the lock exists to prevent — the two
|
||||||
// instances share a memory space and can't silently diverge from each
|
// instances share a memory space and can't silently diverge from each
|
||||||
// other beyond what their callers already see. Warn and take over.
|
// other beyond what their callers already see. Warn and take over —
|
||||||
|
// unless the record proves the previous instance already let go, in
|
||||||
|
// which case there is nothing to warn about.
|
||||||
if (existing.pid === myPid && existing.hostname === hostname && !options?.force) {
|
if (existing.pid === myPid && existing.hostname === hostname && !options?.force) {
|
||||||
console.warn(
|
if (releasedCleanly) {
|
||||||
`[brainy] Re-acquiring writer lock for ${this.rootDir} held by the same process (PID ${existing.pid}). ` +
|
console.warn(
|
||||||
`If you intended to keep the previous Brainy instance alive, this is a bug — close it first.`
|
`[brainy] Clearing the leftover writer lock for ${this.rootDir} — an earlier ` +
|
||||||
)
|
`instance in this process (PID ${existing.pid}) RELEASED it cleanly at ` +
|
||||||
|
`${closeRecord!.closedAt} but could not remove the file. Nothing to recover.`
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
console.warn(
|
||||||
|
`[brainy] Re-acquiring writer lock for ${this.rootDir} held by the same process (PID ${existing.pid}). ` +
|
||||||
|
`If you intended to keep the previous Brainy instance alive, this is a bug — close it first.`
|
||||||
|
)
|
||||||
|
}
|
||||||
const info: WriterLockInfo = {
|
const info: WriterLockInfo = {
|
||||||
pid: myPid,
|
pid: myPid,
|
||||||
hostname,
|
hostname,
|
||||||
|
|
@ -1902,11 +1987,18 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
rootDir: this.rootDir
|
rootDir: this.rootDir
|
||||||
}
|
}
|
||||||
await this.writeFileAtomic(lockFile, JSON.stringify(info, null, 2))
|
await this.writeFileAtomic(lockFile, JSON.stringify(info, null, 2))
|
||||||
|
await this.clearWriterCloseRecord()
|
||||||
this.installWriterLock(info)
|
this.installWriterLock(info)
|
||||||
return info
|
return info
|
||||||
}
|
}
|
||||||
|
|
||||||
const stale = !options?.force && (await this.isWriterLockStale(existing))
|
// A cleanly-released lock is stale by RECORD, not by inference. Only
|
||||||
|
// when no record vouches for this lock do we fall back to pid
|
||||||
|
// liveness, and then we say THAT honestly too: an unrecorded lock
|
||||||
|
// means the writer did not complete its close, so the store was not
|
||||||
|
// closed cleanly and this open pays recovery.
|
||||||
|
const stale =
|
||||||
|
releasedCleanly || (!options?.force && (await this.isWriterLockStale(existing)))
|
||||||
if (!options?.force && !stale) {
|
if (!options?.force && !stale) {
|
||||||
// Consumer-facing error contract: callers detect this case via
|
// Consumer-facing error contract: callers detect this case via
|
||||||
// err.code and read the holder's details from err.lockInfo.
|
// err.code and read the holder's details from err.lockInfo.
|
||||||
|
|
@ -1917,8 +2009,16 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
options?.force
|
options?.force
|
||||||
? `[brainy] Force-overwriting writer lock for ${this.rootDir} ` +
|
? `[brainy] Force-overwriting writer lock for ${this.rootDir} ` +
|
||||||
`(was held by PID ${existing.pid} on ${existing.hostname}).`
|
`(was held by PID ${existing.pid} on ${existing.hostname}).`
|
||||||
: `[brainy] Overwriting stale writer lock for ${this.rootDir} ` +
|
: releasedCleanly
|
||||||
`(PID ${existing.pid} on ${existing.hostname} appears dead).`
|
? `[brainy] Clearing the leftover writer lock for ${this.rootDir} — ` +
|
||||||
|
`PID ${existing.pid} on ${existing.hostname} RELEASED it cleanly at ` +
|
||||||
|
`${closeRecord!.closedAt} but could not remove the file. ` +
|
||||||
|
`Nothing to recover.`
|
||||||
|
: `[brainy] Overwriting stale writer lock for ${this.rootDir} ` +
|
||||||
|
`(PID ${existing.pid} on ${existing.hostname} is gone and left NO ` +
|
||||||
|
`clean-close record — that writer did not finish closing, so this ` +
|
||||||
|
`store was not closed cleanly; open will run crash recovery and ` +
|
||||||
|
`report its wall).`
|
||||||
)
|
)
|
||||||
// Takeover: verify the file still holds the lock we judged (a live
|
// Takeover: verify the file still holds the lock we judged (a live
|
||||||
// successor may have claimed meanwhile), then remove it and fall
|
// successor may have claimed meanwhile), then remove it and fall
|
||||||
|
|
@ -1972,6 +2072,12 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
await fs.promises.unlink(claimTmp).catch(() => {})
|
await fs.promises.unlink(claimTmp).catch(() => {})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CONSUME the previous writer's clean-close record. It described the
|
||||||
|
// lock generation that just ended; leaving it in place would let it
|
||||||
|
// vouch for OUR lock if this process later dies without closing —
|
||||||
|
// turning a real crash into a "closed cleanly" verdict. One unlink.
|
||||||
|
await this.clearWriterCloseRecord()
|
||||||
|
|
||||||
this.installWriterLock(info)
|
this.installWriterLock(info)
|
||||||
return info
|
return info
|
||||||
}
|
}
|
||||||
|
|
@ -2095,13 +2201,27 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE)
|
const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE)
|
||||||
|
const released = this.writerLockInfo
|
||||||
try {
|
try {
|
||||||
// Only delete if we still own it — avoid clobbering a successor that
|
// Only delete if we still own it — avoid clobbering a successor that
|
||||||
// claimed the lock via force-override.
|
// claimed the lock via force-override.
|
||||||
const current = await this.readWriterLock()
|
const current = await this.readWriterLock()
|
||||||
if (current && current.pid === this.writerLockInfo.pid && current.hostname === this.writerLockInfo.hostname) {
|
const ours =
|
||||||
|
current === null ||
|
||||||
|
(current.pid === released.pid && current.hostname === released.hostname)
|
||||||
|
if (current && ours) {
|
||||||
await fs.promises.unlink(lockFile)
|
await fs.promises.unlink(lockFile)
|
||||||
}
|
}
|
||||||
|
// THE CLEAN-CLOSE RECORD (see WriterCloseRecord). Written whenever this
|
||||||
|
// instance gives up a lock nobody else has taken — the unlink above
|
||||||
|
// having succeeded OR the file already being gone. The next open reads
|
||||||
|
// it instead of guessing from pid liveness: a recorded release is an
|
||||||
|
// orderly shutdown, an absent record is a writer that never finished
|
||||||
|
// closing. Not written when a successor holds the lock: our release is
|
||||||
|
// then a no-op and a record would slander their live lock.
|
||||||
|
if (ours) {
|
||||||
|
await this.writeWriterCloseRecord(released)
|
||||||
|
}
|
||||||
} catch (err: any) {
|
} catch (err: any) {
|
||||||
if (err.code !== 'ENOENT') {
|
if (err.code !== 'ENOENT') {
|
||||||
console.warn('[brainy] Failed to release writer lock file:', err)
|
console.warn('[brainy] Failed to release writer lock file:', err)
|
||||||
|
|
@ -2111,6 +2231,97 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Read the clean-close record at `locks/_writer.close`, or
|
||||||
|
* `null` when it is absent or unparseable. A torn record is treated as
|
||||||
|
* absent — the conservative direction, since an unreadable record can
|
||||||
|
* vouch for nothing.
|
||||||
|
* @returns The record, or null.
|
||||||
|
*/
|
||||||
|
public async readWriterCloseRecord(): Promise<WriterCloseRecord | null> {
|
||||||
|
await this.ensureInitialized()
|
||||||
|
const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE)
|
||||||
|
try {
|
||||||
|
const raw = await fs.promises.readFile(recordFile, 'utf-8')
|
||||||
|
const parsed = JSON.parse(raw) as WriterCloseRecord
|
||||||
|
if (
|
||||||
|
typeof parsed?.pid !== 'number' ||
|
||||||
|
typeof parsed?.hostname !== 'string' ||
|
||||||
|
typeof parsed?.startedAt !== 'string' ||
|
||||||
|
typeof parsed?.closedAt !== 'string'
|
||||||
|
) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
return parsed
|
||||||
|
} catch (err: any) {
|
||||||
|
if (err.code === 'ENOENT') return null
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Whether a clean-close record describes the very lock
|
||||||
|
* generation `lock` represents. The match is pid + hostname + `startedAt`:
|
||||||
|
* `startedAt` is the lock generation's identity, so a record can never
|
||||||
|
* vouch for a LATER lock taken by the same pid on the same host (the
|
||||||
|
* same-process re-open path mints a fresh `startedAt`).
|
||||||
|
* @param record - The clean-close record read from disk.
|
||||||
|
* @param lock - The lock file's contents.
|
||||||
|
*/
|
||||||
|
private closeRecordVouchesFor(record: WriterCloseRecord, lock: WriterLockInfo): boolean {
|
||||||
|
return (
|
||||||
|
record.pid === lock.pid &&
|
||||||
|
record.hostname === lock.hostname &&
|
||||||
|
record.startedAt === lock.startedAt
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Write the clean-close record for a lock this instance just
|
||||||
|
* released. Atomic (temp + rename) so a concurrent opener never reads half
|
||||||
|
* a record. A failure here costs the next open nothing but the honest
|
||||||
|
* fallback (pid liveness), so it warns rather than failing the close.
|
||||||
|
* @param released - The lock info this instance held.
|
||||||
|
*/
|
||||||
|
private async writeWriterCloseRecord(released: WriterLockInfo): Promise<void> {
|
||||||
|
const record: WriterCloseRecord = {
|
||||||
|
pid: released.pid,
|
||||||
|
hostname: released.hostname,
|
||||||
|
startedAt: released.startedAt,
|
||||||
|
closedAt: new Date().toISOString(),
|
||||||
|
version: released.version
|
||||||
|
}
|
||||||
|
const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE)
|
||||||
|
try {
|
||||||
|
await this.writeFileAtomic(recordFile, JSON.stringify(record, null, 2))
|
||||||
|
} catch (err) {
|
||||||
|
// ENOENT = the lock directory is gone, i.e. the whole store was removed
|
||||||
|
// under us. There is no next open to inform.
|
||||||
|
if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return
|
||||||
|
console.warn(
|
||||||
|
`[brainy] Failed to write the writer clean-close record for ${this.rootDir} — ` +
|
||||||
|
`the next open will fall back to pid liveness and may report this orderly ` +
|
||||||
|
`shutdown as a crash:`,
|
||||||
|
err
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Remove the clean-close record. Called by every successful
|
||||||
|
* lock claim so a record never outlives the lock generation it describes.
|
||||||
|
*/
|
||||||
|
private async clearWriterCloseRecord(): Promise<void> {
|
||||||
|
const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE)
|
||||||
|
try {
|
||||||
|
await fs.promises.unlink(recordFile)
|
||||||
|
} catch (err: any) {
|
||||||
|
if (err.code !== 'ENOENT') {
|
||||||
|
console.warn('[brainy] Failed to clear the writer clean-close record:', err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public override async readWriterLock(): Promise<WriterLockInfo | null> {
|
public override async readWriterLock(): Promise<WriterLockInfo | null> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE)
|
const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE)
|
||||||
|
|
@ -2197,36 +2408,115 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Start watching for cross-process flush requests. Called by Brainy.init()
|
* Start watching for cross-process flush requests. Called by Brainy.init()
|
||||||
* in writer mode. Polls `locks/_flush_requests/` every
|
* in writer mode. Each new `.req` file in `locks/_flush_requests/` triggers
|
||||||
* FLUSH_WATCH_INTERVAL_MS — each new `.req` file triggers the supplied
|
* the supplied callback (`brain.flush()`), after which an `.ack` is written
|
||||||
* callback (`brain.flush()`), after which an `.ack` is written to
|
* to `locks/_flush_responses/` with the same request ID. Stale `.req` files
|
||||||
* `locks/_flush_responses/` with the same request ID. Stale `.req` files
|
* (>FLUSH_REQUEST_TTL_MS) are garbage-collected on each sweep.
|
||||||
* (>FLUSH_REQUEST_TTL_MS) are garbage-collected on every tick.
|
*
|
||||||
|
* THE WATCH IS EVENT-DRIVEN, NOT A POLL. It used to `readdir` the request
|
||||||
|
* directory every 500 ms, per brain, for the entire life of every writer —
|
||||||
|
* armed on every non-reader brain whether or not any inspector process
|
||||||
|
* existed. MEASURED on a production process holding 21 brains: 42 directory
|
||||||
|
* reads per second on a completely idle service, plus a stale-request GC
|
||||||
|
* pass on every one of them. The engine does no periodic work without a
|
||||||
|
* cause, and a request that has not been made is not a cause.
|
||||||
|
*
|
||||||
|
* `fs.watch` (inotify on Linux) delivers the arrival itself, so a request is
|
||||||
|
* seen SOONER than the old poll saw it. Two honest concessions ride with it:
|
||||||
|
* - a slow SAFETY SWEEP (FLUSH_SAFETY_SWEEP_MS) still runs, because
|
||||||
|
* `fs.watch` can miss events on network and fuse filesystems and because
|
||||||
|
* the stale-request GC needs some tick of its own. At 30s that is 0.7
|
||||||
|
* reads/s across 21 brains where the poll cost 42.
|
||||||
|
* - a filesystem that cannot watch at all falls back to the ORIGINAL
|
||||||
|
* 500 ms poll, narrated once, because correctness outranks idle cost:
|
||||||
|
* an inspector whose request is never seen waits forever.
|
||||||
*/
|
*/
|
||||||
public override startFlushRequestWatcher(onRequest: () => Promise<void>): void {
|
public override startFlushRequestWatcher(onRequest: () => Promise<void>): void {
|
||||||
if (this.flushWatcherInterval) return // already watching
|
// Already watching — or already ARMING. The arm is asynchronous (the
|
||||||
|
// request directory is created before it can be watched), so neither the
|
||||||
|
// watcher nor the interval exists yet during that window; the callback is
|
||||||
|
// the flag that covers it. Without this a second call in the window would
|
||||||
|
// leave two watchers and two sweeps running for the life of the store.
|
||||||
|
if (this.flushWatcherInterval || this.flushWatcher || this.flushWatcherOnRequest) return
|
||||||
this.flushWatcherOnRequest = onRequest
|
this.flushWatcherOnRequest = onRequest
|
||||||
|
|
||||||
const reqDir = path.join(this.lockDir, FileSystemStorage.FLUSH_REQUEST_DIR)
|
const reqDir = path.join(this.lockDir, FileSystemStorage.FLUSH_REQUEST_DIR)
|
||||||
const ackDir = path.join(this.lockDir, FileSystemStorage.FLUSH_RESPONSE_DIR)
|
const ackDir = path.join(this.lockDir, FileSystemStorage.FLUSH_RESPONSE_DIR)
|
||||||
|
|
||||||
// Ensure both dirs exist up front so the first .req drop doesn't race with mkdir.
|
const sweep = (): void => {
|
||||||
this.ensureDirectoryExists(reqDir).catch(() => {})
|
if (this.flushWatcherInFlight) return // skip overlapping sweep
|
||||||
this.ensureDirectoryExists(ackDir).catch(() => {})
|
|
||||||
|
|
||||||
this.flushWatcherInterval = setInterval(() => {
|
|
||||||
if (this.flushWatcherInFlight) return // skip overlapping tick
|
|
||||||
this.flushWatcherInFlight = true
|
this.flushWatcherInFlight = true
|
||||||
this.processFlushRequests(reqDir, ackDir).finally(() => {
|
this.processFlushRequests(reqDir, ackDir).finally(() => {
|
||||||
this.flushWatcherInFlight = false
|
this.flushWatcherInFlight = false
|
||||||
})
|
})
|
||||||
}, FileSystemStorage.FLUSH_WATCH_INTERVAL_MS)
|
}
|
||||||
|
|
||||||
|
// Ensure both dirs exist up front so the first .req drop doesn't race with
|
||||||
|
// mkdir — and so there is a directory to watch.
|
||||||
|
void this.ensureDirectoryExists(reqDir)
|
||||||
|
.then(() => this.ensureDirectoryExists(ackDir))
|
||||||
|
.then(() => {
|
||||||
|
if (this.flushWatcherOnRequest !== onRequest) return // stopped meanwhile
|
||||||
|
try {
|
||||||
|
const watcher = fs.watch(reqDir, () => sweep())
|
||||||
|
this.flushWatcher = watcher
|
||||||
|
watcher.on('error', (err: Error) => {
|
||||||
|
// A watch that dies mid-life must not leave the door deaf.
|
||||||
|
console.warn(
|
||||||
|
`[brainy] Flush-request watch failed (${err.message}) — falling back to polling.`
|
||||||
|
)
|
||||||
|
this.flushWatcher?.close()
|
||||||
|
this.flushWatcher = undefined
|
||||||
|
// The SAFETY sweep must go first. It is already armed at 30s, and
|
||||||
|
// startFlushRequestPolling() declines to arm over an existing
|
||||||
|
// interval — so leaving it would quietly leave this store answering
|
||||||
|
// flush requests on a 30s cadence instead of the 500ms one the door
|
||||||
|
// promises. A degrade nobody asked for is still a degrade.
|
||||||
|
if (this.flushWatcherInterval) {
|
||||||
|
clearInterval(this.flushWatcherInterval)
|
||||||
|
this.flushWatcherInterval = undefined
|
||||||
|
}
|
||||||
|
this.startFlushRequestPolling(sweep)
|
||||||
|
})
|
||||||
|
if (typeof watcher.unref === 'function') watcher.unref()
|
||||||
|
// The safety sweep: missed events on exotic filesystems, and the
|
||||||
|
// stale-request GC.
|
||||||
|
this.flushWatcherInterval = setInterval(sweep, FileSystemStorage.FLUSH_SAFETY_SWEEP_MS)
|
||||||
|
if (typeof this.flushWatcherInterval.unref === 'function') {
|
||||||
|
this.flushWatcherInterval.unref()
|
||||||
|
}
|
||||||
|
// One sweep now: a request may have been dropped before the watch armed.
|
||||||
|
sweep()
|
||||||
|
} catch (err) {
|
||||||
|
console.warn(
|
||||||
|
`[brainy] Flush-request directory cannot be watched on this filesystem ` +
|
||||||
|
`(${(err as Error).message}) — polling every ` +
|
||||||
|
`${FileSystemStorage.FLUSH_WATCH_INTERVAL_MS}ms instead.`
|
||||||
|
)
|
||||||
|
this.startFlushRequestPolling(sweep)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.catch(() => {
|
||||||
|
// The request directory could not be created; nothing to watch. A
|
||||||
|
// cross-process flush request cannot be made either, so there is
|
||||||
|
// nothing to miss.
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The original 500 ms poll — the fallback when a directory cannot be watched. */
|
||||||
|
private startFlushRequestPolling(sweep: () => void): void {
|
||||||
|
if (this.flushWatcherInterval) return
|
||||||
|
this.flushWatcherInterval = setInterval(sweep, FileSystemStorage.FLUSH_WATCH_INTERVAL_MS)
|
||||||
if (typeof this.flushWatcherInterval.unref === 'function') {
|
if (typeof this.flushWatcherInterval.unref === 'function') {
|
||||||
this.flushWatcherInterval.unref()
|
this.flushWatcherInterval.unref()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public override stopFlushRequestWatcher(): void {
|
public override stopFlushRequestWatcher(): void {
|
||||||
|
if (this.flushWatcher) {
|
||||||
|
this.flushWatcher.close()
|
||||||
|
this.flushWatcher = undefined
|
||||||
|
}
|
||||||
if (this.flushWatcherInterval) {
|
if (this.flushWatcherInterval) {
|
||||||
clearInterval(this.flushWatcherInterval)
|
clearInterval(this.flushWatcherInterval)
|
||||||
this.flushWatcherInterval = undefined
|
this.flushWatcherInterval = undefined
|
||||||
|
|
@ -2623,25 +2913,29 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
this.allCountsDerivedBy = undefined
|
this.allCountsDerivedBy = undefined
|
||||||
this.allCountsSuspect = true
|
this.allCountsSuspect = true
|
||||||
needsPersist = true
|
needsPersist = true
|
||||||
prodLog.warn(
|
prodLog.narrate(
|
||||||
'[FileSystemStorage] canonical count ledger was derived under the legacy ' +
|
'[FileSystemStorage] canonical count ledger was derived under the legacy ' +
|
||||||
'container rule — marked suspect; a sanctioned recount (repairIndex) restores ' +
|
'container rule — it counts one entity per id DIRECTORY, so every ghost/scar ' +
|
||||||
'exact denominators'
|
'container inflates it. Marked suspect, and an honest recount is scheduled to ' +
|
||||||
|
'run in the background after this open; until it lands, do not subtract ' +
|
||||||
|
'against these ALL scalars.'
|
||||||
)
|
)
|
||||||
|
// A suspect ledger used to stay wrong for the life of the store,
|
||||||
|
// waiting for an operator to run repairIndex. A downstream index
|
||||||
|
// heal took its "remaining" figure from these inflated
|
||||||
|
// denominators and reported work that did not exist. The ledger
|
||||||
|
// now HEALS ITSELF — in the background, because a denominator is
|
||||||
|
// a derived scalar and no read is ever served from it.
|
||||||
|
this.scheduleCountLedgerDerivation('legacy container-rule ledger')
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
const nouns = await this.scanCanonicalEntities('nouns')
|
// No ALL scalars at all. There is nothing to serve in the meantime —
|
||||||
const verbs = await this.scanCanonicalEntities('verbs')
|
// a zero would read as an empty store — so the scalars stay unknown
|
||||||
this.totalNounCountAll = nouns.count
|
// and SUSPECT until the background derivation lands. The open does
|
||||||
this.totalVerbCountAll = verbs.count
|
// not wait for it: an id-tree walk is O(ids) and this file has been
|
||||||
this.allCountsSuspect = false
|
// the whole reason a 24k-id store opened in silence.
|
||||||
this.allCountsDerivedBy = 'identity-record'
|
this.allCountsSuspect = true
|
||||||
console.warn(
|
this.scheduleCountLedgerDerivation('counts.json predates the ALL-visibility ledger')
|
||||||
`[FileSystemStorage] counts.json predates the ALL-visibility count ledger — ` +
|
|
||||||
`derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` +
|
|
||||||
`every tier) and persisted; no further scan.`
|
|
||||||
)
|
|
||||||
needsPersist = true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// The vectored-noun scalar (shipped after the ALL scalars above — a
|
// The vectored-noun scalar (shipped after the ALL scalars above — a
|
||||||
|
|
@ -2654,14 +2948,12 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
if (typeof counts.totalVectoredNounCount === 'number') {
|
if (typeof counts.totalVectoredNounCount === 'number') {
|
||||||
this.totalVectoredNounCount = counts.totalVectoredNounCount
|
this.totalVectoredNounCount = counts.totalVectoredNounCount
|
||||||
} else {
|
} else {
|
||||||
const vectored = await this.scanVectoredNounCount()
|
// O(nouns) CONTENT reads — the most expensive derivation of the
|
||||||
this.totalVectoredNounCount = vectored
|
// three, and the one most likely to have been the silent minutes at
|
||||||
console.warn(
|
// the front of a large store's open. Background, suspect until it
|
||||||
`[FileSystemStorage] counts.json predates the vectored-noun count ledger — ` +
|
// lands, same as the ALL scalars.
|
||||||
`derived once by reading every noun's vectors.json (${vectored} vectored) and ` +
|
this.allCountsSuspect = true
|
||||||
`persisted; no further scan.`
|
this.scheduleCountLedgerDerivation('counts.json predates the vectored-noun ledger')
|
||||||
)
|
|
||||||
needsPersist = true
|
|
||||||
}
|
}
|
||||||
if (needsPersist) {
|
if (needsPersist) {
|
||||||
await this.persistCounts()
|
await this.persistCounts()
|
||||||
|
|
@ -2690,6 +2982,22 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
* Initialize counts by scanning disk (only done once)
|
* Initialize counts by scanning disk (only done once)
|
||||||
*/
|
*/
|
||||||
private async initializeCountsFromDisk(): Promise<void> {
|
private async initializeCountsFromDisk(): Promise<void> {
|
||||||
|
const startedAt = Date.now()
|
||||||
|
// THIS ONE CANNOT LEAVE THE FOREGROUND, and the reason is worth stating:
|
||||||
|
// it derives `totalNounCount` / `totalVerbCount`, the scalars
|
||||||
|
// `getNounCount()` and `getVerbCount()` RETURN. Backgrounding it would
|
||||||
|
// make a populated store answer "0 entities" until the walk landed — a
|
||||||
|
// wrong answer, not a slow one, and the serving law grades a failure by
|
||||||
|
// whether an answer could be wrong. The ALL-visibility denominators, which
|
||||||
|
// no read is served from, DO run in the background (see
|
||||||
|
// scheduleCountLedgerDerivation). What this walk owes the operator instead
|
||||||
|
// is narration: it announces itself, and reports its wall.
|
||||||
|
prodLog.narrate(
|
||||||
|
`[FileSystemStorage] no usable counts.json — deriving the entity counters from ` +
|
||||||
|
`the canonical id tree now. This is O(ids) listings plus one vectors.json read ` +
|
||||||
|
`per noun, and it BLOCKS the open because getNounCount()/getVerbCount() are ` +
|
||||||
|
`served from it. It runs once; the result is persisted.`
|
||||||
|
)
|
||||||
try {
|
try {
|
||||||
// Count the CANONICAL 8.0 layout (`entities/<kind>/<shard>/<id>/…`) —
|
// Count the CANONICAL 8.0 layout (`entities/<kind>/<shard>/<id>/…`) —
|
||||||
// the tree saveNoun/getNouns actually read and write. The previous scan
|
// the tree saveNoun/getNouns actually read and write. The previous scan
|
||||||
|
|
@ -2737,6 +3045,11 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
await this.persistCounts()
|
await this.persistCounts()
|
||||||
|
prodLog.narrate(
|
||||||
|
`[FileSystemStorage] counter derivation from the canonical id tree finished in ` +
|
||||||
|
`${Date.now() - startedAt}ms: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs, ` +
|
||||||
|
`${this.totalVectoredNounCount} vectored nouns — persisted, stamped identity-record.`
|
||||||
|
)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error initializing counts from disk:', error)
|
console.error('Error initializing counts from disk:', error)
|
||||||
}
|
}
|
||||||
|
|
@ -2758,6 +3071,118 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
* directories (absolute paths) — nouns feed the type-distribution estimate
|
* directories (absolute paths) — nouns feed the type-distribution estimate
|
||||||
* above. An absent tree (fresh store) counts zero.
|
* above. An absent tree (fresh store) counts zero.
|
||||||
*/
|
*/
|
||||||
|
/**
|
||||||
|
* @description Derive the ALL-visibility count ledger honestly — one entity
|
||||||
|
* per IDENTITY RECORD, never per id directory — IN THE BACKGROUND, once,
|
||||||
|
* and persist the result stamped `identity-record`.
|
||||||
|
*
|
||||||
|
* Why background: these scalars are DENOMINATORS. No read is served from
|
||||||
|
* them, so deriving them cannot be allowed to hold an open hostage — a
|
||||||
|
* store with 24,898 ids spent minutes of a production restart inside walks
|
||||||
|
* exactly like these, in silence, before serving anything. Why at all: a
|
||||||
|
* ledger derived under the old container rule stayed wrong for the life of
|
||||||
|
* the store, and a downstream index heal subtracted against it and reported
|
||||||
|
* remaining work that did not exist (measured on a real store: 14,231
|
||||||
|
* derived against 14,056 identity records — precisely the store's 25 noun
|
||||||
|
* scar directories; verbs 72,729 against 72,679, its 50 verb scars).
|
||||||
|
*
|
||||||
|
* Idempotent: a second call while one is in flight joins the first.
|
||||||
|
* @param reason - What made the ledger untrustworthy, quoted in narration.
|
||||||
|
* @returns Nothing; observe completion with {@link whenCountLedgerSettled}.
|
||||||
|
*/
|
||||||
|
private scheduleCountLedgerDerivation(reason: string): void {
|
||||||
|
if (this.countLedgerDerivation) return
|
||||||
|
this.countLedgerDerivation = (async () => {
|
||||||
|
const startedAt = Date.now()
|
||||||
|
prodLog.narrate(
|
||||||
|
`[FileSystemStorage] count-ledger derivation started in the background ` +
|
||||||
|
`(${reason}) — counting identity records, not id directories; the open does ` +
|
||||||
|
`not wait for it and no read is served from these scalars.`
|
||||||
|
)
|
||||||
|
try {
|
||||||
|
const beforeNouns = this.totalNounCountAll
|
||||||
|
const beforeVerbs = this.totalVerbCountAll
|
||||||
|
const beforeVectored = this.totalVectoredNounCount
|
||||||
|
// A walk that RACED A WRITE cannot prove its number: a row that landed
|
||||||
|
// mid-walk may or may not have been in the shard the walk had already
|
||||||
|
// passed. Rather than persist a figure that might be off by one and
|
||||||
|
// stamp it "exact", the walk is repeated once on a quiet store, and if
|
||||||
|
// the store is never quiet the ledger stays SUSPECT and says so. One
|
||||||
|
// retry, never a spin.
|
||||||
|
let attempt = 0
|
||||||
|
let derived: { nouns: number; verbs: number; vectored: number } | null = null
|
||||||
|
while (attempt < 2 && derived === null) {
|
||||||
|
attempt++
|
||||||
|
const activityBefore = this.ledgerActivityStamp()
|
||||||
|
const nouns = await this.scanCanonicalEntities('nouns')
|
||||||
|
const verbs = await this.scanCanonicalEntities('verbs')
|
||||||
|
const vectored = await this.scanVectoredNounCount()
|
||||||
|
if (this.ledgerActivityStamp() === activityBefore) {
|
||||||
|
derived = { nouns: nouns.count, verbs: verbs.count, vectored }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (derived === null) {
|
||||||
|
this.allCountsSuspect = true
|
||||||
|
prodLog.narrate(
|
||||||
|
`[FileSystemStorage] count-ledger derivation could not finish on a quiet store ` +
|
||||||
|
`after ${attempt} attempts (${Date.now() - startedAt}ms) — writes landed during ` +
|
||||||
|
`every walk. The ALL-visibility scalars stay SUSPECT and must not be subtracted ` +
|
||||||
|
`against; brain.repairIndex() derives them under a recount barrier.`
|
||||||
|
)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
this.totalNounCountAll = derived.nouns
|
||||||
|
this.totalVerbCountAll = derived.verbs
|
||||||
|
this.totalVectoredNounCount = derived.vectored
|
||||||
|
this.allCountsDerivedBy = 'identity-record'
|
||||||
|
this.allCountsSuspect = false
|
||||||
|
await this.persistCounts()
|
||||||
|
prodLog.narrate(
|
||||||
|
`[FileSystemStorage] count-ledger derivation finished in ${Date.now() - startedAt}ms: ` +
|
||||||
|
`${derived.nouns} nouns / ${derived.verbs} verbs / ${derived.vectored} vectored nouns` +
|
||||||
|
(beforeNouns !== derived.nouns ||
|
||||||
|
beforeVerbs !== derived.verbs ||
|
||||||
|
beforeVectored !== derived.vectored
|
||||||
|
? ` (corrected from ${beforeNouns} / ${beforeVerbs} / ${beforeVectored} — the ` +
|
||||||
|
`difference is ghost and scar containers the old rule counted as entities)`
|
||||||
|
: ' (unchanged)') +
|
||||||
|
` — persisted, stamped identity-record, no longer suspect.`
|
||||||
|
)
|
||||||
|
} catch (error) {
|
||||||
|
// The ledger stays suspect and the next open retries. Loud: a
|
||||||
|
// denominator nobody can derive is a fact an operator must have.
|
||||||
|
this.allCountsSuspect = true
|
||||||
|
prodLog.error(
|
||||||
|
`[FileSystemStorage] count-ledger derivation FAILED after ` +
|
||||||
|
`${Date.now() - startedAt}ms — the ALL-visibility scalars remain SUSPECT ` +
|
||||||
|
`and must not be subtracted against; the next open retries:`,
|
||||||
|
error
|
||||||
|
)
|
||||||
|
}
|
||||||
|
})()
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description A cheap witness that the ledger changed while a walk was
|
||||||
|
* running. Every landed write moves one of these live counters, so an
|
||||||
|
* unchanged stamp across a walk means no write landed during it.
|
||||||
|
* @returns A value that differs whenever the live ALL scalars have moved.
|
||||||
|
*/
|
||||||
|
private ledgerActivityStamp(): string {
|
||||||
|
return `${this.totalNounCountAll}:${this.totalVerbCountAll}:${this.totalVectoredNounCount}`
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Resolve once any background count-ledger derivation has
|
||||||
|
* settled (succeeded or failed). Resolves immediately when none was needed.
|
||||||
|
* Exists so tests and operators can observe the ledger's honest value rather
|
||||||
|
* than race it; nothing in the read path waits on this.
|
||||||
|
* @returns A promise that settles with the derivation.
|
||||||
|
*/
|
||||||
|
public async whenCountLedgerSettled(): Promise<void> {
|
||||||
|
await this.countLedgerDerivation
|
||||||
|
}
|
||||||
|
|
||||||
private async scanCanonicalEntities(
|
private async scanCanonicalEntities(
|
||||||
kind: 'nouns' | 'verbs'
|
kind: 'nouns' | 'verbs'
|
||||||
): Promise<{ count: number; sampleDirs: string[] }> {
|
): Promise<{ count: number; sampleDirs: string[] }> {
|
||||||
|
|
@ -2916,10 +3341,15 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
lastUpdated: new Date().toISOString()
|
lastUpdated: new Date().toISOString()
|
||||||
}
|
}
|
||||||
|
|
||||||
await fs.promises.writeFile(
|
// ATOMIC (temp + rename), never a plain writeFile. A direct write
|
||||||
this.countsFilePath,
|
// truncates the file first, so every persist opened a window — measured
|
||||||
JSON.stringify(counts, null, 2)
|
// at roughly 750ms after a flush or close on a real store — in which a
|
||||||
)
|
// concurrent reader saw counts.json EMPTY. An empty file is unparseable,
|
||||||
|
// and an unparseable ledger sends the next open down the full-rescan
|
||||||
|
// path: the cheapest file in the store was costing the most expensive
|
||||||
|
// recovery. The rename is atomic, so a reader sees the old ledger or the
|
||||||
|
// new one, never neither.
|
||||||
|
await this.writeFileAtomic(this.countsFilePath, JSON.stringify(counts, null, 2))
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error persisting counts:', error)
|
console.error('Error persisting counts:', error)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -125,6 +125,36 @@ export interface WriterLockInfo {
|
||||||
rootDir?: string // Convenience for log lines / error messages
|
rootDir?: string // Convenience for log lines / error messages
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* THE CLEAN-CLOSE RECORD. Written by `releaseWriterLock()` at the instant it
|
||||||
|
* gives up the writer lock, naming the lock identity it released. The next
|
||||||
|
* `acquireWriterLock()` reads it and can then say — from a RECORD, not from a
|
||||||
|
* guess — whether the previous writer left on purpose.
|
||||||
|
*
|
||||||
|
* Why a record and not PID liveness: "the recorded PID is no longer alive" is
|
||||||
|
* true of every orderly restart AND of every crash, so the two were reported
|
||||||
|
* identically ("appears dead") and neither could be trusted. Worse, the same
|
||||||
|
* inference fails the other way when the operating system RECYCLES the pid —
|
||||||
|
* a live unrelated process makes a long-dead writer's lock look held, and the
|
||||||
|
* store refuses to open naming a pid that was never Brainy. A record settles
|
||||||
|
* both: matched → the previous writer closed cleanly, nothing to recover;
|
||||||
|
* absent → say so, and name what recovery the open will now run.
|
||||||
|
*
|
||||||
|
* Lifecycle: written at release, consumed (deleted) by the next successful
|
||||||
|
* lock claim — a record must never outlive the lock generation it describes,
|
||||||
|
* or it would vouch for a later crash.
|
||||||
|
*/
|
||||||
|
export interface WriterCloseRecord {
|
||||||
|
pid: number
|
||||||
|
hostname: string
|
||||||
|
/** `startedAt` of the lock this close released — the identity match key. */
|
||||||
|
startedAt: string
|
||||||
|
/** ISO timestamp at which the lock was released. */
|
||||||
|
closedAt: string
|
||||||
|
/** Brainy version that performed the close. */
|
||||||
|
version: string
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* FNV-1a hash returning a 2-char hex bucket (00-ff).
|
* FNV-1a hash returning a 2-char hex bucket (00-ff).
|
||||||
* Distributes system keys across 256 sub-prefixes to avoid
|
* Distributes system keys across 256 sub-prefixes to avoid
|
||||||
|
|
@ -1407,6 +1437,29 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
return this.listObjectsUnderPath(prefix)
|
return this.listObjectsUnderPath(prefix)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description The IMMEDIATE child directory names under a prefix — one
|
||||||
|
* level, no recursion. See the seam's JSDoc (`db/types.ts`) for why a
|
||||||
|
* separate door exists. This default derives them from the recursive
|
||||||
|
* listing, so it is never WRONG, only never faster; the filesystem adapter
|
||||||
|
* overrides it with a single directory read.
|
||||||
|
* @param prefix - Storage-root-relative directory prefix.
|
||||||
|
* @returns The child directory names (not paths), in listing order.
|
||||||
|
*/
|
||||||
|
public async listRawPrefixes(prefix: string): Promise<string[]> {
|
||||||
|
await this.ensureInitialized()
|
||||||
|
const paths = await this.listObjectsUnderPath(prefix)
|
||||||
|
const normalizedPrefix = prefix.endsWith('/') ? prefix : `${prefix}/`
|
||||||
|
const names = new Set<string>()
|
||||||
|
for (const p of paths) {
|
||||||
|
const rest = p.startsWith(normalizedPrefix) ? p.slice(normalizedPrefix.length) : null
|
||||||
|
if (rest === null) continue
|
||||||
|
const slash = rest.search(/[/\\]/)
|
||||||
|
if (slash > 0) names.add(rest.slice(0, slash))
|
||||||
|
}
|
||||||
|
return [...names]
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Remove every object under a storage-root-relative prefix. The filesystem
|
* Remove every object under a storage-root-relative prefix. The filesystem
|
||||||
* adapter overrides this with a recursive directory removal; this default
|
* adapter overrides this with a recursive directory removal; this default
|
||||||
|
|
|
||||||
|
|
@ -215,7 +215,7 @@ export interface ScoreExplanation {
|
||||||
*
|
*
|
||||||
* @example
|
* @example
|
||||||
* ```ts
|
* ```ts
|
||||||
* declare module '@soulcraft/brainy' {
|
* declare module '@soulcraftlabs/brainy' {
|
||||||
* interface SubtypeRegistry {
|
* interface SubtypeRegistry {
|
||||||
* // For NounType.Person, subtype 'employee':
|
* // For NounType.Person, subtype 'employee':
|
||||||
* 'person:employee': { employeeId: string; department: string }
|
* 'person:employee': { employeeId: string; department: string }
|
||||||
|
|
@ -1217,6 +1217,13 @@ export interface RepairFamilyReport {
|
||||||
skipped?: string
|
skipped?: string
|
||||||
/** Why the outcome is what it is when neither `detail` nor `skipped` says it. */
|
/** Why the outcome is what it is when neither `detail` nor `skipped` says it. */
|
||||||
reason?: string
|
reason?: string
|
||||||
|
/**
|
||||||
|
* The phase's own wall, in milliseconds. A repair on a production store ran
|
||||||
|
* for over thirty minutes without a single line of output; an operator had
|
||||||
|
* to read `top` to know it was alive. A receipt that cannot say WHERE the
|
||||||
|
* time went is not a receipt — every row carries its own.
|
||||||
|
*/
|
||||||
|
durationMs?: number
|
||||||
}
|
}
|
||||||
|
|
||||||
/** The full receipt returned by repairIndex(). */
|
/** The full receipt returned by repairIndex(). */
|
||||||
|
|
|
||||||
|
|
@ -65,7 +65,7 @@
|
||||||
* | `_rev` | system-managed revision counter — pass `ifRev` to `update()` for CAS |
|
* | `_rev` | system-managed revision counter — pass `ifRev` to `update()` for CAS |
|
||||||
*
|
*
|
||||||
* @example
|
* @example
|
||||||
* import { RESERVED_ENTITY_FIELDS } from '@soulcraft/brainy'
|
* import { RESERVED_ENTITY_FIELDS } from '@soulcraftlabs/brainy'
|
||||||
* const isReserved = (key: string) =>
|
* const isReserved = (key: string) =>
|
||||||
* (RESERVED_ENTITY_FIELDS as readonly string[]).includes(key)
|
* (RESERVED_ENTITY_FIELDS as readonly string[]).includes(key)
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@
|
||||||
*
|
*
|
||||||
* @example
|
* @example
|
||||||
* ```typescript
|
* ```typescript
|
||||||
* import { BrainyTypes } from '@soulcraft/brainy'
|
* import { BrainyTypes } from '@soulcraftlabs/brainy'
|
||||||
*
|
*
|
||||||
* // Get all available types
|
* // Get all available types
|
||||||
* const nounTypes = BrainyTypes.nouns // ['Person', 'Organization', ...]
|
* const nounTypes = BrainyTypes.nouns // ['Person', 'Organization', ...]
|
||||||
|
|
|
||||||
|
|
@ -153,3 +153,83 @@ export function assessProviderHealth(provider: unknown): ProviderHealthAssessmen
|
||||||
reasons: readiness === 'not-ready' ? ['isReady() returned false'] : []
|
reasons: readiness === 'not-ready' ? ['isReady() returned false'] : []
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description A provider's self-report that it is REBUILDING ITS OWN index
|
||||||
|
* right now. Returned by the optional `rebuildInProgress()` hook.
|
||||||
|
*
|
||||||
|
* The distinction this exists to make: a provider reporting `serving: false`
|
||||||
|
* because it is BROKEN and a provider reporting `serving: false` because it is
|
||||||
|
* BUSY BUILDING ITSELF look identical through `healthReport()` alone, and
|
||||||
|
* brainy treated both the same way — it called `rebuild()` and waited for it,
|
||||||
|
* on the foreground of `init()`. A production store whose metadata provider
|
||||||
|
* had to rebuild paid 641 SECONDS of that wait before `init()` returned, with
|
||||||
|
* every other family idle behind it.
|
||||||
|
*
|
||||||
|
* A provider that reports progress here owns its own rebuild: brainy neither
|
||||||
|
* starts one nor waits for it, `init()` returns, the other families serve, and
|
||||||
|
* THAT family's doors refuse by name — carrying this progress — until the
|
||||||
|
* provider reports itself serving.
|
||||||
|
*
|
||||||
|
* Every field but `phase` is optional and every field is a MEASUREMENT: a
|
||||||
|
* provider reports only what it actually tracks, never an estimate dressed as
|
||||||
|
* a fact.
|
||||||
|
*/
|
||||||
|
export interface ProviderRebuildProgress {
|
||||||
|
/** The provider's own name for what it is doing. Quoted verbatim in refusals. */
|
||||||
|
phase: string
|
||||||
|
/** Units completed so far, if the provider counts them. */
|
||||||
|
done?: number
|
||||||
|
/** Units expected in total, if the provider knows it. */
|
||||||
|
total?: number
|
||||||
|
/** Epoch millis when this rebuild started, if the provider tracks it. */
|
||||||
|
startedAt?: number
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A provider that can report a rebuild it is running itself. */
|
||||||
|
interface MaybeRebuildingProvider {
|
||||||
|
rebuildInProgress?: () => ProviderRebuildProgress | null
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Ask a provider whether it is rebuilding itself right now.
|
||||||
|
* Synchronous, O(1), feature-detected: a provider without the hook reports
|
||||||
|
* nothing and is treated exactly as before.
|
||||||
|
* @param provider - Any index provider, or `null`/`undefined`.
|
||||||
|
* @returns The provider's progress, or `null` when it is not rebuilding (or
|
||||||
|
* does not implement the hook).
|
||||||
|
*/
|
||||||
|
export function assessProviderRebuild(provider: unknown): ProviderRebuildProgress | null {
|
||||||
|
const p = provider as MaybeRebuildingProvider | null | undefined
|
||||||
|
if (p == null || typeof p.rebuildInProgress !== 'function') return null
|
||||||
|
try {
|
||||||
|
const progress = p.rebuildInProgress()
|
||||||
|
if (!progress || typeof progress.phase !== 'string' || progress.phase.length === 0) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
return progress
|
||||||
|
} catch {
|
||||||
|
// A throwing hook says nothing trustworthy about a rebuild; fall through to
|
||||||
|
// the ordinary health verdict rather than inventing one.
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Render a rebuild progress report as one operator-facing clause,
|
||||||
|
* for a refusal message. Includes only what the provider actually measured.
|
||||||
|
* @param progress - The provider's report.
|
||||||
|
* @returns A clause such as `rebuilding ("metadata shadow build", 4,096/14,056, 12s elapsed)`.
|
||||||
|
*/
|
||||||
|
export function describeRebuildProgress(progress: ProviderRebuildProgress): string {
|
||||||
|
const parts: string[] = [`"${progress.phase}"`]
|
||||||
|
if (typeof progress.done === 'number' && typeof progress.total === 'number') {
|
||||||
|
parts.push(`${progress.done.toLocaleString()}/${progress.total.toLocaleString()}`)
|
||||||
|
} else if (typeof progress.done === 'number') {
|
||||||
|
parts.push(`${progress.done.toLocaleString()} done`)
|
||||||
|
}
|
||||||
|
if (typeof progress.startedAt === 'number') {
|
||||||
|
parts.push(`${Math.round((Date.now() - progress.startedAt) / 1000)}s elapsed`)
|
||||||
|
}
|
||||||
|
return `rebuilding (${parts.join(', ')})`
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -266,6 +266,26 @@ export const prodLog = {
|
||||||
console.error(message, ...args)
|
console.error(message, ...args)
|
||||||
},
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* THE NARRATION CHANNEL — always visible, exactly like `error`.
|
||||||
|
*
|
||||||
|
* `warn`/`info`/`log` below are clamped to ERROR in any environment that
|
||||||
|
* looks like production (see isProductionEnvironment), which is the right
|
||||||
|
* default for chatter and the wrong one for the two things an operator is
|
||||||
|
* entitled to hear from a database no matter what: WHY IT IS SLOW and WHAT
|
||||||
|
* IT IS DOING ABOUT IT. A production service opening a 16 GB store spent
|
||||||
|
* three minutes emitting nothing at all — the phase timings that would have
|
||||||
|
* named the slow phase were written to `warn` and thrown away by the log
|
||||||
|
* level. Progress and cost narration goes here; it is never a per-record
|
||||||
|
* line, always a phase, a wall, or a bounded-cadence heartbeat.
|
||||||
|
*
|
||||||
|
* `silent: true` still silences it — that is the consumer's explicit
|
||||||
|
* request, not a cost default.
|
||||||
|
*/
|
||||||
|
narrate: (message?: any, ...args: any[]) => {
|
||||||
|
console.warn(message, ...args)
|
||||||
|
},
|
||||||
|
|
||||||
// These are suppressed in production unless BRAINY_LOG_LEVEL is set
|
// These are suppressed in production unless BRAINY_LOG_LEVEL is set
|
||||||
warn: (message?: any, ...args: any[]) => smartConsole.warn(message, ...args),
|
warn: (message?: any, ...args: any[]) => smartConsole.warn(message, ...args),
|
||||||
info: (message?: any, ...args: any[]) => smartConsole.info(message, ...args),
|
info: (message?: any, ...args: any[]) => smartConsole.info(message, ...args),
|
||||||
|
|
|
||||||
|
|
@ -2241,6 +2241,74 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ===== ARRAY SET OPERATORS =====
|
||||||
|
// An element-indexed array field makes all three exact on the
|
||||||
|
// index path. They were previously ABSENT from this switch, so
|
||||||
|
// `fieldResults` kept its initial `[]` and the whole find()
|
||||||
|
// returned an empty page — a documented, matcher-implemented
|
||||||
|
// operator answering silently wrong. Served here instead.
|
||||||
|
|
||||||
|
// hasAll: [a, b] — the field's array contains EVERY operand:
|
||||||
|
// the intersection of each element's posting set.
|
||||||
|
case 'hasAll': {
|
||||||
|
if (!Array.isArray(operand)) {
|
||||||
|
fieldResults = []
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if (operand.length === 0) {
|
||||||
|
// Vacuously true of every row that HAS the field.
|
||||||
|
const anyBitmap = (this.columnStore && this.columnStore.hasField(field))
|
||||||
|
? await this.columnStore.rangeQuery(field)
|
||||||
|
: await this.getExistsBitmapLegacy(field)
|
||||||
|
fieldResults = this.idMapper.intsIterableToUuids(anyBitmap)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
let intersection: Set<string> | null = null
|
||||||
|
for (const item of operand) {
|
||||||
|
const ids = new Set(await this.getIds(field, item))
|
||||||
|
if (intersection === null) {
|
||||||
|
intersection = ids
|
||||||
|
} else {
|
||||||
|
for (const id of [...intersection]) {
|
||||||
|
if (!ids.has(id)) intersection.delete(id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (intersection.size === 0) break
|
||||||
|
}
|
||||||
|
fieldResults = intersection ? [...intersection] : []
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
// noneOf: [a, b] — the field's value is NONE of the operands:
|
||||||
|
// the complement of their union.
|
||||||
|
case 'noneOf': {
|
||||||
|
if (!Array.isArray(operand)) {
|
||||||
|
fieldResults = []
|
||||||
|
break
|
||||||
|
}
|
||||||
|
const excludeInts: number[] = []
|
||||||
|
for (const value of operand) {
|
||||||
|
for (const uuid of await this.getIds(field, value)) {
|
||||||
|
const intId = this.idMapper.getInt(uuid)
|
||||||
|
if (intId !== undefined) excludeInts.push(intId)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fieldResults = this.complementIds(excludeInts)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
// excludes: value — the field's array does NOT contain the value:
|
||||||
|
// the complement of `contains`.
|
||||||
|
case 'excludes': {
|
||||||
|
const excludeInts: number[] = []
|
||||||
|
for (const uuid of await this.getIds(field, operand)) {
|
||||||
|
const intId = this.idMapper.getInt(uuid)
|
||||||
|
if (intId !== undefined) excludeInts.push(intId)
|
||||||
|
}
|
||||||
|
fieldResults = this.complementIds(excludeInts)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
// ===== MISSING OPERATOR =====
|
// ===== MISSING OPERATOR =====
|
||||||
// missing: boolean - equivalent to exists: !boolean
|
// missing: boolean - equivalent to exists: !boolean
|
||||||
case 'missing': {
|
case 'missing': {
|
||||||
|
|
@ -2257,6 +2325,27 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
||||||
}
|
}
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ===== EVERYTHING ELSE: REFUSED BY NAME, NEVER ANSWERED EMPTY ====
|
||||||
|
// An equality/range posting index cannot evaluate a substring, a
|
||||||
|
// pattern or an array length without reading every row, and this
|
||||||
|
// path exists precisely to avoid that. It used to fall out of the
|
||||||
|
// switch with `fieldResults` still `[]`, so `find({ where: { name:
|
||||||
|
// { startsWith: 'a' } } })` returned an empty page and looked like
|
||||||
|
// an answer. An accepted operator either works or refuses — the
|
||||||
|
// matcher's own support for these operators governs in-memory
|
||||||
|
// filtering, never an index-backed find().
|
||||||
|
default:
|
||||||
|
throw new BrainyError(
|
||||||
|
`Filter operator "${op}" on field "${rawField}" cannot be served by the ` +
|
||||||
|
`metadata index: an equality/range posting index cannot evaluate substrings, ` +
|
||||||
|
`patterns or array lengths without reading every row. It is REFUSED rather ` +
|
||||||
|
`than answered with an empty page. Filter on an indexable operator ` +
|
||||||
|
`(equals/eq, notEquals/ne, oneOf/in, noneOf, greaterThan/gt, ` +
|
||||||
|
`greaterThanOrEqual/gte, lessThan/lt, lessThanOrEqual/lte, between, contains, ` +
|
||||||
|
`excludes, hasAll, exists, missing) and narrow the rest in your own code.`,
|
||||||
|
'INVALID_QUERY'
|
||||||
|
)
|
||||||
}
|
}
|
||||||
// Intersect this operator's matches with the running set (AND semantics
|
// Intersect this operator's matches with the running set (AND semantics
|
||||||
// for multiple operators on the same field).
|
// for multiple operators on the same field).
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
/**
|
/**
|
||||||
* @module utils/version
|
* @module utils/version
|
||||||
* @description Resolves the running `@soulcraft/brainy` package version. Brainy 8.0
|
* @description Resolves the running `@soulcraftlabs/brainy` package version. Brainy 8.0
|
||||||
* targets Node-like runtimes only (Node.js, Bun, Deno — all expose `node:fs`), so the
|
* targets Node-like runtimes only (Node.js, Bun, Deno — all expose `node:fs`), so the
|
||||||
* version is read **synchronously** from `package.json` on first call and cached.
|
* version is read **synchronously** from `package.json` on first call and cached.
|
||||||
*
|
*
|
||||||
|
|
@ -83,3 +83,27 @@ export function getAugmentationVersion(service: string): { augmentation: string;
|
||||||
version: getBrainyVersion()
|
version: getBrainyVersion()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The API-contract version this build implements — a single integer that two
|
||||||
|
* engines can compare without probing prototypes.
|
||||||
|
*
|
||||||
|
* A MINOR release is ADDITIVE: doors and error codes may be added, never
|
||||||
|
* removed or narrowed, and the contract integer does not move. A MAJOR release
|
||||||
|
* is what a REQUIRED door's removal or a behavioural narrowing costs, and it
|
||||||
|
* bumps this integer. A consumer pinning `brainyContract` in a peer range is
|
||||||
|
* therefore pinning "what I may call", not "which build I run".
|
||||||
|
*
|
||||||
|
* Declared in package.json as `"brainyContract"` so a manifest, a tool, or a
|
||||||
|
* sibling package can read it without importing the engine, and returned here
|
||||||
|
* so a running process can state its own.
|
||||||
|
*/
|
||||||
|
export const BRAINY_CONTRACT_VERSION = 1 as const
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description The API-contract version this build implements.
|
||||||
|
* @returns The contract integer — see {@link BRAINY_CONTRACT_VERSION}.
|
||||||
|
*/
|
||||||
|
export function contractVersion(): number {
|
||||||
|
return BRAINY_CONTRACT_VERSION
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { Readable, Writable } from 'stream'
|
import { Readable, Writable } from 'stream'
|
||||||
|
import { prodLog } from '../utils/logger.js'
|
||||||
import crypto from 'crypto'
|
import crypto from 'crypto'
|
||||||
import { v4 as uuidv4 } from '../universal/uuid.js'
|
import { v4 as uuidv4 } from '../universal/uuid.js'
|
||||||
import { Brainy } from '../brainy.js'
|
import { Brainy } from '../brainy.js'
|
||||||
|
|
@ -66,6 +67,20 @@ export class VirtualFileSystem implements IVirtualFileSystem {
|
||||||
private config: Required<Omit<VFSConfig, 'rootEntityId'>> & { rootEntityId?: string }
|
private config: Required<Omit<VFSConfig, 'rootEntityId'>> & { rootEntityId?: string }
|
||||||
private rootEntityId?: string
|
private rootEntityId?: string
|
||||||
private initialized = false
|
private initialized = false
|
||||||
|
/**
|
||||||
|
* The one-time old-root sweep, in flight. See {@link sweepOldRootsIfNeeded}.
|
||||||
|
*/
|
||||||
|
private rootSweep?: Promise<void>
|
||||||
|
/**
|
||||||
|
* Where the completed old-root sweep is recorded. Engine plumbing under
|
||||||
|
* `_system/`, like every other marker there — never enumerated as data.
|
||||||
|
*/
|
||||||
|
private static readonly ROOT_SWEEP_MARKER_PATH = '_system/vfs-root-sweep.json'
|
||||||
|
/**
|
||||||
|
* Below this wall, a sweep that removed nothing says nothing — see
|
||||||
|
* {@link sweepOldRootsIfNeeded}.
|
||||||
|
*/
|
||||||
|
private static readonly ROOT_SWEEP_NARRATE_MS = 1_000
|
||||||
private currentUser: string = 'system' // Track current user for collaboration
|
private currentUser: string = 'system' // Track current user for collaboration
|
||||||
|
|
||||||
// Knowledge Layer features available via augmentation (brain.use('knowledge'))
|
// Knowledge Layer features available via augmentation (brain.use('knowledge'))
|
||||||
|
|
@ -143,8 +158,17 @@ export class VirtualFileSystem implements IVirtualFileSystem {
|
||||||
// Create or find root entity
|
// Create or find root entity
|
||||||
this.rootEntityId = await this.initializeRoot()
|
this.rootEntityId = await this.initializeRoot()
|
||||||
|
|
||||||
// Clean up old UUID-based roots (one-time migration)
|
// Clean up old UUID-based roots — ONCE PER STORE, BEHIND THE DOORS.
|
||||||
await this.cleanupOldRoots()
|
// This is a migration sweep for roots created before the fixed root id
|
||||||
|
// existed. It ran on EVERY open, forever: a filtered find over the whole
|
||||||
|
// store hunting for duplicates that a store has either always had or
|
||||||
|
// never will. MEASURED on a 14,056-noun / 72,679-verb store: the phase it
|
||||||
|
// dominates cost 43-53 SECONDS of every open, warm reopens included.
|
||||||
|
// Now: a durable marker records that the sweep has run, and a store
|
||||||
|
// carrying it never sweeps again; a store without one sweeps in the
|
||||||
|
// BACKGROUND (the sweep only removes duplicate roots — nothing serves
|
||||||
|
// from them — and it was always declared non-critical).
|
||||||
|
this.rootSweep = this.sweepOldRootsIfNeeded()
|
||||||
|
|
||||||
// Initialize projection registry with auto-discovery of built-in projections
|
// Initialize projection registry with auto-discovery of built-in projections
|
||||||
this.projectionRegistry = new ProjectionRegistry()
|
this.projectionRegistry = new ProjectionRegistry()
|
||||||
|
|
@ -394,7 +418,100 @@ export class VirtualFileSystem implements IVirtualFileSystem {
|
||||||
*
|
*
|
||||||
* This is a one-time migration helper that can be removed in future versions.
|
* This is a one-time migration helper that can be removed in future versions.
|
||||||
*/
|
*/
|
||||||
private async cleanupOldRoots(): Promise<void> {
|
/**
|
||||||
|
* @description Run the old-root sweep at most once per store, in the
|
||||||
|
* background, and record that it ran. See the call site in {@link init} for
|
||||||
|
* the measurement that made this necessary.
|
||||||
|
* @returns A promise that settles when the sweep has finished (or was
|
||||||
|
* skipped); nothing in the read path awaits it.
|
||||||
|
*/
|
||||||
|
private async sweepOldRootsIfNeeded(): Promise<void> {
|
||||||
|
const store = this.rawObjectStore()
|
||||||
|
if (store === null) {
|
||||||
|
// A storage adapter with no raw-object door cannot carry the marker.
|
||||||
|
// Sweep every open, as before — correctness over cost.
|
||||||
|
await this.cleanupOldRoots()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const marker = await store.readRawObject(VirtualFileSystem.ROOT_SWEEP_MARKER_PATH)
|
||||||
|
if (marker !== null && marker !== undefined) return
|
||||||
|
} catch {
|
||||||
|
// Unreadable marker: sweep, and rewrite it below.
|
||||||
|
}
|
||||||
|
// NARRATION HAS A THRESHOLD, like every other line this engine emits on the
|
||||||
|
// always-visible channel. On a fresh or small store this sweep finds
|
||||||
|
// nothing and costs a millisecond, and announcing it — twice — on a
|
||||||
|
// channel a production log level deliberately CANNOT silence would train
|
||||||
|
// operators to ignore the one channel that exists to be impossible to
|
||||||
|
// ignore. It speaks when it has something to say: duplicates removed, or a
|
||||||
|
// wall long enough that somebody watching a slow first open deserves to
|
||||||
|
// know what is running. Otherwise it does its work and stays quiet.
|
||||||
|
const startedAt = Date.now()
|
||||||
|
const duplicatesRemoved = await this.cleanupOldRoots()
|
||||||
|
const elapsedMs = Date.now() - startedAt
|
||||||
|
try {
|
||||||
|
await store.writeRawObject(VirtualFileSystem.ROOT_SWEEP_MARKER_PATH, {
|
||||||
|
sweptAt: new Date().toISOString(),
|
||||||
|
durationMs: elapsedMs
|
||||||
|
})
|
||||||
|
if (duplicatesRemoved > 0 || elapsedMs >= VirtualFileSystem.ROOT_SWEEP_NARRATE_MS) {
|
||||||
|
prodLog.narrate(
|
||||||
|
`[VFS] one-time old-root sweep complete in ${elapsedMs}ms` +
|
||||||
|
(duplicatesRemoved > 0
|
||||||
|
? `, ${duplicatesRemoved} pre-fixed-id root(s) removed`
|
||||||
|
: '') +
|
||||||
|
' and recorded — no future open pays for it.'
|
||||||
|
)
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
// Unrecorded sweep = the next open sweeps again. Conservative, and said
|
||||||
|
// out loud rather than quietly repeated forever.
|
||||||
|
prodLog.narrate(
|
||||||
|
`[VFS] old-root sweep finished in ${Date.now() - startedAt}ms but could NOT be ` +
|
||||||
|
`recorded (${(error as Error).message}) — the next open will sweep again.`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Settle once the background old-root sweep has finished.
|
||||||
|
* Resolves immediately when the store already carried the marker. Exists so
|
||||||
|
* tests and operators can observe the sweep instead of racing it; no read
|
||||||
|
* path waits on it.
|
||||||
|
* @returns A promise that settles with the sweep.
|
||||||
|
*/
|
||||||
|
public async whenRootSweepSettled(): Promise<void> {
|
||||||
|
await this.rootSweep
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description The brain's storage adapter, narrowed to the raw-object door
|
||||||
|
* this migration marker needs. Boundary: `Brainy.storage` is private, and
|
||||||
|
* this is the same reach-in the engine uses elsewhere for exactly this kind
|
||||||
|
* of engine-internal artifact. Returns null when the adapter has no
|
||||||
|
* raw-object door.
|
||||||
|
*/
|
||||||
|
private rawObjectStore(): {
|
||||||
|
readRawObject: (key: string) => Promise<unknown>
|
||||||
|
writeRawObject: (key: string, value: unknown) => Promise<void>
|
||||||
|
} | null {
|
||||||
|
const storage = (this.brain as unknown as { storage?: Record<string, unknown> }).storage
|
||||||
|
if (
|
||||||
|
storage &&
|
||||||
|
typeof storage.readRawObject === 'function' &&
|
||||||
|
typeof storage.writeRawObject === 'function'
|
||||||
|
) {
|
||||||
|
return storage as unknown as {
|
||||||
|
readRawObject: (key: string) => Promise<unknown>
|
||||||
|
writeRawObject: (key: string, value: unknown) => Promise<void>
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
private async cleanupOldRoots(): Promise<number> {
|
||||||
|
let removed = 0
|
||||||
try {
|
try {
|
||||||
// Find any old VFS roots with UUID-based IDs (not our fixed ID)
|
// Find any old VFS roots with UUID-based IDs (not our fixed ID)
|
||||||
const oldRoots = await this.brain.find({
|
const oldRoots = await this.brain.find({
|
||||||
|
|
@ -416,6 +533,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
|
||||||
for (const duplicate of duplicates) {
|
for (const duplicate of duplicates) {
|
||||||
try {
|
try {
|
||||||
await this.brain.remove(duplicate.id)
|
await this.brain.remove(duplicate.id)
|
||||||
|
removed++
|
||||||
console.log(`VFS: Deleted old root ${duplicate.id.substring(0, 8)}`)
|
console.log(`VFS: Deleted old root ${duplicate.id.substring(0, 8)}`)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.warn(`VFS: Failed to delete old root ${duplicate.id}:`, error)
|
console.warn(`VFS: Failed to delete old root ${duplicate.id}:`, error)
|
||||||
|
|
@ -428,6 +546,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
|
||||||
// Non-critical error - log and continue
|
// Non-critical error - log and continue
|
||||||
console.warn('VFS: Cleanup of old roots failed (non-critical):', error)
|
console.warn('VFS: Cleanup of old roots failed (non-critical):', error)
|
||||||
}
|
}
|
||||||
|
return removed
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
251
tests/integration/count-ledger-identity-record.test.ts
Normal file
251
tests/integration/count-ledger-identity-record.test.ts
Normal file
|
|
@ -0,0 +1,251 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/count-ledger-identity-record
|
||||||
|
* @description THE COUNT LEDGER COUNTS RECORDS, NOT DIRECTORIES — and heals
|
||||||
|
* itself when it was derived the other way.
|
||||||
|
*
|
||||||
|
* Measured on a real store: the ALL-visibility ledger read 14,231 nouns
|
||||||
|
* against 14,056 identity records, and 72,729 verbs against 72,679 — exactly
|
||||||
|
* that store's 25 noun and 50 verb SCAR directories (empty `<id>/` containers
|
||||||
|
* left by a pre-8.3.1 partial delete). Two copies of the SAME archive derived
|
||||||
|
* different numbers, because each had been persisted at a different moment
|
||||||
|
* under the old container rule. A downstream index heal subtracted against
|
||||||
|
* those denominators and reported remaining work that did not exist.
|
||||||
|
*
|
||||||
|
* The membership predicate is the IDENTITY RECORD (the metadata content leg).
|
||||||
|
* The scan already applies it; what is pinned here is that a ledger persisted
|
||||||
|
* under the OLD rule does not go on lying — it is corrected in the background,
|
||||||
|
* without blocking the open, and two copies of one archive agree.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, afterEach } from 'vitest'
|
||||||
|
import {
|
||||||
|
mkdtempSync,
|
||||||
|
mkdirSync,
|
||||||
|
rmSync,
|
||||||
|
writeFileSync,
|
||||||
|
readFileSync,
|
||||||
|
cpSync,
|
||||||
|
existsSync
|
||||||
|
} from 'node:fs'
|
||||||
|
import { tmpdir } from 'node:os'
|
||||||
|
import { join } from 'node:path'
|
||||||
|
import { Brainy } from '../../src/brainy.js'
|
||||||
|
import { NounType } from '../../src/types/graphTypes.js'
|
||||||
|
import { FileSystemStorage as FileSystemStorageClass } from '../../src/storage/adapters/fileSystemStorage.js'
|
||||||
|
import type { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js'
|
||||||
|
|
||||||
|
const NOUN_COUNT = 6
|
||||||
|
const NOUN_SCARS = 3
|
||||||
|
const VERB_SCARS = 2
|
||||||
|
/** A REAL two-hex shard — the scan skips any directory that is not one. */
|
||||||
|
const SCAR_SHARD = 'ab'
|
||||||
|
|
||||||
|
function makeTempDir(): string {
|
||||||
|
return mkdtempSync(join(tmpdir(), 'brainy-count-ledger-'))
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The FileSystemStorage behind a brain. */
|
||||||
|
function storageOf(brain: Brainy): FileSystemStorage {
|
||||||
|
return (brain as unknown as { storage: FileSystemStorage }).storage
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add `count` empty `<id>/` container directories under
|
||||||
|
* `entities/<kind>/<shard>/` — scars, exactly as a partial delete leaves them.
|
||||||
|
*/
|
||||||
|
function addScarContainers(dir: string, kind: 'nouns' | 'verbs', count: number): void {
|
||||||
|
for (let i = 0; i < count; i++) {
|
||||||
|
const id = `${SCAR_SHARD}5ca4000-0000-0000-0000-00000000000${i}`
|
||||||
|
mkdirSync(join(dir, 'entities', kind, SCAR_SHARD, id), { recursive: true })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Add one GHOST container: a `vectors.json` leg with no identity record. */
|
||||||
|
function addGhostContainer(dir: string): void {
|
||||||
|
const id = `${SCAR_SHARD}9405700-0000-0000-0000-000000000000`
|
||||||
|
const idDir = join(dir, 'entities', 'nouns', SCAR_SHARD, id)
|
||||||
|
mkdirSync(idDir, { recursive: true })
|
||||||
|
writeFileSync(join(idDir, 'vectors.json'), JSON.stringify({ id, vector: [0.1, 0.2] }))
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Rewrite counts.json into the LEGACY shape: ALL scalars inflated by the
|
||||||
|
* containers, and no `allCountsDerivedBy` stamp — exactly what a store carried
|
||||||
|
* when it was last written by a build that counted directories.
|
||||||
|
*/
|
||||||
|
function writeLegacyCountsLedger(dir: string, inflateNouns: number, inflateVerbs: number): void {
|
||||||
|
const file = join(dir, '_system', 'counts.json')
|
||||||
|
const counts = JSON.parse(readFileSync(file, 'utf-8'))
|
||||||
|
counts.totalNounCountAll = (counts.totalNounCountAll ?? 0) + inflateNouns
|
||||||
|
counts.totalVerbCountAll = (counts.totalVerbCountAll ?? 0) + inflateVerbs
|
||||||
|
delete counts.allCountsDerivedBy
|
||||||
|
delete counts.allCountsSuspect
|
||||||
|
writeFileSync(file, JSON.stringify(counts, null, 2))
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Seed a store and return the HONEST ledger it holds when freshly written —
|
||||||
|
* the baseline the correction must return to. Read from the engine rather than
|
||||||
|
* hardcoded: an open creates its own rows (the VFS root), and a pin that
|
||||||
|
* asserts a literal would be pinning that incidental fact instead of the rule.
|
||||||
|
*/
|
||||||
|
async function seedStore(dir: string): Promise<{ nouns: number; verbs: number }> {
|
||||||
|
const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||||
|
await brain.init()
|
||||||
|
const ids: string[] = []
|
||||||
|
for (let i = 0; i < NOUN_COUNT; i++) {
|
||||||
|
ids.push(await brain.add({ data: `entity number ${i}`, type: NounType.Concept }))
|
||||||
|
}
|
||||||
|
await brain.relate({ from: ids[0], to: ids[1], type: 'relatedTo' } as never)
|
||||||
|
await brain.relate({ from: ids[1], to: ids[2], type: 'relatedTo' } as never)
|
||||||
|
await brain.flush()
|
||||||
|
const ledger = await storageOf(brain).getCanonicalCounts()
|
||||||
|
const baseline = { nouns: ledger.nouns.all, verbs: ledger.verbs.all }
|
||||||
|
await brain.close()
|
||||||
|
return baseline
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Make the ledger walk take `ms` so a test can observe the open completing
|
||||||
|
* WITHOUT it. Patches the prototype before any brain is constructed; returns
|
||||||
|
* the restore function.
|
||||||
|
*/
|
||||||
|
function slowTheLedgerWalk(ms: number): () => void {
|
||||||
|
const proto = (
|
||||||
|
FileSystemStorageClass as unknown as {
|
||||||
|
prototype: Record<string, (...args: unknown[]) => Promise<unknown>>
|
||||||
|
}
|
||||||
|
).prototype
|
||||||
|
const real = proto.scanCanonicalEntities
|
||||||
|
proto.scanCanonicalEntities = async function slow(this: unknown, ...args: unknown[]) {
|
||||||
|
await new Promise((r) => setTimeout(r, ms))
|
||||||
|
return real.apply(this, args)
|
||||||
|
}
|
||||||
|
return () => { proto.scanCanonicalEntities = real }
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('the canonical count ledger', () => {
|
||||||
|
const dirs: string[] = []
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
for (const d of dirs.splice(0)) {
|
||||||
|
try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ }
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
function trackDir(): string {
|
||||||
|
const dir = makeTempDir()
|
||||||
|
dirs.push(dir)
|
||||||
|
return dir
|
||||||
|
}
|
||||||
|
|
||||||
|
it('corrects a legacy container-rule ledger in the background, counting identity records', async () => {
|
||||||
|
const dir = trackDir()
|
||||||
|
const baseline = await seedStore(dir)
|
||||||
|
|
||||||
|
// Scars and a ghost: containers with no identity record.
|
||||||
|
addScarContainers(dir, 'nouns', NOUN_SCARS)
|
||||||
|
addScarContainers(dir, 'verbs', VERB_SCARS)
|
||||||
|
addGhostContainer(dir)
|
||||||
|
// The ledger as the old rule left it: every container counted.
|
||||||
|
writeLegacyCountsLedger(dir, NOUN_SCARS + 1, VERB_SCARS)
|
||||||
|
|
||||||
|
const restore = slowTheLedgerWalk(1_500)
|
||||||
|
let brain: Brainy
|
||||||
|
try {
|
||||||
|
const openStarted = Date.now()
|
||||||
|
brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||||
|
await brain.init()
|
||||||
|
const openMs = Date.now() - openStarted
|
||||||
|
const storage = storageOf(brain)
|
||||||
|
|
||||||
|
// THE OPEN DID NOT WAIT. Two walks of 1.5s each would have added 3s.
|
||||||
|
expect(openMs).toBeLessThan(2_500)
|
||||||
|
// And while it runs, the scalars say so instead of being subtracted against.
|
||||||
|
const atOpen = await storage.getCanonicalCounts()
|
||||||
|
expect(atOpen.suspect).toBe(true)
|
||||||
|
expect(atOpen.nouns.all).toBe(baseline.nouns + NOUN_SCARS + 1)
|
||||||
|
|
||||||
|
await storage.whenCountLedgerSettled()
|
||||||
|
} finally {
|
||||||
|
restore()
|
||||||
|
}
|
||||||
|
const storage = storageOf(brain!)
|
||||||
|
|
||||||
|
const healed = await storage.getCanonicalCounts()
|
||||||
|
expect(healed.nouns.all).toBe(baseline.nouns)
|
||||||
|
expect(healed.verbs.all).toBe(baseline.verbs)
|
||||||
|
expect(healed.suspect).toBe(false)
|
||||||
|
|
||||||
|
// And it is PERSISTED with the honest stamp — the correction survives a
|
||||||
|
// reopen instead of being re-derived (or re-lost) every time.
|
||||||
|
await brain!.close()
|
||||||
|
const persisted = JSON.parse(readFileSync(join(dir, '_system', 'counts.json'), 'utf-8'))
|
||||||
|
expect(persisted.totalNounCountAll).toBe(baseline.nouns)
|
||||||
|
expect(persisted.totalVerbCountAll).toBe(baseline.verbs)
|
||||||
|
expect(persisted.allCountsDerivedBy).toBe('identity-record')
|
||||||
|
|
||||||
|
const reopened = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||||
|
await reopened.init()
|
||||||
|
const afterReopen = await storageOf(reopened).getCanonicalCounts()
|
||||||
|
expect(afterReopen.nouns.all).toBe(baseline.nouns)
|
||||||
|
expect(afterReopen.suspect).toBe(false)
|
||||||
|
await reopened.close()
|
||||||
|
}, 180_000)
|
||||||
|
|
||||||
|
it('derives the same number from two copies of one archive', async () => {
|
||||||
|
const source = trackDir()
|
||||||
|
const baseline = await seedStore(source)
|
||||||
|
addScarContainers(source, 'nouns', NOUN_SCARS)
|
||||||
|
addGhostContainer(source)
|
||||||
|
|
||||||
|
// Two copies of the SAME bytes, each carrying a DIFFERENT legacy ledger —
|
||||||
|
// the situation that made one archive report 14,231 and its twin 14,081.
|
||||||
|
const copyA = trackDir()
|
||||||
|
const copyB = trackDir()
|
||||||
|
cpSync(source, copyA, { recursive: true })
|
||||||
|
cpSync(source, copyB, { recursive: true })
|
||||||
|
writeLegacyCountsLedger(copyA, NOUN_SCARS + 1, 0)
|
||||||
|
writeLegacyCountsLedger(copyB, 1, 0)
|
||||||
|
|
||||||
|
const derived: number[] = []
|
||||||
|
for (const dir of [copyA, copyB]) {
|
||||||
|
const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||||
|
await brain.init()
|
||||||
|
const storage = storageOf(brain)
|
||||||
|
await storage.whenCountLedgerSettled()
|
||||||
|
derived.push((await storage.getCanonicalCounts()).nouns.all)
|
||||||
|
await brain.close()
|
||||||
|
}
|
||||||
|
expect(derived[0]).toBe(derived[1])
|
||||||
|
expect(derived[0]).toBe(baseline.nouns)
|
||||||
|
}, 180_000)
|
||||||
|
|
||||||
|
it('writes counts.json atomically — no reader ever sees it empty', async () => {
|
||||||
|
const dir = trackDir()
|
||||||
|
await seedStore(dir)
|
||||||
|
const file = join(dir, '_system', 'counts.json')
|
||||||
|
expect(existsSync(file)).toBe(true)
|
||||||
|
|
||||||
|
const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||||
|
await brain.init()
|
||||||
|
const storage = storageOf(brain)
|
||||||
|
|
||||||
|
// Watch the ledger across many persists. A truncating write leaves a
|
||||||
|
// window in which the file parses as nothing; a temp+rename never does.
|
||||||
|
let sawUnparseable = 0
|
||||||
|
const watcher = setInterval(() => {
|
||||||
|
try {
|
||||||
|
JSON.parse(readFileSync(file, 'utf-8'))
|
||||||
|
} catch {
|
||||||
|
sawUnparseable++
|
||||||
|
}
|
||||||
|
}, 1)
|
||||||
|
for (let i = 0; i < 40; i++) {
|
||||||
|
await (storage as unknown as { persistCounts: () => Promise<void> }).persistCounts()
|
||||||
|
}
|
||||||
|
clearInterval(watcher)
|
||||||
|
await brain.close()
|
||||||
|
expect(sawUnparseable).toBe(0)
|
||||||
|
}, 180_000)
|
||||||
|
})
|
||||||
|
|
@ -57,7 +57,11 @@ describe('entity-tree family stamp', () => {
|
||||||
const invariants = (stamp.members as any).invariants
|
const invariants = (stamp.members as any).invariants
|
||||||
expect(invariants.nounCount).toBe(await brain.storage.getNounCount())
|
expect(invariants.nounCount).toBe(await brain.storage.getNounCount())
|
||||||
expect(invariants.verbCount).toBe(await brain.storage.getVerbCount())
|
expect(invariants.verbCount).toBe(await brain.storage.getVerbCount())
|
||||||
expect(stamp.sourceGeneration).toBe(brain.generation())
|
// THE SOURCE IS COMMITTED TRUTH, never the allocated counter. Stamping the
|
||||||
|
// counter labelled the stamp with a generation a write in flight had merely
|
||||||
|
// claimed, so every crash inside a write window produced a spurious verdict
|
||||||
|
// at the next open (see the torn-tail pins below).
|
||||||
|
expect(stamp.sourceGeneration).toBe(brain.generationStore.committedGeneration())
|
||||||
expect(stamp.generation).toBeGreaterThanOrEqual(1)
|
expect(stamp.generation).toBeGreaterThanOrEqual(1)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
@ -112,6 +116,96 @@ describe('entity-tree family stamp', () => {
|
||||||
expect(stillIncoherent).toEqual([])
|
expect(stillIncoherent).toEqual([])
|
||||||
})
|
})
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Rewrite the on-disk stamp so its `sourceGeneration` sits ABOVE the store's
|
||||||
|
* committed watermark — the durable shape a torn generation-log tail leaves
|
||||||
|
* behind (the stamp's fsync outlived the tail's). Fabricated rather than
|
||||||
|
* crash-produced so the pin is deterministic; the seeded-SIGKILL lane
|
||||||
|
* (`scripts/crash-consistency.mjs` in the engine repo) produces the same
|
||||||
|
* shape from a real abrupt termination.
|
||||||
|
*/
|
||||||
|
const fabricateTear = (ahead: number): FamilyStamp => {
|
||||||
|
const file = path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`)
|
||||||
|
const zlib = require('node:zlib')
|
||||||
|
const raw = JSON.parse(zlib.gunzipSync(fs.readFileSync(file)).toString('utf-8')) as FamilyStamp
|
||||||
|
const torn: FamilyStamp = { ...raw, sourceGeneration: raw.sourceGeneration + ahead }
|
||||||
|
fs.writeFileSync(file, zlib.gzipSync(JSON.stringify(torn)))
|
||||||
|
return torn
|
||||||
|
}
|
||||||
|
|
||||||
|
it('a torn generation-log tail is a TERMINAL VERDICT at open: narrated, demoted, never a wait', async () => {
|
||||||
|
for (let i = 0; i < 3; i++)
|
||||||
|
await brain.add({ data: `torn${i}`, type: 'document', metadata: { i } })
|
||||||
|
await brain.close()
|
||||||
|
const torn = fabricateTear(5)
|
||||||
|
|
||||||
|
const warn = vi.spyOn(prodLog, 'warn')
|
||||||
|
const startedAt = Date.now()
|
||||||
|
brain = await open()
|
||||||
|
const openMs = Date.now() - startedAt
|
||||||
|
|
||||||
|
const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL'))
|
||||||
|
expect(tearLines.length).toBe(1)
|
||||||
|
const said = String(tearLines[0][0])
|
||||||
|
// Narrated PRECISELY: both generations, the file, and the named cure.
|
||||||
|
expect(said).toContain(`source generation ${torn.sourceGeneration}`)
|
||||||
|
expect(said).toContain(`committed generation ${brain.generationStore.committedGeneration()}`)
|
||||||
|
expect(said).toContain(ENTITY_TREE_STAMP_PATH)
|
||||||
|
expect(said).toContain('DEMOTED')
|
||||||
|
expect(said).toMatch(/repairIndex\(\)/)
|
||||||
|
// Terminal, not a wait: the demotion is O(1) straight-line work, so a tear
|
||||||
|
// cannot turn an open into the 8-minute spin this class was reported as.
|
||||||
|
expect(openMs).toBeLessThan(30_000)
|
||||||
|
|
||||||
|
// The store SERVES — a tear in a stamp never locks an owner out of the
|
||||||
|
// canonical tree the stamp merely describes.
|
||||||
|
expect((await brain.find({ type: 'document', limit: 100 })).length).toBe(3)
|
||||||
|
|
||||||
|
// The demotion CONVERGED: the stamp now names committed truth, and the
|
||||||
|
// next open is quiet. A verdict that re-narrates every open is a wait
|
||||||
|
// wearing a different hat.
|
||||||
|
const restamped = (await readFamilyStamp(brain.storage, ENTITY_TREE_STAMP_PATH)) as FamilyStamp
|
||||||
|
expect(restamped.sourceGeneration).toBe(brain.generationStore.committedGeneration())
|
||||||
|
await brain.close()
|
||||||
|
const warn2 = vi.spyOn(prodLog, 'warn')
|
||||||
|
brain = await open()
|
||||||
|
expect(warn2.mock.calls.filter((c) => String(c[0]).includes('TORN'))).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a READ-ONLY open on a torn tail refuses to guess: terminal verdict + named cure, no re-stamp', async () => {
|
||||||
|
await brain.add({ data: 'ro', type: 'document', metadata: {} })
|
||||||
|
await brain.close()
|
||||||
|
const torn = fabricateTear(3)
|
||||||
|
|
||||||
|
const warn = vi.spyOn(prodLog, 'warn')
|
||||||
|
const reader: any = await Brainy.openReadOnly({
|
||||||
|
requireSubtype: false,
|
||||||
|
storage: { type: 'filesystem', path: dir },
|
||||||
|
silent: true,
|
||||||
|
dimensions: 384
|
||||||
|
})
|
||||||
|
const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL'))
|
||||||
|
expect(tearLines.length).toBe(1)
|
||||||
|
const said = String(tearLines[0][0])
|
||||||
|
expect(said).toContain('READ-ONLY')
|
||||||
|
expect(said).toContain('UNVERIFIED')
|
||||||
|
expect(said).toMatch(/repairIndex\(\)/)
|
||||||
|
await reader.close()
|
||||||
|
|
||||||
|
// A reader never rewrites the store: read the bytes back off disk (not
|
||||||
|
// through a writer open, which would demote them) — the torn stamp is
|
||||||
|
// exactly as it was found.
|
||||||
|
const onDisk = JSON.parse(
|
||||||
|
require('node:zlib')
|
||||||
|
.gunzipSync(fs.readFileSync(path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`)))
|
||||||
|
.toString('utf-8')
|
||||||
|
) as FamilyStamp
|
||||||
|
expect(onDisk.sourceGeneration).toBe(torn.sourceGeneration)
|
||||||
|
expect(onDisk.generation).toBe(torn.generation)
|
||||||
|
|
||||||
|
brain = await open()
|
||||||
|
})
|
||||||
|
|
||||||
it('the one verifier handles both member modes', () => {
|
it('the one verifier handles both member modes', () => {
|
||||||
const rollup: FamilyStamp = {
|
const rollup: FamilyStamp = {
|
||||||
family: 'x',
|
family: 'x',
|
||||||
|
|
@ -127,7 +221,13 @@ describe('entity-tree family stamp', () => {
|
||||||
stampSource: 5,
|
stampSource: 5,
|
||||||
head: 9
|
head: 9
|
||||||
})
|
})
|
||||||
expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 }).state).toBe('incoherent') // ahead of head
|
// AHEAD is its own class — a torn generation-log tail, never folded in
|
||||||
|
// with `incoherent`: the two have opposite cures (recount vs demote).
|
||||||
|
expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 })).toEqual({
|
||||||
|
state: 'torn',
|
||||||
|
stampSource: 5,
|
||||||
|
head: 3
|
||||||
|
})
|
||||||
expect(verifyFamilyStamp(null, 5, {})).toEqual({ state: 'absent' })
|
expect(verifyFamilyStamp(null, 5, {})).toEqual({ state: 'absent' })
|
||||||
|
|
||||||
const enumerated: FamilyStamp = {
|
const enumerated: FamilyStamp = {
|
||||||
|
|
|
||||||
151
tests/integration/filter-operator-conformance.test.ts
Normal file
151
tests/integration/filter-operator-conformance.test.ts
Normal file
|
|
@ -0,0 +1,151 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/filter-operator-conformance
|
||||||
|
* @description THE OPERATOR SET, AND WHAT EACH TOKEN DOES ON THE INDEX PATH.
|
||||||
|
*
|
||||||
|
* The contract-1 manifest splits this engine's `where` operators three ways —
|
||||||
|
* served, served-beyond-baseline, refused-by-name — and two engines must agree
|
||||||
|
* token for token. This lane is the machine-checkable side of that agreement:
|
||||||
|
* it asserts the EXACT accepted set (so a manifest can be diffed against a run
|
||||||
|
* rather than against prose), and it pins each of the three classes.
|
||||||
|
*
|
||||||
|
* The defect it closes: the metadata index's operator switch had no default
|
||||||
|
* case, so an operator it does not implement — `hasAll`, `noneOf`, `excludes`,
|
||||||
|
* `startsWith`, `endsWith`, `matches`, `length` — left the field's match set at
|
||||||
|
* its initial `[]` and `find()` returned an empty page. A documented operator,
|
||||||
|
* implemented in the in-memory matcher, answering silently wrong. Three of the
|
||||||
|
* seven are now SERVED on the index path; the other four are REFUSED BY NAME,
|
||||||
|
* because an equality/range posting index cannot evaluate a substring, a
|
||||||
|
* pattern or an array length without reading every row.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, afterEach } from 'vitest'
|
||||||
|
import { mkdtempSync, rmSync, readFileSync } from 'node:fs'
|
||||||
|
import { tmpdir } from 'node:os'
|
||||||
|
import { join } from 'node:path'
|
||||||
|
import { Brainy } from '../../src/brainy.js'
|
||||||
|
import { NounType } from '../../src/types/graphTypes.js'
|
||||||
|
import { contractVersion, BRAINY_CONTRACT_VERSION } from '../../src/utils/version.js'
|
||||||
|
|
||||||
|
/** The accepted `where` value-operator tokens, as a sorted list. */
|
||||||
|
const ACCEPTED_OPERATORS = [
|
||||||
|
'between', 'contains', 'endsWith', 'eq', 'equals', 'excludes', 'exists',
|
||||||
|
'greaterThan', 'greaterThanOrEqual', 'gt', 'gte', 'hasAll', 'in', 'length',
|
||||||
|
'lessThan', 'lessThanOrEqual', 'lt', 'lte', 'matches', 'missing', 'ne',
|
||||||
|
'noneOf', 'notEquals', 'oneOf', 'startsWith'
|
||||||
|
] as const
|
||||||
|
|
||||||
|
/** Served on the index path with exact posting-set semantics. */
|
||||||
|
const SERVED_ON_INDEX = [
|
||||||
|
'between', 'contains', 'eq', 'equals', 'exists', 'greaterThan',
|
||||||
|
'greaterThanOrEqual', 'gt', 'gte', 'in', 'lessThan', 'lessThanOrEqual',
|
||||||
|
'lt', 'lte', 'missing', 'ne', 'notEquals', 'oneOf',
|
||||||
|
'excludes', 'hasAll', 'noneOf'
|
||||||
|
] as const
|
||||||
|
|
||||||
|
/** Accepted by name, refused by the index path — never answered empty. */
|
||||||
|
const REFUSED_BY_INDEX = ['endsWith', 'length', 'matches', 'startsWith'] as const
|
||||||
|
|
||||||
|
describe('filter operator conformance', () => {
|
||||||
|
const dirs: string[] = []
|
||||||
|
const brains: Brainy[] = []
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
for (const b of brains.splice(0)) {
|
||||||
|
try { await b.close() } catch { /* already closed */ }
|
||||||
|
}
|
||||||
|
for (const d of dirs.splice(0)) {
|
||||||
|
try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ }
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
async function seeded(): Promise<Brainy> {
|
||||||
|
const dir = mkdtempSync(join(tmpdir(), 'brainy-operators-'))
|
||||||
|
dirs.push(dir)
|
||||||
|
const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||||
|
brains.push(brain)
|
||||||
|
await brain.init()
|
||||||
|
await brain.add({
|
||||||
|
data: 'a document about ferrets',
|
||||||
|
type: NounType.Document,
|
||||||
|
metadata: { tags: ['ferret', 'small', 'furry'], team: 'alpha' }
|
||||||
|
})
|
||||||
|
await brain.add({
|
||||||
|
data: 'a document about whales',
|
||||||
|
type: NounType.Document,
|
||||||
|
metadata: { tags: ['whale', 'large'], team: 'beta' }
|
||||||
|
})
|
||||||
|
await brain.flush()
|
||||||
|
return brain
|
||||||
|
}
|
||||||
|
|
||||||
|
it('the accepted operator set is exactly these 25 tokens', async () => {
|
||||||
|
const brain = await seeded()
|
||||||
|
// The engine names its own valid set in the refusal it raises for an
|
||||||
|
// unknown token — the honest place to read it from.
|
||||||
|
let message = ''
|
||||||
|
try {
|
||||||
|
await brain.find({ where: { team: { notIn: ['alpha'] } } } as never)
|
||||||
|
} catch (err) {
|
||||||
|
message = (err as Error).message
|
||||||
|
}
|
||||||
|
expect(message).toMatch(/Unknown filter operator "notIn"/)
|
||||||
|
const listed = (message.match(/Valid operators: ([^.]+)\./)?.[1] ?? '')
|
||||||
|
.split(',')
|
||||||
|
.map((t) => t.trim())
|
||||||
|
.filter(Boolean)
|
||||||
|
.sort()
|
||||||
|
expect(listed).toEqual([...ACCEPTED_OPERATORS].sort())
|
||||||
|
expect(listed.length).toBe(25)
|
||||||
|
// Four tokens a sibling manifest listed as served aliases are NOT in this
|
||||||
|
// engine's set and never have been — they raise INVALID_QUERY.
|
||||||
|
for (const absent of ['is', 'isNot', 'greaterEqual', 'lessEqual']) {
|
||||||
|
expect(listed).not.toContain(absent)
|
||||||
|
await expect(
|
||||||
|
brain.find({ where: { team: { [absent]: 'alpha' } } } as never)
|
||||||
|
).rejects.toThrow(/Unknown filter operator/)
|
||||||
|
}
|
||||||
|
}, 120_000)
|
||||||
|
|
||||||
|
it('serves hasAll, noneOf and excludes on the index path — never an empty page', async () => {
|
||||||
|
const brain = await seeded()
|
||||||
|
|
||||||
|
const hasAll = await brain.find({ where: { tags: { hasAll: ['ferret', 'furry'] } } } as never)
|
||||||
|
expect(hasAll.length).toBe(1)
|
||||||
|
expect((hasAll[0] as { metadata?: Record<string, unknown> }).metadata?.team).toBe('alpha')
|
||||||
|
|
||||||
|
const noneOf = await brain.find({ where: { team: { noneOf: ['alpha'] } } } as never)
|
||||||
|
expect(noneOf.length).toBe(1)
|
||||||
|
expect((noneOf[0] as { metadata?: Record<string, unknown> }).metadata?.team).toBe('beta')
|
||||||
|
|
||||||
|
const excludes = await brain.find({ where: { tags: { excludes: 'whale' } } } as never)
|
||||||
|
expect(excludes.length).toBe(1)
|
||||||
|
expect((excludes[0] as { metadata?: Record<string, unknown> }).metadata?.team).toBe('alpha')
|
||||||
|
|
||||||
|
// hasAll with an operand nothing carries is EMPTY because it is empty —
|
||||||
|
// the honest zero, reached by evaluating the operator.
|
||||||
|
const none = await brain.find({ where: { tags: { hasAll: ['ferret', 'whale'] } } } as never)
|
||||||
|
expect(none.length).toBe(0)
|
||||||
|
}, 120_000)
|
||||||
|
|
||||||
|
it('refuses the four index-unserveable operators BY NAME', async () => {
|
||||||
|
const brain = await seeded()
|
||||||
|
for (const op of REFUSED_BY_INDEX) {
|
||||||
|
const operand = op === 'length' ? 3 : 'a'
|
||||||
|
await expect(
|
||||||
|
brain.find({ where: { team: { [op]: operand } } } as never),
|
||||||
|
`${op} must refuse, never answer an empty page`
|
||||||
|
).rejects.toThrow(new RegExp(`Filter operator "${op}".*cannot be served by the metadata index`, 's'))
|
||||||
|
}
|
||||||
|
}, 120_000)
|
||||||
|
|
||||||
|
it('declares its contract version in code and in package.json', async () => {
|
||||||
|
expect(contractVersion()).toBe(1)
|
||||||
|
expect(BRAINY_CONTRACT_VERSION).toBe(1)
|
||||||
|
const pkg = JSON.parse(readFileSync(join(process.cwd(), 'package.json'), 'utf-8'))
|
||||||
|
expect(pkg.brainyContract).toBe(contractVersion())
|
||||||
|
})
|
||||||
|
|
||||||
|
it('the three classes partition the accepted set', () => {
|
||||||
|
expect([...SERVED_ON_INDEX, ...REFUSED_BY_INDEX].sort()).toEqual([...ACCEPTED_OPERATORS].sort())
|
||||||
|
})
|
||||||
|
})
|
||||||
94
tests/integration/flush-watcher-event-driven.test.ts
Normal file
94
tests/integration/flush-watcher-event-driven.test.ts
Normal file
|
|
@ -0,0 +1,94 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/flush-watcher-event-driven
|
||||||
|
* @description THE FLUSH-REQUEST WATCH IS EVENT-DRIVEN.
|
||||||
|
*
|
||||||
|
* It used to `readdir` the request directory every 500 ms, per brain, for the
|
||||||
|
* life of every writer — armed on every non-reader brain whether or not any
|
||||||
|
* inspector process existed. MEASURED on a production process holding 21
|
||||||
|
* brains: 42 directory reads per second on a completely idle service, plus a
|
||||||
|
* stale-request GC pass on every one of them.
|
||||||
|
*
|
||||||
|
* The law: a request that has not been made is not a cause. The arrival itself
|
||||||
|
* wakes the watcher, so the request is seen SOONER than the poll saw it, and a
|
||||||
|
* slow safety sweep covers filesystems that drop watch events and the GC.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, afterEach, vi } from 'vitest'
|
||||||
|
import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from 'node:fs'
|
||||||
|
import * as nodeFs from 'node:fs'
|
||||||
|
import { tmpdir } from 'node:os'
|
||||||
|
import { join } from 'node:path'
|
||||||
|
import { Brainy } from '../../src/brainy.js'
|
||||||
|
import { NounType } from '../../src/types/graphTypes.js'
|
||||||
|
|
||||||
|
describe('the flush-request watcher', () => {
|
||||||
|
const dirs: string[] = []
|
||||||
|
const brains: Brainy[] = []
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
for (const b of brains.splice(0)) {
|
||||||
|
try { await b.close() } catch { /* already closed */ }
|
||||||
|
}
|
||||||
|
for (const d of dirs.splice(0)) {
|
||||||
|
try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ }
|
||||||
|
}
|
||||||
|
vi.restoreAllMocks()
|
||||||
|
})
|
||||||
|
|
||||||
|
async function openWriter(): Promise<{ brain: Brainy; dir: string }> {
|
||||||
|
const dir = mkdtempSync(join(tmpdir(), 'brainy-flush-watch-'))
|
||||||
|
dirs.push(dir)
|
||||||
|
const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||||
|
brains.push(brain)
|
||||||
|
await brain.init()
|
||||||
|
await brain.add({ data: 'a row', type: NounType.Concept })
|
||||||
|
await brain.flush()
|
||||||
|
return { brain, dir }
|
||||||
|
}
|
||||||
|
|
||||||
|
it('does not poll the request directory on an idle writer', async () => {
|
||||||
|
const { dir } = await openWriter()
|
||||||
|
const reqDir = join(dir, 'locks', '_flush_requests')
|
||||||
|
|
||||||
|
// Count real reads of the request directory over a window far longer than
|
||||||
|
// the old 500ms poll (which would have made ~16 of them).
|
||||||
|
const realReaddir = nodeFs.promises.readdir
|
||||||
|
let requestDirReads = 0
|
||||||
|
const spy = vi
|
||||||
|
.spyOn(nodeFs.promises, 'readdir')
|
||||||
|
.mockImplementation((async (p: unknown, ...rest: unknown[]) => {
|
||||||
|
if (String(p) === reqDir) requestDirReads++
|
||||||
|
return (realReaddir as unknown as (...a: unknown[]) => Promise<unknown>)(p, ...rest)
|
||||||
|
}) as typeof nodeFs.promises.readdir)
|
||||||
|
|
||||||
|
await new Promise((r) => setTimeout(r, 8_000))
|
||||||
|
spy.mockRestore()
|
||||||
|
|
||||||
|
// The old poll: 500ms → ~16 reads. The safety sweep is 30s → 0 in this window.
|
||||||
|
expect(requestDirReads).toBeLessThanOrEqual(1)
|
||||||
|
}, 120_000)
|
||||||
|
|
||||||
|
it('answers a request that arrives, without waiting for the sweep', async () => {
|
||||||
|
const { brain, dir } = await openWriter()
|
||||||
|
const reqDir = join(dir, 'locks', '_flush_requests')
|
||||||
|
const ackDir = join(dir, 'locks', '_flush_responses')
|
||||||
|
mkdirSync(reqDir, { recursive: true })
|
||||||
|
|
||||||
|
// Drop a request exactly as an out-of-process inspector does.
|
||||||
|
const id = 'test-request-0001'
|
||||||
|
writeFileSync(join(reqDir, `${id}.req`), JSON.stringify({ at: Date.now() }))
|
||||||
|
|
||||||
|
// The ack must land far sooner than the 30s safety sweep.
|
||||||
|
const deadline = Date.now() + 10_000
|
||||||
|
let acked = false
|
||||||
|
while (Date.now() < deadline) {
|
||||||
|
try {
|
||||||
|
const entries = await nodeFs.promises.readdir(ackDir)
|
||||||
|
if (entries.some((e) => e.startsWith(id))) { acked = true; break }
|
||||||
|
} catch { /* dir not created yet */ }
|
||||||
|
await new Promise((r) => setTimeout(r, 100))
|
||||||
|
}
|
||||||
|
expect(acked, 'the watcher must answer an arriving request').toBe(true)
|
||||||
|
void brain
|
||||||
|
}, 120_000)
|
||||||
|
})
|
||||||
|
|
@ -188,7 +188,7 @@ describe('health gate (b) — unledgered is unknown: never blocks a serving prov
|
||||||
describe('health gate (c) — degraded-but-serving narrates once per generation', () => {
|
describe('health gate (c) — degraded-but-serving narrates once per generation', () => {
|
||||||
// PER-FAMILY LAW (10.4.1): a metadata find() consults the METADATA leg only — the
|
// PER-FAMILY LAW (10.4.1): a metadata find() consults the METADATA leg only — the
|
||||||
// degraded report lives on the family the read actually consults.
|
// degraded report lives on the family the read actually consults.
|
||||||
it('a heal:"repair" failure serves; narrates once per generation, twice across a generation bump', async () => {
|
it('a heal:"repair" failure serves; narrates once per DISTINCT VERDICT, not once per generation bump', async () => {
|
||||||
const brain = new Brainy(createTestConfig({ silent: true }))
|
const brain = new Brainy(createTestConfig({ silent: true }))
|
||||||
await brain.init()
|
await brain.init()
|
||||||
brains.push(brain)
|
brains.push(brain)
|
||||||
|
|
@ -197,12 +197,13 @@ describe('health gate (c) — degraded-but-serving narrates once per generation'
|
||||||
|
|
||||||
const internals = internalsOf(brain)
|
const internals = internalsOf(brain)
|
||||||
let generation = 1
|
let generation = 1
|
||||||
|
let detail = 'counter drift'
|
||||||
internals.metadataIndex.healthReport = () =>
|
internals.metadataIndex.healthReport = () =>
|
||||||
healthReport({
|
healthReport({
|
||||||
provider: 'vector',
|
provider: 'vector',
|
||||||
serving: true,
|
serving: true,
|
||||||
healthy: false,
|
healthy: false,
|
||||||
invariants: [invariant({ name: 'stale-vector-counter', holds: false, heal: 'repair', detail: 'counter drift' })],
|
invariants: [invariant({ name: 'stale-vector-counter', holds: false, heal: 'repair', detail })],
|
||||||
generation
|
generation
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
@ -212,11 +213,22 @@ describe('health gate (c) — degraded-but-serving narrates once per generation'
|
||||||
|
|
||||||
await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1)
|
await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1)
|
||||||
await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1)
|
await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1)
|
||||||
expect(countNarrations()).toBe(1) // same generation both times — one narration
|
expect(countNarrations()).toBe(1) // same verdict both times — one narration
|
||||||
|
|
||||||
|
// THE DEDUPE KEY IS THE VERDICT, NOT THE COUNTER. A provider's `generation`
|
||||||
|
// bumps on every ledger mutation and every rebuild boundary, so keying the
|
||||||
|
// narration on it re-printed an UNCHANGED health line on every read that
|
||||||
|
// consulted a busy provider — and, in the other direction, let a provider
|
||||||
|
// that never bumped suppress a line whose reasons had genuinely changed.
|
||||||
|
// An unchanged verdict is silent however the counter moves:
|
||||||
generation = 2
|
generation = 2
|
||||||
await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1)
|
await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1)
|
||||||
expect(countNarrations()).toBe(2) // generation bumped — a second narration
|
expect(countNarrations()).toBe(1) // generation bumped, verdict identical — still silent
|
||||||
|
|
||||||
|
// ...and a CHANGED verdict is always heard, bump or no bump:
|
||||||
|
detail = 'counter drift widened to 12 rows'
|
||||||
|
await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1)
|
||||||
|
expect(countNarrations()).toBe(2) // the reasons changed — a new narration
|
||||||
|
|
||||||
delete internals.metadataIndex.healthReport
|
delete internals.metadataIndex.healthReport
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,7 @@ import { describe, it, expect, afterEach } from 'vitest'
|
||||||
import * as fs from 'node:fs'
|
import * as fs from 'node:fs'
|
||||||
import * as path from 'node:path'
|
import * as path from 'node:path'
|
||||||
import * as os from 'node:os'
|
import * as os from 'node:os'
|
||||||
|
import * as zlib from 'node:zlib'
|
||||||
import { Brainy } from '../../src/brainy.js'
|
import { Brainy } from '../../src/brainy.js'
|
||||||
import { NounType } from '../../src/types/graphTypes.js'
|
import { NounType } from '../../src/types/graphTypes.js'
|
||||||
import { GenerationStore } from '../../src/db/generationStore.js'
|
import { GenerationStore } from '../../src/db/generationStore.js'
|
||||||
|
|
@ -57,6 +58,107 @@ describe('history repacking — the two-tier lifecycle', () => {
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
/**
|
||||||
|
* THE HOLE, END TO END — the shape a real store carries.
|
||||||
|
*
|
||||||
|
* A forensic fixture was measured with generation directories 1..2503
|
||||||
|
* present except for exactly one: 1416. Its fact-log segment already showed
|
||||||
|
* the tell — `seg-...1410.bfl` declaring firstGeneration 1410, lastGeneration
|
||||||
|
* 1940 (531 generations) while recording only 530 facts.
|
||||||
|
*
|
||||||
|
* Before the fix, repacking such a store folded ACROSS that hole: the batch
|
||||||
|
* skipped 1416 (no readable delta) and the sealed segment declared a range
|
||||||
|
* spanning it anyway. The next open merged that declared range back into
|
||||||
|
* committedRanges, re-admitting 1416 as committed history, and every
|
||||||
|
* subsequent auto-compaction pass then asked the packed tier for a frame
|
||||||
|
* that was never written — producing, on EVERY run, the non-fatal narration
|
||||||
|
*
|
||||||
|
* Auto-compaction of generational history failed (non-fatal): generation
|
||||||
|
* N is inside sealed segment seg-....bgs's declared range but has no frame
|
||||||
|
* — packed history is damaged
|
||||||
|
*
|
||||||
|
* This pin removes a generation directory to make the same hole, then
|
||||||
|
* requires repack + reopen + compaction to complete cleanly.
|
||||||
|
*/
|
||||||
|
it('a missing generation directory does not poison the packed tier', async () => {
|
||||||
|
const dir = tempDir()
|
||||||
|
// `retention: 'all'` throughout: close() otherwise auto-compacts the
|
||||||
|
// history away, and this pin needs the cold generations still on disk so
|
||||||
|
// there is something to punch a hole in. The live window stays at its
|
||||||
|
// production default for the build phase, so nothing folds yet.
|
||||||
|
const archival = async (): Promise<Brainy> => {
|
||||||
|
const b = new Brainy({
|
||||||
|
requireSubtype: false,
|
||||||
|
storage: { type: 'filesystem', path: dir },
|
||||||
|
embeddingFunction: stub,
|
||||||
|
retention: 'all'
|
||||||
|
})
|
||||||
|
await b.init()
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
const brain = await archival()
|
||||||
|
|
||||||
|
const id = await brain.add({
|
||||||
|
data: 'holed-entity',
|
||||||
|
type: NounType.Document,
|
||||||
|
metadata: { v: 0 }
|
||||||
|
})
|
||||||
|
// One flush per update: single-op writes coalesce inside a flush window,
|
||||||
|
// so a history deep enough to have a middle needs the windows separated.
|
||||||
|
for (let v = 1; v <= 12; v++) {
|
||||||
|
await brain.update({ id, metadata: { v } })
|
||||||
|
await brain.flush()
|
||||||
|
}
|
||||||
|
await brain.close()
|
||||||
|
|
||||||
|
// Punch the hole: delete ONE generation directory in the middle of the
|
||||||
|
// cold range, exactly as the real store presents it.
|
||||||
|
const genRoot = path.join(dir, '_generations')
|
||||||
|
const numeric = fs
|
||||||
|
.readdirSync(genRoot, { withFileTypes: true })
|
||||||
|
.filter((e) => e.isDirectory() && /^\d+$/.test(e.name))
|
||||||
|
.map((e) => Number(e.name))
|
||||||
|
.sort((a, b) => a - b)
|
||||||
|
expect(numeric.length).toBeGreaterThan(6)
|
||||||
|
const victim = numeric[Math.floor(numeric.length / 2)]
|
||||||
|
fs.rmSync(path.join(genRoot, String(victim)), { recursive: true, force: true })
|
||||||
|
|
||||||
|
// Now shrink the live window and reopen. close() repacks automatically
|
||||||
|
// (brainy.ts phase 0b), so this is the production sequence exactly: a
|
||||||
|
// store with a hole in its history gets folded by ordinary housekeeping,
|
||||||
|
// with nobody asking for it.
|
||||||
|
;(GenerationStore as any).REPACK_LIVE_WINDOW = 3
|
||||||
|
const reopened = await archival()
|
||||||
|
const result = await reopened.repackHistory()
|
||||||
|
expect(result.foldedGenerations).toBeGreaterThan(0)
|
||||||
|
|
||||||
|
const segDir = path.join(dir, SEGMENTS_PREFIX)
|
||||||
|
const manifestPath = ['manifest.json', 'manifest.json.gz']
|
||||||
|
.map((f) => path.join(segDir, f))
|
||||||
|
.find((p) => fs.existsSync(p))!
|
||||||
|
const raw = manifestPath.endsWith('.gz')
|
||||||
|
? zlib.gunzipSync(fs.readFileSync(manifestPath)).toString('utf8')
|
||||||
|
: fs.readFileSync(manifestPath, 'utf8')
|
||||||
|
const manifest = JSON.parse(raw) as {
|
||||||
|
segments: Array<{ firstGeneration: number; lastGeneration: number; frames: number }>
|
||||||
|
}
|
||||||
|
|
||||||
|
// THE LAW: every sealed segment declares exactly as many generations as it
|
||||||
|
// holds frames, and none of them spans the victim.
|
||||||
|
for (const s of manifest.segments) {
|
||||||
|
expect(s.lastGeneration - s.firstGeneration + 1).toBe(s.frames)
|
||||||
|
expect(victim >= s.firstGeneration && victim <= s.lastGeneration).toBe(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
await reopened.close()
|
||||||
|
|
||||||
|
// And the pass that used to fail on every run now completes: reopen (which
|
||||||
|
// re-seeds committedRanges from the packed tier) then compact history.
|
||||||
|
const third = await openBrain(dir)
|
||||||
|
await expect(third.compactHistory({ maxGenerations: 2 })).resolves.toBeDefined()
|
||||||
|
await third.close()
|
||||||
|
})
|
||||||
|
|
||||||
it('repack preserves every historical read across cold reopen; folded dirs are gone', async () => {
|
it('repack preserves every historical read across cold reopen; folded dirs are gone', async () => {
|
||||||
;(GenerationStore as any).REPACK_LIVE_WINDOW = 3
|
;(GenerationStore as any).REPACK_LIVE_WINDOW = 3
|
||||||
const dir = tempDir()
|
const dir = tempDir()
|
||||||
|
|
|
||||||
|
|
@ -21,10 +21,16 @@ describe('Hybrid Search with VFS', () => {
|
||||||
testDir = path.join(os.tmpdir(), `brainy-hybrid-vfs-test-${Date.now()}`)
|
testDir = path.join(os.tmpdir(), `brainy-hybrid-vfs-test-${Date.now()}`)
|
||||||
fs.mkdirSync(testDir, { recursive: true })
|
fs.mkdirSync(testDir, { recursive: true })
|
||||||
|
|
||||||
|
// `storage.path`, NOT the pre-8.0 `options.basePath` alias. That alias was
|
||||||
|
// removed at the 8.0 major and configures nothing, so this suite silently
|
||||||
|
// opened the DEFAULT store instead of its own temp directory — sharing one
|
||||||
|
// on-disk brain with every other run on the machine, accumulating tens of
|
||||||
|
// thousands of rows, and eventually failing on that shared store's graph
|
||||||
|
// adjacency rather than on anything it was written to test.
|
||||||
brain = new Brainy({ requireSubtype: false,
|
brain = new Brainy({ requireSubtype: false,
|
||||||
storage: {
|
storage: {
|
||||||
type: 'filesystem',
|
type: 'filesystem',
|
||||||
options: { basePath: testDir }
|
path: testDir
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue