Compare commits

..

No commits in common. "main" and "release/10.4.1" have entirely different histories.

209 changed files with 1368 additions and 20100 deletions

View file

@ -2,7 +2,7 @@
## What Is Brainy ## What Is Brainy
@soulcraftlabs/brainy (v7.17.0) is a Universal Knowledge Protocol -- a Triple Intelligence database combining vector search, graph traversal, and metadata filtering in a single library. Published to npm as a public MIT-licensed package. @soulcraft/brainy (v7.17.0) is a Universal Knowledge Protocol -- a Triple Intelligence database combining vector search, graph traversal, and metadata filtering in a single library. Published to npm as a public MIT-licensed package.
## Core Architecture ## Core Architecture

View file

@ -5,10 +5,6 @@ name: CI
# sequential, so tag-triggered matrix jobs (~22 min) would queue AHEAD of the # sequential, so tag-triggered matrix jobs (~22 min) would queue AHEAD of the
# tag's publish-source run and starve every release (observed on 8.10.3 and # tag's publish-source run and starve every release (observed on 8.10.3 and
# 9.0.0: the publish sat behind the tag's own redundant CI). # 9.0.0: the publish sat behind the tag's own redundant CI).
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: true
on: on:
push: push:
branches: ['**'] branches: ['**']

View file

@ -1,148 +0,0 @@
name: Delta Gate
# On-demand candidate-vs-control gate on the capped functional CI lane
# (label: gate-functional). That lane is Bun-only host-mode — there is no
# Node.js runtime available to it, so this workflow deliberately avoids every
# JS-based action (checkout/setup-node/setup-bun/upload-artifact all require
# one) and does everything with plain git + bun in shell steps instead.
#
# Verdict lines a caller should grep for in the run log:
# COLLECTED patch=<n> control=<n> — collection-truncation guard inputs
# NEW-RED-COUNT:<n> — failures on candidate absent from control
# DELTA-GATE: CLEAN | NEW REDS | INVALID | STOPPED-BY-REGISTRY-TRIPWIRE
#
# The lane's own housekeeping stops the runner and drops a marker file when
# host pressure (I/O, registry latency, disk budget) trips — never ours to
# interpret as a red or a green. The final step checks for that marker before
# it says anything about pass/fail.
on:
workflow_dispatch:
inputs:
candidate:
description: 'Candidate ref (branch or sha) to gate'
required: true
type: string
control:
description: 'Control sha to diff against'
required: true
type: string
# workflow_dispatch needs Actions-unit write on the dispatching credential;
# push does not (it runs from the pushed ref's own tree), so a plain push
# to a release or CI branch is the fallback trigger while that grant is
# outstanding — see the ref-resolution step below for what it gates against.
push:
branches: ['rel/**', 'ci/**']
concurrency:
group: delta-gate
cancel-in-progress: false
jobs:
delta-gate:
name: Delta gate — candidate vs control
runs-on: gate-functional
timeout-minutes: 120
steps:
- name: Resolve candidate/control refs
id: refs
run: |
candidate="${{ github.event.inputs.candidate }}"
control="${{ github.event.inputs.control }}"
# workflow_dispatch supplies both explicitly; a push event carries
# neither — fall back to the pushed commit as candidate and the
# last released, known-good tip (10.4.9) as control, so a plain
# push still produces a meaningful gate instead of an empty ref.
if [ -z "$candidate" ]; then candidate="${{ github.sha }}"; fi
if [ -z "$control" ]; then control="eec90bdd"; fi
echo "candidate=$candidate" >> "$GITHUB_OUTPUT"
echo "control=$control" >> "$GITHUB_OUTPUT"
echo "Resolved (trigger=${{ github.event_name }}): candidate=$candidate control=$control"
- name: Clean any residue from a prior run
run: rm -rf "ob-cand-${{ github.run_id }}" "ob-ctrl-${{ github.run_id }}" "/tmp/ob-${{ github.run_id }}-"*
- name: Clone + test — candidate
id: patch
run: |
set -o pipefail
git clone --quiet "https://source.soulcraft.com/soulcraftlabs/open-brainy.git" "ob-cand-${{ github.run_id }}"
cd "ob-cand-${{ github.run_id }}"
git checkout --quiet "${{ steps.refs.outputs.candidate }}"
git log --oneline -1
bun install
rc=0
bun x vitest run > "/tmp/ob-${{ github.run_id }}-patch.log" 2>&1 || rc=$?
echo "PATCH-RC:$rc"
grep -aE "Tests .*(passed|failed)" "/tmp/ob-${{ github.run_id }}-patch.log" | tail -1
grep -aE "^ FAIL |^\s+×" "/tmp/ob-${{ github.run_id }}-patch.log" | sed -E "s/ [0-9]+ms$//" | sed -E "s/^\s+//" | sort -u > "/tmp/ob-${{ github.run_id }}-patch.fail"
echo "PATCH-FAILING:$(wc -l < "/tmp/ob-${{ github.run_id }}-patch.fail")"
- name: Clone + test — control
id: control
run: |
set -o pipefail
git clone --quiet "https://source.soulcraft.com/soulcraftlabs/open-brainy.git" "ob-ctrl-${{ github.run_id }}"
cd "ob-ctrl-${{ github.run_id }}"
git checkout --quiet "${{ steps.refs.outputs.control }}"
git log --oneline -1
bun install
rc=0
bun x vitest run > "/tmp/ob-${{ github.run_id }}-control.log" 2>&1 || rc=$?
echo "CONTROL-RC:$rc"
grep -aE "Tests .*(passed|failed)" "/tmp/ob-${{ github.run_id }}-control.log" | tail -1
grep -aE "^ FAIL |^\s+×" "/tmp/ob-${{ github.run_id }}-control.log" | sed -E "s/ [0-9]+ms$//" | sed -E "s/^\s+//" | sort -u > "/tmp/ob-${{ github.run_id }}-control.fail"
echo "CONTROL-FAILING:$(wc -l < "/tmp/ob-${{ github.run_id }}-control.fail")"
- name: Delta gate verdict
if: always()
run: |
set -o pipefail
# The lane's own tripwire wins over anything we would otherwise say:
# a bare failure/timeout above with this marker present is host
# pressure, never a real red and never a real green.
if [ -f /srv/gate-lane/TRIPWIRE-STOPPED ]; then
echo "DELTA-GATE: STOPPED-BY-REGISTRY-TRIPWIRE"
head -1 /srv/gate-lane/TRIPWIRE-STOPPED
exit 3
fi
patch_log="/tmp/ob-${{ github.run_id }}-patch.log"
control_log="/tmp/ob-${{ github.run_id }}-control.log"
patch_fail="/tmp/ob-${{ github.run_id }}-patch.fail"
control_fail="/tmp/ob-${{ github.run_id }}-control.fail"
if [ ! -s "$patch_log" ] || [ ! -s "$control_log" ]; then
echo "DELTA-GATE: INVALID — a leg produced no log (see the two steps above for the real cause)"
exit 2
fi
pt=$(grep -aoE "\(([0-9]+)\)$" "$patch_log" | tail -1 | tr -d "()")
ct=$(grep -aoE "\(([0-9]+)\)$" "$control_log" | tail -1 | tr -d "()")
echo "COLLECTED patch=${pt:-0} control=${ct:-0}"
if [ "${pt:-0}" -lt 3000 ] || [ "${ct:-0}" -lt 3000 ]; then
echo "DELTA-GATE: INVALID — truncated collection"
exit 2
fi
echo "=== NEW REDS ==="
comm -23 "$patch_fail" "$control_fail"
new=$(comm -23 "$patch_fail" "$control_fail" | wc -l)
echo "NEW-RED-COUNT:$new"
echo "=== full candidate fail list ==="
cat "$patch_fail"
echo "=== full control fail list ==="
cat "$control_fail"
if [ "$new" -eq 0 ]; then
echo "DELTA-GATE: CLEAN"
else
echo "DELTA-GATE: NEW REDS"
exit 1
fi
- name: Clean up (mind the lane's disk budget)
if: always()
run: rm -rf "ob-cand-${{ github.run_id }}" "ob-ctrl-${{ github.run_id }}" "/tmp/ob-${{ github.run_id }}-"*

View file

@ -12,11 +12,6 @@ on:
push: push:
tags: tags:
- 'v*' - 'v*'
workflow_dispatch:
inputs:
ref_reason:
description: 'why this manual run (e.g. tag event dropped)'
required: false
jobs: jobs:
publish: publish:
@ -37,7 +32,7 @@ jobs:
run: | run: |
set -eo pipefail set -eo pipefail
SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraftlabs/npm/" SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/"
VERSION="$(node -p "require('./package.json').version")" VERSION="$(node -p "require('./package.json').version")"
# The dist-tag follows the version: a prerelease (any hyphen — # The dist-tag follows the version: a prerelease (any hyphen —
# 10.4.0-rc.1) publishes under 'rc' and must NEVER move 'latest' — # 10.4.0-rc.1) publishes under 'rc' and must NEVER move 'latest' —
@ -48,13 +43,13 @@ jobs:
case "$VERSION" in case "$VERSION" in
*-*) NPM_TAG="rc" ;; *-*) NPM_TAG="rc" ;;
esac esac
echo "Publishing @soulcraftlabs/brainy@${VERSION} to The Source registry (dist-tag: ${NPM_TAG})..." echo "Publishing @soulcraft/brainy@${VERSION} to The Source registry (dist-tag: ${NPM_TAG})..."
TMPRC="$(mktemp)" TMPRC="$(mktemp)"
chmod 600 "$TMPRC" chmod 600 "$TMPRC"
{ {
echo "@soulcraftlabs:registry=${SOURCE_NPM_REG}" echo "@soulcraft:registry=${SOURCE_NPM_REG}"
echo "//source.soulcraft.com/api/packages/soulcraftlabs/npm/:_authToken=${FORGE_NPM_TOKEN}" echo "//source.soulcraft.com/api/packages/soulcraft/npm/:_authToken=${FORGE_NPM_TOKEN}"
} > "$TMPRC" } > "$TMPRC"
# The release script bumps package.json's version before it tags, so # The release script bumps package.json's version before it tags, so
@ -69,7 +64,7 @@ jobs:
# exit code: a benign duplicate publish (a prior run, or a mirror, already # exit code: a benign duplicate publish (a prior run, or a mirror, already
# landed this exact version) reports failure even though the registry # landed this exact version) reports failure even though the registry
# already holds the right content. # already holds the right content.
LANDED_VERSION="$(npm view "@soulcraftlabs/brainy@${VERSION}" version --userconfig "$TMPRC" 2>/dev/null || echo "")" LANDED_VERSION="$(npm view "@soulcraft/brainy@${VERSION}" version --userconfig "$TMPRC" 2>/dev/null || echo "")"
rm -f "$TMPRC" rm -f "$TMPRC"
if [ "$LANDED_VERSION" != "$VERSION" ]; then if [ "$LANDED_VERSION" != "$VERSION" ]; then
@ -78,7 +73,7 @@ jobs:
fi fi
if [ "$PUBLISH_OK" = true ]; then if [ "$PUBLISH_OK" = true ]; then
echo "Published and verified @soulcraftlabs/brainy@${VERSION} on The Source registry." echo "Published and verified @soulcraft/brainy@${VERSION} on The Source registry."
else else
echo "::warning::npm publish reported failure, but readback confirms @soulcraftlabs/brainy@${VERSION} is already live on The Source (a prior run or mirror landed it) — treating this run as successful, since the registry content is correct. Any OTHER failure mode would have failed the readback check above instead." echo "::warning::npm publish reported failure, but readback confirms @soulcraft/brainy@${VERSION} is already live on The Source (a prior run or mirror landed it) — treating this run as successful, since the registry content is correct. Any OTHER failure mode would have failed the readback check above instead."
fi fi

View file

@ -2,155 +2,23 @@
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
### [10.4.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0...v10.4.1) (2026-08-26)
### [10.4.13](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.12...v10.4.13) (2026-09-03)
- A shutdown that holds its listener until the exit decision, and a test suite that closes every brain it opens
- fix(shutdown): the engine's signal handler keeps its listener registered until the exit decision is made — closing the last live instance no longer deregisters the handler mid-run, so a second signal delivery during a clean shutdown can never kill the process after the work is done (a2ea21b3)
- fix(release): the release wall entry commits under an explicit git identity read from the developer's checkout; a host with no identity refuses by name instead of failing inside git (aac853d3)
- test(hygiene): every brain a test file creates is closed by that file — 40 files fixed, the leaks that let a stray cadence narrate into later files are gone; brains whose init() was expected to fail are closed too (6eb5e448)
### [10.4.12](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.11...v10.4.12) (2026-09-03)
- Mixed-kind fields index exactly, arrays to 256, a drained loop is not a shutdown, and finds project from the column store
- fix(index): a metadata field holds every value kind it was written with — one posting column per (field, kind); an equality filter reads the query value's own kind, a range routes by its bounds; nothing is refused and nothing is silently dropped; an index written by the old shape opens unchanged (a128f0ed)
- fix(metadata): metadata arrays index up to 256 elements; a longer array refuses at write time by name (MetadataArrayTooLargeError) — a vector parked in metadata now throws; move it to `vector` (e435da78)
- fix(shutdown): beforeExit runs a non-closing flush only — a script that never calls close() exits with the writer lock on disk and no clean-shutdown marker, and the next open evicts the stale lock and folds the log, bounded; SIGTERM and SIGINT are unchanged (6baa4d7f)
- feat(find): field projection — find({fields}) and get({fields}) resolve scalars from the column store on every leg, including vector-leg finds; absent fields stay absent (ad0f493f)
- fix(find): orderBy is the order on every find path, not only the metadata-only one (5e720d17)
- fix(metadata): the legacy sparse range path orders values, or refuses by name — never ranks by hash (a7eb7f52)
- fix(close): a read-only brain writes nothing under `_system/` (f27a7776)
- fix(contract): the flush gate's internals are private, not doors (72c8ee6a)
- test(hygiene): the triple-intelligence correctness cases sit in the gate; the idle and connected-find pins name the brain they measure (28083981)
- ci(release): the rail writes its own wall entry into the shared releases repo — never hand-written again (adcb883e)
### [10.4.11](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.9...v10.4.11) (2026-09-02)
- ci: superseded pushes cancel their own runs (concurrency per ref) (6053f6d4)
- test(batch): the batch-size-limit tests add unvectored items — they test batching, not embedding (a1423c6d)
- fix(flush): the gate settles its waiter from the machine, never from a chain (dea3ec20)
- test(batch): the batch-vs-individual timing assertion runs in the perf lane, not the correctness gate (ebb3a4bf)
- test(gate): the coverage guard counts the perf lane's config as a gate (2c5e3474)
- chore(contract): emit the 10.4.11 manifest (4142f368)
- fix(close): a read-only brain writes no clean-shutdown evidence — the marker is the writer's word about itself (367ca721)
- fix(generation-store): commitTransaction refuses while single-ops are pending — the order invariant is enforced, not assumed (a79db434)
- test(shutdown): pin one owner per brain — real processes, real signals (da951990)
- fix(shutdown): one owner per brain — the signal handler defers to close(), and flush is single-flight (ec644bde)
- fix(vfs): a path-scoped search is a served range over the path, not a refused prefix match (65493ba2)
- ci(test): perf and scale benchmarks leave the correctness gate (dee46b35)
- test(open): pin the pending-embed checkpoint — stuck id, crash matrix, torn fallback (1fb51093)
- perf(open): the pending-embed fold is bounded by a checkpoint of the SET, not an empty-only mark (15d4f65d)
- perf(open): a sealed segment the manifest proves is below the bound is never read (bc70c43d)
- fix(find): a page the metadata block already cut is not cut again (905c267c)
- fix(find): the hybrid legs rank inside the filter, and only the page is read (b1c70544)
- ci(delta-gate): add a push fallback trigger alongside workflow_dispatch (67ae0046)
- ci: add the delta-gate workflow for the capped functional lane (9922631d)
- docs(plugin): the planner door's hiddenIds contract is the answer, not the mechanism (2633e8d5)
- feat(engine): a protected factory for the generation store — a subclass may substitute one that keeps the contract (f763317a)
- fix(find): near() searches around the anchor's own vector, and refuses by name without one (a8c5fbf9)
- Merge remote-tracking branches 'origin/fix/planner-provider-door' and 'origin/fix/containment-batching' into rel/10.4.10-candidate (34f1886f)
- feat(plugin): an optional planFindPage door — an index that can plan a find answers it in one call (4d5f823f)
- perf(vfs): repairContainment's reconcile is one paged edge walk, not one graph call per file (3e60aded)
### [10.4.9](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.6...v10.4.9) (2026-09-02)
- Merge branch 'fix/pending-embed-low-water' into rel/10.4.9-candidate (2648f56d)
- fix(open): pending-embed recovery keeps the crash-recovery contract — foreground, bounded by the mark (8a2ebacf)
- Merge branches 'fix/connected-find-order', 'fix/pending-embed-low-water' and 'fix/related-verb-array' into rel/10.4.9-candidate (d5147ed6)
- fix(graph): the verb fast paths honour every requested type, source, and target (6a89adc4)
- perf(open): pending-embed recovery is bounded by a low-water mark and runs behind the doors (88e79729)
- fix(find): connected finds are graph-first — neighbours, then the filter over those ids, then the page (077cbc0b)
- fix(storage): counts persistence is single-flight, coalesced, and never races its own temp file (5e3b343a)
### [10.4.6](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.5...v10.4.6) (2026-08-31)
- fix(transact): metadata-index ops take their JSON-safe view at the crossing, not at construction (73500e7d)
### [10.4.5](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.4...v10.4.5) (2026-08-31)
- build(release): the docs-push step retires — this engine documents itself in its own repository (d6bcb14f)
- fix(generations): a sealed segment may only declare the generations it holds (a963a744)
- fix(recovery): a torn generation-log tail is a terminal verdict, never a wait (c9930871)
### [10.4.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.3...v10.4.4) (2026-08-28)
- fix(vfs): the old-root sweep narrates only when it has something to say (d49148e1)
- fix(tests): the health-gate pin follows the verdict, and the VFS suite uses its own store (42e2da25)
- Merge branch 'next/open-lazy-open-and-counts' (5ebd3b40)
- docs: the contract manifest stands alone; public docs describe this engine only (a8c724a2)
- docs(releases): 10.4.4 consumer notes — correctness and observability, with the performance line stated exactly (61a46927)
- docs: measurements in public history carry numbers, not provenance (02c61636)
- feat(open): name the two steps that hold the vfs-bootstrap phase (2cf38010)
- fix(storage): a dead flush watch falls back to the 500ms poll, not the 30s sweep (5c22f950)
- fix(storage): the flush watcher cannot arm twice in its async window (16d2e1a9)
- perf(idle): the flush-request watch is event-driven; the heartbeat is observability (fb1da1c5)
- perf(open): answer "are there any entities?" with one directory read (417ddb51)
- perf(generations): discover generations by directory name, not by walking the log (9dd39921)
- fix(flush): clear() and repairIndex() set the dirty witness themselves (e4c27fbc)
- feat(open): the open names the STEP that cost the time, not just the phase (5a091cca)
- perf(vfs): the old-root sweep runs once per store, not once per open (4a67aa0f)
- chore: keep the generated neural stamps at main's values (c1f09723)
- feat(contract): declare contract 1, serve three operators, refuse four by name (48802ba3)
- fix(open): a provider rebuilding itself is a third state, not a CRITICAL (50676c02)
- feat(open): open never waits for a provider that is rebuilding itself (131daa08)
- perf(flush): an idle brain does no work — no periodic flush without a write (f5a6cb3f)
- feat(repair): repairIndex narrates every phase and its receipt carries the walls (3fffd9c6)
- fix(storage): a suspect count ledger heals itself, and counts.json is written atomically (f4e2d34b)
- feat(open): the open narrates itself, on a channel production cannot clamp (afe08a1f)
- fix(storage): a clean close is recorded, and the writer lock is always given up (e652162c)
- docs: repository links point at soulcraftlabs/open-brainy — the soulcraft/brainy path becomes the native engine's repo tonight (38c3397b)
### [10.4.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.2...v10.4.3) (2026-08-27)
- Merge branch 'next/open-brainy-rename' (a58372f0)
- chore: rename to @soulcraftlabs/brainy for Open Brainy on The Source (a99b1e83)
- docs(releases): 10.4.3 — Open Brainy's first release under the new name, same engine as 10.4.2; The Source is the one registry (9f248b24)
### [10.4.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.2-rc.1...v10.4.2) (2026-08-27)
- docs(releases): 10.4.1 and 10.4.2 consumer notes; 10.4.2 is the last MIT release under this name, Open Brainy continues at @soulcraftlabs/brainy (a082e0ef)
### [10.4.2-rc.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.1...v10.4.2-rc.1) (2026-08-27)
- Merge branch 'next/zero-norm-unvector-door' (9b84ef5b)
- fix(vectors): a zero-norm vector is not a vector, canonical side included, plus the sanctioned unvector door (0de76659)
- fix(hnsw): skip unvectored rows on rebuild; refuse empty vectors in the index (8fc553b1)
- fix(storage): derive the canonical count ledger from identity records, stamp the derivation rule, and mark legacy-derived ledgers suspect at load (fd6b4ce4)
- Merge branch 'next/enumeration-identity-rekey' (204d74c1)
- fix(storage): enumeration re-keys on the identity record, not the vector leg (f8d8ce16)
- fix(init): rethrow plugin activation failures with the original error as cause so the originating frame survives to the caller (2496e09a)
- Merge branch 'next/vfs-root-zero-norm' (4c7b0fab)
- fix(vfs): the VFS root never persists a zero-norm vector (c6cc0de9)
- build: derive generated-file stamps from git commit time, not wall clock (8a5c1245)
- Merge remote-tracking branch 'origin/release/10.4.1' (aad9e2ee)
- docs(concepts): the serving law — a failure is graded by whether an answer could be wrong, never by the cost of the fix; reads refuse per family (2914e0eb)
- chore(release): 10.4.1-rc.1 (7870dc40)
### [10.4.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0...v10.4.1) (2026-08-26)
- fix(reads): the read gate is per-family; a write carrying unchanged data never re-embeds (c039411e) - fix(reads): the read gate is per-family; a write carrying unchanged data never re-embeds (c039411e)
- docs(guide): the docs pipeline publishes through the ingest API — the separate deploy step is retired (21e506e8) - docs(guide): the docs pipeline publishes through the ingest API — the separate deploy step is retired (21e506e8)
### [10.4.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.4...v10.4.0) (2026-08-26) ### [10.4.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.4...v10.4.0) (2026-08-26)
- docs(releases): the 10.4.0 entry catches up to the late trains — repair routing, the vector ledger and open-gate leg, the loud config guard, the JSON-safe crossing (834149ed) - docs(releases): the 10.4.0 entry catches up to the late trains — repair routing, the vector ledger and open-gate leg, the loud config guard, the JSON-safe crossing (834149ed)
### [10.4.0-rc.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.3...v10.4.0-rc.4) (2026-08-25) ### [10.4.0-rc.4](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.3...v10.4.0-rc.4) (2026-08-25)
- feat(vector): the vectored-noun scalar joins the count ledger; the open gate closes the vector leg (9730835b) - feat(vector): the vectored-noun scalar joins the count ledger; the open gate closes the vector leg (9730835b)
### [10.4.0-rc.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.2...v10.4.0-rc.3) (2026-08-25) ### [10.4.0-rc.3](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.2...v10.4.0-rc.3) (2026-08-25)
- fix(update-seam): the metadata crossing never carries BigInt endpoint ints (f4780c8e) - fix(update-seam): the metadata crossing never carries BigInt endpoint ints (f4780c8e)
- Merge branch 'worktree-agent-ad3aff0dffd17a6eb' (f14da34b) - Merge branch 'worktree-agent-ad3aff0dffd17a6eb' (f14da34b)
@ -159,7 +27,7 @@ All notable changes to this project will be documented in this file. See [standa
- feat(open-path): init never gates on the embedding model; open goes concurrent; slow opens narrate (96624f40) - feat(open-path): init never gates on the embedding model; open goes concurrent; slow opens narrate (96624f40)
### [10.4.0-rc.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.1...v10.4.0-rc.2) (2026-08-25) ### [10.4.0-rc.2](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.1...v10.4.0-rc.2) (2026-08-25)
- test(readiness): the report helper's clock freezes — two independently-built reports compared across a millisecond tick made the plant lane red (39b916a3) - test(readiness): the report helper's clock freezes — two independently-built reports compared across a millisecond tick made the plant lane red (39b916a3)
- feat(repair): a heal:'repair' verdict routes to the provider's own incremental repair() (553e0d97) - feat(repair): a heal:'repair' verdict routes to the provider's own incremental repair() (553e0d97)
@ -170,7 +38,7 @@ All notable changes to this project will be documented in this file. See [standa
- feat(health): the gate reads the named report — reads refuse loudly, never rebuild; open serves before it returns; the ceremony door (f8f64780) - feat(health): the gate reads the named report — reads refuse loudly, never rebuild; open serves before it returns; the ceremony door (f8f64780)
### [10.4.0-rc.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.3.1...v10.4.0-rc.1) (2026-08-24) ### [10.4.0-rc.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.3.1...v10.4.0-rc.1) (2026-08-24)
- ci(publish): the home dist-tag follows the version — a prerelease publishes under 'rc' and never moves 'latest' (a1376e4a) - ci(publish): the home dist-tag follows the version — a prerelease publishes under 'rc' and never moves 'latest' (a1376e4a)
- chore(release): --source-only — a home-only prerelease mode (The Source, never the storefront) (dcbad176) - chore(release): --source-only — a home-only prerelease mode (The Source, never the storefront) (dcbad176)
@ -183,13 +51,13 @@ All notable changes to this project will be documented in this file. See [standa
- ci(gate): the machine-health preflight and the truncation verdict guard (1e046aa1) - ci(gate): the machine-health preflight and the truncation verdict guard (1e046aa1)
### [10.3.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.3.0...v10.3.1) (2026-08-18) ### [10.3.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.3.0...v10.3.1) (2026-08-18)
- docs(releases): the 10.3.1 consumer entry — the fold that behaves (900cc895) - docs(releases): the 10.3.1 consumer entry — the fold that behaves (900cc895)
- fix(recovery): the fold streams and narrates; the checkpoint chain arms at the flip (ed7d1db9) - fix(recovery): the fold streams and narrates; the checkpoint chain arms at the flip (ed7d1db9)
### [10.3.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.2.0...v10.3.0) (2026-08-18) ### [10.3.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.2.0...v10.3.0) (2026-08-18)
- docs(releases): the 10.3.0 consumer entry — the trust-and-provenance release (97d75649) - docs(releases): the 10.3.0 consumer entry — the trust-and-provenance release (97d75649)
- fix(locks): the fence keys ownership on pid+hostname — a same-process re-open never fences its predecessor (0991cf28) - fix(locks): the fence keys ownership on pid+hostname — a same-process re-open never fences its predecessor (0991cf28)
@ -198,14 +66,14 @@ All notable changes to this project will be documented in this file. See [standa
- feat(log): system commits carry their origin; the attested per-id reconcile door (9ac9e706) - feat(log): system commits carry their origin; the attested per-id reconcile door (9ac9e706)
### [10.2.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.1.0...v10.2.0) (2026-08-17) ### [10.2.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.1.0...v10.2.0) (2026-08-17)
- docs(releases): the 10.2.0 consumer entry — adoption completes in one call (97538e1f) - docs(releases): the 10.2.0 consumer entry — adoption completes in one call (97538e1f)
- ci: the correctness plant runs integration + conformance on every push — a release never waits on a second machine (b17fdc8e) - ci: the correctness plant runs integration + conformance on every push — a release never waits on a second machine (b17fdc8e)
- fix(adoption): the baseline backfill runs to completion — one call adopts a pre-log baseline of any size (a5a18838) - fix(adoption): the baseline backfill runs to completion — one call adopts a pre-log baseline of any size (a5a18838)
### [10.1.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.0.0...v10.1.0) (2026-08-13) ### [10.1.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.0.0...v10.1.0) (2026-08-13)
- docs(releases): the 10.1.0 consumer entry — bounded recovery, restore founding, the two write-path cures (7d3c8696) - docs(releases): the 10.1.0 consumer entry — bounded recovery, restore founding, the two write-path cures (7d3c8696)
- fix(restore): a restore is an unclean event — the swap runs quiesced and the snapshot's durability stamps never survive it (9ca80667) - fix(restore): a restore is an unclean event — the swap runs quiesced and the snapshot's durability stamps never survive it (9ca80667)
@ -214,7 +82,7 @@ All notable changes to this project will be documented in this file. See [standa
- feat(query): the sparse-store cut — where on a never-carried field serves operator truth, never a refusal (7b67db4d) - feat(query): the sparse-store cut — where on a never-carried field serves operator truth, never a refusal (7b67db4d)
### [10.0.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v9.0.0...v10.0.0) (2026-08-12) ### [10.0.0](https://source.soulcraft.com/soulcraft/brainy/compare/v9.0.0...v10.0.0) (2026-08-12)
- fix(adoption): the baseline backfill cures hydration-law drift — existing brains reach the crash-safe default with zero operator steps (25f0dd96) - fix(adoption): the baseline backfill cures hydration-law drift — existing brains reach the crash-safe default with zero operator steps (25f0dd96)
- fix(adoption): the reserved-root mint exemption — int 0 is legitimate for exactly one id (2abe8b38) - fix(adoption): the reserved-root mint exemption — int 0 is legitimate for exactly one id (2abe8b38)
@ -246,7 +114,7 @@ All notable changes to this project will be documented in this file. See [standa
- test: version-coupling pins go major-agnostic — the 8.x literals broke at the 9.0.0 bump while the coupling law itself behaved correctly (8a6807e8) - test: version-coupling pins go major-agnostic — the 8.x literals broke at the 9.0.0 bump while the coupling law itself behaved correctly (8a6807e8)
### [9.0.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.11.0...v9.0.0) (2026-08-04) ### [9.0.0](https://source.soulcraft.com/soulcraft/brainy/compare/v8.11.0...v9.0.0) (2026-08-04)
- docs: 9.0 namespace-migration guide — the simple story + the mechanical sweep checklist, published for humans and tooling alike (61ab9db2) - docs: 9.0 namespace-migration guide — the simple story + the mechanical sweep checklist, published for humans and tooling alike (61ab9db2)
- fix(release): storefront leg republishes CI's exact forge artifact — byte-identity by construction, verified by cross-registry shasum before the ceremony reports success (d89df2ed) - fix(release): storefront leg republishes CI's exact forge artifact — byte-identity by construction, verified by cross-registry shasum before the ceremony reports success (d89df2ed)
@ -281,7 +149,7 @@ All notable changes to this project will be documented in this file. See [standa
- feat: scanFacts liveness contract — first batch or loud failure within a documented bound (f8e6da2b) - feat: scanFacts liveness contract — first batch or loud failure within a documented bound (f8e6da2b)
### [8.11.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.1...v8.11.0) (2026-07-27) ### [8.11.0](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.1...v8.11.0) (2026-07-27)
- docs: the last two archived-host links point home (91ef1c8b) - docs: the last two archived-host links point home (91ef1c8b)
- feat: includeHidden — export carries every visibility tier for migration-grade canon completeness (63c1eeb9) - feat: includeHidden — export carries every visibility tier for migration-grade canon completeness (63c1eeb9)
@ -290,19 +158,19 @@ All notable changes to this project will be documented in this file. See [standa
- ci: run the pipeline on the forge (999d0ebb) - ci: run the pipeline on the forge (999d0ebb)
### [8.10.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.2...v8.10.3) (2026-08-03) ### [8.10.3](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.2...v8.10.3) (2026-08-03)
- docs: dedupe the 8.10.2 release-notes entry the cherry doubled onto the branch (8c956608) - docs: dedupe the 8.10.2 release-notes entry the cherry doubled onto the branch (8c956608)
- fix: user metadata named 'level' is a real field everywhere — the engine-internal node layer no longer shadows it in sort/filter/aggregation, and the indexing views stop stamping a phantom 0 into its column; index epoch 2 rebuilds existing brains at first open (958a0859) - fix: user metadata named 'level' is a real field everywhere — the engine-internal node layer no longer shadows it in sort/filter/aggregation, and the indexing views stop stamping a phantom 0 into its column; index epoch 2 rebuilds existing brains at first open (958a0859)
### [8.10.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.1...v8.10.2) (2026-07-29) ### [8.10.2](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.1...v8.10.2) (2026-07-29)
- docs: 8.10.2 consumer release notes — update() write granularity, PathResolver idle-log fix, graph-lsm key recognition (a0123b5b) - docs: 8.10.2 consumer release notes — update() write granularity, PathResolver idle-log fix, graph-lsm key recognition (a0123b5b)
- fix: metadata-only update() never rewrites the noun record — the unconditional whole-vector save turned per-entity stat touches into full rewrites+fsync, amplifying read-heavy sweeps into disk saturation on a production deployment (5b65eb82) - fix: metadata-only update() never rewrites the noun record — the unconditional whole-vector save turned per-entity stat touches into full rewrites+fsync, amplifying read-heavy sweeps into disk saturation on a production deployment (5b65eb82)
### [8.10.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.0...v8.10.1) (2026-07-24) ### [8.10.1](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.0...v8.10.1) (2026-07-24)
- refactor: remove the orphaned transaction-result type left behind by the dead-path removal (edf123a5) - refactor: remove the orphaned transaction-result type left behind by the dead-path removal (edf123a5)
- fix: warm() metadata surface routes through the active provider (warm hook added to the metadata contract); add maintenanceDebt() observability surface (5b2cbf74) - fix: warm() metadata surface routes through the active provider (warm hook added to the metadata contract); add maintenanceDebt() observability surface (5b2cbf74)

View file

@ -12,13 +12,13 @@ Handoff file: `/home/dpsifr/.strategy/PLATFORM-HANDOFF.md`
**Brainy's current open actions:** None. MIT open-source — no platform-specific actions. **Brainy's current open actions:** None. MIT open-source — no platform-specific actions.
**Current version:** run `npm view @soulcraftlabs/brainy version --registry https://source.soulcraft.com/api/packages/soulcraftlabs/npm/` (never trust a hardcoded number here — this line went stale for months); consumer-facing changes tracked in `RELEASES.md` **Current version:** run `npm view @soulcraft/brainy version` (never trust a hardcoded number here — this line went stale for months); consumer-facing changes tracked in `RELEASES.md`
--- ---
## Project Overview ## Project Overview
Brainy is a Universal Knowledge Protocol -- a Triple Intelligence database that combines vector similarity search, graph traversal, and metadata filtering into a single TypeScript library. Published as `@soulcraftlabs/brainy` on The Source (source.soulcraft.com registry) under the MIT license. Brainy is a Universal Knowledge Protocol -- a Triple Intelligence database that combines vector similarity search, graph traversal, and metadata filtering into a single TypeScript library. Published as `@soulcraft/brainy` on npm under the MIT license.
## Getting Started ## Getting Started

View file

@ -6,7 +6,7 @@ may find elsewhere in the repo's history.
## Where the project lives ## Where the project lives
The source of truth is a self-hosted forge: **source.soulcraft.com/soulcraftlabs/open-brainy**. The source of truth is a self-hosted forge: **source.soulcraft.com/soulcraft/brainy**.
It's anonymously readable and cloneable — no account needed to browse, clone, It's anonymously readable and cloneable — no account needed to browse, clone,
or build. or build.
@ -31,7 +31,7 @@ fine) to talk through the approach saves everyone rework.
## Development setup ## Development setup
```bash ```bash
git clone https://source.soulcraft.com/soulcraftlabs/open-brainy.git git clone https://source.soulcraft.com/soulcraft/brainy.git
cd brainy cd brainy
npm install npm install
npm run build npm run build
@ -41,20 +41,6 @@ npm test
Tests run on [Vitest](https://vitest.dev/). `npm test` runs the unit suite; Tests run on [Vitest](https://vitest.dev/). `npm test` runs the unit suite;
see `package.json` for `test:integration`, `test:coverage`, and friends. see `package.json` for `test:integration`, `test:coverage`, and friends.
## Test gate
The release gate is a bare `vitest run` (no `--config` flag) — the same
command the delta gate and CI's checks invoke. It carries the full
correctness suite and nothing else: wall-clock/scale benchmarks
(`tests/performance/**`, `tests/critical-performance-benchmark.test.ts`,
`tests/api/performance-benchmarks.test.ts`) and the two tests whose outcome
depends on the host machine or network rather than the code
(`tests/package-size-limit.test.ts` shells out to the `npm` CLI;
`tests/model-loading.test.ts` makes a real network call to download a model)
are excluded from it, because a timing threshold or a flaky network call has
no business failing a correctness check. That whole family runs on demand,
in its own exclusive slot, via `npm run test:perf`.
## Standards ## Standards
- **Strict TypeScript.** No `any` escape hatches to dodge the type checker. - **Strict TypeScript.** No `any` escape hatches to dodge the type checker.
@ -71,17 +57,6 @@ in its own exclusive slot, via `npm run test:perf`.
description states a number, cite the benchmark that produced it (see description states a number, cite the benchmark that produced it (see
[docs/performance-envelopes.md](docs/performance-envelopes.md) for the [docs/performance-envelopes.md](docs/performance-envelopes.md) for the
pattern). Don't state an estimate as if it were measured. pattern). Don't state an estimate as if it were measured.
- **Measurements carry numbers, not provenance.** Public commit messages and
docs give the SHAPE a number was taken at and never where it was taken: no
hostnames, no store or deployment identities, no operational anecdotes about
someone's running system. "A 14,056-noun / 72,679-verb production-shaped
store, measured solo under an exclusive lock" tells a reader everything the
number depends on; the machine it ran on and whose data it was tell them
nothing except where somebody's infrastructure lives.
- **Documents that answer or reference a confidential specification never enter
this repository, even summarized.** The public docs describe THIS engine and
the published contract, and nothing else — a summary of a private document is
still that document's contents.
## License ## License

View file

@ -1,14 +1,9 @@
<p align="center"> <p align="center">
<img src="https://source.soulcraft.com/soulcraftlabs/open-brainy/raw/branch/main/brainy.png" alt="Brainy" width="180"> <img src="https://source.soulcraft.com/soulcraft/brainy/raw/branch/main/brainy.png" alt="Brainy" width="180">
</p> </p>
<h1 align="center">Brainy</h1> <h1 align="center">Brainy</h1>
> **Frozen at 10.4.13 (2026-09-03).** This repository is the reference implementation of the Brainy store format and API,
> published under the MIT license. Version 10.4.13 is its last release; the repository is read-only from here. The engine
> continues as `@soulcraft/brainy`, which bundles this layer as owned code; every published version of this package stays
> available on The Source. Use this repository to read a Brainy store independently or to verify the conformance contract.
<p align="center"> <p align="center">
<b>Three database paradigms. One API. Zero configuration.</b><br> <b>Three database paradigms. One API. Zero configuration.</b><br>
The in-process knowledge database for TypeScript — vector search, graph traversal,<br> The in-process knowledge database for TypeScript — vector search, graph traversal,<br>
@ -16,9 +11,9 @@
</p> </p>
<p align="center"> <p align="center">
<a href="https://source.soulcraft.com/soulcraftlabs/-/packages/npm/brainy"><img src="https://img.shields.io/badge/package-The%20Source-2c3e50.svg" alt="Package on The Source"></a> <a href="https://www.npmjs.com/package/@soulcraft/brainy"><img src="https://img.shields.io/npm/v/@soulcraft/brainy.svg" alt="npm version"></a>
<a href="https://source.soulcraft.com/soulcraftlabs/open-brainy"><img src="https://img.shields.io/badge/repo-open--brainy-2c3e50.svg" alt="Repository"></a> <a href="https://www.npmjs.com/package/@soulcraft/brainy"><img src="https://img.shields.io/npm/dm/@soulcraft/brainy.svg" alt="npm downloads"></a>
<a href="https://source.soulcraft.com/soulcraftlabs/open-brainy/actions"><img src="https://source.soulcraft.com/soulcraftlabs/open-brainy/actions/workflows/ci.yml/badge.svg?branch=main" alt="CI"></a> <a href="https://source.soulcraft.com/soulcraft/brainy/actions"><img src="https://source.soulcraft.com/soulcraft/brainy/actions/workflows/ci.yml/badge.svg?branch=main" alt="CI"></a>
<a href="https://soulcraft.com/docs"><img src="https://img.shields.io/badge/docs-soulcraft.com-blue.svg" alt="Documentation"></a> <a href="https://soulcraft.com/docs"><img src="https://img.shields.io/badge/docs-soulcraft.com-blue.svg" alt="Documentation"></a>
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="MIT License"></a> <a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="MIT License"></a>
<a href="https://www.typescriptlang.org/"><img src="https://img.shields.io/badge/%3C%2F%3E-TypeScript-%230074c1.svg" alt="TypeScript"></a> <a href="https://www.typescriptlang.org/"><img src="https://img.shields.io/badge/%3C%2F%3E-TypeScript-%230074c1.svg" alt="TypeScript"></a>
@ -35,8 +30,6 @@
--- ---
**Open Brainy** is the MIT engine — the open API, client library, types, and protocol; an openly specified canonical on-disk format; and this TypeScript reference engine, scoped as a single-node engine for stores up to roughly one million rows. `@soulcraft/brainy` 10.4.2 was the last release under the old package name — the name passes to the native engine, **Brainy**, at 11.0.0: the same API over the same open format at production scale, and it requires a license.
Built because we were tired of stitching a vector store to a graph database to a document store — and spending weeks on plumbing before writing a line of business logic. Brainy indexes every fact **three ways at once** and lets one call query them together: Built because we were tired of stitching a vector store to a graph database to a document store — and spending weeks on plumbing before writing a line of business logic. Brainy indexes every fact **three ways at once** and lets one call query them together:
| You write | Brainy indexes it as | You query it with | | You write | Brainy indexes it as | You query it with |
@ -52,14 +45,12 @@ It runs **inside your process** — no server, no Docker, nothing to operate —
## Quick start ## Quick start
```bash ```bash
bun add @soulcraftlabs/brainy # Bun ≥ 1.1 — recommended bun add @soulcraft/brainy # Bun ≥ 1.1 — recommended
npm install @soulcraftlabs/brainy # Node.js ≥ 22 npm install @soulcraft/brainy # Node.js ≥ 22
``` ```
> **Registry**: add `@soulcraftlabs:registry=https://source.soulcraft.com/api/packages/soulcraftlabs/npm/` to your `.npmrc` (anonymous read).
```javascript ```javascript
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
const brain = new Brainy() // in-memory; one line swaps to disk const brain = new Brainy() // in-memory; one line swaps to disk
await brain.init() await brain.init()

View file

@ -1,17 +1,7 @@
# @soulcraft/brainy — Release Notes for Consumers # @soulcraft/brainy — Release Notes for Consumers
> **Frozen at 10.4.13 (2026-09-03).** 10.4.13 is the last release of `@soulcraftlabs/brainy`; this repository is read-only from here.
> Release notes for the product engine continue on its own wall.
Machine-readable release notes are published at
https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/open-brainy.json
(this engine) and
https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/brainy.json
(the product engine) — read by HQ's `/hq/releases` door, and the source of
truth ahead of this file.
This file is the **quick reference for downstream sessions** tracking Brainy changes. This file is the **quick reference for downstream sessions** tracking Brainy changes.
Full auto-generated changelog: `CHANGELOG.md` · Releases: https://source.soulcraft.com/soulcraftlabs/open-brainy/releases Full auto-generated changelog: `CHANGELOG.md` · Releases: https://source.soulcraft.com/soulcraft/brainy/releases
**How to use:** Brainy is the underlying data engine for downstream applications. Read this when: **How to use:** Brainy is the underlying data engine for downstream applications. Read this when:
- Upgrading `@soulcraft/brainy` in your application - Upgrading `@soulcraft/brainy` in your application
@ -41,227 +31,6 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the
--- ---
## v10.4.4 — 2026-08-28
**A correctness and observability release.** The headline is not speed: it is that a
restart now tells you the truth about itself, a store stops lying about how much it
holds, and the engine stops doing work nobody asked for. There is a performance
improvement and it is modest; it is stated exactly below rather than rounded up.
### The dark restart — fixed at the root
A service could stop cleanly, exit 0, having awaited `close()` on every store it held,
and its next boot would announce `Overwriting stale writer lock … appears dead` for
every one of them. Nothing had crashed. Two deployments hit this; the same defect also
made those boots pay a crash-recovery fold they did not owe.
The cause was not the lock. `close()` released it correctly — when it got there. A
failure part-way through close skipped both the release AND the clean-shutdown marker,
and "the recorded pid is gone" reads identically for an orderly restart and a crash.
- `close()` is now two parts and the second is unconditional: the flush-request watcher,
the **writer lock**, the VFS timers and the terminal `closed` flag are released whether
the durable steps succeeded or not. The original failure is narrated with what it costs
the next open, then rethrown.
- Releasing the lock writes a **clean-close record** naming the lock generation it gave
up. The next open reads that record instead of guessing: recorded → nothing to recover;
absent → it says so, and names the recovery it is about to run. This also ends two
long-standing false alarms — a recycled pid locking a store out of its own reopen, and
`Re-acquiring writer lock … this is a bug` after a perfectly clean close.
- The signal path stopped failing in a batch. One store's failing flush used to strand
every remaining store's lock and markers — at exit code 0. Now: per-store isolation, the
generation store's close (the marker) is part of shutdown, the lock goes in a `finally`,
and the handler no longer calls `process.exit()` when the host application has its own
signal handler, a race that truncated the host's own shutdown mid-flight.
### The count ledger stops lying, and `counts.json` is written atomically
The all-tier scalars are the denominator a coverage check subtracts against. A ledger
derived under the old rule — one entity per id DIRECTORY — counted ghost and scar
containers as rows, and was only FLAGGED suspect: it went on serving wrong numbers for
the life of the store. Two copies of one archive could disagree, and a downstream index
heal reported remaining work that did not exist.
- Such a ledger now derives itself honestly **in the background** after the open, counting
identity records, and persists the correction stamped. Nothing waits for it, because no
read is served from a denominator.
- A derivation that raced a write refuses to stamp its number: one retry on a quiet store,
then the ledger stays SUSPECT and names `repairIndex()` as the door that recounts under
a barrier.
- `counts.json` is written temp+rename. A truncating write left a window in which a
concurrent reader saw the file EMPTY — and an unparseable ledger sends the next open
down the full-rescan path, so the cheapest file in the store was buying the most
expensive recovery.
### An open and a repair narrate themselves — on a channel a log level cannot silence
A store could open for three minutes and print nothing at all. The phase timings existed;
they were written to a channel that every production-looking environment clamps away.
- Narration moved to an always-visible channel. An open now heartbeats the phase it is in,
names each phase as it ends with what it was paying for, and names the expensive STEP
inside a phase. `repairIndex()` does the same and its receipt carries a per-family
`durationMs` — a repair that ran for half an hour with no output could only be watched
through `top`.
- A brain nobody has written to now does nothing: a flush over a clean store is a no-op
and says nothing, the graph index's auto-flush asks before it acts, and the
cross-process flush-request watch is **event-driven** (`fs.watch`) instead of polling a
directory every 500 ms per store forever, with a slow safety sweep behind it and a
narrated fall back to polling where a filesystem cannot be watched.
- A provider that is REBUILDING ITSELF is no longer confused with a broken one. `init()`
does not wait for it, every other family serves, and that family's doors refuse **by
name, carrying the provider's own progress**, saying plainly that they open by
themselves and no action is needed. Health narration dedupes by content, so an unchanged
verdict is silent however a provider's generation counter moves.
### For operators — one behaviour change
**Four `where` operators that previously returned an empty page now raise
`INVALID_QUERY`:** `startsWith`, `endsWith`, `matches` and `length`. An equality/range
posting index cannot evaluate a substring, a pattern or an array length without reading
every row, and it now refuses by name instead of answering with an empty result that
looks like an answer.
**Three that previously returned an empty page are now SERVED:** `hasAll`, `noneOf` and
`excludes`. All 25 accepted operator tokens now agree between this engine and its
accelerated counterpart.
### Performance — stated exactly
Measured on a 14,056-noun / 72,679-verb production-shaped store, both builds solo under
an exclusive lock:
- **Warm reopen after a clean close: 85.7 s → 77.0 s (10.2%).** The whole of that gain is
one fix — generation discovery reads directory NAMES instead of recursively walking the
entire generation log (9.2 s, and it scales with history rather than row count). The
VFS phase is **unchanged**.
- **Cold open: 31.4 s** (518.1 s → 486.7 s), of which the count-ledger derivation moving
off the critical path accounts for storage-init dropping 5,941 ms → 25 ms.
- **A dominant ~38 s remains, diagnosed and NOT fixed.** It is not the VFS — the VFS's own
init is under 2 s of that phase. It is the log-authority adoption and/or the
pending-embed log recovery, both now instrumented so the next measurement names the
culprit outright.
Continuing work, named so nobody has to rediscover it: that ~38 s term; making the
generation store's committed-range set lazy; the hydration path that substitutes
`Date.now()` for an unreadable stored timestamp (inventing data); and a VFS path-prefix
filter built with a `$startsWith` spelling no operator set accepts, so
`searchFiles({ path })` throws today.
---
## v10.4.3 — 2026-08-27 (Open Brainy's first release)
**`@soulcraftlabs/brainy` 10.4.3 is the same engine as `@soulcraft/brainy` 10.4.2, byte for
byte — only the name, the registry, and the pointers changed.** Install:
```bash
npm install @soulcraftlabs/brainy
```
with the registry line in your `.npmrc` (anonymous read):
```
@soulcraftlabs:registry=https://source.soulcraft.com/api/packages/soulcraftlabs/npm/
```
- **The Source is the one registry.** Open Brainy publishes to source.soulcraft.com only; the
npmjs republish step is retired from the release rail. Existing npmjs versions of
`@soulcraft/brainy` stay as they are and receive no new versions.
- **The repository moved** to `soulcraftlabs/open-brainy` on The Source; the old path redirects.
- **No engine change.** Everything in the 10.4.2 notes applies unchanged; adoption is one
install-line change (`@soulcraft/brainy``@soulcraftlabs/brainy`), which downstream
applications make together with their native-engine bump.
## v10.4.2 — 2026-08-27 (a zero-norm vector is not a vector)
**This is the last release of the MIT engine under the `@soulcraft/brainy` name.**
The MIT package continues as **Open Brainy**`@soulcraftlabs/brainy`: the open API,
client library, types and protocol, an openly specified canonical format, and the TypeScript
reference engine, scoped honestly as a single-node engine for stores up to roughly one
million rows. The `@soulcraft/brainy` name passes to the native engine, **Brainy**, at a
major version bump; that engine implements the same API over the same open format at
production scale, requires a license, and refuses loudly without one. Nothing changes
for existing installs until that major ships; the move is announced with it.
Six fixes, one law: a vector with no magnitude carries no information, so it must
never reach a vector index — in any engine — and the canonical store must say so.
- **The permanently-unvectored row.** `add({ ..., vector: [] })` (and the same item
shape in `addMany` / `transact`) is now the sanctioned "no vector" row: persisted
with an empty vector leg, never embedded, never indexed, counted as unvectored in
the canonical ledger. Metadata-only rows — telemetry tallies, counters, plumbing —
no longer need a placeholder vector and never enter the vector leg. `vector: []`
together with `deferEmbedding: true` is refused with a typed error (a supplied
vector has nothing to defer). Previously `vector: []` threw a dimension error.
- **The unvector door.** `update({ id, vector: [] })` (and its `transact()` twin) is
the sanctioned way to strip a vector from an existing row: canonical vector → `[]`,
removal from the vector index, the vectored ledger decremented exactly once — and
idempotent, so a resumed cleanup pass may simply re-issue. It never re-embeds, and
it clears a pending deferred-embed marker durably so the background worker cannot
re-vector the row later. Note that a rebuild never sheds vectors (it re-derives the
index from canonical rows); shedding historical vectors needs this door.
- **Zero-norm vectors are normalized at the write.** An explicit all-zero vector on
any write path is persisted as unvectored (`[]`) with one warning naming the row;
the vector-index operations keep their own refusal as a second line. The engine's
own VFS root, which used to persist a deliberate all-zero placeholder (harmless
under cosine distance, a false attractor under a downstream engine's
squared-euclidean serving — a production incident this week), is now created
unvectored, and an existing store's legacy root is migrated on open by a single
fixed-path read before the health gate runs — never a walk.
- **Enumeration keys on the identity record.** `getNouns()` / `getVerbs()` and the
cursor walks behind them enumerate by the metadata record, the same key the
canonical ledger counts by — previously the walk keyed on the vector file, so a
row holding metadata but no vector was counted yet never yielded (a permanent
"missing" phantom in coverage math), while an orphaned vector-only directory
could be yielded as a phantom id. The recovery fold also never deletes an existing
vector when it replays a metadata-only after-image (preserve-if-absent). One
documented gap remains: a verb's endpoints live only in its vector leg, so a
metadata-only verb is counted and loudly skipped, never fabricated — the fix is a
canonical-format change and lands with the open format.
- **The ledger's one-time derivation counts identity records.** Stores upgraded from
pre-ledger versions derived their ALL-visibility scalars once by counting id
directories, which included ghost and scar containers left by an old partial-delete
defect — an inflated denominator whose coverage row could never reach exact. The
derivation now counts only directories holding a metadata record, `counts.json`
carries a derivation-rule stamp, and a ledger derived under the old rule is marked
`suspect` at open (one O(1) field read, one warning) so the online `repairIndex()`
path clears it with a real recount.
- **The vector index refuses what it cannot hold.** `rebuild()` skips unvectored and
zero-norm rows (one summary line), re-pins the vector dimension from the first real
vector after a restart (previously a restart left the pin unset, so a wrong-length
insert became the new pin instead of being rejected), and `addItem` / `updateItem`
throw a typed `EmptyVectorIndexError` on a length-0 vector instead of ever storing
a vector-less node.
- **Smaller:** a failing plugin activation now rethrows with the original error as
`cause` (the originating file and line survive to the caller's log); build
generators stamp from the repository history of their inputs instead of wall clock,
so two builds of the same tree are byte-identical.
Adoption: one restart, paired with its native-engine release. The first open of an
existing store runs the legacy-root migration (one narrated line) and, on stores that
upgraded from pre-ledger versions, marks the ledger suspect until the next sanctioned
recount — no rebuild in either case.
## v10.4.1 — 2026-08-26 (reads refuse per family; an unchanged write never re-embeds)
Two production defects from the same week, fixed together as a patch to 10.4.0.
- **The read gate is per family.** A read now refuses only when the index family it
actually consults is unhealthy: a metadata filter is served while the vector leg is
rebuilding; a semantic query is refused only by the vector family; a graph
traversal only by the graph family. Previously any unhealthy family refused every
read on the brain — under a long vector rebuild, a production deployment's
metadata-only reads were refused for the duration, and the retries became a write
pump of their own.
- **Unchanged data never re-embeds.** `update()` compares the incoming `data`
structurally with the stored record; an update carrying identical data (a common
shape for periodic upserts) no longer embeds again and no longer churns the vector
leg. Previously every such update re-embedded and re-inserted, which under load
saturated the vector index with near-identical vectors.
Adoption: one restart, paired with its native-engine release.
## v10.4.0 — 2026-08-25 (the health report has a name) ## v10.4.0 — 2026-08-25 (the health report has a name)
Three related cures, one root cause: an index deciding whether it could be trusted Three related cures, one root cause: an index deciding whether it could be trusted

View file

@ -30,7 +30,7 @@ commit to backporting fixes to unsupported lines.
## Scope ## Scope
This policy covers the `@soulcraftlabs/brainy` package itself — the code in This policy covers the `@soulcraft/brainy` package itself — the code in
this repository. If you're evaluating a deployment that also uses this repository. If you're evaluating a deployment that also uses
`@soulcraft/cor`, report issues in that package the same way, to the same `@soulcraft/cor`, report issues in that package the same way, to the same
address; we'll route internally. address; we'll route internally.

View file

@ -3,7 +3,7 @@
/** /**
* Modern TypeScript CLI Runner * Modern TypeScript CLI Runner
* *
* This is the entry point after npm install @soulcraftlabs/brainy * This is the entry point after npm install @soulcraft/brainy
* It runs the compiled TypeScript CLI code * It runs the compiled TypeScript CLI code
*/ */

View file

@ -3,7 +3,7 @@
"configVersion": 0, "configVersion": 0,
"workspaces": { "workspaces": {
"": { "": {
"name": "@soulcraftlabs/brainy", "name": "@soulcraft/brainy",
"dependencies": { "dependencies": {
"@aws-sdk/client-s3": "^3.540.0", "@aws-sdk/client-s3": "^3.540.0",
"@azure/identity": "^4.0.0", "@azure/identity": "^4.0.0",

View file

@ -25,13 +25,13 @@
### Prerequisites ### Prerequisites
```bash ```bash
npm install @soulcraftlabs/brainy npm install @soulcraft/brainy
``` ```
### Your First Neural Database ### Your First Neural Database
```typescript ```typescript
import { Brainy, NounType } from '@soulcraftlabs/brainy' import { Brainy, NounType } from '@soulcraft/brainy'
// Step 1: Create and initialize Brainy // Step 1: Create and initialize Brainy
const brain = new Brainy({ const brain = new Brainy({
@ -143,7 +143,7 @@ Once you're comfortable with basic operations, move to **Level 2** to learn abou
### Building a Knowledge Graph ### Building a Knowledge Graph
```typescript ```typescript
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
const brain = new Brainy({ storage: { type: 'memory' } }) const brain = new Brainy({ storage: { type: 'memory' } })
await brain.init() await brain.init()
@ -314,7 +314,7 @@ Ready for AI-powered search and clustering? Move to **Level 3**.
### Triple Intelligence in Action ### Triple Intelligence in Action
```typescript ```typescript
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
const brain = new Brainy({ storage: { type: 'memory' } }) const brain = new Brainy({ storage: { type: 'memory' } })
await brain.init() await brain.init()
@ -529,7 +529,7 @@ Want to treat files as intelligent entities? Learn the **Virtual Filesystem** in
### Files as Intelligent Entities ### Files as Intelligent Entities
```typescript ```typescript
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
const brain = new Brainy({ storage: { type: 'memory' } }) const brain = new Brainy({ storage: { type: 'memory' } })
await brain.init() await brain.init()
@ -832,7 +832,7 @@ Ready for production deployment? Level 5 covers **planet-scale architecture**.
### Production-Ready Deployment ### Production-Ready Deployment
```typescript ```typescript
import { Brainy, NounType } from '@soulcraftlabs/brainy' import { Brainy, NounType } from '@soulcraft/brainy'
// 1. PRODUCTION STORAGE - Filesystem with off-site snapshots // 1. PRODUCTION STORAGE - Filesystem with off-site snapshots
console.log('Initializing production storage...\n') console.log('Initializing production storage...\n')

View file

@ -369,71 +369,6 @@ return results.slice(offset, offset + limit)
// → Auto-correction: Use most likely alternative based on affinity data // → Auto-correction: Use most likely alternative based on affinity data
``` ```
## Field Projection (`fields`)
`find()` and `get()` accept a `fields` list. Without it they return the whole
record; with it they return only the fields you name — and, where the index can
supply them, without opening the canonical record at all.
```ts
// A list page: two user fields and one engine scalar. No document bodies.
await brain.find({
where: { kind: 'post' },
fields: ['title', 'slug', 'system.createdAt'],
limit: 50
})
await brain.get(id, { fields: ['title'] })
```
### Why it exists
A list view that renders a title and a date does not need the body, but without
a projection every row hydrates its full record and throws almost all of it
away. On a posts list that is the dominant cost of the query.
### The rules
| | |
|---|---|
| **`fields` absent** | The full record, byte-identical to before. Nothing changes. |
| **Field names** | The one addressing law: a bare name is user metadata (`'title'`), `system.*` is an engine scalar (`'system.createdAt'`). |
| **A field the row lacks** | Simply **absent** from the result. Never an error. |
| **Identity** | Every row keeps its `id` (and `score` on `find`) regardless — a row you cannot identify is not a row. |
| **Where values come from** | The **column store**, which holds raw values. Never the sparse index, which buckets timestamps for range queries. |
| **A field the column cannot serve** | The canonical record is read for that field only. Correct, just not free. |
### Missing fields are absent, not errors
This is deliberate and differs from `orderBy`, which throws
`UnresolvableFieldError` for an unknown field. A typo in `orderBy` silently
changes the ordering, so it must be loud. A projection asks "give me these if
you have them", and an optional field must not turn a list into a failure — so
`fields` uses the permissive path.
### Cost
When every named field is column-served, a projected page performs **zero**
canonical reads. When one is not, only that read happens and the rest still come
from the index. Both are pinned by counting reads rather than timing them, in
`tests/integration/find-fields-projection.test.ts`.
### `related()` takes no `fields`
A `Relation` carries `from` and `to` as **ids** and hydrates no entity record,
so there is nothing for a projection to trim. Projecting the endpoints would be
a new capability rather than a projection of an existing one.
### For engine implementers
Projection is served through an optional provider door,
`getScalarsForIds(ids, fields)` on `MetadataIndexProvider`. The contract is in
`src/plugin.ts`; the short version is **return only what you can serve exactly,
and say what you served**. The caller diffs the answer against the request and
reads records for the remainder, so omission costs a read while a wrong value is
a wrong answer nobody can see. An engine without the door still works — every
field falls back to the record.
## Performance Characteristics ## Performance Characteristics
### Query Performance by Type ### Query Performance by Type
@ -1282,7 +1217,7 @@ where: {
await brain.find({ type: 'Document' }) await brain.find({ type: 'Document' })
// ✅ Correct: Use NounType enum // ✅ Correct: Use NounType enum
import { NounType } from '@soulcraftlabs/brainy' import { NounType } from '@soulcraft/brainy'
await brain.find({ type: NounType.Document }) await brain.find({ type: NounType.Document })
// ❌ Error: Operator not recognized // ❌ Error: Operator not recognized

View file

@ -153,13 +153,13 @@ brainy-data/
### Step 1: Update Brainy Package ### Step 1: Update Brainy Package
```bash ```bash
npm install @soulcraftlabs/brainy@latest npm install @soulcraft/brainy@latest
``` ```
**Check your version:** **Check your version:**
```bash ```bash
npm list @soulcraftlabs/brainy npm list @soulcraft/brainy
# Should show: @soulcraftlabs/brainy@4.0.0 # Should show: @soulcraft/brainy@4.0.0
``` ```
### Step 2: No Code Changes Required! ✅ ### Step 2: No Code Changes Required! ✅
@ -374,7 +374,7 @@ If you encounter issues, you can rollback:
```bash ```bash
# Reinstall v3 # Reinstall v3
npm install @soulcraftlabs/brainy@^3.50.0 npm install @soulcraft/brainy@^3.50.0
# Restart application # Restart application
``` ```
@ -389,7 +389,7 @@ rm -rf ./data
cp -r ./data-backup ./data cp -r ./data-backup ./data
# Reinstall v3 # Reinstall v3
npm install @soulcraftlabs/brainy@^3.50.0 npm install @soulcraft/brainy@^3.50.0
``` ```
## Common Migration Scenarios ## Common Migration Scenarios
@ -539,7 +539,7 @@ console.log('Storage type:', status.type)
**Migration Checklist:** **Migration Checklist:**
- ✅ Backup data - ✅ Backup data
- ✅ Update npm package (`npm install @soulcraftlabs/brainy@latest`) - ✅ Update npm package (`npm install @soulcraft/brainy@latest`)
- ✅ Restart application (automatic migration) - ✅ Restart application (automatic migration)
- ✅ Verify data integrity - ✅ Verify data integrity
- ✅ Enable lifecycle policies - ✅ Enable lifecycle policies

View file

@ -46,7 +46,7 @@ If no plugin provides a given key, brainy uses its built-in JavaScript implement
### 1. Implement the `BrainyPlugin` interface ### 1. Implement the `BrainyPlugin` interface
```typescript ```typescript
import type { BrainyPlugin, BrainyPluginContext } from '@soulcraftlabs/brainy/plugin' import type { BrainyPlugin, BrainyPluginContext } from '@soulcraft/brainy/plugin'
const myPlugin: BrainyPlugin = { const myPlugin: BrainyPlugin = {
name: 'my-brainy-plugin', // Must be unique (typically your npm package name) name: 'my-brainy-plugin', // Must be unique (typically your npm package name)
@ -90,7 +90,7 @@ await brain.init()
**Programmatic registration:** For plugins not installed as npm packages, use `brain.use()`: **Programmatic registration:** For plugins not installed as npm packages, use `brain.use()`:
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
import myPlugin from './my-plugin.js' import myPlugin from './my-plugin.js'
const brain = new Brainy() const brain = new Brainy()
@ -272,10 +272,10 @@ When provided by an optional native acceleration plugin (such as `@soulcraft/cor
#### `cache` #### `cache`
**Type:** `UnifiedCache` **Type:** `UnifiedCache`
Replaces the global `UnifiedCache` singleton used for VFS path resolution, semantic caching, and vector index caching. Must implement the `UnifiedCache` interface (available from `@soulcraftlabs/brainy/internals`). Replaces the global `UnifiedCache` singleton used for VFS path resolution, semantic caching, and vector index caching. Must implement the `UnifiedCache` interface (available from `@soulcraft/brainy/internals`).
```typescript ```typescript
import type { UnifiedCache } from '@soulcraftlabs/brainy/internals' import type { UnifiedCache } from '@soulcraft/brainy/internals'
context.registerProvider('cache', myNativeCache) context.registerProvider('cache', myNativeCache)
``` ```
@ -325,8 +325,8 @@ Plugins can register custom storage backends that users reference by name.
### Implementing a Storage Adapter ### Implementing a Storage Adapter
```typescript ```typescript
import type { StorageAdapterFactory } from '@soulcraftlabs/brainy/plugin' import type { StorageAdapterFactory } from '@soulcraft/brainy/plugin'
import type { StorageAdapter } from '@soulcraftlabs/brainy' import type { StorageAdapter } from '@soulcraft/brainy'
class MyStorageAdapter implements StorageAdapter { class MyStorageAdapter implements StorageAdapter {
async init(): Promise<void> { /* ... */ } async init(): Promise<void> { /* ... */ }
@ -360,9 +360,9 @@ Brainy provides three entry points for plugin developers:
| Import Path | Contents | Stability | | Import Path | Contents | Stability |
|-------------|----------|-----------| |-------------|----------|-----------|
| `@soulcraftlabs/brainy` | Public API, types, StorageAdapter | Stable (semver) | | `@soulcraft/brainy` | Public API, types, StorageAdapter | Stable (semver) |
| `@soulcraftlabs/brainy/plugin` | BrainyPlugin, BrainyPluginContext, StorageAdapterFactory | Stable (semver) | | `@soulcraft/brainy/plugin` | BrainyPlugin, BrainyPluginContext, StorageAdapterFactory | Stable (semver) |
| `@soulcraftlabs/brainy/internals` | UnifiedCache, EntityIdMapper, logger utilities | Internal (may change between minor versions) | | `@soulcraft/brainy/internals` | UnifiedCache, EntityIdMapper, logger utilities | Internal (may change between minor versions) |
## Diagnostics ## Diagnostics
@ -440,7 +440,7 @@ A minimal but useful plugin that provides SIMD-accelerated distance calculations
```typescript ```typescript
// simd-distance-plugin/src/plugin.ts // simd-distance-plugin/src/plugin.ts
import type { BrainyPlugin, BrainyPluginContext } from '@soulcraftlabs/brainy/plugin' import type { BrainyPlugin, BrainyPluginContext } from '@soulcraft/brainy/plugin'
// Hypothetical native module // Hypothetical native module
import { simdCosineDistance } from './native.js' import { simdCosineDistance } from './native.js'
@ -470,7 +470,7 @@ export default simdDistancePlugin
"main": "./dist/plugin.js", "main": "./dist/plugin.js",
"types": "./dist/plugin.d.ts", "types": "./dist/plugin.d.ts",
"peerDependencies": { "peerDependencies": {
"@soulcraftlabs/brainy": ">=7.0.0" "@soulcraft/brainy": ">=7.0.0"
} }
} }
``` ```
@ -478,7 +478,7 @@ export default simdDistancePlugin
Usage: Usage:
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brain = new Brainy({ plugins: ['brainy-simd-distance'] }) const brain = new Brainy({ plugins: ['brainy-simd-distance'] })
await brain.init() await brain.init()

View file

@ -54,7 +54,7 @@ After 40 API calls:
```typescript ```typescript
// server.ts // server.ts
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
// SINGLETON INSTANCE // SINGLETON INSTANCE
let brainInstance: Brainy | null = null let brainInstance: Brainy | null = null
@ -174,7 +174,7 @@ process.on('SIGTERM', async () => {
```typescript ```typescript
// server.ts - Clean Bun implementation // server.ts - Clean Bun implementation
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
let brain: Brainy | null = null let brain: Brainy | null = null

View file

@ -5,7 +5,7 @@
## Quick Start ## Quick Start
```typescript ```typescript
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()

View file

@ -99,7 +99,7 @@ Examples:
```bash ```bash
# 1. Deprecate wrong version on npm # 1. Deprecate wrong version on npm
npm deprecate @soulcraftlabs/brainy@X.X.X "Incorrect version - use Y.Y.Y" npm deprecate @soulcraft/brainy@X.X.X "Incorrect version - use Y.Y.Y"
# 2. Fix version in package.json # 2. Fix version in package.json
# 3. Republish correct version # 3. Republish correct version

View file

@ -13,7 +13,7 @@
### In-Memory ### In-Memory
```typescript ```typescript
import Brainy from '@soulcraftlabs/brainy' import Brainy from '@soulcraft/brainy'
const brain = new Brainy({ storage: { type: 'memory' } }) const brain = new Brainy({ storage: { type: 'memory' } })
``` ```
@ -43,7 +43,7 @@ The native vector provider (via the optional `@soulcraft/cor` package) extends t
Numbers below are **measured** by `tests/benchmarks/find-composition-scale.js` (a single Numbers below are **measured** by `tests/benchmarks/find-composition-scale.js` (a single
Node 22 process, in-memory storage, 384-dim vectors, `balanced` recall). They are the Node 22 process, in-memory storage, 384-dim vectors, `balanced` recall). They are the
open-core (pure-TypeScript) path — what you get from `@soulcraftlabs/brainy` with no native open-core (pure-TypeScript) path — what you get from `@soulcraft/brainy` with no native
provider installed. Run it yourself: `node --max-old-space-size=8192 tests/benchmarks/find-composition-scale.js 100000`. provider installed. Run it yourself: `node --max-old-space-size=8192 tests/benchmarks/find-composition-scale.js 100000`.
`find()` query latency, p50 / p95 (200 queries each): `find()` query latency, p50 / p95 (200 queries each):

File diff suppressed because it is too large Load diff

View file

@ -24,7 +24,7 @@ next:
## Quick Start ## Quick Start
```typescript ```typescript
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
const brain = new Brainy() // Zero config! const brain = new Brainy() // Zero config!
await brain.init() // VFS auto-initialized! await brain.init() // VFS auto-initialized!
@ -1010,7 +1010,7 @@ await db.release() // unpin + free cached materialization
### Db API errors ### Db API errors
All exported from `@soulcraftlabs/brainy`: All exported from `@soulcraft/brainy`:
| Error | Thrown by | Meaning | | Error | Thrown by | Meaning |
|---|---|---| |---|---|---|
@ -1918,11 +1918,11 @@ isn't serving throws instead of rebuilding mid-query:
| `MetadataIndexNotReadyError` | `find({ where })` | Metadata/field index isn't serving | | `MetadataIndexNotReadyError` | `find({ where })` | Metadata/field index isn't serving |
| `VectorIndexNotReadyError` | `find({ query })`, `similar()` | Vector index isn't serving | | `VectorIndexNotReadyError` | `find({ query })`, `similar()` | Vector index isn't serving |
All three are exported from `@soulcraftlabs/brainy`. Catch them to distinguish All three are exported from `@soulcraft/brainy`. Catch them to distinguish
"index not ready" from a genuine empty result: "index not ready" from a genuine empty result:
```typescript ```typescript
import { MetadataIndexNotReadyError } from '@soulcraftlabs/brainy' import { MetadataIndexNotReadyError } from '@soulcraft/brainy'
try { try {
const rows = await brain.find({ where: { status: 'active' } }) const rows = await brain.find({ where: { status: 'active' } })
@ -2208,7 +2208,7 @@ For the full taxonomy with all 169 types and their descriptions, see:
- **📖 Documentation:** [Full Documentation](../) - **📖 Documentation:** [Full Documentation](../)
- **🐛 Issues:** [GitHub Issues](https://github.com/soulcraftlabs/brainy/issues) - **🐛 Issues:** [GitHub Issues](https://github.com/soulcraftlabs/brainy/issues)
- **💬 Discussions:** [GitHub Discussions](https://github.com/soulcraftlabs/brainy/discussions) - **💬 Discussions:** [GitHub Discussions](https://github.com/soulcraftlabs/brainy/discussions)
- **📦 NPM:** [@soulcraftlabs/brainy](https://www.npmjs.com/package/@soulcraftlabs/brainy) - **📦 NPM:** [@soulcraft/brainy](https://www.npmjs.com/package/@soulcraft/brainy)
- **⭐ GitHub:** [Star us](https://github.com/soulcraftlabs/brainy) - **⭐ GitHub:** [Star us](https://github.com/soulcraftlabs/brainy)
--- ---

View file

@ -217,40 +217,6 @@ membership queries at scale:
`__words__` for tokenized text…). `__words__` for tokenized text…).
- `_blobs/_column_index/{field}/L0-NNNNNN.bin` — the actual level-0 run - `_blobs/_column_index/{field}/L0-NNNNNN.bin` — the actual level-0 run
segments, stored through the shared `_blobs/<key>.bin` binary convention. segments, stored through the shared `_blobs/<key>.bin` binary convention.
- `_column_index/{field}/k/{kind}/…` — the same two files again, for a
**second value kind** on the same field (see below). Absent for a field that
holds one kind, which is nearly all of them.
### One posting column per (field, kind)
A field is not obliged to hold one type of value. `category` may carry
`'electronics'` on some rows and `5` on others, and both are real values of
that field. A segment, though, has one encoding — i64, f64, UTF-8, or boolean
— so a field that holds several kinds gets **one column per kind**:
- The first kind a field ever sees owns the plain `_column_index/{field}/`
layout above. A single-kind field is therefore byte-identical to what earlier
versions wrote, and an index written before typed postings opens unchanged.
- Every later kind gets its own column beside it at
`_column_index/{field}/k/{kind}/`, where `{kind}` is `number`, `string` or
`boolean`.
What that buys at query time:
| | |
|---|---|
| **Equality** | Answered from the column matching the **query value's own kind**. `where {category: 5}` reads the number postings; `where {category: '5'}` reads the string postings. Neither borrows the other's rows — a row written with the number `5` is not a row whose category is the text `'5'`. |
| **A kind the field never held** | Matches nothing. That is the true answer, not a coerced one. |
| **Ranges** | Routed by the kind of the bounds: numeric bounds read the numeric postings and ignore the field's strings. An **unbounded** range is the "has any value here" probe behind `exists`, and reads every kind. |
| **`orderBy`** | A number and a string have no order between them, so a mixed field orders by kind first (number, string, boolean) and by value within a kind. A single-kind field sorts exactly as it always did. |
| **Numbers** | One kind, one column: an integer column is written as i64 and widens to f64 the first time a non-integer arrives, so `4.5` is stored as itself rather than rounded. |
`null` and `undefined` are not kinds and are never posted; their absence is
what the `exists` / `missing` operators read.
Older readers are unaffected by the additional columns: they see the field's
primary column exactly where it has always been, and a `k/{kind}` directory is
simply a name they never query.
Sparse per-field indexes, roaring-bitmap chunks, and zone-map/bloom segments Sparse per-field indexes, roaring-bitmap chunks, and zone-map/bloom segments
additionally live as bucketed keys under `_system/idx/` (see §3). Which path additionally live as bucketed keys under `_system/idx/` (see §3). Which path
@ -302,7 +268,7 @@ locks/_flush_responses/ # writer answers with <uuid>.ack
| **Counts/statistics** | Per-type and per-subtype maps | `_system/{type,subtype,verb-subtype}-statistics.json.gz`, `counts.json` | Recomputable by scanning entities (`brainy inspect repair`) | | **Counts/statistics** | Per-type and per-subtype maps | `_system/{type,subtype,verb-subtype}-statistics.json.gz`, `counts.json` | Recomputable by scanning entities (`brainy inspect repair`) |
A pluggable index provider (the 8.0 plugin contract in A pluggable index provider (the 8.0 plugin contract in
`@soulcraftlabs/brainy/plugin`) may replace any of the JS implementations; the `@soulcraft/brainy/plugin`) may replace any of the JS implementations; the
persisted formats above are contract-bound so JS and native implementations persisted formats above are contract-bound so JS and native implementations
can interleave on the same directory. can interleave on the same directory.

View file

@ -126,7 +126,7 @@ class TypeAwareMetadataIndex {
**The Design**: Specify types clearly in your API calls: **The Design**: Specify types clearly in your API calls:
```typescript ```typescript
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
// Add entity with explicit type // Add entity with explicit type
await brain.add({ await brain.add({
@ -231,7 +231,7 @@ class OrgEnrichmentAugmentation {
**Brainy's Approach**: Extract **typed** concepts: **Brainy's Approach**: Extract **typed** concepts:
```typescript ```typescript
import { NaturalLanguageProcessor } from '@soulcraftlabs/brainy' import { NaturalLanguageProcessor } from '@soulcraft/brainy'
const nlp = new NaturalLanguageProcessor() const nlp = new NaturalLanguageProcessor()
const concepts = await nlp.extractConcepts("Alice works at Google in San Francisco") const concepts = await nlp.extractConcepts("Alice works at Google in San Francisco")
@ -382,7 +382,7 @@ import {
getVerbTypes, getVerbTypes,
BrainyTypes, BrainyTypes,
suggestType suggestType
} from '@soulcraftlabs/brainy' } from '@soulcraft/brainy'
// Get all available noun types // Get all available noun types
const nounTypes = getNounTypes() const nounTypes = getNounTypes()

View file

@ -127,7 +127,7 @@ For reference, a clean migration path:
`isMultiProcessSafe` type-guard. Keep `hasStorageMethod` for `isMultiProcessSafe` type-guard. Keep `hasStorageMethod` for
build/install artifact protection. build/install artifact protection.
5. Document the new contract in `concepts/storage-adapters.md`. 5. Document the new contract in `concepts/storage-adapters.md`.
6. Major-version-bump the `@soulcraftlabs/brainy` peerDep range expected by 6. Major-version-bump the `@soulcraft/brainy` peerDep range expected by
plugins. plugins.
Estimated work: ~half a day of code, ~2 hours of doc/example updates, Estimated work: ~half a day of code, ~2 hours of doc/example updates,

View file

@ -20,7 +20,7 @@ next:
Every example on this page is written against the real Brainy 8.0 API. The setup is always the same: Every example on this page is written against the real Brainy 8.0 API. The setup is always the same:
```typescript ```typescript
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()
@ -40,7 +40,7 @@ Brainy's **Noun-Verb Taxonomy** achieves broad coverage of human knowledge throu
- **Multi-hop Graph Traversals = Relationship Complexity** - **Multi-hop Graph Traversals = Relationship Complexity**
- **Result: Model data across virtually any industry** - **Result: Model data across virtually any industry**
Every piece of information can be represented as entities (nouns) connected by relationships (verbs) carrying properties (metadata). The standardized type system from `@soulcraftlabs/brainy` (`NounType`, `VerbType`) gives those nouns and verbs a stable, shared name. Every piece of information can be represented as entities (nouns) connected by relationships (verbs) carrying properties (metadata). The standardized type system from `@soulcraft/brainy` (`NounType`, `VerbType`) gives those nouns and verbs a stable, shared name.
## The Power of Standardization: Universal Interoperability ## The Power of Standardization: Universal Interoperability

View file

@ -35,7 +35,7 @@ constructor and `init()`.
## Instant Start ## Instant Start
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
// That's it. No config needed. // That's it. No config needed.
const brain = new Brainy() const brain = new Brainy()

View file

@ -167,7 +167,7 @@ await brain.find({ orderBy: 'createdAt' })
`UnresolvableFieldError` is exported from the package root: `UnresolvableFieldError` is exported from the package root:
```typescript ```typescript
import { UnresolvableFieldError } from '@soulcraftlabs/brainy' import { UnresolvableFieldError } from '@soulcraft/brainy'
try { try {
await brain.find({ orderBy: 'createdAt' }) await brain.find({ orderBy: 'createdAt' })

View file

@ -68,19 +68,6 @@ a maintenance window, a divergence `repairIndex()` will clean up on its own
schedule. `serving: false` is not benign. It means this provider is refusing to schedule. `serving: false` is not benign. It means this provider is refusing to
answer, on its own word, right now. answer, on its own word, right now.
**How a failure gets its grade — the serving law.** A provider grades `heal` by
one question only: *could an answer be wrong?* — never *how expensive is the
fix?* A missing-postings shortfall, however large, is `heal: 'repair'` (re-post
exactly what the ledger names, reads serving throughout); it can never withhold
serving just because healing it takes work. `serving` is withheld only by a
small, named set of rebuild-graded conditions — the index not initialized, its
durable state absent, a manifest naming files that are not resident, a replay
that did not complete cleanly — the states in which an answer could genuinely be
wrong. And a read is only ever refused by the family it actually consults: a
metadata filter is answered by the metadata index alone, vector search by the
vector index, traversal by the graph index — one family's refusal never blocks
another family's reads.
## Reads refuse — they never rebuild ## Reads refuse — they never rebuild
A query that reaches a not-serving provider does not trigger a rebuild from inside A query that reaches a not-serving provider does not trigger a rebuild from inside
@ -95,7 +82,7 @@ catchable error naming the reason:
| `MetadataIndexNotReadyError` | `find({ where })` | The metadata/field index isn't serving — a filtered read would otherwise return `[]` indistinguishable from "no matches" | | `MetadataIndexNotReadyError` | `find({ where })` | The metadata/field index isn't serving — a filtered read would otherwise return `[]` indistinguishable from "no matches" |
| `VectorIndexNotReadyError` | `find({ query })`, `similar()` | The vector index isn't serving — a semantic search would otherwise return `[]` indistinguishable from "nothing similar" | | `VectorIndexNotReadyError` | `find({ query })`, `similar()` | The vector index isn't serving — a semantic search would otherwise return `[]` indistinguishable from "nothing similar" |
All three are exported from `@soulcraftlabs/brainy`. Catch them where your application All three are exported from `@soulcraft/brainy`. Catch them where your application
needs to distinguish "this index isn't ready yet" from "there's genuinely nothing needs to distinguish "this index isn't ready yet" from "there's genuinely nothing
here" — a health dashboard, a retry policy, an operator alert. The fix is always here" — a health dashboard, a retry policy, an operator alert. The fix is always
the same: reconcile the index, either by reopening the brain (which brings every the same: reconcile the index, either by reopening the brain (which brings every

View file

@ -95,15 +95,8 @@ The heartbeat interval rewrites the lock file every 10 seconds. The timer
is unref'd, so it does not keep the event loop alive on its own. is unref'd, so it does not keep the event loop alive on its own.
On normal shutdown the writer releases the lock in `close()`. The shutdown On normal shutdown the writer releases the lock in `close()`. The shutdown
hooks Brainy registers for `SIGTERM` and `SIGINT` close every live brain by hooks Brainy registers for `SIGTERM`, `SIGINT`, and `beforeExit` also
that same `close()`, so a container restart doesn't strand the directory. release the lock so a container restart doesn't strand the directory.
`beforeExit` is not one of them. Node emits it whenever the event loop has
no ref'd work left — a state a healthy script reaches routinely, because
Brainy's own idle and cadence timers are unref'd — and a drained event loop
is not a shutdown. That hook only persists derived state with a non-closing
`flush()`: it closes nothing, releases no lock, and leaves every brain open
and usable. If you want a shutdown, call `close()` or send `SIGTERM`.
## How to inspect a live writer ## How to inspect a live writer

View file

@ -61,7 +61,7 @@ The only required override is the capability flag. Returning `true` from
to call `acquireWriterLock()` at init. to call `acquireWriterLock()` at init.
```typescript ```typescript
import { FileSystemStorage } from '@soulcraftlabs/brainy' import { FileSystemStorage } from '@soulcraft/brainy'
export class MmapFileSystemStorage extends FileSystemStorage { export class MmapFileSystemStorage extends FileSystemStorage {
public supportsMultiProcessLocking(): boolean { public supportsMultiProcessLocking(): boolean {
@ -79,7 +79,7 @@ If your storage is **not filesystem-backed** (a custom
network backend), extend `BaseStorage` directly: network backend), extend `BaseStorage` directly:
```typescript ```typescript
import { BaseStorage } from '@soulcraftlabs/brainy' import { BaseStorage } from '@soulcraft/brainy'
export class MyCloudStorage extends BaseStorage { export class MyCloudStorage extends BaseStorage {
// BaseStorage's default no-op implementations of the multi-process // BaseStorage's default no-op implementations of the multi-process
@ -101,7 +101,7 @@ The defensive check at every new-storage-method call site (`brainy.ts`,
`hasStorageMethod(name)`) does **not** exist to handle "plugin bundles a `hasStorageMethod(name)`) does **not** exist to handle "plugin bundles a
stale BaseStorage." Plugins ship a dist that preserves the dynamic ESM stale BaseStorage." Plugins ship a dist that preserves the dynamic ESM
import (verify in your plugin's `dist/`: `import { FileSystemStorage } from import (verify in your plugin's `dist/`: `import { FileSystemStorage } from
'@soulcraftlabs/brainy'` is not rewritten to a vendored copy). The prototype '@soulcraft/brainy'` is not rewritten to a vendored copy). The prototype
chain at runtime resolves to whatever Brainy version your consumer has chain at runtime resolves to whatever Brainy version your consumer has
installed. installed.
@ -109,8 +109,8 @@ installed.
the prototype chain at the consumer-app level: the prototype chain at the consumer-app level:
- **Stale `node_modules`** — a lingering install from before the consumer - **Stale `node_modules`** — a lingering install from before the consumer
upgraded Brainy. The package.json says `@soulcraftlabs/brainy@7.22.0` but upgraded Brainy. The package.json says `@soulcraft/brainy@7.22.0` but
`node_modules/@soulcraftlabs/brainy` is still 7.20.x. `node_modules/@soulcraft/brainy` is still 7.20.x.
- **Lockfile drift**`bun.lockb` / `package-lock.json` pins a brainy - **Lockfile drift**`bun.lockb` / `package-lock.json` pins a brainy
version older than the package.json range, and `bun install` honors the version older than the package.json range, and `bun install` honors the
lockfile. lockfile.
@ -131,7 +131,7 @@ and the warning names the adapter class plus a remediation hint:
methods on its prototype chain. Writer locking and the flush-request RPC are methods on its prototype chain. Writer locking and the flush-request RPC are
disabled for this directory. Likely fix: clean install (`rm -rf node_modules disabled for this directory. Likely fix: clean install (`rm -rf node_modules
bun.lockb && bun install`) or rebuild your container image to refresh bun.lockb && bun install`) or rebuild your container image to refresh
`@soulcraftlabs/brainy` to ≥7.21. See docs/concepts/storage-adapters.md. `@soulcraft/brainy` to ≥7.21. See docs/concepts/storage-adapters.md.
``` ```
## Authoring a new storage adapter — minimum checklist ## Authoring a new storage adapter — minimum checklist
@ -168,7 +168,7 @@ bun.lockb && bun install`) or rebuild your container image to refresh
install time — fix install, not your plugin. install time — fix install, not your plugin.
6. **Pin your peer dep generously.** `"peerDependencies": { 6. **Pin your peer dep generously.** `"peerDependencies": {
"@soulcraftlabs/brainy": "^7.21.0" }` accepts any compatible 7.x. Don't pin "@soulcraft/brainy": "^7.21.0" }` accepts any compatible 7.x. Don't pin
to an exact patch unless you're tracking a known regression. to an exact patch unless you're tracking a known regression.
## Future direction ## Future direction
@ -185,5 +185,5 @@ follow-up; consumers don't need to anticipate the change.
heartbeat semantics, what the lock protects. heartbeat semantics, what the lock protects.
- [`guides/inspection`](../guides/inspection.md) — `brainy inspect` and the - [`guides/inspection`](../guides/inspection.md) — `brainy inspect` and the
read-only mode. read-only mode.
- `node_modules/@soulcraftlabs/brainy/dist/storage/baseStorage.d.ts` — the - `node_modules/@soulcraft/brainy/dist/storage/baseStorage.d.ts` — the
authoritative type signatures for every method this page references. authoritative type signatures for every method this page references.

View file

@ -22,7 +22,7 @@ they share a single scan.
## Quick Start ## Quick Start
```typescript ```typescript
import { Brainy, NounType } from '@soulcraftlabs/brainy' import { Brainy, NounType } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()

View file

@ -8,7 +8,7 @@ Brainy is **framework-friendly** - designed to drop into the server side of any
Brainy embeds an HNSW vector index, a graph engine, and a filesystem-backed persistence layer. These belong on the server: Brainy embeds an HNSW vector index, a graph engine, and a filesystem-backed persistence layer. These belong on the server:
- **Zero configuration**: Just `import { Brainy } from '@soulcraftlabs/brainy'` - **Zero configuration**: Just `import { Brainy } from '@soulcraft/brainy'`
- **Auto storage detection**: `new Brainy()` auto-selects filesystem persistence on Node - **Auto storage detection**: `new Brainy()` auto-selects filesystem persistence on Node
- **Cleaner code**: No browser polyfills, no conditional client/server imports - **Cleaner code**: No browser polyfills, no conditional client/server imports
- **Better DX**: One instance shared across your server routes - **Better DX**: One instance shared across your server routes
@ -18,13 +18,13 @@ Brainy embeds an HNSW vector index, a graph engine, and a filesystem-backed pers
### Install Brainy ### Install Brainy
```bash ```bash
npm install @soulcraftlabs/brainy npm install @soulcraft/brainy
``` ```
### Basic Integration ### Basic Integration
```javascript ```javascript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
// Run on the server (API route, server component, backend service) // Run on the server (API route, server component, backend service)
// new Brainy() auto-detects filesystem persistence on Node // new Brainy() auto-detects filesystem persistence on Node
@ -105,7 +105,7 @@ On the server, create one Brainy instance and reuse it across requests. This mod
```javascript ```javascript
// lib/brain.server.js // lib/brain.server.js
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
let brainPromise let brainPromise
@ -163,7 +163,7 @@ On the server, create one Brainy instance and reuse it across requests:
```javascript ```javascript
// server/brain.js (server-only module) // server/brain.js (server-only module)
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
let brainPromise let brainPromise
@ -248,7 +248,7 @@ The matching backend endpoint uses Brainy directly (Node/Bun):
```typescript ```typescript
// server: api/search // server: api/search
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brain = new Brainy() // auto-detects filesystem persistence on Node const brain = new Brainy() // auto-detects filesystem persistence on Node
await brain.init() await brain.init()
@ -266,7 +266,7 @@ In Next.js, Brainy lives in server code only: API routes, server components, or
```javascript ```javascript
// lib/brain.server.js (imported only by server code) // lib/brain.server.js (imported only by server code)
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
let brainPromise let brainPromise
@ -318,7 +318,7 @@ Brainy runs in a server-only module (`*.server.js`); the component fetches resul
```javascript ```javascript
// src/lib/server/brain.js (server-only — note the .server suffix) // src/lib/server/brain.js (server-only — note the .server suffix)
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
let brainPromise let brainPromise
@ -432,7 +432,7 @@ import { defineConfig } from 'vite'
export default defineConfig({ export default defineConfig({
ssr: { ssr: {
external: ['@soulcraftlabs/brainy'] external: ['@soulcraft/brainy']
} }
}) })
``` ```
@ -440,7 +440,7 @@ export default defineConfig({
```javascript ```javascript
// rollup.config.js (server bundle) // rollup.config.js (server bundle)
export default { export default {
external: ['@soulcraftlabs/brainy', 'node:fs', 'node:path', 'node:crypto'] external: ['@soulcraft/brainy', 'node:fs', 'node:path', 'node:crypto']
} }
``` ```
@ -466,7 +466,7 @@ export async function load({ url }) {
```javascript ```javascript
// For build-time usage (runs in Node during the build) // For build-time usage (runs in Node during the build)
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
export async function generateStaticProps() { export async function generateStaticProps() {
const brain = new Brainy({ const brain = new Brainy({
@ -513,7 +513,7 @@ export async function generateStaticProps() {
### Issue: Large client bundle size ### Issue: Large client bundle size
**Cause**: A client module is pulling in Brainy. **Cause**: A client module is pulling in Brainy.
**Solution**: Move the `import { Brainy } from '@soulcraftlabs/brainy'` into a server-only module so it never reaches the browser bundle. **Solution**: Move the `import { Brainy } from '@soulcraft/brainy'` into a server-only module so it never reaches the browser bundle.
### Issue: SSR hydration mismatch ### Issue: SSR hydration mismatch
**Solution**: Run the search on the server (loader / server action / API route) and pass the results down as props, so server and client render the same markup. **Solution**: Run the search on the server (loader / server action / API route) and pass the results down as props, so server and client render the same markup.

View file

@ -9,7 +9,7 @@ Brainy's import is **ONE magical method** that understands EVERYTHING:
## The Ultimate Simplicity ## The Ultimate Simplicity
```javascript ```javascript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()

View file

@ -13,7 +13,7 @@ Brainy provides real-time progress tracking for **all 7 supported file formats**
### Basic Progress Tracking ### Basic Progress Tracking
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
import * as fs from 'fs' import * as fs from 'fs'
const brain = await Brainy.create() const brain = await Brainy.create()

View file

@ -7,7 +7,7 @@
## Basic Import ## Basic Import
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()
@ -187,7 +187,7 @@ await brain.import(file, {
## Complete Example ## Complete Example
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
import * as fs from 'fs' import * as fs from 'fs'
async function importCatalog() { async function importCatalog() {

View file

@ -108,7 +108,7 @@ check fails — useful for piping into monitoring or CI.
## Programmatic inspection ## Programmatic inspection
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const reader = await Brainy.openReadOnly({ const reader = await Brainy.openReadOnly({
storage: { type: 'filesystem', path: '/data/brain' } storage: { type: 'filesystem', path: '/data/brain' }

View file

@ -21,21 +21,21 @@ next:
## Install ## Install
```bash ```bash
npm install @soulcraftlabs/brainy npm install @soulcraft/brainy
``` ```
Or with your preferred package manager: Or with your preferred package manager:
```bash ```bash
bun add @soulcraftlabs/brainy bun add @soulcraft/brainy
yarn add @soulcraftlabs/brainy yarn add @soulcraft/brainy
pnpm add @soulcraftlabs/brainy pnpm add @soulcraft/brainy
``` ```
## Verify ## Verify
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()
@ -52,7 +52,7 @@ npm install @soulcraft/cor
``` ```
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brain = new Brainy({ plugins: ['@soulcraft/cor'] }) const brain = new Brainy({ plugins: ['@soulcraft/cor'] })
await brain.init() // native providers registered during init await brain.init() // native providers registered during init
@ -71,7 +71,7 @@ remains available on npm if you need it.
Brainy ships with full TypeScript types. No `@types/` package needed: Brainy ships with full TypeScript types. No `@types/` package needed:
```typescript ```typescript
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()

View file

@ -66,7 +66,7 @@ const results = await brain.search("query")
**New diagnostics for capacity planning and performance tuning.** **New diagnostics for capacity planning and performance tuning.**
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()
@ -112,7 +112,7 @@ Recommendations: ${stats.recommendations.join(', ')}
### Step 1: Update Package ### Step 1: Update Package
```bash ```bash
npm install @soulcraftlabs/brainy@latest npm install @soulcraft/brainy@latest
``` ```
### Step 2: Restart Your Application ### Step 2: Restart Your Application
@ -134,7 +134,7 @@ npm run start
### Check Adaptive Sizing is Working ### Check Adaptive Sizing is Working
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()
@ -218,7 +218,7 @@ For debugging or compatibility testing:
If you need to rollback to v3.35.0: If you need to rollback to v3.35.0:
```bash ```bash
npm install @soulcraftlabs/brainy@3.35.0 npm install @soulcraft/brainy@3.35.0
``` ```
**Note:** We don't anticipate any issues, but rollback is straightforward if needed. **Note:** We don't anticipate any issues, but rollback is straightforward if needed.
@ -367,7 +367,7 @@ if (stats.fairness.fairnessViolation) {
## Next Steps ## Next Steps
1. ✅ **Upgrade:** `npm install @soulcraftlabs/brainy@latest` 1. ✅ **Upgrade:** `npm install @soulcraft/brainy@latest`
2. 📊 **Monitor:** Use `getCacheStats()` to verify performance improvements 2. 📊 **Monitor:** Use `getCacheStats()` to verify performance improvements
3. 🎯 **Tune:** Adjust based on recommendations (if needed) 3. 🎯 **Tune:** Adjust based on recommendations (if needed)
4. 📖 **Read:** [Operations Guide](../operations/capacity-planning.md) for capacity planning 4. 📖 **Read:** [Operations Guide](../operations/capacity-planning.md) for capacity planning

View file

@ -37,7 +37,7 @@ This single WASM file contains everything needed for sentence embeddings.
```bash ```bash
# Bun as a runtime — supported and recommended # Bun as a runtime — supported and recommended
bun add @soulcraftlabs/brainy bun add @soulcraft/brainy
bun run server.ts bun run server.ts
``` ```

View file

@ -80,7 +80,7 @@ If you read raw stored records (fact-log scanners, export tooling), use
the exported shape-aware splitters — they handle both record eras: the exported shape-aware splitters — they handle both record eras:
```typescript ```typescript
import { splitNounMetadataRecord } from '@soulcraftlabs/brainy' import { splitNounMetadataRecord } from '@soulcraft/brainy'
const { reserved, custom } = splitNounMetadataRecord(rawRecord) const { reserved, custom } = splitNounMetadataRecord(rawRecord)
// reserved = engine fields · custom = the user's bag, ANY names // reserved = engine fields · custom = the user's bag, ANY names
``` ```
@ -88,7 +88,7 @@ const { reserved, custom } = splitNounMetadataRecord(rawRecord)
Feature detection (never version-sniff): Feature detection (never version-sniff):
```typescript ```typescript
import * as brainy from '@soulcraftlabs/brainy' import * as brainy from '@soulcraft/brainy'
const lawActive = 'FIELD_ADDRESSING_CAPABILITY' in brainy // 'field-addressing/v1' const lawActive = 'FIELD_ADDRESSING_CAPABILITY' in brainy // 'field-addressing/v1'
``` ```

View file

@ -9,7 +9,7 @@ Complete guide to integrating Brainy with Next.js applications, covering App Rou
```bash ```bash
npx create-next-app@latest my-brainy-app npx create-next-app@latest my-brainy-app
cd my-brainy-app cd my-brainy-app
npm install @soulcraftlabs/brainy npm install @soulcraft/brainy
``` ```
### Basic Setup ### Basic Setup
@ -18,7 +18,7 @@ npm install @soulcraftlabs/brainy
// app/components/BrainyProvider.jsx // app/components/BrainyProvider.jsx
'use client' 'use client'
import { createContext, useContext, useEffect, useState } from 'react' import { createContext, useContext, useEffect, useState } from 'react'
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const BrainyContext = createContext() const BrainyContext = createContext()
@ -271,7 +271,7 @@ export default function SearchPage() {
```javascript ```javascript
// app/api/search/route.js (App Router) // app/api/search/route.js (App Router)
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
let brain = null let brain = null
@ -332,7 +332,7 @@ export async function GET() {
```javascript ```javascript
// pages/api/search.js (Pages Router) // pages/api/search.js (Pages Router)
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
let brain = null let brain = null
@ -374,7 +374,7 @@ export default async function handler(req, res) {
```javascript ```javascript
// app/api/data/route.js // app/api/data/route.js
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
let brain = null let brain = null
@ -418,7 +418,7 @@ export async function POST(request) {
```jsx ```jsx
// app/actions/brainy.js // app/actions/brainy.js
'use server' 'use server'
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
let brain = null let brain = null
@ -630,7 +630,7 @@ CMD ["npm", "start"]
/** @type {import('next').NextConfig} */ /** @type {import('next').NextConfig} */
const nextConfig = { const nextConfig = {
experimental: { experimental: {
serverComponentsExternalPackages: ['@soulcraftlabs/brainy'] serverComponentsExternalPackages: ['@soulcraft/brainy']
}, },
webpack: (config, { isServer }) => { webpack: (config, { isServer }) => {
if (!isServer) { if (!isServer) {
@ -797,7 +797,7 @@ export function rateLimit(req, limit = 100, window = 60000) {
// app/contexts/BrainyContext.jsx // app/contexts/BrainyContext.jsx
'use client' 'use client'
import { createContext, useContext, useReducer, useEffect } from 'react' import { createContext, useContext, useReducer, useEffect } from 'react'
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const BrainyContext = createContext() const BrainyContext = createContext()
@ -873,7 +873,7 @@ import { BrainyProvider } from '../app/components/BrainyProvider'
import { Search } from '../app/components/Search' import { Search } from '../app/components/Search'
// Mock Brainy // Mock Brainy
jest.mock('@soulcraftlabs/brainy', () => ({ jest.mock('@soulcraft/brainy', () => ({
Brainy: jest.fn().mockImplementation(() => ({ Brainy: jest.fn().mockImplementation(() => ({
init: jest.fn().mockResolvedValue(undefined), init: jest.fn().mockResolvedValue(undefined),
find: jest.fn().mockResolvedValue([ find: jest.fn().mockResolvedValue([

View file

@ -32,7 +32,7 @@ Brainy 7.31.0 adds a per-entity revision counter so multiple writers can coordin
Every distributed-job scheduler eventually wants this exact loop: Every distributed-job scheduler eventually wants this exact loop:
```ts ```ts
import { Brainy, RevisionConflictError } from '@soulcraftlabs/brainy' import { Brainy, RevisionConflictError } from '@soulcraft/brainy'
const LOCK_ID = '...uuid for this job slot...' const LOCK_ID = '...uuid for this job slot...'
@ -137,7 +137,7 @@ await brain.addIfMissing({ // ← not a real API
It's race-prone as a plain read-then-write: two concurrent imports both see "not found," both insert, you get duplicates. Without a unique-index primitive (which Brainy doesn't have today), close the race with whole-store CAS — read at a pinned generation, then commit only if nothing moved: It's race-prone as a plain read-then-write: two concurrent imports both see "not found," both insert, you get duplicates. Without a unique-index primitive (which Brainy doesn't have today), close the race with whole-store CAS — read at a pinned generation, then commit only if nothing moved:
```ts ```ts
import { GenerationConflictError } from '@soulcraftlabs/brainy' import { GenerationConflictError } from '@soulcraft/brainy'
async function addIfMissingByEmail(email: string, data: string) { async function addIfMissingByEmail(email: string, data: string) {
for (let attempt = 0; attempt < 5; attempt++) { for (let attempt = 0; attempt < 5; attempt++) {

View file

@ -18,13 +18,13 @@ Get Brainy running in under a minute.
## 1. Install ## 1. Install
```bash ```bash
npm install @soulcraftlabs/brainy npm install @soulcraft/brainy
``` ```
## 2. Initialize ## 2. Initialize
```typescript ```typescript
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()
@ -67,7 +67,7 @@ await brain.relate({
## 5. Query with Triple Intelligence ## 5. Query with Triple Intelligence
```typescript ```typescript
import type { Result } from '@soulcraftlabs/brainy' import type { Result } from '@soulcraft/brainy'
// All three search paradigms in one call // All three search paradigms in one call
const results: Result[] = await brain.find({ const results: Result[] = await brain.find({

View file

@ -11,7 +11,7 @@
### One Interface for Everything ### One Interface for Everything
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brain = await Brainy.create() const brain = await Brainy.create()
@ -78,7 +78,7 @@ interface ImportProgress {
```typescript ```typescript
import { useState } from 'react' import { useState } from 'react'
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
function UniversalImportProgress({ file }: { file: File }) { function UniversalImportProgress({ file }: { file: File }) {
const [progress, setProgress] = useState({ const [progress, setProgress] = useState({
@ -177,7 +177,7 @@ function UniversalImportProgress({ file }: { file: File }) {
```typescript ```typescript
import ora from 'ora' import ora from 'ora'
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
async function importWithProgress(filePath: string) { async function importWithProgress(filePath: string) {
const spinner = ora('Starting import...').start() const spinner = ora('Starting import...').start()

View file

@ -28,7 +28,7 @@ on-disk layout (memory's "disk" is a JS Map).
## Quick start ## Quick start
```ts ```ts
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
// Filesystem (recommended for any persistent workload): // Filesystem (recommended for any persistent workload):
const brain = new Brainy({ const brain = new Brainy({
@ -134,7 +134,7 @@ config; the `type` is optional.
If you want to skip the factory: If you want to skip the factory:
```ts ```ts
import { FileSystemStorage, MemoryStorage } from '@soulcraftlabs/brainy' import { FileSystemStorage, MemoryStorage } from '@soulcraft/brainy'
const fsStorage = new FileSystemStorage('./brainy-data') const fsStorage = new FileSystemStorage('./brainy-data')
const memStorage = new MemoryStorage() const memStorage = new MemoryStorage()

View file

@ -34,7 +34,7 @@ Three layers solve this:
### Write ### Write
```typescript ```typescript
import { Brainy, NounType } from '@soulcraftlabs/brainy' import { Brainy, NounType } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()
@ -240,7 +240,7 @@ await brain.migrateField({
A realistic adoption sequence for a brain that started without these primitives: A realistic adoption sequence for a brain that started without these primitives:
```typescript ```typescript
import { Brainy, NounType } from '@soulcraftlabs/brainy' import { Brainy, NounType } from '@soulcraft/brainy'
const brain = new Brainy({ storage: { type: 'filesystem', path: './brain-data' } }) const brain = new Brainy({ storage: { type: 'filesystem', path: './brain-data' } })
await brain.init() await brain.init()

View file

@ -25,7 +25,7 @@ content — and how 8.0 recovers it for you.
## TL;DR ## TL;DR
- **Just upgrade to `@soulcraftlabs/brainy@8.0.12` (or later) and open the store.** - **Just upgrade to `@soulcraft/brainy@8.0.12` (or later) and open the store.**
If a previous upgrade left VFS content stranded, 8.0.12 **heals it on open**, If a previous upgrade left VFS content stranded, 8.0.12 **heals it on open**,
with no operator action. with no operator action.
- Want to force or script it? Call **`await brain.vfs.adoptOrphanedBlobs()`**. - Want to force or script it? Call **`await brain.vfs.adoptOrphanedBlobs()`**.
@ -90,7 +90,7 @@ So the operator action for a stranded store is simply: **upgrade to 8.0.12 and
open it.** open it.**
```ts ```ts
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
// Opening the store is all that is required — recovery runs during init(). // Opening the store is all that is required — recovery runs during init().
const brain = new Brainy({ storage: { type: 'filesystem', path: '/data/my-store' } }) const brain = new Brainy({ storage: { type: 'filesystem', path: '/data/my-store' } })
@ -182,5 +182,5 @@ and opening each store is sufficient.
The recovery is copy-only, so no rollback of the recovery itself is ever needed. The recovery is copy-only, so no rollback of the recovery itself is ever needed.
If you need to roll back the **whole** 7→8 upgrade, restore the directory from If you need to roll back the **whole** 7→8 upgrade, restore the directory from
your pre-upgrade backup (retained automatically while recovery is incomplete, or your pre-upgrade backup (retained automatically while recovery is incomplete, or
your own snapshot) and pin `@soulcraftlabs/brainy@7.x`. 8.0 does not keep the old your own snapshot) and pin `@soulcraft/brainy@7.x`. 8.0 does not keep the old
branch layout in place, so a directory-level restore is the rollback path. branch layout in place, so a directory-level restore is the rollback path.

View file

@ -12,7 +12,7 @@ Complete guide to integrating Brainy with Vue.js applications, covering Vue 3, N
npm create vue@latest my-brainy-app npm create vue@latest my-brainy-app
cd my-brainy-app cd my-brainy-app
npm install npm install
npm install @soulcraftlabs/brainy npm install @soulcraft/brainy
``` ```
### Basic Setup ### Basic Setup
@ -574,7 +574,7 @@ Nuxt's server engine (Nitro) is the natural home for Brainy: it runs on Node/Bun
```javascript ```javascript
// server/utils/brain.js (server-only — Nitro never bundles this into the client) // server/utils/brain.js (server-only — Nitro never bundles this into the client)
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
let brainPromise let brainPromise
@ -1201,7 +1201,7 @@ import vue from '@vitejs/plugin-vue'
export default defineConfig({ export default defineConfig({
plugins: [vue()], plugins: [vue()],
ssr: { ssr: {
external: ['@soulcraftlabs/brainy'] external: ['@soulcraft/brainy']
} }
}) })
``` ```

View file

@ -24,7 +24,7 @@ Brainy's neural extraction system uses a **4-signal ensemble architecture** to c
### Method 1: Brain Instance (Recommended) ### Method 1: Brain Instance (Recommended)
```typescript ```typescript
import { Brainy, NounType } from '@soulcraftlabs/brainy' import { Brainy, NounType } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()
@ -62,9 +62,9 @@ const people = await brain.extractEntities('...', {
import { import {
SmartExtractor, SmartExtractor,
SmartRelationshipExtractor SmartRelationshipExtractor
} from '@soulcraftlabs/brainy' } from '@soulcraft/brainy'
// Or use subpath imports: // Or use subpath imports:
import { SmartExtractor } from '@soulcraftlabs/brainy/neural/SmartExtractor' import { SmartExtractor } from '@soulcraft/brainy/neural/SmartExtractor'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()
@ -176,7 +176,7 @@ const withVectors = await brain.extractEntities(text, {
**Direct entity type classifier.** Use when you have pre-detected candidates or need custom configuration. **Direct entity type classifier.** Use when you have pre-detected candidates or need custom configuration.
```typescript ```typescript
import { SmartExtractor, FormatContext } from '@soulcraftlabs/brainy' import { SmartExtractor, FormatContext } from '@soulcraft/brainy'
const extractor = new SmartExtractor(brain, { const extractor = new SmartExtractor(brain, {
minConfidence: 0.7, // Threshold minConfidence: 0.7, // Threshold
@ -229,7 +229,7 @@ interface ExtractionResult {
**Relationship type classifier.** Determines verb/relationship types between entities. **Relationship type classifier.** Determines verb/relationship types between entities.
```typescript ```typescript
import { SmartRelationshipExtractor } from '@soulcraftlabs/brainy' import { SmartRelationshipExtractor } from '@soulcraft/brainy'
const relExtractor = new SmartRelationshipExtractor(brain, { const relExtractor = new SmartRelationshipExtractor(brain, {
minConfidence: 0.6, minConfidence: 0.6,
@ -286,7 +286,7 @@ const rel = await relExtractor.infer(
**Full extraction orchestrator.** Handles candidate detection, classification, and deduplication. **Full extraction orchestrator.** Handles candidate detection, classification, and deduplication.
```typescript ```typescript
import { NeuralEntityExtractor } from '@soulcraftlabs/brainy' import { NeuralEntityExtractor } from '@soulcraft/brainy'
const extractor = new NeuralEntityExtractor(brain) const extractor = new NeuralEntityExtractor(brain)
@ -607,7 +607,7 @@ const locations = entities.filter(e => e.type === NounType.Location)
### Example 2: Excel Data Classification ### Example 2: Excel Data Classification
```typescript ```typescript
import { SmartExtractor } from '@soulcraftlabs/brainy' import { SmartExtractor } from '@soulcraft/brainy'
const extractor = new SmartExtractor(brain) const extractor = new SmartExtractor(brain)
@ -629,7 +629,7 @@ for (let i = 0; i < cells.length; i++) {
### Example 3: Relationship Extraction ### Example 3: Relationship Extraction
```typescript ```typescript
import { SmartRelationshipExtractor } from '@soulcraftlabs/brainy' import { SmartRelationshipExtractor } from '@soulcraft/brainy'
const relExtractor = new SmartRelationshipExtractor(brain) const relExtractor = new SmartRelationshipExtractor(brain)

View file

@ -204,8 +204,8 @@ await brain.add({ data: { name: 'Entity' }, type: NounType.Thing })
### Basic Add Operation ### Basic Add Operation
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
import { NounType } from '@soulcraftlabs/brainy/types' import { NounType } from '@soulcraft/brainy/types'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()
@ -428,7 +428,7 @@ await brain.relate({ ... }) // a crash here leaves the entity unlinked
```typescript ```typescript
import { describe, it, expect } from 'vitest' import { describe, it, expect } from 'vitest'
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
describe('Transaction Tests', () => { describe('Transaction Tests', () => {
it('should rollback on failure', async () => { it('should rollback on failure', async () => {

View file

@ -23,7 +23,7 @@ The Universal Display Augmentation is a powerful AI-powered system that automati
### Basic Usage ### Basic Usage
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brainy = new Brainy() const brainy = new Brainy()
await brainy.init() await brainy.init()

View file

@ -71,9 +71,9 @@ Let's build a projection that organizes files by priority (high, medium, low):
### Step 1: Create the Strategy Class ### Step 1: Create the Strategy Class
```typescript ```typescript
import { BaseProjectionStrategy } from '@soulcraftlabs/brainy/vfs/semantic' import { BaseProjectionStrategy } from '@soulcraft/brainy/vfs/semantic'
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
import { VirtualFileSystem, VFSEntity } from '@soulcraftlabs/brainy/vfs' import { VirtualFileSystem, VFSEntity } from '@soulcraft/brainy/vfs'
export class PriorityProjection extends BaseProjectionStrategy { export class PriorityProjection extends BaseProjectionStrategy {
readonly name = 'priority' readonly name = 'priority'
@ -141,7 +141,7 @@ export class PriorityProjection extends BaseProjectionStrategy {
### Step 2: Register the Strategy ### Step 2: Register the Strategy
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
import { PriorityProjection } from './PriorityProjection' import { PriorityProjection } from './PriorityProjection'
const brain = new Brainy() const brain = new Brainy()
@ -537,7 +537,7 @@ Use the projection's resolve cache:
```typescript ```typescript
import { describe, it, expect, beforeAll } from 'vitest' import { describe, it, expect, beforeAll } from 'vitest'
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
import { PriorityProjection } from './PriorityProjection' import { PriorityProjection } from './PriorityProjection'
describe('PriorityProjection', () => { describe('PriorityProjection', () => {
@ -714,7 +714,7 @@ async resolve(brain, vfs, value: string) {
3. Use appropriate limits: Don't fetch more than needed 3. Use appropriate limits: Don't fetch more than needed
### Type errors ### Type errors
1. Import correct types: `import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy'` 1. Import correct types: `import { Brainy, VirtualFileSystem } from '@soulcraft/brainy'`
2. Use `as VFSEntity` when mapping results 2. Use `as VFSEntity` when mapping results
3. Check BaseProjectionStrategy import 3. Check BaseProjectionStrategy import

View file

@ -14,11 +14,11 @@ A file explorer that:
## ⚡ Step 1: Basic Setup (1 minute) ## ⚡ Step 1: Basic Setup (1 minute)
```bash ```bash
npm install @soulcraftlabs/brainy npm install @soulcraft/brainy
``` ```
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
// ✅ CORRECT: Use filesystem storage for production // ✅ CORRECT: Use filesystem storage for production
const brain = new Brainy({ const brain = new Brainy({
@ -115,7 +115,7 @@ Here's a complete React component using the correct patterns:
```tsx ```tsx
import React, { useState, useEffect } from 'react' import React, { useState, useEffect } from 'react'
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
export function FileExplorer() { export function FileExplorer() {
const [brain, setBrain] = useState(null) const [brain, setBrain] = useState(null)
@ -288,8 +288,8 @@ Your file explorer is now working! Here's what to explore next:
### "Module not found" errors ### "Module not found" errors
```bash ```bash
# Make sure you're using the right import # Make sure you're using the right import
npm ls @soulcraftlabs/brainy # Check version npm ls @soulcraft/brainy # Check version
npm install @soulcraftlabs/brainy@latest # Update if needed npm install @soulcraft/brainy@latest # Update if needed
``` ```
### "VFS not initialized" errors ### "VFS not initialized" errors

View file

@ -24,7 +24,7 @@ Brainy VFS is a revolutionary virtual filesystem that runs on top of Brainy's ne
## Quick Start ## Quick Start
```javascript ```javascript
import { VirtualFileSystem } from '@soulcraftlabs/brainy/vfs' import { VirtualFileSystem } from '@soulcraft/brainy/vfs'
// Initialize the VFS // Initialize the VFS
const vfs = new VirtualFileSystem({ const vfs = new VirtualFileSystem({
@ -381,7 +381,7 @@ Brainy VFS fully leverages Brainy's revolutionary Triple Intelligence system:
## Installation ## Installation
```bash ```bash
npm install @soulcraftlabs/brainy npm install @soulcraft/brainy
``` ```
## Requirements ## Requirements

View file

@ -135,7 +135,7 @@ Mount VFS as a native filesystem on Linux/Mac/Windows.
```typescript ```typescript
// Planned (research phase) // Planned (research phase)
import { mountVFS } from '@soulcraftlabs/brainy/vfs/fuse' import { mountVFS } from '@soulcraft/brainy/vfs/fuse'
await mountVFS(vfs, { await mountVFS(vfs, {
mountPoint: '/mnt/brainy', mountPoint: '/mnt/brainy',
@ -160,7 +160,7 @@ These features would benefit from community contributions. If you're interested
### Express.js Static Middleware ### Express.js Static Middleware
```typescript ```typescript
// Wanted: Community contribution // Wanted: Community contribution
import { createStaticMiddleware } from '@soulcraftlabs/brainy/vfs/express' import { createStaticMiddleware } from '@soulcraft/brainy/vfs/express'
app.use('/files', createStaticMiddleware(vfs, { app.use('/files', createStaticMiddleware(vfs, {
index: ['index.html', 'index.md'], index: ['index.html', 'index.md'],
@ -172,7 +172,7 @@ app.use('/files', createStaticMiddleware(vfs, {
### VSCode Extension ### VSCode Extension
```typescript ```typescript
// Wanted: Community contribution // Wanted: Community contribution
import { VFSProvider } from '@soulcraftlabs/brainy/vfs/vscode' import { VFSProvider } from '@soulcraft/brainy/vfs/vscode'
const provider = new VFSProvider(vfs) const provider = new VFSProvider(vfs)
vscode.workspace.registerFileSystemProvider('brainy', provider) vscode.workspace.registerFileSystemProvider('brainy', provider)

View file

@ -327,7 +327,7 @@ console.log(id1 === id2 && id2 === id3) // true
Create your own semantic dimensions: Create your own semantic dimensions:
```typescript ```typescript
import { BaseProjectionStrategy } from '@soulcraftlabs/brainy/vfs/semantic' import { BaseProjectionStrategy } from '@soulcraft/brainy/vfs/semantic'
class PriorityProjection extends BaseProjectionStrategy { class PriorityProjection extends BaseProjectionStrategy {
readonly name = 'priority' readonly name = 'priority'

View file

@ -7,7 +7,7 @@ Brainy's Virtual Filesystem (VFS) provides a POSIX-like filesystem interface tha
## Quick Start ## Quick Start
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
// Initialize Brainy // Initialize Brainy
const brain = new Brainy({ const brain = new Brainy({
@ -598,7 +598,7 @@ const user = await store.findById('users', 'user123')
VFS uses standard POSIX-style errors: VFS uses standard POSIX-style errors:
```typescript ```typescript
import { VFSError, VFSErrorCode } from '@soulcraftlabs/brainy' import { VFSError, VFSErrorCode } from '@soulcraft/brainy'
try { try {
await vfs.readFile('/nonexistent.txt') await vfs.readFile('/nonexistent.txt')

View file

@ -280,7 +280,7 @@ GitBridge provides Git import/export capabilities:
#### GitBridge Usage #### GitBridge Usage
```javascript ```javascript
// Import and instantiate GitBridge // Import and instantiate GitBridge
import { GitBridge } from '@soulcraftlabs/brainy' import { GitBridge } from '@soulcraft/brainy'
const gitBridge = new GitBridge(vfs, brain) const gitBridge = new GitBridge(vfs, brain)
// Export VFS to Git repository structure // Export VFS to Git repository structure
@ -452,7 +452,7 @@ This ordering prevents race conditions where file writes might fail because pare
## Complete Example ## Complete Example
```javascript ```javascript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
async function vfsExample() { async function vfsExample() {
// Initialize // Initialize

View file

@ -196,5 +196,5 @@ await brain.relate({
Always import and use the type enums: Always import and use the type enums:
```javascript ```javascript
import { NounType, VerbType } from '@soulcraftlabs/brainy' import { NounType, VerbType } from '@soulcraft/brainy'
``` ```

View file

@ -5,7 +5,7 @@
The Brainy VFS is automatically initialized during `brain.init()`. No separate initialization needed! The Brainy VFS is automatically initialized during `brain.init()`. No separate initialization needed!
```javascript ```javascript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
// Create and initialize Brainy // Create and initialize Brainy
const brain = new Brainy({ const brain = new Brainy({
@ -71,7 +71,7 @@ VFS stores files as entities and relationships in the same graph as everything e
## Complete Example ## Complete Example
```javascript ```javascript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
async function useVFS() { async function useVFS() {
// Initialize Brainy // Initialize Brainy
@ -100,7 +100,7 @@ useVFS().catch(console.error)
## TypeScript Usage ## TypeScript Usage
```typescript ```typescript
import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy' import { Brainy, VirtualFileSystem } from '@soulcraft/brainy'
class FileManager { class FileManager {
private brain: Brainy private brain: Brainy

View file

@ -37,7 +37,7 @@ Brainy VFS provides safe, tree-aware methods that prevent these issues:
### Method 1: Use `getDirectChildren()` (Recommended) ### Method 1: Use `getDirectChildren()` (Recommended)
```typescript ```typescript
import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy' import { Brainy, VirtualFileSystem } from '@soulcraft/brainy'
const brain = new Brainy() const brain = new Brainy()
await brain.init() await brain.init()
@ -97,7 +97,7 @@ Here's a complete example using React:
```tsx ```tsx
import React, { useState, useEffect } from 'react' import React, { useState, useEffect } from 'react'
import { VirtualFileSystem } from '@soulcraftlabs/brainy' import { VirtualFileSystem } from '@soulcraft/brainy'
interface FileNode { interface FileNode {
name: string name: string
@ -177,7 +177,7 @@ function TreeView({ node, onToggle, expanded }) {
If you must build trees manually from flat lists, use the `VFSTreeUtils`: If you must build trees manually from flat lists, use the `VFSTreeUtils`:
```typescript ```typescript
import { VFSTreeUtils } from '@soulcraftlabs/brainy/vfs' import { VFSTreeUtils } from '@soulcraft/brainy/vfs'
// Get all entities somehow // Get all entities somehow
const allEntities = await vfs.getDescendants('/root') const allEntities = await vfs.getDescendants('/root')

View file

@ -7,7 +7,7 @@
* the Bluesky firehose with Brainy's distributed architecture * the Bluesky firehose with Brainy's distributed architecture
*/ */
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
import { WebSocket } from 'ws' import { WebSocket } from 'ws'
// ===================================================== // =====================================================

View file

@ -14,7 +14,7 @@
* ts-node examples/monitor-cache-performance.ts * ts-node examples/monitor-cache-performance.ts
*/ */
import { Brainy, NounType } from '@soulcraftlabs/brainy' import { Brainy, NounType } from '@soulcraft/brainy'
// ANSI color codes for pretty output // ANSI color codes for pretty output
const colors = { const colors = {

View file

@ -5,7 +5,7 @@ Connect Brainy to spreadsheets, BI tools, and external systems with zero configu
## Quick Start ## Quick Start
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brain = new Brainy({ integrations: true }) const brain = new Brainy({ integrations: true })
await brain.init() await brain.init()
@ -178,7 +178,7 @@ Webhooks include `X-Brainy-Signature` header with HMAC-SHA256 signature.
### Minimal (in-memory): ### Minimal (in-memory):
```typescript ```typescript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brain = new Brainy({ integrations: true }) const brain = new Brainy({ integrations: true })
await brain.init() await brain.init()
@ -194,7 +194,7 @@ console.log(brain.hub.getInstructions())
```typescript ```typescript
import express from 'express' import express from 'express'
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const app = express() const app = express()
const brain = new Brainy({ const brain = new Brainy({
@ -232,7 +232,7 @@ app.listen(3000, () => {
```typescript ```typescript
import { Hono } from 'hono' import { Hono } from 'hono'
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const app = new Hono() const app = new Hono()

View file

@ -99,7 +99,7 @@ Add the `BRAINY_URL` script property in Apps Script settings.
The simplest way to enable all integrations: The simplest way to enable all integrations:
```javascript ```javascript
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const brain = new Brainy({ integrations: true }) const brain = new Brainy({ integrations: true })
await brain.init() await brain.init()
@ -112,7 +112,7 @@ With Express:
```javascript ```javascript
import express from 'express' import express from 'express'
import { Brainy } from '@soulcraftlabs/brainy' import { Brainy } from '@soulcraft/brainy'
const app = express() const app = express()
const brain = new Brainy({ integrations: true }) const brain = new Brainy({ integrations: true })

8
package-lock.json generated
View file

@ -1,12 +1,12 @@
{ {
"name": "@soulcraftlabs/brainy", "name": "@soulcraft/brainy",
"version": "10.4.13", "version": "10.4.1",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "@soulcraftlabs/brainy", "name": "@soulcraft/brainy",
"version": "10.4.13", "version": "10.4.1",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"@msgpack/msgpack": "^3.1.2", "@msgpack/msgpack": "^3.1.2",

View file

@ -1,7 +1,6 @@
{ {
"name": "@soulcraftlabs/brainy", "name": "@soulcraft/brainy",
"version": "10.4.13", "version": "10.4.1",
"brainyContract": 1,
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
"main": "dist/index.js", "main": "dist/index.js",
"module": "dist/index.js", "module": "dist/index.js",
@ -88,7 +87,7 @@
"test:watch": "NODE_OPTIONS='--max-old-space-size=8192' vitest --config tests/configs/vitest.unit.config.ts", "test:watch": "NODE_OPTIONS='--max-old-space-size=8192' vitest --config tests/configs/vitest.unit.config.ts",
"test:coverage": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.unit.config.ts --coverage", "test:coverage": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.unit.config.ts --coverage",
"test:unit": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.unit.config.ts", "test:unit": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.unit.config.ts",
"test:perf": "vitest run --config tests/configs/vitest.perf.config.ts", "test:perf": "vitest run tests/unit/performance --reporter=basic",
"test:integration": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.integration.config.ts", "test:integration": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.integration.config.ts",
"test:semantic": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.semantic.config.ts", "test:semantic": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.semantic.config.ts",
"test:all": "npm run test:unit && npm run test:integration", "test:all": "npm run test:unit && npm run test:integration",
@ -127,16 +126,15 @@
"license": "MIT", "license": "MIT",
"private": false, "private": false,
"publishConfig": { "publishConfig": {
"access": "public", "access": "public"
"registry": "https://source.soulcraft.com/api/packages/soulcraftlabs/npm/"
}, },
"homepage": "https://source.soulcraft.com/soulcraftlabs/open-brainy", "homepage": "https://source.soulcraft.com/soulcraft/brainy",
"bugs": { "bugs": {
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/issues" "url": "https://source.soulcraft.com/soulcraft/brainy/issues"
}, },
"repository": { "repository": {
"type": "git", "type": "git",
"url": "git+https://source.soulcraft.com/soulcraftlabs/open-brainy.git" "url": "git+https://source.soulcraft.com/soulcraft/brainy.git"
}, },
"files": [ "files": [
"dist/**/*.js", "dist/**/*.js",

View file

@ -10,7 +10,6 @@ import { TransformerEmbedding } from '../src/utils/embedding.js'
import * as fs from 'fs/promises' import * as fs from 'fs/promises'
import * as path from 'path' import * as path from 'path'
import { fileURLToPath } from 'url' import { fileURLToPath } from 'url'
import { resolveDeterministicStamp } from './lib/deterministicStamp.js'
const __dirname = path.dirname(fileURLToPath(import.meta.url)) const __dirname = path.dirname(fileURLToPath(import.meta.url))
@ -99,21 +98,12 @@ async function buildEmbeddedPatterns() {
const uint8 = new Uint8Array(buffer) const uint8 = new Uint8Array(buffer)
const base64 = Buffer.from(uint8).toString('base64') const base64 = Buffer.from(uint8).toString('base64')
// Deterministic stamp: derived from the git commit time of this
// generator's inputs, never from wall-clock time — two builds of the
// same source tree must produce byte-identical output.
const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedPatterns.ts')
const generatedStamp = resolveDeterministicStamp(
[path.join(__dirname, 'buildEmbeddedPatterns.ts'), libraryPath],
outputPath
)
// Generate TypeScript file with everything embedded // Generate TypeScript file with everything embedded
const tsContent = `/** const tsContent = `/**
* 🧠 BRAINY EMBEDDED PATTERNS * 🧠 BRAINY EMBEDDED PATTERNS
* *
* AUTO-GENERATED - DO NOT EDIT * AUTO-GENERATED - DO NOT EDIT
* Generated: ${generatedStamp} * Generated: ${new Date().toISOString()}
* Patterns: ${libraryData.patterns.length} * Patterns: ${libraryData.patterns.length}
* Coverage: 94-98% of all queries * Coverage: 94-98% of all queries
* *
@ -207,6 +197,7 @@ prodLog.info(\`🧠 Brainy Pattern Library loaded: \${EMBEDDED_PATTERNS.length}
` `
// Write the TypeScript file // Write the TypeScript file
const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedPatterns.ts')
await fs.writeFile(outputPath, tsContent) await fs.writeFile(outputPath, tsContent)
// Report statistics // Report statistics

View file

@ -11,7 +11,6 @@ import * as fs from 'fs/promises'
import * as path from 'path' import * as path from 'path'
import { fileURLToPath } from 'url' import { fileURLToPath } from 'url'
import { NounType, VerbType } from '../src/types/graphTypes.js' import { NounType, VerbType } from '../src/types/graphTypes.js'
import { resolveDeterministicStamp } from './lib/deterministicStamp.js'
const __dirname = path.dirname(fileURLToPath(import.meta.url)) const __dirname = path.dirname(fileURLToPath(import.meta.url))
@ -374,24 +373,12 @@ async function buildTypeEmbeddings() {
const uint8 = new Uint8Array(buffer) const uint8 = new Uint8Array(buffer)
const base64 = Buffer.from(uint8).toString('base64') const base64 = Buffer.from(uint8).toString('base64')
// Deterministic stamp: derived from the git commit time of this
// generator's inputs, never from wall-clock time — two builds of the
// same source tree must produce byte-identical output.
const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedTypeEmbeddings.ts')
const generatedStamp = resolveDeterministicStamp(
[
path.join(__dirname, 'buildTypeEmbeddings.ts'),
path.join(__dirname, '..', 'src', 'types', 'graphTypes.ts')
],
outputPath
)
// Generate TypeScript file // Generate TypeScript file
const tsContent = `/** const tsContent = `/**
* 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS
* *
* AUTO-GENERATED - DO NOT EDIT * AUTO-GENERATED - DO NOT EDIT
* Generated: ${generatedStamp} * Generated: ${new Date().toISOString()}
* Noun Types: ${nounTypes.length} * Noun Types: ${nounTypes.length}
* Verb Types: ${verbTypes.length} * Verb Types: ${verbTypes.length}
* *
@ -408,7 +395,7 @@ export const TYPE_METADATA = {
verbTypes: ${verbTypes.length}, verbTypes: ${verbTypes.length},
totalTypes: ${totalTypes}, totalTypes: ${totalTypes},
embeddingDimensions: ${embeddingDim}, embeddingDimensions: ${embeddingDim},
generatedAt: "${generatedStamp}", generatedAt: "${new Date().toISOString()}",
sizeBytes: { sizeBytes: {
embeddings: ${buffer.byteLength}, embeddings: ${buffer.byteLength},
base64: ${base64.length} base64: ${base64.length}
@ -507,6 +494,7 @@ prodLog.info(\`🧠 Brainy Type Embeddings loaded: \${TYPE_METADATA.nounTypes} n
` `
// Write the TypeScript file // Write the TypeScript file
const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedTypeEmbeddings.ts')
await fs.writeFile(outputPath, tsContent) await fs.writeFile(outputPath, tsContent)
// Report statistics // Report statistics

View file

@ -1,128 +0,0 @@
#!/usr/bin/env node
/**
* Emit this build's API-contract manifest to docs/api-contract.json.
*
* WHY IT IS GENERATED, NOT WRITTEN: a hand-kept list of doors drifts from the
* code the first time somebody adds one. This reads the surface the build
* actually exposes the prototype's own methods and accessors, the exported
* error classes, the `where` operator sets, the field-addressing vocabulary,
* the health verdicts so a diff between two engines' manifests is a diff
* between two engines, never between two authors.
*
* Requirement marking (required / optional per door) is NOT derivable from the
* surface it is a commitment, recorded with the contract's owner rather than
* here. This manifest carries the surface; the promise lives with the contract.
*
* Usage: node scripts/emit-contract-manifest.mjs [--check]
* --check exits non-zero when the committed manifest is stale.
*/
import { writeFileSync, readFileSync, existsSync } from 'node:fs'
import { join, dirname } from 'node:path'
import { fileURLToPath } from 'node:url'
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..')
const OUT = join(ROOT, 'docs', 'api-contract.json')
const { Brainy } = await import(join(ROOT, 'dist', 'brainy.js'))
const errorsModule = await import(join(ROOT, 'dist', 'errors', 'brainyError.js'))
const versionModule = await import(join(ROOT, 'dist', 'utils', 'version.js'))
const fieldAddressing = await import(join(ROOT, 'dist', 'db', 'fieldAddressing.js'))
/** Every own method and accessor on the class's prototype, minus the private ones. */
function surfaceOf(ctor) {
const doors = []
for (const name of Object.getOwnPropertyNames(ctor.prototype)) {
if (name === 'constructor' || name.startsWith('_')) continue
const descriptor = Object.getOwnPropertyDescriptor(ctor.prototype, name)
if (!descriptor) continue
if (typeof descriptor.value === 'function') {
doors.push({ name, kind: 'method', arity: descriptor.value.length })
} else if (descriptor.get) {
doors.push({ name, kind: 'accessor' })
}
}
return doors.sort((a, b) => a.name.localeCompare(b.name))
}
const errors = Object.entries(errorsModule)
.filter(([name, value]) => typeof value === 'function' && /Error$/.test(name))
.map(([name]) => name)
.sort()
// The operator sets, read from the engine's own refusal message so the
// manifest can never disagree with the validator.
const filterSource = readFileSync(join(ROOT, 'src', 'utils', 'metadataFilter.ts'), 'utf-8')
const acceptedMatch = filterSource.match(/const VALUE_OPERATORS = new Set<string>\(\[([\s\S]*?)\]\)/)
if (!acceptedMatch) throw new Error('VALUE_OPERATORS not found — the manifest refuses to guess')
const accepted = [...acceptedMatch[1].matchAll(/'([^']+)'/g)].map((m) => m[1]).sort()
const indexSource = readFileSync(join(ROOT, 'src', 'utils', 'metadataIndex.ts'), 'utf-8')
const refusedByIndex = ['endsWith', 'length', 'matches', 'startsWith'].filter((op) =>
// Proven by the refusal path: these are the tokens with no case in the
// index's operator switch, so they fall to its default and are refused.
!new RegExp(`case '${op}':`).test(indexSource)
)
const servedOnIndex = accepted.filter((op) => !refusedByIndex.includes(op))
const manifest = {
contractVersion: versionModule.contractVersion(),
engine: '@soulcraftlabs/brainy',
compatibility: {
minor:
'additive — a new optional door, a new served operator, a new error class; every existing implementation still conforms',
major:
'breaking — a door removed, an answer narrowed, an ordering law changed, an optional door promoted to required, or an operator moved from served to refused'
},
doors: surfaceOf(Brainy),
errors,
operators: {
accepted,
servedOnIndexPath: servedOnIndex,
refusedByIndexPath: refusedByIndex,
combinators: ['allOf', 'anyOf', 'not']
},
fieldAddressing: {
systemKeyPrefix: 'system.',
systemEntityScalars: [...(fieldAddressing.SYSTEM_ENTITY_SCALARS ?? [])].sort(),
systemRelationScalars: [...(fieldAddressing.SYSTEM_RELATION_SCALARS ?? [])].sort(),
plumbingFields: [...(fieldAddressing.PLUMBING_FIELDS ?? [])].sort()
},
health: {
verdicts: ['pass', 'warn', 'fail'],
healKinds: ['none', 'repair', 'rebuild'],
servingWithholdingInvariants: [
'index-initialized',
'durable-state-present',
'manifest-residency',
'replay-clean',
'strand-latch'
]
}
}
const rendered = `${JSON.stringify(manifest, null, 2)}\n`
if (process.argv.includes('--check')) {
if (!existsSync(OUT)) {
console.error(`docs/api-contract.json is missing — run: node scripts/emit-contract-manifest.mjs`)
process.exit(1)
}
if (readFileSync(OUT, 'utf-8') !== rendered) {
console.error(
`docs/api-contract.json is STALE — the public surface changed. Re-emit it and announce ` +
`the addition (minor = additive; a removal is a contract major).`
)
process.exit(1)
}
console.log(`docs/api-contract.json is current (${manifest.doors.length} doors, contract ${manifest.contractVersion}).`)
process.exit(0)
}
writeFileSync(OUT, rendered)
console.log(
`Wrote docs/api-contract.json — contract ${manifest.contractVersion}, ` +
`${manifest.doors.length} doors, ${manifest.errors.length} error classes, ` +
`${manifest.operators.accepted.length} operators ` +
`(${manifest.operators.refusedByIndexPath.length} refused by the index path).`
)

View file

@ -1,118 +0,0 @@
/**
* Deterministic generation-stamp resolution for Brainy's build-time code
* generators.
*
* Two builds of the same source tree must produce byte-identical output.
* A wall-clock stamp (`new Date()`) breaks that guarantee, so every
* generator that writes a "Generated:" header or a `generatedAt` field
* into its output must resolve the stamp through this module instead.
*
* Resolution order:
* 1. The newest git commit timestamp among the generator's input files
* (the generator script itself always counts as an input).
* 2. If git metadata is unavailable (for example, building from a
* published npm tarball with no `.git` directory), the stamp already
* recorded in the previously generated output file.
* 3. If neither is available, the fixed epoch string
* `1970-01-01T00:00:00.000Z`.
*
* Every fallback logs a line to stderr deterministic degradation is
* loud, never a silent divergence.
*/
import { execFileSync } from 'child_process'
import * as fs from 'fs'
const EPOCH_STAMP = '1970-01-01T00:00:00.000Z'
const STAMP_PATTERN = /\*\s*Generated:\s*(\S+)/
/**
* Resolve the deterministic stamp for a generator run.
*
* @param inputPaths Absolute paths to every file whose content determines
* the generator's output, including the generator script itself.
* @param previousOutputPath Absolute path to the previously generated
* file, used for the existing-stamp fallback when git is unavailable.
* @returns An ISO-8601 timestamp string that is deterministic for a given
* source tree.
*/
export function resolveDeterministicStamp(
inputPaths: string[],
previousOutputPath: string
): string {
const gitStamp = newestGitCommitTimestamp(inputPaths)
if (gitStamp) {
return gitStamp
}
const existingStamp = readExistingStamp(previousOutputPath)
if (existingStamp) {
process.stderr.write(
`[deterministic-stamp] no git commit history found for generator inputs; ` +
`reusing existing stamp from ${previousOutputPath}: ${existingStamp}\n`
)
return existingStamp
}
process.stderr.write(
`[deterministic-stamp] no git commit history and no previous output at ` +
`${previousOutputPath}; falling back to fixed epoch stamp ${EPOCH_STAMP}\n`
)
return EPOCH_STAMP
}
/**
* Find the newest git commit timestamp among the given input paths.
* Returns null if git is unavailable, the tree is not a git repository,
* or none of the inputs have any commit history yet.
*/
function newestGitCommitTimestamp(inputPaths: string[]): string | null {
let newest: string | null = null
for (const inputPath of inputPaths) {
if (!fs.existsSync(inputPath)) {
continue
}
let out: string
try {
out = execFileSync(
'git',
['log', '-1', '--format=%cI', '--', inputPath],
{ stdio: ['ignore', 'pipe', 'ignore'] }
)
.toString()
.trim()
} catch {
// git missing, not a repository, or no permissions — handled by the
// caller's fallback chain.
continue
}
if (!out) {
// Path exists but has no commit history yet (e.g. newly created,
// uncommitted file).
continue
}
if (!newest || new Date(out).getTime() > new Date(newest).getTime()) {
newest = out
}
}
return newest
}
/**
* Parse the `* Generated: <ISO timestamp>` header out of a previously
* generated file, if one exists.
*/
function readExistingStamp(outputPath: string): string | null {
if (!fs.existsSync(outputPath)) {
return null
}
const content = fs.readFileSync(outputPath, 'utf-8')
const match = content.match(STAMP_PATTERN)
return match ? match[1] : null
}

View file

@ -15,11 +15,11 @@ NC='\033[0m' # No Color
RELEASE_TYPE="${1:-patch}" # patch, minor, or major RELEASE_TYPE="${1:-patch}" # patch, minor, or major
SKIP_TESTS=false SKIP_TESTS=false
DRY_RUN=false DRY_RUN=false
# --source-only is now a no-op: The Source is the one registry, so every # --source-only: the HOME leg only — tag, CI's publish to The Source, and the
# release already ships Source-only — tag, CI's publish to The Source, the # release page; NO storefront (npmjs) publish, NO pair verification, NO docs
# release page, and the docs push, with no separate storefront leg to skip. # push. The pair-gate shape: a prerelease the fleet's other engine devDeps
# The flag is still accepted (for backward-compatible invocations) and just # from our own registry while the pair is proven, never a public artifact.
# prints a notice; it no longer changes behavior. # Refused for a non-prerelease version — a public floor is always a pair.
SOURCE_ONLY=false SOURCE_ONLY=false
for arg in "$@"; do for arg in "$@"; do
@ -109,7 +109,7 @@ else
;; ;;
*) *)
echo -e "${RED}❌ Invalid release type: ${RELEASE_TYPE}${NC}" echo -e "${RED}❌ Invalid release type: ${RELEASE_TYPE}${NC}"
echo "Usage: ./scripts/release.sh [patch|minor|major|<explicit-version>] [--dry-run] [--source-only (no-op; The Source is the one registry)]" echo "Usage: ./scripts/release.sh [patch|minor|major|<explicit-version>] [--dry-run] [--source-only (prereleases only)]"
exit 1 exit 1
;; ;;
esac esac
@ -129,7 +129,11 @@ if [ "$PRERELEASE" = true ]; then
echo -e "${YELLOW}⚠️ Prerelease → npm dist-tag '${NPM_TAG}', GitHub prerelease${NC}" echo -e "${YELLOW}⚠️ Prerelease → npm dist-tag '${NPM_TAG}', GitHub prerelease${NC}"
fi fi
if [ "$SOURCE_ONLY" = true ]; then if [ "$SOURCE_ONLY" = true ]; then
echo -e "${YELLOW}⚠️ The Source is the one registry; --source-only is implied${NC}" if [ "$PRERELEASE" != true ]; then
echo -e "${RED}❌ --source-only is for prereleases only: a non-prerelease version is a public floor and always ships as the byte-identical pair.${NC}"
exit 1
fi
echo -e "${YELLOW}⚠️ --source-only → The Source (home) ONLY: no npmjs publish, no pair verification, no docs push${NC}"
fi fi
echo "" echo ""
@ -154,26 +158,13 @@ else
fi fi
# Create new changelog entry # Create new changelog entry
RELEASE_DATE=$(date +%Y-%m-%d) CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraft/brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) ($(date +%Y-%m-%d))
CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) (${RELEASE_DATE})
${COMMITS} ${COMMITS}
" "
# A CURATED entry wins over the generated one. When a release is cut from a
# lineage that diverged from the previous tag (a candidate branch carrying
# main's history), `git log <last-tag>..HEAD` lists every commit the tag never
# saw — old notes, already-shipped fixes under new hashes, merge commits — and a
# wall entry derived from it would misreport the release. If CHANGELOG.md
# already carries a `### [NEW_VERSION]` heading, it was written on purpose:
# keep it, and skip the generated prepend entirely.
CURATED_ENTRY=false
if grep -qE "^### \[${NEW_VERSION}\]" CHANGELOG.md 2>/dev/null; then
CURATED_ENTRY=true
echo -e "${YELLOW}CHANGELOG already carries a curated ### [${NEW_VERSION}] entry — keeping it, not generating one from commits${NC}"
fi
# Prepend to CHANGELOG.md after header # Prepend to CHANGELOG.md after header
if [ "$CURATED_ENTRY" = false ] && [ -f "CHANGELOG.md" ]; then if [ -f "CHANGELOG.md" ]; then
# Read header (first 4 lines) # Read header (first 4 lines)
HEADER=$(head -n 4 CHANGELOG.md) HEADER=$(head -n 4 CHANGELOG.md)
# Read rest of file # Read rest of file
@ -187,19 +178,6 @@ if [ "$CURATED_ENTRY" = false ] && [ -f "CHANGELOG.md" ]; then
fi fi
echo -e "${GREEN}✅ CHANGELOG updated${NC}\n" echo -e "${GREEN}✅ CHANGELOG updated${NC}\n"
# Step 6b: Update the releases wall entry — mechanical, derived from the
# CHANGELOG entry just composed. The fleet's HQ page reads open-brainy.json
# from the one shared releases repo, soulcraftlabs/releases on The Source —
# this used to be hand-written after every release (David: never again —
# make it a step of the rail, landed in the one shared home; this repo no
# longer hosts its own copy). This step clones/fetches that repo into a
# local cache, prepends the entry, and pushes it directly — a real
# cross-repo push, refusing loudly (never skipping) on any
# clone/validation/commit/push failure.
echo -e "${BLUE}5⃣▸ Updating the releases wall...${NC}"
node scripts/wall-entry.mjs --product open-brainy --version "${NEW_VERSION}" --date "${RELEASE_DATE}" --from-changelog CHANGELOG.md
echo -e "${GREEN}✅ Releases wall updated${NC}\n"
# Step 7: Create release commit # Step 7: Create release commit
echo -e "${BLUE}6⃣ Creating release commit...${NC}" echo -e "${BLUE}6⃣ Creating release commit...${NC}"
git add package.json package-lock.json CHANGELOG.md git add package.json package-lock.json CHANGELOG.md
@ -231,9 +209,9 @@ echo -e "${GREEN}✅ Pushed to origin${NC}\n"
# .forgejo/workflows/publish-source.yml, which builds and publishes on The # .forgejo/workflows/publish-source.yml, which builds and publishes on The
# Source's own runner (datacenter-side: seconds, not the laptop's WAN timing # Source's own runner (datacenter-side: seconds, not the laptop's WAN timing
# out on an 87MB tarball PUT). The laptop holds no home-registry publish # out on an 87MB tarball PUT). The laptop holds no home-registry publish
# credential anymore; it only waits for CI's result before continuing on to # credential anymore; it only waits for CI's result before trusting the
# the release page and the docs push. # home/npmjs pair enough to publish the storefront leg.
SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraftlabs/npm/" SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/"
SOURCE_POLL_INTERVAL_S=15 SOURCE_POLL_INTERVAL_S=15
SOURCE_POLL_MAX_ATTEMPTS=200 # 200 × 15s = 50 minutes — the runner is sequential and a busy day's ci.yml SOURCE_POLL_MAX_ATTEMPTS=200 # 200 × 15s = 50 minutes — the runner is sequential and a busy day's ci.yml
# backlog has twice exceeded the old 20-minute window (8.10.3, 9.0.0); # backlog has twice exceeded the old 20-minute window (8.10.3, 9.0.0);
@ -241,7 +219,7 @@ SOURCE_POLL_MAX_ATTEMPTS=200 # 200 × 15s = 50 minutes — the runner is sequen
echo -e "${BLUE}9⃣ Waiting for CI to publish v${NEW_VERSION} to The Source registry (home)...${NC}" echo -e "${BLUE}9⃣ Waiting for CI to publish v${NEW_VERSION} to The Source registry (home)...${NC}"
SOURCE_LANDED=false SOURCE_LANDED=false
for ((attempt = 1; attempt <= SOURCE_POLL_MAX_ATTEMPTS; attempt++)); do for ((attempt = 1; attempt <= SOURCE_POLL_MAX_ATTEMPTS; attempt++)); do
LANDED_VERSION=$(npm view "@soulcraftlabs/brainy@${NEW_VERSION}" version "--@soulcraftlabs:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "") LANDED_VERSION=$(npm view "@soulcraft/brainy@${NEW_VERSION}" version "--@soulcraft:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "")
if [ "$LANDED_VERSION" = "$NEW_VERSION" ]; then if [ "$LANDED_VERSION" = "$NEW_VERSION" ]; then
SOURCE_LANDED=true SOURCE_LANDED=true
break break
@ -254,16 +232,62 @@ if [ "$SOURCE_LANDED" = true ]; then
echo -e "${GREEN}✅ CI published v${NEW_VERSION} to The Source${NC}\n" echo -e "${GREEN}✅ CI published v${NEW_VERSION} to The Source${NC}\n"
else else
echo -e "${RED}❌ CI's home publish did not land — check the workflow run on The Source; the pair must not diverge.${NC}" echo -e "${RED}❌ CI's home publish did not land — check the workflow run on The Source; the pair must not diverge.${NC}"
echo -e "${RED} v${NEW_VERSION} was tagged and pushed, but @soulcraftlabs/brainy@${NEW_VERSION} never became visible on the${NC}" echo -e "${RED} v${NEW_VERSION} was tagged and pushed, but @soulcraft/brainy@${NEW_VERSION} never became visible on the${NC}"
echo -e "${RED} Source registry after ${SOURCE_POLL_MAX_ATTEMPTS} attempts, ${SOURCE_POLL_INTERVAL_S}s apart. Aborting.${NC}" echo -e "${RED} Source registry after ${SOURCE_POLL_MAX_ATTEMPTS} attempts, ${SOURCE_POLL_INTERVAL_S}s apart. Aborting before npmjs.${NC}"
exit 1 exit 1
fi fi
if [ "$SOURCE_ONLY" = true ]; then
echo -e "${YELLOW}9⃣½ Storefront (npmjs) leg SKIPPED — --source-only: v${NEW_VERSION} lives on The Source under dist-tag '${NPM_TAG}' only${NC}\n"
else
echo -e "${BLUE}9⃣½ Publishing to npmjs (storefront, dist-tag: ${NPM_TAG})...${NC}"
# BYTE-IDENTITY LAW: the storefront republishes CI's EXACT artifact — download
# the tarball The Source serves and publish that file, never a fresh local pack
# (a local rebuild can differ byte-wise, and the fleet verifies the pair by
# shasum across registries).
STOREFRONT_TMP="$(mktemp -d)"
(cd "$STOREFRONT_TMP" && npm pack "@soulcraft/brainy@${NEW_VERSION}" "--@soulcraft:registry=${SOURCE_NPM_REG}" >/dev/null)
SOURCE_TARBALL="$(ls "$STOREFRONT_TMP"/soulcraft-brainy-*.tgz)"
echo -e "${BLUE} home artifact: $(sha256sum "$SOURCE_TARBALL" | cut -d' ' -f1)${NC}"
npm publish "$SOURCE_TARBALL" --tag "$NPM_TAG" "--@soulcraft:registry=https://registry.npmjs.org/"
rm -rf "$STOREFRONT_TMP"
# Brainy is the only PUBLIC @soulcraft package — verify visibility after every publish.
npm access get status @soulcraft/brainy "--@soulcraft:registry=https://registry.npmjs.org/" || true
# Verify the pair is byte-identical by registry-reported shasum — divergence
# here means the storefront leg must be treated as failed, loudly. RETRIED
# with raw curl: npmjs metadata propagates with a lag measured in minutes,
# and a one-shot npm-view probe fired a false DIVERGENCE on 10.0.0 while a
# raw curl of the registry document already confirmed byte-identity. The
# probe now reads the registry JSON directly (no npm cache in the path) and
# gives propagation up to 5 minutes before calling the pair divergent.
NPMJS_VERIFY_ATTEMPTS=20
NPMJS_VERIFY_INTERVAL_S=15 # 20 × 15s = 5 minutes of propagation grace
SOURCE_SHA=$(npm view "@soulcraft/brainy@${NEW_VERSION}" dist.shasum "--@soulcraft:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "source-unavailable")
PAIR_IDENTICAL=false
for ((attempt = 1; attempt <= NPMJS_VERIFY_ATTEMPTS; attempt++)); do
NPMJS_SHA=$(curl -fsSL "https://registry.npmjs.org/@soulcraft%2Fbrainy" 2>/dev/null \
| node -e "let d='';process.stdin.on('data',c=>d+=c).on('end',()=>{try{const v=JSON.parse(d).versions[process.argv[1]];console.log(v?v.dist.shasum:'')}catch{console.log('')}})" "${NEW_VERSION}" \
|| echo "")
if [ -n "$NPMJS_SHA" ] && [ "$SOURCE_SHA" = "$NPMJS_SHA" ]; then
PAIR_IDENTICAL=true
break
fi
echo -e "${YELLOW} … npmjs metadata not settled (attempt ${attempt}/${NPMJS_VERIFY_ATTEMPTS}: '${NPMJS_SHA:-absent}' vs '${SOURCE_SHA}'); retrying in ${NPMJS_VERIFY_INTERVAL_S}s${NC}"
sleep "$NPMJS_VERIFY_INTERVAL_S"
done
if [ "$PAIR_IDENTICAL" = true ]; then
echo -e "${GREEN}✅ Published to npmjs — byte-identical pair (shasum ${NPMJS_SHA})${NC}\n"
else
echo -e "${RED}❌ REGISTRY DIVERGENCE: The Source shasum ${SOURCE_SHA} != npmjs shasum ${NPMJS_SHA} after ${NPMJS_VERIFY_ATTEMPTS} attempts — investigate before announcing${NC}\n"
exit 1
fi
fi
# Step 11: Release object on The Source (presentational — the tag, CHANGELOG, # Step 11: Release object on The Source (presentational — the tag, CHANGELOG,
# and RELEASES.md are the record; this just gives The Source's UI a release page). # and RELEASES.md are the record; this just gives The Source's UI a release page).
echo -e "${BLUE}🔟 Creating release page on The Source...${NC}" echo -e "${BLUE}🔟 Creating release page on The Source...${NC}"
if [ -n "${FORGEJO_RELEASE_TOKEN:-}" ]; then if [ -n "${FORGEJO_RELEASE_TOKEN:-}" ]; then
if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraftlabs/open-brainy/releases" \ if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraft/brainy/releases" \
-H "Authorization: token ${FORGEJO_RELEASE_TOKEN}" -H "Content-Type: application/json" \ -H "Authorization: token ${FORGEJO_RELEASE_TOKEN}" -H "Content-Type: application/json" \
-d "{\"tag_name\":\"v${NEW_VERSION}\",\"name\":\"v${NEW_VERSION}\",\"prerelease\":${PRERELEASE}}" >/dev/null; then -d "{\"tag_name\":\"v${NEW_VERSION}\",\"name\":\"v${NEW_VERSION}\",\"prerelease\":${PRERELEASE}}" >/dev/null; then
echo -e "${GREEN}✅ Release page created on The Source${NC}\n" echo -e "${GREEN}✅ Release page created on The Source${NC}\n"
@ -274,15 +298,29 @@ else
echo -e "${RED}⚠️ FORGEJO_RELEASE_TOKEN unset — no release page created; tag + CHANGELOG remain the record${NC}\n" echo -e "${RED}⚠️ FORGEJO_RELEASE_TOKEN unset — no release page created; tag + CHANGELOG remain the record${NC}\n"
fi fi
# Step 12 RETIRED (2026-08-31, CORTEX-SITE-BRAINY-RENAME round 12, David-ruled): # Step 12: Push public docs to the soulcraft.com docs ingest door
# soulcraft.com/docs carries the paid product's documentation only. This # (VENUE-DOCS-RELEASE-PUSH). Skips with a loud warning when
# engine's documentation home is THIS repository — README and docs/ — and the # DOCS_INGEST_SECRET is unset; fails loudly (without undoing the publish —
# site serves 301s for the slugs this rail used to push. The push script stays # that already happened) when a push errors, so the docs site never
# in the tree for history; the rail no longer calls it. # silently trails npm.
echo -e "${BLUE}Docs step: this engine documents itself in its own repo (site push retired 2026-08-31)${NC}" if [ "$SOURCE_ONLY" = true ]; then
echo -e "${YELLOW}1⃣2⃣ Docs push SKIPPED — --source-only (a home-only prerelease publishes no public docs)${NC}\n"
else
echo -e "${BLUE}1⃣2⃣ Pushing public docs to soulcraft.com/docs...${NC}"
if node scripts/push-docs.js; then
echo -e "${GREEN}✅ Docs push step done${NC}\n"
else
echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n"
fi
fi
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}" echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}"
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo "" echo ""
echo -e "🏠 The Source: ${BLUE}https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v${NEW_VERSION}${NC}" if [ "$SOURCE_ONLY" = true ]; then
echo -e "📦 npmjs: ${YELLOW}not published (--source-only)${NC}"
else
echo -e "📦 npm: ${BLUE}https://www.npmjs.com/package/@soulcraft/brainy/v/${NEW_VERSION}${NC}"
fi
echo -e "🏠 The Source: ${BLUE}https://source.soulcraft.com/soulcraft/brainy/releases/tag/v${NEW_VERSION}${NC}"

View file

@ -1,539 +0,0 @@
#!/usr/bin/env node
/**
* @module scripts/wall-entry
* @description The releases-wall entry, made mechanical. The fleet's HQ page
* reads one public JSON per product from the ONE releases repo on The Source
* (soulcraftlabs/releases, files <product>.json at its root shape
* {product, entries:[{version, date, headline, items, url, thumb?}]}), at
* https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/<product>.json.
* Those entries were hand-written after every release, then briefly written
* into this repo's own releases/<product>.json; this script is the one door
* that composes an entry and lands it in the shared repo, so it is never
* hand-written and never forked across repos again.
*
* Two modes:
*
* 1. Generate + publish (default):
* node wall-entry.mjs --product <p> --version <v> --date <YYYY-MM-DD> \
* --from-changelog <CHANGELOG.md>
* Derives an entry from the CHANGELOG.md entry for <v> (headline = the
* entry's first bullet, items = every bullet, trimmed of its trailing
* commit hash), then:
* - clones (or, if a cached clone already exists, fetches and resets)
* the releases repo into a local cache directory,
* - prepends the entry to <cache>/<p>.json, newest first replacing
* any existing entry for the same version so a re-run is idempotent,
* - validates the file's shape before and after,
* - commits the change as "chore(wall): <p> <v>" and pushes main.
* A failure at any step (clone, validation, commit, push, a
* non-fast-forward remote) exits non-zero naming the cure. Nothing is
* ever skipped the wall either lands correctly or the release fails.
*
* 2. Dry run:
* node wall-entry.mjs --dry-run --product <p> --version <v> \
* --date <YYYY-MM-DD> --from-changelog <CHANGELOG.md>
* Derives the entry exactly as above and prints it, along with the file
* it would be written to, but touches no clone and no remote usable
* from a fresh checkout with no cache and no network.
*
* 3. Validate only (--check):
* node wall-entry.mjs --check --file <path/to/product.json>
* Validates an arbitrary wall file's exact key set (top-level and
* per-entry), field types, and strict-descending semver ordering with
* no duplicates. Read-only; never writes. Exit 0 = clean, exit 1 =
* named violations printed to stderr.
*
* The remote and the local cache directory are each overridable
* (--remote / --cache-dir, or WALL_ENTRY_RELEASES_REMOTE /
* WALL_ENTRY_RELEASES_CACHE_DIR) so tests can point at a throwaway local
* bare repo and a throwaway cache directory never the real remote or the
* real developer cache.
*
* No dependencies beyond the system `git` binary CHANGELOG parsing,
* semver comparison, and JSON shape checking are all hand-rolled below.
*/
import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs'
import { execFileSync } from 'node:child_process'
import { homedir } from 'node:os'
import { dirname, join } from 'node:path'
const DEFAULT_REMOTE = 'git@source.soulcraft.com:soulcraftlabs/releases.git'
/** @returns {string} */
function defaultCacheDir() {
const base = process.env.XDG_CACHE_HOME || join(homedir(), '.cache')
return join(base, 'soulcraft-releases')
}
// Required on every entry; "thumb" is optional (may be absent, or present as
// string | null) — matching the HQ contract's {..., thumb?}.
const ENTRY_REQUIRED_KEYS = ['version', 'date', 'headline', 'items', 'url']
const ENTRY_OPTIONAL_KEYS = ['thumb']
const ENTRY_ALLOWED_KEYS = [...ENTRY_REQUIRED_KEYS, ...ENTRY_OPTIONAL_KEYS]
const FILE_KEYS = ['product', 'entries']
// The public permalink pattern, by product. Every entry MUST carry an https
// permalink: HQ's parser rejects a wall whose entries carry url: null (the
// whole feed became unreadable on 2026-09-02). A product whose forge repo is
// private links its PUBLIC package page on The Source instead of a release
// page that would 404 for HQ's readers.
const RELEASE_URL_PATTERNS = {
'open-brainy': (version) => `https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v${version}`,
'brainy': (version) => `https://source.soulcraft.com/soulcraft/-/packages/npm/@soulcraft%2Fbrainy/${version}`,
}
/**
* Parse argv into a flag map. `--flag value` sets a string; `--flag` alone
* (end of argv, or followed by another `--flag`) sets boolean true.
* @param {string[]} argv
* @returns {Record<string, string | true>}
*/
function parseArgs(argv) {
/** @type {Record<string, string | true>} */
const args = {}
for (let i = 0; i < argv.length; i++) {
const a = argv[i]
if (!a.startsWith('--')) continue
const key = a.slice(2)
const next = argv[i + 1]
if (next === undefined || next.startsWith('--')) {
args[key] = true
} else {
args[key] = next
i++
}
}
return args
}
/**
* Print a loud, named error and exit 1. Every refusal in this script goes
* through here so the failure mode is always the same shape: "wall-entry: <what>".
* @param {string} message
* @returns {never}
*/
function fail(message) {
console.error(`wall-entry: ${message}`)
process.exit(1)
}
/**
* @param {string} version
* @returns {{major: number, minor: number, patch: number, pre: string | null} | null}
*/
function parseSemver(version) {
const m = /^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$/.exec(version)
if (!m) return null
return { major: Number(m[1]), minor: Number(m[2]), patch: Number(m[3]), pre: m[4] ?? null }
}
/**
* @param {string} a
* @param {string} b
* @returns {number} positive if a > b, negative if a < b, 0 if equal.
*/
function compareSemver(a, b) {
const pa = parseSemver(a)
const pb = parseSemver(b)
if (!pa || !pb) throw new Error(`cannot compare non-semver versions "${a}" vs "${b}"`)
if (pa.major !== pb.major) return pa.major - pb.major
if (pa.minor !== pb.minor) return pa.minor - pb.minor
if (pa.patch !== pb.patch) return pa.patch - pb.patch
if (pa.pre === pb.pre) return 0
if (pa.pre === null) return 1 // a release outranks any prerelease of the same core version
if (pb.pre === null) return -1
return pa.pre < pb.pre ? -1 : pa.pre > pb.pre ? 1 : 0
}
/**
* Validate a wall file's full shape: top-level keys ("product", "entries"
* no more, no less), per-entry keys and field types ("thumb" optional), and
* strict-descending semver ordering with no duplicates. Collects every
* violation instead of failing on the first, so a caller reports the whole
* picture in one pass.
* @param {unknown} data
* @returns {string[]} Violation messages; empty means the file is clean.
*/
function validateShape(data) {
/** @type {string[]} */
const errors = []
if (typeof data !== 'object' || data === null || Array.isArray(data)) {
return ['top level: expected a JSON object']
}
const obj = /** @type {Record<string, unknown>} */ (data)
const topKeys = Object.keys(obj)
const missingTop = FILE_KEYS.filter((k) => !(k in obj))
const extraTop = topKeys.filter((k) => !FILE_KEYS.includes(k))
if (missingTop.length) errors.push(`top level: missing key(s) ${missingTop.join(', ')}`)
if (extraTop.length) errors.push(`top level: unexpected key(s) ${extraTop.join(', ')}`)
if (typeof obj.product !== 'string' || obj.product.trim() === '') {
errors.push('top level: "product" must be a non-empty string')
}
if (!Array.isArray(obj.entries)) {
errors.push('top level: "entries" must be an array')
return errors // nothing further to check without an array
}
const entries = /** @type {unknown[]} */ (obj.entries)
entries.forEach((rawEntry, i) => {
const label = `entries[${i}]`
if (typeof rawEntry !== 'object' || rawEntry === null || Array.isArray(rawEntry)) {
errors.push(`${label}: expected an object`)
return
}
const entry = /** @type {Record<string, unknown>} */ (rawEntry)
const keys = Object.keys(entry)
const missing = ENTRY_REQUIRED_KEYS.filter((k) => !(k in entry))
const extra = keys.filter((k) => !ENTRY_ALLOWED_KEYS.includes(k))
if (missing.length) errors.push(`${label}: missing key(s) ${missing.join(', ')}`)
if (extra.length) errors.push(`${label}: unexpected key(s) ${extra.join(', ')}`)
if (typeof entry.version !== 'string' || !parseSemver(entry.version)) {
errors.push(`${label}: "version" must be a semver string (got ${JSON.stringify(entry.version)})`)
}
if (typeof entry.date !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(entry.date) || Number.isNaN(Date.parse(entry.date))) {
errors.push(`${label}: "date" must be a YYYY-MM-DD string (got ${JSON.stringify(entry.date)})`)
}
if (typeof entry.headline !== 'string' || entry.headline.trim() === '') {
errors.push(`${label}: "headline" must be a non-empty string`)
}
if (!Array.isArray(entry.items) || entry.items.length === 0 || entry.items.some((it) => typeof it !== 'string' || it.trim() === '')) {
errors.push(`${label}: "items" must be a non-empty array of non-empty strings`)
}
if (typeof entry.url !== 'string' || !/^https:\/\/\S+$/.test(entry.url)) {
errors.push(`${label}: "url" must be an https permalink — never null; HQ's parser rejects the whole feed`)
}
if ('thumb' in entry && !(entry.thumb === null || typeof entry.thumb === 'string')) {
errors.push(`${label}: "thumb" must be a string or null when present`)
}
})
// Ordering: newest first, strictly descending, no duplicate versions —
// checked only over entries whose version parsed (a bad version is
// already reported above; comparing it too would just be noise).
const versioned = entries
.map((e, i) => ({ i, version: /** @type {any} */ (e)?.version }))
.filter((e) => typeof e.version === 'string' && parseSemver(e.version))
for (let i = 0; i < versioned.length - 1; i++) {
const a = versioned[i]
const b = versioned[i + 1]
const cmp = compareSemver(a.version, b.version)
if (cmp === 0) {
errors.push(`entries[${a.i}] and entries[${b.i}]: duplicate version ${a.version}`)
} else if (cmp < 0) {
errors.push(`entries[${a.i}] (${a.version}) sits above entries[${b.i}] (${b.version}) — not newest-first`)
}
}
return errors
}
/**
* Extract one version's entry body from a standard-version-style CHANGELOG.md
* (headings `### [version](url) (date)`, followed by `- bullet (hash)` lines
* until the next heading or EOF).
* @param {string} changelog
* @param {string} version
* @returns {string[]} Bullet lines, trimmed of their leading "- " and
* trailing " (hash)".
*/
function extractChangelogBullets(changelog, version) {
const lines = changelog.split('\n')
const headingRe = /^### \[([^\]]+)\]\(.*\)\s*\(\d{4}-\d{2}-\d{2}\)\s*$/
let start = -1
for (let i = 0; i < lines.length; i++) {
const m = headingRe.exec(lines[i])
if (m && m[1] === version) {
start = i + 1
break
}
}
if (start === -1) {
fail(
`version ${version} has no CHANGELOG entry yet — run this after the CHANGELOG step composes "### [${version}]", not before`,
)
}
/** @type {string[]} */
const bullets = []
for (let i = start; i < lines.length; i++) {
if (headingRe.test(lines[i])) break // next entry starts
const bulletMatch = /^- (.+?)(?:\s\(([0-9a-f]{6,40})\))?$/.exec(lines[i].trim())
if (lines[i].trim().startsWith('- ') && bulletMatch) {
const text = bulletMatch[1].trim()
if (text) bullets.push(text)
}
}
if (bullets.length === 0) {
fail(`version ${version}'s CHANGELOG entry has no bullets to derive a headline/items from`)
}
return bullets
}
/**
* Derive a wall entry from a CHANGELOG.md.
* @param {{product: string, version: string, date: string, changelogPath: string, url?: string, thumb?: string | null}} opts
* @returns {{version: string, date: string, headline: string, items: string[], url: string, thumb: string | null}}
*/
function deriveEntry({ product, version, date, changelogPath, url, thumb }) {
if (!parseSemver(version)) fail(`--version "${version}" is not a semver string`)
if (!/^\d{4}-\d{2}-\d{2}$/.test(date) || Number.isNaN(Date.parse(date))) {
fail(`--date "${date}" is not a YYYY-MM-DD date`)
}
if (!existsSync(changelogPath)) fail(`--from-changelog "${changelogPath}" does not exist`)
const changelog = readFileSync(changelogPath, 'utf8')
const items = extractChangelogBullets(changelog, version)
const headline = items[0]
const pattern = RELEASE_URL_PATTERNS[product]
if (url === undefined && pattern === undefined) {
throw new Error(`wall-entry: no permalink pattern for product "${product}" — add one to RELEASE_URL_PATTERNS or pass --url; entries never carry url: null`)
}
const resolvedUrl = url !== undefined ? url : pattern(version)
const resolvedThumb = thumb !== undefined ? thumb : null
return { version, date, headline, items, url: resolvedUrl, thumb: resolvedThumb }
}
/**
* Load and shape-validate a wall file.
* @param {string} filePath
* @returns {Record<string, any>}
*/
function loadWallFile(filePath) {
if (!existsSync(filePath)) fail(`"${filePath}" does not exist`)
/** @type {unknown} */
let data
try {
data = JSON.parse(readFileSync(filePath, 'utf8'))
} catch (err) {
fail(`"${filePath}" is not valid JSON: ${/** @type {Error} */ (err).message}`)
}
const errors = validateShape(data)
if (errors.length) {
fail(`"${filePath}" fails shape validation —\n ${errors.join('\n ')}`)
}
return /** @type {Record<string, any>} */ (data)
}
/**
* Run a git command, throwing an Error whose message is git's own stderr
* (trimmed) on failure every caller wraps this to name the cure.
* @param {string[]} args
* @param {string} cwd
* @returns {string} stdout, trimmed.
*/
function git(args, cwd) {
try {
return execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim()
} catch (err) {
const stderr = /** @type {any} */ (err).stderr
const message = (typeof stderr === 'string' && stderr.trim()) || /** @type {Error} */ (err).message
throw new Error(message)
}
}
/**
* Resolve the git identity for the wall commit from the repository the rail
* is actually running in the developer's own checkout (`process.cwd()`;
* `release.sh` invokes this script from the repo root with no `cd`), via
* git's normal config precedence (repo-local, then global, then system).
* Never guessed and never left to git's own "who are you?" prompt: a host
* with no configured identity anywhere (a bare CI box, say) must refuse
* loudly rather than have git manufacture a placeholder identity or hang.
* @returns {{name: string, email: string}}
*/
function resolveWallCommitIdentity() {
const repo = process.cwd()
let name = ''
let email = ''
try {
name = git(['config', 'user.name'], repo)
} catch {
name = ''
}
try {
email = git(['config', 'user.email'], repo)
} catch {
email = ''
}
if (!name || !email) {
fail('no git identity for the wall commit — set user.name/user.email')
}
return { name, email }
}
/**
* Ensure a clean, up-to-date local clone of the releases repo at
* `cacheDir`, checked out on `main` cloning fresh if `cacheDir` has no
* `.git`, otherwise fetching and hard-resetting onto `origin/main` (so a
* stray local commit or edit left by a previous failed run can never leak
* into the next one).
* @param {string} remote
* @param {string} cacheDir
*/
function ensureReleasesClone(remote, cacheDir) {
if (existsSync(join(cacheDir, '.git'))) {
try {
git(['remote', 'set-url', 'origin', remote], cacheDir)
git(['fetch', '--prune', 'origin'], cacheDir)
git(['checkout', 'main'], cacheDir)
git(['reset', '--hard', 'origin/main'], cacheDir)
git(['clean', '-fd'], cacheDir)
} catch (err) {
fail(
`cannot refresh the cached releases checkout at "${cacheDir}" from "${remote}" — ${/** @type {Error} */ (err).message}\n` +
` cure: delete "${cacheDir}" and re-run so it re-clones from scratch, or confirm SSH access with "ssh -T git@source.soulcraft.com"`,
)
}
return
}
mkdirSync(dirname(cacheDir), { recursive: true })
try {
git(['clone', remote, cacheDir], dirname(cacheDir))
} catch (err) {
fail(
`cannot clone "${remote}" — ${/** @type {Error} */ (err).message}\n` +
` cure: confirm SSH access with "ssh -T git@source.soulcraft.com" and that the soulcraftlabs/releases repo exists yet`,
)
}
try {
git(['checkout', 'main'], cacheDir)
} catch (err) {
fail(
`cloned "${remote}" into "${cacheDir}" but could not check out "main" — ${/** @type {Error} */ (err).message}\n` +
` cure: confirm the releases repo's default branch is named "main"`,
)
}
}
/**
* Prepend `entry` to the wall at `<cacheDir>/<product>.json`, replacing any
* existing entry for the same version (idempotent re-runs), validating
* before and after, committing, and pushing or refusing loudly, naming
* the cure, at whichever step fails.
* @param {{version: string, date: string, headline: string, items: string[], url: string, thumb: string | null}} entry
* @param {string} product
* @param {string} remote
* @param {string} cacheDir
*/
function publishEntry(entry, product, remote, cacheDir) {
ensureReleasesClone(remote, cacheDir)
const filePath = join(cacheDir, `${product}.json`)
if (!existsSync(filePath)) {
fail(
`"${filePath}" does not exist in the releases repo — cure: seed "${product}.json" at the repo root first (it must exist before any release rail can prepend to it)`,
)
}
const wall = loadWallFile(filePath)
if (wall.product !== product) {
fail(`"${filePath}" has product "${wall.product}", but --product "${product}" was given — refusing a cross-product write`)
}
const replacing = wall.entries.some((e) => e.version === entry.version)
wall.entries = [entry, ...wall.entries.filter((e) => e.version !== entry.version)]
const postErrors = validateShape(wall)
if (postErrors.length) {
fail(`the entry for ${entry.version} would leave "${filePath}" invalid —\n ${postErrors.join('\n ')}`)
}
writeFileSync(filePath, JSON.stringify(wall, null, 2) + '\n', 'utf8')
const status = git(['status', '--porcelain', '--', `${product}.json`], cacheDir)
if (status === '') {
console.log(`wall-entry: "${product}.json" already carries an identical entry for ${entry.version} — nothing to commit or push`)
return
}
const identity = resolveWallCommitIdentity()
try {
git(['add', `${product}.json`], cacheDir)
git(
['-c', `user.name=${identity.name}`, '-c', `user.email=${identity.email}`, 'commit', '-m', `chore(wall): ${product} ${entry.version}`],
cacheDir,
)
} catch (err) {
fail(`cannot commit the wall entry in "${cacheDir}" — ${/** @type {Error} */ (err).message}\n cure: inspect "${cacheDir}" by hand and re-run once its git state is clean`)
}
try {
git(['push', 'origin', 'main'], cacheDir)
} catch (err) {
fail(
`push to "${remote}" failed (likely a non-fast-forward — another release landed on main first) — ${/** @type {Error} */ (err).message}\n` +
` cure: re-run this release step; it re-fetches and resets onto the latest origin/main before retrying`,
)
}
const sha = git(['rev-parse', 'HEAD'], cacheDir)
console.log(
`wall-entry: ${replacing ? 'replaced' : 'wrote'} v${entry.version} in "${product}.json" (${wall.entries.length} entries, newest first) — pushed ${sha} to ${remote} main`,
)
}
function main() {
const args = parseArgs(process.argv.slice(2))
if (args.check) {
const filePath = /** @type {string | undefined} */ (args.file)
if (!filePath) fail('--check needs --file <path>')
const wall = loadWallFile(/** @type {string} */ (filePath))
console.log(`wall-entry --check: "${filePath}" OK — product "${wall.product}", ${wall.entries.length} entries, newest-first, no duplicates`)
process.exit(0)
}
// Generate mode (default, also covers --dry-run): --product, --version,
// --date, --from-changelog required.
const product = /** @type {string | undefined} */ (args.product)
const version = /** @type {string | undefined} */ (args.version)
const date = /** @type {string | undefined} */ (args.date)
const fromChangelog = /** @type {string | undefined} */ (args['from-changelog'])
const missing = []
if (!product) missing.push('--product')
if (!version) missing.push('--version')
if (!date) missing.push('--date')
if (!fromChangelog) missing.push('--from-changelog')
if (missing.length) {
fail(
`missing required flag(s): ${missing.join(', ')}\n` +
'Usage:\n' +
' wall-entry.mjs --product <p> --version <v> --date <YYYY-MM-DD> --from-changelog <CHANGELOG.md> [--dry-run]\n' +
' wall-entry.mjs --check --file <path/to/product.json>',
)
}
const urlArg = args.url === true ? undefined : /** @type {string | undefined} */ (args.url)
const thumbArg = args.thumb === true ? undefined : /** @type {string | undefined} */ (args.thumb)
const entry = deriveEntry({
product: /** @type {string} */ (product),
version: /** @type {string} */ (version),
date: /** @type {string} */ (date),
changelogPath: /** @type {string} */ (fromChangelog),
url: urlArg,
thumb: thumbArg,
})
const remote = /** @type {string} */ (args.remote ?? process.env.WALL_ENTRY_RELEASES_REMOTE ?? DEFAULT_REMOTE)
const cacheDir = /** @type {string} */ (args['cache-dir'] ?? process.env.WALL_ENTRY_RELEASES_CACHE_DIR ?? defaultCacheDir())
if (args['dry-run']) {
console.log(`wall-entry --dry-run: would write to "${join(cacheDir, `${product}.json`)}" in ${remote} (main), pushed as "chore(wall): ${product} ${version}"`)
console.log(JSON.stringify(entry, null, 2))
process.exit(0)
}
publishEntry(entry, /** @type {string} */ (product), remote, cacheDir)
}
main()

File diff suppressed because it is too large Load diff

View file

@ -872,27 +872,6 @@ export interface StorageAdapter {
*/ */
noteVectorLanded?(id: string): Promise<void> noteVectorLanded?(id: string): Promise<void>
/**
* OPTIONAL narrow ledger hook, the mirror of {@link noteVectorLanded}:
* record that a canonical noun's vector was just REMOVED rewritten from
* a real (non-empty) vector to the "unvectored" empty-array shape. Exists
* for the ONE sanctioned reverse migration this engine supports: the VFS
* root's zero-norm fix (see `VirtualFileSystem.doInitializeRoot()` and
* `Brainy.unvectorNounForRootMigration()`), which rewrites a pre-fix
* store's all-zero placeholder root vector to `[]` and must decrement
* `vectors.all` through this hook so the coverage ledger never drifts.
* NOT a general-purpose "I removed a vector" callback ordinary
* application data has no sanctioned path from vectored back to
* unvectored (`update()` refuses an empty vector as a dimension
* mismatch by design). Callers MUST call this only when the noun held a
* REAL vector immediately before this write (the caller already holds
* that fact for free, from its own pre-write read never an added read).
* A backend without vectored-noun tracking is a no-op via this method's
* absence (feature-detected).
* @param id - The noun whose vector was just removed.
*/
noteVectorUnlanded?(id: string): Promise<void>
/** /**
* Get noun with metadata combined * Get noun with metadata combined
* @returns Combined HNSWNounWithMetadata or null * @returns Combined HNSWNounWithMetadata or null

View file

@ -28,7 +28,7 @@
* speculative `with()` overlay; the canonical storage walk only ever answers * speculative `with()` overlay; the canonical storage walk only ever answers
* "what is live right now." * "what is live right now."
* *
* All are exported from the package root (`@soulcraftlabs/brainy`). * All are exported from the package root (`@soulcraft/brainy`).
*/ */
/** /**
@ -351,63 +351,3 @@ export class PendingFlushDurabilityError extends Error {
this.failedAttempts = failedAttempts this.failedAttempts = failedAttempts
} }
} }
/**
* @description Thrown by {@link GenerationStore.commitTransaction} when the
* PENDING single-op tier is non-empty i.e. one or more `commitSingleOp()`
* generations are buffered in memory, not yet flushed to
* `committedRanges` via `flushPendingSingleOps()`.
*
* The invariant `reservedGensAsc()` (and everything built on it
* `resolveManyAt`, `resolveAt`, `changedBetween`, the hot-tail window) relies
* on is documented, not enforced by types: pending generations must always be
* numerically greater than every committed one, because the ONLY sanctioned
* callers of `commitTransaction()` `Brainy.transact()` and
* `Brainy.compactHistory()` flush the pending tier FIRST. A caller that
* invokes `commitTransaction()` directly while single-ops are still pending
* breaks that invariant: the new commit lands in `committedRanges` ABOVE
* generations still sitting in `pendingGens`, so the committed-then-pending
* concatenation `reservedGensAsc()` yields is no longer ascending. The
* concrete failure this produces is silent, not a crash: `resolveManyAt`
* walks committed ranges before pending ones, so it can report a NEWER
* generation as the "first after" a pin than an older, still-pending one that
* actually touched the id first a wrong before-image at a point-in-time
* read, without a compensating error to warn a caller anything went wrong.
*
* This error refuses the commit outright, before any staging I/O: nothing is
* written, the generation counter reservation is untouched, and
* `committedRanges`/`pendingGens` are exactly as they were. Call
* `flushPendingSingleOps()` first (or go through `Brainy.transact()`, which
* already does).
*
* @example
* try {
* await generationStore.commitTransaction({ touched, execute })
* } catch (err) {
* if (err instanceof PendingSingleOpsUnflushedError) {
* await generationStore.flushPendingSingleOps()
* await generationStore.commitTransaction({ touched, execute }) // now safe
* }
* }
*/
export class PendingSingleOpsUnflushedError extends Error {
/** How many un-flushed single-op generations were buffered at refusal time. */
public readonly pendingCount: number
/**
* @param pendingCount - `pendingGens.length` at the moment of refusal (always 1).
*/
constructor(pendingCount: number) {
super(
`commitTransaction() refused: ${pendingCount} pending single-op generation(s) ` +
`are still buffered and un-flushed. Flush the pending single-op tier before ` +
`committing a transaction — Brainy.transact() does this automatically; a ` +
`direct commitTransaction() call with pending generations would leave the ` +
`generation order unsorted (committed generations landing above lower, ` +
`still-pending ones) and make point-in-time reads (resolveManyAt/resolveAt) ` +
`return the wrong before-image. Call flushPendingSingleOps() first, then retry.`
)
this.name = 'PendingSingleOpsUnflushedError'
this.pendingCount = pendingCount
}
}

View file

@ -40,10 +40,7 @@
* The manifest (`_generations/facts/manifest.json`, JSON forensics stay * The manifest (`_generations/facts/manifest.json`, JSON forensics stay
* terminal-readable) is the single source of truth for the segment SET; * terminal-readable) is the single source of truth for the segment SET;
* rotation flips it atomically (write-new fsync rename) BEFORE the new * rotation flips it atomically (write-new fsync rename) BEFORE the new
* tail's first byte exists, so no segment file is ever unaccounted for. Its * tail's first byte exists, so no segment file is ever unaccounted for.
* per-segment `firstGeneration`/`lastGeneration` are LOAD-BEARING at open: a
* recovery pass looking for facts above a bound reads only the segments those
* bounds cannot rule out (the prune law see `segmentsHoldingFactsAbove`).
* *
* ## Mixed-version logs (the v2 live-write cutover) * ## Mixed-version logs (the v2 live-write cutover)
* *
@ -692,74 +689,6 @@ function parseSegment(
return { facts, validBytes: offset, formatVersion: FACT_LOG_FORMAT_V1 } return { facts, validBytes: offset, formatVersion: FACT_LOG_FORMAT_V1 }
} }
/**
* THE PRUNE LAW which segment files a pass looking for facts ABOVE
* `committedGeneration` actually has to read, and how many the manifest's own
* recorded bounds took off the table.
*
* A sealed segment's `lastGeneration` is written at SEAL time and never
* mutated upward afterwards ({@link FactLog.rotate}, unchanged since the log
* was introduced): the tail's bytes are fsynced FIRST (`await this.sync()`
* "sealed segments are always fully durable"), the entry is then built from
* the content that fsync covered, and only then does the manifest flip
* atomically (tmp+rename) and fsynced which in the SAME write re-points
* `tailSegment` at a new file, so the sealed file is never appended to again.
* A crash anywhere in that order is safe in the pruning direction: crash
* before the manifest write and the segment is still the TAIL (read whole);
* crash after it and the entry describes bytes that were already durable. The
* only later mutation of a sealed segment is `open()`'s straddle truncation,
* which REMOVES facts and re-derives the entry from the actual bytes so a
* recorded bound can drift DOWN with its file, never up.
*
* Therefore: `lastGeneration = L` proves the file holds no fact above L, and
* a pass above `committedGeneration >= L` can skip it whole no read, no
* CRC decode, no msgpack. What the manifest cannot PROVE is never pruned: an
* entry with no numeric `lastGeneration` (a legacy or hand-repaired manifest)
* is read, and the unsealed tail is always read.
*
* This is the difference between an open that costs O(whole fact log) and one
* that costs O(the facts that could matter). MEASURED in production: a 16k-row
* brain at generation ~478,819 paid 34-37s of segment reads and CRC decoding
* in `generation-store-open-fold` on EVERY open to answer a question whose
* answer, after a clean close, is always "nothing".
*/
function segmentsHoldingFactsAbove(
stored: FactsManifest,
committedGeneration: number
): { files: string[]; pruned: number } {
const files: string[] = []
let pruned = 0
for (const entry of stored.segments) {
const last = (entry as Partial<SegmentEntry>).lastGeneration
if (typeof last === 'number' && Number.isFinite(last) && last <= committedGeneration) {
pruned++
continue
}
files.push(entry.file)
}
if (stored.tailSegment) files.push(stored.tailSegment)
return { files, pruned }
}
/**
* Say what the open actually read. One line, and only when the log holds more
* than one segment (a single-segment log has nothing to prune and nothing to
* report) the operator's receipt that the open is paying for the tail, not
* for the whole history.
*/
function narrateAboveScan(
pass: string,
committedGeneration: number,
read: number,
pruned: number
): void {
if (read + pruned <= 1) return
prodLog.narrate(
`[FactLog] ${pass} above generation ${committedGeneration}: ${read} segment(s) read, ` +
`${pruned} pruned of ${read + pruned} (sealed at or below the bound)`
)
}
/** /**
* The generation fact log. One instance per open store; every method assumes * The generation fact log. One instance per open store; every method assumes
* the single-writer discipline the generation store already enforces (calls * the single-writer discipline the generation store already enforces (calls
@ -825,6 +754,22 @@ export class FactLog {
return this.manifest.brainId !== undefined || this.tailVersion === FACT_LOG_FORMAT_V2 return this.manifest.brainId !== undefined || this.tailVersion === FACT_LOG_FORMAT_V2
} }
/**
* Open the log and reconcile it to committed truth: read the manifest,
* establish the tail's intact content (torn-tail scan), then TRUNCATE any
* fact with `generation > committedGeneration` those never committed (a
* crash between fact-append and the commit point). After open, the log is
* exactly the committed prefix.
*/
/**
* Read (without truncating) every intact fact ABOVE a generation the
* log-authority recovery surface: after a crash, facts beyond the
* manifest watermark that survived with valid CRCs are ACKED writes in
* durable-at-ack mode, and the owner REPLAYS them instead of letting
* open() truncate them. Must be called BEFORE open() (it reads the raw
* segments directly; the torn tail's invalid suffix is ignored exactly
* like open() would).
*/
/** /**
* STREAMING twin of {@link FactLog.peekFactsAbove} for the recovery fold: * STREAMING twin of {@link FactLog.peekFactsAbove} for the recovery fold:
* yields facts above the bound one SEGMENT at a time, ascending, without * yields facts above the bound one SEGMENT at a time, ascending, without
@ -834,18 +779,13 @@ export class FactLog {
* Works manifest-direct (safe before {@link FactLog.open}). Ordering is * Works manifest-direct (safe before {@link FactLog.open}). Ordering is
* structural (segments rotate in order; appends are ordered within one) and * structural (segments rotate in order; appends are ordered within one) and
* ASSERTED a violation aborts loudly, never a silent misordered replay. * ASSERTED a violation aborts loudly, never a silent misordered replay.
*
* Reads only the segments that CAN hold a fact above the bound see
* {@link segmentsHoldingFactsAbove}. A bounded fold above a high checkpoint
* therefore reads its own tail, not the whole history it already proved
* durable.
*/ */
async *streamFactsAbove(committedGeneration: number): AsyncGenerator<CommitFact[], void> { async *streamFactsAbove(committedGeneration: number): AsyncGenerator<CommitFact[], void> {
const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null
if (!stored || typeof stored !== 'object' || !Array.isArray(stored.segments)) return if (!stored || typeof stored !== 'object' || !Array.isArray(stored.segments)) return
if (stored.formatVersion !== FACTS_FORMAT_VERSION) return if (stored.formatVersion !== FACTS_FORMAT_VERSION) return
const { files, pruned } = segmentsHoldingFactsAbove(stored, committedGeneration) const files = [...stored.segments.map((s) => s.file)]
narrateAboveScan('recovery fold', committedGeneration, files.length, pruned) if (stored.tailSegment) files.push(stored.tailSegment)
let lastGen = committedGeneration let lastGen = committedGeneration
for (const file of files) { for (const file of files) {
const bytes = await this.storage.readRawBytes(`${FACTS_PREFIX}/${file}`) const bytes = await this.storage.readRawBytes(`${FACTS_PREFIX}/${file}`)
@ -867,27 +807,13 @@ export class FactLog {
} }
} }
/**
* Read (without truncating) every intact fact ABOVE a generation the
* log-authority recovery surface: after a crash, facts beyond the
* manifest watermark that survived with valid CRCs are ACKED writes in
* durable-at-ack mode, and the owner REPLAYS them instead of letting
* open() truncate them. Must be called BEFORE open() (it reads the raw
* segments directly; the torn tail's invalid suffix is ignored exactly
* like open() would).
*
* Reads only the segments that CAN hold such a fact see
* {@link segmentsHoldingFactsAbove}. This runs on EVERY log-authority open,
* including the clean one where the answer is always empty, so the segments
* the manifest already proves irrelevant are never opened at all.
*/
async peekFactsAbove(committedGeneration: number): Promise<CommitFact[]> { async peekFactsAbove(committedGeneration: number): Promise<CommitFact[]> {
const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null
if (!stored || typeof stored !== 'object' || !Array.isArray(stored.segments)) return [] if (!stored || typeof stored !== 'object' || !Array.isArray(stored.segments)) return []
if (stored.formatVersion !== FACTS_FORMAT_VERSION) return [] if (stored.formatVersion !== FACTS_FORMAT_VERSION) return []
const out: CommitFact[] = [] const out: CommitFact[] = []
const { files, pruned } = segmentsHoldingFactsAbove(stored, committedGeneration) const files = [...stored.segments.map((s) => s.file)]
narrateAboveScan('above-manifest peek', committedGeneration, files.length, pruned) if (stored.tailSegment) files.push(stored.tailSegment)
for (const file of files) { for (const file of files) {
const bytes = await this.storage.readRawBytes(`${FACTS_PREFIX}/${file}`) const bytes = await this.storage.readRawBytes(`${FACTS_PREFIX}/${file}`)
if (bytes === null) continue if (bytes === null) continue
@ -900,13 +826,6 @@ export class FactLog {
return out return out
} }
/**
* Open the log and reconcile it to committed truth: read the manifest,
* establish the tail's intact content (torn-tail scan), then TRUNCATE any
* fact with `generation > committedGeneration` those never committed (a
* crash between fact-append and the commit point). After open, the log is
* exactly the committed prefix.
*/
async open(committedGeneration: number): Promise<void> { async open(committedGeneration: number): Promise<void> {
const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null
if (stored && typeof stored === 'object' && Array.isArray(stored.segments)) { if (stored && typeof stored === 'object' && Array.isArray(stored.segments)) {

View file

@ -12,11 +12,9 @@
* the verified surface is a small set of rollup invariants (entity/ * the verified surface is a small set of rollup invariants (entity/
* relationship counts) plus `sourceGeneration`. * relationship counts) plus `sourceGeneration`.
* *
* `sourceGeneration` is the COMMITTED generation of the source-of-truth log * `sourceGeneration` is the generation of the source-of-truth log this
* this projection reflects never the allocated counter, which names a * projection reflects open-time coherence becomes a COMPARISON (stamp vs
* generation that may never commit (see {@link StampVerdict.torn}) so * log head), not a walk:
* open-time coherence becomes a COMPARISON (stamp vs committed head), not a
* walk:
* *
* - equal + invariants hold coherent, serve. * - equal + invariants hold coherent, serve.
* - behind the projection missed the tail (crash between commit and stamp); * - behind the projection missed the tail (crash between commit and stamp);
@ -26,9 +24,6 @@
* - invariants FAIL at equal generation genuine incoherence: loud, and the * - invariants FAIL at equal generation genuine incoherence: loud, and the
* repair ritual (`repairIndex()`, whose recount rebuilds the rollups from a * repair ritual (`repairIndex()`, whose recount rebuilds the rollups from a
* canonical walk) heals it. * canonical walk) heals it.
* - AHEAD a torn generation-log tail: the stamp's fsync outlived the log
* tail's. TERMINAL, never a wait the generation the stamp names does not
* exist to arrive.
* *
* Stamps are JSON on purpose every incident gets debugged by reading a * Stamps are JSON on purpose every incident gets debugged by reading a
* stamp in a terminal. * stamp in a terminal.
@ -75,12 +70,6 @@ export type StampVerdict =
| { state: 'coherent' } | { state: 'coherent' }
| { state: 'absent' } // legacy store — first stamp writes at the next flush | { state: 'absent' } // legacy store — first stamp writes at the next flush
| { state: 'behind'; stampSource: number; head: number } | { state: 'behind'; stampSource: number; head: number }
/**
* TORN GENERATION-LOG TAIL: the stamp witnesses a source generation the
* store's committed watermark can no longer show. TERMINAL there is no
* generation to wait for, so the open demotes (or refuses) and never spins.
*/
| { state: 'torn'; stampSource: number; head: number }
| { state: 'incoherent'; failures: string[] } | { state: 'incoherent'; failures: string[] }
| { state: 'unverifiable'; reason: string } // a FAULT reading the stamp — never conflated with absence | { state: 'unverifiable'; reason: string } // a FAULT reading the stamp — never conflated with absence
@ -129,15 +118,12 @@ export function verifyFamilyStamp(
): StampVerdict { ): StampVerdict {
if (stamp === null) return { state: 'absent' } if (stamp === null) return { state: 'absent' }
if (stamp.sourceGeneration > head) { if (stamp.sourceGeneration > head) {
// A stamp AHEAD of committed truth witnesses a generation the store can no // A stamp AHEAD of the log claims state that never committed — the
// longer show: the stamp's fsync survived a crash that the log tail did // projection was stamped against truth that a crash rolled back.
// not. This is the TORN GENERATION-LOG TAIL — its own class, never folded return {
// in with `incoherent` (a count that drifted at a generation both sides state: 'incoherent',
// agree on), because the two have opposite cures: incoherence is recounted, failures: [`sourceGeneration ${stamp.sourceGeneration} is ahead of the log head ${head}`]
// a tear is DEMOTED. It is also terminal by construction — there is no }
// generation the open can wait for, because the one the stamp names is
// gone.
return { state: 'torn', stampSource: stamp.sourceGeneration, head }
} }
if (stamp.sourceGeneration < head) { if (stamp.sourceGeneration < head) {
return { state: 'behind', stampSource: stamp.sourceGeneration, head } return { state: 'behind', stampSource: stamp.sourceGeneration, head }

View file

@ -147,60 +147,6 @@ export class GenerationSegmentStore {
return this.coveringSegment(gen) !== null return this.coveringSegment(gen) !== null
} }
/**
* @description True when `meta` declares more generations than it holds
* frames a segment sealed by a writer that folded across a hole. The
* manifest records `frames` at fold time, so this is an O(1) comparison
* against the declared span and needs no I/O.
*/
private isSparse(meta: SegmentMeta): boolean {
return meta.lastGeneration - meta.firstGeneration + 1 !== meta.frames
}
/**
* @description The generations this tier ACTUALLY holds, as coalesced
* ascending intervals not what the segments declare.
*
* Dense segments (every one a current writer produces) contribute their
* declared range with no I/O. A SPARSE segment one sealed before the
* density law was enforced, whose declared range spans generations it has
* no frame for has its real generation list read from its sidecar and
* contributed instead, with the discrepancy narrated once.
*
* This is what keeps a store that already carries the damage from wedging.
* `open()` seeds `committedRanges` from these intervals, so a hole is never
* re-admitted as a committed generation, and the auto-compaction pass that
* used to fail on every run with "packed history is damaged" simply never
* asks for the missing frame.
*
* @returns Ascending, non-overlapping `[first, last]` intervals.
*/
async actualRanges(): Promise<Array<[number, number]>> {
const out: Array<[number, number]> = []
for (const meta of this.manifest.segments) {
if (!this.isSparse(meta)) {
out.push([meta.firstGeneration, meta.lastGeneration])
continue
}
const missing = meta.lastGeneration - meta.firstGeneration + 1 - meta.frames
prodLog.warn(
`[GenerationSegments] sealed segment ${meta.file} declares generations ` +
`${meta.firstGeneration}..${meta.lastGeneration} but holds only ${meta.frames} ` +
`frame(s) — ${missing} generation(s) in that span were never folded into it. ` +
`Serving the frames it actually holds; the declared span is not treated as ` +
`committed history. (Written by a pre-density-law writer that folded across a ` +
`gap; the segment itself is intact and no record is lost.)`
)
const idx = await this.sidecarFor(meta)
for (const [gen] of idx.generations) {
const last = out[out.length - 1]
if (last !== undefined && gen === last[1] + 1) last[1] = gen
else out.push([gen, gen])
}
}
return out
}
/** /**
* Fold consecutive generations into ONE new sealed segment + sidecar and * Fold consecutive generations into ONE new sealed segment + sidecar and
* append it to the manifest atomically. Caller guarantees: `gens` is * append it to the manifest atomically. Caller guarantees: `gens` is
@ -218,38 +164,6 @@ export class GenerationSegmentStore {
throw new Error('[GenerationSegments] fold() input must be strictly ascending') throw new Error('[GenerationSegments] fold() input must be strictly ascending')
} }
} }
// THE DENSITY LAW, MADE MECHANICAL.
//
// A sealed segment declares a CONTIGUOUS range [firstGeneration,
// lastGeneration] and every reader treats that range as containment:
// `coveringSegment` is an interval test, `hasGeneration` returns true for
// anything inside it, and `open()` seeds committedRanges from it. So a
// segment folded from a SPARSE input silently claims generations it does
// not hold, and the first read of one of those holes throws
// "inside sealed segment ... but has no frame — packed history is damaged".
//
// That is exactly how the damage was produced. `repackHistory` skipped
// generations mid-batch — ones absent from committedRanges, ones still in
// the pending buffer, ones whose tx.json would not read — and handed the
// survivors here, where the range was computed from the first and last of
// them. Worse, the mis-declared range was then merged back into
// committedRanges at the next open, which is what turned a quiet hole into
// a repeating auto-compaction failure on every subsequent run.
//
// Callers now split at discontinuities; this refusal is what keeps any
// future caller from reintroducing the class. A refusal here loses
// nothing — the generations stay in the live tier, readable, and the next
// pass folds them correctly.
for (let i = 1; i < gens.length; i++) {
if (gens[i].generation !== gens[i - 1].generation + 1) {
throw new Error(
`[GenerationSegments] fold() input is not contiguous: ${gens[i - 1].generation}` +
`${gens[i].generation} skips ${gens[i].generation - gens[i - 1].generation - 1} ` +
`generation(s). A sealed segment declares a dense range, so folding a sparse ` +
`batch would claim generations it does not hold. Split the batch at the gap.`
)
}
}
const last = this.manifest.segments[this.manifest.segments.length - 1] const last = this.manifest.segments[this.manifest.segments.length - 1]
if (last && gens[0].generation <= last.lastGeneration) { if (last && gens[0].generation <= last.lastGeneration) {
throw new Error( throw new Error(
@ -450,37 +364,12 @@ export class GenerationSegmentStore {
return this.decodeFrame(payload) return this.decodeFrame(payload)
} }
} }
// Inside the covering range but with no frame. Two very different causes, // In the covering range but not present: the packed tier is dense by
// and conflating them is what made this class wedge every maintenance pass // construction (fold packs every generation it is handed, including
// on the affected stores. // record-less ones) — absence inside a sealed range is damage.
//
// (1) A SPARSE SEGMENT — the manifest's own `frames` count is smaller than
// the span it declares. That segment was sealed by a writer that
// folded across a hole (the class this file's density law now bars).
// The segment is INTACT and nothing is lost; it simply never held this
// generation. Answering "not packed" is the honest answer, and it lets
// the caller's two-tier read decide what a genuinely absent generation
// means, instead of every compaction pass dying on a repeating throw.
// `actualRanges()` keeps such holes out of committedRanges at open, so
// in a healed store nobody asks this question in the first place.
//
// (2) A DENSE SEGMENT missing a frame it says it has — the manifest and
// the sidecar disagree about a segment that claims to be complete.
// That IS damage, and it stays loud.
if (this.isSparse(meta)) {
prodLog.warn(
`[GenerationSegments] generation ${gen} falls inside sealed segment ${meta.file}'s ` +
`declared range ${meta.firstGeneration}..${meta.lastGeneration}, but that segment ` +
`holds ${meta.frames} frame(s) for a ${meta.lastGeneration - meta.firstGeneration + 1}` +
`-generation span — it was sealed across a gap and never held this generation. ` +
`Reporting it as unpacked rather than as damage; no record is lost.`
)
return null
}
throw new Error( throw new Error(
`[GenerationSegments] generation ${gen} is inside sealed segment ${meta.file}'s declared ` + `[GenerationSegments] generation ${gen} is inside sealed segment ${meta.file}'s declared ` +
`range but has no frame, and that segment declares a complete ${meta.frames}-frame ` + `range but has no frame — packed history is damaged`
`span — the manifest and the sidecar disagree; packed history is damaged`
) )
} }

View file

@ -32,13 +32,7 @@
*/ */
import { prodLog } from '../utils/logger.js' import { prodLog } from '../utils/logger.js'
import { import { GenerationCompactedError, GenerationConflictError, PendingFlushDurabilityError, StoreInconsistentError } from './errors.js'
GenerationCompactedError,
GenerationConflictError,
PendingFlushDurabilityError,
PendingSingleOpsUnflushedError,
StoreInconsistentError
} from './errors.js'
import type { UnreconciledRecord } from './errors.js' import type { UnreconciledRecord } from './errors.js'
import { TransactionRollbackError } from '../transaction/errors.js' import { TransactionRollbackError } from '../transaction/errors.js'
import type { import type {
@ -102,35 +96,6 @@ export const FOLD_CHECKPOINT_PATH = '_system/fold-checkpoint.json'
/** Storage-root-relative prefix of the per-generation record directories. */ /** Storage-root-relative prefix of the per-generation record directories. */
export const GENERATIONS_PREFIX = '_generations' export const GENERATIONS_PREFIX = '_generations'
/**
* @description Split an ascending list of fold candidates into maximal
* CONTIGUOUS runs `[7,8,9,12,13]` becomes `[[7,8,9],[12,13]]`.
*
* A sealed segment declares one dense range `[firstGeneration,
* lastGeneration]`, and every reader treats that range as containment. So a
* batch with a hole in it must never become one segment: it would claim a
* generation it does not hold, and the first read of that hole reports the
* packed history as damaged. One run, one segment the ranges then describe
* exactly what the segments contain.
*
* @param gens - Fold candidates, strictly ascending by generation.
* @returns One array per contiguous run, in ascending order. Empty in, empty out.
*/
export function contiguousRuns(gens: FoldGeneration[]): FoldGeneration[][] {
const runs: FoldGeneration[][] = []
let run: FoldGeneration[] = []
for (const g of gens) {
const prev = run[run.length - 1]
if (prev !== undefined && g.generation !== prev.generation + 1) {
runs.push(run)
run = []
}
run.push(g)
}
if (run.length > 0) runs.push(run)
return runs
}
/** /**
* @description Phases of the {@link GenerationStore.commitTransaction} commit * @description Phases of the {@link GenerationStore.commitTransaction} commit
* protocol at which a test-only fault injector can simulate a process crash. * protocol at which a test-only fault injector can simulate a process crash.
@ -572,29 +537,12 @@ export class GenerationStore {
this.horizonGen = finiteGen(manifest?.horizon, 'manifest horizon') this.horizonGen = finiteGen(manifest?.horizon, 'manifest horizon')
this.counter = Math.max(finiteGen(counterFile?.generation, 'generation counter'), this.committed) this.counter = Math.max(finiteGen(counterFile?.generation, 'generation counter'), this.committed)
// Discover existing generation record directories — BY DIRECTORY NAME. // Discover existing generation record directories.
// This used to call listRawObjects(), which recurses the whole const recordPaths = await this.storage.listRawObjects(GENERATIONS_PREFIX)
// `_generations/` tree and returns every file in every generation, to
// extract a set of integers the top-level directory names already spell.
// MEASURED on a real store with an 11 GB generation history: the phase
// this sits in cost 55,538 ms of a WARM REOPEN after a clean close, with
// no fold to blame — this walk is what it was doing. An adapter without
// the one-level door falls back to the recursive listing, unchanged.
const seenGens = new Set<number>() const seenGens = new Set<number>()
const oneLevel = ( for (const p of recordPaths) {
this.storage as { listRawPrefixes?: (prefix: string) => Promise<string[]> } const gen = parseGenerationFromPath(p)
).listRawPrefixes if (gen !== null) seenGens.add(gen)
if (typeof oneLevel === 'function') {
for (const name of await oneLevel.call(this.storage, GENERATIONS_PREFIX)) {
const gen = Number(name)
if (Number.isSafeInteger(gen) && gen >= 0) seenGens.add(gen)
}
} else {
const recordPaths = await this.storage.listRawObjects(GENERATIONS_PREFIX)
for (const p of recordPaths) {
const gen = parseGenerationFromPath(p)
if (gen !== null) seenGens.add(gen)
}
} }
let rolledBack = 0 let rolledBack = 0
@ -704,56 +652,21 @@ export class GenerationStore {
: 'WHOLE-LOG fold' : 'WHOLE-LOG fold'
: 'above-manifest replay' : 'above-manifest replay'
let replayed = 0 let replayed = 0
const foldStartedAt = Date.now()
const replayFact = async (fact: CommitFact): Promise<void> => { const replayFact = async (fact: CommitFact): Promise<void> => {
for (const op of fact.ops) { for (const op of fact.ops) {
let image: { metadata: unknown | null; vector: unknown | null } const image =
if (op.record === null) { op.record === null
// A genuine tombstone (both legs absent) — the fold removes ? { metadata: null, vector: null }
// both legs, exactly like `writeNounRaw`/`writeVerbRaw`'s raw : { metadata: op.record.metadata, vector: op.record.vector }
// exact-restore contract.
image = { metadata: null, vector: null }
} else if (
op.record.metadata !== null &&
(op.record.vector === null || op.record.vector === undefined)
) {
// PRESERVE-IF-ABSENT (population law, ADR-008 G1 — the fold's
// half): a metadata-only after-image must never DELETE an
// existing vector leg through the fold. `writeNounRaw`/
// `writeVerbRaw` are exact-restore primitives — a `vector:
// null` there means "delete", which is exactly right for
// `rollBackUncommittedGeneration`'s before-image restore (a
// transaction abort legitimately un-writes a vector the failed
// transaction added). It is NOT right here: this fold replays
// AFTER-IMAGES, and re-applying an already-intact record must
// be byte-safe (this module's own invariant, see the log-authority
// comment above) — silently erasing a landed vector because one
// replayed fact's vector leg came back null is the exact defect
// that left metadata-counted, never-enumerated rows in a
// production store (confirmed root cause: the enumeration walk
// used to key on the vector leg, so a preserved-but-then-deleted
// vector made the row invisible while the ledger still counted
// it by metadata). A genuine "unvector" has its own sanctioned,
// ledger-correct path (`Brainy.unvectorNounForRootMigration`) —
// never this raw primitive, and never the fold.
const current =
op.kind === 'verb'
? await this.storage.readVerbRaw(op.id)
: await this.storage.readNounRaw(op.id)
image = { metadata: op.record.metadata, vector: current.vector ?? null }
} else {
image = { metadata: op.record.metadata, vector: op.record.vector }
}
if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image) if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image)
else await this.storage.writeNounRaw(op.id, image) else await this.storage.writeNounRaw(op.id, image)
this.noteCheckpointDirty(op.kind, op.id) this.noteCheckpointDirty(op.kind, op.id)
} }
replayed++ replayed++
if (replayed % 1000 === 0) { if (replayed % 1000 === 0) {
prodLog.narrate( prodLog.warn(
`[GenerationStore] recovery fold in progress — ${replayed} fact(s) folded ` + `[GenerationStore] recovery fold in progress — ${replayed} fact(s) folded ` +
`in ${Date.now() - foldStartedAt}ms (at generation ${fact.generation}); ` + `(at generation ${fact.generation}); do not restart, the fold is finite`
`do not restart, the fold is finite`
) )
} }
if (fact.generation > this.committed) { if (fact.generation > this.committed) {
@ -768,7 +681,7 @@ export class GenerationStore {
} }
} }
if (uncleanOpen) { if (uncleanOpen) {
prodLog.narrate( prodLog.warn(
`[GenerationStore] log-authority recovery: ${foldKind} beginning ` + `[GenerationStore] log-authority recovery: ${foldKind} beginning ` +
`(unclean shutdown detected) — streaming replay, bounded memory, ` + `(unclean shutdown detected) — streaming replay, bounded memory, ` +
`progress every 1000 facts. Do not restart the process; a restart ` + `progress every 1000 facts. Do not restart the process; a restart ` +
@ -791,10 +704,9 @@ export class GenerationStore {
} }
await this.storage.writeRawObject(MANIFEST_PATH, manifest) await this.storage.writeRawObject(MANIFEST_PATH, manifest)
await this.storage.syncRawObjects([MANIFEST_PATH]) await this.storage.syncRawObjects([MANIFEST_PATH])
prodLog.narrate( prodLog.warn(
`[GenerationStore] log-authority recovery replayed ${replayed} fact(s) into ` + `[GenerationStore] log-authority recovery replayed ${replayed} fact(s) into ` +
`canonical in ${Date.now() - foldStartedAt}ms (${foldKind}; committed at ` + `canonical (${foldKind}; committed at ${this.committed}) — an acked write is never lost`
`${this.committed}) — an acked write is never lost`
) )
} }
// A recovery fold re-applied (and the barrier below re-syncs) every // A recovery fold re-applied (and the barrier below re-syncs) every
@ -805,16 +717,7 @@ export class GenerationStore {
if (uncleanOpen) await this.advanceFoldCheckpointUnlocked() if (uncleanOpen) await this.advanceFoldCheckpointUnlocked()
// The marker is consumed: any session that can write invalidates it // The marker is consumed: any session that can write invalidates it
// at first commit (see the commit paths); a clean close re-writes it. // at first commit (see the commit paths); a clean close re-writes it.
// A READER NEVER CONSUMES IT. The marker is the writer's own evidence await this.clearCleanShutdownMarker()
// about the writer's own process — clearing it here exists so that
// if THIS session goes on to write and then dies before its next
// clean close, the marker's absence correctly reads as unclean. A
// reader can never write, so it can never leave the store in a state
// its own crash would mis-describe; clearing the marker for it would
// only cost the store's actual writer a needless whole-log fold on
// its next open, for a generation the reader merely observed. Leave
// `_system/` exactly as found.
if (!options?.readOnly) await this.clearCleanShutdownMarker()
} }
await this.factLog.open(this.committed) await this.factLog.open(this.committed)
} else { } else {
@ -828,15 +731,9 @@ export class GenerationStore {
if (storageSupportsFactLog(this.storage)) { if (storageSupportsFactLog(this.storage)) {
this.segments = new GenerationSegmentStore(this.storage) this.segments = new GenerationSegmentStore(this.storage)
await this.segments.open() await this.segments.open()
// ACTUAL ranges, not declared ones. A segment sealed by a pre-density-law const packedRanges = this.segments
// writer can declare a span wider than the frames it holds; seeding .segments()
// committedRanges from the declared span re-admits those holes as .map((s): [number, number] => [s.firstGeneration, Math.min(s.lastGeneration, this.committed)])
// committed generations, and every later maintenance pass then asks for a
// frame that was never written. `actualRanges()` reads the real
// generation list from the sidecar for exactly those segments (and does
// no I/O for the dense ones, which is all of them on a healthy store).
const packedRanges = (await this.segments.actualRanges())
.map((r): [number, number] => [r[0], Math.min(r[1], this.committed)])
.filter(([lo, hi]) => lo <= hi) .filter(([lo, hi]) => lo <= hi)
if (packedRanges.length > 0) { if (packedRanges.length > 0) {
// Merge packed (older) + live (newer) interval sets — both ascending; // Merge packed (older) + live (newer) interval sets — both ascending;
@ -904,11 +801,7 @@ export class GenerationStore {
} }
} }
/** /** Consume the clean-shutdown marker (every open; a clean close re-writes it). */
* Consume the clean-shutdown marker (every WRITER open; a clean close
* re-writes it). Callers must gate this on `!options.readOnly` a reader
* never consumes the marker, see the call site in {@link open}.
*/
private async clearCleanShutdownMarker(): Promise<void> { private async clearCleanShutdownMarker(): Promise<void> {
try { try {
await this.storage.deleteRawObject(CLEAN_SHUTDOWN_PATH) await this.storage.deleteRawObject(CLEAN_SHUTDOWN_PATH)
@ -1370,9 +1263,6 @@ export class GenerationStore {
* @param args.execute - Runs the planned operation batch atomically. * @param args.execute - Runs the planned operation batch atomically.
* @returns The committed generation and its commit timestamp. * @returns The committed generation and its commit timestamp.
* @throws GenerationConflictError when the CAS expectation fails. * @throws GenerationConflictError when the CAS expectation fails.
* @throws PendingSingleOpsUnflushedError when the pending single-op tier is
* non-empty call `flushPendingSingleOps()` first (both `Brainy.transact()`
* and `Brainy.compactHistory()` already do).
*/ */
/** /**
* The generation fact log, or `null` when the storage layer cannot host one. * The generation fact log, or `null` when the storage layer cannot host one.
@ -1447,13 +1337,6 @@ export class GenerationStore {
execute: () => Promise<void> execute: () => Promise<void>
}): Promise<{ generation: number; timestamp: number }> { }): Promise<{ generation: number; timestamp: number }> {
return this.withMutex(async () => { return this.withMutex(async () => {
// The generation-order guard (see assertPendingSingleOpsFlushed): a
// direct commitTransaction() call while single-ops are still pending
// would commit above them, unsorting reservedGensAsc() and corrupting
// point-in-time reads. Both sanctioned callers (Brainy.transact(),
// Brainy.compactHistory()) already flush first, so this is
// behavior-neutral on every real path.
this.assertPendingSingleOpsFlushed()
// A latched history-durability failure compromises the whole generation // A latched history-durability failure compromises the whole generation
// chain — refuse a transact too (advancing the manifest past stuck, // chain — refuse a transact too (advancing the manifest past stuck,
// un-durable single-op generations would be inconsistent). Same loud // un-durable single-op generations would be inconsistent). Same loud
@ -2323,37 +2206,6 @@ export class GenerationStore {
} }
} }
/**
* @description Throw if the pending single-op tier is non-empty. Called at
* the top of {@link commitTransaction} (the ONLY method that appends a
* fresh commit directly into {@link committedRanges} outside recovery) so
* the ordering invariant {@link reservedGensAsc}'s own doc comment states
* "pending generations are always greater than every committed one" is
* ENFORCED there rather than merely assumed.
*
* That invariant holds today only because both sanctioned callers flush the
* pending tier before committing: `Brainy.transact()` (src/brainy.ts,
* `await this.generationStore.flushPendingSingleOps()` immediately before
* its `commitTransaction()` call) and `Brainy.compactHistory()`
* (src/brainy.ts, the same flush immediately before its `compact()` call
* `compact()` itself only ever RECLAIMS an existing committed prefix, so it
* cannot land a commit out of order and needs no guard of its own). A
* caller that reaches `commitTransaction()` by any other path bypassing
* that flush would commit a new generation into `committedRanges` ABOVE
* generations still sitting in `pendingGens`, breaking `reservedGensAsc`'s
* "committed-then-pending is already sorted" assumption and making
* `resolveManyAt`'s single ascending pass (and `resolveAt`'s consumers)
* return the WRONG before-image for a point-in-time read silently, no
* compensating error. Refusing here, before any staging I/O, keeps the
* store untouched (nothing committed, nothing staged, the generation
* counter reservation unaffected) on every path that already flushes.
*/
private assertPendingSingleOpsFlushed(): void {
if (this.pendingGens.length > 0) {
throw new PendingSingleOpsUnflushedError(this.pendingGens.length)
}
}
/** Schedule a coalesced pending-tier flush (size trigger fires immediately on /** Schedule a coalesced pending-tier flush (size trigger fires immediately on
* the next microtask; otherwise a {@link PENDING_FLUSH_DELAY_MS} timer). Both * the next microtask; otherwise a {@link PENDING_FLUSH_DELAY_MS} timer). Both
* defer outside the current mutex section so the flush can re-acquire it. A * defer outside the current mutex section so the flush can re-acquire it. A
@ -2437,13 +2289,6 @@ export class GenerationStore {
* committed-then-pending concatenation is already sorted identical to the old * committed-then-pending concatenation is already sorted identical to the old
* `[...committedGens, ...pendingGens]`. This is the union historical reads * `[...committedGens, ...pendingGens]`. This is the union historical reads
* resolve over so un-flushed single-ops are visible to pins/`asOf`. * resolve over so un-flushed single-ops are visible to pins/`asOf`.
*
* The "flush first" half of that invariant is ENFORCED, not just documented:
* {@link commitTransaction} the only method that lands a fresh commit into
* {@link committedRanges} outside crash recovery refuses via
* {@link assertPendingSingleOpsFlushed} whenever {@link pendingGens} is
* non-empty, so a committed generation can never land above a still-pending
* one and break this ordering.
*/ */
private *reservedGensAsc(): IterableIterator<number> { private *reservedGensAsc(): IterableIterator<number> {
yield* this.committedGensAsc() yield* this.committedGensAsc()
@ -3223,26 +3068,13 @@ export class GenerationStore {
foldInput.push({ generation: gen, timestamp: delta.timestamp, delta, records }) foldInput.push({ generation: gen, timestamp: delta.timestamp, delta, records })
} }
if (foldInput.length === 0) continue if (foldInput.length === 0) continue
// SPLIT AT DISCONTINUITIES. `eligible` is NOT contiguous — three await segments.fold(foldInput)
// filters above punch holes in it: a generation missing from segmentsCreated++
// committedRanges never appears, one still in the pending buffer is // Segment + manifest durable → the live copies retire.
// skipped, and one whose tx.json will not read is skipped. A sealed for (const g of foldInput) {
// segment declares a DENSE range, so folding across such a hole makes await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`)
// the segment claim a generation it does not hold; the next open
// merges that mis-declared range into committedRanges, and every
// subsequent auto-compaction pass then asks for the missing frame and
// fails with "packed history is damaged". Fold each contiguous RUN as
// its own segment instead — same bytes, honest ranges.
for (const run of contiguousRuns(foldInput)) {
if (deadline !== undefined && Date.now() >= deadline) break
await segments.fold(run)
segmentsCreated++
// Segment + manifest durable → the live copies retire.
for (const g of run) {
await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`)
}
folded += run.length
} }
folded += foldInput.length
} }
if (folded > 0) { if (folded > 0) {
prodLog.info( prodLog.info(

View file

@ -450,21 +450,6 @@ export interface GenerationStorage {
deleteRawObject(path: string): Promise<void> deleteRawObject(path: string): Promise<void>
/** List raw object paths under a prefix (normalized, `.gz`-stripped). */ /** List raw object paths under a prefix (normalized, `.gz`-stripped). */
listRawObjects(prefix: string): Promise<string[]> listRawObjects(prefix: string): Promise<string[]>
/**
* OPTIONAL: the IMMEDIATE child directory names under a prefix one level,
* no recursion, no file paths.
*
* Why it exists: discovering which generations are on disk needs only the
* top-level directory NAMES under `_generations/`, but the only door for it
* was `listRawObjects`, which recurses the whole tree and returns every file
* in every generation. On a store with a long history that is a full walk of
* the entire generation log, paid on EVERY open, to learn a set of integers
* the directory names already spell out.
*
* An adapter without this door keeps working the caller falls back to the
* recursive listing.
*/
listRawPrefixes?(prefix: string): Promise<string[]>
/** Remove every object under a prefix (and the directory itself on disk). */ /** Remove every object under a prefix (and the directory itself on disk). */
removeRawPrefix(prefix: string): Promise<void> removeRawPrefix(prefix: string): Promise<void>
/** Durability barrier: fsync the given object paths (no-op in memory). */ /** Durability barrier: fsync the given object paths (no-op in memory). */

View file

@ -128,7 +128,7 @@ async function loadBunAssets(): Promise<ModelAssets> {
} }
// Strategy 2: node_modules path relative to CWD (for installed packages) // Strategy 2: node_modules path relative to CWD (for installed packages)
const nmPath = './node_modules/@soulcraftlabs/brainy/assets/models/all-MiniLM-L6-v2' const nmPath = './node_modules/@soulcraft/brainy/assets/models/all-MiniLM-L6-v2'
pathsToTry.push([ pathsToTry.push([
`${nmPath}/model.safetensors`, `${nmPath}/model.safetensors`,
`${nmPath}/tokenizer.json`, `${nmPath}/tokenizer.json`,
@ -168,9 +168,9 @@ async function loadBunAssets(): Promise<ModelAssets> {
// If all strategies fail, provide helpful error message // If all strategies fail, provide helpful error message
throw new Error( throw new Error(
'Could not load model assets. For bun --compile, ensure model files are accessible:\n' + 'Could not load model assets. For bun --compile, ensure model files are accessible:\n' +
' Option 1: Keep node_modules/@soulcraftlabs/brainy/assets/ alongside your binary\n' + ' Option 1: Keep node_modules/@soulcraft/brainy/assets/ alongside your binary\n' +
' Option 2: Copy assets/ folder to your working directory\n' + ' Option 2: Copy assets/ folder to your working directory\n' +
' Option 3: Use --asset flag: bun build --compile --asset="./node_modules/@soulcraftlabs/brainy/assets/**/*"' ' Option 3: Use --asset flag: bun build --compile --asset="./node_modules/@soulcraft/brainy/assets/**/*"'
) )
} }
@ -190,7 +190,7 @@ async function loadNodeAssets(): Promise<ModelAssets> {
if (!fs.existsSync(assetsDir)) { if (!fs.existsSync(assetsDir)) {
throw new Error( throw new Error(
`Model assets not found: ${assetsDir}\n` + `Model assets not found: ${assetsDir}\n` +
`Ensure @soulcraftlabs/brainy is installed correctly.` `Ensure @soulcraft/brainy is installed correctly.`
) )
} }

View file

@ -405,73 +405,3 @@ export class MigrationInProgressError extends BrainyError {
} }
} }
} }
/**
* THE INDEXABLE-ARRAY BOUND. An array-valued metadata field indexes one posting
* per element, so an unbounded array is an unbounded write a 384-float
* embedding parked in the metadata bag would mint 384 postings for one row.
* The bound exists to keep that out of the index.
*
* 256 is hardcoded on purpose (the zero-config law: no knob). It sits far above
* every legitimate multi-value field the engine has seen tags, authors,
* categories, labels, keyword lists, participant lists and still below the
* narrowest embedding this engine will ever meet (384 dimensions, the smallest
* model it ships), so the two populations do not overlap and no caller has to
* tune it. A vector parked in metadata is refused; a long keyword list is not.
*
* It replaces a limit of 10 that was applied SILENTLY: a row whose `tags` array
* held eleven entries had that field skipped entirely and dropped out of every
* filtered search on it, with no error, no warning and no way to tell the
* difference from "no row matches". A rule this consequential is a law with a
* name and a refusal, not a `continue`.
*
* This is the ONE place the number lives. Every message, warning, doc line and
* pin derives it from here never a literal.
*/
export const MAX_INDEXED_ARRAY_LENGTH = 256
/**
* A metadata field carries an array longer than {@link MAX_INDEXED_ARRAY_LENGTH}.
*
* Thrown at the WRITE door (`add` / `update` / `relate` / `updateRelation`), so
* the caller learns at the moment of writing that the field will not be
* searchable rather than discovering it later as rows that quietly fail to
* match. Carries the field, its length and the bound so a handler can report
* or repair without parsing the message.
*
* The cure is one of: store the long array outside the indexed bag (`data`
* carries arbitrary content and is not indexed element-wise); pass an embedding
* as the first-class `vector` parameter, which is where a vector belongs; or
* shorten the field to the values that are actually queried.
*/
export class MetadataArrayTooLargeError extends BrainyError {
/** The metadata field whose array is too long (its full dotted address). */
public readonly field: string
/** How many elements that array holds. */
public readonly length: number
/** The bound it exceeded — {@link MAX_INDEXED_ARRAY_LENGTH}. */
public readonly limit: number
constructor(site: string, field: string, length: number, limit: number) {
super(
`${site}: metadata field '${field}' holds ${length} array elements, ` +
`over the ${limit}-element indexing bound. An array field indexes one ` +
`posting per element, so an unbounded array is an unbounded write. ` +
`This write is refused rather than indexed partially or skipped silently ` +
`— a skipped field drops the row out of every filtered search on '${field}' ` +
`with no way to tell that from "nothing matched". ` +
`Cures: put the long array in 'data' (stored, not indexed element-wise); ` +
`pass an embedding as the first-class 'vector' parameter; or keep only ` +
`the values you actually query in '${field}'.`,
'VALIDATION',
false
)
this.name = 'MetadataArrayTooLargeError'
this.field = field
this.length = length
this.limit = limit
if (Error.captureStackTrace) {
Error.captureStackTrace(this, MetadataArrayTooLargeError)
}
}
}

View file

@ -14,7 +14,7 @@
* - {@link RelationNotFoundError} a referenced relationship (verb) does * - {@link RelationNotFoundError} a referenced relationship (verb) does
* not exist. * not exist.
* *
* Both are exported from the package root (`@soulcraftlabs/brainy`). * Both are exported from the package root (`@soulcraft/brainy`).
*/ */
/** /**

View file

@ -1052,17 +1052,6 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
*/ */
private startAutoFlush(): void { private startAutoFlush(): void {
this.flushTimer = setInterval(async () => { this.flushTimer = setInterval(async () => {
// NO PERIODIC WORK WITHOUT A CAUSE. Ask first, in two O(1) reads: an
// index nobody has written to since the last flush has nothing to
// write, and calling into the trees (and their logging) on a cadence
// over a quiet store is exactly the idle cost this law exists to
// remove.
if (
!this.lsmTreeVerbsBySource.hasPendingWrites() &&
!this.lsmTreeVerbsByTarget.hasPendingWrites()
) {
return
}
await this.flush() await this.flush()
}, this.config.flushInterval) }, this.config.flushInterval)
// Background maintenance must never keep the host process alive — // Background maintenance must never keep the host process alive —
@ -1105,31 +1094,13 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
} }
/** /**
* Stop the auto-flush interval WITHOUT writing anything. * Clean shutdown
*
* The non-writing half of {@link close}, for a shutdown that must leave the
* store byte-identical a read-only brain's close. `close()` itself is a
* writer: it drains both LSM MemTables to SSTables and stamps the watermark,
* which is exactly right for a writer and forbidden for a reader. A reader
* still has to release this interval, though: it is the one piece of this
* index that outlives the close and could fire against a store the session no
* longer owns.
*
* @returns Nothing.
*/ */
stopBackgroundFlush(): void { async close(): Promise<void> {
if (this.flushTimer) { if (this.flushTimer) {
clearInterval(this.flushTimer) clearInterval(this.flushTimer)
this.flushTimer = undefined this.flushTimer = undefined
} }
}
/**
* Clean shutdown drains both trees and stamps the watermark. THIS WRITES;
* a read-only brain must call {@link stopBackgroundFlush} instead.
*/
async close(): Promise<void> {
this.stopBackgroundFlush()
// Close both LSM-trees (will flush MemTables to SSTables) // Close both LSM-trees (will flush MemTables to SSTables)
if (this.initialized) { if (this.initialized) {

View file

@ -687,17 +687,6 @@ export class LSMTree {
} }
} }
/**
* @description Whether this tree holds anything a flush would write
* the MemTable is non-empty. Synchronous and O(1), so a background cadence
* can ask before it does anything at all: the engine does no periodic work
* without a cause.
* @returns true when a flush would write; false when it would be a no-op.
*/
hasPendingWrites(): boolean {
return !this.memTable.isEmpty()
}
async close(): Promise<void> { async close(): Promise<void> {
this.stopCompactionTimer() this.stopCompactionTimer()

View file

@ -10,7 +10,7 @@ import {
Vector, Vector,
VectorDocument VectorDocument
} from '../coreTypes.js' } from '../coreTypes.js'
import { euclideanDistance, calculateDistancesBatch, isZeroNormVector } from '../utils/index.js' import { euclideanDistance, calculateDistancesBatch } from '../utils/index.js'
import type { BaseStorage } from '../storage/baseStorage.js' import type { BaseStorage } from '../storage/baseStorage.js'
import { getGlobalCache, UnifiedCache } from '../utils/unifiedCache.js' import { getGlobalCache, UnifiedCache } from '../utils/unifiedCache.js'
import { prodLog } from '../utils/logger.js' import { prodLog } from '../utils/logger.js'
@ -64,34 +64,6 @@ export class HnswFlushError extends Error {
} }
} }
/**
* @description Thrown by {@link JsHnswVectorIndex.addItem} / {@link
* JsHnswVectorIndex.updateItem} when handed a length-0 vector. A length-0
* vector is the sanctioned "unvectored" shape for a canonical noun record
* (class-J: a VFS-system row, a deferred embed not yet landed, or any other
* legitimately-vector-less row) but it is NEVER a legal INDEX insert. The
* index itself has no concept of "unvectored"; deciding that a row is
* unvectored and therefore skippable is the FILL/REBUILD/LOAD consumer's job
* (see {@link JsHnswVectorIndex.rebuild}), done BEFORE ever calling addItem.
* A length-0 vector reaching this point is a caller bug: silently accepting
* it would pin `this.dimension = 0` on an empty index (poisoning every real
* insert thereafter with a dimension mismatch) or store a vector-less node
* that a distance calculation can never safely compare against. Loud errors,
* never quiet losses this throws instead of either.
*/
export class EmptyVectorIndexError extends Error {
constructor(public readonly id: string, operation: 'addItem' | 'updateItem') {
super(
`${operation}(${id}): refusing to index a length-0 vector — a length-0 vector is the ` +
`sanctioned "unvectored" shape for a canonical row, but it is never a legal index ` +
`insert. Callers that fill/rebuild/load the index must skip vector.length === 0 rows ` +
`themselves (unvectored = nothing to index, not an error at that layer); reaching ` +
`here with one is a caller bug.`
)
this.name = 'EmptyVectorIndexError'
}
}
/** /**
* Implements {@link VectorIndexProvider}: the vector-index surface Brainy calls * Implements {@link VectorIndexProvider}: the vector-index surface Brainy calls
* on whatever the `'vector'` factory returns (its own `JsHnswVectorIndex`, or a native * on whatever the `'vector'` factory returns (its own `JsHnswVectorIndex`, or a native
@ -608,15 +580,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
throw new Error('Vector is undefined or null') throw new Error('Vector is undefined or null')
} }
// THE INDEX REFUSES A LENGTH-0 VECTOR (see EmptyVectorIndexError's JSDoc):
// an empty vector is the sanctioned "unvectored" shape at the canonical
// layer, never a legal index member. Refusing here — loudly, before the
// dimension pin below — means no future fill/rebuild/load path can ever
// poison `this.dimension` to 0 or park a vector-less node in the graph.
if (vector.length === 0) {
throw new EmptyVectorIndexError(id, 'addItem')
}
// Set dimension on first insert // Set dimension on first insert
if (this.dimension === null) { if (this.dimension === null) {
this.dimension = vector.length this.dimension = vector.length
@ -991,13 +954,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
return return
} }
// Same refusal as addItem (see EmptyVectorIndexError's JSDoc) — an
// in-place relink must never rewrite an already-indexed node down to the
// unvectored shape or poison the pinned dimension.
if (vector.length === 0) {
throw new EmptyVectorIndexError(id, 'updateItem')
}
if (this.dimension === null) { if (this.dimension === null) {
this.dimension = vector.length this.dimension = vector.length
} else if (vector.length !== this.dimension) { } else if (vector.length !== this.dimension) {
@ -1599,15 +1555,7 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
} }
const loaded = await this.storage.getNounVector(noun.id) const loaded = await this.storage.getNounVector(noun.id)
// `loaded` is a length-0 array (not null/undefined) for a canonical row if (!loaded) {
// that is legitimately unvectored — `![]` is FALSE (an empty array is
// truthy), so the bare `!loaded` check below would silently accept it
// as "found" and hand a dimension-0 vector to a distance calculation.
// A node only reaches this lazy-load path because it is a MEMBER of
// the live index (rebuild() now refuses to admit unvectored rows — see
// its JSDoc), so an empty vector here is never legitimate: treat it
// exactly like "not found", loudly.
if (!loaded || loaded.length === 0) {
throw new Error(`Vector not found for noun ${noun.id}`) throw new Error(`Vector not found for noun ${noun.id}`)
} }
@ -1817,56 +1765,9 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
totalCount = result.totalCount || result.items.length totalCount = result.totalCount || result.items.length
// UNVECTORED ROWS ARE NOT AN INDEX MEMBER (the class-J law): a canonical
// noun whose vector leg is `[]` (a VFS-root-style system row, a
// deferred embed not yet landed, or a best-effort fallback for an
// unreadable vector leg) is a normal, enumerable, countable row — it
// is simply not indexed. `storage.getVectorIndexData()` derives its
// {level, connections} answer straight from the noun's OWN record, so
// it returns non-null for every existing noun regardless of whether
// that noun ever actually reached `addItem()` — it cannot be used to
// decide indexability. `nounData.vector.length === 0` is the one
// truthful signal (mirrors the `noun.vector.length > 0` guards in
// {@link getVectorSafe} / {@link getVectorSync}): skip here, counted
// once in a summary line, never per-row spam.
let skippedUnvectored = 0
// Process all nouns at once // Process all nouns at once
for (const nounData of result.items) { for (const nounData of result.items) {
try { try {
if (!Array.isArray(nounData.vector) || nounData.vector.length === 0) {
skippedUnvectored++
continue
}
// THE ZERO-NORM LAW — bulk-rebuild leg: a persisted zero-norm
// vector (a pre-10.4.2 row the canonical write has not yet
// normalized) must never enter the index either, mirroring the
// belt AddToVectorIndexOperation enforces on the live write path.
// Only the canonical vector is authoritative here — persisted
// HNSW graph metadata (level/connections) can outlive an unvector.
if (isZeroNormVector(nounData.vector)) {
prodLog.warn(
`[HNSW] rebuild(): skipping entity ${nounData.id} — persisted vector is ` +
`zero-norm (a zero-norm vector is not a vector and never crosses an ` +
`engine boundary)`
)
continue
}
// Restore the pinned dimension from the first real vector this
// rebuild loads. `addItem`/`updateItem` only pin `this.dimension`
// on a LIVE insert — a fresh rebuild from storage never goes
// through either, so without this the pin stays `null` across a
// restart. A `null` pin means the very next insert (correct OR
// wrong length) silently BECOMES the new pin instead of being
// checked against the store's real dimension — the wrong-length
// case then fails much later and less clearly, inside a distance
// calculation against an already-loaded node, instead of here,
// immediately, with a named expected-vs-got mismatch.
if (this.dimension === null) {
this.dimension = nounData.vector.length
}
// Load HNSW graph data for this entity // Load HNSW graph data for this entity
const hnswData = await this.storage.getVectorIndexData(nounData.id) const hnswData = await this.storage.getVectorIndexData(nounData.id)
@ -1914,10 +1815,7 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
options.onProgress(loadedCount, totalCount) options.onProgress(loadedCount, totalCount)
} }
prodLog.info( prodLog.info(`HNSW: Loaded ${loadedCount.toLocaleString()} nodes (${storageType})`)
`HNSW: Loaded ${loadedCount.toLocaleString()} nodes (${storageType})` +
(skippedUnvectored > 0 ? `${skippedUnvectored.toLocaleString()} unvectored row(s) skipped` : '')
)
} }
// Step 5: CRITICAL - Recover entry point if missing) // Step 5: CRITICAL - Recover entry point if missing)

View file

@ -184,7 +184,6 @@ export {
// Export version utilities // Export version utilities
export { getBrainyVersion } from './utils/version.js' export { getBrainyVersion } from './utils/version.js'
export { contractVersion, BRAINY_CONTRACT_VERSION } from './utils/version.js'
// Export plugin system // Export plugin system
export type { BrainyPlugin, BrainyPluginContext, StorageAdapterFactory } from './plugin.js' export type { BrainyPlugin, BrainyPluginContext, StorageAdapterFactory } from './plugin.js'
@ -203,7 +202,7 @@ export { EntityNotFoundError, RelationNotFoundError } from './errors/notFound.js
// Base error + typed migration-lock error — thrown by any data-plane call while a // Base error + typed migration-lock error — thrown by any data-plane call while a
// brain runs its one-time 7.x→8.0 upgrade; catch to answer HTTP 503 + Retry-After. // brain runs its one-time 7.x→8.0 upgrade; catch to answer HTTP 503 + Retry-After.
export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError, ProtectedArtifactError, DerivedArtifactMissingError, MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from './errors/brainyError.js' export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError, ProtectedArtifactError, DerivedArtifactMissingError } from './errors/brainyError.js'
export type { BrainyErrorType } from './errors/brainyError.js' export type { BrainyErrorType } from './errors/brainyError.js'
// ============= 8.0 Db API — generational MVCC ============= // ============= 8.0 Db API — generational MVCC =============
@ -231,8 +230,7 @@ export {
GenerationCompactedError, GenerationCompactedError,
StoreInconsistentError, StoreInconsistentError,
PendingFlushDurabilityError, PendingFlushDurabilityError,
CanonicalEnumerationUnavailableError, CanonicalEnumerationUnavailableError
PendingSingleOpsUnflushedError
} from './db/errors.js' } from './db/errors.js'
export type { UnreconciledRecord } from './db/errors.js' export type { UnreconciledRecord } from './db/errors.js'
export type { export type {

View file

@ -23,10 +23,7 @@ import type { ColumnStoreProvider, SegmentMeta } from './types.js'
import { import {
ValueType, ValueType,
DEFAULT_FLUSH_THRESHOLD, DEFAULT_FLUSH_THRESHOLD,
FLAG_MULTI_VALUE, FLAG_MULTI_VALUE
POSTING_KINDS,
KIND_PATH_SEGMENT,
type PostingKind
} from './types.js' } from './types.js'
import { ColumnTailBuffer } from './ColumnTailBuffer.js' import { ColumnTailBuffer } from './ColumnTailBuffer.js'
import { ColumnManifest } from './ColumnManifest.js' import { ColumnManifest } from './ColumnManifest.js'
@ -55,89 +52,10 @@ interface HeapEntry {
value: number | string value: number | string
entityIntId: number entityIntId: number
cursorIndex: number cursorIndex: number
/**
* Rank of the posting kind this entry came from, from {@link POSTING_KINDS}.
* A mixed-kind field has no natural total order, so the merge orders by kind
* first and by value within a kind.
*/
kindRank: number
/** Iterator for the cursor — call next() to advance */ /** Iterator for the cursor — call next() to advance */
iterator: Generator<CursorEntry> iterator: Generator<CursorEntry>
} }
/**
* One physical posting column: a (field, kind) pair and the key every internal
* map and every storage path uses for it.
*/
interface KindColumn {
/** The field as the query language names it. */
field: string
/** The kind of value this column holds. */
kind: PostingKind
/**
* Internal map / storage key. The field's PRIMARY kind uses the bare field
* name the historical layout and every other kind uses
* `<field>/<KIND_PATH_SEGMENT>/<kind>`.
*/
key: string
}
/**
* The KIND a value indexes under its JavaScript `typeof` class, not its
* storage encoding.
*
* Anything that is not a number, string or boolean indexes as a string, which
* is the `String(value)` treatment those values already received. `null` and
* `undefined` never reach here: `addEntity` skips them, and their absence is
* what the `exists` / `missing` operators read.
*
* @param value - The value about to be indexed or queried
* @returns The posting kind that owns this value
*/
function kindOfValue(value: unknown): PostingKind {
const t = typeof value
if (t === 'number') return 'number'
if (t === 'boolean') return 'boolean'
return 'string'
}
/**
* The segment encoding a fresh column of this kind starts with.
*
* Only the number kind has a choice: an integer column starts as i64 and
* widens to f64 the first time a non-integer arrives
* ({@link ColumnTailBuffer.promoteToFloat}).
*/
function initialValueTypeFor(kind: PostingKind, firstValue: unknown): ValueType {
switch (kind) {
case 'boolean':
return ValueType.Boolean
case 'string':
return ValueType.String
case 'number':
return Number.isInteger(firstValue) ? ValueType.Number : ValueType.Float
}
}
/**
* The kind a column of this encoding holds the inverse of
* {@link initialValueTypeFor}, used to read a kind back off a manifest written
* before typed postings existed.
*/
function kindOfValueType(valueType: ValueType): PostingKind {
switch (valueType) {
case ValueType.Boolean:
return 'boolean'
case ValueType.String:
return 'string'
case ValueType.Number:
case ValueType.Float:
return 'number'
default:
throw new Error(`Unknown ValueType: ${valueType}`)
}
}
/** /**
* Unified column store coordinator. * Unified column store coordinator.
* *
@ -203,19 +121,9 @@ export class ColumnStore implements ColumnStoreProvider {
*/ */
private deletedEntities: Map<string, RoaringBitmap32> = new Map() private deletedEntities: Map<string, RoaringBitmap32> = new Map()
/** Segment encoding per COLUMN key (not per field — a field has one per kind). */ /** Known field value types (inferred from first write). */
private fieldTypes: Map<string, ValueType> = new Map() private fieldTypes: Map<string, ValueType> = new Map()
/**
* Every posting column a field owns: field kind column key.
*
* This is the map that ends the first-writer type freeze. A field's first
* kind takes the bare field name as its column key, keeping the historical
* on-disk layout; each later kind takes its own column beside it. Nothing is
* coerced across kinds and nothing is dropped for being the wrong type.
*/
private fieldColumns: Map<string, Map<PostingKind, string>> = new Map()
/** Whether init() has completed. */ /** Whether init() has completed. */
private initialized = false private initialized = false
@ -232,128 +140,6 @@ export class ColumnStore implements ColumnStoreProvider {
this.l0CompactionTrigger = config?.l0CompactionTrigger ?? 4 this.l0CompactionTrigger = config?.l0CompactionTrigger ?? 4
} }
// =========================================================================
// Posting columns: (field, kind) → one physical column
// =========================================================================
/**
* Storage / map key for a (field, kind) column.
*
* `primary` is the kind that owns the bare field name. It is whichever kind
* the field saw first, which for an index written before typed postings is
* simply the kind of its single manifest so the historical layout is
* preserved rather than migrated.
*/
private static columnKeyFor(field: string, kind: PostingKind, primary: PostingKind | null): string {
return primary === null || kind === primary
? field
: `${field}/${KIND_PATH_SEGMENT}/${kind}`
}
/**
* Split a discovered manifest path back into its (field, kind) column, or
* `null` when the path names a field's primary column rather than a kind
* column. `<field>/k/<kind>` is the only shape that reads as a kind column,
* and only for a `<kind>` this version knows.
*/
private static parseKindColumnKey(key: string): { field: string; kind: PostingKind } | null {
const marker = `/${KIND_PATH_SEGMENT}/`
const at = key.lastIndexOf(marker)
if (at <= 0) return null
const kind = key.slice(at + marker.length)
if (!POSTING_KINDS.includes(kind as PostingKind)) return null
return { field: key.slice(0, at), kind: kind as PostingKind }
}
/** Record a discovered or freshly created column against its field. */
private registerColumn(field: string, kind: PostingKind, key: string): void {
let byKind = this.fieldColumns.get(field)
if (!byKind) {
byKind = new Map()
this.fieldColumns.set(field, byKind)
}
const existing = byKind.get(kind)
if (existing !== undefined && existing !== key) {
// Two columns claiming one (field, kind) means the layout on disk is not
// one this writer could have produced. Serving it would silently answer
// from half the postings, so say which two and stop.
throw new Error(
`ColumnStore: field '${field}' has two '${kind}' posting columns on ` +
`disk ('${existing}' and '${key}'). The column index layout is ` +
`inconsistent — rebuild/repair the metadata index rather than ` +
`serving from one half of it.`
)
}
byKind.set(kind, key)
}
/** The column key for this (field, kind), or `null` if the field has no such kind. */
private columnKey(field: string, kind: PostingKind): string | null {
return this.fieldColumns.get(field)?.get(kind) ?? null
}
/**
* The column key for this (field, kind), creating the registration if the
* field has not seen this kind before. Write path only.
*/
private ensureColumnKey(field: string, kind: PostingKind): string {
const byKind = this.fieldColumns.get(field)
const existing = byKind?.get(kind)
if (existing !== undefined) return existing
// The primary kind is the one already holding the bare field name, if any.
let primary: PostingKind | null = null
if (byKind) {
for (const [k, key] of byKind) {
if (key === field) { primary = k; break }
}
}
const key = ColumnStore.columnKeyFor(field, kind, primary)
this.registerColumn(field, kind, key)
return key
}
/**
* Every posting column this field owns, in {@link POSTING_KINDS} order.
*
* Read doors that are not about one particular value an unbounded range
* used as an "any value present" probe, distinct values, sorting fan out
* over all of them.
*/
private columnsForField(field: string): KindColumn[] {
const byKind = this.fieldColumns.get(field)
if (!byKind) return []
const out: KindColumn[] = []
for (const kind of POSTING_KINDS) {
const key = byKind.get(kind)
if (key !== undefined) out.push({ field, kind, key })
}
return out
}
/**
* Which value kinds this field actually holds, in {@link POSTING_KINDS}
* order the honest answer to "what type is this field?".
*
* A field that carries both `'electronics'` and `5` reports
* `['number', 'string']`, not whichever of them was written first.
*
* @param field - Field name
* @returns Every kind with at least one posting, or `[]` for an unknown field
*/
getFieldKinds(field: string): PostingKind[] {
return this.columnsForField(field)
.filter((c) => this.columnHasData(c.key))
.map((c) => c.kind)
}
/** Does this physical column hold any postings (persisted or buffered)? */
private columnHasData(key: string): boolean {
const manifest = this.manifests.get(key)
const buffer = this.tailBuffers.get(key)
return (manifest !== undefined && !manifest.isEmpty()) || (buffer !== undefined && buffer.size > 0)
}
/** /**
* Initialize the column store: discover existing field manifests. * Initialize the column store: discover existing field manifests.
*/ */
@ -371,23 +157,11 @@ export class ColumnStore implements ColumnStoreProvider {
}).listObjectsUnderPath(this.basePath + '/') }).listObjectsUnderPath(this.basePath + '/')
for (const path of paths) { for (const path of paths) {
if (path.endsWith('/MANIFEST.json')) { if (path.endsWith('/MANIFEST.json')) {
// The discovered name is a COLUMN key: either a bare field (that const fieldName = path.replace(this.basePath + '/', '').replace('/MANIFEST.json', '')
// field's primary kind, which is every column an index written const manifest = new ColumnManifest(fieldName, this.basePath)
// before typed postings has) or `<field>/k/<kind>` for a second
// kind that arrived on a field later.
const columnKey = path.replace(this.basePath + '/', '').replace('/MANIFEST.json', '')
const manifest = new ColumnManifest(columnKey, this.basePath)
await manifest.load(storage) await manifest.load(storage)
this.manifests.set(columnKey, manifest) this.manifests.set(fieldName, manifest)
this.fieldTypes.set(columnKey, manifest.valueType) this.fieldTypes.set(fieldName, manifest.valueType)
const parsed = ColumnStore.parseKindColumnKey(columnKey)
if (parsed) {
this.registerColumn(parsed.field, parsed.kind, columnKey)
} else {
this.registerColumn(columnKey, kindOfValueType(manifest.valueType), columnKey)
}
const fieldName = columnKey
// Load global deleted bitmap if it exists. Raw blob preferred // Load global deleted bitmap if it exists. Raw blob preferred
// (2.4.0 #4 cortex-shared format); legacy envelope fallback for // (2.4.0 #4 cortex-shared format); legacy envelope fallback for
@ -490,43 +264,26 @@ export class ColumnStore implements ColumnStoreProvider {
/** /**
* Point filter: find entities where field equals value. * Point filter: find entities where field equals value.
* *
* The QUERY VALUE'S OWN KIND picks the posting column, and only that column * Searches all segments + tail buffer, returns union as roaring bitmap.
* is read. `where {category: 5}` answers from the number postings and * Excludes globally deleted entities.
* `where {category: '5'}` from the string postings neither borrows the
* other's rows, because a row written with the number `5` is not a row whose
* category is the text `'5'`.
*
* A field that has never seen this kind matches nothing, which is the true
* answer rather than a coerced one.
*
* Searches all segments + tail buffer of that column, returns the union as a
* roaring bitmap. Excludes globally deleted entities.
*/ */
async filter(field: string, value: unknown): Promise<RoaringBitmap32> { async filter(field: string, value: unknown): Promise<RoaringBitmap32> {
const result = new RoaringBitmap32() const result = new RoaringBitmap32()
const columnKey = this.columnKey(field, kindOfValue(value)) const deleted = this.deletedEntities.get(field)
if (columnKey === null) return result
// The query value takes the column's encoding — a boolean queried against
// a boolean column has to become the 1/0 the column stores.
const encoded = this.normalizeValue(value, this.fieldTypes.get(columnKey) ?? ValueType.String)
if (encoded === undefined) return result
const deleted = this.deletedEntities.get(columnKey)
// Search segments // Search segments
const cursors = await this.getSegmentCursors(columnKey) const cursors = await this.getSegmentCursors(field)
for (const cursor of cursors) { for (const cursor of cursors) {
const ids = cursor.getEntityIdsForValue(encoded) const ids = cursor.getEntityIdsForValue(value as number | string)
for (const id of ids) { for (const id of ids) {
if (!deleted || !deleted.has(id)) result.add(id) if (!deleted || !deleted.has(id)) result.add(id)
} }
} }
// Search tail buffer // Search tail buffer
const tailCursor = this.getTailBufferCursor(columnKey) const tailCursor = this.getTailBufferCursor(field)
if (tailCursor) { if (tailCursor) {
const ids = tailCursor.getEntityIdsForValue(encoded) const ids = tailCursor.getEntityIdsForValue(value as number | string)
for (const id of ids) { for (const id of ids) {
if (!deleted || !deleted.has(id)) result.add(id) if (!deleted || !deleted.has(id)) result.add(id)
} }
@ -535,62 +292,6 @@ export class ColumnStore implements ColumnStoreProvider {
return result return result
} }
/**
* Read this column's value for each of `entityIntIds` the per-id read
* behind `find({ fields })`.
*
* Every other read door here answers "which entities have this value". A
* projection asks the opposite "what value does this entity have" and
* without it a projection has to go to the canonical record for a field the
* column is already holding.
*
* The column is walked ONCE and the wanted ids are picked out as they pass,
* so the cost is O(column) per field rather than O(ids x column). Later
* sources win: the tail buffer holds writes newer than any segment, and
* within the segments a later one supersedes an earlier, exactly as `filter`
* treats them.
*
* Values are EXACT this store keeps raw values, not the bucketed form the
* sparse index uses for range queries which is what makes it safe to
* project from. Deleted entities are skipped; an id with no value in this
* column is simply absent from the result.
*
* @param field - Field name to read.
* @param entityIntIds - Entity integer ids to read values for.
* @returns `entityIntId -> value` for the ids this column holds.
*/
async valuesForIds(
field: string,
entityIntIds: Iterable<number>
): Promise<Map<number, number | string>> {
const wanted = new Set<number>(entityIntIds)
const out = new Map<number, number | string>()
if (wanted.size === 0 || !this.hasField(field)) return out
// Every kind the field holds is read, in POSTING_KINDS order — a value an
// entity wrote as a string is still that entity's value for this field.
for (const column of this.columnsForField(field)) {
const deleted = this.deletedEntities.get(column.key)
const take = (entry: { value: number | string; entityIntId: number }): void => {
if (!wanted.has(entry.entityIntId)) return
if (deleted && deleted.has(entry.entityIntId)) return
out.set(entry.entityIntId, entry.value)
}
// Segments oldest -> newest, then the tail: a later write overwrites an
// earlier one for the same id.
const cursors = await this.getSegmentCursors(column.key)
for (const cursor of cursors) {
for (const entry of cursor.iterateForward()) take(entry)
}
const tailCursor = this.getTailBufferCursor(column.key)
if (tailCursor) {
for (const entry of tailCursor.iterateForward()) take(entry)
}
}
return out
}
/** /**
* Range filter: find entities where field is within the bounds. * Range filter: find entities where field is within the bounds.
* *
@ -610,59 +311,41 @@ export class ColumnStore implements ColumnStoreProvider {
includeMax: boolean = true includeMax: boolean = true
): Promise<RoaringBitmap32> { ): Promise<RoaringBitmap32> {
const result = new RoaringBitmap32() const result = new RoaringBitmap32()
const cursors = await this.getSegmentCursors(field)
const hasMin = min !== undefined && min !== null const hasMin = min !== undefined && min !== null
const hasMax = max !== undefined && max !== null const hasMax = max !== undefined && max !== null
// The BOUNDS pick the column: numeric bounds read the numeric postings, for (const cursor of cursors) {
// string bounds the string postings. An unbounded call is not a range at const lo = hasMin ? min as number | string : cursor.minValue
// all — it is the "has any value here" probe behind `exists` — so it fans const hi = hasMax ? max as number | string : cursor.maxValue
// out over every kind the field holds. if (lo === undefined || hi === undefined) continue
const columns: KindColumn[] = hasMin // Exclusivity applies only to an explicitly provided bound. A bound taken
? this.columnsForKind(field, kindOfValue(min)) // from the segment's own min/max is a real stored value and must stay
: hasMax // inclusive, or the segment's boundary entities would be wrongly dropped.
? this.columnsForKind(field, kindOfValue(max)) const ids = cursor.getEntityIdsInRange(
: this.columnsForField(field) lo,
hi,
hasMin ? includeMin : true,
hasMax ? includeMax : true
)
for (const id of ids) result.add(id)
}
for (const column of columns) { // Tail buffer range: linear scan (tail is small)
const cursors = await this.getSegmentCursors(column.key) const tailCursor = this.getTailBufferCursor(field)
for (const cursor of cursors) { if (tailCursor) {
const lo = hasMin ? min as number | string : cursor.minValue for (const entry of tailCursor.iterateForward()) {
const hi = hasMax ? max as number | string : cursor.maxValue const v = entry.value as any
if (lo === undefined || hi === undefined) continue const loOk = !hasMin || (includeMin ? v >= (min as any) : v > (min as any))
// Exclusivity applies only to an explicitly provided bound. A bound taken const hiOk = !hasMax || (includeMax ? v <= (max as any) : v < (max as any))
// from the segment's own min/max is a real stored value and must stay if (loOk && hiOk) result.add(entry.entityIntId)
// inclusive, or the segment's boundary entities would be wrongly dropped.
const ids = cursor.getEntityIdsInRange(
lo,
hi,
hasMin ? includeMin : true,
hasMax ? includeMax : true
)
for (const id of ids) result.add(id)
}
// Tail buffer range: linear scan (tail is small)
const tailCursor = this.getTailBufferCursor(column.key)
if (tailCursor) {
for (const entry of tailCursor.iterateForward()) {
const v = entry.value as any
const loOk = !hasMin || (includeMin ? v >= (min as any) : v > (min as any))
const hiOk = !hasMax || (includeMax ? v <= (max as any) : v < (max as any))
if (loOk && hiOk) result.add(entry.entityIntId)
}
} }
} }
return result return result
} }
/** The single column for this (field, kind), as a list, or empty if absent. */
private columnsForKind(field: string, kind: PostingKind): KindColumn[] {
const key = this.columnKey(field, kind)
return key === null ? [] : [{ field, kind, key }]
}
/** /**
* Sort top-K: return K entity int IDs in sorted order (u64-safe BigInt). * Sort top-K: return K entity int IDs in sorted order (u64-safe BigInt).
* *
@ -693,21 +376,18 @@ export class ColumnStore implements ColumnStoreProvider {
*/ */
async getFilterValues(field: string): Promise<string[]> { async getFilterValues(field: string): Promise<string[]> {
const valueSet = new Set<string>() const valueSet = new Set<string>()
const cursors = await this.getSegmentCursors(field)
for (const column of this.columnsForField(field)) { for (const cursor of cursors) {
const cursors = await this.getSegmentCursors(column.key) for (const entry of cursor.iterateForward()) {
valueSet.add(String(entry.value))
for (const cursor of cursors) {
for (const entry of cursor.iterateForward()) {
valueSet.add(String(entry.value))
}
} }
}
const tailCursor = this.getTailBufferCursor(column.key) const tailCursor = this.getTailBufferCursor(field)
if (tailCursor) { if (tailCursor) {
for (const entry of tailCursor.iterateForward()) { for (const entry of tailCursor.iterateForward()) {
valueSet.add(String(entry.value)) valueSet.add(String(entry.value))
}
} }
} }
@ -718,7 +398,9 @@ export class ColumnStore implements ColumnStoreProvider {
* Check if a field has any indexed data. * Check if a field has any indexed data.
*/ */
hasField(field: string): boolean { hasField(field: string): boolean {
return this.columnsForField(field).some((c) => this.columnHasData(c.key)) const manifest = this.manifests.get(field)
const buffer = this.tailBuffers.get(field)
return (manifest !== undefined && !manifest.isEmpty()) || (buffer !== undefined && buffer.size > 0)
} }
/** /**
@ -728,11 +410,12 @@ export class ColumnStore implements ColumnStoreProvider {
* store will actually serve queries from. * store will actually serve queries from.
*/ */
getIndexedFields(): string[] { getIndexedFields(): string[] {
// Names FIELDS, not columns: a field carrying two kinds is one name here,
// the same name a caller queries with.
const fields = new Set<string>() const fields = new Set<string>()
for (const [field] of this.fieldColumns) { for (const [field, manifest] of this.manifests) {
if (this.hasField(field)) fields.add(field) if (!manifest.isEmpty()) fields.add(field)
}
for (const [field, buffer] of this.tailBuffers) {
if (buffer.size > 0) fields.add(field)
} }
return Array.from(fields).sort() return Array.from(fields).sort()
} }
@ -747,16 +430,12 @@ export class ColumnStore implements ColumnStoreProvider {
getFieldSizeSummary(): Array<{ field: string; segmentCount: number; tailSize: number }> { getFieldSizeSummary(): Array<{ field: string; segmentCount: number; tailSize: number }> {
const summary: Array<{ field: string; segmentCount: number; tailSize: number }> = [] const summary: Array<{ field: string; segmentCount: number; tailSize: number }> = []
for (const field of this.getIndexedFields()) { for (const field of this.getIndexedFields()) {
// Summed across the field's kind columns — the caller asked about a const manifest = this.manifests.get(field)
// field, and a field's size is all of the postings under its name. const buffer = this.tailBuffers.get(field)
let segmentCount = 0 const segmentCount = manifest && !manifest.isEmpty()
let tailSize = 0 ? manifest.getAllSegments().length
for (const column of this.columnsForField(field)) { : 0
const manifest = this.manifests.get(column.key) const tailSize = buffer ? buffer.size : 0
const buffer = this.tailBuffers.get(column.key)
if (manifest && !manifest.isEmpty()) segmentCount += manifest.getAllSegments().length
if (buffer) tailSize += buffer.size
}
summary.push({ field, segmentCount, tailSize }) summary.push({ field, segmentCount, tailSize })
} }
return summary return summary
@ -784,8 +463,6 @@ export class ColumnStore implements ColumnStoreProvider {
this.segmentCache.clear() this.segmentCache.clear()
this.manifests.clear() this.manifests.clear()
this.deletedEntities.clear() this.deletedEntities.clear()
this.fieldColumns.clear()
this.fieldTypes.clear()
this.initialized = false this.initialized = false
} }
@ -794,64 +471,32 @@ export class ColumnStore implements ColumnStoreProvider {
// ========================================================================= // =========================================================================
/** /**
* Push a single value to the posting column for its (field, KIND). * Push a single value to a field's tail buffer.
* * Creates the buffer and manifest if first write to this field.
* The value's own kind picks the column — a string goes to the field's * Infers ValueType from the first value seen.
* string postings, a number to its number postings so a field carrying
* `'electronics'` and `5` keeps both, each answerable by an equality filter
* of its own kind. Under the first-writer type freeze this method replaced,
* the first value's type became the field's type and every later value of
* another kind was coerced to it or, when coercion failed, dropped with no
* error at all.
*
* Creates the column's buffer and manifest on its first value.
*/ */
private pushToBuffer(field: string, value: unknown, entityIntId: number, isMultiValue: boolean): void { private pushToBuffer(field: string, value: unknown, entityIntId: number, isMultiValue: boolean): void {
const kind = kindOfValue(value) let buffer = this.tailBuffers.get(field)
const columnKey = this.ensureColumnKey(field, kind)
let buffer = this.tailBuffers.get(columnKey)
if (!buffer) { if (!buffer) {
// A reopened column takes its encoding from its manifest — an integer const valueType = this.inferValueType(value)
// column that widened to f64 in an earlier session stays widened. buffer = new ColumnTailBuffer(field, valueType, this.flushThreshold)
const valueType = this.tailBuffers.set(field, buffer)
this.manifests.get(columnKey)?.valueType ?? initialValueTypeFor(kind, value) this.fieldTypes.set(field, valueType)
buffer = new ColumnTailBuffer(columnKey, valueType, this.flushThreshold)
this.tailBuffers.set(columnKey, buffer)
this.fieldTypes.set(columnKey, valueType)
// Ensure manifest exists // Ensure manifest exists
if (!this.manifests.has(columnKey)) { if (!this.manifests.has(field)) {
const manifest = new ColumnManifest(columnKey, this.basePath) const manifest = new ColumnManifest(field, this.basePath)
manifest.valueType = valueType manifest.valueType = valueType
manifest.multiValue = isMultiValue manifest.multiValue = isMultiValue
this.manifests.set(columnKey, manifest) this.manifests.set(field, manifest)
} }
} }
// An integer column widens the first time a non-integer number arrives, so // Normalize value to the column type
// the value is stored as itself instead of rounded to the nearest integer.
if (kind === 'number' && buffer.valueType === ValueType.Number && !Number.isInteger(value)) {
buffer.promoteToFloat()
this.fieldTypes.set(columnKey, ValueType.Float)
const manifest = this.manifests.get(columnKey)
if (manifest) manifest.valueType = ValueType.Float
}
const normalizedValue = this.normalizeValue(value, buffer.valueType) const normalizedValue = this.normalizeValue(value, buffer.valueType)
if (normalizedValue === undefined) { if (normalizedValue !== undefined) {
// Unreachable by construction: the column was chosen BY this value's buffer.add(normalizedValue, entityIntId)
// kind, so the encoding always accepts it. Reaching here would mean a
// value had been silently dropped from the index — the exact failure
// typed postings exist to end — so it is an error, never a skip.
throw new Error(
`ColumnStore: field '${field}' rejected a ${kind} value for its own ` +
`${ValueType[buffer.valueType]} posting column. The value would have ` +
`been dropped from the index while the row stayed readable by id — ` +
`this is a kind-routing bug, not a value the caller may ignore.`
)
} }
buffer.add(normalizedValue, entityIntId)
} }
/** /**
@ -980,15 +625,8 @@ export class ColumnStore implements ColumnStoreProvider {
/** Torn-segment quarantine entries for a field (observability + heal input). */ /** Torn-segment quarantine entries for a field (observability + heal input). */
quarantinedSegments(field: string): Array<{ segment: string; error: string; hits: number }> { quarantinedSegments(field: string): Array<{ segment: string; error: string; hits: number }> {
const out: Array<{ segment: string; error: string; hits: number }> = [] const out: Array<{ segment: string; error: string; hits: number }> = []
// Across every kind column of the field — a torn segment in the string for (const [key, q] of this.segmentQuarantine) {
// postings is this field's torn segment as much as one in the numbers. if (key.startsWith(`${field}:`)) out.push({ segment: key.slice(field.length + 1), error: q.error, hits: q.hits })
for (const column of this.columnsForField(field)) {
const prefix = `${column.key}:`
for (const [key, q] of this.segmentQuarantine) {
if (key.startsWith(prefix)) {
out.push({ segment: key.slice(prefix.length), error: q.error, hits: q.hits })
}
}
} }
return out return out
} }
@ -1160,22 +798,17 @@ export class ColumnStore implements ColumnStoreProvider {
k: number, k: number,
filterBitmap: RoaringBitmap32 | null filterBitmap: RoaringBitmap32 | null
): Promise<number[]> { ): Promise<number[]> {
// Collect cursors across EVERY kind the field holds. A single-kind field — // Collect all cursors (segments + tail buffer)
// nearly all of them — merges exactly the cursors it always did. const segCursors = await this.getSegmentCursors(field)
const tailCursor = this.getTailBufferCursor(field)
// Create iterators for each cursor in the specified direction
const iterators: Generator<CursorEntry>[] = [] const iterators: Generator<CursorEntry>[] = []
const iteratorKindRank: number[] = [] for (const cursor of segCursors) {
for (const column of this.columnsForField(field)) { iterators.push(order === 'asc' ? cursor.iterateForward() : cursor.iterateBackward())
const kindRank = POSTING_KINDS.indexOf(column.kind) }
const segCursors = await this.getSegmentCursors(column.key) if (tailCursor) {
for (const cursor of segCursors) { iterators.push(order === 'asc' ? tailCursor.iterateForward() : tailCursor.iterateBackward())
iterators.push(order === 'asc' ? cursor.iterateForward() : cursor.iterateBackward())
iteratorKindRank.push(kindRank)
}
const tailCursor = this.getTailBufferCursor(column.key)
if (tailCursor) {
iterators.push(order === 'asc' ? tailCursor.iterateForward() : tailCursor.iterateBackward())
iteratorKindRank.push(kindRank)
}
} }
if (iterators.length === 0) return [] if (iterators.length === 0) return []
@ -1189,21 +822,16 @@ export class ColumnStore implements ColumnStoreProvider {
value: next.value.value, value: next.value.value,
entityIntId: next.value.entityIntId, entityIntId: next.value.entityIntId,
cursorIndex: i, cursorIndex: i,
kindRank: iteratorKindRank[i],
iterator: iterators[i] iterator: iterators[i]
}) })
} }
} }
// Heapify. A number and a string have no ordering between them, so a // Heapify
// mixed-kind field orders by KIND first (POSTING_KINDS order) and by value const isString = (this.fieldTypes.get(field) ?? ValueType.Number) === ValueType.String
// within a kind — one defined total order instead of a comparison whose
// answer depends on which value happened to be on the left.
const compare = (a: HeapEntry, b: HeapEntry): number => { const compare = (a: HeapEntry, b: HeapEntry): number => {
let cmp: number let cmp: number
if (a.kindRank !== b.kindRank) { if (isString) {
cmp = a.kindRank - b.kindRank
} else if (POSTING_KINDS[a.kindRank] === 'string') {
cmp = compareCodePoints(String(a.value), String(b.value)) cmp = compareCodePoints(String(a.value), String(b.value))
} else { } else {
cmp = (a.value as number) - (b.value as number) cmp = (a.value as number) - (b.value as number)
@ -1235,7 +863,6 @@ export class ColumnStore implements ColumnStoreProvider {
value: next.value.value, value: next.value.value,
entityIntId: next.value.entityIntId, entityIntId: next.value.entityIntId,
cursorIndex: top.cursorIndex, cursorIndex: top.cursorIndex,
kindRank: top.kindRank,
iterator: top.iterator iterator: top.iterator
} }
} }
@ -1243,11 +870,8 @@ export class ColumnStore implements ColumnStoreProvider {
this.heapDown(heap, 0, compare) this.heapDown(heap, 0, compare)
} }
// Apply global deleted check, filter, and dedup. The deleted bitmap is // Apply global deleted check, filter, and dedup
// per COLUMN, and the entry came from the column its kind names. const deleted = this.deletedEntities.get(field)
const deleted = this.deletedEntities.get(
this.columnKey(field, POSTING_KINDS[top.kindRank]) ?? field
)
if (deleted && deleted.has(top.entityIntId)) continue if (deleted && deleted.has(top.entityIntId)) continue
if (seen.has(top.entityIntId)) continue if (seen.has(top.entityIntId)) continue
if (filterBitmap && !filterBitmap.has(top.entityIntId)) continue if (filterBitmap && !filterBitmap.has(top.entityIntId)) continue
@ -1289,31 +913,35 @@ export class ColumnStore implements ColumnStoreProvider {
} }
/** /**
* Encode a value for the column its own kind selected. * Infer ValueType from a JavaScript value.
* */
* This does NOT convert between kinds. It used to: a string reaching a private inferValueType(value: unknown): ValueType {
* numeric column was run through `Number(value)`, and a number reaching a if (typeof value === 'boolean') return ValueType.Boolean
* numeric column was run through `Math.round`, so `'electronics'` became if (typeof value === 'number') {
* `NaN` and vanished while `4.5` became `5` and answered the wrong query. return Number.isInteger(value) ? ValueType.Number : ValueType.Float
* Kind routing removes the need for either the only work left is picking }
* the encoding the column already committed to. return ValueType.String
* }
* @returns The encoded value, or `undefined` if the value does not belong in
* this column at all which the caller treats as a routing bug and /**
* raises, never as a value to skip. * Normalize a JavaScript value to the column's ValueType.
*/ */
private normalizeValue(value: unknown, type: ValueType): number | string | undefined { private normalizeValue(value: unknown, type: ValueType): number | string | undefined {
switch (type) { switch (type) {
case ValueType.Number: case ValueType.Number:
// Integer column. Non-integers widen it to Float before reaching here. if (typeof value === 'number') return Math.round(value)
return typeof value === 'number' && Number.isInteger(value) ? value : undefined if (typeof value === 'string') { const n = Number(value); return isNaN(n) ? undefined : Math.round(n) }
if (typeof value === 'boolean') return value ? 1 : 0
return undefined
case ValueType.Float: case ValueType.Float:
return typeof value === 'number' ? value : undefined if (typeof value === 'number') return value
if (typeof value === 'string') { const n = Number(value); return isNaN(n) ? undefined : n }
return undefined
case ValueType.Boolean: case ValueType.Boolean:
return typeof value === 'boolean' ? (value ? 1 : 0) : undefined if (typeof value === 'boolean') return value ? 1 : 0
if (typeof value === 'number') return value ? 1 : 0
return undefined
case ValueType.String: case ValueType.String:
// The string kind is also where objects and bigints land, exactly as
// they always did.
return String(value) return String(value)
default: default:
return undefined return undefined

View file

@ -55,12 +55,8 @@ export class ColumnTailBuffer {
/** Field name this buffer is for. */ /** Field name this buffer is for. */
readonly fieldName: string readonly fieldName: string
/** /** Value type determines sort comparator. */
* Value type determines sort comparator and segment encoding. readonly valueType: ValueType
*
* Widened in place by {@link promoteToFloat} never otherwise reassigned.
*/
valueType: ValueType
/** Flush threshold. */ /** Flush threshold. */
readonly threshold: number readonly threshold: number
@ -85,38 +81,6 @@ export class ColumnTailBuffer {
this.threshold = threshold this.threshold = threshold
} }
/**
* Widen an integer column to floating point, losslessly and in place.
*
* The number posting kind holds every JavaScript number, but a segment picks
* ONE encoding: i64 for integers, f64 for the rest. A column that has only
* ever seen integers is written as i64; the first non-integer to arrive
* widens it here, so that value is stored as itself instead of being rounded
* to the nearest integer with no error the rounding that made `4.5` and
* `5.5` both answer `where {score: 5}` and neither answer its own value.
*
* Widening is lossless in both directions it has to be: every value already
* buffered is an integer, and every integer is exactly representable as f64.
* Segments already on disk keep their own i64 encoding in their own headers
* and keep decoding by it only segments written from here on are f64.
*
* @throws Error if called on a column that is not an integer column the
* only legal widening is Number Float, and any other request is a bug in
* the caller's kind routing rather than something to absorb quietly.
*/
promoteToFloat(): void {
if (this.valueType === ValueType.Float) return
if (this.valueType !== ValueType.Number) {
throw new Error(
`ColumnTailBuffer '${this.fieldName}': cannot widen a ` +
`${ValueType[this.valueType]} column to Float — only an integer ` +
`(Number) column widens, and this call means a value reached the ` +
`wrong kind's column`
)
}
this.valueType = ValueType.Float
}
/** /**
* Add a (value, entityIntId) entry to the buffer. * Add a (value, entityIntId) entry to the buffer.
* *

View file

@ -58,53 +58,6 @@ export enum ValueType {
Boolean = 3 Boolean = 3
} }
/**
* The KIND of a value, as the query language sees it.
*
* A kind is a JavaScript `typeof` class, not a storage encoding: `5` and `5.5`
* are one kind (`'number'`) held in one posting column, even though they need
* different segment encodings (i64 vs f64 see {@link ValueType}).
*
* A field holds ONE POSTING COLUMN PER KIND, so `category` may carry string
* values and number values at the same time and answer equality on each. This
* replaces the first-writer type freeze, under which the first value's type
* became the field's type and every later value of another kind was coerced
* or, when coercion failed (`Number('electronics')`), dropped from the index
* with no error: the row stayed readable by id and by vector but vanished from
* every equality filter on that field.
*
* Kinds do not coerce into one another at query time either: `where {c: 5}`
* matches rows written with the NUMBER `5`, and `where {c: '5'}` matches rows
* written with the STRING `'5'`. Neither ever matches the other.
*
* Values that are none of these three (objects, bigints) index as strings
* the same `String(value)` treatment they received before.
*/
export type PostingKind = 'number' | 'string' | 'boolean'
/**
* Every posting kind, in the order that defines cross-kind sort position.
*
* A mixed-kind field has no natural total order a number does not compare
* with a string so `sortTopK` orders by KIND first (numbers, then strings,
* then booleans) and by value within a kind. A single-kind field, which is
* nearly every field, sorts exactly as it always did.
*/
export const POSTING_KINDS: readonly PostingKind[] = ['number', 'string', 'boolean']
/**
* Path segment marking a field's NON-PRIMARY kind columns on disk.
*
* The first kind a field ever sees keeps the historical layout
* `<base>/<field>/MANIFEST.json` and `<base>/<field>/L0-NNNNNN` so every
* index written before typed postings opens unchanged, and the byte-for-byte
* interchange with the native column store is untouched for the single-kind
* fields that are nearly all of them. A second kind arriving on the same field
* gets its own column at `<base>/<field>/k/<kind>/…` rather than overwriting or
* being coerced into the first.
*/
export const KIND_PATH_SEGMENT = 'k'
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Segment header and footer // Segment header and footer
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -314,19 +267,6 @@ export interface ColumnStoreProvider {
*/ */
hasField(field: string): boolean hasField(field: string): boolean
/**
* Which value KINDS this field actually holds, in {@link POSTING_KINDS}
* order the honest answer to "what type is this field?" for a field that
* carries more than one.
*
* OPTIONAL so an implementation written against the pre-typed-postings
* contract still satisfies this interface; feature-detect before calling.
*
* @param field - Field name
* @returns Every kind with at least one posting, or `[]` for an unknown field
*/
getFieldKinds?(field: string): PostingKind[]
/** /**
* Flush all in-memory tail buffers to L0 segments on disk. * Flush all in-memory tail buffers to L0 segments on disk.
* Saves all manifests. * Saves all manifests.

View file

@ -9,7 +9,7 @@
* *
* @example Enable integrations (recommended) * @example Enable integrations (recommended)
* ```typescript * ```typescript
* import { Brainy } from '@soulcraftlabs/brainy' * import { Brainy } from '@soulcraft/brainy'
* *
* const brain = new Brainy({ integrations: true }) * const brain = new Brainy({ integrations: true })
* await brain.init() * await brain.init()

View file

@ -41,7 +41,7 @@ The `BrainyMCPService` has been refactored to separate the core functionality fr
### In Any Environment (Browser, Node.js, Server) ### In Any Environment (Browser, Node.js, Server)
```typescript ```typescript
import { Brainy, BrainyMCPAdapter, MCPAugmentationToolset } from '@soulcraftlabs/brainy' import { Brainy, BrainyMCPAdapter, MCPAugmentationToolset } from '@soulcraft/brainy'
// Create a Brainy instance // Create a Brainy instance
const brainyData = new Brainy() const brainyData = new Brainy()
@ -81,7 +81,7 @@ const toolResponse = await toolset.handleRequest({
### In Browser Environment (Core Functionality Only) ### In Browser Environment (Core Functionality Only)
```typescript ```typescript
import { Brainy, BrainyMCPService } from '@soulcraftlabs/brainy' import { Brainy, BrainyMCPService } from '@soulcraft/brainy'
// Create a Brainy instance // Create a Brainy instance
const brainyData = new Brainy() const brainyData = new Brainy()

View file

@ -2,7 +2,7 @@
* 🧠 BRAINY EMBEDDED PATTERNS * 🧠 BRAINY EMBEDDED PATTERNS
* *
* AUTO-GENERATED - DO NOT EDIT * AUTO-GENERATED - DO NOT EDIT
* Generated: 2026-08-27T09:18:45-07:00 * Generated: 2026-07-02T21:43:26.976Z
* Patterns: 220 * Patterns: 220
* Coverage: 94-98% of all queries * Coverage: 94-98% of all queries
* *

View file

@ -2,7 +2,7 @@
* 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS
* *
* AUTO-GENERATED - DO NOT EDIT * AUTO-GENERATED - DO NOT EDIT
* Generated: 2026-08-27T09:18:45-07:00 * Generated: 2026-02-09T16:59:48.867Z
* Noun Types: 42 * Noun Types: 42
* Verb Types: 127 * Verb Types: 127
* *
@ -19,7 +19,7 @@ export const TYPE_METADATA = {
verbTypes: 127, verbTypes: 127,
totalTypes: 169, totalTypes: 169,
embeddingDimensions: 384, embeddingDimensions: 384,
generatedAt: "2026-08-27T09:18:45-07:00", generatedAt: "2026-02-09T16:59:48.867Z",
sizeBytes: { sizeBytes: {
embeddings: 259584, embeddings: 259584,
base64: 346112 base64: 346112

View file

@ -22,7 +22,7 @@ import type { GraphIndexStats } from './graph/graphAdjacencyIndex.js'
// Re-export the provider contracts that already live closer to their // Re-export the provider contracts that already live closer to their
// implementations so a plugin author (Cor) can import the *entire* // implementations so a plugin author (Cor) can import the *entire*
// provider surface from one stable entrypoint: `@soulcraftlabs/brainy/plugin`. // provider surface from one stable entrypoint: `@soulcraft/brainy/plugin`.
export type { ColumnStoreProvider } from './indexes/columnStore/types.js' export type { ColumnStoreProvider } from './indexes/columnStore/types.js'
export type { export type {
AggregationProvider, AggregationProvider,
@ -41,7 +41,7 @@ export interface BrainyPlugin {
name: string name: string
/** /**
* Optional semver range of `@soulcraftlabs/brainy` this plugin supports * Optional semver range of `@soulcraft/brainy` this plugin supports
* (e.g. `'>=8.0.0 <9.0.0'` or `'^8.0.0'`). When set and the running brainy is * (e.g. `'>=8.0.0 <9.0.0'` or `'^8.0.0'`). When set and the running brainy is
* OUTSIDE the range, `init()` THROWS rather than silently falling back to the * OUTSIDE the range, `init()` THROWS rather than silently falling back to the
* default JS engine. This is the version-coupling guard for the native * default JS engine. This is the version-coupling guard for the native
@ -411,129 +411,7 @@ export interface MetadataIndexProvider {
* @returns The matching id universe as an opaque set. * @returns The matching id universe as an opaque set.
*/ */
getIdSetForFilter?(filter: any): Promise<OpaqueIdSet> getIdSetForFilter?(filter: any): Promise<OpaqueIdSet>
/**
* @description OPTIONAL: evaluate `filter` over `ids` ONLY and return the
* survivors in the caller's order the door a graph-first
* `find({ connected, where })` walks. The neighbour set is the universe there,
* so the filter must cost O(|ids|) membership checks, never a whole-store
* materialization. A native index answers from its roaring filter result
* (membership by entity int); the reference index answers from its own
* `getIdsForFilter`, so the two doors can never disagree. Absent Brainy
* intersects `getIdsForFilter`'s answer with `ids` itself (correct, O(store)).
* @param filter - The same filter shape accepted by `getIdsForFilter`.
* @param ids - The candidate ids (canonical). The answer is a subsequence.
*/
filterIdsWithin?(filter: any, ids: readonly string[]): Promise<string[]>
/**
* @description OPTIONAL: plan and execute a WHOLE `find()` the graph
* traversal, the metadata filter, the ordering and the page and answer the
* page's ids, or `null` for a shape this index does not plan.
*
* The doors above each serve one stage, so a `find()` that consults three of
* them crosses into the index three times and marshals a result set at every
* crossing. An index that can decide the stage ORDER itself does the whole
* thing in one call and materializes ids only for the page a filter
* matching a hundred thousand rows then builds twenty-five id strings instead
* of a hundred thousand.
*
* The contract this door must keep, because Brainy cannot check it:
*
* - **The same answer.** Identical rows, in identical order, to what the
* stage doors would have produced for the same params. This door changes
* which code runs, never what the answer is.
* - **The law of the stages** (`find({ connected })` is graph-first): the
* neighbour set is the candidate universe, the filter is evaluated over
* those ids only, `orderBy` sorts the whole candidate set, and the page is
* cut LAST.
* - **`null` before work, not instead of an answer.** A shape the index does
* not plan must be handed back BEFORE any evaluation, so Brainy serves it
* through the stage doors exactly as it always has. Returning `null` after
* partial work, or an empty page for a shape it could not evaluate, is a
* silent wrong answer.
* - **`emptyAt` names the stage** that produced an empty page `'graph'`,
* `'filter'`, `'visibility'` or `'none'` so Brainy can apply its serving
* law to the right index. An empty answer from an index that is not
* serving must refuse loudly, and Brainy can only re-verify what it is told.
*
* Absent every `find()` is served by the stage doors, which is Brainy's
* own behaviour and the ordering oracle for any implementation of this one.
* @param params - The find params, already normalized by `find()`
* (natural-language parsed, `connected` anchors resolved to canonical ids,
* an empty `where` dropped).
* @param hiddenIds - Ids this read must not return. The contract is the ANSWER, not the
* mechanism: a provider may subtract this set before paging, or derive the
* same exclusion from the params' visibility tiers itself either way the
* page must equal the engine's own answer with none of these ids in it.
* @param graphIndex - The active graph provider, for a `connected` plan.
* @returns The page's ids plus the stage that emptied it, or `null`.
*/
planFindPage?(
params: any,
hiddenIds: readonly string[],
graphIndex: unknown
): Promise<{ ids: string[]; emptyAt: 'graph' | 'filter' | 'visibility' | 'none' } | null>
getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>> getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>>
/**
* @description OPTIONAL: score `query` over `ids` ONLY the text-leg twin of
* {@link filterIdsWithin}, and the door a hybrid `find({ query, where })`
* walks. The metadata filter's universe is the candidate set there, so the
* text leg must cost O(|ids|) membership checks and marshal at most `|ids|`
* rows, never the whole posting list of every query word. A native index
* intersects its own postings with the candidate set (membership by entity
* int) before any string crosses the boundary; the reference index answers
* from its own `getIdsForTextQuery`, so the two doors can never disagree.
* Absent Brainy intersects `getIdsForTextQuery`'s answer with `ids` itself
* (correct, and still hydrate-last, but it marshals the whole answer).
*
* The answer keeps `getIdsForTextQuery`'s contract: `{ id, matchCount }`
* sorted by `matchCount` descending, ties in the order the whole-store answer
* would have produced. Only rows in `ids` may appear.
* @param query - The same text query accepted by `getIdsForTextQuery`.
* @param ids - The candidate ids (canonical). The answer is a subset.
*/
getIdsForTextQueryWithin?(
query: string,
ids: readonly string[]
): Promise<Array<{ id: string; matchCount: number }>>
/**
* @description OPTIONAL: read named SCALAR fields for many ids at once, from
* the index's own value storage, WITHOUT touching the canonical record.
*
* This is the door behind `find/get/related({ fields })`. A list view that
* needs a title and a slug currently hydrates the whole record for every row
* document bodies included and then discards almost all of it. Serving
* the named scalars from the index turns that into an index read.
*
* ## The contract, and the one rule that makes it safe
*
* **Return only what you can serve EXACTLY, and say what you served.** The
* answer is a per-id map of the fields this index actually resolved; the
* caller diffs it against what was requested and reads the canonical record
* for the remainder. An implementation must therefore OMIT a field rather
* than approximate it and omission costs only a record read, while a wrong
* value is a wrong answer nobody can see.
*
* That rule is not hypothetical. This engine's own index buckets
* `system.createdAt` and `system.updatedAt` to the minute for range queries,
* so it cannot serve them exactly and omits them. An engine whose column
* store holds raw values can serve the same fields so the two answer
* differently in COST and identically in CONTENT, which is the only
* difference a projection door is allowed to have.
*
* A field absent from an entity is simply absent from that entity's map. It
* is never an error, and never a `null` standing in for one: absent and
* present-and-null are different answers.
*
* @param ids - Canonical entity ids to read.
* @param fields - Index KEYS (bare = user metadata, `system.*` = engine
* scalar), already address-resolved by the caller.
* @returns `id → { field: value }` for the fields this index served exactly.
* Ids with nothing to serve may be omitted entirely.
*/
getScalarsForIds?(
ids: readonly string[],
fields: readonly string[]
): Promise<Map<string, Record<string, unknown>>>
getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise<string[]> getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise<string[]>
getFilterValues(field: string): Promise<string[]> getFilterValues(field: string): Promise<string[]>
getFilterFields(): Promise<string[]> getFilterFields(): Promise<string[]>

View file

@ -1066,18 +1066,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
protected allCountsSuspect = false protected allCountsSuspect = false
/** One narration per session for the suspect transition (never per delete). */ /** One narration per session for the suspect transition (never per delete). */
private allCountsSuspectNarrated = false private allCountsSuspectNarrated = false
/**
* Which rule produced the ALL scalars currently in memory. `'identity-record'`
* means one counted entity per metadata content leg the honest rule: a
* bare id-directory (a ghost or scar left by a partial-delete defect, no
* content leg) counts zero. Set by the one-time derivation and by the
* sanctioned recount, alongside `allCountsSuspect = false`; left `undefined`
* when a loaded counts.json carries the ALL scalars but no stamp the
* legacy container-rule derivation, which forces `allCountsSuspect = true`
* at load instead. A filesystem concern: `MemoryStorage` has no counts.json
* and never sets this.
*/
protected allCountsDerivedBy?: 'identity-record'
protected entityCounts: Map<string, number> = new Map() // type -> count protected entityCounts: Map<string, number> = new Map() // type -> count
protected verbCounts: Map<string, number> = new Map() // verb type -> count protected verbCounts: Map<string, number> = new Map() // verb type -> count
protected countCache: Map<string, { count: number; timestamp: number }> = new Map() protected countCache: Map<string, { count: number; timestamp: number }> = new Map()
@ -1089,10 +1077,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
// Counts changed since the last persist? Drives the write-through flush. // Counts changed since the last persist? Drives the write-through flush.
protected pendingCountPersist = false protected pendingCountPersist = false
/** The one persist running right now, if any (single-flight law — see flushCounts). */
private countPersistInFlight: Promise<void> | null = null
/** The one trailing persist a burst has queued behind the in-flight one. */
private countPersistTrailing: Promise<void> | null = null
/** /**
* Get total noun count - O(1) operation * Get total noun count - O(1) operation
@ -1168,24 +1152,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
}) })
} }
/**
* OPTIONAL narrow ledger hook (see {@link StorageAdapter.noteVectorUnlanded}):
* the mirror of {@link noteVectorLanded} record a noun's vector was just
* REMOVED (rewritten to the unvectored `[]` shape). Never below zero: a
* caller that (incorrectly) fires this for a noun already unvectored would
* otherwise drive the ledger negative clamped defensively, matching the
* delete path's `if (this.totalVectoredNounCount > 0)` guard.
* @param id - The noun whose vector was just removed (retained for a
* future narration seam; the count itself needs no id-keyed state).
*/
async noteVectorUnlanded(id: string): Promise<void> {
void id
if (this.totalVectoredNounCount > 0) this.totalVectoredNounCount--
this.scheduleCountPersist().catch(() => {
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
})
}
/** /**
* Increment count for entity type - O(1) operation. * Increment count for entity type - O(1) operation.
* Concurrency is handled by the process-global mutex * Concurrency is handled by the process-global mutex
@ -1345,46 +1311,15 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
return return
} }
// SINGLE-FLIGHT, COALESCED. Counts are write-through on every change, so try {
// a burst of writes used to launch one persist per change, all in flight // Persist to storage (implemented by subclass)
// together. Two of them inside the same millisecond shared the atomic await this.persistCounts()
// writer's temp path (`.tmp-<pid>-<ms>`): both wrote it, the first rename this.pendingCountPersist = false
// consumed it, the second rename found nothing — ENOENT, ~1,500 times a } catch (error) {
// day on a busy production brain, with a full ledger write per change console.error('CRITICAL: Failed to flush counts to storage:', error)
// behind it. Now exactly one persist runs at a time; requests that arrive // Keep pending flag set so we retry on next operation
// while it runs collapse into ONE trailing persist that carries the final throw error
// state. A burst of N changes costs at most two writes and never races
// itself.
if (this.countPersistInFlight) {
// The in-flight write may have already serialised a stale snapshot —
// ask for one more pass after it, and let every caller in this burst
// await that same pass.
if (!this.countPersistTrailing) {
this.countPersistTrailing = this.countPersistInFlight
.catch(() => undefined)
.then(() => {
this.countPersistTrailing = null
return this.flushCounts()
})
}
return this.countPersistTrailing
} }
this.countPersistInFlight = (async () => {
try {
// Persist to storage (implemented by subclass)
this.pendingCountPersist = false
await this.persistCounts()
} catch (error) {
// Keep the flag set so the next operation retries.
this.pendingCountPersist = true
console.error('CRITICAL: Failed to flush counts to storage:', error)
throw error
} finally {
this.countPersistInFlight = null
}
})()
return this.countPersistInFlight
} }
/** /**

View file

@ -14,13 +14,10 @@ import {
StorageBatchConfig, StorageBatchConfig,
SYSTEM_DIR, SYSTEM_DIR,
STATISTICS_KEY, STATISTICS_KEY,
WriterLockInfo, WriterLockInfo
WriterCloseRecord
} from '../baseStorage.js' } from '../baseStorage.js'
import { getBrainyVersion } from '../../utils/index.js' import { getBrainyVersion } from '../../utils/index.js'
import { isAbsentError } from '../../utils/errorClassification.js' import { isAbsentError } from '../../utils/errorClassification.js'
import { prodLog } from '../../utils/logger.js'
import { isZeroNormVector } from '../../utils/distance.js'
import { import {
TornRecordError, TornRecordError,
isUnparseablePayloadError, isUnparseablePayloadError,
@ -100,30 +97,7 @@ export class FileSystemStorage extends BaseStorage {
// timer rewrites the lock every 10s so stale-lock detection can tell a dead // timer rewrites the lock every 10s so stale-lock detection can tell a dead
// writer from a slow one. The constant name matches the file path used. // writer from a slow one. The constant name matches the file path used.
private static readonly WRITER_LOCK_FILE = '_writer.lock' private static readonly WRITER_LOCK_FILE = '_writer.lock'
/** private static readonly WRITER_HEARTBEAT_MS = 10_000
* The clean-close record at `locks/_writer.close` (see
* {@link WriterCloseRecord}). Written when the lock is released, consumed by
* the next claim, so an open can distinguish "the previous writer left" from
* "the previous writer died" without inferring either from a pid.
*/
private static readonly WRITER_CLOSE_FILE = '_writer.close'
/**
* How often the lock file's `lastHeartbeat` is rewritten.
*
* THIS IS OBSERVABILITY ONLY, and the cadence follows from that. Staleness
* is decided by PID LIVENESS alone (see isWriterLockStale) and the fence
* compares pid + hostname no decision anywhere reads this timestamp. It
* exists so an operator inspecting a lock file, or reading the
* BRAINY_WRITER_LOCKED error, can judge liveness themselves.
*
* At 10s it was a lock-file WRITE every ten seconds per brain, forever: 2.1
* writes/s across a production process holding 21 idle brains, for a
* human-readable timestamp nothing computes with. At 60s an operator still
* sees a heartbeat inside the minute, at a sixth of the cost. With the
* clean-close record now recording orderly releases explicitly, the
* heartbeat carries even less weight than it did.
*/
private static readonly WRITER_HEARTBEAT_MS = 60_000
private static readonly WRITER_STALE_THRESHOLD_MS = 60_000 private static readonly WRITER_STALE_THRESHOLD_MS = 60_000
private writerLockHeartbeat?: NodeJS.Timeout private writerLockHeartbeat?: NodeJS.Timeout
private writerLockInfo?: WriterLockInfo private writerLockInfo?: WriterLockInfo
@ -136,13 +110,6 @@ export class FileSystemStorage extends BaseStorage {
*/ */
private writerHeartbeatInFlight?: Promise<void> private writerHeartbeatInFlight?: Promise<void>
/**
* The in-flight background count-ledger derivation, if one was needed at
* open. See {@link scheduleCountLedgerDerivation} awaited only by
* {@link whenCountLedgerSettled}, never by a read.
*/
private countLedgerDerivation?: Promise<void>
// Flush-request RPC state. The writer polls `locks/_flush_requests/` for // Flush-request RPC state. The writer polls `locks/_flush_requests/` for
// new `.req` files and emits `.ack` files in `locks/_flush_responses/` after // new `.req` files and emits `.ack` files in `locks/_flush_responses/` after
// flushing. Inspectors call `requestFlushOverFilesystem` to drop a request // flushing. Inspectors call `requestFlushOverFilesystem` to drop a request
@ -151,16 +118,9 @@ export class FileSystemStorage extends BaseStorage {
private static readonly FLUSH_REQUEST_DIR = '_flush_requests' private static readonly FLUSH_REQUEST_DIR = '_flush_requests'
private static readonly FLUSH_RESPONSE_DIR = '_flush_responses' private static readonly FLUSH_RESPONSE_DIR = '_flush_responses'
private static readonly FLUSH_WATCH_INTERVAL_MS = 500 private static readonly FLUSH_WATCH_INTERVAL_MS = 500
/**
* The safety sweep behind the fs.watch: catches events an exotic filesystem
* dropped, and runs the stale-request GC. See startFlushRequestWatcher.
*/
private static readonly FLUSH_SAFETY_SWEEP_MS = 30_000
private static readonly FLUSH_POLL_INTERVAL_MS = 100 private static readonly FLUSH_POLL_INTERVAL_MS = 100
private static readonly FLUSH_REQUEST_TTL_MS = 60_000 private static readonly FLUSH_REQUEST_TTL_MS = 60_000
private flushWatcherInterval?: NodeJS.Timeout private flushWatcherInterval?: NodeJS.Timeout
/** The inotify-backed watch on the request directory, when the FS supports one. */
private flushWatcher?: import('node:fs').FSWatcher
private flushWatcherInFlight = false private flushWatcherInFlight = false
private flushWatcherOnRequest?: () => Promise<void> private flushWatcherOnRequest?: () => Promise<void>
@ -642,20 +602,6 @@ export class FileSystemStorage extends BaseStorage {
* automatically. Returns the pruned container ids so the caller can recompute * automatically. Returns the pruned container ids so the caller can recompute
* counts. * counts.
*/ */
/**
* @description Whether an id directory's file legs include the metadata
* CONTENT leg (`metadata.json` or its `.json.gz` variant) the single
* test that decides whether an `entities/<kind>/<shard>/<id>/` container is
* a live entity or a ghost/scar orphan left by the pre-8.3.1 partial-delete
* defect (see {@link pruneOrphanedEntities}). Shared by the orphan prune
* and {@link scanCanonicalEntities} so the two agree by construction one
* counted entity per identity record, never per bare container.
* @param legs - File names in one `entities/<kind>/<shard>/<id>/` directory.
*/
private hasMetadataContentLeg(legs: string[]): boolean {
return legs.some((f) => f.startsWith('metadata.json'))
}
public async pruneOrphanedEntities(): Promise<{ nouns: string[]; verbs: string[] }> { public async pruneOrphanedEntities(): Promise<{ nouns: string[]; verbs: string[] }> {
await this.ensureInitialized() await this.ensureInitialized()
const pruned: { nouns: string[]; verbs: string[] } = { nouns: [], verbs: [] } const pruned: { nouns: string[]; verbs: string[] } = { nouns: [], verbs: [] }
@ -695,7 +641,7 @@ export class FileSystemStorage extends BaseStorage {
} }
// A live entity has its metadata content leg. No content leg → a // A live entity has its metadata content leg. No content leg → a
// vector-only ghost or an empty scar → prune the whole container. // vector-only ghost or an empty scar → prune the whole container.
if (this.hasMetadataContentLeg(legs)) continue if (legs.some((f) => f.startsWith('metadata.json'))) continue
await fs.promises.rm(idAbs, { recursive: true, force: true }) await fs.promises.rm(idAbs, { recursive: true, force: true })
pruned[kind].push(entry.name) pruned[kind].push(entry.name)
console.warn( console.warn(
@ -709,30 +655,6 @@ export class FileSystemStorage extends BaseStorage {
return pruned return pruned
} }
/**
* @description The IMMEDIATE child directory names under a prefix ONE
* `readdir`, no recursion, no file paths. See the seam's JSDoc
* (`src/db/types.ts`) for what this replaced: discovering the generations on
* disk walked the entire generation log on every open, reading out every
* file in every generation, to learn the set of integers the top-level
* directory names already spell.
* @param prefix - Storage-root-relative directory prefix.
* @returns The child directory names (not paths); empty when the prefix does
* not exist.
*/
public override async listRawPrefixes(prefix: string): Promise<string[]> {
await this.ensureInitialized()
const fullPath = path.join(this.rootDir, prefix)
try {
const entries = await fs.promises.readdir(fullPath, { withFileTypes: true })
return entries.filter((e: { isDirectory: () => boolean }) => e.isDirectory())
.map((e: { name: string }) => e.name)
} catch (error: any) {
if (error?.code === 'ENOENT') return []
throw error
}
}
/** /**
* Primitive operation: List objects under path prefix * Primitive operation: List objects under path prefix
* All metadata operations use this internally via base class routing * All metadata operations use this internally via base class routing
@ -1943,41 +1865,18 @@ export class FileSystemStorage extends BaseStorage {
} }
} }
// THE CLEAN-CLOSE RECORD IS READ BEFORE ANY VERDICT (see
// WriterCloseRecord). A lock file whose release was RECORDED is
// bookkeeping left by an orderly shutdown, not evidence of anything —
// and that is true whether the previous holder was another process or
// an earlier instance in THIS one. A production restart reported
// "Re-acquiring writer lock ... this is a bug" immediately after a clean
// close, sending an operator hunting for a leak that did not exist.
const closeRecord = existing ? await this.readWriterCloseRecord() : null
const releasedCleanly =
existing !== null &&
closeRecord !== null &&
this.closeRecordVouchesFor(closeRecord, existing)
if (existing) { if (existing) {
// Same-process re-open: a second Brainy instance in this Node process // Same-process re-open: a second Brainy instance in this Node process
// (e.g. test "simulate server restart" patterns, or a consumer that // (e.g. test "simulate server restart" patterns, or a consumer that
// explicitly re-instantiates without closing first). This isn't the // explicitly re-instantiates without closing first). This isn't the
// dangerous cross-process case the lock exists to prevent — the two // dangerous cross-process case the lock exists to prevent — the two
// instances share a memory space and can't silently diverge from each // instances share a memory space and can't silently diverge from each
// other beyond what their callers already see. Warn and take over — // other beyond what their callers already see. Warn and take over.
// unless the record proves the previous instance already let go, in
// which case there is nothing to warn about.
if (existing.pid === myPid && existing.hostname === hostname && !options?.force) { if (existing.pid === myPid && existing.hostname === hostname && !options?.force) {
if (releasedCleanly) { console.warn(
console.warn( `[brainy] Re-acquiring writer lock for ${this.rootDir} held by the same process (PID ${existing.pid}). ` +
`[brainy] Clearing the leftover writer lock for ${this.rootDir} — an earlier ` + `If you intended to keep the previous Brainy instance alive, this is a bug — close it first.`
`instance in this process (PID ${existing.pid}) RELEASED it cleanly at ` + )
`${closeRecord!.closedAt} but could not remove the file. Nothing to recover.`
)
} else {
console.warn(
`[brainy] Re-acquiring writer lock for ${this.rootDir} held by the same process (PID ${existing.pid}). ` +
`If you intended to keep the previous Brainy instance alive, this is a bug — close it first.`
)
}
const info: WriterLockInfo = { const info: WriterLockInfo = {
pid: myPid, pid: myPid,
hostname, hostname,
@ -1987,18 +1886,11 @@ export class FileSystemStorage extends BaseStorage {
rootDir: this.rootDir rootDir: this.rootDir
} }
await this.writeFileAtomic(lockFile, JSON.stringify(info, null, 2)) await this.writeFileAtomic(lockFile, JSON.stringify(info, null, 2))
await this.clearWriterCloseRecord()
this.installWriterLock(info) this.installWriterLock(info)
return info return info
} }
// A cleanly-released lock is stale by RECORD, not by inference. Only const stale = !options?.force && (await this.isWriterLockStale(existing))
// when no record vouches for this lock do we fall back to pid
// liveness, and then we say THAT honestly too: an unrecorded lock
// means the writer did not complete its close, so the store was not
// closed cleanly and this open pays recovery.
const stale =
releasedCleanly || (!options?.force && (await this.isWriterLockStale(existing)))
if (!options?.force && !stale) { if (!options?.force && !stale) {
// Consumer-facing error contract: callers detect this case via // Consumer-facing error contract: callers detect this case via
// err.code and read the holder's details from err.lockInfo. // err.code and read the holder's details from err.lockInfo.
@ -2009,16 +1901,8 @@ export class FileSystemStorage extends BaseStorage {
options?.force options?.force
? `[brainy] Force-overwriting writer lock for ${this.rootDir} ` + ? `[brainy] Force-overwriting writer lock for ${this.rootDir} ` +
`(was held by PID ${existing.pid} on ${existing.hostname}).` `(was held by PID ${existing.pid} on ${existing.hostname}).`
: releasedCleanly : `[brainy] Overwriting stale writer lock for ${this.rootDir} ` +
? `[brainy] Clearing the leftover writer lock for ${this.rootDir}` + `(PID ${existing.pid} on ${existing.hostname} appears dead).`
`PID ${existing.pid} on ${existing.hostname} RELEASED it cleanly at ` +
`${closeRecord!.closedAt} but could not remove the file. ` +
`Nothing to recover.`
: `[brainy] Overwriting stale writer lock for ${this.rootDir} ` +
`(PID ${existing.pid} on ${existing.hostname} is gone and left NO ` +
`clean-close record — that writer did not finish closing, so this ` +
`store was not closed cleanly; open will run crash recovery and ` +
`report its wall).`
) )
// Takeover: verify the file still holds the lock we judged (a live // Takeover: verify the file still holds the lock we judged (a live
// successor may have claimed meanwhile), then remove it and fall // successor may have claimed meanwhile), then remove it and fall
@ -2072,12 +1956,6 @@ export class FileSystemStorage extends BaseStorage {
await fs.promises.unlink(claimTmp).catch(() => {}) await fs.promises.unlink(claimTmp).catch(() => {})
} }
// CONSUME the previous writer's clean-close record. It described the
// lock generation that just ended; leaving it in place would let it
// vouch for OUR lock if this process later dies without closing —
// turning a real crash into a "closed cleanly" verdict. One unlink.
await this.clearWriterCloseRecord()
this.installWriterLock(info) this.installWriterLock(info)
return info return info
} }
@ -2201,27 +2079,13 @@ export class FileSystemStorage extends BaseStorage {
return return
} }
const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE) const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE)
const released = this.writerLockInfo
try { try {
// Only delete if we still own it — avoid clobbering a successor that // Only delete if we still own it — avoid clobbering a successor that
// claimed the lock via force-override. // claimed the lock via force-override.
const current = await this.readWriterLock() const current = await this.readWriterLock()
const ours = if (current && current.pid === this.writerLockInfo.pid && current.hostname === this.writerLockInfo.hostname) {
current === null ||
(current.pid === released.pid && current.hostname === released.hostname)
if (current && ours) {
await fs.promises.unlink(lockFile) await fs.promises.unlink(lockFile)
} }
// THE CLEAN-CLOSE RECORD (see WriterCloseRecord). Written whenever this
// instance gives up a lock nobody else has taken — the unlink above
// having succeeded OR the file already being gone. The next open reads
// it instead of guessing from pid liveness: a recorded release is an
// orderly shutdown, an absent record is a writer that never finished
// closing. Not written when a successor holds the lock: our release is
// then a no-op and a record would slander their live lock.
if (ours) {
await this.writeWriterCloseRecord(released)
}
} catch (err: any) { } catch (err: any) {
if (err.code !== 'ENOENT') { if (err.code !== 'ENOENT') {
console.warn('[brainy] Failed to release writer lock file:', err) console.warn('[brainy] Failed to release writer lock file:', err)
@ -2231,97 +2095,6 @@ export class FileSystemStorage extends BaseStorage {
} }
} }
/**
* @description Read the clean-close record at `locks/_writer.close`, or
* `null` when it is absent or unparseable. A torn record is treated as
* absent the conservative direction, since an unreadable record can
* vouch for nothing.
* @returns The record, or null.
*/
public async readWriterCloseRecord(): Promise<WriterCloseRecord | null> {
await this.ensureInitialized()
const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE)
try {
const raw = await fs.promises.readFile(recordFile, 'utf-8')
const parsed = JSON.parse(raw) as WriterCloseRecord
if (
typeof parsed?.pid !== 'number' ||
typeof parsed?.hostname !== 'string' ||
typeof parsed?.startedAt !== 'string' ||
typeof parsed?.closedAt !== 'string'
) {
return null
}
return parsed
} catch (err: any) {
if (err.code === 'ENOENT') return null
return null
}
}
/**
* @description Whether a clean-close record describes the very lock
* generation `lock` represents. The match is pid + hostname + `startedAt`:
* `startedAt` is the lock generation's identity, so a record can never
* vouch for a LATER lock taken by the same pid on the same host (the
* same-process re-open path mints a fresh `startedAt`).
* @param record - The clean-close record read from disk.
* @param lock - The lock file's contents.
*/
private closeRecordVouchesFor(record: WriterCloseRecord, lock: WriterLockInfo): boolean {
return (
record.pid === lock.pid &&
record.hostname === lock.hostname &&
record.startedAt === lock.startedAt
)
}
/**
* @description Write the clean-close record for a lock this instance just
* released. Atomic (temp + rename) so a concurrent opener never reads half
* a record. A failure here costs the next open nothing but the honest
* fallback (pid liveness), so it warns rather than failing the close.
* @param released - The lock info this instance held.
*/
private async writeWriterCloseRecord(released: WriterLockInfo): Promise<void> {
const record: WriterCloseRecord = {
pid: released.pid,
hostname: released.hostname,
startedAt: released.startedAt,
closedAt: new Date().toISOString(),
version: released.version
}
const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE)
try {
await this.writeFileAtomic(recordFile, JSON.stringify(record, null, 2))
} catch (err) {
// ENOENT = the lock directory is gone, i.e. the whole store was removed
// under us. There is no next open to inform.
if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return
console.warn(
`[brainy] Failed to write the writer clean-close record for ${this.rootDir}` +
`the next open will fall back to pid liveness and may report this orderly ` +
`shutdown as a crash:`,
err
)
}
}
/**
* @description Remove the clean-close record. Called by every successful
* lock claim so a record never outlives the lock generation it describes.
*/
private async clearWriterCloseRecord(): Promise<void> {
const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE)
try {
await fs.promises.unlink(recordFile)
} catch (err: any) {
if (err.code !== 'ENOENT') {
console.warn('[brainy] Failed to clear the writer clean-close record:', err)
}
}
}
public override async readWriterLock(): Promise<WriterLockInfo | null> { public override async readWriterLock(): Promise<WriterLockInfo | null> {
await this.ensureInitialized() await this.ensureInitialized()
const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE) const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE)
@ -2400,130 +2173,44 @@ export class FileSystemStorage extends BaseStorage {
* Atomic write via temp-file-then-rename so concurrent readers never see a * Atomic write via temp-file-then-rename so concurrent readers never see a
* half-written lock JSON. Reused by writer-lock writes + heartbeat. * half-written lock JSON. Reused by writer-lock writes + heartbeat.
*/ */
/** Monotonic per-process sequence so two atomic writes never share a temp path. */
private static atomicWriteSeq = 0
private async writeFileAtomic(filePath: string, contents: string): Promise<void> { private async writeFileAtomic(filePath: string, contents: string): Promise<void> {
// pid + timestamp alone collided: two writers of the same target inside const tmp = `${filePath}.tmp-${process.pid}-${Date.now()}`
// one millisecond shared this path, and the loser's rename found the
// winner had already moved it (ENOENT). The sequence makes every call's
// temp path its own.
const tmp = `${filePath}.tmp-${process.pid}-${Date.now()}-${++FileSystemStorage.atomicWriteSeq}`
await fs.promises.writeFile(tmp, contents) await fs.promises.writeFile(tmp, contents)
await fs.promises.rename(tmp, filePath) await fs.promises.rename(tmp, filePath)
} }
/** /**
* Start watching for cross-process flush requests. Called by Brainy.init() * Start watching for cross-process flush requests. Called by Brainy.init()
* in writer mode. Each new `.req` file in `locks/_flush_requests/` triggers * in writer mode. Polls `locks/_flush_requests/` every
* the supplied callback (`brain.flush()`), after which an `.ack` is written * FLUSH_WATCH_INTERVAL_MS each new `.req` file triggers the supplied
* to `locks/_flush_responses/` with the same request ID. Stale `.req` files * callback (`brain.flush()`), after which an `.ack` is written to
* (>FLUSH_REQUEST_TTL_MS) are garbage-collected on each sweep. * `locks/_flush_responses/` with the same request ID. Stale `.req` files
* * (>FLUSH_REQUEST_TTL_MS) are garbage-collected on every tick.
* THE WATCH IS EVENT-DRIVEN, NOT A POLL. It used to `readdir` the request
* directory every 500 ms, per brain, for the entire life of every writer
* armed on every non-reader brain whether or not any inspector process
* existed. MEASURED on a production process holding 21 brains: 42 directory
* reads per second on a completely idle service, plus a stale-request GC
* pass on every one of them. The engine does no periodic work without a
* cause, and a request that has not been made is not a cause.
*
* `fs.watch` (inotify on Linux) delivers the arrival itself, so a request is
* seen SOONER than the old poll saw it. Two honest concessions ride with it:
* - a slow SAFETY SWEEP (FLUSH_SAFETY_SWEEP_MS) still runs, because
* `fs.watch` can miss events on network and fuse filesystems and because
* the stale-request GC needs some tick of its own. At 30s that is 0.7
* reads/s across 21 brains where the poll cost 42.
* - a filesystem that cannot watch at all falls back to the ORIGINAL
* 500 ms poll, narrated once, because correctness outranks idle cost:
* an inspector whose request is never seen waits forever.
*/ */
public override startFlushRequestWatcher(onRequest: () => Promise<void>): void { public override startFlushRequestWatcher(onRequest: () => Promise<void>): void {
// Already watching — or already ARMING. The arm is asynchronous (the if (this.flushWatcherInterval) return // already watching
// request directory is created before it can be watched), so neither the
// watcher nor the interval exists yet during that window; the callback is
// the flag that covers it. Without this a second call in the window would
// leave two watchers and two sweeps running for the life of the store.
if (this.flushWatcherInterval || this.flushWatcher || this.flushWatcherOnRequest) return
this.flushWatcherOnRequest = onRequest this.flushWatcherOnRequest = onRequest
const reqDir = path.join(this.lockDir, FileSystemStorage.FLUSH_REQUEST_DIR) const reqDir = path.join(this.lockDir, FileSystemStorage.FLUSH_REQUEST_DIR)
const ackDir = path.join(this.lockDir, FileSystemStorage.FLUSH_RESPONSE_DIR) const ackDir = path.join(this.lockDir, FileSystemStorage.FLUSH_RESPONSE_DIR)
const sweep = (): void => { // Ensure both dirs exist up front so the first .req drop doesn't race with mkdir.
if (this.flushWatcherInFlight) return // skip overlapping sweep this.ensureDirectoryExists(reqDir).catch(() => {})
this.ensureDirectoryExists(ackDir).catch(() => {})
this.flushWatcherInterval = setInterval(() => {
if (this.flushWatcherInFlight) return // skip overlapping tick
this.flushWatcherInFlight = true this.flushWatcherInFlight = true
this.processFlushRequests(reqDir, ackDir).finally(() => { this.processFlushRequests(reqDir, ackDir).finally(() => {
this.flushWatcherInFlight = false this.flushWatcherInFlight = false
}) })
} }, FileSystemStorage.FLUSH_WATCH_INTERVAL_MS)
// Ensure both dirs exist up front so the first .req drop doesn't race with
// mkdir — and so there is a directory to watch.
void this.ensureDirectoryExists(reqDir)
.then(() => this.ensureDirectoryExists(ackDir))
.then(() => {
if (this.flushWatcherOnRequest !== onRequest) return // stopped meanwhile
try {
const watcher = fs.watch(reqDir, () => sweep())
this.flushWatcher = watcher
watcher.on('error', (err: Error) => {
// A watch that dies mid-life must not leave the door deaf.
console.warn(
`[brainy] Flush-request watch failed (${err.message}) — falling back to polling.`
)
this.flushWatcher?.close()
this.flushWatcher = undefined
// The SAFETY sweep must go first. It is already armed at 30s, and
// startFlushRequestPolling() declines to arm over an existing
// interval — so leaving it would quietly leave this store answering
// flush requests on a 30s cadence instead of the 500ms one the door
// promises. A degrade nobody asked for is still a degrade.
if (this.flushWatcherInterval) {
clearInterval(this.flushWatcherInterval)
this.flushWatcherInterval = undefined
}
this.startFlushRequestPolling(sweep)
})
if (typeof watcher.unref === 'function') watcher.unref()
// The safety sweep: missed events on exotic filesystems, and the
// stale-request GC.
this.flushWatcherInterval = setInterval(sweep, FileSystemStorage.FLUSH_SAFETY_SWEEP_MS)
if (typeof this.flushWatcherInterval.unref === 'function') {
this.flushWatcherInterval.unref()
}
// One sweep now: a request may have been dropped before the watch armed.
sweep()
} catch (err) {
console.warn(
`[brainy] Flush-request directory cannot be watched on this filesystem ` +
`(${(err as Error).message}) — polling every ` +
`${FileSystemStorage.FLUSH_WATCH_INTERVAL_MS}ms instead.`
)
this.startFlushRequestPolling(sweep)
}
})
.catch(() => {
// The request directory could not be created; nothing to watch. A
// cross-process flush request cannot be made either, so there is
// nothing to miss.
})
}
/** The original 500 ms poll — the fallback when a directory cannot be watched. */
private startFlushRequestPolling(sweep: () => void): void {
if (this.flushWatcherInterval) return
this.flushWatcherInterval = setInterval(sweep, FileSystemStorage.FLUSH_WATCH_INTERVAL_MS)
if (typeof this.flushWatcherInterval.unref === 'function') { if (typeof this.flushWatcherInterval.unref === 'function') {
this.flushWatcherInterval.unref() this.flushWatcherInterval.unref()
} }
} }
public override stopFlushRequestWatcher(): void { public override stopFlushRequestWatcher(): void {
if (this.flushWatcher) {
this.flushWatcher.close()
this.flushWatcher = undefined
}
if (this.flushWatcherInterval) { if (this.flushWatcherInterval) {
clearInterval(this.flushWatcherInterval) clearInterval(this.flushWatcherInterval)
this.flushWatcherInterval = undefined this.flushWatcherInterval = undefined
@ -2903,46 +2590,19 @@ export class FileSystemStorage extends BaseStorage {
) { ) {
this.totalNounCountAll = counts.totalNounCountAll this.totalNounCountAll = counts.totalNounCountAll
this.totalVerbCountAll = counts.totalVerbCountAll this.totalVerbCountAll = counts.totalVerbCountAll
if (counts.allCountsDerivedBy === 'identity-record') { this.allCountsSuspect = counts.allCountsSuspect === true
// Derived (or recounted) under the honest rule — one counted
// entity per metadata content leg. Trust the persisted suspect
// flag as-is; an unprovable delete since may still have set it.
this.allCountsDerivedBy = 'identity-record'
this.allCountsSuspect = counts.allCountsSuspect === true
} else {
// The ALL scalars exist but predate the identity-record stamp —
// they were derived under the legacy rule that counted one
// entity per id DIRECTORY, so orphaned ghost/scar containers (a
// pre-8.3.1 partial-delete defect — see pruneOrphanedEntities())
// were counted as entities too. O(1) field read, NEVER a walk
// here: force suspect and name it loudly. A sanctioned recount
// (repairIndex) restores exact denominators and clears this.
this.allCountsDerivedBy = undefined
this.allCountsSuspect = true
needsPersist = true
prodLog.narrate(
'[FileSystemStorage] canonical count ledger was derived under the legacy ' +
'container rule — it counts one entity per id DIRECTORY, so every ghost/scar ' +
'container inflates it. Marked suspect, and an honest recount is scheduled to ' +
'run in the background after this open; until it lands, do not subtract ' +
'against these ALL scalars.'
)
// A suspect ledger used to stay wrong for the life of the store,
// waiting for an operator to run repairIndex. A downstream index
// heal took its "remaining" figure from these inflated
// denominators and reported work that did not exist. The ledger
// now HEALS ITSELF — in the background, because a denominator is
// a derived scalar and no read is ever served from it.
this.scheduleCountLedgerDerivation('legacy container-rule ledger')
}
} else { } else {
// No ALL scalars at all. There is nothing to serve in the meantime — const nouns = await this.scanCanonicalEntities('nouns')
// a zero would read as an empty store — so the scalars stay unknown const verbs = await this.scanCanonicalEntities('verbs')
// and SUSPECT until the background derivation lands. The open does this.totalNounCountAll = nouns.count
// not wait for it: an id-tree walk is O(ids) and this file has been this.totalVerbCountAll = verbs.count
// the whole reason a 24k-id store opened in silence. this.allCountsSuspect = false
this.allCountsSuspect = true console.warn(
this.scheduleCountLedgerDerivation('counts.json predates the ALL-visibility ledger') `[FileSystemStorage] counts.json predates the ALL-visibility count ledger — ` +
`derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` +
`every tier) and persisted; no further scan.`
)
needsPersist = true
} }
// The vectored-noun scalar (shipped after the ALL scalars above — a // The vectored-noun scalar (shipped after the ALL scalars above — a
@ -2955,12 +2615,14 @@ export class FileSystemStorage extends BaseStorage {
if (typeof counts.totalVectoredNounCount === 'number') { if (typeof counts.totalVectoredNounCount === 'number') {
this.totalVectoredNounCount = counts.totalVectoredNounCount this.totalVectoredNounCount = counts.totalVectoredNounCount
} else { } else {
// O(nouns) CONTENT reads — the most expensive derivation of the const vectored = await this.scanVectoredNounCount()
// three, and the one most likely to have been the silent minutes at this.totalVectoredNounCount = vectored
// the front of a large store's open. Background, suspect until it console.warn(
// lands, same as the ALL scalars. `[FileSystemStorage] counts.json predates the vectored-noun count ledger — ` +
this.allCountsSuspect = true `derived once by reading every noun's vectors.json (${vectored} vectored) and ` +
this.scheduleCountLedgerDerivation('counts.json predates the vectored-noun ledger') `persisted; no further scan.`
)
needsPersist = true
} }
if (needsPersist) { if (needsPersist) {
await this.persistCounts() await this.persistCounts()
@ -2989,22 +2651,6 @@ export class FileSystemStorage extends BaseStorage {
* Initialize counts by scanning disk (only done once) * Initialize counts by scanning disk (only done once)
*/ */
private async initializeCountsFromDisk(): Promise<void> { private async initializeCountsFromDisk(): Promise<void> {
const startedAt = Date.now()
// THIS ONE CANNOT LEAVE THE FOREGROUND, and the reason is worth stating:
// it derives `totalNounCount` / `totalVerbCount`, the scalars
// `getNounCount()` and `getVerbCount()` RETURN. Backgrounding it would
// make a populated store answer "0 entities" until the walk landed — a
// wrong answer, not a slow one, and the serving law grades a failure by
// whether an answer could be wrong. The ALL-visibility denominators, which
// no read is served from, DO run in the background (see
// scheduleCountLedgerDerivation). What this walk owes the operator instead
// is narration: it announces itself, and reports its wall.
prodLog.narrate(
`[FileSystemStorage] no usable counts.json — deriving the entity counters from ` +
`the canonical id tree now. This is O(ids) listings plus one vectors.json read ` +
`per noun, and it BLOCKS the open because getNounCount()/getVerbCount() are ` +
`served from it. It runs once; the result is persisted.`
)
try { try {
// Count the CANONICAL 8.0 layout (`entities/<kind>/<shard>/<id>/…`) — // Count the CANONICAL 8.0 layout (`entities/<kind>/<shard>/<id>/…`) —
// the tree saveNoun/getNouns actually read and write. The previous scan // the tree saveNoun/getNouns actually read and write. The previous scan
@ -3021,7 +2667,6 @@ export class FileSystemStorage extends BaseStorage {
this.totalNounCountAll = nouns.count this.totalNounCountAll = nouns.count
this.totalVerbCountAll = verbs.count this.totalVerbCountAll = verbs.count
this.allCountsSuspect = false this.allCountsSuspect = false
this.allCountsDerivedBy = 'identity-record'
// Vectored-noun scalar: presence needs each noun's vectors.json CONTENT // Vectored-noun scalar: presence needs each noun's vectors.json CONTENT
// (a deferred-embed noun's file exists but holds an empty vector until // (a deferred-embed noun's file exists but holds an empty vector until
// its embed lands), so this is a full O(nouns) content scan — see // its embed lands), so this is a full O(nouns) content scan — see
@ -3052,11 +2697,6 @@ export class FileSystemStorage extends BaseStorage {
} }
await this.persistCounts() await this.persistCounts()
prodLog.narrate(
`[FileSystemStorage] counter derivation from the canonical id tree finished in ` +
`${Date.now() - startedAt}ms: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs, ` +
`${this.totalVectoredNounCount} vectored nouns — persisted, stamped identity-record.`
)
} catch (error) { } catch (error) {
console.error('Error initializing counts from disk:', error) console.error('Error initializing counts from disk:', error)
} }
@ -3064,132 +2704,11 @@ export class FileSystemStorage extends BaseStorage {
/** /**
* Walk the canonical `entities/<kind>/<2-hex-shard>/<id>/` tree, counting * Walk the canonical `entities/<kind>/<2-hex-shard>/<id>/` tree, counting
* one entity per id directory that holds the metadata CONTENT leg * one entity per id directory (the layout `getNounVectorPath`/`getNouns`
* (`metadata.json` or its `.json.gz` variant see * use). Returns up to 100 sampled entity directories (absolute paths)
* {@link hasMetadataContentLeg}). A bare container a ghost (a stale * nouns feed the type-distribution estimate above. An absent tree (fresh
* `vectors.json` left with no metadata leg) or a scar (an empty directory), * store) counts zero.
* both artifacts of the pre-8.3.1 partial-delete defect counts ZERO: the
* identity record IS the population (ADR-008 G1), never the directory.
* This is the ONE-TIME legacy derivation walk (see callers); a prior
* version of this scan counted every id directory regardless of content,
* over-counting any store carrying orphaned containers see
* `allCountsDerivedBy` for how a counts.json derived under that old rule is
* marked suspect on load. Returns up to 100 sampled *counted* entity
* directories (absolute paths) nouns feed the type-distribution estimate
* above. An absent tree (fresh store) counts zero.
*/ */
/**
* @description Derive the ALL-visibility count ledger honestly one entity
* per IDENTITY RECORD, never per id directory IN THE BACKGROUND, once,
* and persist the result stamped `identity-record`.
*
* Why background: these scalars are DENOMINATORS. No read is served from
* them, so deriving them cannot be allowed to hold an open hostage a
* store with 24,898 ids spent minutes of a production restart inside walks
* exactly like these, in silence, before serving anything. Why at all: a
* ledger derived under the old container rule stayed wrong for the life of
* the store, and a downstream index heal subtracted against it and reported
* remaining work that did not exist (measured on a real store: 14,231
* derived against 14,056 identity records precisely the store's 25 noun
* scar directories; verbs 72,729 against 72,679, its 50 verb scars).
*
* Idempotent: a second call while one is in flight joins the first.
* @param reason - What made the ledger untrustworthy, quoted in narration.
* @returns Nothing; observe completion with {@link whenCountLedgerSettled}.
*/
private scheduleCountLedgerDerivation(reason: string): void {
if (this.countLedgerDerivation) return
this.countLedgerDerivation = (async () => {
const startedAt = Date.now()
prodLog.narrate(
`[FileSystemStorage] count-ledger derivation started in the background ` +
`(${reason}) — counting identity records, not id directories; the open does ` +
`not wait for it and no read is served from these scalars.`
)
try {
const beforeNouns = this.totalNounCountAll
const beforeVerbs = this.totalVerbCountAll
const beforeVectored = this.totalVectoredNounCount
// A walk that RACED A WRITE cannot prove its number: a row that landed
// mid-walk may or may not have been in the shard the walk had already
// passed. Rather than persist a figure that might be off by one and
// stamp it "exact", the walk is repeated once on a quiet store, and if
// the store is never quiet the ledger stays SUSPECT and says so. One
// retry, never a spin.
let attempt = 0
let derived: { nouns: number; verbs: number; vectored: number } | null = null
while (attempt < 2 && derived === null) {
attempt++
const activityBefore = this.ledgerActivityStamp()
const nouns = await this.scanCanonicalEntities('nouns')
const verbs = await this.scanCanonicalEntities('verbs')
const vectored = await this.scanVectoredNounCount()
if (this.ledgerActivityStamp() === activityBefore) {
derived = { nouns: nouns.count, verbs: verbs.count, vectored }
}
}
if (derived === null) {
this.allCountsSuspect = true
prodLog.narrate(
`[FileSystemStorage] count-ledger derivation could not finish on a quiet store ` +
`after ${attempt} attempts (${Date.now() - startedAt}ms) — writes landed during ` +
`every walk. The ALL-visibility scalars stay SUSPECT and must not be subtracted ` +
`against; brain.repairIndex() derives them under a recount barrier.`
)
return
}
this.totalNounCountAll = derived.nouns
this.totalVerbCountAll = derived.verbs
this.totalVectoredNounCount = derived.vectored
this.allCountsDerivedBy = 'identity-record'
this.allCountsSuspect = false
await this.persistCounts()
prodLog.narrate(
`[FileSystemStorage] count-ledger derivation finished in ${Date.now() - startedAt}ms: ` +
`${derived.nouns} nouns / ${derived.verbs} verbs / ${derived.vectored} vectored nouns` +
(beforeNouns !== derived.nouns ||
beforeVerbs !== derived.verbs ||
beforeVectored !== derived.vectored
? ` (corrected from ${beforeNouns} / ${beforeVerbs} / ${beforeVectored} — the ` +
`difference is ghost and scar containers the old rule counted as entities)`
: ' (unchanged)') +
` — persisted, stamped identity-record, no longer suspect.`
)
} catch (error) {
// The ledger stays suspect and the next open retries. Loud: a
// denominator nobody can derive is a fact an operator must have.
this.allCountsSuspect = true
prodLog.error(
`[FileSystemStorage] count-ledger derivation FAILED after ` +
`${Date.now() - startedAt}ms — the ALL-visibility scalars remain SUSPECT ` +
`and must not be subtracted against; the next open retries:`,
error
)
}
})()
}
/**
* @description A cheap witness that the ledger changed while a walk was
* running. Every landed write moves one of these live counters, so an
* unchanged stamp across a walk means no write landed during it.
* @returns A value that differs whenever the live ALL scalars have moved.
*/
private ledgerActivityStamp(): string {
return `${this.totalNounCountAll}:${this.totalVerbCountAll}:${this.totalVectoredNounCount}`
}
/**
* @description Resolve once any background count-ledger derivation has
* settled (succeeded or failed). Resolves immediately when none was needed.
* Exists so tests and operators can observe the ledger's honest value rather
* than race it; nothing in the read path waits on this.
* @returns A promise that settles with the derivation.
*/
public async whenCountLedgerSettled(): Promise<void> {
await this.countLedgerDerivation
}
private async scanCanonicalEntities( private async scanCanonicalEntities(
kind: 'nouns' | 'verbs' kind: 'nouns' | 'verbs'
): Promise<{ count: number; sampleDirs: string[] }> { ): Promise<{ count: number; sampleDirs: string[] }> {
@ -3205,21 +2724,9 @@ export class FileSystemStorage extends BaseStorage {
const ids = await fs.promises.readdir(shardPath, { withFileTypes: true }) const ids = await fs.promises.readdir(shardPath, { withFileTypes: true })
for (const entry of ids) { for (const entry of ids) {
if (!entry.isDirectory()) continue if (!entry.isDirectory()) continue
const idAbs = path.join(shardPath, entry.name)
let legs: string[]
try {
legs = await fs.promises.readdir(idAbs)
} catch (error: any) {
if (error?.code === 'ENOENT') continue
throw error
}
// No metadata content leg → a ghost or scar container → not an
// entity. Same test pruneOrphanedEntities() uses, so the two agree
// by construction.
if (!this.hasMetadataContentLeg(legs)) continue
count++ count++
if (sampleDirs.length < SAMPLE_MAX) { if (sampleDirs.length < SAMPLE_MAX) {
sampleDirs.push(idAbs) sampleDirs.push(path.join(shardPath, entry.name))
} }
} }
} }
@ -3274,20 +2781,14 @@ export class FileSystemStorage extends BaseStorage {
} }
/** /**
* Count canonical nouns holding a REAL (non-empty, non-zero-norm) vector * Count canonical nouns holding a REAL (non-empty) vector the vectored-
* the vectored-noun ledger scalar. UNLIKE {@link scanCanonicalEntities}, * noun ledger scalar. UNLIKE {@link scanCanonicalEntities}, presence
* presence cannot be decided from the id-directory listing alone: a * cannot be decided from the id-directory listing alone: a deferred-embed
* deferred-embed noun's `vectors.json` EXISTS (written at `add()` time * noun's `vectors.json` EXISTS (written at `add()` time with `vector: []`)
* with `vector: []`) until its embed LANDS, so this walk reads every * until its embed LANDS, so this walk reads every noun's `vectors.json`
* noun's `vectors.json` CONTENT O(nouns) reads, not O(ids) listing. * CONTENT O(nouns) reads, not O(ids) listing. Used ONLY for a one-time
* ZERO-NORM LAW: a real all-zero vector is not a vector it never counts * legacy-counts.json derivation or a lost/corrupted counts.json recovery;
* here either (Brainy's write paths normalize an explicit zero-norm * the result is persisted so this scan never repeats.
* vector to `[]` at write time, but a store created before that fix may
* still carry legacy all-zero rows on disk; this derivation must agree
* with the live ledger's definition of "vectored" regardless of when the
* row was written). Used ONLY for a one-time legacy-counts.json derivation
* or a lost/corrupted counts.json recovery; the result is persisted so
* this scan never repeats.
*/ */
private async scanVectoredNounCount(): Promise<number> { private async scanVectoredNounCount(): Promise<number> {
const base = path.join(this.rootDir, 'entities', 'nouns') const base = path.join(this.rootDir, 'entities', 'nouns')
@ -3301,12 +2802,7 @@ export class FileSystemStorage extends BaseStorage {
for (const entry of ids) { for (const entry of ids) {
if (!entry.isDirectory()) continue if (!entry.isDirectory()) continue
const record = await this.readEntityVectorRaw(path.join(shardPath, entry.name)) const record = await this.readEntityVectorRaw(path.join(shardPath, entry.name))
if ( if (record && Array.isArray(record.vector) && record.vector.length > 0) {
record &&
Array.isArray(record.vector) &&
record.vector.length > 0 &&
!isZeroNormVector(record.vector)
) {
vectored++ vectored++
} }
} }
@ -3338,25 +2834,13 @@ export class FileSystemStorage extends BaseStorage {
// scanVectoredNounCount()'s JSDoc). // scanVectoredNounCount()'s JSDoc).
totalVectoredNounCount: this.totalVectoredNounCount, totalVectoredNounCount: this.totalVectoredNounCount,
allCountsSuspect: this.allCountsSuspect, allCountsSuspect: this.allCountsSuspect,
// Derivation-rule stamp for the ALL scalars above — 'identity-record'
// when they were counted one-per-metadata-content-leg (the honest
// rule); omitted (JSON.stringify drops `undefined`) when the current
// in-memory scalars came from a legacy container-rule counts.json
// that hasn't been through a sanctioned recount yet, so a future load
// keeps naming them suspect rather than trusting an unproven value.
allCountsDerivedBy: this.allCountsDerivedBy,
lastUpdated: new Date().toISOString() lastUpdated: new Date().toISOString()
} }
// ATOMIC (temp + rename), never a plain writeFile. A direct write await fs.promises.writeFile(
// truncates the file first, so every persist opened a window — measured this.countsFilePath,
// at roughly 750ms after a flush or close on a real store — in which a JSON.stringify(counts, null, 2)
// concurrent reader saw counts.json EMPTY. An empty file is unparseable, )
// and an unparseable ledger sends the next open down the full-rescan
// path: the cheapest file in the store was costing the most expensive
// recovery. The rename is atomic, so a reader sees the old ledger or the
// new one, never neither.
await this.writeFileAtomic(this.countsFilePath, JSON.stringify(counts, null, 2))
} catch (error) { } catch (error) {
console.error('Error persisting counts:', error) console.error('Error persisting counts:', error)
} }

View file

@ -125,36 +125,6 @@ export interface WriterLockInfo {
rootDir?: string // Convenience for log lines / error messages rootDir?: string // Convenience for log lines / error messages
} }
/**
* THE CLEAN-CLOSE RECORD. Written by `releaseWriterLock()` at the instant it
* gives up the writer lock, naming the lock identity it released. The next
* `acquireWriterLock()` reads it and can then say from a RECORD, not from a
* guess whether the previous writer left on purpose.
*
* Why a record and not PID liveness: "the recorded PID is no longer alive" is
* true of every orderly restart AND of every crash, so the two were reported
* identically ("appears dead") and neither could be trusted. Worse, the same
* inference fails the other way when the operating system RECYCLES the pid
* a live unrelated process makes a long-dead writer's lock look held, and the
* store refuses to open naming a pid that was never Brainy. A record settles
* both: matched the previous writer closed cleanly, nothing to recover;
* absent say so, and name what recovery the open will now run.
*
* Lifecycle: written at release, consumed (deleted) by the next successful
* lock claim a record must never outlive the lock generation it describes,
* or it would vouch for a later crash.
*/
export interface WriterCloseRecord {
pid: number
hostname: string
/** `startedAt` of the lock this close released — the identity match key. */
startedAt: string
/** ISO timestamp at which the lock was released. */
closedAt: string
/** Brainy version that performed the close. */
version: string
}
/** /**
* FNV-1a hash returning a 2-char hex bucket (00-ff). * FNV-1a hash returning a 2-char hex bucket (00-ff).
* Distributes system keys across 256 sub-prefixes to avoid * Distributes system keys across 256 sub-prefixes to avoid
@ -233,40 +203,6 @@ function idFromVectorPath(path: string): string {
return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix
} }
/**
* @description Extract the entity id embedded in a metadata path
* (`entities/{nouns|verbs}/{shard}/{id}/metadata.json`) the IDENTITY-RECORD
* mirror of {@link idFromVectorPath}. The cursored noun/verb walks key their
* population on this file (ADR-008 G1: the metadata record IS the population;
* the vector leg is optional), so walk ordering and cursor resume derive the
* id from THIS path, never the vector path a row with metadata and no
* vector file must still be listed, ordered, and resumable.
* @param path - A metadata path (full or prefix-relative; must end with `/metadata.json`).
* @returns The entity id (the path segment immediately before `/metadata.json`).
*/
function idFromMetadataPath(path: string): string {
const withoutSuffix = path.replace(/\/metadata\.json$/, '')
const lastSlash = withoutSuffix.lastIndexOf('/')
return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix
}
/**
* @description The sanctioned UNVECTORED shape for a noun hydrated during
* enumeration when its identity record (metadata.json) exists but its vector
* leg (vectors.json) does not a fold-born metadata-only after-image, or any
* row genuinely without a vector yet. Mirrors the shape
* `unvectorNounForRootMigration` (src/brainy.ts) writes for the sanctioned
* unvector path (`{ vector: [], connections: new Map(), level: 0 }`), so a
* walk-yielded unvectored row is byte-shape-identical to one produced by that
* migration. Callers already handle `vector: []` as first-class
* (validateAddParams exempts it; index gates key on `length > 0`).
* @param id - The noun id.
* @returns A structurally-valid, vector-empty `HNSWNoun`.
*/
function unvectoredNoun(id: string): HNSWNoun {
return { id, vector: [], connections: new Map<number, Set<string>>(), level: 0 }
}
/** /**
* Get ID-first path for verb metadata * Get ID-first path for verb metadata
* No type parameter needed - direct O(1) lookup by ID * No type parameter needed - direct O(1) lookup by ID
@ -1437,29 +1373,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
return this.listObjectsUnderPath(prefix) return this.listObjectsUnderPath(prefix)
} }
/**
* @description The IMMEDIATE child directory names under a prefix one
* level, no recursion. See the seam's JSDoc (`db/types.ts`) for why a
* separate door exists. This default derives them from the recursive
* listing, so it is never WRONG, only never faster; the filesystem adapter
* overrides it with a single directory read.
* @param prefix - Storage-root-relative directory prefix.
* @returns The child directory names (not paths), in listing order.
*/
public async listRawPrefixes(prefix: string): Promise<string[]> {
await this.ensureInitialized()
const paths = await this.listObjectsUnderPath(prefix)
const normalizedPrefix = prefix.endsWith('/') ? prefix : `${prefix}/`
const names = new Set<string>()
for (const p of paths) {
const rest = p.startsWith(normalizedPrefix) ? p.slice(normalizedPrefix.length) : null
if (rest === null) continue
const slash = rest.search(/[/\\]/)
if (slash > 0) names.add(rest.slice(0, slash))
}
return [...names]
}
/** /**
* Remove every object under a storage-root-relative prefix. The filesystem * Remove every object under a storage-root-relative prefix. The filesystem
* adapter overrides this with a recursive directory removal; this default * adapter overrides this with a recursive directory removal; this default
@ -1540,18 +1453,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
* rollups are derived state with their own rebuild paths * rollups are derived state with their own rebuild paths
* (`rebuildTypeCounts()` / `rebuildSubtypeCounts()`). * (`rebuildTypeCounts()` / `rebuildSubtypeCounts()`).
* *
* EXACT-RESTORE PRIMITIVE `vector: null` DELETES the vector leg, on
* purpose: `GenerationStore.rollBackUncommittedGeneration()` depends on
* this to legitimately un-write a vector a failed transaction added. This
* is deliberately NOT "preserve if absent" a caller replaying an
* AFTER-IMAGE (the recovery fold, `GenerationStore`'s `replayFact`) must
* apply preserve-if-absent itself BEFORE calling this, by reading the
* current vector and carrying it forward when the after-image's own
* vector leg is null/undefined but its metadata is not (see `replayFact`
* for the implementation and full rationale). A caller that genuinely
* wants to unvector a row uses the sanctioned, ledger-correct path
* (`Brainy.unvectorNounForRootMigration`) never this primitive.
*
* @param id - The entity id. * @param id - The entity id.
* @param record - Raw stored objects as returned by {@link BaseStorage.readNounRaw}. * @param record - Raw stored objects as returned by {@link BaseStorage.readNounRaw}.
*/ */
@ -1588,9 +1489,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
/** /**
* Restore a relationship's raw stored objects byte-for-byte (verb-side * Restore a relationship's raw stored objects byte-for-byte (verb-side
* mirror of {@link BaseStorage.writeNounRaw}; same bookkeeping caveats, * mirror of {@link BaseStorage.writeNounRaw}; same bookkeeping caveats).
* same EXACT-RESTORE contract `vector: null` deletes, on purpose; the
* fold's preserve-if-absent logic lives at its call site, not here).
* *
* @param id - The relationship id. * @param id - The relationship id.
* @param record - Raw stored objects as returned by {@link BaseStorage.readVerbRaw}. * @param record - Raw stored objects as returned by {@link BaseStorage.readVerbRaw}.
@ -2284,18 +2183,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// Stable within-shard order (by noun id) so offset windows and cursor resume // Stable within-shard order (by noun id) so offset windows and cursor resume
// are deterministic; ids come from the path so skipped nouns are never read. // are deterministic; ids come from the path so skipped nouns are never read.
//
// IDENTITY-KEYED WALK (population law, ADR-008 G1): the metadata record
// (not the vector) IS the population — a noun with metadata and no vector
// file (a fold-born after-image, see writeNounRaw's preserve-if-absent
// contract) must still enumerate. Keying on metadata.json here means the
// ledger recount (rebuildTypeCounts' `allNouns`, also metadata.json-keyed)
// and this walk agree on population by construction. Ordering is
// unaffected for a healthy store: every vectored noun has both legs, so
// the id set and sort order are identical to the old vectors.json keying.
const entries = nounFiles const entries = nounFiles
.filter((p) => p.includes('/metadata.json')) .filter((p) => p.includes('/vectors.json'))
.map((p) => ({ path: p, id: idFromMetadataPath(p) })) .map((p) => ({ path: p, id: idFromVectorPath(p) }))
.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)) .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
// Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor
@ -2320,24 +2210,13 @@ export abstract class BaseStorage extends BaseStorageAdapter {
) { ) {
const batch = toHydrate.slice(i, i + BaseStorage.HYDRATE_CONCURRENCY) const batch = toHydrate.slice(i, i + BaseStorage.HYDRATE_CONCURRENCY)
const hydrated = await Promise.all( const hydrated = await Promise.all(
batch.map(async ({ path: metadataPath, id }) => { batch.map(async ({ path: nounPath }) => {
try { try {
const metadata = await this.readCanonicalObject(metadataPath) const noun = await this.readCanonicalObject(nounPath)
if (!noun) return null
const deserialized = this.deserializeNoun(noun)
const metadata = await this.getNounMetadata(deserialized.id)
if (!metadata) return null if (!metadata) return null
// The vector leg is OPTIONAL (population law): a metadata-only
// row hydrates with the sanctioned unvectored shape rather than
// being dropped from the walk. A fault reading the vector leg
// is treated the same as absence — best-effort, matching the
// canonical recount's tolerance for an unreadable vectors.json
// (rebuildTypeCounts) — a vector-leg problem never hides an
// otherwise-good identity record.
let deserialized: HNSWNoun
try {
const vectorRecord = await this.readCanonicalObject(getNounVectorPath(id))
deserialized = vectorRecord ? this.deserializeNoun(vectorRecord) : unvectoredNoun(id)
} catch {
deserialized = unvectoredNoun(id)
}
return { deserialized, metadata } return { deserialized, metadata }
} catch (error) { } catch (error) {
// A TORN record must surface typed — a paginated read that // A TORN record must surface typed — a paginated read that
@ -2347,9 +2226,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// walk's job is to HEAL PAST it — skip the victim, serve the rest. // walk's job is to HEAL PAST it — skip the victim, serve the rest.
// Identity point-reads (get-by-id) still throw typed upstream. // Identity point-reads (get-by-id) still throw typed upstream.
if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ }
// Skip nouns whose IDENTITY record fails to load (the metadata // Skip nouns that fail to load
// read above) — that is the one leg this walk cannot proceed
// without.
return null return null
} }
}) })
@ -2470,14 +2347,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
const shardDir = `entities/nouns/${shardHex}` const shardDir = `entities/nouns/${shardHex}`
try { try {
const nounFiles = await this.listCanonicalObjects(shardDir) const nounFiles = await this.listCanonicalObjects(shardDir)
// IDENTITY-KEYED WALK (population law, ADR-008 G1) — see the matching
// comment in getNounsWithPagination: metadata.json is the population;
// the vector leg is optional, so a metadata-only row must still be
// listed (and here, for the unfiltered case, needs ZERO reads either
// way — the id comes straight from the path).
const entries = nounFiles const entries = nounFiles
.filter((p) => p.includes('/metadata.json')) .filter((p) => p.includes('/vectors.json'))
.map((p) => idFromMetadataPath(p)) .map((p) => idFromVectorPath(p))
.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)) .sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))
const toWalk = const toWalk =
cursor && shard === cursor.shard ? entries.filter((id) => id > cursor.id) : entries cursor && shard === cursor.shard ? entries.filter((id) => id > cursor.id) : entries
@ -2688,79 +2560,23 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// Stable within-shard order (by verb id) so offset windows and cursor resume // Stable within-shard order (by verb id) so offset windows and cursor resume
// are deterministic and consistent across calls. Ids come from the path, so // are deterministic and consistent across calls. Ids come from the path, so
// verbs skipped by the cursor are never read. // verbs skipped by the cursor are never read.
//
// IDENTITY-KEYED WALK (population law, ADR-008 G1) — the noun mirror of
// this comment in getNounsWithPagination applies here too: metadata.json
// is the population; keying on it here means this walk and the ledger
// recount (rebuildTypeCounts' `allVerbs`, already metadata.json-keyed)
// agree on population by construction. Unchanged for a healthy store —
// `relate()` always writes both legs of a verb in the same commit, so
// the id set and order match the old vectors.json keying exactly; this
// only additionally surfaces a fold-born metadata-only row (see
// writeVerbRaw's preserve-if-absent contract).
const entries = verbFiles const entries = verbFiles
.filter((p) => p.includes('/metadata.json')) .filter((p) => p.includes('/vectors.json'))
.map((p) => ({ path: p, id: idFromMetadataPath(p) })) .map((p) => ({ path: p, id: idFromVectorPath(p) }))
.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)) .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
for (const { path: metadataPath, id: verbId } of entries) { for (const { path: verbPath, id: verbId } of entries) {
if (collected.length >= peekCount) break if (collected.length >= peekCount) break
// Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor id // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor id
// (later shards are processed in full). No read for skipped verbs. // (later shards are processed in full). No read for skipped verbs.
if (cursor && shard === cursor.shard && verbId <= cursor.id) continue if (cursor && shard === cursor.shard && verbId <= cursor.id) continue
try { try {
// Identity leg first — required. A verb this walk cannot read const rawVerb = await this.readCanonicalObject(verbPath)
// metadata for cannot be hydrated at all (same as before). if (!rawVerb) continue
const metadata = await this.readCanonicalObject(metadataPath)
if (!metadata) continue
// The vector leg is the verb's STRUCTURAL core (verb/sourceId/ // Deserialize connections Map from JSON storage format
// targetId live there — see coreTypes.ts HNSWVerb), unlike a const verb = this.deserializeVerb(rawVerb)
// noun's vector, which is pure embedding data. `relate()` always
// writes both legs atomically and verbs have no deferred-embed
// path, so a healthy store's verbs always have both. A vector-leg
// absence here can only be a fold-born after-image (see
// writeVerbRaw's preserve-if-absent contract) — and unlike a
// noun, this walk cannot safely FABRICATE sourceId/targetId to
// synthesize a structurally-valid verb (an empty-string endpoint
// would silently create a phantom edge — worse than omission).
// If the metadata record happens to carry its own sourceId/
// targetId (never true for current production writes, but not
// disallowed — e.g. a future schema or a repair tool could
// populate them), reconstruct from those; otherwise this row is
// loudly skipped — counted by the ledger, but not returned as an
// item, until a repair can supply the missing endpoints.
const rawVerb = await this.readCanonicalObject(getVerbVectorPath(verbId))
let verb: HNSWVerb
if (rawVerb) {
verb = this.deserializeVerb(rawVerb)
} else {
const metaSourceId = (metadata as Record<string, unknown>).sourceId
const metaTargetId = (metadata as Record<string, unknown>).targetId
const metaVerbType = (metadata as Record<string, unknown>).verb
if (
typeof metaSourceId === 'string' && metaSourceId.length > 0 &&
typeof metaTargetId === 'string' && metaTargetId.length > 0 &&
typeof metaVerbType === 'string' && metaVerbType.length > 0
) {
verb = {
id: verbId,
vector: [],
connections: new Map<number, Set<string>>(),
verb: metaVerbType as VerbType,
sourceId: metaSourceId,
targetId: metaTargetId
}
} else {
prodLog.error(
`[BaseStorage] getVerbsWithPagination: verb ${verbId} has a metadata ` +
`record but no vector leg and no recoverable sourceId/targetId — ` +
`skipping (counted by the ledger, not yielded; needs repair).`
)
continue
}
}
// Apply type filter // Apply type filter
if (filterVerbTypes && !filterVerbTypes.has(verb.verb)) { if (filterVerbTypes && !filterVerbTypes.has(verb.verb)) {
@ -2777,6 +2593,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
continue continue
} }
// Load metadata
const metadata = await this.getVerbMetadata(verb.id)
// Apply subtype filter (requires metadata — checked AFTER load) // Apply subtype filter (requires metadata — checked AFTER load)
if (filterSubtypes) { if (filterSubtypes) {
const subtype = metadata?.subtype as string | undefined const subtype = metadata?.subtype as string | undefined
@ -2942,33 +2761,19 @@ export abstract class BaseStorage extends BaseStorageAdapter {
!options.filter.service && !options.filter.service &&
!options.filter.metadata !options.filter.metadata
) { ) {
const sourceIds = Array.isArray(options.filter.sourceId) const sourceId = Array.isArray(options.filter.sourceId)
? options.filter.sourceId ? options.filter.sourceId[0]
: [options.filter.sourceId] : options.filter.sourceId
// EVERY requested verb type is honoured — an array used to collapse to const verbType = Array.isArray(options.filter.verbType)
// its first element here, silently dropping the rest of the ask. ? options.filter.verbType[0]
const verbTypes = new Set( : options.filter.verbType
Array.isArray(options.filter.verbType)
? options.filter.verbType
: [options.filter.verbType]
)
// Get verbs by source (union over every requested source), filter by the // Get verbs by source, then filter by type (O(1) graph lookup + O(n) type filter),
// requested type SET (O(1) graph lookup + O(n) type filter), then apply // then apply the subtype / visibility metadata filters on the candidate set.
// the subtype / visibility metadata filters on the candidate set. const verbsBySource = await this.getVerbsBySource_internal(sourceId)
const bySource: HNSWVerbWithMetadata[] = []
const seenVerbIds = new Set<string>()
for (const oneSource of sourceIds) {
for (const v of await this.getVerbsBySource_internal(oneSource)) {
if (!seenVerbIds.has(v.id)) {
seenVerbIds.add(v.id)
bySource.push(v)
}
}
}
const filteredVerbs = this.applyVerbMetadataFilters( const filteredVerbs = this.applyVerbMetadataFilters(
bySource.filter(v => verbTypes.has(v.verb)), verbsBySource.filter(v => v.verb === verbType),
options.filter options.filter
) )
@ -2999,22 +2804,16 @@ export abstract class BaseStorage extends BaseStorageAdapter {
!options.filter.service && !options.filter.service &&
!options.filter.metadata !options.filter.metadata
) { ) {
// EVERY requested source is honoured — an array used to collapse to const sourceId = Array.isArray(options.filter.sourceId)
// its first element here, silently dropping the rest of the ask. ? options.filter.sourceId[0]
const onlySourceIds = Array.isArray(options.filter.sourceId) : options.filter.sourceId
? options.filter.sourceId
: [options.filter.sourceId] // Get verbs by source directly (hydrated with metadata), then apply the
const sourceUnion: HNSWVerbWithMetadata[] = [] // subtype / visibility metadata filters on the O(degree) candidate set.
const seenSourceVerbIds = new Set<string>() const verbsBySource = this.applyVerbMetadataFilters(
for (const oneSource of onlySourceIds) { await this.getVerbsBySource_internal(sourceId),
for (const v of await this.getVerbsBySource_internal(oneSource)) { options.filter
if (!seenSourceVerbIds.has(v.id)) { )
seenSourceVerbIds.add(v.id)
sourceUnion.push(v)
}
}
}
const verbsBySource = this.applyVerbMetadataFilters(sourceUnion, options.filter)
// Apply pagination // Apply pagination
const paginatedVerbs = verbsBySource.slice(offset, offset + limit) const paginatedVerbs = verbsBySource.slice(offset, offset + limit)
@ -3043,22 +2842,16 @@ export abstract class BaseStorage extends BaseStorageAdapter {
!options.filter.service && !options.filter.service &&
!options.filter.metadata !options.filter.metadata
) { ) {
// EVERY requested target is honoured — an array used to collapse to const targetId = Array.isArray(options.filter.targetId)
// its first element here, silently dropping the rest of the ask. ? options.filter.targetId[0]
const onlyTargetIds = Array.isArray(options.filter.targetId) : options.filter.targetId
? options.filter.targetId
: [options.filter.targetId] // Get verbs by target directly (hydrated with metadata), then apply the
const targetUnion: HNSWVerbWithMetadata[] = [] // subtype / visibility metadata filters on the O(degree) candidate set.
const seenTargetVerbIds = new Set<string>() const verbsByTarget = this.applyVerbMetadataFilters(
for (const oneTarget of onlyTargetIds) { await this.getVerbsByTarget_internal(targetId),
for (const v of await this.getVerbsByTarget_internal(oneTarget)) { options.filter
if (!seenTargetVerbIds.has(v.id)) { )
seenTargetVerbIds.add(v.id)
targetUnion.push(v)
}
}
}
const verbsByTarget = this.applyVerbMetadataFilters(targetUnion, options.filter)
// Apply pagination // Apply pagination
const paginatedVerbs = verbsByTarget.slice(offset, offset + limit) const paginatedVerbs = verbsByTarget.slice(offset, offset + limit)
@ -3087,25 +2880,16 @@ export abstract class BaseStorage extends BaseStorageAdapter {
!options.filter.service && !options.filter.service &&
!options.filter.metadata !options.filter.metadata
) { ) {
// EVERY requested verb type is honoured — an array used to collapse to const verbType = Array.isArray(options.filter.verbType)
// its first element here, silently dropping the rest of the ask. ? options.filter.verbType[0]
const verbTypes = Array.isArray(options.filter.verbType) : options.filter.verbType
? options.filter.verbType
: [options.filter.verbType]
// Get verbs by each requested type (hydrated with metadata), deduped by // Get verbs by type directly (hydrated with metadata), then apply the
// id, then apply the subtype / visibility metadata filters on the set. // subtype / visibility metadata filters on the candidate set.
const byType: HNSWVerbWithMetadata[] = [] const verbsByType = this.applyVerbMetadataFilters(
const seenTypeVerbIds = new Set<string>() await this.getVerbsByType_internal(verbType),
for (const oneType of verbTypes) { options.filter
for (const v of await this.getVerbsByType_internal(oneType)) { )
if (!seenTypeVerbIds.has(v.id)) {
seenTypeVerbIds.add(v.id)
byType.push(v)
}
}
}
const verbsByType = this.applyVerbMetadataFilters(byType, options.filter)
// Apply pagination // Apply pagination
const paginatedVerbs = verbsByType.slice(offset, offset + limit) const paginatedVerbs = verbsByType.slice(offset, offset + limit)
@ -4908,10 +4692,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
this.totalVerbCountAll = allVerbs this.totalVerbCountAll = allVerbs
this.totalVectoredNounCount = allVectoredNouns this.totalVectoredNounCount = allVectoredNouns
this.allCountsSuspect = false this.allCountsSuspect = false
// This walk counts one entity per metadata.json record (never per bare
// container) — the identity-record rule. Stamp it so a future load
// trusts these scalars instead of naming them suspect at open.
this.allCountsDerivedBy = 'identity-record'
this.countCache.clear() this.countCache.clear()
await this.persistCounts() await this.persistCounts()

Some files were not shown because too many files have changed in this diff Show more