Compare commits
No commits in common. "v10.4.0-rc.1" and "v10.3.1" have entirely different histories.
v10.4.0-rc
...
v10.3.1
22 changed files with 87 additions and 1412 deletions
|
|
@ -34,16 +34,7 @@ jobs:
|
||||||
|
|
||||||
SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/"
|
SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/"
|
||||||
VERSION="$(node -p "require('./package.json').version")"
|
VERSION="$(node -p "require('./package.json').version")"
|
||||||
# The dist-tag follows the version: a prerelease (any hyphen —
|
echo "Publishing @soulcraft/brainy@${VERSION} to The Source registry..."
|
||||||
# 10.4.0-rc.1) publishes under 'rc' and must NEVER move 'latest' —
|
|
||||||
# every consumer resolving 'latest' from this registry would otherwise
|
|
||||||
# be handed a release candidate. Same rule scripts/release.sh applies
|
|
||||||
# to the storefront leg.
|
|
||||||
NPM_TAG="latest"
|
|
||||||
case "$VERSION" in
|
|
||||||
*-*) NPM_TAG="rc" ;;
|
|
||||||
esac
|
|
||||||
echo "Publishing @soulcraft/brainy@${VERSION} to The Source registry (dist-tag: ${NPM_TAG})..."
|
|
||||||
|
|
||||||
TMPRC="$(mktemp)"
|
TMPRC="$(mktemp)"
|
||||||
chmod 600 "$TMPRC"
|
chmod 600 "$TMPRC"
|
||||||
|
|
@ -56,7 +47,7 @@ jobs:
|
||||||
# this tag's checkout already carries the version being published —
|
# this tag's checkout already carries the version being published —
|
||||||
# nothing here re-derives it from the tag name.
|
# nothing here re-derives it from the tag name.
|
||||||
PUBLISH_OK=true
|
PUBLISH_OK=true
|
||||||
if ! npm publish --tag "$NPM_TAG" --userconfig "$TMPRC"; then
|
if ! npm publish --tag latest --userconfig "$TMPRC"; then
|
||||||
PUBLISH_OK=false
|
PUBLISH_OK=false
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
|
||||||
13
CHANGELOG.md
13
CHANGELOG.md
|
|
@ -2,19 +2,6 @@
|
||||||
|
|
||||||
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
||||||
|
|
||||||
### [10.4.0-rc.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.3.1...v10.4.0-rc.1) (2026-08-24)
|
|
||||||
|
|
||||||
- ci(publish): the home dist-tag follows the version — a prerelease publishes under 'rc' and never moves 'latest' (a1376e4a)
|
|
||||||
- chore(release): --source-only — a home-only prerelease mode (The Source, never the storefront) (dcbad176)
|
|
||||||
- test(fold-checkpoint): the ARM-AT-FLIP pin arms its crash instead of racing the pending-flush timer (4176439b)
|
|
||||||
- fix(health): one contract for a throwing probe — heal is none, serving is not withheld; repair report gains missing/rebuilt/reason (116550eb)
|
|
||||||
- feat(storage): the canonical count ledger — ALL-visibility scalars, unclamped totals, suspect-on-unprovable-delete (7c8c8be3)
|
|
||||||
- fix(delete): the null-metadata skip closes — index legs run id-keyed or narrate, never silently strand postings (607e9f54)
|
|
||||||
- feat(repair): repairIndex returns the per-family receipt and narrates its summary (8d45f964)
|
|
||||||
- fix(reads): the readiness gate guards every index read surface — serving empty from a not-ready provider is unrepresentable (40e7119b)
|
|
||||||
- ci(gate): the machine-health preflight and the truncation verdict guard (1e046aa1)
|
|
||||||
|
|
||||||
|
|
||||||
### [10.3.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.3.0...v10.3.1) (2026-08-18)
|
### [10.3.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.3.0...v10.3.1) (2026-08-18)
|
||||||
|
|
||||||
- docs(releases): the 10.3.1 consumer entry — the fold that behaves (900cc895)
|
- docs(releases): the 10.3.1 consumer entry — the fold that behaves (900cc895)
|
||||||
|
|
|
||||||
4
package-lock.json
generated
4
package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
||||||
{
|
{
|
||||||
"name": "@soulcraft/brainy",
|
"name": "@soulcraft/brainy",
|
||||||
"version": "10.4.0-rc.1",
|
"version": "10.3.1",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "@soulcraft/brainy",
|
"name": "@soulcraft/brainy",
|
||||||
"version": "10.4.0-rc.1",
|
"version": "10.3.1",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@msgpack/msgpack": "^3.1.2",
|
"@msgpack/msgpack": "^3.1.2",
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "@soulcraft/brainy",
|
"name": "@soulcraft/brainy",
|
||||||
"version": "10.4.0-rc.1",
|
"version": "10.3.1",
|
||||||
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
|
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"module": "dist/index.js",
|
"module": "dist/index.js",
|
||||||
|
|
|
||||||
|
|
@ -1,85 +0,0 @@
|
||||||
# Gate Guards
|
|
||||||
|
|
||||||
Two standalone scripts that stand between a test/build gate and a false
|
|
||||||
verdict: one refuses to let the gate start on a noisy machine, the other
|
|
||||||
refuses to let a truncated or crashed vitest run be read as green.
|
|
||||||
|
|
||||||
## Why these exist
|
|
||||||
|
|
||||||
Both guards exist because of the 2026-08-13 lost-day ledger: a gate ran on
|
|
||||||
a machine under load, and separately a vitest worker pool died mid-suite
|
|
||||||
while still printing a plausible-looking summary line, and in both cases
|
|
||||||
the bad result was trusted and acted on for the better part of a day before
|
|
||||||
anyone noticed. Neither failure mode announces itself — a loaded machine
|
|
||||||
still finishes and reports numbers, and a truncated test run still prints a
|
|
||||||
`Test Files` / `Tests` line — so both guards check the evidence explicitly
|
|
||||||
rather than trusting that a gate finishing means the gate was valid.
|
|
||||||
|
|
||||||
## gate-preflight.sh
|
|
||||||
|
|
||||||
Run before any gate lane starts. Exits 1 the moment the machine isn't
|
|
||||||
gate-clean, with one `FATAL:` line per violation naming the exact offender
|
|
||||||
(the pid and command, the path, the measured value). Prints one `OK:` line
|
|
||||||
per check that passes. `WARNING:` lines mark checks that were skipped, not
|
|
||||||
failures.
|
|
||||||
|
|
||||||
Checks:
|
|
||||||
|
|
||||||
| # | Check | Default threshold | Override |
|
|
||||||
|---|-------|--------------------|----------|
|
|
||||||
| a | 1-minute load average | `nproc / 2` | `GATE_MAX_LOAD` |
|
|
||||||
| b | any non-allowlisted process over 50% of one core | 50% | `GATE_ALLOW_REGEX` (extra pattern matched against the process's args) |
|
|
||||||
| c | cpu0 scaling governor must be `performance` | — | none (warns and skips if the sysfs path is absent) |
|
|
||||||
| d | free space on `/` and `/tmp` | 10G each | `GATE_SKIP_DISK_CHECK=1` to skip entirely |
|
|
||||||
|
|
||||||
The allowlist for check (b) is always: this script's own process tree
|
|
||||||
(its ancestors and its direct child processes), `sshd`, `systemd`, and
|
|
||||||
kernel threads (recognizable by args wrapped in brackets, e.g.
|
|
||||||
`[kworker/0:1]`). `GATE_ALLOW_REGEX` extends it — it does not replace it.
|
|
||||||
|
|
||||||
## vitest-verdict-check.sh
|
|
||||||
|
|
||||||
Run after every vitest lane, against that lane's captured log. Fails
|
|
||||||
loudly, quoting the exact line or string that tripped it, when the log's
|
|
||||||
own summary can't be trusted:
|
|
||||||
|
|
||||||
- no `Test Files` (or, in `--count-tests` mode, `Tests`) summary line is
|
|
||||||
present at all
|
|
||||||
- the parenthesized total in that line doesn't match what was expected
|
|
||||||
- fewer files/tests are accounted for (passed + failed + skipped) than the
|
|
||||||
total claims — a truncated run
|
|
||||||
- the log contains `Unhandled Error` or `Timeout calling` anywhere — a dead
|
|
||||||
worker pool, regardless of what the summary line claims
|
|
||||||
|
|
||||||
```
|
|
||||||
vitest-verdict-check.sh <log-file> <expected-file-count>
|
|
||||||
vitest-verdict-check.sh --count-tests <log-file> <minimum-test-count>
|
|
||||||
```
|
|
||||||
|
|
||||||
The first form checks `Test Files` for an exact match. The second checks
|
|
||||||
`Tests` for a minimum (a floor, not an exact count, since the total number
|
|
||||||
of individual tests moves more often than the number of test files).
|
|
||||||
|
|
||||||
## Wiring into a CI lane
|
|
||||||
|
|
||||||
```sh
|
|
||||||
# Before any lane that will report a verdict:
|
|
||||||
scripts/gate/gate-preflight.sh || exit 1
|
|
||||||
|
|
||||||
# Run the suite, capturing its output:
|
|
||||||
npx vitest run tests/unit 2>&1 | tee /tmp/unit.log
|
|
||||||
|
|
||||||
# After every vitest lane, check the log against the actual file count:
|
|
||||||
EXPECTED_FILES=$(ls tests/unit/**/*.test.ts | wc -l)
|
|
||||||
scripts/gate/vitest-verdict-check.sh /tmp/unit.log "$EXPECTED_FILES" || exit 1
|
|
||||||
```
|
|
||||||
|
|
||||||
## Exit-code contract
|
|
||||||
|
|
||||||
| Script | Exit 0 | Exit 1 |
|
|
||||||
|--------|--------|--------|
|
|
||||||
| `gate-preflight.sh` | machine is gate-clean | one or more `FATAL:` violations printed |
|
|
||||||
| `vitest-verdict-check.sh` | log's summary is trustworthy and matches | usage error, missing/unreadable log, or one or more `FATAL:` violations printed |
|
|
||||||
|
|
||||||
Non-zero from either script means: do not trust the gate that was about to
|
|
||||||
run, or the result of the one that just ran.
|
|
||||||
|
|
@ -1,206 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
# Brainy Gate Preflight
|
|
||||||
# Refuses to let a test/build gate run on a machine that isn't clean enough
|
|
||||||
# to trust the numbers it produces. See scripts/gate/README.md for why (the
|
|
||||||
# 2026-08-13 lost-day ledger).
|
|
||||||
#
|
|
||||||
# Checks: 1-minute load average, any non-allowlisted process pinning a core,
|
|
||||||
# the cpu0 scaling governor, and free space on / and /tmp.
|
|
||||||
#
|
|
||||||
# Exit 0 and print one OK line per passing check when the machine is clean.
|
|
||||||
# Exit 1 and print one FATAL line per violation, naming the offender, when
|
|
||||||
# it is not.
|
|
||||||
#
|
|
||||||
# Known trap: a helper function whose last executed statement is a `while`
|
|
||||||
# (or any command whose own exit status happens to be nonzero) hands that
|
|
||||||
# status back as the function's return value. Called as a plain statement,
|
|
||||||
# that silently kills this script under `set -e`. Every helper below ends
|
|
||||||
# on an explicit `return 0` as its own statement, never on a loop or test.
|
|
||||||
#
|
|
||||||
# The same failure mode hides in plainer-looking lines too: `var=$(cmd)` is
|
|
||||||
# a bare assignment, so `set -e` DOES treat a nonzero `cmd` (or, under
|
|
||||||
# `pipefail`, a nonzero stage anywhere in `cmd`'s pipeline) as a failure of
|
|
||||||
# that statement and kills the script right there — even mid-loop, even
|
|
||||||
# when the "failure" is routine (a process that exited before a second
|
|
||||||
# lookup, a path that doesn't exist). Every such assignment below is paired
|
|
||||||
# with an explicit `|| var=""` fallback so a routine miss degrades to an
|
|
||||||
# empty value instead of an exit.
|
|
||||||
|
|
||||||
VIOLATIONS=0
|
|
||||||
ANCESTOR_PIDS=""
|
|
||||||
|
|
||||||
fatal() {
|
|
||||||
echo "FATAL: $1"
|
|
||||||
VIOLATIONS=$((VIOLATIONS + 1))
|
|
||||||
}
|
|
||||||
|
|
||||||
ok() {
|
|
||||||
echo "OK: $1"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Walks this process's parent chain up to pid 1, then takes one snapshot of
|
|
||||||
# its direct children (the ps/read pipeline in check_processes), and
|
|
||||||
# records both in ANCESTOR_PIDS — so the process-scan below can recognize
|
|
||||||
# its own tree (the shell/terminal/session that launched it, plus its own
|
|
||||||
# helper commands) instead of flagging it. Children are captured once, up
|
|
||||||
# front, rather than re-queried per row later, so a helper command that has
|
|
||||||
# already exited by the time it's looked up can't be mistaken for a miss.
|
|
||||||
build_ancestor_pids() {
|
|
||||||
local pid="$$"
|
|
||||||
local ppid child
|
|
||||||
ANCESTOR_PIDS=" $pid "
|
|
||||||
while [ "$pid" != "1" ]; do
|
|
||||||
ppid=$(ps -o ppid= -p "$pid" 2>/dev/null | tr -d ' ') || ppid=""
|
|
||||||
if [ -z "$ppid" ]; then
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
ANCESTOR_PIDS="${ANCESTOR_PIDS}${ppid} "
|
|
||||||
pid="$ppid"
|
|
||||||
done
|
|
||||||
|
|
||||||
while IFS= read -r child; do
|
|
||||||
[ -z "$child" ] && continue
|
|
||||||
ANCESTOR_PIDS="${ANCESTOR_PIDS}${child} "
|
|
||||||
done < <(ps --ppid "$$" -o pid= 2>/dev/null || true)
|
|
||||||
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
# (a) 1-minute load average vs. threshold (default: nproc / 2).
|
|
||||||
check_load() {
|
|
||||||
local max_load="${GATE_MAX_LOAD:-}"
|
|
||||||
if [ -z "$max_load" ]; then
|
|
||||||
max_load=$(( $(nproc) / 2 ))
|
|
||||||
if [ "$max_load" -lt 1 ]; then
|
|
||||||
max_load=1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
local load_1m
|
|
||||||
load_1m=$(cut -d' ' -f1 /proc/loadavg)
|
|
||||||
|
|
||||||
if awk -v l="$load_1m" -v m="$max_load" 'BEGIN { exit !(l > m) }'; then
|
|
||||||
fatal "1-minute load average ${load_1m} exceeds threshold ${max_load} (GATE_MAX_LOAD=${max_load})"
|
|
||||||
else
|
|
||||||
ok "1-minute load average ${load_1m} is within threshold ${max_load}"
|
|
||||||
fi
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
# (b) any process outside the allowlist pinning more than half a core.
|
|
||||||
# Parsed with `read` into named fields, not an awk/cut chain — a fixed-column
|
|
||||||
# awk/cut split on `ps` output duplicated fields the first time this was
|
|
||||||
# tried, because process args vary in word count. `read` with a fixed list
|
|
||||||
# of variables dumps everything left over into the last one (args), which
|
|
||||||
# handles that correctly.
|
|
||||||
check_processes() {
|
|
||||||
local max_pcpu=50
|
|
||||||
local extra_regex="${GATE_ALLOW_REGEX:-}"
|
|
||||||
local violation_found=0
|
|
||||||
local line pcpu pid args pcpu_int
|
|
||||||
|
|
||||||
while IFS= read -r line; do
|
|
||||||
[ -z "$line" ] && continue
|
|
||||||
read -r pcpu pid args <<< "$line"
|
|
||||||
|
|
||||||
# Kernel threads report their comm in brackets, e.g. "[kworker/0:1]".
|
|
||||||
case "$args" in
|
|
||||||
\[*\]) continue ;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
# This script's own tree: its ancestors (shell, terminal, session) and
|
|
||||||
# its direct children, both captured once by build_ancestor_pids.
|
|
||||||
case " $ANCESTOR_PIDS " in
|
|
||||||
*" $pid "*) continue ;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
case "$args" in
|
|
||||||
*sshd*|*systemd*) continue ;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
if [ -n "$extra_regex" ] && [[ "$args" =~ $extra_regex ]]; then
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
|
|
||||||
pcpu_int="${pcpu%.*}"
|
|
||||||
if [ -z "$pcpu_int" ]; then
|
|
||||||
pcpu_int=0
|
|
||||||
fi
|
|
||||||
if [ "$pcpu_int" -gt "$max_pcpu" ]; then
|
|
||||||
fatal "pid ${pid} ('${args}') is using ${pcpu}% of one core"
|
|
||||||
violation_found=1
|
|
||||||
fi
|
|
||||||
done < <(ps -eo pcpu,pid,args --sort=-pcpu | tail -n +2)
|
|
||||||
|
|
||||||
if [ "$violation_found" -eq 0 ]; then
|
|
||||||
ok "no process outside the allowlist exceeds ${max_pcpu}% of one core"
|
|
||||||
fi
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
# (c) cpu0 scaling governor must be "performance". Skipped with a warning
|
|
||||||
# (not a violation) when the sysfs path doesn't exist on this machine.
|
|
||||||
check_governor() {
|
|
||||||
local gov_path="/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor"
|
|
||||||
if [ ! -r "$gov_path" ]; then
|
|
||||||
echo "WARNING: ${gov_path} not present; skipping governor check"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
local governor
|
|
||||||
governor=$(cat "$gov_path" 2>/dev/null) || governor=""
|
|
||||||
if [ "$governor" != "performance" ]; then
|
|
||||||
fatal "cpu0 governor is '${governor}', not 'performance'"
|
|
||||||
else
|
|
||||||
ok "cpu0 governor is 'performance'"
|
|
||||||
fi
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
# (d) free-space floors on / and /tmp (default 10G each). Skip entirely via
|
|
||||||
# GATE_SKIP_DISK_CHECK=1.
|
|
||||||
check_disk() {
|
|
||||||
if [ "${GATE_SKIP_DISK_CHECK:-0}" = "1" ]; then
|
|
||||||
echo "WARNING: disk free-space check skipped (GATE_SKIP_DISK_CHECK=1)"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
local floor_gb=10
|
|
||||||
local floor_bytes=$((floor_gb * 1024 * 1024 * 1024))
|
|
||||||
local path avail_bytes avail_gb
|
|
||||||
|
|
||||||
for path in / /tmp; do
|
|
||||||
avail_bytes=$(df --output=avail -B1 "$path" 2>/dev/null | tail -n 1 | tr -d ' ') || avail_bytes=""
|
|
||||||
if [ -z "$avail_bytes" ]; then
|
|
||||||
echo "WARNING: could not determine free space on ${path}; skipping"
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
if [ "$avail_bytes" -lt "$floor_bytes" ]; then
|
|
||||||
avail_gb=$((avail_bytes / 1024 / 1024 / 1024))
|
|
||||||
fatal "${path} has only ${avail_gb}G free, below the ${floor_gb}G floor"
|
|
||||||
else
|
|
||||||
ok "${path} has enough free space (floor ${floor_gb}G)"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
echo "Brainy gate preflight"
|
|
||||||
echo "----------------------"
|
|
||||||
|
|
||||||
build_ancestor_pids
|
|
||||||
check_load
|
|
||||||
check_processes
|
|
||||||
check_governor
|
|
||||||
check_disk
|
|
||||||
|
|
||||||
echo "----------------------"
|
|
||||||
if [ "$VIOLATIONS" -gt 0 ]; then
|
|
||||||
echo "FATAL: gate preflight failed with ${VIOLATIONS} violation(s) — machine is not gate-clean"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "gate preflight passed — machine is gate-clean"
|
|
||||||
exit 0
|
|
||||||
|
|
@ -1,158 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
# Brainy Vitest Verdict Check
|
|
||||||
# Confirms a vitest run's own summary line is trustworthy before anything
|
|
||||||
# downstream treats a green run as green. See scripts/gate/README.md for why
|
|
||||||
# (the 2026-08-13 lost-day ledger).
|
|
||||||
#
|
|
||||||
# Usage:
|
|
||||||
# vitest-verdict-check.sh <log-file> <expected-file-count>
|
|
||||||
# vitest-verdict-check.sh --count-tests <log-file> <minimum-test-count>
|
|
||||||
#
|
|
||||||
# The first form checks the "Test Files" summary line's total against an
|
|
||||||
# exact expected count. The second checks the "Tests" summary line's total
|
|
||||||
# against a minimum. Both also fail on any sign the worker pool died
|
|
||||||
# mid-run, whether or not a summary line still made it into the log.
|
|
||||||
#
|
|
||||||
# Exit 0 and print one OK line per passing check when the log is clean.
|
|
||||||
# Exit 1 and print one FATAL line per violation, quoting the exact line or
|
|
||||||
# string that tripped it, when it is not.
|
|
||||||
#
|
|
||||||
# Known trap (shared with gate-preflight.sh): every helper below ends on an
|
|
||||||
# explicit `return 0` as its own statement, never on a loop or test, so a
|
|
||||||
# helper's last command can never hand its own exit status back as the
|
|
||||||
# function's under `set -e`. The same applies to `var=$(cmd)` assignments
|
|
||||||
# mid-helper: a bare assignment IS checked by `set -e`, so a `grep` that
|
|
||||||
# legitimately finds nothing (exit 1) would otherwise kill the script
|
|
||||||
# instead of just leaving the variable empty — every such assignment below
|
|
||||||
# is paired with an explicit `|| true` inside the substitution.
|
|
||||||
|
|
||||||
usage() {
|
|
||||||
echo "Usage: $0 <log-file> <expected-file-count>"
|
|
||||||
echo " $0 --count-tests <log-file> <minimum-test-count>"
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
MODE="files"
|
|
||||||
if [ "${1:-}" = "--count-tests" ]; then
|
|
||||||
MODE="tests"
|
|
||||||
shift
|
|
||||||
fi
|
|
||||||
|
|
||||||
LOG_FILE="${1:-}"
|
|
||||||
THRESHOLD="${2:-}"
|
|
||||||
|
|
||||||
if [ -z "$LOG_FILE" ] || [ -z "$THRESHOLD" ]; then
|
|
||||||
usage
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ ! -f "$LOG_FILE" ]; then
|
|
||||||
echo "FATAL: log file '${LOG_FILE}' does not exist"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if ! [[ "$THRESHOLD" =~ ^[0-9]+$ ]]; then
|
|
||||||
echo "FATAL: threshold '${THRESHOLD}' is not a non-negative integer"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
VIOLATIONS=0
|
|
||||||
|
|
||||||
fatal() {
|
|
||||||
echo "FATAL: $1"
|
|
||||||
VIOLATIONS=$((VIOLATIONS + 1))
|
|
||||||
}
|
|
||||||
|
|
||||||
ok() {
|
|
||||||
echo "OK: $1"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Vitest colorizes its summary with ANSI escapes; strip them before parsing
|
|
||||||
# anything, or the color codes end up embedded in the fields we grep for.
|
|
||||||
CLEAN_LOG="$(sed 's/\x1b\[[0-9;]*m//g' "$LOG_FILE")"
|
|
||||||
|
|
||||||
# Worker-pool death: if either string appears, the run's own summary line —
|
|
||||||
# even if present and even if its numbers look fine — cannot be trusted,
|
|
||||||
# because the process died mid-suite and vitest's own accounting is what
|
|
||||||
# died with it.
|
|
||||||
check_worker_death() {
|
|
||||||
if echo "$CLEAN_LOG" | grep -q "Unhandled Error"; then
|
|
||||||
fatal "log contains 'Unhandled Error' — worker pool died mid-run"
|
|
||||||
fi
|
|
||||||
if echo "$CLEAN_LOG" | grep -q "Timeout calling"; then
|
|
||||||
fatal "log contains 'Timeout calling' — worker pool died mid-run"
|
|
||||||
fi
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
# Shared shape between the "Test Files" and "Tests" summary lines:
|
|
||||||
# <label> <n> passed | <n> failed | <n> skipped (<total>)
|
|
||||||
# `compare` is "eq" (total must equal threshold) or "min" (total must be at
|
|
||||||
# least threshold).
|
|
||||||
check_summary_line() {
|
|
||||||
local label="$1"
|
|
||||||
local threshold="$2"
|
|
||||||
local compare="$3"
|
|
||||||
local summary_line total accounted n
|
|
||||||
|
|
||||||
summary_line=$(echo "$CLEAN_LOG" | grep -E "^[[:space:]]*${label}[[:space:]]+" | tail -n 1 || true)
|
|
||||||
|
|
||||||
if [ -z "$summary_line" ]; then
|
|
||||||
fatal "no '${label}' summary line found in ${LOG_FILE}"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
total=$(echo "$summary_line" | grep -oE '\([0-9]+\)' | tr -d '()' | tail -n 1 || true)
|
|
||||||
if [ -z "$total" ]; then
|
|
||||||
fatal "'${label}' summary line has no parenthesized total: \"${summary_line}\""
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ "$compare" = "eq" ]; then
|
|
||||||
if [ "$total" -ne "$threshold" ]; then
|
|
||||||
fatal "'${label}' total is ${total}, expected ${threshold}: \"${summary_line}\""
|
|
||||||
else
|
|
||||||
ok "'${label}' total matches expected ${threshold}"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
if [ "$total" -lt "$threshold" ]; then
|
|
||||||
fatal "'${label}' total is ${total}, below minimum ${threshold}: \"${summary_line}\""
|
|
||||||
else
|
|
||||||
ok "'${label}' total ${total} meets minimum ${threshold}"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
accounted=0
|
|
||||||
for n in $(echo "$summary_line" | grep -oE '[0-9]+ (passed|failed|skipped)' | grep -oE '^[0-9]+'); do
|
|
||||||
accounted=$((accounted + n))
|
|
||||||
done
|
|
||||||
|
|
||||||
if [ "$accounted" -lt "$total" ]; then
|
|
||||||
fatal "'${label}' line accounts for only ${accounted} of ${total} — truncated run: \"${summary_line}\""
|
|
||||||
else
|
|
||||||
ok "'${label}' line accounts for all ${total}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
echo "Brainy vitest verdict check: ${LOG_FILE}"
|
|
||||||
echo "----------------------"
|
|
||||||
|
|
||||||
check_worker_death
|
|
||||||
|
|
||||||
if [ "$MODE" = "files" ]; then
|
|
||||||
check_summary_line "Test Files" "$THRESHOLD" "eq"
|
|
||||||
else
|
|
||||||
check_summary_line "Tests" "$THRESHOLD" "min"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "----------------------"
|
|
||||||
if [ "$VIOLATIONS" -gt 0 ]; then
|
|
||||||
echo "FATAL: vitest verdict check failed with ${VIOLATIONS} violation(s) for ${LOG_FILE}"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "vitest verdict check passed for ${LOG_FILE}"
|
|
||||||
exit 0
|
|
||||||
|
|
@ -15,12 +15,6 @@ NC='\033[0m' # No Color
|
||||||
RELEASE_TYPE="${1:-patch}" # patch, minor, or major
|
RELEASE_TYPE="${1:-patch}" # patch, minor, or major
|
||||||
SKIP_TESTS=false
|
SKIP_TESTS=false
|
||||||
DRY_RUN=false
|
DRY_RUN=false
|
||||||
# --source-only: the HOME leg only — tag, CI's publish to The Source, and the
|
|
||||||
# release page; NO storefront (npmjs) publish, NO pair verification, NO docs
|
|
||||||
# push. The pair-gate shape: a prerelease the fleet's other engine devDeps
|
|
||||||
# from our own registry while the pair is proven, never a public artifact.
|
|
||||||
# Refused for a non-prerelease version — a public floor is always a pair.
|
|
||||||
SOURCE_ONLY=false
|
|
||||||
|
|
||||||
for arg in "$@"; do
|
for arg in "$@"; do
|
||||||
case $arg in
|
case $arg in
|
||||||
|
|
@ -30,9 +24,6 @@ for arg in "$@"; do
|
||||||
--dry-run)
|
--dry-run)
|
||||||
DRY_RUN=true
|
DRY_RUN=true
|
||||||
;;
|
;;
|
||||||
--source-only)
|
|
||||||
SOURCE_ONLY=true
|
|
||||||
;;
|
|
||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
|
|
||||||
|
|
@ -109,7 +100,7 @@ else
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
echo -e "${RED}❌ Invalid release type: ${RELEASE_TYPE}${NC}"
|
echo -e "${RED}❌ Invalid release type: ${RELEASE_TYPE}${NC}"
|
||||||
echo "Usage: ./scripts/release.sh [patch|minor|major|<explicit-version>] [--dry-run] [--source-only (prereleases only)]"
|
echo "Usage: ./scripts/release.sh [patch|minor|major|<explicit-version>] [--dry-run]"
|
||||||
exit 1
|
exit 1
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
@ -128,13 +119,6 @@ echo -e "${BLUE}New version: ${NEW_VERSION}${NC}"
|
||||||
if [ "$PRERELEASE" = true ]; then
|
if [ "$PRERELEASE" = true ]; then
|
||||||
echo -e "${YELLOW}⚠️ Prerelease → npm dist-tag '${NPM_TAG}', GitHub prerelease${NC}"
|
echo -e "${YELLOW}⚠️ Prerelease → npm dist-tag '${NPM_TAG}', GitHub prerelease${NC}"
|
||||||
fi
|
fi
|
||||||
if [ "$SOURCE_ONLY" = true ]; then
|
|
||||||
if [ "$PRERELEASE" != true ]; then
|
|
||||||
echo -e "${RED}❌ --source-only is for prereleases only: a non-prerelease version is a public floor and always ships as the byte-identical pair.${NC}"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
echo -e "${YELLOW}⚠️ --source-only → The Source (home) ONLY: no npmjs publish, no pair verification, no docs push${NC}"
|
|
||||||
fi
|
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
if [ "$DRY_RUN" = true ]; then
|
if [ "$DRY_RUN" = true ]; then
|
||||||
|
|
@ -237,34 +221,31 @@ else
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ "$SOURCE_ONLY" = true ]; then
|
echo -e "${BLUE}9️⃣½ Publishing to npmjs (storefront, dist-tag: ${NPM_TAG})...${NC}"
|
||||||
echo -e "${YELLOW}9️⃣½ Storefront (npmjs) leg SKIPPED — --source-only: v${NEW_VERSION} lives on The Source under dist-tag '${NPM_TAG}' only${NC}\n"
|
# BYTE-IDENTITY LAW: the storefront republishes CI's EXACT artifact — download
|
||||||
else
|
# the tarball The Source serves and publish that file, never a fresh local pack
|
||||||
echo -e "${BLUE}9️⃣½ Publishing to npmjs (storefront, dist-tag: ${NPM_TAG})...${NC}"
|
# (a local rebuild can differ byte-wise, and the fleet verifies the pair by
|
||||||
# BYTE-IDENTITY LAW: the storefront republishes CI's EXACT artifact — download
|
# shasum across registries).
|
||||||
# the tarball The Source serves and publish that file, never a fresh local pack
|
STOREFRONT_TMP="$(mktemp -d)"
|
||||||
# (a local rebuild can differ byte-wise, and the fleet verifies the pair by
|
(cd "$STOREFRONT_TMP" && npm pack "@soulcraft/brainy@${NEW_VERSION}" "--@soulcraft:registry=${SOURCE_NPM_REG}" >/dev/null)
|
||||||
# shasum across registries).
|
SOURCE_TARBALL="$(ls "$STOREFRONT_TMP"/soulcraft-brainy-*.tgz)"
|
||||||
STOREFRONT_TMP="$(mktemp -d)"
|
echo -e "${BLUE} home artifact: $(sha256sum "$SOURCE_TARBALL" | cut -d' ' -f1)${NC}"
|
||||||
(cd "$STOREFRONT_TMP" && npm pack "@soulcraft/brainy@${NEW_VERSION}" "--@soulcraft:registry=${SOURCE_NPM_REG}" >/dev/null)
|
npm publish "$SOURCE_TARBALL" --tag "$NPM_TAG" "--@soulcraft:registry=https://registry.npmjs.org/"
|
||||||
SOURCE_TARBALL="$(ls "$STOREFRONT_TMP"/soulcraft-brainy-*.tgz)"
|
rm -rf "$STOREFRONT_TMP"
|
||||||
echo -e "${BLUE} home artifact: $(sha256sum "$SOURCE_TARBALL" | cut -d' ' -f1)${NC}"
|
# Brainy is the only PUBLIC @soulcraft package — verify visibility after every publish.
|
||||||
npm publish "$SOURCE_TARBALL" --tag "$NPM_TAG" "--@soulcraft:registry=https://registry.npmjs.org/"
|
npm access get status @soulcraft/brainy "--@soulcraft:registry=https://registry.npmjs.org/" || true
|
||||||
rm -rf "$STOREFRONT_TMP"
|
# Verify the pair is byte-identical by registry-reported shasum — divergence
|
||||||
# Brainy is the only PUBLIC @soulcraft package — verify visibility after every publish.
|
# here means the storefront leg must be treated as failed, loudly. RETRIED
|
||||||
npm access get status @soulcraft/brainy "--@soulcraft:registry=https://registry.npmjs.org/" || true
|
# with raw curl: npmjs metadata propagates with a lag measured in minutes,
|
||||||
# Verify the pair is byte-identical by registry-reported shasum — divergence
|
# and a one-shot npm-view probe fired a false DIVERGENCE on 10.0.0 while a
|
||||||
# here means the storefront leg must be treated as failed, loudly. RETRIED
|
# raw curl of the registry document already confirmed byte-identity. The
|
||||||
# with raw curl: npmjs metadata propagates with a lag measured in minutes,
|
# probe now reads the registry JSON directly (no npm cache in the path) and
|
||||||
# and a one-shot npm-view probe fired a false DIVERGENCE on 10.0.0 while a
|
# gives propagation up to 5 minutes before calling the pair divergent.
|
||||||
# raw curl of the registry document already confirmed byte-identity. The
|
NPMJS_VERIFY_ATTEMPTS=20
|
||||||
# probe now reads the registry JSON directly (no npm cache in the path) and
|
NPMJS_VERIFY_INTERVAL_S=15 # 20 × 15s = 5 minutes of propagation grace
|
||||||
# gives propagation up to 5 minutes before calling the pair divergent.
|
SOURCE_SHA=$(npm view "@soulcraft/brainy@${NEW_VERSION}" dist.shasum "--@soulcraft:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "source-unavailable")
|
||||||
NPMJS_VERIFY_ATTEMPTS=20
|
PAIR_IDENTICAL=false
|
||||||
NPMJS_VERIFY_INTERVAL_S=15 # 20 × 15s = 5 minutes of propagation grace
|
for ((attempt = 1; attempt <= NPMJS_VERIFY_ATTEMPTS; attempt++)); do
|
||||||
SOURCE_SHA=$(npm view "@soulcraft/brainy@${NEW_VERSION}" dist.shasum "--@soulcraft:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "source-unavailable")
|
|
||||||
PAIR_IDENTICAL=false
|
|
||||||
for ((attempt = 1; attempt <= NPMJS_VERIFY_ATTEMPTS; attempt++)); do
|
|
||||||
NPMJS_SHA=$(curl -fsSL "https://registry.npmjs.org/@soulcraft%2Fbrainy" 2>/dev/null \
|
NPMJS_SHA=$(curl -fsSL "https://registry.npmjs.org/@soulcraft%2Fbrainy" 2>/dev/null \
|
||||||
| node -e "let d='';process.stdin.on('data',c=>d+=c).on('end',()=>{try{const v=JSON.parse(d).versions[process.argv[1]];console.log(v?v.dist.shasum:'')}catch{console.log('')}})" "${NEW_VERSION}" \
|
| node -e "let d='';process.stdin.on('data',c=>d+=c).on('end',()=>{try{const v=JSON.parse(d).versions[process.argv[1]];console.log(v?v.dist.shasum:'')}catch{console.log('')}})" "${NEW_VERSION}" \
|
||||||
|| echo "")
|
|| echo "")
|
||||||
|
|
@ -274,13 +255,12 @@ else
|
||||||
fi
|
fi
|
||||||
echo -e "${YELLOW} … npmjs metadata not settled (attempt ${attempt}/${NPMJS_VERIFY_ATTEMPTS}: '${NPMJS_SHA:-absent}' vs '${SOURCE_SHA}'); retrying in ${NPMJS_VERIFY_INTERVAL_S}s${NC}"
|
echo -e "${YELLOW} … npmjs metadata not settled (attempt ${attempt}/${NPMJS_VERIFY_ATTEMPTS}: '${NPMJS_SHA:-absent}' vs '${SOURCE_SHA}'); retrying in ${NPMJS_VERIFY_INTERVAL_S}s${NC}"
|
||||||
sleep "$NPMJS_VERIFY_INTERVAL_S"
|
sleep "$NPMJS_VERIFY_INTERVAL_S"
|
||||||
done
|
done
|
||||||
if [ "$PAIR_IDENTICAL" = true ]; then
|
if [ "$PAIR_IDENTICAL" = true ]; then
|
||||||
echo -e "${GREEN}✅ Published to npmjs — byte-identical pair (shasum ${NPMJS_SHA})${NC}\n"
|
echo -e "${GREEN}✅ Published to npmjs — byte-identical pair (shasum ${NPMJS_SHA})${NC}\n"
|
||||||
else
|
else
|
||||||
echo -e "${RED}❌ REGISTRY DIVERGENCE: The Source shasum ${SOURCE_SHA} != npmjs shasum ${NPMJS_SHA} after ${NPMJS_VERIFY_ATTEMPTS} attempts — investigate before announcing${NC}\n"
|
echo -e "${RED}❌ REGISTRY DIVERGENCE: The Source shasum ${SOURCE_SHA} != npmjs shasum ${NPMJS_SHA} after ${NPMJS_VERIFY_ATTEMPTS} attempts — investigate before announcing${NC}\n"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Step 11: Release object on The Source (presentational — the tag, CHANGELOG,
|
# Step 11: Release object on The Source (presentational — the tag, CHANGELOG,
|
||||||
|
|
@ -303,24 +283,16 @@ fi
|
||||||
# DOCS_INGEST_SECRET is unset; fails loudly (without undoing the publish —
|
# DOCS_INGEST_SECRET is unset; fails loudly (without undoing the publish —
|
||||||
# that already happened) when a push errors, so the docs site never
|
# that already happened) when a push errors, so the docs site never
|
||||||
# silently trails npm.
|
# silently trails npm.
|
||||||
if [ "$SOURCE_ONLY" = true ]; then
|
echo -e "${BLUE}1️⃣2️⃣ Pushing public docs to soulcraft.com/docs...${NC}"
|
||||||
echo -e "${YELLOW}1️⃣2️⃣ Docs push SKIPPED — --source-only (a home-only prerelease publishes no public docs)${NC}\n"
|
if node scripts/push-docs.js; then
|
||||||
else
|
|
||||||
echo -e "${BLUE}1️⃣2️⃣ Pushing public docs to soulcraft.com/docs...${NC}"
|
|
||||||
if node scripts/push-docs.js; then
|
|
||||||
echo -e "${GREEN}✅ Docs push step done${NC}\n"
|
echo -e "${GREEN}✅ Docs push step done${NC}\n"
|
||||||
else
|
else
|
||||||
echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n"
|
echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n"
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||||
echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}"
|
echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}"
|
||||||
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||||
echo ""
|
echo ""
|
||||||
if [ "$SOURCE_ONLY" = true ]; then
|
echo -e "📦 npm: ${BLUE}https://www.npmjs.com/package/@soulcraft/brainy/v/${NEW_VERSION}${NC}"
|
||||||
echo -e "📦 npmjs: ${YELLOW}not published (--source-only)${NC}"
|
|
||||||
else
|
|
||||||
echo -e "📦 npm: ${BLUE}https://www.npmjs.com/package/@soulcraft/brainy/v/${NEW_VERSION}${NC}"
|
|
||||||
fi
|
|
||||||
echo -e "🏠 The Source: ${BLUE}https://source.soulcraft.com/soulcraft/brainy/releases/tag/v${NEW_VERSION}${NC}"
|
echo -e "🏠 The Source: ${BLUE}https://source.soulcraft.com/soulcraft/brainy/releases/tag/v${NEW_VERSION}${NC}"
|
||||||
|
|
|
||||||
197
src/brainy.ts
197
src/brainy.ts
|
|
@ -144,9 +144,7 @@ import {
|
||||||
ScoreExplanation,
|
ScoreExplanation,
|
||||||
FillSubtypeRule,
|
FillSubtypeRule,
|
||||||
FillSubtypeRules,
|
FillSubtypeRules,
|
||||||
FillSubtypesResult,
|
FillSubtypesResult
|
||||||
RepairReport,
|
|
||||||
RepairFamilyReport
|
|
||||||
} from './types/brainy.types.js'
|
} from './types/brainy.types.js'
|
||||||
import { NounType, VerbType, TypeUtils } from './types/graphTypes.js'
|
import { NounType, VerbType, TypeUtils } from './types/graphTypes.js'
|
||||||
import {
|
import {
|
||||||
|
|
@ -3692,22 +3690,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// stored, so remove() deletes the same entity. A real UUID passes through.
|
// stored, so remove() deletes the same entity. A real UUID passes through.
|
||||||
id = resolveEntityId(id)
|
id = resolveEntityId(id)
|
||||||
|
|
||||||
// Get entity metadata and related verbs before deletion. TORN-TOLERANT:
|
// Get entity metadata and related verbs before deletion
|
||||||
// a torn record must still be deletable (the delete IS the cure) — a
|
const metadata = await this.storage.getNounMetadata(id)
|
||||||
// torn pre-read reads as null and the null-path below handles it loudly.
|
const noun = await this.storage.getNoun(id)
|
||||||
let metadata: any = null
|
|
||||||
let noun: any = null
|
|
||||||
try {
|
|
||||||
metadata = await this.storage.getNounMetadata(id)
|
|
||||||
} catch (err) {
|
|
||||||
if ((err as { code?: string }).code !== 'TORN_RECORD') throw err
|
|
||||||
prodLog.warn(`[Brainy] remove(${id}): metadata pre-read is TORN — deleting anyway; index legs run id-keyed`)
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
noun = await this.storage.getNoun(id)
|
|
||||||
} catch (err) {
|
|
||||||
if ((err as { code?: string }).code !== 'TORN_RECORD') throw err
|
|
||||||
}
|
|
||||||
const verbs = await this.storage.getVerbsBySource(id)
|
const verbs = await this.storage.getVerbsBySource(id)
|
||||||
const targetVerbs = await this.storage.getVerbsByTarget(id)
|
const targetVerbs = await this.storage.getVerbsByTarget(id)
|
||||||
const allVerbs = [...verbs, ...targetVerbs]
|
const allVerbs = [...verbs, ...targetVerbs]
|
||||||
|
|
@ -3725,61 +3710,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Operation 2: Remove from metadata index. THE NULL-METADATA SKIP IS
|
// Operation 2: Remove from metadata index
|
||||||
// CLOSED (a posting-leak class, confirmed at this site): when the
|
|
||||||
// pre-read missed, the leg no longer silently skips —
|
|
||||||
// - a provider exposing removeEntityById (the id-keyed contract)
|
|
||||||
// gets it: exact per-entity retraction via its reverse record;
|
|
||||||
// - the JS index gets removeFromIndex(id) — safe id-keyed cleanup
|
|
||||||
// (deleted bitmap + id mapper; field stats reconcile at rebuild);
|
|
||||||
// - a NATIVE provider WITHOUT the contract is never called
|
|
||||||
// metadata-omitted (that path walks its value space) — the skip
|
|
||||||
// happens, but NARRATED and tracked in the degraded set so
|
|
||||||
// repairIndex reconciles it. Silence is the only thing outlawed.
|
|
||||||
if (metadata) {
|
if (metadata) {
|
||||||
tx.addOperation(
|
tx.addOperation(
|
||||||
new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata, this.indexWriteGeneration)
|
new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata, this.indexWriteGeneration)
|
||||||
)
|
)
|
||||||
} else {
|
|
||||||
const prov = this.metadataIndex as unknown as {
|
|
||||||
removeEntityById?: (id: string) => Promise<void>
|
|
||||||
removeFromIndex?: (id: string, metadata?: unknown, generation?: bigint) => Promise<void>
|
|
||||||
}
|
|
||||||
if (typeof prov.removeEntityById === 'function') {
|
|
||||||
const g = this.indexWriteGeneration
|
|
||||||
tx.addOperation({
|
|
||||||
name: 'RemoveEntityByIdTombstone',
|
|
||||||
execute: async () => {
|
|
||||||
await prov.removeEntityById!(id)
|
|
||||||
return async () => {
|
|
||||||
// Undo of an id-keyed tombstone on an absent row: nothing
|
|
||||||
// to restore (the row had no readable metadata to re-post).
|
|
||||||
void g
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
} else if (this.metadataIndex instanceof MetadataIndexManager) {
|
|
||||||
const gv = this.indexWriteGeneration
|
|
||||||
tx.addOperation({
|
|
||||||
name: 'IdKeyedIndexCleanup',
|
|
||||||
execute: async () => {
|
|
||||||
await prov.removeFromIndex!(id, undefined, typeof gv === 'function' ? gv() : gv)
|
|
||||||
return async () => {}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
prodLog.warn(
|
|
||||||
`[Brainy] remove(${id}): no metadata at delete — id-keyed index cleanup ran ` +
|
|
||||||
`(deleted bitmap + id mapper); field statistics reconcile at the next rebuild/repairIndex.`
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
this._indexDegradedIds.add(id)
|
|
||||||
prodLog.warn(
|
|
||||||
`[Brainy] remove(${id}): no metadata at delete and this provider has no id-keyed ` +
|
|
||||||
`removal — its postings for this id are NOT tombstoned yet (tracked as degraded; ` +
|
|
||||||
`repairIndex() reconciles). Never calling a metadata-omitted native removal: that ` +
|
|
||||||
`path walks the store's value space.`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Operation 3: Delete noun (full removal). The pre-read metadata rides
|
// Operation 3: Delete noun (full removal). The pre-read metadata rides
|
||||||
|
|
@ -3972,12 +3907,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
uuid: string,
|
uuid: string,
|
||||||
options?: { direction?: 'in' | 'out' | 'both'; limit?: number; offset?: number }
|
options?: { direction?: 'in' | 'out' | 'both'; limit?: number; offset?: number }
|
||||||
): Promise<string[]> {
|
): Promise<string[]> {
|
||||||
// READ-SURFACE READINESS GATE (the 4.2.4 blackout's brainy half): every
|
|
||||||
// index read funnels through this helper, so the gate here makes
|
|
||||||
// serve-while-not-ready UNREPRESENTABLE — a production store once acked
|
|
||||||
// writes while every non-find() read served empty from a not-ready
|
|
||||||
// provider for 15 minutes. Fast path after the latch is one boolean.
|
|
||||||
await this.ensureIndexesLoaded()
|
|
||||||
const entityInt = this.graphEntityInt(uuid)
|
const entityInt = this.graphEntityInt(uuid)
|
||||||
if (entityInt === undefined) return []
|
if (entityInt === undefined) return []
|
||||||
const neighborInts = await this.graphIndex.getNeighbors(entityInt, options)
|
const neighborInts = await this.graphIndex.getNeighbors(entityInt, options)
|
||||||
|
|
@ -11878,12 +11807,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
filter: unknown,
|
filter: unknown,
|
||||||
opts?: { limit?: number; offset?: number }
|
opts?: { limit?: number; offset?: number }
|
||||||
): Promise<string[]> {
|
): Promise<string[]> {
|
||||||
// READ-SURFACE READINESS GATE (the 4.2.4 blackout's brainy half): every
|
|
||||||
// index read funnels through this helper, so the gate here makes
|
|
||||||
// serve-while-not-ready UNREPRESENTABLE — a production store once acked
|
|
||||||
// writes while every non-find() read served empty from a not-ready
|
|
||||||
// provider for 15 minutes. Fast path after the latch is one boolean.
|
|
||||||
await this.ensureIndexesLoaded()
|
|
||||||
try {
|
try {
|
||||||
return await this.metadataIndex.getIdsForFilter(filter, opts)
|
return await this.metadataIndex.getIdsForFilter(filter, opts)
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|
@ -14221,29 +14144,16 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
const report = await fn.call(provider)
|
const report = await fn.call(provider)
|
||||||
if (report && Array.isArray(report.invariants)) reports.push(report)
|
if (report && Array.isArray(report.invariants)) reports.push(report)
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
// ONE CONTRACT FOR A THROWING PROBE, both engines: a probe that throws
|
|
||||||
// is `heal: 'none'` with the error in `detail` — flakiness can never
|
|
||||||
// buy a rebuild, and a thrown check never changes `serving` (the
|
|
||||||
// provider's serving verdict is composed by the provider, not inferred
|
|
||||||
// from a probe that failed to run). This catch used to synthesize
|
|
||||||
// `heal: 'rebuild'` — the read-triggered dark-rebuild lever one
|
|
||||||
// transient exception away — while the native composer said 'none';
|
|
||||||
// two components disagreeing on what a throw means is how a flaky
|
|
||||||
// probe became an outage. `healthy: false` stays: an unrunnable probe
|
|
||||||
// is a named, loud, unverified state, never a clean bill.
|
|
||||||
const name = typeof (provider as { name?: string })?.name === 'string'
|
|
||||||
? (provider as { name: string }).name
|
|
||||||
: 'unknown'
|
|
||||||
reports.push({
|
reports.push({
|
||||||
provider: name,
|
provider: 'unknown',
|
||||||
healthy: false,
|
healthy: false,
|
||||||
serving: true,
|
serving: false,
|
||||||
invariants: [
|
invariants: [
|
||||||
{
|
{
|
||||||
name: 'validate-invariants-threw',
|
name: 'validate-invariants-threw',
|
||||||
holds: false,
|
holds: false,
|
||||||
detail: `validateInvariants() threw (contract violation — it must never throw): ${(err as Error).message}`,
|
detail: `validateInvariants() threw (contract violation — it must never throw): ${(err as Error).message}`,
|
||||||
heal: 'none'
|
heal: 'rebuild'
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
checkedAt: Date.now(),
|
checkedAt: Date.now(),
|
||||||
|
|
@ -14369,12 +14279,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
verbTypes?: Set<VerbType>,
|
verbTypes?: Set<VerbType>,
|
||||||
limit?: number
|
limit?: number
|
||||||
): Promise<string[]> {
|
): Promise<string[]> {
|
||||||
// READ-SURFACE READINESS GATE (the 4.2.4 blackout's brainy half): every
|
|
||||||
// index read funnels through this helper, so the gate here makes
|
|
||||||
// serve-while-not-ready UNREPRESENTABLE — a production store once acked
|
|
||||||
// writes while every non-find() read served empty from a not-ready
|
|
||||||
// provider for 15 minutes. Fast path after the latch is one boolean.
|
|
||||||
await this.ensureIndexesLoaded()
|
|
||||||
// 8.0 BigInt boundary: unmapped node → no relations.
|
// 8.0 BigInt boundary: unmapped node → no relations.
|
||||||
const nodeInt = this.graphEntityInt(nodeId)
|
const nodeInt = this.graphEntityInt(nodeId)
|
||||||
if (nodeInt === undefined) return []
|
if (nodeInt === undefined) return []
|
||||||
|
|
@ -16313,15 +16217,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start lazy rebuild (with mutex to prevent concurrent rebuilds).
|
// Start lazy rebuild (with mutex to prevent concurrent rebuilds)
|
||||||
// ALWAYS narrated (prodLog, never the silent-suppressible console): a
|
|
||||||
// read that triggers an index build must be visible to the operator —
|
|
||||||
// fifteen silent minutes of a production blackout taught this line.
|
|
||||||
prodLog.warn(
|
|
||||||
`[Brainy] first read on this instance is building the derived indexes ` +
|
|
||||||
`(deferred at open by disableAutoRebuild) — reads WAIT and then serve; ` +
|
|
||||||
`nothing serves empty. Bounded by store size; progress under [MetadataIndex]/[GraphIndex].`
|
|
||||||
)
|
|
||||||
this.lazyRebuildInProgress = true
|
this.lazyRebuildInProgress = true
|
||||||
this.lazyRebuildPromise = this.rebuildIndexesIfNeeded(true)
|
this.lazyRebuildPromise = this.rebuildIndexesIfNeeded(true)
|
||||||
.then(() => {
|
.then(() => {
|
||||||
|
|
@ -17166,13 +17062,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
async repairIndex(): Promise<RepairReport> {
|
async repairIndex(): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
const startedAt = Date.now()
|
|
||||||
const families: RepairFamilyReport[] = []
|
|
||||||
const record = (family: string, entry: Omit<RepairFamilyReport, 'family'>): void => {
|
|
||||||
families.push({ family, ...entry })
|
|
||||||
}
|
|
||||||
|
|
||||||
// Prune orphaned canonical containers left by the pre-8.3.1 partial-delete
|
// Prune orphaned canonical containers left by the pre-8.3.1 partial-delete
|
||||||
// defect: a delete that removed the metadata (content) leg but left the
|
// defect: a delete that removed the metadata (content) leg but left the
|
||||||
|
|
@ -17188,23 +17079,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
}
|
}
|
||||||
if (typeof pruner.pruneOrphanedEntities === 'function') {
|
if (typeof pruner.pruneOrphanedEntities === 'function') {
|
||||||
const orphans = await pruner.pruneOrphanedEntities()
|
const orphans = await pruner.pruneOrphanedEntities()
|
||||||
const pruned = orphans.nouns.length + orphans.verbs.length
|
if (orphans.nouns.length + orphans.verbs.length > 0) {
|
||||||
record('orphaned-containers', {
|
|
||||||
checked: true,
|
|
||||||
healed: pruned,
|
|
||||||
...(pruned > 0
|
|
||||||
? { detail: `${orphans.nouns.length} noun + ${orphans.verbs.length} verb container(s) pruned` }
|
|
||||||
: {})
|
|
||||||
})
|
|
||||||
if (pruned > 0) {
|
|
||||||
prodLog.warn(
|
prodLog.warn(
|
||||||
`[Brainy] repairIndex() pruned ${orphans.nouns.length} orphaned noun + ` +
|
`[Brainy] repairIndex() pruned ${orphans.nouns.length} orphaned noun + ` +
|
||||||
`${orphans.verbs.length} orphaned verb container(s) left by a pre-8.3.1 ` +
|
`${orphans.verbs.length} orphaned verb container(s) left by a pre-8.3.1 ` +
|
||||||
`partial delete.`
|
`partial delete.`
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
record('orphaned-containers', { checked: false, healed: 0, skipped: 'storage has no container model' })
|
|
||||||
}
|
}
|
||||||
// SANCTIONED RECOUNT — unconditional, not gated on orphans found: the
|
// SANCTIONED RECOUNT — unconditional, not gated on orphans found: the
|
||||||
// persisted counters can be inflated over perfectly clean shelves (deletes
|
// persisted counters can be inflated over perfectly clean shelves (deletes
|
||||||
|
|
@ -17215,14 +17096,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// canonical walk and persists them.
|
// canonical walk and persists them.
|
||||||
await pruner.rebuildTypeCounts?.()
|
await pruner.rebuildTypeCounts?.()
|
||||||
await pruner.rebuildSubtypeCounts?.()
|
await pruner.rebuildSubtypeCounts?.()
|
||||||
record('count-rollups', {
|
|
||||||
checked: typeof pruner.rebuildTypeCounts === 'function',
|
|
||||||
healed: 0,
|
|
||||||
detail: typeof pruner.rebuildTypeCounts === 'function'
|
|
||||||
? 'recomputed from one canonical walk (unconditional)'
|
|
||||||
: undefined,
|
|
||||||
...(typeof pruner.rebuildTypeCounts !== 'function' ? { skipped: 'storage has no count rollups' } : {})
|
|
||||||
})
|
|
||||||
|
|
||||||
// The recount changed the rollup truth — re-stamp the entity tree so the
|
// The recount changed the rollup truth — re-stamp the entity tree so the
|
||||||
// stamp's invariants match the healed counters (repair leaves a coherent
|
// stamp's invariants match the healed counters (repair leaves a coherent
|
||||||
|
|
@ -17236,13 +17109,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// containment edges are touched. Loud per repair.
|
// containment edges are touched. Loud per repair.
|
||||||
if (this._vfsInitialized && this._vfs) {
|
if (this._vfsInitialized && this._vfs) {
|
||||||
const containment = await this._vfs.repairContainment()
|
const containment = await this._vfs.repairContainment()
|
||||||
record('vfs-containment', {
|
|
||||||
checked: true,
|
|
||||||
healed: containment.removed + containment.restored,
|
|
||||||
...(containment.removed + containment.restored > 0
|
|
||||||
? { detail: `${containment.removed} stale edge(s) removed, ${containment.restored} restored` }
|
|
||||||
: {})
|
|
||||||
})
|
|
||||||
if (containment.removed + containment.restored > 0) {
|
if (containment.removed + containment.restored > 0) {
|
||||||
prodLog.warn(
|
prodLog.warn(
|
||||||
`[Brainy] repairIndex() reconciled VFS containment: removed ${containment.removed} ` +
|
`[Brainy] repairIndex() reconciled VFS containment: removed ${containment.removed} ` +
|
||||||
|
|
@ -17251,19 +17117,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!this._vfsInitialized || !this._vfs) {
|
|
||||||
record('vfs-containment', { checked: false, healed: 0, skipped: 'VFS not initialized' })
|
|
||||||
}
|
|
||||||
|
|
||||||
await this.metadataIndex.detectAndRepairCorruption()
|
await this.metadataIndex.detectAndRepairCorruption()
|
||||||
record('metadata-corruption', { checked: true, healed: 0, detail: 'detect-and-repair pass ran (see its own narration for repairs)' })
|
|
||||||
// Lift a failed-rollback write-quarantine: force a full rebuild so the
|
// Lift a failed-rollback write-quarantine: force a full rebuild so the
|
||||||
// derived indexes are provably reconciled with canonical, then clear the
|
// derived indexes are provably reconciled with canonical, then clear the
|
||||||
// flag so writes resume.
|
// flag so writes resume.
|
||||||
if (this.storeInconsistency) {
|
if (this.storeInconsistency) {
|
||||||
await this.rebuildIndexesIfNeeded(true)
|
await this.rebuildIndexesIfNeeded(true)
|
||||||
const cleared = this.storeInconsistency
|
const cleared = this.storeInconsistency
|
||||||
record('write-quarantine', { checked: true, healed: 1, detail: `lifted (${cleared.records.length} record(s) reconciled)` })
|
|
||||||
this.storeInconsistency = null
|
this.storeInconsistency = null
|
||||||
prodLog.warn(
|
prodLog.warn(
|
||||||
`[Brainy] repairIndex() reconciled the store and LIFTED the write-quarantine ` +
|
`[Brainy] repairIndex() reconciled the store and LIFTED the write-quarantine ` +
|
||||||
|
|
@ -17280,38 +17140,20 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
validateInvariants?: () => Promise<ProviderInvariantReport>
|
validateInvariants?: () => Promise<ProviderInvariantReport>
|
||||||
rebuild?: () => Promise<void>
|
rebuild?: () => Promise<void>
|
||||||
} | null
|
} | null
|
||||||
if (!p || typeof p.validateInvariants !== 'function' || typeof p.rebuild !== 'function') {
|
if (!p || typeof p.validateInvariants !== 'function' || typeof p.rebuild !== 'function') continue
|
||||||
record(`provider:${(provider as { constructor?: { name?: string } })?.constructor?.name ?? 'unknown'}`, {
|
|
||||||
checked: false, healed: 0, skipped: 'no validateInvariants/rebuild contract'
|
|
||||||
})
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
let report: ProviderInvariantReport
|
let report: ProviderInvariantReport
|
||||||
try {
|
try {
|
||||||
report = await p.validateInvariants()
|
report = await p.validateInvariants()
|
||||||
} catch (err) {
|
} catch {
|
||||||
record(`provider:unknown`, { checked: false, healed: 0, skipped: `validateInvariants threw: ${(err as Error).message}` })
|
|
||||||
continue // a throwing validateInvariants is surfaced by validateIndexConsistency; skip repair here
|
continue // a throwing validateInvariants is surfaced by validateIndexConsistency; skip repair here
|
||||||
}
|
}
|
||||||
if (report.healthy) {
|
if (report.healthy) continue
|
||||||
record(`provider:${report.provider}`, { checked: true, healed: 0 })
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if (report.invariants.some((i) => !i.holds && i.heal === 'rebuild')) {
|
if (report.invariants.some((i) => !i.holds && i.heal === 'rebuild')) {
|
||||||
record(`provider:${report.provider}`, {
|
|
||||||
checked: true, healed: 1,
|
|
||||||
detail: `rebuilt from canonical (failing: ${report.invariants.filter((i) => !i.holds).map((i) => i.name).join(', ')})`
|
|
||||||
})
|
|
||||||
prodLog.warn(
|
prodLog.warn(
|
||||||
`[Brainy] repairIndex(): provider '${report.provider}' has a failing invariant ` +
|
`[Brainy] repairIndex(): provider '${report.provider}' has a failing invariant ` +
|
||||||
`requiring a rebuild — reconciling its derived state from canonical.`
|
`requiring a rebuild — reconciling its derived state from canonical.`
|
||||||
)
|
)
|
||||||
await p.rebuild()
|
await p.rebuild()
|
||||||
} else {
|
|
||||||
record(`provider:${report.provider}`, {
|
|
||||||
checked: true, healed: 0,
|
|
||||||
detail: `unhealthy without a rebuild verdict (failing: ${report.invariants.filter((i) => !i.holds).map((i) => `${i.name}→${i.heal}`).join(', ')})`
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// detectAndRepairCorruption() above rebuilt the derived indexes from
|
// detectAndRepairCorruption() above rebuilt the derived indexes from
|
||||||
|
|
@ -17322,18 +17164,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
this._indexDegradedIds.clear()
|
this._indexDegradedIds.clear()
|
||||||
this._indexRebuildFailed = null
|
this._indexRebuildFailed = null
|
||||||
this._degradedReadWarned = false
|
this._degradedReadWarned = false
|
||||||
record('degraded-read-state', { checked: true, healed: 1, detail: 'degraded ids cleared, read-path warning re-armed' })
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const healedTotal = families.reduce((n, f) => n + f.healed, 0)
|
|
||||||
const report: RepairReport = { families, healedTotal, durationMs: Date.now() - startedAt }
|
|
||||||
prodLog.warn(
|
|
||||||
`[Brainy] repairIndex complete in ${report.durationMs}ms — ` +
|
|
||||||
`${families.filter((f) => f.checked).length}/${families.length} families checked, ` +
|
|
||||||
`${healedTotal} heal(s): ` +
|
|
||||||
families.map((f) => `${f.family}=${f.checked ? f.healed : 'skipped'}`).join(', ')
|
|
||||||
)
|
|
||||||
return report
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -792,19 +792,6 @@ export interface DerivedFamilyDeclaration {
|
||||||
rebuildable?: boolean
|
rebuildable?: boolean
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* @description The canonical count ledger a storage adapter maintains on its
|
|
||||||
* write path: per family, the user-facing `counted` scalar and the
|
|
||||||
* ALL-visibility `all` scalar (every tier — the coverage-ledger denominator).
|
|
||||||
* See {@link StorageAdapter.getCanonicalCounts}.
|
|
||||||
*/
|
|
||||||
export interface CanonicalCounts {
|
|
||||||
nouns: { counted: number; all: number }
|
|
||||||
verbs: { counted: number; all: number }
|
|
||||||
/** An unprovable delete has left the `all` scalars unverified since the last recount. */
|
|
||||||
suspect: boolean
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface StorageAdapter {
|
export interface StorageAdapter {
|
||||||
init(): Promise<void>
|
init(): Promise<void>
|
||||||
|
|
||||||
|
|
@ -1306,19 +1293,6 @@ export interface StorageAdapter {
|
||||||
*/
|
*/
|
||||||
getVerbCount(): Promise<number>
|
getVerbCount(): Promise<number>
|
||||||
|
|
||||||
/**
|
|
||||||
* The canonical count ledger — O(1), no I/O. `counted` mirrors
|
|
||||||
* `getNounCount()` / `getVerbCount()` (public + internal tiers); `all` is
|
|
||||||
* the ALL-visibility scalar every unfiltered storage walk is measured
|
|
||||||
* against — the denominator a derived-index provider's coverage ledger
|
|
||||||
* subtracts from. `suspect` is `true` when an unprovable delete has left
|
|
||||||
* `all` unverified since the last sanctioned recount (`repairIndex()`).
|
|
||||||
* Optional: adapters without the ledger omit it; a consumer treats absence
|
|
||||||
* as "no denominator", never as zero.
|
|
||||||
* @returns Both scalars per family plus the suspect flag.
|
|
||||||
*/
|
|
||||||
getCanonicalCounts?(): Promise<CanonicalCounts>
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* OPTIONAL — create a pre-upgrade backup of the whole store and return its
|
* OPTIONAL — create a pre-upgrade backup of the whole store and return its
|
||||||
* location, or `null` when there is nothing to back up (empty store). On the
|
* location, or `null` when there is nothing to back up (empty store). On the
|
||||||
|
|
|
||||||
|
|
@ -80,9 +80,7 @@ export type {
|
||||||
AggregationOp,
|
AggregationOp,
|
||||||
TimeWindowGranularity,
|
TimeWindowGranularity,
|
||||||
GroupByDimension,
|
GroupByDimension,
|
||||||
AggregationProvider,
|
AggregationProvider
|
||||||
RepairReport,
|
|
||||||
RepairFamilyReport,
|
|
||||||
} from './types/brainy.types.js'
|
} from './types/brainy.types.js'
|
||||||
|
|
||||||
// Read-barrier contract (waitForIndexed): the leg names, the options, and
|
// Read-barrier contract (waitForIndexed): the leg names, the options, and
|
||||||
|
|
|
||||||
|
|
@ -12,8 +12,7 @@ import {
|
||||||
HNSWNounWithMetadata,
|
HNSWNounWithMetadata,
|
||||||
HNSWVerbWithMetadata,
|
HNSWVerbWithMetadata,
|
||||||
NounMetadata,
|
NounMetadata,
|
||||||
VerbMetadata,
|
VerbMetadata
|
||||||
CanonicalCounts,
|
|
||||||
} from '../../coreTypes.js'
|
} from '../../coreTypes.js'
|
||||||
import { StorageBatchConfig } from '../baseStorage.js'
|
import { StorageBatchConfig } from '../baseStorage.js'
|
||||||
import { extractFieldNamesFromJson, mapToStandardField } from '../../utils/fieldNameTracking.js'
|
import { extractFieldNamesFromJson, mapToStandardField } from '../../utils/fieldNameTracking.js'
|
||||||
|
|
@ -1029,28 +1028,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
// Universal count tracking - O(1) operations
|
// Universal count tracking - O(1) operations
|
||||||
protected totalNounCount = 0
|
protected totalNounCount = 0
|
||||||
protected totalVerbCount = 0
|
protected totalVerbCount = 0
|
||||||
/**
|
|
||||||
* The ALL-visibility canonical scalars — every noun / verb the unfiltered
|
|
||||||
* storage walk yields, system and internal tiers included. These are the
|
|
||||||
* denominators a derived-index provider's coverage ledger subtracts from
|
|
||||||
* (`posted === all` is the whole-store coverage verdict); the user-facing
|
|
||||||
* `totalNounCount` / `totalVerbCount` skip hidden tiers by design and can
|
|
||||||
* never serve as a ledger denominator. Maintained on the write path
|
|
||||||
* (every new record +1, every proven delete −1), persisted beside the
|
|
||||||
* counted scalars, recomputed by the sanctioned recount. Never clamped.
|
|
||||||
*/
|
|
||||||
protected totalNounCountAll = 0
|
|
||||||
protected totalVerbCountAll = 0
|
|
||||||
/**
|
|
||||||
* `true` when a delete could not prove whether the record existed (no
|
|
||||||
* canonical read, no caller-provided prior) — the ALL scalar may be off by
|
|
||||||
* the unprovable deletes since. Loud, persisted, and cleared only by the
|
|
||||||
* sanctioned recount; a consumer reading the scalar as a ledger denominator
|
|
||||||
* must treat a suspect scalar as unverified, never as exact.
|
|
||||||
*/
|
|
||||||
protected allCountsSuspect = false
|
|
||||||
/** One narration per session for the suspect transition (never per delete). */
|
|
||||||
private allCountsSuspectNarrated = false
|
|
||||||
protected entityCounts: Map<string, number> = new Map() // type -> count
|
protected entityCounts: Map<string, number> = new Map() // type -> count
|
||||||
protected verbCounts: Map<string, number> = new Map() // verb type -> count
|
protected verbCounts: Map<string, number> = new Map() // verb type -> count
|
||||||
protected countCache: Map<string, { count: number; timestamp: number }> = new Map()
|
protected countCache: Map<string, { count: number; timestamp: number }> = new Map()
|
||||||
|
|
@ -1079,43 +1056,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
return this.totalVerbCount
|
return this.totalVerbCount
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* The canonical count ledger — O(1), no I/O. `counted` is the user-facing
|
|
||||||
* scalar (public/internal tiers, what `getNounCount()` returns); `all` is
|
|
||||||
* the ALL-visibility scalar every unfiltered storage walk is measured
|
|
||||||
* against (the coverage-ledger denominator for derived-index providers).
|
|
||||||
* `suspect` is `true` when an unprovable delete has made `all` unverified
|
|
||||||
* since the last sanctioned recount (`rebuildTypeCounts`).
|
|
||||||
* @returns Both scalars per family plus the suspect flag.
|
|
||||||
*/
|
|
||||||
async getCanonicalCounts(): Promise<CanonicalCounts> {
|
|
||||||
return {
|
|
||||||
nouns: { counted: this.totalNounCount, all: this.totalNounCountAll },
|
|
||||||
verbs: { counted: this.totalVerbCount, all: this.totalVerbCountAll },
|
|
||||||
suspect: this.allCountsSuspect
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Mark the ALL scalars unverified after a delete that could not prove the
|
|
||||||
* record existed. Narrates ONCE per session (the flag is what persists);
|
|
||||||
* the sanctioned recount clears it.
|
|
||||||
* @param family - Which family's delete was unprovable.
|
|
||||||
* @param id - The id whose existence could not be established.
|
|
||||||
*/
|
|
||||||
protected markAllCountsSuspect(family: 'noun' | 'verb', id: string): void {
|
|
||||||
this.allCountsSuspect = true
|
|
||||||
if (!this.allCountsSuspectNarrated) {
|
|
||||||
this.allCountsSuspectNarrated = true
|
|
||||||
console.warn(
|
|
||||||
`[Storage] ${family} delete of ${id} could not prove the record existed ` +
|
|
||||||
`(no canonical read, no prior record) — the ALL-visibility count ledger is ` +
|
|
||||||
`SUSPECT until brain.repairIndex() recounts. Further unprovable deletes ` +
|
|
||||||
`this session are counted silently under the same flag.`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Increment count for entity type - O(1) operation.
|
* Increment count for entity type - O(1) operation.
|
||||||
* Concurrency is handled by the process-global mutex
|
* Concurrency is handled by the process-global mutex
|
||||||
|
|
|
||||||
|
|
@ -2561,33 +2561,6 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
this.totalNounCount = counts.totalNounCount || 0
|
this.totalNounCount = counts.totalNounCount || 0
|
||||||
this.totalVerbCount = counts.totalVerbCount || 0
|
this.totalVerbCount = counts.totalVerbCount || 0
|
||||||
|
|
||||||
// The ALL-visibility scalars (ledger denominators). A counts.json
|
|
||||||
// written before they existed carries neither key: derive both ONCE
|
|
||||||
// from the canonical id tree (an id-directory listing — O(ids), no
|
|
||||||
// record reads), persist, and never scan again. Absent keys are a
|
|
||||||
// legacy file, not a zero — a zero here would make every provider's
|
|
||||||
// coverage ledger read "over-posted" on a populated store.
|
|
||||||
if (
|
|
||||||
typeof counts.totalNounCountAll === 'number' &&
|
|
||||||
typeof counts.totalVerbCountAll === 'number'
|
|
||||||
) {
|
|
||||||
this.totalNounCountAll = counts.totalNounCountAll
|
|
||||||
this.totalVerbCountAll = counts.totalVerbCountAll
|
|
||||||
this.allCountsSuspect = counts.allCountsSuspect === true
|
|
||||||
} else {
|
|
||||||
const nouns = await this.scanCanonicalEntities('nouns')
|
|
||||||
const verbs = await this.scanCanonicalEntities('verbs')
|
|
||||||
this.totalNounCountAll = nouns.count
|
|
||||||
this.totalVerbCountAll = verbs.count
|
|
||||||
this.allCountsSuspect = false
|
|
||||||
console.warn(
|
|
||||||
`[FileSystemStorage] counts.json predates the ALL-visibility count ledger — ` +
|
|
||||||
`derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` +
|
|
||||||
`every tier) and persisted; no further scan.`
|
|
||||||
)
|
|
||||||
await this.persistCounts()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Also populate the cache for backward compatibility
|
// Also populate the cache for backward compatibility
|
||||||
this.countCache.set('nouns_count', {
|
this.countCache.set('nouns_count', {
|
||||||
count: this.totalNounCount,
|
count: this.totalNounCount,
|
||||||
|
|
@ -2623,10 +2596,6 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
this.totalNounCount = nouns.count
|
this.totalNounCount = nouns.count
|
||||||
const verbs = await this.scanCanonicalEntities('verbs')
|
const verbs = await this.scanCanonicalEntities('verbs')
|
||||||
this.totalVerbCount = verbs.count
|
this.totalVerbCount = verbs.count
|
||||||
// The id-tree scan counts every tier — it IS the ALL-visibility ledger.
|
|
||||||
this.totalNounCountAll = nouns.count
|
|
||||||
this.totalVerbCountAll = verbs.count
|
|
||||||
this.allCountsSuspect = false
|
|
||||||
|
|
||||||
// Sample some entities for the type distribution (don't read all).
|
// Sample some entities for the type distribution (don't read all).
|
||||||
// Read the metadata files DIRECTLY with fs — this runs inside init(),
|
// Read the metadata files DIRECTLY with fs — this runs inside init(),
|
||||||
|
|
@ -2724,11 +2693,6 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
verbCounts: Object.fromEntries(this.verbCounts),
|
verbCounts: Object.fromEntries(this.verbCounts),
|
||||||
totalNounCount: this.totalNounCount,
|
totalNounCount: this.totalNounCount,
|
||||||
totalVerbCount: this.totalVerbCount,
|
totalVerbCount: this.totalVerbCount,
|
||||||
// ALL-visibility ledger scalars (+ the suspect flag) — absent in files
|
|
||||||
// written before the ledger existed; initializeCounts() derives them once.
|
|
||||||
totalNounCountAll: this.totalNounCountAll,
|
|
||||||
totalVerbCountAll: this.totalVerbCountAll,
|
|
||||||
allCountsSuspect: this.allCountsSuspect,
|
|
||||||
lastUpdated: new Date().toISOString()
|
lastUpdated: new Date().toISOString()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -540,10 +540,6 @@ export class MemoryStorage extends BaseStorage {
|
||||||
|
|
||||||
this.totalNounCount = totalNouns
|
this.totalNounCount = totalNouns
|
||||||
this.totalVerbCount = totalVerbs
|
this.totalVerbCount = totalVerbs
|
||||||
// A scan of every canonical record IS the ALL-visibility count.
|
|
||||||
this.totalNounCountAll = totalNouns
|
|
||||||
this.totalVerbCountAll = totalVerbs
|
|
||||||
this.allCountsSuspect = false
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -2273,14 +2273,11 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
|
|
||||||
// totalCount must be the TRUE dataset total, not this peeked page. For the
|
// totalCount must be the TRUE dataset total, not this peeked page. For the
|
||||||
// unfiltered case the authoritative total is the O(1) counter maintained on
|
// unfiltered case the authoritative total is the O(1) counter maintained on
|
||||||
// every add/delete (rehydrated on init) — the ALL-visibility scalar, because
|
// every add/delete (rehydrated on init); `Math.max` guards a stale counter. A
|
||||||
// this walk is unfiltered by tier (system/internal records are in `collected`).
|
// filtered scan has no cheap exact total, so it keeps the collected length.
|
||||||
// Never clamped: `Math.max(scalar, scanned)` could only ever move the scalar
|
const totalCount = filter
|
||||||
// UP, so an inflated counter could never correct itself and a divergence was
|
? collected.length
|
||||||
// hidden instead of reported. A scalar that disagrees with the walk is the
|
: Math.max(this.totalNounCount, collected.length)
|
||||||
// canonical-count-ledger invariant's job, healed by the sanctioned recount.
|
|
||||||
// A filtered scan has no cheap exact total, so it keeps the collected length.
|
|
||||||
const totalCount = filter ? collected.length : this.totalNounCountAll
|
|
||||||
|
|
||||||
// nextCursor = the (shard, id) of the last RETURNED noun, so the next call
|
// nextCursor = the (shard, id) of the last RETURNED noun, so the next call
|
||||||
// resumes immediately after it (works for both cursor and offset callers).
|
// resumes immediately after it (works for both cursor and offset callers).
|
||||||
|
|
@ -2412,8 +2409,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
const pagePairs = collected.slice(windowStart, windowStart + limit)
|
const pagePairs = collected.slice(windowStart, windowStart + limit)
|
||||||
const ids = pagePairs.map((p) => p.id)
|
const ids = pagePairs.map((p) => p.id)
|
||||||
const hasMore = collected.length > windowStart + limit
|
const hasMore = collected.length > windowStart + limit
|
||||||
// ALL-visibility scalar, unclamped — same law as getNouns() above.
|
const totalCount = filter ? collected.length : Math.max(this.totalNounCount, collected.length)
|
||||||
const totalCount = filter ? collected.length : this.totalNounCountAll
|
|
||||||
|
|
||||||
let nextCursor: string | undefined = undefined
|
let nextCursor: string | undefined = undefined
|
||||||
if (hasMore && pagePairs.length > 0) {
|
if (hasMore && pagePairs.length > 0) {
|
||||||
|
|
@ -2645,13 +2641,13 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
const hasMore = collected.length > windowStart + limit
|
const hasMore = collected.length > windowStart + limit
|
||||||
|
|
||||||
// totalCount must be the TRUE dataset total, not this peeked page. For the
|
// totalCount must be the TRUE dataset total, not this peeked page. For the
|
||||||
// unfiltered scan the authoritative total is the O(1) ALL-visibility counter
|
// unfiltered scan the authoritative total is the O(1) `totalVerbCount` counter
|
||||||
// (`totalVerbCountAll`: isNew-gated, EVERY tier, rehydrated on init) — the walk
|
// (isNew-gated, visibility-filtered, rehydrated on init); `Math.max` guards a
|
||||||
// itself is unfiltered by tier, so the user-facing `totalVerbCount` (which skips
|
// stale counter from under-reporting. A filtered scan has no cheap exact total,
|
||||||
// system/internal edges) would undercount it on every store with a VFS. Never
|
// so it keeps the collected length (a lower bound).
|
||||||
// clamped (see getNouns): a divergence is reported, not hidden. A filtered scan
|
const totalCount = filter
|
||||||
// has no cheap exact total, so it keeps the collected length (a lower bound).
|
? collected.length
|
||||||
const totalCount = filter ? collected.length : this.totalVerbCountAll
|
: Math.max(this.totalVerbCount, collected.length)
|
||||||
|
|
||||||
// nextCursor encodes the (shard, id) of the LAST RETURNED verb so the next call
|
// nextCursor encodes the (shard, id) of the LAST RETURNED verb so the next call
|
||||||
// resumes immediately after it — for both cursor and offset callers (an offset
|
// resumes immediately after it — for both cursor and offset callers (an offset
|
||||||
|
|
@ -3459,19 +3455,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
const wasCounted = isNew ? false : isCountedVisibility(existingMetadata?.visibility)
|
const wasCounted = isNew ? false : isCountedVisibility(existingMetadata?.visibility)
|
||||||
const isCounted = isCountedVisibility(newVisibility)
|
const isCounted = isCountedVisibility(newVisibility)
|
||||||
|
|
||||||
// ALL-visibility ledger: every NEW canonical record is +1 regardless of tier
|
|
||||||
// (the unfiltered walk yields it, so the denominator must count it). The
|
|
||||||
// counted branch below persists for public/internal records; a hidden new
|
|
||||||
// record persists here so the ALL scalar never lags the tree.
|
|
||||||
if (isNew) {
|
|
||||||
this.totalNounCountAll++
|
|
||||||
if (!(metadata.noun && isCounted)) {
|
|
||||||
this.scheduleCountPersist().catch(() => {
|
|
||||||
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// CRITICAL FIX: Increment count for new entities
|
// CRITICAL FIX: Increment count for new entities
|
||||||
// This runs AFTER metadata is saved, guaranteeing type information is available
|
// This runs AFTER metadata is saved, guaranteeing type information is available
|
||||||
// Uses synchronous increment since storage operations are already serialized
|
// Uses synchronous increment since storage operations are already serialized
|
||||||
|
|
@ -3875,20 +3858,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
await this.deleteCanonicalObject(path)
|
await this.deleteCanonicalObject(path)
|
||||||
const record = read ?? priorRecord
|
const record = read ?? priorRecord
|
||||||
|
|
||||||
// ALL-visibility ledger: a PROVEN delete (the record was read, or the caller
|
|
||||||
// carried its prior image) is −1 regardless of tier. A delete that can prove
|
|
||||||
// nothing never guesses — it marks the ledger suspect (loud, persisted) and the
|
|
||||||
// sanctioned recount restores exactness.
|
|
||||||
if (record) {
|
|
||||||
if (this.totalNounCountAll > 0) this.totalNounCountAll--
|
|
||||||
else this.markAllCountsSuspect('noun', id)
|
|
||||||
} else {
|
|
||||||
this.markAllCountsSuspect('noun', id)
|
|
||||||
}
|
|
||||||
this.scheduleCountPersist().catch(() => {
|
|
||||||
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
|
|
||||||
})
|
|
||||||
|
|
||||||
const priorType = record?.noun as NounType | undefined
|
const priorType = record?.noun as NounType | undefined
|
||||||
// 8.0 visibility: an internal/system entity was never added to `nounCountsByType`
|
// 8.0 visibility: an internal/system entity was never added to `nounCountsByType`
|
||||||
// (gated in `saveNounMetadata_internal()`), so it must not be decremented here either.
|
// (gated in `saveNounMetadata_internal()`), so it must not be decremented here either.
|
||||||
|
|
@ -4022,9 +3991,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
// Fixes Bug #2: Count synchronization failure during relate() and import()
|
// Fixes Bug #2: Count synchronization failure during relate() and import()
|
||||||
// 8.0: skip the user-facing total for internal/system edges (counts.json + getVerbCount()).
|
// 8.0: skip the user-facing total for internal/system edges (counts.json + getVerbCount()).
|
||||||
if (isNew) {
|
if (isNew) {
|
||||||
// ALL-visibility ledger: every new edge is +1 regardless of tier (the
|
|
||||||
// unfiltered walk yields VFS/system edges too; the denominator must count them).
|
|
||||||
this.totalVerbCountAll++
|
|
||||||
if (isCounted) {
|
if (isCounted) {
|
||||||
this.incrementVerbCount(verbType)
|
this.incrementVerbCount(verbType)
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -4086,18 +4052,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
await this.deleteCanonicalObject(path)
|
await this.deleteCanonicalObject(path)
|
||||||
const record = read ?? priorRecord
|
const record = read ?? priorRecord
|
||||||
|
|
||||||
// ALL-visibility ledger: proven delete −1 regardless of tier; an unprovable
|
|
||||||
// delete marks the ledger suspect instead of guessing (see deleteNounMetadata).
|
|
||||||
if (record) {
|
|
||||||
if (this.totalVerbCountAll > 0) this.totalVerbCountAll--
|
|
||||||
else this.markAllCountsSuspect('verb', id)
|
|
||||||
} else {
|
|
||||||
this.markAllCountsSuspect('verb', id)
|
|
||||||
}
|
|
||||||
this.scheduleCountPersist().catch(() => {
|
|
||||||
// Ignore persist errors — in-memory count is authoritative; a later op retries.
|
|
||||||
})
|
|
||||||
|
|
||||||
const priorVerb = record?.verb as VerbType | undefined
|
const priorVerb = record?.verb as VerbType | undefined
|
||||||
// Symmetric count decrement (previously OMITTED — verb deletes touched neither the
|
// Symmetric count decrement (previously OMITTED — verb deletes touched neither the
|
||||||
// scalar total nor the per-type bucket, so both inflated permanently). A COUNTED
|
// scalar total nor the per-type bucket, so both inflated permanently). A COUNTED
|
||||||
|
|
@ -4543,12 +4497,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
// walk, every counter rollup rebuilt and persisted from it.
|
// walk, every counter rollup rebuilt and persisted from it.
|
||||||
const countedNouns = new Map<string, number>()
|
const countedNouns = new Map<string, number>()
|
||||||
const countedVerbs = new Map<string, number>()
|
const countedVerbs = new Map<string, number>()
|
||||||
// ALL-visibility scalars: one per canonical record the walk yields, every
|
|
||||||
// tier, readable or not — the same population the unfiltered getNouns()/
|
|
||||||
// getVerbs() walks enumerate, so `totalCount` and this recount agree by
|
|
||||||
// construction.
|
|
||||||
let allNouns = 0
|
|
||||||
let allVerbs = 0
|
|
||||||
|
|
||||||
// Scan noun shards
|
// Scan noun shards
|
||||||
for (let shard = 0; shard < 256; shard++) {
|
for (let shard = 0; shard < 256; shard++) {
|
||||||
|
|
@ -4560,7 +4508,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
|
|
||||||
for (const path of paths) {
|
for (const path of paths) {
|
||||||
if (!path.includes('/metadata.json')) continue
|
if (!path.includes('/metadata.json')) continue
|
||||||
allNouns++
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const metadata = await this.readCanonicalObject(path)
|
const metadata = await this.readCanonicalObject(path)
|
||||||
|
|
@ -4593,7 +4540,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
|
|
||||||
for (const path of paths) {
|
for (const path of paths) {
|
||||||
if (!path.includes('/metadata.json')) continue
|
if (!path.includes('/metadata.json')) continue
|
||||||
allVerbs++
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const metadata = await this.readCanonicalObject(path)
|
const metadata = await this.readCanonicalObject(path)
|
||||||
|
|
@ -4630,24 +4576,10 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
this.verbCounts = countedVerbs
|
this.verbCounts = countedVerbs
|
||||||
this.totalNounCount = totalNouns
|
this.totalNounCount = totalNouns
|
||||||
this.totalVerbCount = totalVerbs
|
this.totalVerbCount = totalVerbs
|
||||||
// The ALL scalars are exact again and the suspect flag clears — this walk
|
|
||||||
// IS the proof an unprovable delete could not give.
|
|
||||||
const nounsAllBefore = this.totalNounCountAll
|
|
||||||
const verbsAllBefore = this.totalVerbCountAll
|
|
||||||
this.totalNounCountAll = allNouns
|
|
||||||
this.totalVerbCountAll = allVerbs
|
|
||||||
this.allCountsSuspect = false
|
|
||||||
this.countCache.clear()
|
this.countCache.clear()
|
||||||
await this.persistCounts()
|
await this.persistCounts()
|
||||||
|
|
||||||
prodLog.info(
|
prodLog.info(`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs (scalar + per-type persisted)`)
|
||||||
`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs (user-facing); ` +
|
|
||||||
`ALL-visibility ledger ${allNouns} nouns / ${allVerbs} verbs` +
|
|
||||||
(nounsAllBefore !== allNouns || verbsAllBefore !== allVerbs
|
|
||||||
? ` (corrected from ${nounsAllBefore} / ${verbsAllBefore})`
|
|
||||||
: ' (unchanged)') +
|
|
||||||
` — scalar + per-type persisted`
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -1192,40 +1192,6 @@ export interface RelateManyParams<T = any> {
|
||||||
/**
|
/**
|
||||||
* Batch result
|
* Batch result
|
||||||
*/
|
*/
|
||||||
/**
|
|
||||||
* One family's row in a {@link RepairReport} — what repairIndex() checked,
|
|
||||||
* what it healed, and why anything was skipped. The receipts venue's graph
|
|
||||||
* trust program asked for: a repair that cannot show its work is a repair
|
|
||||||
* nobody can trust.
|
|
||||||
*/
|
|
||||||
export interface RepairFamilyReport {
|
|
||||||
family: string
|
|
||||||
/** The family was actually examined (false = skipped; see `skipped`/`reason`). */
|
|
||||||
checked: boolean
|
|
||||||
/** Items re-posted / corrected in place — the incremental heal count. */
|
|
||||||
healed: number
|
|
||||||
/**
|
|
||||||
* What the check found missing or divergent, when it can name it: an exact
|
|
||||||
* count plus a capped sample of ids (never the whole list — a report is a
|
|
||||||
* verdict, not a dump). Absent when the family has nothing to name.
|
|
||||||
*/
|
|
||||||
missing?: { count: number; sample: string[] }
|
|
||||||
/** A full generational rebuild of this family ran (as opposed to an incremental heal). */
|
|
||||||
rebuilt?: boolean
|
|
||||||
detail?: string
|
|
||||||
/** Why the family was not checked (`checked: false`). */
|
|
||||||
skipped?: string
|
|
||||||
/** Why the outcome is what it is when neither `detail` nor `skipped` says it. */
|
|
||||||
reason?: string
|
|
||||||
}
|
|
||||||
|
|
||||||
/** The full receipt returned by repairIndex(). */
|
|
||||||
export interface RepairReport {
|
|
||||||
families: RepairFamilyReport[]
|
|
||||||
healedTotal: number
|
|
||||||
durationMs: number
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface BatchResult<T = any> {
|
export interface BatchResult<T = any> {
|
||||||
successful: T[] // Successfully processed items
|
successful: T[] // Successfully processed items
|
||||||
failed: Array<{ // Failed items with errors
|
failed: Array<{ // Failed items with errors
|
||||||
|
|
|
||||||
|
|
@ -1,182 +0,0 @@
|
||||||
/**
|
|
||||||
* @module tests/integration/canonical-count-ledger
|
|
||||||
* @description The canonical count ledger — the denominators a derived-index
|
|
||||||
* provider's coverage ledger subtracts from. Laws under test:
|
|
||||||
* (1) THE ALL-VISIBILITY SCALAR IS THE UNFILTERED WALK'S TOTAL — the
|
|
||||||
* storage-level `getNouns()` / `getVerbs()` `totalCount` counts EVERY tier
|
|
||||||
* (system, internal, public) because the walk yields every tier; the
|
|
||||||
* user-facing `getNounCount()` / `getVerbCount()` keep skipping hidden
|
|
||||||
* tiers. A ledger built on the user-facing scalar would read "over-posted"
|
|
||||||
* on every store with a VFS — the mismatch this pin makes unbuildable.
|
|
||||||
* (2) NEVER CLAMPED — `Math.max(scalar, scanned)` could only move a scalar up,
|
|
||||||
* so an inflated counter hid forever. An inflated scalar is now VISIBLE
|
|
||||||
* (totalCount ≠ walk) and the sanctioned recount heals it, durably.
|
|
||||||
* (3) NEVER GUESSED — a delete that cannot prove the record existed marks the
|
|
||||||
* ledger SUSPECT (persisted) instead of decrementing on faith; the recount
|
|
||||||
* clears the flag with proof.
|
|
||||||
* (4) LEGACY FILES DERIVE ONCE — a counts.json written before the ledger is
|
|
||||||
* upgraded from the canonical id tree at open, then persisted.
|
|
||||||
*/
|
|
||||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
|
||||||
import * as fs from 'node:fs'
|
|
||||||
import * as os from 'node:os'
|
|
||||||
import * as path from 'node:path'
|
|
||||||
import { Brainy } from '../../src/index.js'
|
|
||||||
|
|
||||||
/** Count canonical `<root>/entities/<kind>/<shard>/<id>` directories — every tier. */
|
|
||||||
function countIdDirs(root: string, kind: 'nouns' | 'verbs'): number {
|
|
||||||
const base = path.join(root, 'entities', kind)
|
|
||||||
if (!fs.existsSync(base)) return 0
|
|
||||||
let n = 0
|
|
||||||
for (const shard of fs.readdirSync(base)) {
|
|
||||||
const shardDir = path.join(base, shard)
|
|
||||||
if (!fs.statSync(shardDir).isDirectory()) continue
|
|
||||||
for (const id of fs.readdirSync(shardDir)) {
|
|
||||||
if (fs.statSync(path.join(shardDir, id)).isDirectory()) n++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return n
|
|
||||||
}
|
|
||||||
|
|
||||||
const countsPath = (root: string) => path.join(root, '_system', 'counts.json')
|
|
||||||
|
|
||||||
describe('canonical count ledger — ALL-visibility scalars, unclamped totals, recount heals', () => {
|
|
||||||
let dir: string
|
|
||||||
let brain: any
|
|
||||||
|
|
||||||
const open = async () => {
|
|
||||||
const b: any = new Brainy({
|
|
||||||
requireSubtype: false,
|
|
||||||
storage: { type: 'filesystem', path: dir },
|
|
||||||
silent: true,
|
|
||||||
dimensions: 384
|
|
||||||
})
|
|
||||||
await b.init()
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
|
|
||||||
beforeEach(async () => {
|
|
||||||
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
|
||||||
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-ledger-'))
|
|
||||||
brain = await open()
|
|
||||||
})
|
|
||||||
afterEach(async () => {
|
|
||||||
await brain.close?.().catch(() => {})
|
|
||||||
fs.rmSync(dir, { recursive: true, force: true })
|
|
||||||
})
|
|
||||||
|
|
||||||
it('the unfiltered walk totalCount is the ALL scalar (every tier); the user-facing count stays counted', async () => {
|
|
||||||
const a = await brain.add({ data: 'public a', type: 'document' })
|
|
||||||
const b = await brain.add({ data: 'internal b', type: 'document', visibility: 'internal' })
|
|
||||||
await brain.relate({ from: a, to: b, type: 'relatedTo', visibility: 'internal' })
|
|
||||||
await brain.vfs.writeFile('/docs/x.txt', 'hello') // VFS: system-tier nouns + Contains edges
|
|
||||||
await brain.flush()
|
|
||||||
|
|
||||||
const ledger = await brain.storage.getCanonicalCounts()
|
|
||||||
expect(ledger.suspect).toBe(false)
|
|
||||||
expect(ledger.nouns.all).toBe(countIdDirs(dir, 'nouns'))
|
|
||||||
expect(ledger.verbs.all).toBe(countIdDirs(dir, 'verbs'))
|
|
||||||
expect(ledger.nouns.counted).toBe(await brain.storage.getNounCount())
|
|
||||||
expect(ledger.verbs.counted).toBe(await brain.storage.getVerbCount())
|
|
||||||
// Hidden tiers exist (the VFS root at minimum, the internal noun, the internal edge):
|
|
||||||
expect(ledger.nouns.all).toBeGreaterThan(ledger.nouns.counted)
|
|
||||||
expect(ledger.verbs.all).toBeGreaterThan(ledger.verbs.counted)
|
|
||||||
|
|
||||||
// The storage-level unfiltered walks report the ALL scalar, and a full page equals it.
|
|
||||||
const nouns = await brain.storage.getNouns({ pagination: { limit: 1000, offset: 0 } })
|
|
||||||
expect(nouns.totalCount).toBe(ledger.nouns.all)
|
|
||||||
expect(nouns.items.length).toBe(ledger.nouns.all)
|
|
||||||
const verbs = await brain.storage.getVerbs({ pagination: { limit: 1000, offset: 0 } })
|
|
||||||
expect(verbs.totalCount).toBe(ledger.verbs.all)
|
|
||||||
expect(verbs.items.length).toBe(ledger.verbs.all)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('proven deletes move the ALL scalar for every tier and the ledger stays exact and unsuspect', async () => {
|
|
||||||
const p = await brain.add({ data: 'public p', type: 'document' })
|
|
||||||
const q = await brain.add({ data: 'internal q', type: 'document', visibility: 'internal' })
|
|
||||||
await brain.relate({ from: p, to: q, type: 'relatedTo' })
|
|
||||||
await brain.flush()
|
|
||||||
const before = await brain.storage.getCanonicalCounts()
|
|
||||||
|
|
||||||
await brain.remove(q) // cascades the edge
|
|
||||||
await brain.remove(p)
|
|
||||||
await brain.flush()
|
|
||||||
|
|
||||||
const after = await brain.storage.getCanonicalCounts()
|
|
||||||
expect(after.nouns.all).toBe(before.nouns.all - 2)
|
|
||||||
expect(after.verbs.all).toBe(before.verbs.all - 1)
|
|
||||||
expect(after.nouns.all).toBe(countIdDirs(dir, 'nouns'))
|
|
||||||
expect(after.verbs.all).toBe(countIdDirs(dir, 'verbs'))
|
|
||||||
expect(after.nouns.counted).toBe(before.nouns.counted - 1)
|
|
||||||
expect(after.suspect).toBe(false)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('a legacy counts.json without the ALL keys is derived once from the id tree and persisted', async () => {
|
|
||||||
await brain.add({ data: 'one', type: 'document' })
|
|
||||||
await brain.add({ data: 'two', type: 'document', visibility: 'internal' })
|
|
||||||
await brain.vfs.writeFile('/a.txt', 'x')
|
|
||||||
await brain.flush()
|
|
||||||
await brain.close()
|
|
||||||
|
|
||||||
const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
|
|
||||||
expect(typeof raw.totalNounCountAll).toBe('number')
|
|
||||||
delete raw.totalNounCountAll
|
|
||||||
delete raw.totalVerbCountAll
|
|
||||||
delete raw.allCountsSuspect
|
|
||||||
fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2))
|
|
||||||
|
|
||||||
brain = await open()
|
|
||||||
const ledger = await brain.storage.getCanonicalCounts()
|
|
||||||
expect(ledger.nouns.all).toBe(countIdDirs(dir, 'nouns'))
|
|
||||||
expect(ledger.verbs.all).toBe(countIdDirs(dir, 'verbs'))
|
|
||||||
expect(ledger.suspect).toBe(false)
|
|
||||||
const persisted = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
|
|
||||||
expect(persisted.totalNounCountAll).toBe(ledger.nouns.all)
|
|
||||||
expect(persisted.totalVerbCountAll).toBe(ledger.verbs.all)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('an inflated ALL scalar is VISIBLE (unclamped) and healed by repairIndex(), surviving reopen', async () => {
|
|
||||||
for (let i = 0; i < 3; i++) await brain.add({ data: `real ${i}`, type: 'document' })
|
|
||||||
await brain.flush()
|
|
||||||
const truth = countIdDirs(dir, 'nouns')
|
|
||||||
|
|
||||||
;(brain.storage as any).totalNounCountAll = truth + 40
|
|
||||||
await (brain.storage as any).persistCounts()
|
|
||||||
await brain.close()
|
|
||||||
brain = await open()
|
|
||||||
|
|
||||||
// The lie survives reopen AND is observable: totalCount disagrees with the walk.
|
|
||||||
const page = await brain.storage.getNouns({ pagination: { limit: 1000, offset: 0 } })
|
|
||||||
expect(page.totalCount).toBe(truth + 40)
|
|
||||||
expect(page.items.length).toBe(truth)
|
|
||||||
|
|
||||||
await brain.repairIndex()
|
|
||||||
expect((await brain.storage.getCanonicalCounts()).nouns.all).toBe(truth)
|
|
||||||
expect((await brain.storage.getNouns({ pagination: { limit: 1000, offset: 0 } })).totalCount).toBe(truth)
|
|
||||||
|
|
||||||
await brain.close()
|
|
||||||
brain = await open()
|
|
||||||
expect((await brain.storage.getCanonicalCounts()).nouns.all).toBe(truth)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('an unprovable delete marks the ledger SUSPECT (persisted); the recount clears it with proof', async () => {
|
|
||||||
await brain.add({ data: 'anchor', type: 'document' })
|
|
||||||
await brain.flush()
|
|
||||||
const truth = countIdDirs(dir, 'nouns')
|
|
||||||
|
|
||||||
// A ghost: no canonical record, no prior image — nothing to prove existence with.
|
|
||||||
await brain.storage.deleteNounMetadata('00000000-dead-4dea-8dea-000000000000')
|
|
||||||
let ledger = await brain.storage.getCanonicalCounts()
|
|
||||||
expect(ledger.suspect).toBe(true)
|
|
||||||
expect(ledger.nouns.all).toBe(truth) // never decremented on faith
|
|
||||||
|
|
||||||
await brain.close()
|
|
||||||
brain = await open()
|
|
||||||
expect((await brain.storage.getCanonicalCounts()).suspect).toBe(true) // the flag persists
|
|
||||||
|
|
||||||
await brain.repairIndex()
|
|
||||||
ledger = await brain.storage.getCanonicalCounts()
|
|
||||||
expect(ledger.suspect).toBe(false)
|
|
||||||
expect(ledger.nouns.all).toBe(truth)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
@ -23,7 +23,6 @@ import { Brainy } from '../../src/brainy.js'
|
||||||
import { NounType } from '../../src/types/graphTypes.js'
|
import { NounType } from '../../src/types/graphTypes.js'
|
||||||
import {
|
import {
|
||||||
abandonAsCrashed,
|
abandonAsCrashed,
|
||||||
armCrash,
|
|
||||||
dropCanonicalNoun,
|
dropCanonicalNoun,
|
||||||
makeTempDir,
|
makeTempDir,
|
||||||
openBrain,
|
openBrain,
|
||||||
|
|
@ -178,33 +177,15 @@ describe('fold-checkpoint bound — crash recovery folds (checkpoint, head], nev
|
||||||
expect(founded, 'checkpoint founded at flip').toBe(committedOf(brain))
|
expect(founded, 'checkpoint founded at flip').toBe(committedOf(brain))
|
||||||
|
|
||||||
// First post-flip boot, unclean (the production first-restart shape):
|
// First post-flip boot, unclean (the production first-restart shape):
|
||||||
// a post-flip write above the checkpoint is restored FROM ITS AT-ACK FACT;
|
// a post-flip write above the checkpoint is restored FROM ITS AT-ACK FACT
|
||||||
// the pre-flip row (its baseline fact ≤ checkpoint, its bytes barrier-
|
// (deliberately NOT flushed — a flush would barrier-sync it and advance
|
||||||
// synced at the flip) is OUTSIDE the fold — vaporizing it synthetically
|
// the stamp over it, making its loss synthetic); the pre-flip row (its
|
||||||
// proves the bound.
|
// baseline fact ≤ checkpoint, its bytes barrier-synced at the flip) is
|
||||||
//
|
// OUTSIDE the fold — vaporizing it synthetically proves the bound.
|
||||||
// THE CRASH IS ARMED, NOT RACED. The post-flip write "dies" at exactly
|
const postFlip = await brain.add({ data: 'post-flip write', type: NounType.Document, metadata: { era: 'log' } })
|
||||||
// `singleop-after-fact-append`: its fact is in the log and at-ack synced,
|
|
||||||
// and NO pending flush was ever scheduled — so the checkpoint provably
|
|
||||||
// still reads the flip's stamp when the bytes are dropped. The earlier
|
|
||||||
// shape (`add()` then abandon) raced the store's 50ms pending-flush
|
|
||||||
// timer: on a loaded box the flush won, barrier-synced the row, advanced
|
|
||||||
// the stamp over it — and the fold, CORRECTLY bounded, did not restore
|
|
||||||
// bytes the test had synthetically destroyed after they were stamped
|
|
||||||
// durable. The plant lane caught it; the engine was right, the pin was
|
|
||||||
// timing-dependent.
|
|
||||||
const postFlip = `post-flip-${Date.now().toString(36)}-0000-4000-8000-000000000000`
|
|
||||||
const arm = armCrash(brain, 'singleop-after-fact-append')
|
|
||||||
await expect(
|
|
||||||
brain.add({ id: postFlip, data: 'post-flip write', type: NounType.Document, metadata: { era: 'log' } })
|
|
||||||
).rejects.toThrow('simulated process crash at singleop-after-fact-append')
|
|
||||||
expect(arm.fired).toContain('singleop-after-fact-append')
|
|
||||||
expect(readCheckpoint(dir), 'the stamp did not move — nothing flushed after the flip').toBe(founded)
|
|
||||||
await abandonAsCrashed(liveBrains.pop()!)
|
await abandonAsCrashed(liveBrains.pop()!)
|
||||||
// The post-flip row's canonical bytes lived only in the pending tier's
|
|
||||||
// RAM (written at flush, never reached) — the crash takes them for real;
|
|
||||||
// nothing to drop. Only the pre-flip row is vaporized synthetically.
|
|
||||||
dropCanonicalNoun(dir, preFlip)
|
dropCanonicalNoun(dir, preFlip)
|
||||||
|
dropCanonicalNoun(dir, postFlip)
|
||||||
|
|
||||||
const reopened = await openBrain(dir, { logAuthority: 'adopt' })
|
const reopened = await openBrain(dir, { logAuthority: 'adopt' })
|
||||||
liveBrains.push(reopened)
|
liveBrains.push(reopened)
|
||||||
|
|
|
||||||
|
|
@ -1,62 +0,0 @@
|
||||||
/**
|
|
||||||
* @module tests/integration/null-metadata-delete
|
|
||||||
* @description The null-metadata delete skip is CLOSED. remove() used to
|
|
||||||
* guard its index legs with `if (metadata)` — a row whose canonical
|
|
||||||
* metadata was unreadable at delete time (torn, or a leg lost to an old
|
|
||||||
* defect) kept its postings FOREVER, silently. Now: the JS index gets an
|
|
||||||
* id-keyed cleanup (deleted bitmap + id mapper), the id-keyed native
|
|
||||||
* contract is used when a provider offers it, and the one remaining
|
|
||||||
* skip-shape (native without the contract) is narrated and tracked, never
|
|
||||||
* silent. Pinned: a metadata-less row with live postings deletes cleanly
|
|
||||||
* and leaves the query universe.
|
|
||||||
*/
|
|
||||||
import { describe, it, expect, afterEach } from 'vitest'
|
|
||||||
import { mkdtempSync, rmSync } from 'node:fs'
|
|
||||||
import { tmpdir } from 'node:os'
|
|
||||||
import { join } from 'node:path'
|
|
||||||
import { Brainy } from '../../src/index.js'
|
|
||||||
import { NounType } from '../../src/types/graphTypes.js'
|
|
||||||
|
|
||||||
type RawBox = {
|
|
||||||
storage: {
|
|
||||||
readNounRaw(id: string): Promise<{ metadata: unknown; vector: unknown }>
|
|
||||||
writeNounRaw(id: string, r: { metadata: unknown; vector: unknown }): Promise<void>
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const dirs: string[] = []
|
|
||||||
const brains: Brainy[] = []
|
|
||||||
afterEach(async () => {
|
|
||||||
for (const b of brains.splice(0)) await b.close().catch(() => {})
|
|
||||||
for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true })
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('null-metadata delete', () => {
|
|
||||||
it('a row whose metadata leg is gone still deletes — id-keyed cleanup, no silent skip, gone from the query universe', async () => {
|
|
||||||
const dir = mkdtempSync(join(tmpdir(), 'brainy-nullmeta-del-'))
|
|
||||||
dirs.push(dir)
|
|
||||||
const brain = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false, silent: true })
|
|
||||||
await brain.init()
|
|
||||||
brains.push(brain)
|
|
||||||
|
|
||||||
const keep = await brain.add({ data: 'survivor', type: NounType.Document, metadata: { team: 'atlas' } })
|
|
||||||
const victim = await brain.add({ data: 'doomed', type: NounType.Document, metadata: { team: 'atlas' } })
|
|
||||||
await brain.flush()
|
|
||||||
expect((await brain.find({ where: { team: 'atlas' } })).length).toBe(2)
|
|
||||||
|
|
||||||
// Manufacture the shape: the victim's metadata leg vanishes behind the
|
|
||||||
// engine's back (vector leg + postings stay live).
|
|
||||||
const storage = (brain as unknown as RawBox).storage
|
|
||||||
const raw = await storage.readNounRaw(victim)
|
|
||||||
await storage.writeNounRaw(victim, { metadata: null, vector: raw.vector })
|
|
||||||
|
|
||||||
// THE PIN: the delete neither throws nor silently strands postings.
|
|
||||||
await brain.remove(victim)
|
|
||||||
await brain.flush()
|
|
||||||
|
|
||||||
const after = await brain.find({ where: { team: 'atlas' } })
|
|
||||||
expect(after.length, 'victim left the query universe; survivor serves').toBe(1)
|
|
||||||
expect(after[0].id).toBe(keep)
|
|
||||||
expect(await brain.get(victim)).toBeNull()
|
|
||||||
}, 120000)
|
|
||||||
})
|
|
||||||
|
|
@ -1,68 +0,0 @@
|
||||||
/**
|
|
||||||
* @module tests/integration/read-surface-readiness
|
|
||||||
* @description THE READ-SURFACE READINESS GATE (a production blackout's
|
|
||||||
* brainy half): with `disableAutoRebuild: true`, init defers index builds —
|
|
||||||
* and before this gate, only find() waited for the lazy rebuild while
|
|
||||||
* related() and every VFS path served EMPTY from the not-ready providers
|
|
||||||
* (writes acked into canonical, readback empty — fifteen live minutes).
|
|
||||||
* The pins: on a fresh instance over a populated store, the FIRST read on
|
|
||||||
* every surface serves truth (it waits for the build), never empty.
|
|
||||||
*/
|
|
||||||
import { describe, it, expect, afterEach } from 'vitest'
|
|
||||||
import { mkdtempSync, rmSync } from 'node:fs'
|
|
||||||
import { tmpdir } from 'node:os'
|
|
||||||
import { join } from 'node:path'
|
|
||||||
import { Brainy } from '../../src/index.js'
|
|
||||||
import { NounType, VerbType } from '../../src/types/graphTypes.js'
|
|
||||||
|
|
||||||
const dirs: string[] = []
|
|
||||||
const brains: Brainy[] = []
|
|
||||||
afterEach(async () => {
|
|
||||||
for (const b of brains.splice(0)) await b.close().catch(() => {})
|
|
||||||
for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true })
|
|
||||||
})
|
|
||||||
|
|
||||||
async function openLazy(dir: string): Promise<Brainy> {
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: { type: 'filesystem', path: dir },
|
|
||||||
requireSubtype: false,
|
|
||||||
silent: true,
|
|
||||||
disableAutoRebuild: true
|
|
||||||
})
|
|
||||||
await brain.init()
|
|
||||||
brains.push(brain)
|
|
||||||
return brain
|
|
||||||
}
|
|
||||||
|
|
||||||
describe('read-surface readiness gate', () => {
|
|
||||||
it('related() as the FIRST read on a fresh lazy instance serves truth, never empty', async () => {
|
|
||||||
const dir = mkdtempSync(join(tmpdir(), 'brainy-readgate-'))
|
|
||||||
dirs.push(dir)
|
|
||||||
const writer = await openLazy(dir)
|
|
||||||
const a = await writer.add({ data: 'hub row', type: NounType.Document, metadata: { n: 1 } })
|
|
||||||
const b = await writer.add({ data: 'leaf row', type: NounType.Document, metadata: { n: 2 } })
|
|
||||||
await writer.relate({ from: a, to: b, type: VerbType.RelatedTo })
|
|
||||||
await writer.flush()
|
|
||||||
await brains.pop()!.close()
|
|
||||||
|
|
||||||
// Fresh instance: indexes deferred at open. The production shape called
|
|
||||||
// related() FIRST (no find() to trigger the old, only gate).
|
|
||||||
const reader = await openLazy(dir)
|
|
||||||
const rels = await reader.related({ from: a })
|
|
||||||
expect(rels.length, 'the FIRST related() read waits for the build and serves').toBeGreaterThan(0)
|
|
||||||
expect(rels.some((r) => r.to === b || (r as { target?: string }).target === b)).toBe(true)
|
|
||||||
}, 120000)
|
|
||||||
|
|
||||||
it('a metadata-filtered read as the FIRST read serves truth, never empty', async () => {
|
|
||||||
const dir = mkdtempSync(join(tmpdir(), 'brainy-readgate2-'))
|
|
||||||
dirs.push(dir)
|
|
||||||
const writer = await openLazy(dir)
|
|
||||||
await writer.add({ data: 'tagged row', type: NounType.Document, metadata: { team: 'atlas' } })
|
|
||||||
await writer.flush()
|
|
||||||
await brains.pop()!.close()
|
|
||||||
|
|
||||||
const reader = await openLazy(dir)
|
|
||||||
const rows = await reader.find({ where: { team: 'atlas' } })
|
|
||||||
expect(rows.length, 'filtered find on a cold lazy instance serves').toBe(1)
|
|
||||||
}, 120000)
|
|
||||||
})
|
|
||||||
|
|
@ -1,71 +0,0 @@
|
||||||
/**
|
|
||||||
* @module tests/integration/repair-report
|
|
||||||
* @description repairIndex() returns the per-family receipt (checked /
|
|
||||||
* healed / skipped-with-reason per family) and narrates a summary — the
|
|
||||||
* "repair that shows its work" half of the graph-trust program's ask. A
|
|
||||||
* repair nobody can audit is a repair nobody can trust.
|
|
||||||
*/
|
|
||||||
import { describe, it, expect, afterEach } from 'vitest'
|
|
||||||
import { mkdtempSync, rmSync } from 'node:fs'
|
|
||||||
import { tmpdir } from 'node:os'
|
|
||||||
import { join } from 'node:path'
|
|
||||||
import { Brainy } from '../../src/index.js'
|
|
||||||
import { NounType } from '../../src/types/graphTypes.js'
|
|
||||||
|
|
||||||
type RawBox = {
|
|
||||||
storage: { writeNounRaw(id: string, r: { metadata: unknown; vector: unknown }): Promise<void> }
|
|
||||||
}
|
|
||||||
|
|
||||||
const dirs: string[] = []
|
|
||||||
const brains: Brainy[] = []
|
|
||||||
afterEach(async () => {
|
|
||||||
for (const b of brains.splice(0)) await b.close().catch(() => {})
|
|
||||||
for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true })
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('repairIndex per-family receipt', () => {
|
|
||||||
it('a healthy store gets a complete zero-heal receipt — every family accounted, none silent', async () => {
|
|
||||||
const dir = mkdtempSync(join(tmpdir(), 'brainy-repair-clean-'))
|
|
||||||
dirs.push(dir)
|
|
||||||
const brain = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false, silent: true })
|
|
||||||
await brain.init()
|
|
||||||
brains.push(brain)
|
|
||||||
await brain.add({ data: 'healthy row', type: NounType.Document, metadata: { n: 1 } })
|
|
||||||
await brain.flush()
|
|
||||||
|
|
||||||
const report = await brain.repairIndex()
|
|
||||||
expect(report.families.length, 'every family reports a row').toBeGreaterThanOrEqual(5)
|
|
||||||
const names = report.families.map((f) => f.family)
|
|
||||||
for (const expected of ['orphaned-containers', 'count-rollups', 'metadata-corruption']) {
|
|
||||||
expect(names, `family ${expected} accounted`).toContain(expected)
|
|
||||||
}
|
|
||||||
// Every row is either checked or carries its skip reason — no silent rows.
|
|
||||||
for (const f of report.families) {
|
|
||||||
expect(f.checked || !!f.skipped, `${f.family} is checked or explains itself`).toBe(true)
|
|
||||||
}
|
|
||||||
expect(report.healedTotal).toBe(0)
|
|
||||||
expect(report.durationMs).toBeGreaterThanOrEqual(0)
|
|
||||||
}, 120000)
|
|
||||||
|
|
||||||
it('a manufactured ghost container appears in the receipt as a heal', async () => {
|
|
||||||
const dir = mkdtempSync(join(tmpdir(), 'brainy-repair-ghost-'))
|
|
||||||
dirs.push(dir)
|
|
||||||
const brain = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false, silent: true })
|
|
||||||
await brain.init()
|
|
||||||
brains.push(brain)
|
|
||||||
await brain.add({ data: 'real row', type: NounType.Document, metadata: { n: 1 } })
|
|
||||||
await brain.flush()
|
|
||||||
// The pre-8.3.1 ghost shape: a vector leg with no content leg.
|
|
||||||
const storage = (brain as unknown as RawBox).storage
|
|
||||||
await storage.writeNounRaw('00000000-0000-7000-8000-00000000dead', {
|
|
||||||
metadata: null,
|
|
||||||
vector: { vector: [0.1, 0.2], noun: 'document' }
|
|
||||||
})
|
|
||||||
|
|
||||||
const report = await brain.repairIndex()
|
|
||||||
const orphans = report.families.find((f) => f.family === 'orphaned-containers')
|
|
||||||
expect(orphans?.checked).toBe(true)
|
|
||||||
expect(orphans!.healed, 'the ghost was pruned and receipted').toBeGreaterThan(0)
|
|
||||||
expect(report.healedTotal).toBeGreaterThan(0)
|
|
||||||
}, 120000)
|
|
||||||
})
|
|
||||||
|
|
@ -77,31 +77,6 @@ describe('validateIndexConsistency delegates to provider validateInvariants() (P
|
||||||
delete brain.index.validateInvariants
|
delete brain.index.validateInvariants
|
||||||
})
|
})
|
||||||
|
|
||||||
it('ONE CONTRACT FOR A THROWING PROBE: heal is none (flakiness never buys a rebuild) and serving is not withheld', async () => {
|
|
||||||
// The probe that fails to RUN must never be read as "the index is broken,
|
|
||||||
// rebuild it" — that synthesized heal:'rebuild' was the dark-rebuild lever
|
|
||||||
// one transient exception away, and the native composer already said
|
|
||||||
// 'none' for the same event. Both engines now agree: named, loud,
|
|
||||||
// unverified — and never a rebuild, never a withheld serve.
|
|
||||||
brain.index.validateInvariants = async () => { throw new Error('transient: mmap window busy') }
|
|
||||||
const v = await brain.validateIndexConsistency()
|
|
||||||
const thrown = v.providers?.find((p: ProviderInvariantReport) =>
|
|
||||||
p.invariants.some((i) => i.name === 'validate-invariants-threw')
|
|
||||||
)
|
|
||||||
expect(thrown).toBeDefined()
|
|
||||||
expect(thrown!.healthy).toBe(false)
|
|
||||||
expect(thrown!.serving).toBe(true)
|
|
||||||
const inv = thrown!.invariants.find((i) => i.name === 'validate-invariants-threw')!
|
|
||||||
expect(inv.holds).toBe(false)
|
|
||||||
expect(inv.heal).toBe('none')
|
|
||||||
expect(inv.detail).toMatch(/transient: mmap window busy/)
|
|
||||||
// No provider report in the set recommends a rebuild for this event.
|
|
||||||
expect(
|
|
||||||
v.providers!.flatMap((p: ProviderInvariantReport) => p.invariants).some((i) => i.heal === 'rebuild')
|
|
||||||
).toBe(false)
|
|
||||||
delete brain.index.validateInvariants
|
|
||||||
})
|
|
||||||
|
|
||||||
it('providers without validateInvariants() are omitted (JS baseline unchanged)', async () => {
|
it('providers without validateInvariants() are omitted (JS baseline unchanged)', async () => {
|
||||||
const v = await brain.validateIndexConsistency()
|
const v = await brain.validateIndexConsistency()
|
||||||
expect(v.providers).toBeUndefined()
|
expect(v.providers).toBeUndefined()
|
||||||
|
|
|
||||||
Reference in a new issue