Compare commits
No commits in common. "v10.4.0-rc.1" and "v10.3.1" have entirely different histories.
v10.4.0-rc
...
v10.3.1
22 changed files with 87 additions and 1412 deletions
|
|
@ -34,16 +34,7 @@ jobs:
|
|||
|
||||
SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/"
|
||||
VERSION="$(node -p "require('./package.json').version")"
|
||||
# The dist-tag follows the version: a prerelease (any hyphen —
|
||||
# 10.4.0-rc.1) publishes under 'rc' and must NEVER move 'latest' —
|
||||
# every consumer resolving 'latest' from this registry would otherwise
|
||||
# be handed a release candidate. Same rule scripts/release.sh applies
|
||||
# to the storefront leg.
|
||||
NPM_TAG="latest"
|
||||
case "$VERSION" in
|
||||
*-*) NPM_TAG="rc" ;;
|
||||
esac
|
||||
echo "Publishing @soulcraft/brainy@${VERSION} to The Source registry (dist-tag: ${NPM_TAG})..."
|
||||
echo "Publishing @soulcraft/brainy@${VERSION} to The Source registry..."
|
||||
|
||||
TMPRC="$(mktemp)"
|
||||
chmod 600 "$TMPRC"
|
||||
|
|
@ -56,7 +47,7 @@ jobs:
|
|||
# this tag's checkout already carries the version being published —
|
||||
# nothing here re-derives it from the tag name.
|
||||
PUBLISH_OK=true
|
||||
if ! npm publish --tag "$NPM_TAG" --userconfig "$TMPRC"; then
|
||||
if ! npm publish --tag latest --userconfig "$TMPRC"; then
|
||||
PUBLISH_OK=false
|
||||
fi
|
||||
|
||||
|
|
|
|||
13
CHANGELOG.md
13
CHANGELOG.md
|
|
@ -2,19 +2,6 @@
|
|||
|
||||
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
||||
|
||||
### [10.4.0-rc.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.3.1...v10.4.0-rc.1) (2026-08-24)
|
||||
|
||||
- ci(publish): the home dist-tag follows the version — a prerelease publishes under 'rc' and never moves 'latest' (a1376e4a)
|
||||
- chore(release): --source-only — a home-only prerelease mode (The Source, never the storefront) (dcbad176)
|
||||
- test(fold-checkpoint): the ARM-AT-FLIP pin arms its crash instead of racing the pending-flush timer (4176439b)
|
||||
- fix(health): one contract for a throwing probe — heal is none, serving is not withheld; repair report gains missing/rebuilt/reason (116550eb)
|
||||
- feat(storage): the canonical count ledger — ALL-visibility scalars, unclamped totals, suspect-on-unprovable-delete (7c8c8be3)
|
||||
- fix(delete): the null-metadata skip closes — index legs run id-keyed or narrate, never silently strand postings (607e9f54)
|
||||
- feat(repair): repairIndex returns the per-family receipt and narrates its summary (8d45f964)
|
||||
- fix(reads): the readiness gate guards every index read surface — serving empty from a not-ready provider is unrepresentable (40e7119b)
|
||||
- ci(gate): the machine-health preflight and the truncation verdict guard (1e046aa1)
|
||||
|
||||
|
||||
### [10.3.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.3.0...v10.3.1) (2026-08-18)
|
||||
|
||||
- docs(releases): the 10.3.1 consumer entry — the fold that behaves (900cc895)
|
||||
|
|
|
|||
4
package-lock.json
generated
4
package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"name": "@soulcraft/brainy",
|
||||
"version": "10.4.0-rc.1",
|
||||
"version": "10.3.1",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@soulcraft/brainy",
|
||||
"version": "10.4.0-rc.1",
|
||||
"version": "10.3.1",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@msgpack/msgpack": "^3.1.2",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@soulcraft/brainy",
|
||||
"version": "10.4.0-rc.1",
|
||||
"version": "10.3.1",
|
||||
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
|
||||
"main": "dist/index.js",
|
||||
"module": "dist/index.js",
|
||||
|
|
|
|||
|
|
@ -1,85 +0,0 @@
|
|||
# Gate Guards
|
||||
|
||||
Two standalone scripts that stand between a test/build gate and a false
|
||||
verdict: one refuses to let the gate start on a noisy machine, the other
|
||||
refuses to let a truncated or crashed vitest run be read as green.
|
||||
|
||||
## Why these exist
|
||||
|
||||
Both guards exist because of the 2026-08-13 lost-day ledger: a gate ran on
|
||||
a machine under load, and separately a vitest worker pool died mid-suite
|
||||
while still printing a plausible-looking summary line, and in both cases
|
||||
the bad result was trusted and acted on for the better part of a day before
|
||||
anyone noticed. Neither failure mode announces itself — a loaded machine
|
||||
still finishes and reports numbers, and a truncated test run still prints a
|
||||
`Test Files` / `Tests` line — so both guards check the evidence explicitly
|
||||
rather than trusting that a gate finishing means the gate was valid.
|
||||
|
||||
## gate-preflight.sh
|
||||
|
||||
Run before any gate lane starts. Exits 1 the moment the machine isn't
|
||||
gate-clean, with one `FATAL:` line per violation naming the exact offender
|
||||
(the pid and command, the path, the measured value). Prints one `OK:` line
|
||||
per check that passes. `WARNING:` lines mark checks that were skipped, not
|
||||
failures.
|
||||
|
||||
Checks:
|
||||
|
||||
| # | Check | Default threshold | Override |
|
||||
|---|-------|--------------------|----------|
|
||||
| a | 1-minute load average | `nproc / 2` | `GATE_MAX_LOAD` |
|
||||
| b | any non-allowlisted process over 50% of one core | 50% | `GATE_ALLOW_REGEX` (extra pattern matched against the process's args) |
|
||||
| c | cpu0 scaling governor must be `performance` | — | none (warns and skips if the sysfs path is absent) |
|
||||
| d | free space on `/` and `/tmp` | 10G each | `GATE_SKIP_DISK_CHECK=1` to skip entirely |
|
||||
|
||||
The allowlist for check (b) is always: this script's own process tree
|
||||
(its ancestors and its direct child processes), `sshd`, `systemd`, and
|
||||
kernel threads (recognizable by args wrapped in brackets, e.g.
|
||||
`[kworker/0:1]`). `GATE_ALLOW_REGEX` extends it — it does not replace it.
|
||||
|
||||
## vitest-verdict-check.sh
|
||||
|
||||
Run after every vitest lane, against that lane's captured log. Fails
|
||||
loudly, quoting the exact line or string that tripped it, when the log's
|
||||
own summary can't be trusted:
|
||||
|
||||
- no `Test Files` (or, in `--count-tests` mode, `Tests`) summary line is
|
||||
present at all
|
||||
- the parenthesized total in that line doesn't match what was expected
|
||||
- fewer files/tests are accounted for (passed + failed + skipped) than the
|
||||
total claims — a truncated run
|
||||
- the log contains `Unhandled Error` or `Timeout calling` anywhere — a dead
|
||||
worker pool, regardless of what the summary line claims
|
||||
|
||||
```
|
||||
vitest-verdict-check.sh <log-file> <expected-file-count>
|
||||
vitest-verdict-check.sh --count-tests <log-file> <minimum-test-count>
|
||||
```
|
||||
|
||||
The first form checks `Test Files` for an exact match. The second checks
|
||||
`Tests` for a minimum (a floor, not an exact count, since the total number
|
||||
of individual tests moves more often than the number of test files).
|
||||
|
||||
## Wiring into a CI lane
|
||||
|
||||
```sh
|
||||
# Before any lane that will report a verdict:
|
||||
scripts/gate/gate-preflight.sh || exit 1
|
||||
|
||||
# Run the suite, capturing its output:
|
||||
npx vitest run tests/unit 2>&1 | tee /tmp/unit.log
|
||||
|
||||
# After every vitest lane, check the log against the actual file count:
|
||||
EXPECTED_FILES=$(ls tests/unit/**/*.test.ts | wc -l)
|
||||
scripts/gate/vitest-verdict-check.sh /tmp/unit.log "$EXPECTED_FILES" || exit 1
|
||||
```
|
||||
|
||||
## Exit-code contract
|
||||
|
||||
| Script | Exit 0 | Exit 1 |
|
||||
|--------|--------|--------|
|
||||
| `gate-preflight.sh` | machine is gate-clean | one or more `FATAL:` violations printed |
|
||||
| `vitest-verdict-check.sh` | log's summary is trustworthy and matches | usage error, missing/unreadable log, or one or more `FATAL:` violations printed |
|
||||
|
||||
Non-zero from either script means: do not trust the gate that was about to
|
||||
run, or the result of the one that just ran.
|
||||
|
|
@ -1,206 +0,0 @@
|
|||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# Brainy Gate Preflight
|
||||
# Refuses to let a test/build gate run on a machine that isn't clean enough
|
||||
# to trust the numbers it produces. See scripts/gate/README.md for why (the
|
||||
# 2026-08-13 lost-day ledger).
|
||||
#
|
||||
# Checks: 1-minute load average, any non-allowlisted process pinning a core,
|
||||
# the cpu0 scaling governor, and free space on / and /tmp.
|
||||
#
|
||||
# Exit 0 and print one OK line per passing check when the machine is clean.
|
||||
# Exit 1 and print one FATAL line per violation, naming the offender, when
|
||||
# it is not.
|
||||
#
|
||||
# Known trap: a helper function whose last executed statement is a `while`
|
||||
# (or any command whose own exit status happens to be nonzero) hands that
|
||||
# status back as the function's return value. Called as a plain statement,
|
||||
# that silently kills this script under `set -e`. Every helper below ends
|
||||
# on an explicit `return 0` as its own statement, never on a loop or test.
|
||||
#
|
||||
# The same failure mode hides in plainer-looking lines too: `var=$(cmd)` is
|
||||
# a bare assignment, so `set -e` DOES treat a nonzero `cmd` (or, under
|
||||
# `pipefail`, a nonzero stage anywhere in `cmd`'s pipeline) as a failure of
|
||||
# that statement and kills the script right there — even mid-loop, even
|
||||
# when the "failure" is routine (a process that exited before a second
|
||||
# lookup, a path that doesn't exist). Every such assignment below is paired
|
||||
# with an explicit `|| var=""` fallback so a routine miss degrades to an
|
||||
# empty value instead of an exit.
|
||||
|
||||
VIOLATIONS=0
|
||||
ANCESTOR_PIDS=""
|
||||
|
||||
fatal() {
|
||||
echo "FATAL: $1"
|
||||
VIOLATIONS=$((VIOLATIONS + 1))
|
||||
}
|
||||
|
||||
ok() {
|
||||
echo "OK: $1"
|
||||
}
|
||||
|
||||
# Walks this process's parent chain up to pid 1, then takes one snapshot of
|
||||
# its direct children (the ps/read pipeline in check_processes), and
|
||||
# records both in ANCESTOR_PIDS — so the process-scan below can recognize
|
||||
# its own tree (the shell/terminal/session that launched it, plus its own
|
||||
# helper commands) instead of flagging it. Children are captured once, up
|
||||
# front, rather than re-queried per row later, so a helper command that has
|
||||
# already exited by the time it's looked up can't be mistaken for a miss.
|
||||
build_ancestor_pids() {
|
||||
local pid="$$"
|
||||
local ppid child
|
||||
ANCESTOR_PIDS=" $pid "
|
||||
while [ "$pid" != "1" ]; do
|
||||
ppid=$(ps -o ppid= -p "$pid" 2>/dev/null | tr -d ' ') || ppid=""
|
||||
if [ -z "$ppid" ]; then
|
||||
break
|
||||
fi
|
||||
ANCESTOR_PIDS="${ANCESTOR_PIDS}${ppid} "
|
||||
pid="$ppid"
|
||||
done
|
||||
|
||||
while IFS= read -r child; do
|
||||
[ -z "$child" ] && continue
|
||||
ANCESTOR_PIDS="${ANCESTOR_PIDS}${child} "
|
||||
done < <(ps --ppid "$$" -o pid= 2>/dev/null || true)
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# (a) 1-minute load average vs. threshold (default: nproc / 2).
|
||||
check_load() {
|
||||
local max_load="${GATE_MAX_LOAD:-}"
|
||||
if [ -z "$max_load" ]; then
|
||||
max_load=$(( $(nproc) / 2 ))
|
||||
if [ "$max_load" -lt 1 ]; then
|
||||
max_load=1
|
||||
fi
|
||||
fi
|
||||
|
||||
local load_1m
|
||||
load_1m=$(cut -d' ' -f1 /proc/loadavg)
|
||||
|
||||
if awk -v l="$load_1m" -v m="$max_load" 'BEGIN { exit !(l > m) }'; then
|
||||
fatal "1-minute load average ${load_1m} exceeds threshold ${max_load} (GATE_MAX_LOAD=${max_load})"
|
||||
else
|
||||
ok "1-minute load average ${load_1m} is within threshold ${max_load}"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# (b) any process outside the allowlist pinning more than half a core.
|
||||
# Parsed with `read` into named fields, not an awk/cut chain — a fixed-column
|
||||
# awk/cut split on `ps` output duplicated fields the first time this was
|
||||
# tried, because process args vary in word count. `read` with a fixed list
|
||||
# of variables dumps everything left over into the last one (args), which
|
||||
# handles that correctly.
|
||||
check_processes() {
|
||||
local max_pcpu=50
|
||||
local extra_regex="${GATE_ALLOW_REGEX:-}"
|
||||
local violation_found=0
|
||||
local line pcpu pid args pcpu_int
|
||||
|
||||
while IFS= read -r line; do
|
||||
[ -z "$line" ] && continue
|
||||
read -r pcpu pid args <<< "$line"
|
||||
|
||||
# Kernel threads report their comm in brackets, e.g. "[kworker/0:1]".
|
||||
case "$args" in
|
||||
\[*\]) continue ;;
|
||||
esac
|
||||
|
||||
# This script's own tree: its ancestors (shell, terminal, session) and
|
||||
# its direct children, both captured once by build_ancestor_pids.
|
||||
case " $ANCESTOR_PIDS " in
|
||||
*" $pid "*) continue ;;
|
||||
esac
|
||||
|
||||
case "$args" in
|
||||
*sshd*|*systemd*) continue ;;
|
||||
esac
|
||||
|
||||
if [ -n "$extra_regex" ] && [[ "$args" =~ $extra_regex ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
pcpu_int="${pcpu%.*}"
|
||||
if [ -z "$pcpu_int" ]; then
|
||||
pcpu_int=0
|
||||
fi
|
||||
if [ "$pcpu_int" -gt "$max_pcpu" ]; then
|
||||
fatal "pid ${pid} ('${args}') is using ${pcpu}% of one core"
|
||||
violation_found=1
|
||||
fi
|
||||
done < <(ps -eo pcpu,pid,args --sort=-pcpu | tail -n +2)
|
||||
|
||||
if [ "$violation_found" -eq 0 ]; then
|
||||
ok "no process outside the allowlist exceeds ${max_pcpu}% of one core"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# (c) cpu0 scaling governor must be "performance". Skipped with a warning
|
||||
# (not a violation) when the sysfs path doesn't exist on this machine.
|
||||
check_governor() {
|
||||
local gov_path="/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor"
|
||||
if [ ! -r "$gov_path" ]; then
|
||||
echo "WARNING: ${gov_path} not present; skipping governor check"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local governor
|
||||
governor=$(cat "$gov_path" 2>/dev/null) || governor=""
|
||||
if [ "$governor" != "performance" ]; then
|
||||
fatal "cpu0 governor is '${governor}', not 'performance'"
|
||||
else
|
||||
ok "cpu0 governor is 'performance'"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# (d) free-space floors on / and /tmp (default 10G each). Skip entirely via
|
||||
# GATE_SKIP_DISK_CHECK=1.
|
||||
check_disk() {
|
||||
if [ "${GATE_SKIP_DISK_CHECK:-0}" = "1" ]; then
|
||||
echo "WARNING: disk free-space check skipped (GATE_SKIP_DISK_CHECK=1)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local floor_gb=10
|
||||
local floor_bytes=$((floor_gb * 1024 * 1024 * 1024))
|
||||
local path avail_bytes avail_gb
|
||||
|
||||
for path in / /tmp; do
|
||||
avail_bytes=$(df --output=avail -B1 "$path" 2>/dev/null | tail -n 1 | tr -d ' ') || avail_bytes=""
|
||||
if [ -z "$avail_bytes" ]; then
|
||||
echo "WARNING: could not determine free space on ${path}; skipping"
|
||||
continue
|
||||
fi
|
||||
if [ "$avail_bytes" -lt "$floor_bytes" ]; then
|
||||
avail_gb=$((avail_bytes / 1024 / 1024 / 1024))
|
||||
fatal "${path} has only ${avail_gb}G free, below the ${floor_gb}G floor"
|
||||
else
|
||||
ok "${path} has enough free space (floor ${floor_gb}G)"
|
||||
fi
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
echo "Brainy gate preflight"
|
||||
echo "----------------------"
|
||||
|
||||
build_ancestor_pids
|
||||
check_load
|
||||
check_processes
|
||||
check_governor
|
||||
check_disk
|
||||
|
||||
echo "----------------------"
|
||||
if [ "$VIOLATIONS" -gt 0 ]; then
|
||||
echo "FATAL: gate preflight failed with ${VIOLATIONS} violation(s) — machine is not gate-clean"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "gate preflight passed — machine is gate-clean"
|
||||
exit 0
|
||||
|
|
@ -1,158 +0,0 @@
|
|||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# Brainy Vitest Verdict Check
|
||||
# Confirms a vitest run's own summary line is trustworthy before anything
|
||||
# downstream treats a green run as green. See scripts/gate/README.md for why
|
||||
# (the 2026-08-13 lost-day ledger).
|
||||
#
|
||||
# Usage:
|
||||
# vitest-verdict-check.sh <log-file> <expected-file-count>
|
||||
# vitest-verdict-check.sh --count-tests <log-file> <minimum-test-count>
|
||||
#
|
||||
# The first form checks the "Test Files" summary line's total against an
|
||||
# exact expected count. The second checks the "Tests" summary line's total
|
||||
# against a minimum. Both also fail on any sign the worker pool died
|
||||
# mid-run, whether or not a summary line still made it into the log.
|
||||
#
|
||||
# Exit 0 and print one OK line per passing check when the log is clean.
|
||||
# Exit 1 and print one FATAL line per violation, quoting the exact line or
|
||||
# string that tripped it, when it is not.
|
||||
#
|
||||
# Known trap (shared with gate-preflight.sh): every helper below ends on an
|
||||
# explicit `return 0` as its own statement, never on a loop or test, so a
|
||||
# helper's last command can never hand its own exit status back as the
|
||||
# function's under `set -e`. The same applies to `var=$(cmd)` assignments
|
||||
# mid-helper: a bare assignment IS checked by `set -e`, so a `grep` that
|
||||
# legitimately finds nothing (exit 1) would otherwise kill the script
|
||||
# instead of just leaving the variable empty — every such assignment below
|
||||
# is paired with an explicit `|| true` inside the substitution.
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 <log-file> <expected-file-count>"
|
||||
echo " $0 --count-tests <log-file> <minimum-test-count>"
|
||||
exit 1
|
||||
}
|
||||
|
||||
MODE="files"
|
||||
if [ "${1:-}" = "--count-tests" ]; then
|
||||
MODE="tests"
|
||||
shift
|
||||
fi
|
||||
|
||||
LOG_FILE="${1:-}"
|
||||
THRESHOLD="${2:-}"
|
||||
|
||||
if [ -z "$LOG_FILE" ] || [ -z "$THRESHOLD" ]; then
|
||||
usage
|
||||
fi
|
||||
|
||||
if [ ! -f "$LOG_FILE" ]; then
|
||||
echo "FATAL: log file '${LOG_FILE}' does not exist"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! [[ "$THRESHOLD" =~ ^[0-9]+$ ]]; then
|
||||
echo "FATAL: threshold '${THRESHOLD}' is not a non-negative integer"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
VIOLATIONS=0
|
||||
|
||||
fatal() {
|
||||
echo "FATAL: $1"
|
||||
VIOLATIONS=$((VIOLATIONS + 1))
|
||||
}
|
||||
|
||||
ok() {
|
||||
echo "OK: $1"
|
||||
}
|
||||
|
||||
# Vitest colorizes its summary with ANSI escapes; strip them before parsing
|
||||
# anything, or the color codes end up embedded in the fields we grep for.
|
||||
CLEAN_LOG="$(sed 's/\x1b\[[0-9;]*m//g' "$LOG_FILE")"
|
||||
|
||||
# Worker-pool death: if either string appears, the run's own summary line —
|
||||
# even if present and even if its numbers look fine — cannot be trusted,
|
||||
# because the process died mid-suite and vitest's own accounting is what
|
||||
# died with it.
|
||||
check_worker_death() {
|
||||
if echo "$CLEAN_LOG" | grep -q "Unhandled Error"; then
|
||||
fatal "log contains 'Unhandled Error' — worker pool died mid-run"
|
||||
fi
|
||||
if echo "$CLEAN_LOG" | grep -q "Timeout calling"; then
|
||||
fatal "log contains 'Timeout calling' — worker pool died mid-run"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# Shared shape between the "Test Files" and "Tests" summary lines:
|
||||
# <label> <n> passed | <n> failed | <n> skipped (<total>)
|
||||
# `compare` is "eq" (total must equal threshold) or "min" (total must be at
|
||||
# least threshold).
|
||||
check_summary_line() {
|
||||
local label="$1"
|
||||
local threshold="$2"
|
||||
local compare="$3"
|
||||
local summary_line total accounted n
|
||||
|
||||
summary_line=$(echo "$CLEAN_LOG" | grep -E "^[[:space:]]*${label}[[:space:]]+" | tail -n 1 || true)
|
||||
|
||||
if [ -z "$summary_line" ]; then
|
||||
fatal "no '${label}' summary line found in ${LOG_FILE}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
total=$(echo "$summary_line" | grep -oE '\([0-9]+\)' | tr -d '()' | tail -n 1 || true)
|
||||
if [ -z "$total" ]; then
|
||||
fatal "'${label}' summary line has no parenthesized total: \"${summary_line}\""
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ "$compare" = "eq" ]; then
|
||||
if [ "$total" -ne "$threshold" ]; then
|
||||
fatal "'${label}' total is ${total}, expected ${threshold}: \"${summary_line}\""
|
||||
else
|
||||
ok "'${label}' total matches expected ${threshold}"
|
||||
fi
|
||||
else
|
||||
if [ "$total" -lt "$threshold" ]; then
|
||||
fatal "'${label}' total is ${total}, below minimum ${threshold}: \"${summary_line}\""
|
||||
else
|
||||
ok "'${label}' total ${total} meets minimum ${threshold}"
|
||||
fi
|
||||
fi
|
||||
|
||||
accounted=0
|
||||
for n in $(echo "$summary_line" | grep -oE '[0-9]+ (passed|failed|skipped)' | grep -oE '^[0-9]+'); do
|
||||
accounted=$((accounted + n))
|
||||
done
|
||||
|
||||
if [ "$accounted" -lt "$total" ]; then
|
||||
fatal "'${label}' line accounts for only ${accounted} of ${total} — truncated run: \"${summary_line}\""
|
||||
else
|
||||
ok "'${label}' line accounts for all ${total}"
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
echo "Brainy vitest verdict check: ${LOG_FILE}"
|
||||
echo "----------------------"
|
||||
|
||||
check_worker_death
|
||||
|
||||
if [ "$MODE" = "files" ]; then
|
||||
check_summary_line "Test Files" "$THRESHOLD" "eq"
|
||||
else
|
||||
check_summary_line "Tests" "$THRESHOLD" "min"
|
||||
fi
|
||||
|
||||
echo "----------------------"
|
||||
if [ "$VIOLATIONS" -gt 0 ]; then
|
||||
echo "FATAL: vitest verdict check failed with ${VIOLATIONS} violation(s) for ${LOG_FILE}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "vitest verdict check passed for ${LOG_FILE}"
|
||||
exit 0
|
||||
|
|
@ -15,12 +15,6 @@ NC='\033[0m' # No Color
|
|||
RELEASE_TYPE="${1:-patch}" # patch, minor, or major
|
||||
SKIP_TESTS=false
|
||||
DRY_RUN=false
|
||||
# --source-only: the HOME leg only — tag, CI's publish to The Source, and the
|
||||
# release page; NO storefront (npmjs) publish, NO pair verification, NO docs
|
||||
# push. The pair-gate shape: a prerelease the fleet's other engine devDeps
|
||||
# from our own registry while the pair is proven, never a public artifact.
|
||||
# Refused for a non-prerelease version — a public floor is always a pair.
|
||||
SOURCE_ONLY=false
|
||||
|
||||
for arg in "$@"; do
|
||||
case $arg in
|
||||
|
|
@ -30,9 +24,6 @@ for arg in "$@"; do
|
|||
--dry-run)
|
||||
DRY_RUN=true
|
||||
;;
|
||||
--source-only)
|
||||
SOURCE_ONLY=true
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
|
|
@ -109,7 +100,7 @@ else
|
|||
;;
|
||||
*)
|
||||
echo -e "${RED}❌ Invalid release type: ${RELEASE_TYPE}${NC}"
|
||||
echo "Usage: ./scripts/release.sh [patch|minor|major|<explicit-version>] [--dry-run] [--source-only (prereleases only)]"
|
||||
echo "Usage: ./scripts/release.sh [patch|minor|major|<explicit-version>] [--dry-run]"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
|
@ -128,13 +119,6 @@ echo -e "${BLUE}New version: ${NEW_VERSION}${NC}"
|
|||
if [ "$PRERELEASE" = true ]; then
|
||||
echo -e "${YELLOW}⚠️ Prerelease → npm dist-tag '${NPM_TAG}', GitHub prerelease${NC}"
|
||||
fi
|
||||
if [ "$SOURCE_ONLY" = true ]; then
|
||||
if [ "$PRERELEASE" != true ]; then
|
||||
echo -e "${RED}❌ --source-only is for prereleases only: a non-prerelease version is a public floor and always ships as the byte-identical pair.${NC}"
|
||||
exit 1
|
||||
fi
|
||||
echo -e "${YELLOW}⚠️ --source-only → The Source (home) ONLY: no npmjs publish, no pair verification, no docs push${NC}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
if [ "$DRY_RUN" = true ]; then
|
||||
|
|
@ -237,50 +221,46 @@ else
|
|||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$SOURCE_ONLY" = true ]; then
|
||||
echo -e "${YELLOW}9️⃣½ Storefront (npmjs) leg SKIPPED — --source-only: v${NEW_VERSION} lives on The Source under dist-tag '${NPM_TAG}' only${NC}\n"
|
||||
else
|
||||
echo -e "${BLUE}9️⃣½ Publishing to npmjs (storefront, dist-tag: ${NPM_TAG})...${NC}"
|
||||
# BYTE-IDENTITY LAW: the storefront republishes CI's EXACT artifact — download
|
||||
# the tarball The Source serves and publish that file, never a fresh local pack
|
||||
# (a local rebuild can differ byte-wise, and the fleet verifies the pair by
|
||||
# shasum across registries).
|
||||
STOREFRONT_TMP="$(mktemp -d)"
|
||||
(cd "$STOREFRONT_TMP" && npm pack "@soulcraft/brainy@${NEW_VERSION}" "--@soulcraft:registry=${SOURCE_NPM_REG}" >/dev/null)
|
||||
SOURCE_TARBALL="$(ls "$STOREFRONT_TMP"/soulcraft-brainy-*.tgz)"
|
||||
echo -e "${BLUE} home artifact: $(sha256sum "$SOURCE_TARBALL" | cut -d' ' -f1)${NC}"
|
||||
npm publish "$SOURCE_TARBALL" --tag "$NPM_TAG" "--@soulcraft:registry=https://registry.npmjs.org/"
|
||||
rm -rf "$STOREFRONT_TMP"
|
||||
# Brainy is the only PUBLIC @soulcraft package — verify visibility after every publish.
|
||||
npm access get status @soulcraft/brainy "--@soulcraft:registry=https://registry.npmjs.org/" || true
|
||||
# Verify the pair is byte-identical by registry-reported shasum — divergence
|
||||
# here means the storefront leg must be treated as failed, loudly. RETRIED
|
||||
# with raw curl: npmjs metadata propagates with a lag measured in minutes,
|
||||
# and a one-shot npm-view probe fired a false DIVERGENCE on 10.0.0 while a
|
||||
# raw curl of the registry document already confirmed byte-identity. The
|
||||
# probe now reads the registry JSON directly (no npm cache in the path) and
|
||||
# gives propagation up to 5 minutes before calling the pair divergent.
|
||||
NPMJS_VERIFY_ATTEMPTS=20
|
||||
NPMJS_VERIFY_INTERVAL_S=15 # 20 × 15s = 5 minutes of propagation grace
|
||||
SOURCE_SHA=$(npm view "@soulcraft/brainy@${NEW_VERSION}" dist.shasum "--@soulcraft:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "source-unavailable")
|
||||
PAIR_IDENTICAL=false
|
||||
for ((attempt = 1; attempt <= NPMJS_VERIFY_ATTEMPTS; attempt++)); do
|
||||
NPMJS_SHA=$(curl -fsSL "https://registry.npmjs.org/@soulcraft%2Fbrainy" 2>/dev/null \
|
||||
| node -e "let d='';process.stdin.on('data',c=>d+=c).on('end',()=>{try{const v=JSON.parse(d).versions[process.argv[1]];console.log(v?v.dist.shasum:'')}catch{console.log('')}})" "${NEW_VERSION}" \
|
||||
|| echo "")
|
||||
if [ -n "$NPMJS_SHA" ] && [ "$SOURCE_SHA" = "$NPMJS_SHA" ]; then
|
||||
PAIR_IDENTICAL=true
|
||||
break
|
||||
fi
|
||||
echo -e "${YELLOW} … npmjs metadata not settled (attempt ${attempt}/${NPMJS_VERIFY_ATTEMPTS}: '${NPMJS_SHA:-absent}' vs '${SOURCE_SHA}'); retrying in ${NPMJS_VERIFY_INTERVAL_S}s${NC}"
|
||||
sleep "$NPMJS_VERIFY_INTERVAL_S"
|
||||
done
|
||||
if [ "$PAIR_IDENTICAL" = true ]; then
|
||||
echo -e "${GREEN}✅ Published to npmjs — byte-identical pair (shasum ${NPMJS_SHA})${NC}\n"
|
||||
else
|
||||
echo -e "${RED}❌ REGISTRY DIVERGENCE: The Source shasum ${SOURCE_SHA} != npmjs shasum ${NPMJS_SHA} after ${NPMJS_VERIFY_ATTEMPTS} attempts — investigate before announcing${NC}\n"
|
||||
exit 1
|
||||
echo -e "${BLUE}9️⃣½ Publishing to npmjs (storefront, dist-tag: ${NPM_TAG})...${NC}"
|
||||
# BYTE-IDENTITY LAW: the storefront republishes CI's EXACT artifact — download
|
||||
# the tarball The Source serves and publish that file, never a fresh local pack
|
||||
# (a local rebuild can differ byte-wise, and the fleet verifies the pair by
|
||||
# shasum across registries).
|
||||
STOREFRONT_TMP="$(mktemp -d)"
|
||||
(cd "$STOREFRONT_TMP" && npm pack "@soulcraft/brainy@${NEW_VERSION}" "--@soulcraft:registry=${SOURCE_NPM_REG}" >/dev/null)
|
||||
SOURCE_TARBALL="$(ls "$STOREFRONT_TMP"/soulcraft-brainy-*.tgz)"
|
||||
echo -e "${BLUE} home artifact: $(sha256sum "$SOURCE_TARBALL" | cut -d' ' -f1)${NC}"
|
||||
npm publish "$SOURCE_TARBALL" --tag "$NPM_TAG" "--@soulcraft:registry=https://registry.npmjs.org/"
|
||||
rm -rf "$STOREFRONT_TMP"
|
||||
# Brainy is the only PUBLIC @soulcraft package — verify visibility after every publish.
|
||||
npm access get status @soulcraft/brainy "--@soulcraft:registry=https://registry.npmjs.org/" || true
|
||||
# Verify the pair is byte-identical by registry-reported shasum — divergence
|
||||
# here means the storefront leg must be treated as failed, loudly. RETRIED
|
||||
# with raw curl: npmjs metadata propagates with a lag measured in minutes,
|
||||
# and a one-shot npm-view probe fired a false DIVERGENCE on 10.0.0 while a
|
||||
# raw curl of the registry document already confirmed byte-identity. The
|
||||
# probe now reads the registry JSON directly (no npm cache in the path) and
|
||||
# gives propagation up to 5 minutes before calling the pair divergent.
|
||||
NPMJS_VERIFY_ATTEMPTS=20
|
||||
NPMJS_VERIFY_INTERVAL_S=15 # 20 × 15s = 5 minutes of propagation grace
|
||||
SOURCE_SHA=$(npm view "@soulcraft/brainy@${NEW_VERSION}" dist.shasum "--@soulcraft:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "source-unavailable")
|
||||
PAIR_IDENTICAL=false
|
||||
for ((attempt = 1; attempt <= NPMJS_VERIFY_ATTEMPTS; attempt++)); do
|
||||
NPMJS_SHA=$(curl -fsSL "https://registry.npmjs.org/@soulcraft%2Fbrainy" 2>/dev/null \
|
||||
| node -e "let d='';process.stdin.on('data',c=>d+=c).on('end',()=>{try{const v=JSON.parse(d).versions[process.argv[1]];console.log(v?v.dist.shasum:'')}catch{console.log('')}})" "${NEW_VERSION}" \
|
||||
|| echo "")
|
||||
if [ -n "$NPMJS_SHA" ] && [ "$SOURCE_SHA" = "$NPMJS_SHA" ]; then
|
||||
PAIR_IDENTICAL=true
|
||||
break
|
||||
fi
|
||||
echo -e "${YELLOW} … npmjs metadata not settled (attempt ${attempt}/${NPMJS_VERIFY_ATTEMPTS}: '${NPMJS_SHA:-absent}' vs '${SOURCE_SHA}'); retrying in ${NPMJS_VERIFY_INTERVAL_S}s${NC}"
|
||||
sleep "$NPMJS_VERIFY_INTERVAL_S"
|
||||
done
|
||||
if [ "$PAIR_IDENTICAL" = true ]; then
|
||||
echo -e "${GREEN}✅ Published to npmjs — byte-identical pair (shasum ${NPMJS_SHA})${NC}\n"
|
||||
else
|
||||
echo -e "${RED}❌ REGISTRY DIVERGENCE: The Source shasum ${SOURCE_SHA} != npmjs shasum ${NPMJS_SHA} after ${NPMJS_VERIFY_ATTEMPTS} attempts — investigate before announcing${NC}\n"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Step 11: Release object on The Source (presentational — the tag, CHANGELOG,
|
||||
|
|
@ -303,24 +283,16 @@ fi
|
|||
# DOCS_INGEST_SECRET is unset; fails loudly (without undoing the publish —
|
||||
# that already happened) when a push errors, so the docs site never
|
||||
# silently trails npm.
|
||||
if [ "$SOURCE_ONLY" = true ]; then
|
||||
echo -e "${YELLOW}1️⃣2️⃣ Docs push SKIPPED — --source-only (a home-only prerelease publishes no public docs)${NC}\n"
|
||||
echo -e "${BLUE}1️⃣2️⃣ Pushing public docs to soulcraft.com/docs...${NC}"
|
||||
if node scripts/push-docs.js; then
|
||||
echo -e "${GREEN}✅ Docs push step done${NC}\n"
|
||||
else
|
||||
echo -e "${BLUE}1️⃣2️⃣ Pushing public docs to soulcraft.com/docs...${NC}"
|
||||
if node scripts/push-docs.js; then
|
||||
echo -e "${GREEN}✅ Docs push step done${NC}\n"
|
||||
else
|
||||
echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n"
|
||||
fi
|
||||
echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n"
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}"
|
||||
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
echo ""
|
||||
if [ "$SOURCE_ONLY" = true ]; then
|
||||
echo -e "📦 npmjs: ${YELLOW}not published (--source-only)${NC}"
|
||||
else
|
||||
echo -e "📦 npm: ${BLUE}https://www.npmjs.com/package/@soulcraft/brainy/v/${NEW_VERSION}${NC}"
|
||||
fi
|
||||
echo -e "📦 npm: ${BLUE}https://www.npmjs.com/package/@soulcraft/brainy/v/${NEW_VERSION}${NC}"
|
||||
echo -e "🏠 The Source: ${BLUE}https://source.soulcraft.com/soulcraft/brainy/releases/tag/v${NEW_VERSION}${NC}"
|
||||
|
|
|
|||
197
src/brainy.ts
197
src/brainy.ts
|
|
@ -144,9 +144,7 @@ import {
|
|||
ScoreExplanation,
|
||||
FillSubtypeRule,
|
||||
FillSubtypeRules,
|
||||
FillSubtypesResult,
|
||||
RepairReport,
|
||||
RepairFamilyReport
|
||||
FillSubtypesResult
|
||||
} from './types/brainy.types.js'
|
||||
import { NounType, VerbType, TypeUtils } from './types/graphTypes.js'
|
||||
import {
|
||||
|
|
@ -3692,22 +3690,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// stored, so remove() deletes the same entity. A real UUID passes through.
|
||||
id = resolveEntityId(id)
|
||||
|
||||
// Get entity metadata and related verbs before deletion. TORN-TOLERANT:
|
||||
// a torn record must still be deletable (the delete IS the cure) — a
|
||||
// torn pre-read reads as null and the null-path below handles it loudly.
|
||||
let metadata: any = null
|
||||
let noun: any = null
|
||||
try {
|
||||
metadata = await this.storage.getNounMetadata(id)
|
||||
} catch (err) {
|
||||
if ((err as { code?: string }).code !== 'TORN_RECORD') throw err
|
||||
prodLog.warn(`[Brainy] remove(${id}): metadata pre-read is TORN — deleting anyway; index legs run id-keyed`)
|
||||
}
|
||||
try {
|
||||
noun = await this.storage.getNoun(id)
|
||||
} catch (err) {
|
||||
if ((err as { code?: string }).code !== 'TORN_RECORD') throw err
|
||||
}
|
||||
// Get entity metadata and related verbs before deletion
|
||||
const metadata = await this.storage.getNounMetadata(id)
|
||||
const noun = await this.storage.getNoun(id)
|
||||
const verbs = await this.storage.getVerbsBySource(id)
|
||||
const targetVerbs = await this.storage.getVerbsByTarget(id)
|
||||
const allVerbs = [...verbs, ...targetVerbs]
|
||||
|
|
@ -3725,61 +3710,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
)
|
||||
}
|
||||
|
||||
// Operation 2: Remove from metadata index. THE NULL-METADATA SKIP IS
|
||||
// CLOSED (a posting-leak class, confirmed at this site): when the
|
||||
// pre-read missed, the leg no longer silently skips —
|
||||
// - a provider exposing removeEntityById (the id-keyed contract)
|
||||
// gets it: exact per-entity retraction via its reverse record;
|
||||
// - the JS index gets removeFromIndex(id) — safe id-keyed cleanup
|
||||
// (deleted bitmap + id mapper; field stats reconcile at rebuild);
|
||||
// - a NATIVE provider WITHOUT the contract is never called
|
||||
// metadata-omitted (that path walks its value space) — the skip
|
||||
// happens, but NARRATED and tracked in the degraded set so
|
||||
// repairIndex reconciles it. Silence is the only thing outlawed.
|
||||
// Operation 2: Remove from metadata index
|
||||
if (metadata) {
|
||||
tx.addOperation(
|
||||
new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata, this.indexWriteGeneration)
|
||||
)
|
||||
} else {
|
||||
const prov = this.metadataIndex as unknown as {
|
||||
removeEntityById?: (id: string) => Promise<void>
|
||||
removeFromIndex?: (id: string, metadata?: unknown, generation?: bigint) => Promise<void>
|
||||
}
|
||||
if (typeof prov.removeEntityById === 'function') {
|
||||
const g = this.indexWriteGeneration
|
||||
tx.addOperation({
|
||||
name: 'RemoveEntityByIdTombstone',
|
||||
execute: async () => {
|
||||
await prov.removeEntityById!(id)
|
||||
return async () => {
|
||||
// Undo of an id-keyed tombstone on an absent row: nothing
|
||||
// to restore (the row had no readable metadata to re-post).
|
||||
void g
|
||||
}
|
||||
}
|
||||
})
|
||||
} else if (this.metadataIndex instanceof MetadataIndexManager) {
|
||||
const gv = this.indexWriteGeneration
|
||||
tx.addOperation({
|
||||
name: 'IdKeyedIndexCleanup',
|
||||
execute: async () => {
|
||||
await prov.removeFromIndex!(id, undefined, typeof gv === 'function' ? gv() : gv)
|
||||
return async () => {}
|
||||
}
|
||||
})
|
||||
prodLog.warn(
|
||||
`[Brainy] remove(${id}): no metadata at delete — id-keyed index cleanup ran ` +
|
||||
`(deleted bitmap + id mapper); field statistics reconcile at the next rebuild/repairIndex.`
|
||||
)
|
||||
} else {
|
||||
this._indexDegradedIds.add(id)
|
||||
prodLog.warn(
|
||||
`[Brainy] remove(${id}): no metadata at delete and this provider has no id-keyed ` +
|
||||
`removal — its postings for this id are NOT tombstoned yet (tracked as degraded; ` +
|
||||
`repairIndex() reconciles). Never calling a metadata-omitted native removal: that ` +
|
||||
`path walks the store's value space.`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Operation 3: Delete noun (full removal). The pre-read metadata rides
|
||||
|
|
@ -3972,12 +3907,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
uuid: string,
|
||||
options?: { direction?: 'in' | 'out' | 'both'; limit?: number; offset?: number }
|
||||
): Promise<string[]> {
|
||||
// READ-SURFACE READINESS GATE (the 4.2.4 blackout's brainy half): every
|
||||
// index read funnels through this helper, so the gate here makes
|
||||
// serve-while-not-ready UNREPRESENTABLE — a production store once acked
|
||||
// writes while every non-find() read served empty from a not-ready
|
||||
// provider for 15 minutes. Fast path after the latch is one boolean.
|
||||
await this.ensureIndexesLoaded()
|
||||
const entityInt = this.graphEntityInt(uuid)
|
||||
if (entityInt === undefined) return []
|
||||
const neighborInts = await this.graphIndex.getNeighbors(entityInt, options)
|
||||
|
|
@ -11878,12 +11807,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
filter: unknown,
|
||||
opts?: { limit?: number; offset?: number }
|
||||
): Promise<string[]> {
|
||||
// READ-SURFACE READINESS GATE (the 4.2.4 blackout's brainy half): every
|
||||
// index read funnels through this helper, so the gate here makes
|
||||
// serve-while-not-ready UNREPRESENTABLE — a production store once acked
|
||||
// writes while every non-find() read served empty from a not-ready
|
||||
// provider for 15 minutes. Fast path after the latch is one boolean.
|
||||
await this.ensureIndexesLoaded()
|
||||
try {
|
||||
return await this.metadataIndex.getIdsForFilter(filter, opts)
|
||||
} catch (err) {
|
||||
|
|
@ -14221,29 +14144,16 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
const report = await fn.call(provider)
|
||||
if (report && Array.isArray(report.invariants)) reports.push(report)
|
||||
} catch (err) {
|
||||
// ONE CONTRACT FOR A THROWING PROBE, both engines: a probe that throws
|
||||
// is `heal: 'none'` with the error in `detail` — flakiness can never
|
||||
// buy a rebuild, and a thrown check never changes `serving` (the
|
||||
// provider's serving verdict is composed by the provider, not inferred
|
||||
// from a probe that failed to run). This catch used to synthesize
|
||||
// `heal: 'rebuild'` — the read-triggered dark-rebuild lever one
|
||||
// transient exception away — while the native composer said 'none';
|
||||
// two components disagreeing on what a throw means is how a flaky
|
||||
// probe became an outage. `healthy: false` stays: an unrunnable probe
|
||||
// is a named, loud, unverified state, never a clean bill.
|
||||
const name = typeof (provider as { name?: string })?.name === 'string'
|
||||
? (provider as { name: string }).name
|
||||
: 'unknown'
|
||||
reports.push({
|
||||
provider: name,
|
||||
provider: 'unknown',
|
||||
healthy: false,
|
||||
serving: true,
|
||||
serving: false,
|
||||
invariants: [
|
||||
{
|
||||
name: 'validate-invariants-threw',
|
||||
holds: false,
|
||||
detail: `validateInvariants() threw (contract violation — it must never throw): ${(err as Error).message}`,
|
||||
heal: 'none'
|
||||
heal: 'rebuild'
|
||||
}
|
||||
],
|
||||
checkedAt: Date.now(),
|
||||
|
|
@ -14369,12 +14279,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
verbTypes?: Set<VerbType>,
|
||||
limit?: number
|
||||
): Promise<string[]> {
|
||||
// READ-SURFACE READINESS GATE (the 4.2.4 blackout's brainy half): every
|
||||
// index read funnels through this helper, so the gate here makes
|
||||
// serve-while-not-ready UNREPRESENTABLE — a production store once acked
|
||||
// writes while every non-find() read served empty from a not-ready
|
||||
// provider for 15 minutes. Fast path after the latch is one boolean.
|
||||
await this.ensureIndexesLoaded()
|
||||
// 8.0 BigInt boundary: unmapped node → no relations.
|
||||
const nodeInt = this.graphEntityInt(nodeId)
|
||||
if (nodeInt === undefined) return []
|
||||
|
|
@ -16313,15 +16217,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
return
|
||||
}
|
||||
|
||||
// Start lazy rebuild (with mutex to prevent concurrent rebuilds).
|
||||
// ALWAYS narrated (prodLog, never the silent-suppressible console): a
|
||||
// read that triggers an index build must be visible to the operator —
|
||||
// fifteen silent minutes of a production blackout taught this line.
|
||||
prodLog.warn(
|
||||
`[Brainy] first read on this instance is building the derived indexes ` +
|
||||
`(deferred at open by disableAutoRebuild) — reads WAIT and then serve; ` +
|
||||
`nothing serves empty. Bounded by store size; progress under [MetadataIndex]/[GraphIndex].`
|
||||
)
|
||||
// Start lazy rebuild (with mutex to prevent concurrent rebuilds)
|
||||
this.lazyRebuildInProgress = true
|
||||
this.lazyRebuildPromise = this.rebuildIndexesIfNeeded(true)
|
||||
.then(() => {
|
||||
|
|
@ -17166,13 +17062,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
)
|
||||
}
|
||||
|
||||
async repairIndex(): Promise<RepairReport> {
|
||||
async repairIndex(): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
const startedAt = Date.now()
|
||||
const families: RepairFamilyReport[] = []
|
||||
const record = (family: string, entry: Omit<RepairFamilyReport, 'family'>): void => {
|
||||
families.push({ family, ...entry })
|
||||
}
|
||||
|
||||
// Prune orphaned canonical containers left by the pre-8.3.1 partial-delete
|
||||
// defect: a delete that removed the metadata (content) leg but left the
|
||||
|
|
@ -17188,23 +17079,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
if (typeof pruner.pruneOrphanedEntities === 'function') {
|
||||
const orphans = await pruner.pruneOrphanedEntities()
|
||||
const pruned = orphans.nouns.length + orphans.verbs.length
|
||||
record('orphaned-containers', {
|
||||
checked: true,
|
||||
healed: pruned,
|
||||
...(pruned > 0
|
||||
? { detail: `${orphans.nouns.length} noun + ${orphans.verbs.length} verb container(s) pruned` }
|
||||
: {})
|
||||
})
|
||||
if (pruned > 0) {
|
||||
if (orphans.nouns.length + orphans.verbs.length > 0) {
|
||||
prodLog.warn(
|
||||
`[Brainy] repairIndex() pruned ${orphans.nouns.length} orphaned noun + ` +
|
||||
`${orphans.verbs.length} orphaned verb container(s) left by a pre-8.3.1 ` +
|
||||
`partial delete.`
|
||||
)
|
||||
}
|
||||
} else {
|
||||
record('orphaned-containers', { checked: false, healed: 0, skipped: 'storage has no container model' })
|
||||
}
|
||||
// SANCTIONED RECOUNT — unconditional, not gated on orphans found: the
|
||||
// persisted counters can be inflated over perfectly clean shelves (deletes
|
||||
|
|
@ -17215,14 +17096,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// canonical walk and persists them.
|
||||
await pruner.rebuildTypeCounts?.()
|
||||
await pruner.rebuildSubtypeCounts?.()
|
||||
record('count-rollups', {
|
||||
checked: typeof pruner.rebuildTypeCounts === 'function',
|
||||
healed: 0,
|
||||
detail: typeof pruner.rebuildTypeCounts === 'function'
|
||||
? 'recomputed from one canonical walk (unconditional)'
|
||||
: undefined,
|
||||
...(typeof pruner.rebuildTypeCounts !== 'function' ? { skipped: 'storage has no count rollups' } : {})
|
||||
})
|
||||
|
||||
// The recount changed the rollup truth — re-stamp the entity tree so the
|
||||
// stamp's invariants match the healed counters (repair leaves a coherent
|
||||
|
|
@ -17236,13 +17109,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// containment edges are touched. Loud per repair.
|
||||
if (this._vfsInitialized && this._vfs) {
|
||||
const containment = await this._vfs.repairContainment()
|
||||
record('vfs-containment', {
|
||||
checked: true,
|
||||
healed: containment.removed + containment.restored,
|
||||
...(containment.removed + containment.restored > 0
|
||||
? { detail: `${containment.removed} stale edge(s) removed, ${containment.restored} restored` }
|
||||
: {})
|
||||
})
|
||||
if (containment.removed + containment.restored > 0) {
|
||||
prodLog.warn(
|
||||
`[Brainy] repairIndex() reconciled VFS containment: removed ${containment.removed} ` +
|
||||
|
|
@ -17251,19 +17117,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
}
|
||||
|
||||
if (!this._vfsInitialized || !this._vfs) {
|
||||
record('vfs-containment', { checked: false, healed: 0, skipped: 'VFS not initialized' })
|
||||
}
|
||||
|
||||
await this.metadataIndex.detectAndRepairCorruption()
|
||||
record('metadata-corruption', { checked: true, healed: 0, detail: 'detect-and-repair pass ran (see its own narration for repairs)' })
|
||||
// Lift a failed-rollback write-quarantine: force a full rebuild so the
|
||||
// derived indexes are provably reconciled with canonical, then clear the
|
||||
// flag so writes resume.
|
||||
if (this.storeInconsistency) {
|
||||
await this.rebuildIndexesIfNeeded(true)
|
||||
const cleared = this.storeInconsistency
|
||||
record('write-quarantine', { checked: true, healed: 1, detail: `lifted (${cleared.records.length} record(s) reconciled)` })
|
||||
this.storeInconsistency = null
|
||||
prodLog.warn(
|
||||
`[Brainy] repairIndex() reconciled the store and LIFTED the write-quarantine ` +
|
||||
|
|
@ -17280,38 +17140,20 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
validateInvariants?: () => Promise<ProviderInvariantReport>
|
||||
rebuild?: () => Promise<void>
|
||||
} | null
|
||||
if (!p || typeof p.validateInvariants !== 'function' || typeof p.rebuild !== 'function') {
|
||||
record(`provider:${(provider as { constructor?: { name?: string } })?.constructor?.name ?? 'unknown'}`, {
|
||||
checked: false, healed: 0, skipped: 'no validateInvariants/rebuild contract'
|
||||
})
|
||||
continue
|
||||
}
|
||||
if (!p || typeof p.validateInvariants !== 'function' || typeof p.rebuild !== 'function') continue
|
||||
let report: ProviderInvariantReport
|
||||
try {
|
||||
report = await p.validateInvariants()
|
||||
} catch (err) {
|
||||
record(`provider:unknown`, { checked: false, healed: 0, skipped: `validateInvariants threw: ${(err as Error).message}` })
|
||||
} catch {
|
||||
continue // a throwing validateInvariants is surfaced by validateIndexConsistency; skip repair here
|
||||
}
|
||||
if (report.healthy) {
|
||||
record(`provider:${report.provider}`, { checked: true, healed: 0 })
|
||||
continue
|
||||
}
|
||||
if (report.healthy) continue
|
||||
if (report.invariants.some((i) => !i.holds && i.heal === 'rebuild')) {
|
||||
record(`provider:${report.provider}`, {
|
||||
checked: true, healed: 1,
|
||||
detail: `rebuilt from canonical (failing: ${report.invariants.filter((i) => !i.holds).map((i) => i.name).join(', ')})`
|
||||
})
|
||||
prodLog.warn(
|
||||
`[Brainy] repairIndex(): provider '${report.provider}' has a failing invariant ` +
|
||||
`requiring a rebuild — reconciling its derived state from canonical.`
|
||||
)
|
||||
await p.rebuild()
|
||||
} else {
|
||||
record(`provider:${report.provider}`, {
|
||||
checked: true, healed: 0,
|
||||
detail: `unhealthy without a rebuild verdict (failing: ${report.invariants.filter((i) => !i.holds).map((i) => `${i.name}→${i.heal}`).join(', ')})`
|
||||
})
|
||||
}
|
||||
}
|
||||
// detectAndRepairCorruption() above rebuilt the derived indexes from
|
||||
|
|
@ -17322,18 +17164,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
this._indexDegradedIds.clear()
|
||||
this._indexRebuildFailed = null
|
||||
this._degradedReadWarned = false
|
||||
record('degraded-read-state', { checked: true, healed: 1, detail: 'degraded ids cleared, read-path warning re-armed' })
|
||||
}
|
||||
|
||||
const healedTotal = families.reduce((n, f) => n + f.healed, 0)
|
||||
const report: RepairReport = { families, healedTotal, durationMs: Date.now() - startedAt }
|
||||
prodLog.warn(
|
||||
`[Brainy] repairIndex complete in ${report.durationMs}ms — ` +
|
||||
`${families.filter((f) => f.checked).length}/${families.length} families checked, ` +
|
||||
`${healedTotal} heal(s): ` +
|
||||
families.map((f) => `${f.family}=${f.checked ? f.healed : 'skipped'}`).join(', ')
|
||||
)
|
||||
return report
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -792,19 +792,6 @@ export interface DerivedFamilyDeclaration {
|
|||
rebuildable?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* @description The canonical count ledger a storage adapter maintains on its
|
||||
* write path: per family, the user-facing `counted` scalar and the
|
||||
* ALL-visibility `all` scalar (every tier — the coverage-ledger denominator).
|
||||
* See {@link StorageAdapter.getCanonicalCounts}.
|
||||
*/
|
||||
export interface CanonicalCounts {
|
||||
nouns: { counted: number; all: number }
|
||||
verbs: { counted: number; all: number }
|
||||
/** An unprovable delete has left the `all` scalars unverified since the last recount. */
|
||||
suspect: boolean
|
||||
}
|
||||
|
||||
export interface StorageAdapter {
|
||||
init(): Promise<void>
|
||||
|
||||
|
|
@ -1306,19 +1293,6 @@ export interface StorageAdapter {
|
|||
*/
|
||||
getVerbCount(): Promise<number>
|
||||
|
||||
/**
|
||||
* The canonical count ledger — O(1), no I/O. `counted` mirrors
|
||||
* `getNounCount()` / `getVerbCount()` (public + internal tiers); `all` is
|
||||
* the ALL-visibility scalar every unfiltered storage walk is measured
|
||||
* against — the denominator a derived-index provider's coverage ledger
|
||||
* subtracts from. `suspect` is `true` when an unprovable delete has left
|
||||
* `all` unverified since the last sanctioned recount (`repairIndex()`).
|
||||
* Optional: adapters without the ledger omit it; a consumer treats absence
|
||||
* as "no denominator", never as zero.
|
||||
* @returns Both scalars per family plus the suspect flag.
|
||||
*/
|
||||
getCanonicalCounts?(): Promise<CanonicalCounts>
|
||||
|
||||
/**
|
||||
* OPTIONAL — create a pre-upgrade backup of the whole store and return its
|
||||
* location, or `null` when there is nothing to back up (empty store). On the
|
||||
|
|
|
|||
|
|
@ -80,9 +80,7 @@ export type {
|
|||
AggregationOp,
|
||||
TimeWindowGranularity,
|
||||
GroupByDimension,
|
||||
AggregationProvider,
|
||||
RepairReport,
|
||||
RepairFamilyReport,
|
||||
AggregationProvider
|
||||
} from './types/brainy.types.js'
|
||||
|
||||
// Read-barrier contract (waitForIndexed): the leg names, the options, and
|
||||
|
|
|
|||
|
|
@ -12,8 +12,7 @@ import {
|
|||
HNSWNounWithMetadata,
|
||||
HNSWVerbWithMetadata,
|
||||
NounMetadata,
|
||||
VerbMetadata,
|
||||
CanonicalCounts,
|
||||
VerbMetadata
|
||||
} from '../../coreTypes.js'
|
||||
import { StorageBatchConfig } from '../baseStorage.js'
|
||||
import { extractFieldNamesFromJson, mapToStandardField } from '../../utils/fieldNameTracking.js'
|
||||
|
|
@ -1029,28 +1028,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
// Universal count tracking - O(1) operations
|
||||
protected totalNounCount = 0
|
||||
protected totalVerbCount = 0
|
||||
/**
|
||||
* The ALL-visibility canonical scalars — every noun / verb the unfiltered
|
||||
* storage walk yields, system and internal tiers included. These are the
|
||||
* denominators a derived-index provider's coverage ledger subtracts from
|
||||
* (`posted === all` is the whole-store coverage verdict); the user-facing
|
||||
* `totalNounCount` / `totalVerbCount` skip hidden tiers by design and can
|
||||
* never serve as a ledger denominator. Maintained on the write path
|
||||
* (every new record +1, every proven delete −1), persisted beside the
|
||||
* counted scalars, recomputed by the sanctioned recount. Never clamped.
|
||||
*/
|
||||
protected totalNounCountAll = 0
|
||||
protected totalVerbCountAll = 0
|
||||
/**
|
||||
* `true` when a delete could not prove whether the record existed (no
|
||||
* canonical read, no caller-provided prior) — the ALL scalar may be off by
|
||||
* the unprovable deletes since. Loud, persisted, and cleared only by the
|
||||
* sanctioned recount; a consumer reading the scalar as a ledger denominator
|
||||
* must treat a suspect scalar as unverified, never as exact.
|
||||
*/
|
||||
protected allCountsSuspect = false
|
||||
/** One narration per session for the suspect transition (never per delete). */
|
||||
private allCountsSuspectNarrated = false
|
||||
protected entityCounts: Map<string, number> = new Map() // type -> count
|
||||
protected verbCounts: Map<string, number> = new Map() // verb type -> count
|
||||
protected countCache: Map<string, { count: number; timestamp: number }> = new Map()
|
||||
|
|
@ -1079,43 +1056,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
return this.totalVerbCount
|
||||
}
|
||||
|
||||
/**
|
||||
* The canonical count ledger — O(1), no I/O. `counted` is the user-facing
|
||||
* scalar (public/internal tiers, what `getNounCount()` returns); `all` is
|
||||
* the ALL-visibility scalar every unfiltered storage walk is measured
|
||||
* against (the coverage-ledger denominator for derived-index providers).
|
||||
* `suspect` is `true` when an unprovable delete has made `all` unverified
|
||||
* since the last sanctioned recount (`rebuildTypeCounts`).
|
||||
* @returns Both scalars per family plus the suspect flag.
|
||||
*/
|
||||
async getCanonicalCounts(): Promise<CanonicalCounts> {
|
||||
return {
|
||||
nouns: { counted: this.totalNounCount, all: this.totalNounCountAll },
|
||||
verbs: { counted: this.totalVerbCount, all: this.totalVerbCountAll },
|
||||
suspect: this.allCountsSuspect
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark the ALL scalars unverified after a delete that could not prove the
|
||||
* record existed. Narrates ONCE per session (the flag is what persists);
|
||||
* the sanctioned recount clears it.
|
||||
* @param family - Which family's delete was unprovable.
|
||||
* @param id - The id whose existence could not be established.
|
||||
*/
|
||||
protected markAllCountsSuspect(family: 'noun' | 'verb', id: string): void {
|
||||
this.allCountsSuspect = true
|
||||
if (!this.allCountsSuspectNarrated) {
|
||||
this.allCountsSuspectNarrated = true
|
||||
console.warn(
|
||||
`[Storage] ${family} delete of ${id} could not prove the record existed ` +
|
||||
`(no canonical read, no prior record) — the ALL-visibility count ledger is ` +
|
||||
`SUSPECT until brain.repairIndex() recounts. Further unprovable deletes ` +
|
||||
`this session are counted silently under the same flag.`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Increment count for entity type - O(1) operation.
|
||||
* Concurrency is handled by the process-global mutex
|
||||
|
|
|
|||
|
|
@ -2561,33 +2561,6 @@ export class FileSystemStorage extends BaseStorage {
|
|||
this.totalNounCount = counts.totalNounCount || 0
|
||||
this.totalVerbCount = counts.totalVerbCount || 0
|
||||
|
||||
// The ALL-visibility scalars (ledger denominators). A counts.json
|
||||
// written before they existed carries neither key: derive both ONCE
|
||||
// from the canonical id tree (an id-directory listing — O(ids), no
|
||||
// record reads), persist, and never scan again. Absent keys are a
|
||||
// legacy file, not a zero — a zero here would make every provider's
|
||||
// coverage ledger read "over-posted" on a populated store.
|
||||
if (
|
||||
typeof counts.totalNounCountAll === 'number' &&
|
||||
typeof counts.totalVerbCountAll === 'number'
|
||||
) {
|
||||
this.totalNounCountAll = counts.totalNounCountAll
|
||||
this.totalVerbCountAll = counts.totalVerbCountAll
|
||||
this.allCountsSuspect = counts.allCountsSuspect === true
|
||||
} else {
|
||||
const nouns = await this.scanCanonicalEntities('nouns')
|
||||
const verbs = await this.scanCanonicalEntities('verbs')
|
||||
this.totalNounCountAll = nouns.count
|
||||
this.totalVerbCountAll = verbs.count
|
||||
this.allCountsSuspect = false
|
||||
console.warn(
|
||||
`[FileSystemStorage] counts.json predates the ALL-visibility count ledger — ` +
|
||||
`derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` +
|
||||
`every tier) and persisted; no further scan.`
|
||||
)
|
||||
await this.persistCounts()
|
||||
}
|
||||
|
||||
// Also populate the cache for backward compatibility
|
||||
this.countCache.set('nouns_count', {
|
||||
count: this.totalNounCount,
|
||||
|
|
@ -2623,10 +2596,6 @@ export class FileSystemStorage extends BaseStorage {
|
|||
this.totalNounCount = nouns.count
|
||||
const verbs = await this.scanCanonicalEntities('verbs')
|
||||
this.totalVerbCount = verbs.count
|
||||
// The id-tree scan counts every tier — it IS the ALL-visibility ledger.
|
||||
this.totalNounCountAll = nouns.count
|
||||
this.totalVerbCountAll = verbs.count
|
||||
this.allCountsSuspect = false
|
||||
|
||||
// Sample some entities for the type distribution (don't read all).
|
||||
// Read the metadata files DIRECTLY with fs — this runs inside init(),
|
||||
|
|
@ -2724,11 +2693,6 @@ export class FileSystemStorage extends BaseStorage {
|
|||
verbCounts: Object.fromEntries(this.verbCounts),
|
||||
totalNounCount: this.totalNounCount,
|
||||
totalVerbCount: this.totalVerbCount,
|
||||
// ALL-visibility ledger scalars (+ the suspect flag) — absent in files
|
||||
// written before the ledger existed; initializeCounts() derives them once.
|
||||
totalNounCountAll: this.totalNounCountAll,
|
||||
totalVerbCountAll: this.totalVerbCountAll,
|
||||
allCountsSuspect: this.allCountsSuspect,
|
||||
lastUpdated: new Date().toISOString()
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -540,10 +540,6 @@ export class MemoryStorage extends BaseStorage {
|
|||
|
||||
this.totalNounCount = totalNouns
|
||||
this.totalVerbCount = totalVerbs
|
||||
// A scan of every canonical record IS the ALL-visibility count.
|
||||
this.totalNounCountAll = totalNouns
|
||||
this.totalVerbCountAll = totalVerbs
|
||||
this.allCountsSuspect = false
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -2273,14 +2273,11 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
|
||||
// totalCount must be the TRUE dataset total, not this peeked page. For the
|
||||
// unfiltered case the authoritative total is the O(1) counter maintained on
|
||||
// every add/delete (rehydrated on init) — the ALL-visibility scalar, because
|
||||
// this walk is unfiltered by tier (system/internal records are in `collected`).
|
||||
// Never clamped: `Math.max(scalar, scanned)` could only ever move the scalar
|
||||
// UP, so an inflated counter could never correct itself and a divergence was
|
||||
// hidden instead of reported. A scalar that disagrees with the walk is the
|
||||
// canonical-count-ledger invariant's job, healed by the sanctioned recount.
|
||||
// A filtered scan has no cheap exact total, so it keeps the collected length.
|
||||
const totalCount = filter ? collected.length : this.totalNounCountAll
|
||||
// every add/delete (rehydrated on init); `Math.max` guards a stale counter. A
|
||||
// filtered scan has no cheap exact total, so it keeps the collected length.
|
||||
const totalCount = filter
|
||||
? collected.length
|
||||
: Math.max(this.totalNounCount, collected.length)
|
||||
|
||||
// nextCursor = the (shard, id) of the last RETURNED noun, so the next call
|
||||
// resumes immediately after it (works for both cursor and offset callers).
|
||||
|
|
@ -2412,8 +2409,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
const pagePairs = collected.slice(windowStart, windowStart + limit)
|
||||
const ids = pagePairs.map((p) => p.id)
|
||||
const hasMore = collected.length > windowStart + limit
|
||||
// ALL-visibility scalar, unclamped — same law as getNouns() above.
|
||||
const totalCount = filter ? collected.length : this.totalNounCountAll
|
||||
const totalCount = filter ? collected.length : Math.max(this.totalNounCount, collected.length)
|
||||
|
||||
let nextCursor: string | undefined = undefined
|
||||
if (hasMore && pagePairs.length > 0) {
|
||||
|
|
@ -2645,13 +2641,13 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
const hasMore = collected.length > windowStart + limit
|
||||
|
||||
// totalCount must be the TRUE dataset total, not this peeked page. For the
|
||||
// unfiltered scan the authoritative total is the O(1) ALL-visibility counter
|
||||
// (`totalVerbCountAll`: isNew-gated, EVERY tier, rehydrated on init) — the walk
|
||||
// itself is unfiltered by tier, so the user-facing `totalVerbCount` (which skips
|
||||
// system/internal edges) would undercount it on every store with a VFS. Never
|
||||
// clamped (see getNouns): a divergence is reported, not hidden. A filtered scan
|
||||
// has no cheap exact total, so it keeps the collected length (a lower bound).
|
||||
const totalCount = filter ? collected.length : this.totalVerbCountAll
|
||||
// unfiltered scan the authoritative total is the O(1) `totalVerbCount` counter
|
||||
// (isNew-gated, visibility-filtered, rehydrated on init); `Math.max` guards a
|
||||
// stale counter from under-reporting. A filtered scan has no cheap exact total,
|
||||
// so it keeps the collected length (a lower bound).
|
||||
const totalCount = filter
|
||||
? collected.length
|
||||
: Math.max(this.totalVerbCount, collected.length)
|
||||
|
||||
// nextCursor encodes the (shard, id) of the LAST RETURNED verb so the next call
|
||||
// resumes immediately after it — for both cursor and offset callers (an offset
|
||||
|
|
@ -3459,19 +3455,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
const wasCounted = isNew ? false : isCountedVisibility(existingMetadata?.visibility)
|
||||
const isCounted = isCountedVisibility(newVisibility)
|
||||
|
||||
// ALL-visibility ledger: every NEW canonical record is +1 regardless of tier
|
||||
// (the unfiltered walk yields it, so the denominator must count it). The
|
||||
// counted branch below persists for public/internal records; a hidden new
|
||||
// record persists here so the ALL scalar never lags the tree.
|
||||
if (isNew) {
|
||||
this.totalNounCountAll++
|
||||
if (!(metadata.noun && isCounted)) {
|
||||
this.scheduleCountPersist().catch(() => {
|
||||
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// CRITICAL FIX: Increment count for new entities
|
||||
// This runs AFTER metadata is saved, guaranteeing type information is available
|
||||
// Uses synchronous increment since storage operations are already serialized
|
||||
|
|
@ -3875,20 +3858,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
await this.deleteCanonicalObject(path)
|
||||
const record = read ?? priorRecord
|
||||
|
||||
// ALL-visibility ledger: a PROVEN delete (the record was read, or the caller
|
||||
// carried its prior image) is −1 regardless of tier. A delete that can prove
|
||||
// nothing never guesses — it marks the ledger suspect (loud, persisted) and the
|
||||
// sanctioned recount restores exactness.
|
||||
if (record) {
|
||||
if (this.totalNounCountAll > 0) this.totalNounCountAll--
|
||||
else this.markAllCountsSuspect('noun', id)
|
||||
} else {
|
||||
this.markAllCountsSuspect('noun', id)
|
||||
}
|
||||
this.scheduleCountPersist().catch(() => {
|
||||
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
|
||||
})
|
||||
|
||||
const priorType = record?.noun as NounType | undefined
|
||||
// 8.0 visibility: an internal/system entity was never added to `nounCountsByType`
|
||||
// (gated in `saveNounMetadata_internal()`), so it must not be decremented here either.
|
||||
|
|
@ -4022,9 +3991,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
// Fixes Bug #2: Count synchronization failure during relate() and import()
|
||||
// 8.0: skip the user-facing total for internal/system edges (counts.json + getVerbCount()).
|
||||
if (isNew) {
|
||||
// ALL-visibility ledger: every new edge is +1 regardless of tier (the
|
||||
// unfiltered walk yields VFS/system edges too; the denominator must count them).
|
||||
this.totalVerbCountAll++
|
||||
if (isCounted) {
|
||||
this.incrementVerbCount(verbType)
|
||||
} else {
|
||||
|
|
@ -4086,18 +4052,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
await this.deleteCanonicalObject(path)
|
||||
const record = read ?? priorRecord
|
||||
|
||||
// ALL-visibility ledger: proven delete −1 regardless of tier; an unprovable
|
||||
// delete marks the ledger suspect instead of guessing (see deleteNounMetadata).
|
||||
if (record) {
|
||||
if (this.totalVerbCountAll > 0) this.totalVerbCountAll--
|
||||
else this.markAllCountsSuspect('verb', id)
|
||||
} else {
|
||||
this.markAllCountsSuspect('verb', id)
|
||||
}
|
||||
this.scheduleCountPersist().catch(() => {
|
||||
// Ignore persist errors — in-memory count is authoritative; a later op retries.
|
||||
})
|
||||
|
||||
const priorVerb = record?.verb as VerbType | undefined
|
||||
// Symmetric count decrement (previously OMITTED — verb deletes touched neither the
|
||||
// scalar total nor the per-type bucket, so both inflated permanently). A COUNTED
|
||||
|
|
@ -4543,12 +4497,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
// walk, every counter rollup rebuilt and persisted from it.
|
||||
const countedNouns = new Map<string, number>()
|
||||
const countedVerbs = new Map<string, number>()
|
||||
// ALL-visibility scalars: one per canonical record the walk yields, every
|
||||
// tier, readable or not — the same population the unfiltered getNouns()/
|
||||
// getVerbs() walks enumerate, so `totalCount` and this recount agree by
|
||||
// construction.
|
||||
let allNouns = 0
|
||||
let allVerbs = 0
|
||||
|
||||
// Scan noun shards
|
||||
for (let shard = 0; shard < 256; shard++) {
|
||||
|
|
@ -4560,7 +4508,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
|
||||
for (const path of paths) {
|
||||
if (!path.includes('/metadata.json')) continue
|
||||
allNouns++
|
||||
|
||||
try {
|
||||
const metadata = await this.readCanonicalObject(path)
|
||||
|
|
@ -4593,7 +4540,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
|
||||
for (const path of paths) {
|
||||
if (!path.includes('/metadata.json')) continue
|
||||
allVerbs++
|
||||
|
||||
try {
|
||||
const metadata = await this.readCanonicalObject(path)
|
||||
|
|
@ -4630,24 +4576,10 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
this.verbCounts = countedVerbs
|
||||
this.totalNounCount = totalNouns
|
||||
this.totalVerbCount = totalVerbs
|
||||
// The ALL scalars are exact again and the suspect flag clears — this walk
|
||||
// IS the proof an unprovable delete could not give.
|
||||
const nounsAllBefore = this.totalNounCountAll
|
||||
const verbsAllBefore = this.totalVerbCountAll
|
||||
this.totalNounCountAll = allNouns
|
||||
this.totalVerbCountAll = allVerbs
|
||||
this.allCountsSuspect = false
|
||||
this.countCache.clear()
|
||||
await this.persistCounts()
|
||||
|
||||
prodLog.info(
|
||||
`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs (user-facing); ` +
|
||||
`ALL-visibility ledger ${allNouns} nouns / ${allVerbs} verbs` +
|
||||
(nounsAllBefore !== allNouns || verbsAllBefore !== allVerbs
|
||||
? ` (corrected from ${nounsAllBefore} / ${verbsAllBefore})`
|
||||
: ' (unchanged)') +
|
||||
` — scalar + per-type persisted`
|
||||
)
|
||||
prodLog.info(`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs (scalar + per-type persisted)`)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -1192,40 +1192,6 @@ export interface RelateManyParams<T = any> {
|
|||
/**
|
||||
* Batch result
|
||||
*/
|
||||
/**
|
||||
* One family's row in a {@link RepairReport} — what repairIndex() checked,
|
||||
* what it healed, and why anything was skipped. The receipts venue's graph
|
||||
* trust program asked for: a repair that cannot show its work is a repair
|
||||
* nobody can trust.
|
||||
*/
|
||||
export interface RepairFamilyReport {
|
||||
family: string
|
||||
/** The family was actually examined (false = skipped; see `skipped`/`reason`). */
|
||||
checked: boolean
|
||||
/** Items re-posted / corrected in place — the incremental heal count. */
|
||||
healed: number
|
||||
/**
|
||||
* What the check found missing or divergent, when it can name it: an exact
|
||||
* count plus a capped sample of ids (never the whole list — a report is a
|
||||
* verdict, not a dump). Absent when the family has nothing to name.
|
||||
*/
|
||||
missing?: { count: number; sample: string[] }
|
||||
/** A full generational rebuild of this family ran (as opposed to an incremental heal). */
|
||||
rebuilt?: boolean
|
||||
detail?: string
|
||||
/** Why the family was not checked (`checked: false`). */
|
||||
skipped?: string
|
||||
/** Why the outcome is what it is when neither `detail` nor `skipped` says it. */
|
||||
reason?: string
|
||||
}
|
||||
|
||||
/** The full receipt returned by repairIndex(). */
|
||||
export interface RepairReport {
|
||||
families: RepairFamilyReport[]
|
||||
healedTotal: number
|
||||
durationMs: number
|
||||
}
|
||||
|
||||
export interface BatchResult<T = any> {
|
||||
successful: T[] // Successfully processed items
|
||||
failed: Array<{ // Failed items with errors
|
||||
|
|
|
|||
|
|
@ -1,182 +0,0 @@
|
|||
/**
|
||||
* @module tests/integration/canonical-count-ledger
|
||||
* @description The canonical count ledger — the denominators a derived-index
|
||||
* provider's coverage ledger subtracts from. Laws under test:
|
||||
* (1) THE ALL-VISIBILITY SCALAR IS THE UNFILTERED WALK'S TOTAL — the
|
||||
* storage-level `getNouns()` / `getVerbs()` `totalCount` counts EVERY tier
|
||||
* (system, internal, public) because the walk yields every tier; the
|
||||
* user-facing `getNounCount()` / `getVerbCount()` keep skipping hidden
|
||||
* tiers. A ledger built on the user-facing scalar would read "over-posted"
|
||||
* on every store with a VFS — the mismatch this pin makes unbuildable.
|
||||
* (2) NEVER CLAMPED — `Math.max(scalar, scanned)` could only move a scalar up,
|
||||
* so an inflated counter hid forever. An inflated scalar is now VISIBLE
|
||||
* (totalCount ≠ walk) and the sanctioned recount heals it, durably.
|
||||
* (3) NEVER GUESSED — a delete that cannot prove the record existed marks the
|
||||
* ledger SUSPECT (persisted) instead of decrementing on faith; the recount
|
||||
* clears the flag with proof.
|
||||
* (4) LEGACY FILES DERIVE ONCE — a counts.json written before the ledger is
|
||||
* upgraded from the canonical id tree at open, then persisted.
|
||||
*/
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import * as fs from 'node:fs'
|
||||
import * as os from 'node:os'
|
||||
import * as path from 'node:path'
|
||||
import { Brainy } from '../../src/index.js'
|
||||
|
||||
/** Count canonical `<root>/entities/<kind>/<shard>/<id>` directories — every tier. */
|
||||
function countIdDirs(root: string, kind: 'nouns' | 'verbs'): number {
|
||||
const base = path.join(root, 'entities', kind)
|
||||
if (!fs.existsSync(base)) return 0
|
||||
let n = 0
|
||||
for (const shard of fs.readdirSync(base)) {
|
||||
const shardDir = path.join(base, shard)
|
||||
if (!fs.statSync(shardDir).isDirectory()) continue
|
||||
for (const id of fs.readdirSync(shardDir)) {
|
||||
if (fs.statSync(path.join(shardDir, id)).isDirectory()) n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
const countsPath = (root: string) => path.join(root, '_system', 'counts.json')
|
||||
|
||||
describe('canonical count ledger — ALL-visibility scalars, unclamped totals, recount heals', () => {
|
||||
let dir: string
|
||||
let brain: any
|
||||
|
||||
const open = async () => {
|
||||
const b: any = new Brainy({
|
||||
requireSubtype: false,
|
||||
storage: { type: 'filesystem', path: dir },
|
||||
silent: true,
|
||||
dimensions: 384
|
||||
})
|
||||
await b.init()
|
||||
return b
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
||||
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-ledger-'))
|
||||
brain = await open()
|
||||
})
|
||||
afterEach(async () => {
|
||||
await brain.close?.().catch(() => {})
|
||||
fs.rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('the unfiltered walk totalCount is the ALL scalar (every tier); the user-facing count stays counted', async () => {
|
||||
const a = await brain.add({ data: 'public a', type: 'document' })
|
||||
const b = await brain.add({ data: 'internal b', type: 'document', visibility: 'internal' })
|
||||
await brain.relate({ from: a, to: b, type: 'relatedTo', visibility: 'internal' })
|
||||
await brain.vfs.writeFile('/docs/x.txt', 'hello') // VFS: system-tier nouns + Contains edges
|
||||
await brain.flush()
|
||||
|
||||
const ledger = await brain.storage.getCanonicalCounts()
|
||||
expect(ledger.suspect).toBe(false)
|
||||
expect(ledger.nouns.all).toBe(countIdDirs(dir, 'nouns'))
|
||||
expect(ledger.verbs.all).toBe(countIdDirs(dir, 'verbs'))
|
||||
expect(ledger.nouns.counted).toBe(await brain.storage.getNounCount())
|
||||
expect(ledger.verbs.counted).toBe(await brain.storage.getVerbCount())
|
||||
// Hidden tiers exist (the VFS root at minimum, the internal noun, the internal edge):
|
||||
expect(ledger.nouns.all).toBeGreaterThan(ledger.nouns.counted)
|
||||
expect(ledger.verbs.all).toBeGreaterThan(ledger.verbs.counted)
|
||||
|
||||
// The storage-level unfiltered walks report the ALL scalar, and a full page equals it.
|
||||
const nouns = await brain.storage.getNouns({ pagination: { limit: 1000, offset: 0 } })
|
||||
expect(nouns.totalCount).toBe(ledger.nouns.all)
|
||||
expect(nouns.items.length).toBe(ledger.nouns.all)
|
||||
const verbs = await brain.storage.getVerbs({ pagination: { limit: 1000, offset: 0 } })
|
||||
expect(verbs.totalCount).toBe(ledger.verbs.all)
|
||||
expect(verbs.items.length).toBe(ledger.verbs.all)
|
||||
})
|
||||
|
||||
it('proven deletes move the ALL scalar for every tier and the ledger stays exact and unsuspect', async () => {
|
||||
const p = await brain.add({ data: 'public p', type: 'document' })
|
||||
const q = await brain.add({ data: 'internal q', type: 'document', visibility: 'internal' })
|
||||
await brain.relate({ from: p, to: q, type: 'relatedTo' })
|
||||
await brain.flush()
|
||||
const before = await brain.storage.getCanonicalCounts()
|
||||
|
||||
await brain.remove(q) // cascades the edge
|
||||
await brain.remove(p)
|
||||
await brain.flush()
|
||||
|
||||
const after = await brain.storage.getCanonicalCounts()
|
||||
expect(after.nouns.all).toBe(before.nouns.all - 2)
|
||||
expect(after.verbs.all).toBe(before.verbs.all - 1)
|
||||
expect(after.nouns.all).toBe(countIdDirs(dir, 'nouns'))
|
||||
expect(after.verbs.all).toBe(countIdDirs(dir, 'verbs'))
|
||||
expect(after.nouns.counted).toBe(before.nouns.counted - 1)
|
||||
expect(after.suspect).toBe(false)
|
||||
})
|
||||
|
||||
it('a legacy counts.json without the ALL keys is derived once from the id tree and persisted', async () => {
|
||||
await brain.add({ data: 'one', type: 'document' })
|
||||
await brain.add({ data: 'two', type: 'document', visibility: 'internal' })
|
||||
await brain.vfs.writeFile('/a.txt', 'x')
|
||||
await brain.flush()
|
||||
await brain.close()
|
||||
|
||||
const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
|
||||
expect(typeof raw.totalNounCountAll).toBe('number')
|
||||
delete raw.totalNounCountAll
|
||||
delete raw.totalVerbCountAll
|
||||
delete raw.allCountsSuspect
|
||||
fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2))
|
||||
|
||||
brain = await open()
|
||||
const ledger = await brain.storage.getCanonicalCounts()
|
||||
expect(ledger.nouns.all).toBe(countIdDirs(dir, 'nouns'))
|
||||
expect(ledger.verbs.all).toBe(countIdDirs(dir, 'verbs'))
|
||||
expect(ledger.suspect).toBe(false)
|
||||
const persisted = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
|
||||
expect(persisted.totalNounCountAll).toBe(ledger.nouns.all)
|
||||
expect(persisted.totalVerbCountAll).toBe(ledger.verbs.all)
|
||||
})
|
||||
|
||||
it('an inflated ALL scalar is VISIBLE (unclamped) and healed by repairIndex(), surviving reopen', async () => {
|
||||
for (let i = 0; i < 3; i++) await brain.add({ data: `real ${i}`, type: 'document' })
|
||||
await brain.flush()
|
||||
const truth = countIdDirs(dir, 'nouns')
|
||||
|
||||
;(brain.storage as any).totalNounCountAll = truth + 40
|
||||
await (brain.storage as any).persistCounts()
|
||||
await brain.close()
|
||||
brain = await open()
|
||||
|
||||
// The lie survives reopen AND is observable: totalCount disagrees with the walk.
|
||||
const page = await brain.storage.getNouns({ pagination: { limit: 1000, offset: 0 } })
|
||||
expect(page.totalCount).toBe(truth + 40)
|
||||
expect(page.items.length).toBe(truth)
|
||||
|
||||
await brain.repairIndex()
|
||||
expect((await brain.storage.getCanonicalCounts()).nouns.all).toBe(truth)
|
||||
expect((await brain.storage.getNouns({ pagination: { limit: 1000, offset: 0 } })).totalCount).toBe(truth)
|
||||
|
||||
await brain.close()
|
||||
brain = await open()
|
||||
expect((await brain.storage.getCanonicalCounts()).nouns.all).toBe(truth)
|
||||
})
|
||||
|
||||
it('an unprovable delete marks the ledger SUSPECT (persisted); the recount clears it with proof', async () => {
|
||||
await brain.add({ data: 'anchor', type: 'document' })
|
||||
await brain.flush()
|
||||
const truth = countIdDirs(dir, 'nouns')
|
||||
|
||||
// A ghost: no canonical record, no prior image — nothing to prove existence with.
|
||||
await brain.storage.deleteNounMetadata('00000000-dead-4dea-8dea-000000000000')
|
||||
let ledger = await brain.storage.getCanonicalCounts()
|
||||
expect(ledger.suspect).toBe(true)
|
||||
expect(ledger.nouns.all).toBe(truth) // never decremented on faith
|
||||
|
||||
await brain.close()
|
||||
brain = await open()
|
||||
expect((await brain.storage.getCanonicalCounts()).suspect).toBe(true) // the flag persists
|
||||
|
||||
await brain.repairIndex()
|
||||
ledger = await brain.storage.getCanonicalCounts()
|
||||
expect(ledger.suspect).toBe(false)
|
||||
expect(ledger.nouns.all).toBe(truth)
|
||||
})
|
||||
})
|
||||
|
|
@ -23,7 +23,6 @@ import { Brainy } from '../../src/brainy.js'
|
|||
import { NounType } from '../../src/types/graphTypes.js'
|
||||
import {
|
||||
abandonAsCrashed,
|
||||
armCrash,
|
||||
dropCanonicalNoun,
|
||||
makeTempDir,
|
||||
openBrain,
|
||||
|
|
@ -178,33 +177,15 @@ describe('fold-checkpoint bound — crash recovery folds (checkpoint, head], nev
|
|||
expect(founded, 'checkpoint founded at flip').toBe(committedOf(brain))
|
||||
|
||||
// First post-flip boot, unclean (the production first-restart shape):
|
||||
// a post-flip write above the checkpoint is restored FROM ITS AT-ACK FACT;
|
||||
// the pre-flip row (its baseline fact ≤ checkpoint, its bytes barrier-
|
||||
// synced at the flip) is OUTSIDE the fold — vaporizing it synthetically
|
||||
// proves the bound.
|
||||
//
|
||||
// THE CRASH IS ARMED, NOT RACED. The post-flip write "dies" at exactly
|
||||
// `singleop-after-fact-append`: its fact is in the log and at-ack synced,
|
||||
// and NO pending flush was ever scheduled — so the checkpoint provably
|
||||
// still reads the flip's stamp when the bytes are dropped. The earlier
|
||||
// shape (`add()` then abandon) raced the store's 50ms pending-flush
|
||||
// timer: on a loaded box the flush won, barrier-synced the row, advanced
|
||||
// the stamp over it — and the fold, CORRECTLY bounded, did not restore
|
||||
// bytes the test had synthetically destroyed after they were stamped
|
||||
// durable. The plant lane caught it; the engine was right, the pin was
|
||||
// timing-dependent.
|
||||
const postFlip = `post-flip-${Date.now().toString(36)}-0000-4000-8000-000000000000`
|
||||
const arm = armCrash(brain, 'singleop-after-fact-append')
|
||||
await expect(
|
||||
brain.add({ id: postFlip, data: 'post-flip write', type: NounType.Document, metadata: { era: 'log' } })
|
||||
).rejects.toThrow('simulated process crash at singleop-after-fact-append')
|
||||
expect(arm.fired).toContain('singleop-after-fact-append')
|
||||
expect(readCheckpoint(dir), 'the stamp did not move — nothing flushed after the flip').toBe(founded)
|
||||
// a post-flip write above the checkpoint is restored FROM ITS AT-ACK FACT
|
||||
// (deliberately NOT flushed — a flush would barrier-sync it and advance
|
||||
// the stamp over it, making its loss synthetic); the pre-flip row (its
|
||||
// baseline fact ≤ checkpoint, its bytes barrier-synced at the flip) is
|
||||
// OUTSIDE the fold — vaporizing it synthetically proves the bound.
|
||||
const postFlip = await brain.add({ data: 'post-flip write', type: NounType.Document, metadata: { era: 'log' } })
|
||||
await abandonAsCrashed(liveBrains.pop()!)
|
||||
// The post-flip row's canonical bytes lived only in the pending tier's
|
||||
// RAM (written at flush, never reached) — the crash takes them for real;
|
||||
// nothing to drop. Only the pre-flip row is vaporized synthetically.
|
||||
dropCanonicalNoun(dir, preFlip)
|
||||
dropCanonicalNoun(dir, postFlip)
|
||||
|
||||
const reopened = await openBrain(dir, { logAuthority: 'adopt' })
|
||||
liveBrains.push(reopened)
|
||||
|
|
|
|||
|
|
@ -1,62 +0,0 @@
|
|||
/**
|
||||
* @module tests/integration/null-metadata-delete
|
||||
* @description The null-metadata delete skip is CLOSED. remove() used to
|
||||
* guard its index legs with `if (metadata)` — a row whose canonical
|
||||
* metadata was unreadable at delete time (torn, or a leg lost to an old
|
||||
* defect) kept its postings FOREVER, silently. Now: the JS index gets an
|
||||
* id-keyed cleanup (deleted bitmap + id mapper), the id-keyed native
|
||||
* contract is used when a provider offers it, and the one remaining
|
||||
* skip-shape (native without the contract) is narrated and tracked, never
|
||||
* silent. Pinned: a metadata-less row with live postings deletes cleanly
|
||||
* and leaves the query universe.
|
||||
*/
|
||||
import { describe, it, expect, afterEach } from 'vitest'
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { Brainy } from '../../src/index.js'
|
||||
import { NounType } from '../../src/types/graphTypes.js'
|
||||
|
||||
type RawBox = {
|
||||
storage: {
|
||||
readNounRaw(id: string): Promise<{ metadata: unknown; vector: unknown }>
|
||||
writeNounRaw(id: string, r: { metadata: unknown; vector: unknown }): Promise<void>
|
||||
}
|
||||
}
|
||||
|
||||
const dirs: string[] = []
|
||||
const brains: Brainy[] = []
|
||||
afterEach(async () => {
|
||||
for (const b of brains.splice(0)) await b.close().catch(() => {})
|
||||
for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
describe('null-metadata delete', () => {
|
||||
it('a row whose metadata leg is gone still deletes — id-keyed cleanup, no silent skip, gone from the query universe', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'brainy-nullmeta-del-'))
|
||||
dirs.push(dir)
|
||||
const brain = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false, silent: true })
|
||||
await brain.init()
|
||||
brains.push(brain)
|
||||
|
||||
const keep = await brain.add({ data: 'survivor', type: NounType.Document, metadata: { team: 'atlas' } })
|
||||
const victim = await brain.add({ data: 'doomed', type: NounType.Document, metadata: { team: 'atlas' } })
|
||||
await brain.flush()
|
||||
expect((await brain.find({ where: { team: 'atlas' } })).length).toBe(2)
|
||||
|
||||
// Manufacture the shape: the victim's metadata leg vanishes behind the
|
||||
// engine's back (vector leg + postings stay live).
|
||||
const storage = (brain as unknown as RawBox).storage
|
||||
const raw = await storage.readNounRaw(victim)
|
||||
await storage.writeNounRaw(victim, { metadata: null, vector: raw.vector })
|
||||
|
||||
// THE PIN: the delete neither throws nor silently strands postings.
|
||||
await brain.remove(victim)
|
||||
await brain.flush()
|
||||
|
||||
const after = await brain.find({ where: { team: 'atlas' } })
|
||||
expect(after.length, 'victim left the query universe; survivor serves').toBe(1)
|
||||
expect(after[0].id).toBe(keep)
|
||||
expect(await brain.get(victim)).toBeNull()
|
||||
}, 120000)
|
||||
})
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
/**
|
||||
* @module tests/integration/read-surface-readiness
|
||||
* @description THE READ-SURFACE READINESS GATE (a production blackout's
|
||||
* brainy half): with `disableAutoRebuild: true`, init defers index builds —
|
||||
* and before this gate, only find() waited for the lazy rebuild while
|
||||
* related() and every VFS path served EMPTY from the not-ready providers
|
||||
* (writes acked into canonical, readback empty — fifteen live minutes).
|
||||
* The pins: on a fresh instance over a populated store, the FIRST read on
|
||||
* every surface serves truth (it waits for the build), never empty.
|
||||
*/
|
||||
import { describe, it, expect, afterEach } from 'vitest'
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { Brainy } from '../../src/index.js'
|
||||
import { NounType, VerbType } from '../../src/types/graphTypes.js'
|
||||
|
||||
const dirs: string[] = []
|
||||
const brains: Brainy[] = []
|
||||
afterEach(async () => {
|
||||
for (const b of brains.splice(0)) await b.close().catch(() => {})
|
||||
for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
async function openLazy(dir: string): Promise<Brainy> {
|
||||
const brain = new Brainy({
|
||||
storage: { type: 'filesystem', path: dir },
|
||||
requireSubtype: false,
|
||||
silent: true,
|
||||
disableAutoRebuild: true
|
||||
})
|
||||
await brain.init()
|
||||
brains.push(brain)
|
||||
return brain
|
||||
}
|
||||
|
||||
describe('read-surface readiness gate', () => {
|
||||
it('related() as the FIRST read on a fresh lazy instance serves truth, never empty', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'brainy-readgate-'))
|
||||
dirs.push(dir)
|
||||
const writer = await openLazy(dir)
|
||||
const a = await writer.add({ data: 'hub row', type: NounType.Document, metadata: { n: 1 } })
|
||||
const b = await writer.add({ data: 'leaf row', type: NounType.Document, metadata: { n: 2 } })
|
||||
await writer.relate({ from: a, to: b, type: VerbType.RelatedTo })
|
||||
await writer.flush()
|
||||
await brains.pop()!.close()
|
||||
|
||||
// Fresh instance: indexes deferred at open. The production shape called
|
||||
// related() FIRST (no find() to trigger the old, only gate).
|
||||
const reader = await openLazy(dir)
|
||||
const rels = await reader.related({ from: a })
|
||||
expect(rels.length, 'the FIRST related() read waits for the build and serves').toBeGreaterThan(0)
|
||||
expect(rels.some((r) => r.to === b || (r as { target?: string }).target === b)).toBe(true)
|
||||
}, 120000)
|
||||
|
||||
it('a metadata-filtered read as the FIRST read serves truth, never empty', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'brainy-readgate2-'))
|
||||
dirs.push(dir)
|
||||
const writer = await openLazy(dir)
|
||||
await writer.add({ data: 'tagged row', type: NounType.Document, metadata: { team: 'atlas' } })
|
||||
await writer.flush()
|
||||
await brains.pop()!.close()
|
||||
|
||||
const reader = await openLazy(dir)
|
||||
const rows = await reader.find({ where: { team: 'atlas' } })
|
||||
expect(rows.length, 'filtered find on a cold lazy instance serves').toBe(1)
|
||||
}, 120000)
|
||||
})
|
||||
|
|
@ -1,71 +0,0 @@
|
|||
/**
|
||||
* @module tests/integration/repair-report
|
||||
* @description repairIndex() returns the per-family receipt (checked /
|
||||
* healed / skipped-with-reason per family) and narrates a summary — the
|
||||
* "repair that shows its work" half of the graph-trust program's ask. A
|
||||
* repair nobody can audit is a repair nobody can trust.
|
||||
*/
|
||||
import { describe, it, expect, afterEach } from 'vitest'
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { Brainy } from '../../src/index.js'
|
||||
import { NounType } from '../../src/types/graphTypes.js'
|
||||
|
||||
type RawBox = {
|
||||
storage: { writeNounRaw(id: string, r: { metadata: unknown; vector: unknown }): Promise<void> }
|
||||
}
|
||||
|
||||
const dirs: string[] = []
|
||||
const brains: Brainy[] = []
|
||||
afterEach(async () => {
|
||||
for (const b of brains.splice(0)) await b.close().catch(() => {})
|
||||
for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
describe('repairIndex per-family receipt', () => {
|
||||
it('a healthy store gets a complete zero-heal receipt — every family accounted, none silent', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'brainy-repair-clean-'))
|
||||
dirs.push(dir)
|
||||
const brain = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false, silent: true })
|
||||
await brain.init()
|
||||
brains.push(brain)
|
||||
await brain.add({ data: 'healthy row', type: NounType.Document, metadata: { n: 1 } })
|
||||
await brain.flush()
|
||||
|
||||
const report = await brain.repairIndex()
|
||||
expect(report.families.length, 'every family reports a row').toBeGreaterThanOrEqual(5)
|
||||
const names = report.families.map((f) => f.family)
|
||||
for (const expected of ['orphaned-containers', 'count-rollups', 'metadata-corruption']) {
|
||||
expect(names, `family ${expected} accounted`).toContain(expected)
|
||||
}
|
||||
// Every row is either checked or carries its skip reason — no silent rows.
|
||||
for (const f of report.families) {
|
||||
expect(f.checked || !!f.skipped, `${f.family} is checked or explains itself`).toBe(true)
|
||||
}
|
||||
expect(report.healedTotal).toBe(0)
|
||||
expect(report.durationMs).toBeGreaterThanOrEqual(0)
|
||||
}, 120000)
|
||||
|
||||
it('a manufactured ghost container appears in the receipt as a heal', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'brainy-repair-ghost-'))
|
||||
dirs.push(dir)
|
||||
const brain = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false, silent: true })
|
||||
await brain.init()
|
||||
brains.push(brain)
|
||||
await brain.add({ data: 'real row', type: NounType.Document, metadata: { n: 1 } })
|
||||
await brain.flush()
|
||||
// The pre-8.3.1 ghost shape: a vector leg with no content leg.
|
||||
const storage = (brain as unknown as RawBox).storage
|
||||
await storage.writeNounRaw('00000000-0000-7000-8000-00000000dead', {
|
||||
metadata: null,
|
||||
vector: { vector: [0.1, 0.2], noun: 'document' }
|
||||
})
|
||||
|
||||
const report = await brain.repairIndex()
|
||||
const orphans = report.families.find((f) => f.family === 'orphaned-containers')
|
||||
expect(orphans?.checked).toBe(true)
|
||||
expect(orphans!.healed, 'the ghost was pruned and receipted').toBeGreaterThan(0)
|
||||
expect(report.healedTotal).toBeGreaterThan(0)
|
||||
}, 120000)
|
||||
})
|
||||
|
|
@ -77,31 +77,6 @@ describe('validateIndexConsistency delegates to provider validateInvariants() (P
|
|||
delete brain.index.validateInvariants
|
||||
})
|
||||
|
||||
it('ONE CONTRACT FOR A THROWING PROBE: heal is none (flakiness never buys a rebuild) and serving is not withheld', async () => {
|
||||
// The probe that fails to RUN must never be read as "the index is broken,
|
||||
// rebuild it" — that synthesized heal:'rebuild' was the dark-rebuild lever
|
||||
// one transient exception away, and the native composer already said
|
||||
// 'none' for the same event. Both engines now agree: named, loud,
|
||||
// unverified — and never a rebuild, never a withheld serve.
|
||||
brain.index.validateInvariants = async () => { throw new Error('transient: mmap window busy') }
|
||||
const v = await brain.validateIndexConsistency()
|
||||
const thrown = v.providers?.find((p: ProviderInvariantReport) =>
|
||||
p.invariants.some((i) => i.name === 'validate-invariants-threw')
|
||||
)
|
||||
expect(thrown).toBeDefined()
|
||||
expect(thrown!.healthy).toBe(false)
|
||||
expect(thrown!.serving).toBe(true)
|
||||
const inv = thrown!.invariants.find((i) => i.name === 'validate-invariants-threw')!
|
||||
expect(inv.holds).toBe(false)
|
||||
expect(inv.heal).toBe('none')
|
||||
expect(inv.detail).toMatch(/transient: mmap window busy/)
|
||||
// No provider report in the set recommends a rebuild for this event.
|
||||
expect(
|
||||
v.providers!.flatMap((p: ProviderInvariantReport) => p.invariants).some((i) => i.heal === 'rebuild')
|
||||
).toBe(false)
|
||||
delete brain.index.validateInvariants
|
||||
})
|
||||
|
||||
it('providers without validateInvariants() are omitted (JS baseline unchanged)', async () => {
|
||||
const v = await brain.validateIndexConsistency()
|
||||
expect(v.providers).toBeUndefined()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue