Compare commits
89 commits
rel/10.4.5
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 1882532cb7 | |||
| 6eb5e4483d | |||
| ba10aaf52e | |||
| 656d9f6f92 | |||
| 7e1ddee4f7 | |||
| da7d2498bc | |||
| 4e058720b4 | |||
| be307a1579 | |||
| de79d6b5a4 | |||
| d6e7453f1f | |||
| 4c344782a7 | |||
| 4c81d7d4c3 | |||
| dadfa61b5f | |||
| e766ed0a84 | |||
| 87d3a945a5 | |||
| bc82d36294 | |||
| d7444ae804 | |||
| e435da787d | |||
| 97b5ea2d5d | |||
| aa457d7159 | |||
| adcb883e67 | |||
| a2820e81af | |||
| 10a6e81a88 | |||
| e49a73e529 | |||
| 72c8ee6acd | |||
| f27a777615 | |||
| 0d5ab6077d | |||
| a7eb7f5222 | |||
| 5e720d17ae | |||
| 69bda5b7cb | |||
| be77a10bfe | |||
| ad0f493f7a | |||
| 6597c146f7 | |||
| 7932175503 | |||
| 2808398164 | |||
| 6baa4d7f6c | |||
| 85b1fa5c1a | |||
| 8752f11f4d | |||
| 61bc5f423b | |||
| 3835a0e702 | |||
| 27759a1be9 | |||
| 6053f6d423 | |||
| a1423c6da7 | |||
| dea3ec2031 | |||
| 08758c254f | |||
| 3dadbec8f2 | |||
| ebb3a4bf13 | |||
| 2c5e34748e | |||
| 4142f36872 | |||
| 367ca721a5 | |||
| a79db434ac | |||
| da9519903a | |||
| ec644bde56 | |||
| 65493ba2de | |||
| dee46b35c8 | |||
| 1fb5109351 | |||
| 15d4f65dcf | |||
| bc70c43d02 | |||
| 905c267c47 | |||
| b1c7054467 | |||
| 67ae0046de | |||
| 9922631d1f | |||
| 2633e8d5e1 | |||
| f763317af7 | |||
| a8c5fbf9dc | |||
| 34f1886f7c | |||
| 4f1e27c9a0 | |||
| 297a3d7657 | |||
| eec90bdd69 | |||
| 2648f56ddf | |||
| 8a2ebacf02 | |||
| 7ab670b525 | |||
| f097cbf6f2 | |||
| 4d5f823f47 | |||
| 3e60aded36 | |||
| d5147ed608 | |||
| 6a89adc468 | |||
| 88e79729d3 | |||
| e64e2bc175 | |||
| 077cbc0b6f | |||
| 5e3b343a0e | |||
| 4014e0f125 | |||
| 73500e7d10 | |||
| 655aa13ea7 | |||
| 39c71ecdac | |||
| 0759c03a82 | |||
| 9a888c37e9 | |||
|
|
298cb6daca | ||
| b8475cc86a |
111 changed files with 11498 additions and 761 deletions
|
|
@ -5,6 +5,10 @@ name: CI
|
||||||
# sequential, so tag-triggered matrix jobs (~22 min) would queue AHEAD of the
|
# sequential, so tag-triggered matrix jobs (~22 min) would queue AHEAD of the
|
||||||
# tag's publish-source run and starve every release (observed on 8.10.3 and
|
# tag's publish-source run and starve every release (observed on 8.10.3 and
|
||||||
# 9.0.0: the publish sat behind the tag's own redundant CI).
|
# 9.0.0: the publish sat behind the tag's own redundant CI).
|
||||||
|
concurrency:
|
||||||
|
group: ci-${{ github.ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches: ['**']
|
branches: ['**']
|
||||||
|
|
|
||||||
148
.forgejo/workflows/delta-gate.yml
Normal file
148
.forgejo/workflows/delta-gate.yml
Normal file
|
|
@ -0,0 +1,148 @@
|
||||||
|
name: Delta Gate
|
||||||
|
|
||||||
|
# On-demand candidate-vs-control gate on the capped functional CI lane
|
||||||
|
# (label: gate-functional). That lane is Bun-only host-mode — there is no
|
||||||
|
# Node.js runtime available to it, so this workflow deliberately avoids every
|
||||||
|
# JS-based action (checkout/setup-node/setup-bun/upload-artifact all require
|
||||||
|
# one) and does everything with plain git + bun in shell steps instead.
|
||||||
|
#
|
||||||
|
# Verdict lines a caller should grep for in the run log:
|
||||||
|
# COLLECTED patch=<n> control=<n> — collection-truncation guard inputs
|
||||||
|
# NEW-RED-COUNT:<n> — failures on candidate absent from control
|
||||||
|
# DELTA-GATE: CLEAN | NEW REDS | INVALID | STOPPED-BY-REGISTRY-TRIPWIRE
|
||||||
|
#
|
||||||
|
# The lane's own housekeeping stops the runner and drops a marker file when
|
||||||
|
# host pressure (I/O, registry latency, disk budget) trips — never ours to
|
||||||
|
# interpret as a red or a green. The final step checks for that marker before
|
||||||
|
# it says anything about pass/fail.
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
candidate:
|
||||||
|
description: 'Candidate ref (branch or sha) to gate'
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
control:
|
||||||
|
description: 'Control sha to diff against'
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
# workflow_dispatch needs Actions-unit write on the dispatching credential;
|
||||||
|
# push does not (it runs from the pushed ref's own tree), so a plain push
|
||||||
|
# to a release or CI branch is the fallback trigger while that grant is
|
||||||
|
# outstanding — see the ref-resolution step below for what it gates against.
|
||||||
|
push:
|
||||||
|
branches: ['rel/**', 'ci/**']
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: delta-gate
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
delta-gate:
|
||||||
|
name: Delta gate — candidate vs control
|
||||||
|
runs-on: gate-functional
|
||||||
|
timeout-minutes: 120
|
||||||
|
steps:
|
||||||
|
- name: Resolve candidate/control refs
|
||||||
|
id: refs
|
||||||
|
run: |
|
||||||
|
candidate="${{ github.event.inputs.candidate }}"
|
||||||
|
control="${{ github.event.inputs.control }}"
|
||||||
|
# workflow_dispatch supplies both explicitly; a push event carries
|
||||||
|
# neither — fall back to the pushed commit as candidate and the
|
||||||
|
# last released, known-good tip (10.4.9) as control, so a plain
|
||||||
|
# push still produces a meaningful gate instead of an empty ref.
|
||||||
|
if [ -z "$candidate" ]; then candidate="${{ github.sha }}"; fi
|
||||||
|
if [ -z "$control" ]; then control="eec90bdd"; fi
|
||||||
|
echo "candidate=$candidate" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "control=$control" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "Resolved (trigger=${{ github.event_name }}): candidate=$candidate control=$control"
|
||||||
|
|
||||||
|
- name: Clean any residue from a prior run
|
||||||
|
run: rm -rf "ob-cand-${{ github.run_id }}" "ob-ctrl-${{ github.run_id }}" "/tmp/ob-${{ github.run_id }}-"*
|
||||||
|
|
||||||
|
- name: Clone + test — candidate
|
||||||
|
id: patch
|
||||||
|
run: |
|
||||||
|
set -o pipefail
|
||||||
|
git clone --quiet "https://source.soulcraft.com/soulcraftlabs/open-brainy.git" "ob-cand-${{ github.run_id }}"
|
||||||
|
cd "ob-cand-${{ github.run_id }}"
|
||||||
|
git checkout --quiet "${{ steps.refs.outputs.candidate }}"
|
||||||
|
git log --oneline -1
|
||||||
|
bun install
|
||||||
|
rc=0
|
||||||
|
bun x vitest run > "/tmp/ob-${{ github.run_id }}-patch.log" 2>&1 || rc=$?
|
||||||
|
echo "PATCH-RC:$rc"
|
||||||
|
grep -aE "Tests .*(passed|failed)" "/tmp/ob-${{ github.run_id }}-patch.log" | tail -1
|
||||||
|
grep -aE "^ FAIL |^\s+×" "/tmp/ob-${{ github.run_id }}-patch.log" | sed -E "s/ [0-9]+ms$//" | sed -E "s/^\s+//" | sort -u > "/tmp/ob-${{ github.run_id }}-patch.fail"
|
||||||
|
echo "PATCH-FAILING:$(wc -l < "/tmp/ob-${{ github.run_id }}-patch.fail")"
|
||||||
|
|
||||||
|
- name: Clone + test — control
|
||||||
|
id: control
|
||||||
|
run: |
|
||||||
|
set -o pipefail
|
||||||
|
git clone --quiet "https://source.soulcraft.com/soulcraftlabs/open-brainy.git" "ob-ctrl-${{ github.run_id }}"
|
||||||
|
cd "ob-ctrl-${{ github.run_id }}"
|
||||||
|
git checkout --quiet "${{ steps.refs.outputs.control }}"
|
||||||
|
git log --oneline -1
|
||||||
|
bun install
|
||||||
|
rc=0
|
||||||
|
bun x vitest run > "/tmp/ob-${{ github.run_id }}-control.log" 2>&1 || rc=$?
|
||||||
|
echo "CONTROL-RC:$rc"
|
||||||
|
grep -aE "Tests .*(passed|failed)" "/tmp/ob-${{ github.run_id }}-control.log" | tail -1
|
||||||
|
grep -aE "^ FAIL |^\s+×" "/tmp/ob-${{ github.run_id }}-control.log" | sed -E "s/ [0-9]+ms$//" | sed -E "s/^\s+//" | sort -u > "/tmp/ob-${{ github.run_id }}-control.fail"
|
||||||
|
echo "CONTROL-FAILING:$(wc -l < "/tmp/ob-${{ github.run_id }}-control.fail")"
|
||||||
|
|
||||||
|
- name: Delta gate verdict
|
||||||
|
if: always()
|
||||||
|
run: |
|
||||||
|
set -o pipefail
|
||||||
|
|
||||||
|
# The lane's own tripwire wins over anything we would otherwise say:
|
||||||
|
# a bare failure/timeout above with this marker present is host
|
||||||
|
# pressure, never a real red and never a real green.
|
||||||
|
if [ -f /srv/gate-lane/TRIPWIRE-STOPPED ]; then
|
||||||
|
echo "DELTA-GATE: STOPPED-BY-REGISTRY-TRIPWIRE"
|
||||||
|
head -1 /srv/gate-lane/TRIPWIRE-STOPPED
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
|
||||||
|
patch_log="/tmp/ob-${{ github.run_id }}-patch.log"
|
||||||
|
control_log="/tmp/ob-${{ github.run_id }}-control.log"
|
||||||
|
patch_fail="/tmp/ob-${{ github.run_id }}-patch.fail"
|
||||||
|
control_fail="/tmp/ob-${{ github.run_id }}-control.fail"
|
||||||
|
|
||||||
|
if [ ! -s "$patch_log" ] || [ ! -s "$control_log" ]; then
|
||||||
|
echo "DELTA-GATE: INVALID — a leg produced no log (see the two steps above for the real cause)"
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
pt=$(grep -aoE "\(([0-9]+)\)$" "$patch_log" | tail -1 | tr -d "()")
|
||||||
|
ct=$(grep -aoE "\(([0-9]+)\)$" "$control_log" | tail -1 | tr -d "()")
|
||||||
|
echo "COLLECTED patch=${pt:-0} control=${ct:-0}"
|
||||||
|
if [ "${pt:-0}" -lt 3000 ] || [ "${ct:-0}" -lt 3000 ]; then
|
||||||
|
echo "DELTA-GATE: INVALID — truncated collection"
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "=== NEW REDS ==="
|
||||||
|
comm -23 "$patch_fail" "$control_fail"
|
||||||
|
new=$(comm -23 "$patch_fail" "$control_fail" | wc -l)
|
||||||
|
echo "NEW-RED-COUNT:$new"
|
||||||
|
|
||||||
|
echo "=== full candidate fail list ==="
|
||||||
|
cat "$patch_fail"
|
||||||
|
echo "=== full control fail list ==="
|
||||||
|
cat "$control_fail"
|
||||||
|
|
||||||
|
if [ "$new" -eq 0 ]; then
|
||||||
|
echo "DELTA-GATE: CLEAN"
|
||||||
|
else
|
||||||
|
echo "DELTA-GATE: NEW REDS"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
- name: Clean up (mind the lane's disk budget)
|
||||||
|
if: always()
|
||||||
|
run: rm -rf "ob-cand-${{ github.run_id }}" "ob-ctrl-${{ github.run_id }}" "/tmp/ob-${{ github.run_id }}-"*
|
||||||
|
|
@ -12,6 +12,11 @@ on:
|
||||||
push:
|
push:
|
||||||
tags:
|
tags:
|
||||||
- 'v*'
|
- 'v*'
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
ref_reason:
|
||||||
|
description: 'why this manual run (e.g. tag event dropped)'
|
||||||
|
required: false
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
publish:
|
publish:
|
||||||
|
|
|
||||||
60
CHANGELOG.md
60
CHANGELOG.md
|
|
@ -2,6 +2,66 @@
|
||||||
|
|
||||||
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
||||||
|
|
||||||
|
|
||||||
|
### [10.4.12](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.11...v10.4.12) (2026-09-03)
|
||||||
|
|
||||||
|
- Mixed-kind fields index exactly, arrays to 256, a drained loop is not a shutdown, and finds project from the column store
|
||||||
|
- fix(index): a metadata field holds every value kind it was written with — one posting column per (field, kind); an equality filter reads the query value's own kind, a range routes by its bounds; nothing is refused and nothing is silently dropped; an index written by the old shape opens unchanged (a128f0ed)
|
||||||
|
- fix(metadata): metadata arrays index up to 256 elements; a longer array refuses at write time by name (MetadataArrayTooLargeError) — a vector parked in metadata now throws; move it to `vector` (e435da78)
|
||||||
|
- fix(shutdown): beforeExit runs a non-closing flush only — a script that never calls close() exits with the writer lock on disk and no clean-shutdown marker, and the next open evicts the stale lock and folds the log, bounded; SIGTERM and SIGINT are unchanged (6baa4d7f)
|
||||||
|
- feat(find): field projection — find({fields}) and get({fields}) resolve scalars from the column store on every leg, including vector-leg finds; absent fields stay absent (ad0f493f)
|
||||||
|
- fix(find): orderBy is the order on every find path, not only the metadata-only one (5e720d17)
|
||||||
|
- fix(metadata): the legacy sparse range path orders values, or refuses by name — never ranks by hash (a7eb7f52)
|
||||||
|
- fix(close): a read-only brain writes nothing under `_system/` (f27a7776)
|
||||||
|
- fix(contract): the flush gate's internals are private, not doors (72c8ee6a)
|
||||||
|
- test(hygiene): the triple-intelligence correctness cases sit in the gate; the idle and connected-find pins name the brain they measure (28083981)
|
||||||
|
- ci(release): the rail writes its own wall entry into the shared releases repo — never hand-written again (adcb883e)
|
||||||
|
|
||||||
|
### [10.4.11](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.9...v10.4.11) (2026-09-02)
|
||||||
|
|
||||||
|
- ci: superseded pushes cancel their own runs (concurrency per ref) (6053f6d4)
|
||||||
|
- test(batch): the batch-size-limit tests add unvectored items — they test batching, not embedding (a1423c6d)
|
||||||
|
- fix(flush): the gate settles its waiter from the machine, never from a chain (dea3ec20)
|
||||||
|
- test(batch): the batch-vs-individual timing assertion runs in the perf lane, not the correctness gate (ebb3a4bf)
|
||||||
|
- test(gate): the coverage guard counts the perf lane's config as a gate (2c5e3474)
|
||||||
|
- chore(contract): emit the 10.4.11 manifest (4142f368)
|
||||||
|
- fix(close): a read-only brain writes no clean-shutdown evidence — the marker is the writer's word about itself (367ca721)
|
||||||
|
- fix(generation-store): commitTransaction refuses while single-ops are pending — the order invariant is enforced, not assumed (a79db434)
|
||||||
|
- test(shutdown): pin one owner per brain — real processes, real signals (da951990)
|
||||||
|
- fix(shutdown): one owner per brain — the signal handler defers to close(), and flush is single-flight (ec644bde)
|
||||||
|
- fix(vfs): a path-scoped search is a served range over the path, not a refused prefix match (65493ba2)
|
||||||
|
- ci(test): perf and scale benchmarks leave the correctness gate (dee46b35)
|
||||||
|
- test(open): pin the pending-embed checkpoint — stuck id, crash matrix, torn fallback (1fb51093)
|
||||||
|
- perf(open): the pending-embed fold is bounded by a checkpoint of the SET, not an empty-only mark (15d4f65d)
|
||||||
|
- perf(open): a sealed segment the manifest proves is below the bound is never read (bc70c43d)
|
||||||
|
- fix(find): a page the metadata block already cut is not cut again (905c267c)
|
||||||
|
- fix(find): the hybrid legs rank inside the filter, and only the page is read (b1c70544)
|
||||||
|
- ci(delta-gate): add a push fallback trigger alongside workflow_dispatch (67ae0046)
|
||||||
|
- ci: add the delta-gate workflow for the capped functional lane (9922631d)
|
||||||
|
- docs(plugin): the planner door's hiddenIds contract is the answer, not the mechanism (2633e8d5)
|
||||||
|
- feat(engine): a protected factory for the generation store — a subclass may substitute one that keeps the contract (f763317a)
|
||||||
|
- fix(find): near() searches around the anchor's own vector, and refuses by name without one (a8c5fbf9)
|
||||||
|
- Merge remote-tracking branches 'origin/fix/planner-provider-door' and 'origin/fix/containment-batching' into rel/10.4.10-candidate (34f1886f)
|
||||||
|
- feat(plugin): an optional planFindPage door — an index that can plan a find answers it in one call (4d5f823f)
|
||||||
|
- perf(vfs): repairContainment's reconcile is one paged edge walk, not one graph call per file (3e60aded)
|
||||||
|
|
||||||
|
|
||||||
|
### [10.4.9](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.6...v10.4.9) (2026-09-02)
|
||||||
|
|
||||||
|
- Merge branch 'fix/pending-embed-low-water' into rel/10.4.9-candidate (2648f56d)
|
||||||
|
- fix(open): pending-embed recovery keeps the crash-recovery contract — foreground, bounded by the mark (8a2ebacf)
|
||||||
|
- Merge branches 'fix/connected-find-order', 'fix/pending-embed-low-water' and 'fix/related-verb-array' into rel/10.4.9-candidate (d5147ed6)
|
||||||
|
- fix(graph): the verb fast paths honour every requested type, source, and target (6a89adc4)
|
||||||
|
- perf(open): pending-embed recovery is bounded by a low-water mark and runs behind the doors (88e79729)
|
||||||
|
- fix(find): connected finds are graph-first — neighbours, then the filter over those ids, then the page (077cbc0b)
|
||||||
|
- fix(storage): counts persistence is single-flight, coalesced, and never races its own temp file (5e3b343a)
|
||||||
|
|
||||||
|
|
||||||
|
### [10.4.6](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.5...v10.4.6) (2026-08-31)
|
||||||
|
|
||||||
|
- fix(transact): metadata-index ops take their JSON-safe view at the crossing, not at construction (73500e7d)
|
||||||
|
|
||||||
|
|
||||||
### [10.4.5](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.4...v10.4.5) (2026-08-31)
|
### [10.4.5](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.4...v10.4.5) (2026-08-31)
|
||||||
|
|
||||||
- build(release): the docs-push step retires — this engine documents itself in its own repository (d6bcb14f)
|
- build(release): the docs-push step retires — this engine documents itself in its own repository (d6bcb14f)
|
||||||
|
|
|
||||||
|
|
@ -41,6 +41,20 @@ npm test
|
||||||
Tests run on [Vitest](https://vitest.dev/). `npm test` runs the unit suite;
|
Tests run on [Vitest](https://vitest.dev/). `npm test` runs the unit suite;
|
||||||
see `package.json` for `test:integration`, `test:coverage`, and friends.
|
see `package.json` for `test:integration`, `test:coverage`, and friends.
|
||||||
|
|
||||||
|
## Test gate
|
||||||
|
|
||||||
|
The release gate is a bare `vitest run` (no `--config` flag) — the same
|
||||||
|
command the delta gate and CI's checks invoke. It carries the full
|
||||||
|
correctness suite and nothing else: wall-clock/scale benchmarks
|
||||||
|
(`tests/performance/**`, `tests/critical-performance-benchmark.test.ts`,
|
||||||
|
`tests/api/performance-benchmarks.test.ts`) and the two tests whose outcome
|
||||||
|
depends on the host machine or network rather than the code
|
||||||
|
(`tests/package-size-limit.test.ts` shells out to the `npm` CLI;
|
||||||
|
`tests/model-loading.test.ts` makes a real network call to download a model)
|
||||||
|
are excluded from it, because a timing threshold or a flaky network call has
|
||||||
|
no business failing a correctness check. That whole family runs on demand,
|
||||||
|
in its own exclusive slot, via `npm run test:perf`.
|
||||||
|
|
||||||
## Standards
|
## Standards
|
||||||
|
|
||||||
- **Strict TypeScript.** No `any` escape hatches to dodge the type checker.
|
- **Strict TypeScript.** No `any` escape hatches to dodge the type checker.
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,12 @@
|
||||||
# @soulcraft/brainy — Release Notes for Consumers
|
# @soulcraft/brainy — Release Notes for Consumers
|
||||||
|
|
||||||
|
Machine-readable release notes are published at
|
||||||
|
https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/open-brainy.json
|
||||||
|
(this engine) and
|
||||||
|
https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/brainy.json
|
||||||
|
(the product engine) — read by HQ's `/hq/releases` door, and the source of
|
||||||
|
truth ahead of this file.
|
||||||
|
|
||||||
This file is the **quick reference for downstream sessions** tracking Brainy changes.
|
This file is the **quick reference for downstream sessions** tracking Brainy changes.
|
||||||
Full auto-generated changelog: `CHANGELOG.md` · Releases: https://source.soulcraft.com/soulcraftlabs/open-brainy/releases
|
Full auto-generated changelog: `CHANGELOG.md` · Releases: https://source.soulcraft.com/soulcraftlabs/open-brainy/releases
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -369,6 +369,71 @@ return results.slice(offset, offset + limit)
|
||||||
// → Auto-correction: Use most likely alternative based on affinity data
|
// → Auto-correction: Use most likely alternative based on affinity data
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Field Projection (`fields`)
|
||||||
|
|
||||||
|
`find()` and `get()` accept a `fields` list. Without it they return the whole
|
||||||
|
record; with it they return only the fields you name — and, where the index can
|
||||||
|
supply them, without opening the canonical record at all.
|
||||||
|
|
||||||
|
```ts
|
||||||
|
// A list page: two user fields and one engine scalar. No document bodies.
|
||||||
|
await brain.find({
|
||||||
|
where: { kind: 'post' },
|
||||||
|
fields: ['title', 'slug', 'system.createdAt'],
|
||||||
|
limit: 50
|
||||||
|
})
|
||||||
|
|
||||||
|
await brain.get(id, { fields: ['title'] })
|
||||||
|
```
|
||||||
|
|
||||||
|
### Why it exists
|
||||||
|
|
||||||
|
A list view that renders a title and a date does not need the body, but without
|
||||||
|
a projection every row hydrates its full record and throws almost all of it
|
||||||
|
away. On a posts list that is the dominant cost of the query.
|
||||||
|
|
||||||
|
### The rules
|
||||||
|
|
||||||
|
| | |
|
||||||
|
|---|---|
|
||||||
|
| **`fields` absent** | The full record, byte-identical to before. Nothing changes. |
|
||||||
|
| **Field names** | The one addressing law: a bare name is user metadata (`'title'`), `system.*` is an engine scalar (`'system.createdAt'`). |
|
||||||
|
| **A field the row lacks** | Simply **absent** from the result. Never an error. |
|
||||||
|
| **Identity** | Every row keeps its `id` (and `score` on `find`) regardless — a row you cannot identify is not a row. |
|
||||||
|
| **Where values come from** | The **column store**, which holds raw values. Never the sparse index, which buckets timestamps for range queries. |
|
||||||
|
| **A field the column cannot serve** | The canonical record is read for that field only. Correct, just not free. |
|
||||||
|
|
||||||
|
### Missing fields are absent, not errors
|
||||||
|
|
||||||
|
This is deliberate and differs from `orderBy`, which throws
|
||||||
|
`UnresolvableFieldError` for an unknown field. A typo in `orderBy` silently
|
||||||
|
changes the ordering, so it must be loud. A projection asks "give me these if
|
||||||
|
you have them", and an optional field must not turn a list into a failure — so
|
||||||
|
`fields` uses the permissive path.
|
||||||
|
|
||||||
|
### Cost
|
||||||
|
|
||||||
|
When every named field is column-served, a projected page performs **zero**
|
||||||
|
canonical reads. When one is not, only that read happens and the rest still come
|
||||||
|
from the index. Both are pinned by counting reads rather than timing them, in
|
||||||
|
`tests/integration/find-fields-projection.test.ts`.
|
||||||
|
|
||||||
|
### `related()` takes no `fields`
|
||||||
|
|
||||||
|
A `Relation` carries `from` and `to` as **ids** and hydrates no entity record,
|
||||||
|
so there is nothing for a projection to trim. Projecting the endpoints would be
|
||||||
|
a new capability rather than a projection of an existing one.
|
||||||
|
|
||||||
|
### For engine implementers
|
||||||
|
|
||||||
|
Projection is served through an optional provider door,
|
||||||
|
`getScalarsForIds(ids, fields)` on `MetadataIndexProvider`. The contract is in
|
||||||
|
`src/plugin.ts`; the short version is **return only what you can serve exactly,
|
||||||
|
and say what you served**. The caller diffs the answer against the request and
|
||||||
|
reads records for the remainder, so omission costs a read while a wrong value is
|
||||||
|
a wrong answer nobody can see. An engine without the door still works — every
|
||||||
|
field falls back to the record.
|
||||||
|
|
||||||
## Performance Characteristics
|
## Performance Characteristics
|
||||||
|
|
||||||
### Query Performance by Type
|
### Query Performance by Type
|
||||||
|
|
|
||||||
|
|
@ -161,6 +161,11 @@
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
"arity": 1
|
"arity": 1
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "captureEmbedCheckpoint",
|
||||||
|
"kind": "method",
|
||||||
|
"arity": 0
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "checkHealth",
|
"name": "checkHealth",
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
|
|
@ -225,6 +230,11 @@
|
||||||
"name": "counts",
|
"name": "counts",
|
||||||
"kind": "accessor"
|
"kind": "accessor"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "createGenerationStore",
|
||||||
|
"kind": "method",
|
||||||
|
"arity": 1
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "createIndex",
|
"name": "createIndex",
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
|
|
@ -258,6 +268,11 @@
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
"arity": 1
|
"arity": 1
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "demoteTornEntityTreeStamp",
|
||||||
|
"kind": "method",
|
||||||
|
"arity": 4
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "detectIdKind",
|
"name": "detectIdKind",
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
|
|
@ -353,11 +368,6 @@
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
"arity": 1
|
"arity": 1
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"name": "executeGraphSearch",
|
|
||||||
"kind": "method",
|
|
||||||
"arity": 2
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"name": "executeProximitySearch",
|
"name": "executeProximitySearch",
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
|
|
@ -368,11 +378,21 @@
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
"arity": 2
|
"arity": 2
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "executeTextSearchScored",
|
||||||
|
"kind": "method",
|
||||||
|
"arity": 3
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "executeVectorSearch",
|
"name": "executeVectorSearch",
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
"arity": 3
|
"arity": 3
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "executeVectorSearchScored",
|
||||||
|
"kind": "method",
|
||||||
|
"arity": 3
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "explain",
|
"name": "explain",
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
|
|
@ -418,6 +438,11 @@
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
"arity": 2
|
"arity": 2
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "filterIdsWithinBelted",
|
||||||
|
"kind": "method",
|
||||||
|
"arity": 2
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "find",
|
"name": "find",
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
|
|
@ -711,6 +736,11 @@
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
"arity": 2
|
"arity": 2
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "hydrateResultPage",
|
||||||
|
"kind": "method",
|
||||||
|
"arity": 2
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "import",
|
"name": "import",
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
|
|
@ -741,6 +771,14 @@
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
"arity": 0
|
"arity": 0
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "isClosed",
|
||||||
|
"kind": "accessor"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "isClosing",
|
||||||
|
"kind": "accessor"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "isEmbeddingReady",
|
"name": "isEmbeddingReady",
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
|
|
@ -799,6 +837,16 @@
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
"arity": 1
|
"arity": 1
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "maybeWriteEmbedCheckpoint",
|
||||||
|
"kind": "method",
|
||||||
|
"arity": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "maybeWriteEmbedLowWater",
|
||||||
|
"kind": "method",
|
||||||
|
"arity": 0
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "metadataIndexRetractionOp",
|
"name": "metadataIndexRetractionOp",
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
|
|
@ -854,6 +902,11 @@
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
"arity": 1
|
"arity": 1
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "noteEmbedCheckpointCadence",
|
||||||
|
"kind": "method",
|
||||||
|
"arity": 0
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "noteWriteForPersistence",
|
"name": "noteWriteForPersistence",
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
|
|
@ -869,6 +922,11 @@
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
"arity": 1
|
"arity": 1
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "pageConnectedIds",
|
||||||
|
"kind": "method",
|
||||||
|
"arity": 2
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "pagination",
|
"name": "pagination",
|
||||||
"kind": "accessor"
|
"kind": "accessor"
|
||||||
|
|
@ -893,6 +951,11 @@
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
"arity": 0
|
"arity": 0
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "pendingResult",
|
||||||
|
"kind": "method",
|
||||||
|
"arity": 2
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "performInit",
|
"name": "performInit",
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
|
|
@ -993,6 +1056,11 @@
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
"arity": 2
|
"arity": 2
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "readPendingEmbedBound",
|
||||||
|
"kind": "method",
|
||||||
|
"arity": 0
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "ready",
|
"name": "ready",
|
||||||
"kind": "accessor"
|
"kind": "accessor"
|
||||||
|
|
@ -1112,6 +1180,11 @@
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
"arity": 2
|
"arity": 2
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "resolveConnectedIds",
|
||||||
|
"kind": "method",
|
||||||
|
"arity": 1
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "resolveDiffEndpoint",
|
"name": "resolveDiffEndpoint",
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
|
|
@ -1155,7 +1228,7 @@
|
||||||
{
|
{
|
||||||
"name": "rrfFusion",
|
"name": "rrfFusion",
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
"arity": 4
|
"arity": 3
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "runAggregationBackfillWalk",
|
"name": "runAggregationBackfillWalk",
|
||||||
|
|
@ -1275,6 +1348,11 @@
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
"arity": 1
|
"arity": 1
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "textIdsWithinBelted",
|
||||||
|
"kind": "method",
|
||||||
|
"arity": 2
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "trackField",
|
"name": "trackField",
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
|
|
@ -1413,12 +1491,23 @@
|
||||||
"name": "wireGraphIdResolver",
|
"name": "wireGraphIdResolver",
|
||||||
"kind": "method",
|
"kind": "method",
|
||||||
"arity": 0
|
"arity": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "writeEmbedCheckpoint",
|
||||||
|
"kind": "method",
|
||||||
|
"arity": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "writeEmbedLowWater",
|
||||||
|
"kind": "method",
|
||||||
|
"arity": 0
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"errors": [
|
"errors": [
|
||||||
"BrainyError",
|
"BrainyError",
|
||||||
"DerivedArtifactMissingError",
|
"DerivedArtifactMissingError",
|
||||||
"GraphIndexNotReadyError",
|
"GraphIndexNotReadyError",
|
||||||
|
"MetadataArrayTooLargeError",
|
||||||
"MetadataIndexNotReadyError",
|
"MetadataIndexNotReadyError",
|
||||||
"MigrationInProgressError",
|
"MigrationInProgressError",
|
||||||
"ProtectedArtifactError",
|
"ProtectedArtifactError",
|
||||||
|
|
|
||||||
|
|
@ -217,6 +217,40 @@ membership queries at scale:
|
||||||
`__words__` for tokenized text…).
|
`__words__` for tokenized text…).
|
||||||
- `_blobs/_column_index/{field}/L0-NNNNNN.bin` — the actual level-0 run
|
- `_blobs/_column_index/{field}/L0-NNNNNN.bin` — the actual level-0 run
|
||||||
segments, stored through the shared `_blobs/<key>.bin` binary convention.
|
segments, stored through the shared `_blobs/<key>.bin` binary convention.
|
||||||
|
- `_column_index/{field}/k/{kind}/…` — the same two files again, for a
|
||||||
|
**second value kind** on the same field (see below). Absent for a field that
|
||||||
|
holds one kind, which is nearly all of them.
|
||||||
|
|
||||||
|
### One posting column per (field, kind)
|
||||||
|
|
||||||
|
A field is not obliged to hold one type of value. `category` may carry
|
||||||
|
`'electronics'` on some rows and `5` on others, and both are real values of
|
||||||
|
that field. A segment, though, has one encoding — i64, f64, UTF-8, or boolean
|
||||||
|
— so a field that holds several kinds gets **one column per kind**:
|
||||||
|
|
||||||
|
- The first kind a field ever sees owns the plain `_column_index/{field}/`
|
||||||
|
layout above. A single-kind field is therefore byte-identical to what earlier
|
||||||
|
versions wrote, and an index written before typed postings opens unchanged.
|
||||||
|
- Every later kind gets its own column beside it at
|
||||||
|
`_column_index/{field}/k/{kind}/`, where `{kind}` is `number`, `string` or
|
||||||
|
`boolean`.
|
||||||
|
|
||||||
|
What that buys at query time:
|
||||||
|
|
||||||
|
| | |
|
||||||
|
|---|---|
|
||||||
|
| **Equality** | Answered from the column matching the **query value's own kind**. `where {category: 5}` reads the number postings; `where {category: '5'}` reads the string postings. Neither borrows the other's rows — a row written with the number `5` is not a row whose category is the text `'5'`. |
|
||||||
|
| **A kind the field never held** | Matches nothing. That is the true answer, not a coerced one. |
|
||||||
|
| **Ranges** | Routed by the kind of the bounds: numeric bounds read the numeric postings and ignore the field's strings. An **unbounded** range is the "has any value here" probe behind `exists`, and reads every kind. |
|
||||||
|
| **`orderBy`** | A number and a string have no order between them, so a mixed field orders by kind first (number, string, boolean) and by value within a kind. A single-kind field sorts exactly as it always did. |
|
||||||
|
| **Numbers** | One kind, one column: an integer column is written as i64 and widens to f64 the first time a non-integer arrives, so `4.5` is stored as itself rather than rounded. |
|
||||||
|
|
||||||
|
`null` and `undefined` are not kinds and are never posted; their absence is
|
||||||
|
what the `exists` / `missing` operators read.
|
||||||
|
|
||||||
|
Older readers are unaffected by the additional columns: they see the field's
|
||||||
|
primary column exactly where it has always been, and a `k/{kind}` directory is
|
||||||
|
simply a name they never query.
|
||||||
|
|
||||||
Sparse per-field indexes, roaring-bitmap chunks, and zone-map/bloom segments
|
Sparse per-field indexes, roaring-bitmap chunks, and zone-map/bloom segments
|
||||||
additionally live as bucketed keys under `_system/idx/` (see §3). Which path
|
additionally live as bucketed keys under `_system/idx/` (see §3). Which path
|
||||||
|
|
|
||||||
|
|
@ -95,8 +95,15 @@ The heartbeat interval rewrites the lock file every 10 seconds. The timer
|
||||||
is unref'd, so it does not keep the event loop alive on its own.
|
is unref'd, so it does not keep the event loop alive on its own.
|
||||||
|
|
||||||
On normal shutdown the writer releases the lock in `close()`. The shutdown
|
On normal shutdown the writer releases the lock in `close()`. The shutdown
|
||||||
hooks Brainy registers for `SIGTERM`, `SIGINT`, and `beforeExit` also
|
hooks Brainy registers for `SIGTERM` and `SIGINT` close every live brain by
|
||||||
release the lock so a container restart doesn't strand the directory.
|
that same `close()`, so a container restart doesn't strand the directory.
|
||||||
|
|
||||||
|
`beforeExit` is not one of them. Node emits it whenever the event loop has
|
||||||
|
no ref'd work left — a state a healthy script reaches routinely, because
|
||||||
|
Brainy's own idle and cadence timers are unref'd — and a drained event loop
|
||||||
|
is not a shutdown. That hook only persists derived state with a non-closing
|
||||||
|
`flush()`: it closes nothing, releases no lock, and leaves every brain open
|
||||||
|
and usable. If you want a shutdown, call `close()` or send `SIGTERM`.
|
||||||
|
|
||||||
## How to inspect a live writer
|
## How to inspect a live writer
|
||||||
|
|
||||||
|
|
|
||||||
4
package-lock.json
generated
4
package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
||||||
{
|
{
|
||||||
"name": "@soulcraftlabs/brainy",
|
"name": "@soulcraftlabs/brainy",
|
||||||
"version": "10.4.5",
|
"version": "10.4.12",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "@soulcraftlabs/brainy",
|
"name": "@soulcraftlabs/brainy",
|
||||||
"version": "10.4.5",
|
"version": "10.4.12",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@msgpack/msgpack": "^3.1.2",
|
"@msgpack/msgpack": "^3.1.2",
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "@soulcraftlabs/brainy",
|
"name": "@soulcraftlabs/brainy",
|
||||||
"version": "10.4.5",
|
"version": "10.4.12",
|
||||||
"brainyContract": 1,
|
"brainyContract": 1,
|
||||||
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
|
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
|
|
@ -88,7 +88,7 @@
|
||||||
"test:watch": "NODE_OPTIONS='--max-old-space-size=8192' vitest --config tests/configs/vitest.unit.config.ts",
|
"test:watch": "NODE_OPTIONS='--max-old-space-size=8192' vitest --config tests/configs/vitest.unit.config.ts",
|
||||||
"test:coverage": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.unit.config.ts --coverage",
|
"test:coverage": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.unit.config.ts --coverage",
|
||||||
"test:unit": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.unit.config.ts",
|
"test:unit": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.unit.config.ts",
|
||||||
"test:perf": "vitest run tests/unit/performance --reporter=basic",
|
"test:perf": "vitest run --config tests/configs/vitest.perf.config.ts",
|
||||||
"test:integration": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.integration.config.ts",
|
"test:integration": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.integration.config.ts",
|
||||||
"test:semantic": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.semantic.config.ts",
|
"test:semantic": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.semantic.config.ts",
|
||||||
"test:all": "npm run test:unit && npm run test:integration",
|
"test:all": "npm run test:unit && npm run test:integration",
|
||||||
|
|
|
||||||
|
|
@ -154,13 +154,26 @@ else
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Create new changelog entry
|
# Create new changelog entry
|
||||||
CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) ($(date +%Y-%m-%d))
|
RELEASE_DATE=$(date +%Y-%m-%d)
|
||||||
|
CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) (${RELEASE_DATE})
|
||||||
|
|
||||||
${COMMITS}
|
${COMMITS}
|
||||||
"
|
"
|
||||||
|
# A CURATED entry wins over the generated one. When a release is cut from a
|
||||||
|
# lineage that diverged from the previous tag (a candidate branch carrying
|
||||||
|
# main's history), `git log <last-tag>..HEAD` lists every commit the tag never
|
||||||
|
# saw — old notes, already-shipped fixes under new hashes, merge commits — and a
|
||||||
|
# wall entry derived from it would misreport the release. If CHANGELOG.md
|
||||||
|
# already carries a `### [NEW_VERSION]` heading, it was written on purpose:
|
||||||
|
# keep it, and skip the generated prepend entirely.
|
||||||
|
CURATED_ENTRY=false
|
||||||
|
if grep -qE "^### \[${NEW_VERSION}\]" CHANGELOG.md 2>/dev/null; then
|
||||||
|
CURATED_ENTRY=true
|
||||||
|
echo -e "${YELLOW}CHANGELOG already carries a curated ### [${NEW_VERSION}] entry — keeping it, not generating one from commits${NC}"
|
||||||
|
fi
|
||||||
|
|
||||||
# Prepend to CHANGELOG.md after header
|
# Prepend to CHANGELOG.md after header
|
||||||
if [ -f "CHANGELOG.md" ]; then
|
if [ "$CURATED_ENTRY" = false ] && [ -f "CHANGELOG.md" ]; then
|
||||||
# Read header (first 4 lines)
|
# Read header (first 4 lines)
|
||||||
HEADER=$(head -n 4 CHANGELOG.md)
|
HEADER=$(head -n 4 CHANGELOG.md)
|
||||||
# Read rest of file
|
# Read rest of file
|
||||||
|
|
@ -174,6 +187,19 @@ if [ -f "CHANGELOG.md" ]; then
|
||||||
fi
|
fi
|
||||||
echo -e "${GREEN}✅ CHANGELOG updated${NC}\n"
|
echo -e "${GREEN}✅ CHANGELOG updated${NC}\n"
|
||||||
|
|
||||||
|
# Step 6b: Update the releases wall entry — mechanical, derived from the
|
||||||
|
# CHANGELOG entry just composed. The fleet's HQ page reads open-brainy.json
|
||||||
|
# from the one shared releases repo, soulcraftlabs/releases on The Source —
|
||||||
|
# this used to be hand-written after every release (David: never again —
|
||||||
|
# make it a step of the rail, landed in the one shared home; this repo no
|
||||||
|
# longer hosts its own copy). This step clones/fetches that repo into a
|
||||||
|
# local cache, prepends the entry, and pushes it directly — a real
|
||||||
|
# cross-repo push, refusing loudly (never skipping) on any
|
||||||
|
# clone/validation/commit/push failure.
|
||||||
|
echo -e "${BLUE}5️⃣▸ Updating the releases wall...${NC}"
|
||||||
|
node scripts/wall-entry.mjs --product open-brainy --version "${NEW_VERSION}" --date "${RELEASE_DATE}" --from-changelog CHANGELOG.md
|
||||||
|
echo -e "${GREEN}✅ Releases wall updated${NC}\n"
|
||||||
|
|
||||||
# Step 7: Create release commit
|
# Step 7: Create release commit
|
||||||
echo -e "${BLUE}6️⃣ Creating release commit...${NC}"
|
echo -e "${BLUE}6️⃣ Creating release commit...${NC}"
|
||||||
git add package.json package-lock.json CHANGELOG.md
|
git add package.json package-lock.json CHANGELOG.md
|
||||||
|
|
@ -237,7 +263,7 @@ fi
|
||||||
# and RELEASES.md are the record; this just gives The Source's UI a release page).
|
# and RELEASES.md are the record; this just gives The Source's UI a release page).
|
||||||
echo -e "${BLUE}🔟 Creating release page on The Source...${NC}"
|
echo -e "${BLUE}🔟 Creating release page on The Source...${NC}"
|
||||||
if [ -n "${FORGEJO_RELEASE_TOKEN:-}" ]; then
|
if [ -n "${FORGEJO_RELEASE_TOKEN:-}" ]; then
|
||||||
if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraft/brainy/releases" \
|
if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraftlabs/open-brainy/releases" \
|
||||||
-H "Authorization: token ${FORGEJO_RELEASE_TOKEN}" -H "Content-Type: application/json" \
|
-H "Authorization: token ${FORGEJO_RELEASE_TOKEN}" -H "Content-Type: application/json" \
|
||||||
-d "{\"tag_name\":\"v${NEW_VERSION}\",\"name\":\"v${NEW_VERSION}\",\"prerelease\":${PRERELEASE}}" >/dev/null; then
|
-d "{\"tag_name\":\"v${NEW_VERSION}\",\"name\":\"v${NEW_VERSION}\",\"prerelease\":${PRERELEASE}}" >/dev/null; then
|
||||||
echo -e "${GREEN}✅ Release page created on The Source${NC}\n"
|
echo -e "${GREEN}✅ Release page created on The Source${NC}\n"
|
||||||
|
|
|
||||||
504
scripts/wall-entry.mjs
Normal file
504
scripts/wall-entry.mjs
Normal file
|
|
@ -0,0 +1,504 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* @module scripts/wall-entry
|
||||||
|
* @description The releases-wall entry, made mechanical. The fleet's HQ page
|
||||||
|
* reads one public JSON per product from the ONE releases repo on The Source
|
||||||
|
* (soulcraftlabs/releases, files <product>.json at its root — shape
|
||||||
|
* {product, entries:[{version, date, headline, items, url, thumb?}]}), at
|
||||||
|
* https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/<product>.json.
|
||||||
|
* Those entries were hand-written after every release, then briefly written
|
||||||
|
* into this repo's own releases/<product>.json; this script is the one door
|
||||||
|
* that composes an entry and lands it in the shared repo, so it is never
|
||||||
|
* hand-written and never forked across repos again.
|
||||||
|
*
|
||||||
|
* Two modes:
|
||||||
|
*
|
||||||
|
* 1. Generate + publish (default):
|
||||||
|
* node wall-entry.mjs --product <p> --version <v> --date <YYYY-MM-DD> \
|
||||||
|
* --from-changelog <CHANGELOG.md>
|
||||||
|
* Derives an entry from the CHANGELOG.md entry for <v> (headline = the
|
||||||
|
* entry's first bullet, items = every bullet, trimmed of its trailing
|
||||||
|
* commit hash), then:
|
||||||
|
* - clones (or, if a cached clone already exists, fetches and resets)
|
||||||
|
* the releases repo into a local cache directory,
|
||||||
|
* - prepends the entry to <cache>/<p>.json, newest first — replacing
|
||||||
|
* any existing entry for the same version so a re-run is idempotent,
|
||||||
|
* - validates the file's shape before and after,
|
||||||
|
* - commits the change as "chore(wall): <p> <v>" and pushes main.
|
||||||
|
* A failure at any step (clone, validation, commit, push, a
|
||||||
|
* non-fast-forward remote) exits non-zero naming the cure. Nothing is
|
||||||
|
* ever skipped — the wall either lands correctly or the release fails.
|
||||||
|
*
|
||||||
|
* 2. Dry run:
|
||||||
|
* node wall-entry.mjs --dry-run --product <p> --version <v> \
|
||||||
|
* --date <YYYY-MM-DD> --from-changelog <CHANGELOG.md>
|
||||||
|
* Derives the entry exactly as above and prints it, along with the file
|
||||||
|
* it would be written to, but touches no clone and no remote — usable
|
||||||
|
* from a fresh checkout with no cache and no network.
|
||||||
|
*
|
||||||
|
* 3. Validate only (--check):
|
||||||
|
* node wall-entry.mjs --check --file <path/to/product.json>
|
||||||
|
* Validates an arbitrary wall file's exact key set (top-level and
|
||||||
|
* per-entry), field types, and strict-descending semver ordering with
|
||||||
|
* no duplicates. Read-only; never writes. Exit 0 = clean, exit 1 =
|
||||||
|
* named violations printed to stderr.
|
||||||
|
*
|
||||||
|
* The remote and the local cache directory are each overridable
|
||||||
|
* (--remote / --cache-dir, or WALL_ENTRY_RELEASES_REMOTE /
|
||||||
|
* WALL_ENTRY_RELEASES_CACHE_DIR) so tests can point at a throwaway local
|
||||||
|
* bare repo and a throwaway cache directory — never the real remote or the
|
||||||
|
* real developer cache.
|
||||||
|
*
|
||||||
|
* No dependencies beyond the system `git` binary — CHANGELOG parsing,
|
||||||
|
* semver comparison, and JSON shape checking are all hand-rolled below.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs'
|
||||||
|
import { execFileSync } from 'node:child_process'
|
||||||
|
import { homedir } from 'node:os'
|
||||||
|
import { dirname, join } from 'node:path'
|
||||||
|
|
||||||
|
const DEFAULT_REMOTE = 'git@source.soulcraft.com:soulcraftlabs/releases.git'
|
||||||
|
|
||||||
|
/** @returns {string} */
|
||||||
|
function defaultCacheDir() {
|
||||||
|
const base = process.env.XDG_CACHE_HOME || join(homedir(), '.cache')
|
||||||
|
return join(base, 'soulcraft-releases')
|
||||||
|
}
|
||||||
|
|
||||||
|
// Required on every entry; "thumb" is optional (may be absent, or present as
|
||||||
|
// string | null) — matching the HQ contract's {..., thumb?}.
|
||||||
|
const ENTRY_REQUIRED_KEYS = ['version', 'date', 'headline', 'items', 'url']
|
||||||
|
const ENTRY_OPTIONAL_KEYS = ['thumb']
|
||||||
|
const ENTRY_ALLOWED_KEYS = [...ENTRY_REQUIRED_KEYS, ...ENTRY_OPTIONAL_KEYS]
|
||||||
|
const FILE_KEYS = ['product', 'entries']
|
||||||
|
|
||||||
|
// The public permalink pattern, by product. Every entry MUST carry an https
|
||||||
|
// permalink: HQ's parser rejects a wall whose entries carry url: null (the
|
||||||
|
// whole feed became unreadable on 2026-09-02). A product whose forge repo is
|
||||||
|
// private links its PUBLIC package page on The Source instead of a release
|
||||||
|
// page that would 404 for HQ's readers.
|
||||||
|
const RELEASE_URL_PATTERNS = {
|
||||||
|
'open-brainy': (version) => `https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v${version}`,
|
||||||
|
'brainy': (version) => `https://source.soulcraft.com/soulcraft/-/packages/npm/@soulcraft%2Fbrainy/${version}`,
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse argv into a flag map. `--flag value` sets a string; `--flag` alone
|
||||||
|
* (end of argv, or followed by another `--flag`) sets boolean true.
|
||||||
|
* @param {string[]} argv
|
||||||
|
* @returns {Record<string, string | true>}
|
||||||
|
*/
|
||||||
|
function parseArgs(argv) {
|
||||||
|
/** @type {Record<string, string | true>} */
|
||||||
|
const args = {}
|
||||||
|
for (let i = 0; i < argv.length; i++) {
|
||||||
|
const a = argv[i]
|
||||||
|
if (!a.startsWith('--')) continue
|
||||||
|
const key = a.slice(2)
|
||||||
|
const next = argv[i + 1]
|
||||||
|
if (next === undefined || next.startsWith('--')) {
|
||||||
|
args[key] = true
|
||||||
|
} else {
|
||||||
|
args[key] = next
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return args
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Print a loud, named error and exit 1. Every refusal in this script goes
|
||||||
|
* through here so the failure mode is always the same shape: "wall-entry: <what>".
|
||||||
|
* @param {string} message
|
||||||
|
* @returns {never}
|
||||||
|
*/
|
||||||
|
function fail(message) {
|
||||||
|
console.error(`wall-entry: ${message}`)
|
||||||
|
process.exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {string} version
|
||||||
|
* @returns {{major: number, minor: number, patch: number, pre: string | null} | null}
|
||||||
|
*/
|
||||||
|
function parseSemver(version) {
|
||||||
|
const m = /^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$/.exec(version)
|
||||||
|
if (!m) return null
|
||||||
|
return { major: Number(m[1]), minor: Number(m[2]), patch: Number(m[3]), pre: m[4] ?? null }
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {string} a
|
||||||
|
* @param {string} b
|
||||||
|
* @returns {number} positive if a > b, negative if a < b, 0 if equal.
|
||||||
|
*/
|
||||||
|
function compareSemver(a, b) {
|
||||||
|
const pa = parseSemver(a)
|
||||||
|
const pb = parseSemver(b)
|
||||||
|
if (!pa || !pb) throw new Error(`cannot compare non-semver versions "${a}" vs "${b}"`)
|
||||||
|
if (pa.major !== pb.major) return pa.major - pb.major
|
||||||
|
if (pa.minor !== pb.minor) return pa.minor - pb.minor
|
||||||
|
if (pa.patch !== pb.patch) return pa.patch - pb.patch
|
||||||
|
if (pa.pre === pb.pre) return 0
|
||||||
|
if (pa.pre === null) return 1 // a release outranks any prerelease of the same core version
|
||||||
|
if (pb.pre === null) return -1
|
||||||
|
return pa.pre < pb.pre ? -1 : pa.pre > pb.pre ? 1 : 0
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validate a wall file's full shape: top-level keys ("product", "entries" —
|
||||||
|
* no more, no less), per-entry keys and field types ("thumb" optional), and
|
||||||
|
* strict-descending semver ordering with no duplicates. Collects every
|
||||||
|
* violation instead of failing on the first, so a caller reports the whole
|
||||||
|
* picture in one pass.
|
||||||
|
* @param {unknown} data
|
||||||
|
* @returns {string[]} Violation messages; empty means the file is clean.
|
||||||
|
*/
|
||||||
|
function validateShape(data) {
|
||||||
|
/** @type {string[]} */
|
||||||
|
const errors = []
|
||||||
|
|
||||||
|
if (typeof data !== 'object' || data === null || Array.isArray(data)) {
|
||||||
|
return ['top level: expected a JSON object']
|
||||||
|
}
|
||||||
|
const obj = /** @type {Record<string, unknown>} */ (data)
|
||||||
|
|
||||||
|
const topKeys = Object.keys(obj)
|
||||||
|
const missingTop = FILE_KEYS.filter((k) => !(k in obj))
|
||||||
|
const extraTop = topKeys.filter((k) => !FILE_KEYS.includes(k))
|
||||||
|
if (missingTop.length) errors.push(`top level: missing key(s) ${missingTop.join(', ')}`)
|
||||||
|
if (extraTop.length) errors.push(`top level: unexpected key(s) ${extraTop.join(', ')}`)
|
||||||
|
|
||||||
|
if (typeof obj.product !== 'string' || obj.product.trim() === '') {
|
||||||
|
errors.push('top level: "product" must be a non-empty string')
|
||||||
|
}
|
||||||
|
if (!Array.isArray(obj.entries)) {
|
||||||
|
errors.push('top level: "entries" must be an array')
|
||||||
|
return errors // nothing further to check without an array
|
||||||
|
}
|
||||||
|
|
||||||
|
const entries = /** @type {unknown[]} */ (obj.entries)
|
||||||
|
entries.forEach((rawEntry, i) => {
|
||||||
|
const label = `entries[${i}]`
|
||||||
|
if (typeof rawEntry !== 'object' || rawEntry === null || Array.isArray(rawEntry)) {
|
||||||
|
errors.push(`${label}: expected an object`)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
const entry = /** @type {Record<string, unknown>} */ (rawEntry)
|
||||||
|
const keys = Object.keys(entry)
|
||||||
|
const missing = ENTRY_REQUIRED_KEYS.filter((k) => !(k in entry))
|
||||||
|
const extra = keys.filter((k) => !ENTRY_ALLOWED_KEYS.includes(k))
|
||||||
|
if (missing.length) errors.push(`${label}: missing key(s) ${missing.join(', ')}`)
|
||||||
|
if (extra.length) errors.push(`${label}: unexpected key(s) ${extra.join(', ')}`)
|
||||||
|
|
||||||
|
if (typeof entry.version !== 'string' || !parseSemver(entry.version)) {
|
||||||
|
errors.push(`${label}: "version" must be a semver string (got ${JSON.stringify(entry.version)})`)
|
||||||
|
}
|
||||||
|
if (typeof entry.date !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(entry.date) || Number.isNaN(Date.parse(entry.date))) {
|
||||||
|
errors.push(`${label}: "date" must be a YYYY-MM-DD string (got ${JSON.stringify(entry.date)})`)
|
||||||
|
}
|
||||||
|
if (typeof entry.headline !== 'string' || entry.headline.trim() === '') {
|
||||||
|
errors.push(`${label}: "headline" must be a non-empty string`)
|
||||||
|
}
|
||||||
|
if (!Array.isArray(entry.items) || entry.items.length === 0 || entry.items.some((it) => typeof it !== 'string' || it.trim() === '')) {
|
||||||
|
errors.push(`${label}: "items" must be a non-empty array of non-empty strings`)
|
||||||
|
}
|
||||||
|
if (typeof entry.url !== 'string' || !/^https:\/\/\S+$/.test(entry.url)) {
|
||||||
|
errors.push(`${label}: "url" must be an https permalink — never null; HQ's parser rejects the whole feed`)
|
||||||
|
}
|
||||||
|
if ('thumb' in entry && !(entry.thumb === null || typeof entry.thumb === 'string')) {
|
||||||
|
errors.push(`${label}: "thumb" must be a string or null when present`)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// Ordering: newest first, strictly descending, no duplicate versions —
|
||||||
|
// checked only over entries whose version parsed (a bad version is
|
||||||
|
// already reported above; comparing it too would just be noise).
|
||||||
|
const versioned = entries
|
||||||
|
.map((e, i) => ({ i, version: /** @type {any} */ (e)?.version }))
|
||||||
|
.filter((e) => typeof e.version === 'string' && parseSemver(e.version))
|
||||||
|
for (let i = 0; i < versioned.length - 1; i++) {
|
||||||
|
const a = versioned[i]
|
||||||
|
const b = versioned[i + 1]
|
||||||
|
const cmp = compareSemver(a.version, b.version)
|
||||||
|
if (cmp === 0) {
|
||||||
|
errors.push(`entries[${a.i}] and entries[${b.i}]: duplicate version ${a.version}`)
|
||||||
|
} else if (cmp < 0) {
|
||||||
|
errors.push(`entries[${a.i}] (${a.version}) sits above entries[${b.i}] (${b.version}) — not newest-first`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return errors
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract one version's entry body from a standard-version-style CHANGELOG.md
|
||||||
|
* (headings `### [version](url) (date)`, followed by `- bullet (hash)` lines
|
||||||
|
* until the next heading or EOF).
|
||||||
|
* @param {string} changelog
|
||||||
|
* @param {string} version
|
||||||
|
* @returns {string[]} Bullet lines, trimmed of their leading "- " and
|
||||||
|
* trailing " (hash)".
|
||||||
|
*/
|
||||||
|
function extractChangelogBullets(changelog, version) {
|
||||||
|
const lines = changelog.split('\n')
|
||||||
|
const headingRe = /^### \[([^\]]+)\]\(.*\)\s*\(\d{4}-\d{2}-\d{2}\)\s*$/
|
||||||
|
let start = -1
|
||||||
|
for (let i = 0; i < lines.length; i++) {
|
||||||
|
const m = headingRe.exec(lines[i])
|
||||||
|
if (m && m[1] === version) {
|
||||||
|
start = i + 1
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (start === -1) {
|
||||||
|
fail(
|
||||||
|
`version ${version} has no CHANGELOG entry yet — run this after the CHANGELOG step composes "### [${version}]", not before`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
/** @type {string[]} */
|
||||||
|
const bullets = []
|
||||||
|
for (let i = start; i < lines.length; i++) {
|
||||||
|
if (headingRe.test(lines[i])) break // next entry starts
|
||||||
|
const bulletMatch = /^- (.+?)(?:\s\(([0-9a-f]{6,40})\))?$/.exec(lines[i].trim())
|
||||||
|
if (lines[i].trim().startsWith('- ') && bulletMatch) {
|
||||||
|
const text = bulletMatch[1].trim()
|
||||||
|
if (text) bullets.push(text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (bullets.length === 0) {
|
||||||
|
fail(`version ${version}'s CHANGELOG entry has no bullets to derive a headline/items from`)
|
||||||
|
}
|
||||||
|
return bullets
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Derive a wall entry from a CHANGELOG.md.
|
||||||
|
* @param {{product: string, version: string, date: string, changelogPath: string, url?: string, thumb?: string | null}} opts
|
||||||
|
* @returns {{version: string, date: string, headline: string, items: string[], url: string, thumb: string | null}}
|
||||||
|
*/
|
||||||
|
function deriveEntry({ product, version, date, changelogPath, url, thumb }) {
|
||||||
|
if (!parseSemver(version)) fail(`--version "${version}" is not a semver string`)
|
||||||
|
if (!/^\d{4}-\d{2}-\d{2}$/.test(date) || Number.isNaN(Date.parse(date))) {
|
||||||
|
fail(`--date "${date}" is not a YYYY-MM-DD date`)
|
||||||
|
}
|
||||||
|
if (!existsSync(changelogPath)) fail(`--from-changelog "${changelogPath}" does not exist`)
|
||||||
|
|
||||||
|
const changelog = readFileSync(changelogPath, 'utf8')
|
||||||
|
const items = extractChangelogBullets(changelog, version)
|
||||||
|
const headline = items[0]
|
||||||
|
|
||||||
|
const pattern = RELEASE_URL_PATTERNS[product]
|
||||||
|
if (url === undefined && pattern === undefined) {
|
||||||
|
throw new Error(`wall-entry: no permalink pattern for product "${product}" — add one to RELEASE_URL_PATTERNS or pass --url; entries never carry url: null`)
|
||||||
|
}
|
||||||
|
const resolvedUrl = url !== undefined ? url : pattern(version)
|
||||||
|
const resolvedThumb = thumb !== undefined ? thumb : null
|
||||||
|
|
||||||
|
return { version, date, headline, items, url: resolvedUrl, thumb: resolvedThumb }
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Load and shape-validate a wall file.
|
||||||
|
* @param {string} filePath
|
||||||
|
* @returns {Record<string, any>}
|
||||||
|
*/
|
||||||
|
function loadWallFile(filePath) {
|
||||||
|
if (!existsSync(filePath)) fail(`"${filePath}" does not exist`)
|
||||||
|
/** @type {unknown} */
|
||||||
|
let data
|
||||||
|
try {
|
||||||
|
data = JSON.parse(readFileSync(filePath, 'utf8'))
|
||||||
|
} catch (err) {
|
||||||
|
fail(`"${filePath}" is not valid JSON: ${/** @type {Error} */ (err).message}`)
|
||||||
|
}
|
||||||
|
const errors = validateShape(data)
|
||||||
|
if (errors.length) {
|
||||||
|
fail(`"${filePath}" fails shape validation —\n ${errors.join('\n ')}`)
|
||||||
|
}
|
||||||
|
return /** @type {Record<string, any>} */ (data)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run a git command, throwing an Error whose message is git's own stderr
|
||||||
|
* (trimmed) on failure — every caller wraps this to name the cure.
|
||||||
|
* @param {string[]} args
|
||||||
|
* @param {string} cwd
|
||||||
|
* @returns {string} stdout, trimmed.
|
||||||
|
*/
|
||||||
|
function git(args, cwd) {
|
||||||
|
try {
|
||||||
|
return execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim()
|
||||||
|
} catch (err) {
|
||||||
|
const stderr = /** @type {any} */ (err).stderr
|
||||||
|
const message = (typeof stderr === 'string' && stderr.trim()) || /** @type {Error} */ (err).message
|
||||||
|
throw new Error(message)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ensure a clean, up-to-date local clone of the releases repo at
|
||||||
|
* `cacheDir`, checked out on `main` — cloning fresh if `cacheDir` has no
|
||||||
|
* `.git`, otherwise fetching and hard-resetting onto `origin/main` (so a
|
||||||
|
* stray local commit or edit left by a previous failed run can never leak
|
||||||
|
* into the next one).
|
||||||
|
* @param {string} remote
|
||||||
|
* @param {string} cacheDir
|
||||||
|
*/
|
||||||
|
function ensureReleasesClone(remote, cacheDir) {
|
||||||
|
if (existsSync(join(cacheDir, '.git'))) {
|
||||||
|
try {
|
||||||
|
git(['remote', 'set-url', 'origin', remote], cacheDir)
|
||||||
|
git(['fetch', '--prune', 'origin'], cacheDir)
|
||||||
|
git(['checkout', 'main'], cacheDir)
|
||||||
|
git(['reset', '--hard', 'origin/main'], cacheDir)
|
||||||
|
git(['clean', '-fd'], cacheDir)
|
||||||
|
} catch (err) {
|
||||||
|
fail(
|
||||||
|
`cannot refresh the cached releases checkout at "${cacheDir}" from "${remote}" — ${/** @type {Error} */ (err).message}\n` +
|
||||||
|
` cure: delete "${cacheDir}" and re-run so it re-clones from scratch, or confirm SSH access with "ssh -T git@source.soulcraft.com"`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
mkdirSync(dirname(cacheDir), { recursive: true })
|
||||||
|
try {
|
||||||
|
git(['clone', remote, cacheDir], dirname(cacheDir))
|
||||||
|
} catch (err) {
|
||||||
|
fail(
|
||||||
|
`cannot clone "${remote}" — ${/** @type {Error} */ (err).message}\n` +
|
||||||
|
` cure: confirm SSH access with "ssh -T git@source.soulcraft.com" and that the soulcraftlabs/releases repo exists yet`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
git(['checkout', 'main'], cacheDir)
|
||||||
|
} catch (err) {
|
||||||
|
fail(
|
||||||
|
`cloned "${remote}" into "${cacheDir}" but could not check out "main" — ${/** @type {Error} */ (err).message}\n` +
|
||||||
|
` cure: confirm the releases repo's default branch is named "main"`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Prepend `entry` to the wall at `<cacheDir>/<product>.json`, replacing any
|
||||||
|
* existing entry for the same version (idempotent re-runs), validating
|
||||||
|
* before and after, committing, and pushing — or refusing loudly, naming
|
||||||
|
* the cure, at whichever step fails.
|
||||||
|
* @param {{version: string, date: string, headline: string, items: string[], url: string, thumb: string | null}} entry
|
||||||
|
* @param {string} product
|
||||||
|
* @param {string} remote
|
||||||
|
* @param {string} cacheDir
|
||||||
|
*/
|
||||||
|
function publishEntry(entry, product, remote, cacheDir) {
|
||||||
|
ensureReleasesClone(remote, cacheDir)
|
||||||
|
|
||||||
|
const filePath = join(cacheDir, `${product}.json`)
|
||||||
|
if (!existsSync(filePath)) {
|
||||||
|
fail(
|
||||||
|
`"${filePath}" does not exist in the releases repo — cure: seed "${product}.json" at the repo root first (it must exist before any release rail can prepend to it)`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
const wall = loadWallFile(filePath)
|
||||||
|
|
||||||
|
if (wall.product !== product) {
|
||||||
|
fail(`"${filePath}" has product "${wall.product}", but --product "${product}" was given — refusing a cross-product write`)
|
||||||
|
}
|
||||||
|
|
||||||
|
const replacing = wall.entries.some((e) => e.version === entry.version)
|
||||||
|
wall.entries = [entry, ...wall.entries.filter((e) => e.version !== entry.version)]
|
||||||
|
|
||||||
|
const postErrors = validateShape(wall)
|
||||||
|
if (postErrors.length) {
|
||||||
|
fail(`the entry for ${entry.version} would leave "${filePath}" invalid —\n ${postErrors.join('\n ')}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
writeFileSync(filePath, JSON.stringify(wall, null, 2) + '\n', 'utf8')
|
||||||
|
|
||||||
|
const status = git(['status', '--porcelain', '--', `${product}.json`], cacheDir)
|
||||||
|
if (status === '') {
|
||||||
|
console.log(`wall-entry: "${product}.json" already carries an identical entry for ${entry.version} — nothing to commit or push`)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
git(['add', `${product}.json`], cacheDir)
|
||||||
|
git(['commit', '-m', `chore(wall): ${product} ${entry.version}`], cacheDir)
|
||||||
|
} catch (err) {
|
||||||
|
fail(`cannot commit the wall entry in "${cacheDir}" — ${/** @type {Error} */ (err).message}\n cure: inspect "${cacheDir}" by hand and re-run once its git state is clean`)
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
git(['push', 'origin', 'main'], cacheDir)
|
||||||
|
} catch (err) {
|
||||||
|
fail(
|
||||||
|
`push to "${remote}" failed (likely a non-fast-forward — another release landed on main first) — ${/** @type {Error} */ (err).message}\n` +
|
||||||
|
` cure: re-run this release step; it re-fetches and resets onto the latest origin/main before retrying`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
const sha = git(['rev-parse', 'HEAD'], cacheDir)
|
||||||
|
console.log(
|
||||||
|
`wall-entry: ${replacing ? 'replaced' : 'wrote'} v${entry.version} in "${product}.json" (${wall.entries.length} entries, newest first) — pushed ${sha} to ${remote} main`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
function main() {
|
||||||
|
const args = parseArgs(process.argv.slice(2))
|
||||||
|
|
||||||
|
if (args.check) {
|
||||||
|
const filePath = /** @type {string | undefined} */ (args.file)
|
||||||
|
if (!filePath) fail('--check needs --file <path>')
|
||||||
|
const wall = loadWallFile(/** @type {string} */ (filePath))
|
||||||
|
console.log(`wall-entry --check: "${filePath}" OK — product "${wall.product}", ${wall.entries.length} entries, newest-first, no duplicates`)
|
||||||
|
process.exit(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate mode (default, also covers --dry-run): --product, --version,
|
||||||
|
// --date, --from-changelog required.
|
||||||
|
const product = /** @type {string | undefined} */ (args.product)
|
||||||
|
const version = /** @type {string | undefined} */ (args.version)
|
||||||
|
const date = /** @type {string | undefined} */ (args.date)
|
||||||
|
const fromChangelog = /** @type {string | undefined} */ (args['from-changelog'])
|
||||||
|
|
||||||
|
const missing = []
|
||||||
|
if (!product) missing.push('--product')
|
||||||
|
if (!version) missing.push('--version')
|
||||||
|
if (!date) missing.push('--date')
|
||||||
|
if (!fromChangelog) missing.push('--from-changelog')
|
||||||
|
if (missing.length) {
|
||||||
|
fail(
|
||||||
|
`missing required flag(s): ${missing.join(', ')}\n` +
|
||||||
|
'Usage:\n' +
|
||||||
|
' wall-entry.mjs --product <p> --version <v> --date <YYYY-MM-DD> --from-changelog <CHANGELOG.md> [--dry-run]\n' +
|
||||||
|
' wall-entry.mjs --check --file <path/to/product.json>',
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
const urlArg = args.url === true ? undefined : /** @type {string | undefined} */ (args.url)
|
||||||
|
const thumbArg = args.thumb === true ? undefined : /** @type {string | undefined} */ (args.thumb)
|
||||||
|
|
||||||
|
const entry = deriveEntry({
|
||||||
|
product: /** @type {string} */ (product),
|
||||||
|
version: /** @type {string} */ (version),
|
||||||
|
date: /** @type {string} */ (date),
|
||||||
|
changelogPath: /** @type {string} */ (fromChangelog),
|
||||||
|
url: urlArg,
|
||||||
|
thumb: thumbArg,
|
||||||
|
})
|
||||||
|
|
||||||
|
const remote = /** @type {string} */ (args.remote ?? process.env.WALL_ENTRY_RELEASES_REMOTE ?? DEFAULT_REMOTE)
|
||||||
|
const cacheDir = /** @type {string} */ (args['cache-dir'] ?? process.env.WALL_ENTRY_RELEASES_CACHE_DIR ?? defaultCacheDir())
|
||||||
|
|
||||||
|
if (args['dry-run']) {
|
||||||
|
console.log(`wall-entry --dry-run: would write to "${join(cacheDir, `${product}.json`)}" in ${remote} (main), pushed as "chore(wall): ${product} ${version}"`)
|
||||||
|
console.log(JSON.stringify(entry, null, 2))
|
||||||
|
process.exit(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
publishEntry(entry, /** @type {string} */ (product), remote, cacheDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
main()
|
||||||
1891
src/brainy.ts
1891
src/brainy.ts
File diff suppressed because it is too large
Load diff
|
|
@ -351,3 +351,63 @@ export class PendingFlushDurabilityError extends Error {
|
||||||
this.failedAttempts = failedAttempts
|
this.failedAttempts = failedAttempts
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Thrown by {@link GenerationStore.commitTransaction} when the
|
||||||
|
* PENDING single-op tier is non-empty — i.e. one or more `commitSingleOp()`
|
||||||
|
* generations are buffered in memory, not yet flushed to
|
||||||
|
* `committedRanges` via `flushPendingSingleOps()`.
|
||||||
|
*
|
||||||
|
* The invariant `reservedGensAsc()` (and everything built on it —
|
||||||
|
* `resolveManyAt`, `resolveAt`, `changedBetween`, the hot-tail window) relies
|
||||||
|
* on is documented, not enforced by types: pending generations must always be
|
||||||
|
* numerically greater than every committed one, because the ONLY sanctioned
|
||||||
|
* callers of `commitTransaction()` — `Brainy.transact()` and
|
||||||
|
* `Brainy.compactHistory()` — flush the pending tier FIRST. A caller that
|
||||||
|
* invokes `commitTransaction()` directly while single-ops are still pending
|
||||||
|
* breaks that invariant: the new commit lands in `committedRanges` ABOVE
|
||||||
|
* generations still sitting in `pendingGens`, so the committed-then-pending
|
||||||
|
* concatenation `reservedGensAsc()` yields is no longer ascending. The
|
||||||
|
* concrete failure this produces is silent, not a crash: `resolveManyAt`
|
||||||
|
* walks committed ranges before pending ones, so it can report a NEWER
|
||||||
|
* generation as the "first after" a pin than an older, still-pending one that
|
||||||
|
* actually touched the id first — a wrong before-image at a point-in-time
|
||||||
|
* read, without a compensating error to warn a caller anything went wrong.
|
||||||
|
*
|
||||||
|
* This error refuses the commit outright, before any staging I/O: nothing is
|
||||||
|
* written, the generation counter reservation is untouched, and
|
||||||
|
* `committedRanges`/`pendingGens` are exactly as they were. Call
|
||||||
|
* `flushPendingSingleOps()` first (or go through `Brainy.transact()`, which
|
||||||
|
* already does).
|
||||||
|
*
|
||||||
|
* @example
|
||||||
|
* try {
|
||||||
|
* await generationStore.commitTransaction({ touched, execute })
|
||||||
|
* } catch (err) {
|
||||||
|
* if (err instanceof PendingSingleOpsUnflushedError) {
|
||||||
|
* await generationStore.flushPendingSingleOps()
|
||||||
|
* await generationStore.commitTransaction({ touched, execute }) // now safe
|
||||||
|
* }
|
||||||
|
* }
|
||||||
|
*/
|
||||||
|
export class PendingSingleOpsUnflushedError extends Error {
|
||||||
|
/** How many un-flushed single-op generations were buffered at refusal time. */
|
||||||
|
public readonly pendingCount: number
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param pendingCount - `pendingGens.length` at the moment of refusal (always ≥ 1).
|
||||||
|
*/
|
||||||
|
constructor(pendingCount: number) {
|
||||||
|
super(
|
||||||
|
`commitTransaction() refused: ${pendingCount} pending single-op generation(s) ` +
|
||||||
|
`are still buffered and un-flushed. Flush the pending single-op tier before ` +
|
||||||
|
`committing a transaction — Brainy.transact() does this automatically; a ` +
|
||||||
|
`direct commitTransaction() call with pending generations would leave the ` +
|
||||||
|
`generation order unsorted (committed generations landing above lower, ` +
|
||||||
|
`still-pending ones) and make point-in-time reads (resolveManyAt/resolveAt) ` +
|
||||||
|
`return the wrong before-image. Call flushPendingSingleOps() first, then retry.`
|
||||||
|
)
|
||||||
|
this.name = 'PendingSingleOpsUnflushedError'
|
||||||
|
this.pendingCount = pendingCount
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -40,7 +40,10 @@
|
||||||
* The manifest (`_generations/facts/manifest.json`, JSON — forensics stay
|
* The manifest (`_generations/facts/manifest.json`, JSON — forensics stay
|
||||||
* terminal-readable) is the single source of truth for the segment SET;
|
* terminal-readable) is the single source of truth for the segment SET;
|
||||||
* rotation flips it atomically (write-new → fsync → rename) BEFORE the new
|
* rotation flips it atomically (write-new → fsync → rename) BEFORE the new
|
||||||
* tail's first byte exists, so no segment file is ever unaccounted for.
|
* tail's first byte exists, so no segment file is ever unaccounted for. Its
|
||||||
|
* per-segment `firstGeneration`/`lastGeneration` are LOAD-BEARING at open: a
|
||||||
|
* recovery pass looking for facts above a bound reads only the segments those
|
||||||
|
* bounds cannot rule out (the prune law — see `segmentsHoldingFactsAbove`).
|
||||||
*
|
*
|
||||||
* ## Mixed-version logs (the v2 live-write cutover)
|
* ## Mixed-version logs (the v2 live-write cutover)
|
||||||
*
|
*
|
||||||
|
|
@ -689,6 +692,74 @@ function parseSegment(
|
||||||
return { facts, validBytes: offset, formatVersion: FACT_LOG_FORMAT_V1 }
|
return { facts, validBytes: offset, formatVersion: FACT_LOG_FORMAT_V1 }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* THE PRUNE LAW — which segment files a pass looking for facts ABOVE
|
||||||
|
* `committedGeneration` actually has to read, and how many the manifest's own
|
||||||
|
* recorded bounds took off the table.
|
||||||
|
*
|
||||||
|
* A sealed segment's `lastGeneration` is written at SEAL time and never
|
||||||
|
* mutated upward afterwards ({@link FactLog.rotate}, unchanged since the log
|
||||||
|
* was introduced): the tail's bytes are fsynced FIRST (`await this.sync()` —
|
||||||
|
* "sealed segments are always fully durable"), the entry is then built from
|
||||||
|
* the content that fsync covered, and only then does the manifest flip —
|
||||||
|
* atomically (tmp+rename) and fsynced — which in the SAME write re-points
|
||||||
|
* `tailSegment` at a new file, so the sealed file is never appended to again.
|
||||||
|
* A crash anywhere in that order is safe in the pruning direction: crash
|
||||||
|
* before the manifest write and the segment is still the TAIL (read whole);
|
||||||
|
* crash after it and the entry describes bytes that were already durable. The
|
||||||
|
* only later mutation of a sealed segment is `open()`'s straddle truncation,
|
||||||
|
* which REMOVES facts and re-derives the entry from the actual bytes — so a
|
||||||
|
* recorded bound can drift DOWN with its file, never up.
|
||||||
|
*
|
||||||
|
* Therefore: `lastGeneration = L` proves the file holds no fact above L, and
|
||||||
|
* a pass above `committedGeneration >= L` can skip it whole — no read, no
|
||||||
|
* CRC decode, no msgpack. What the manifest cannot PROVE is never pruned: an
|
||||||
|
* entry with no numeric `lastGeneration` (a legacy or hand-repaired manifest)
|
||||||
|
* is read, and the unsealed tail is always read.
|
||||||
|
*
|
||||||
|
* This is the difference between an open that costs O(whole fact log) and one
|
||||||
|
* that costs O(the facts that could matter). MEASURED in production: a 16k-row
|
||||||
|
* brain at generation ~478,819 paid 34-37s of segment reads and CRC decoding
|
||||||
|
* in `generation-store-open-fold` on EVERY open — to answer a question whose
|
||||||
|
* answer, after a clean close, is always "nothing".
|
||||||
|
*/
|
||||||
|
function segmentsHoldingFactsAbove(
|
||||||
|
stored: FactsManifest,
|
||||||
|
committedGeneration: number
|
||||||
|
): { files: string[]; pruned: number } {
|
||||||
|
const files: string[] = []
|
||||||
|
let pruned = 0
|
||||||
|
for (const entry of stored.segments) {
|
||||||
|
const last = (entry as Partial<SegmentEntry>).lastGeneration
|
||||||
|
if (typeof last === 'number' && Number.isFinite(last) && last <= committedGeneration) {
|
||||||
|
pruned++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
files.push(entry.file)
|
||||||
|
}
|
||||||
|
if (stored.tailSegment) files.push(stored.tailSegment)
|
||||||
|
return { files, pruned }
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Say what the open actually read. One line, and only when the log holds more
|
||||||
|
* than one segment (a single-segment log has nothing to prune and nothing to
|
||||||
|
* report) — the operator's receipt that the open is paying for the tail, not
|
||||||
|
* for the whole history.
|
||||||
|
*/
|
||||||
|
function narrateAboveScan(
|
||||||
|
pass: string,
|
||||||
|
committedGeneration: number,
|
||||||
|
read: number,
|
||||||
|
pruned: number
|
||||||
|
): void {
|
||||||
|
if (read + pruned <= 1) return
|
||||||
|
prodLog.narrate(
|
||||||
|
`[FactLog] ${pass} above generation ${committedGeneration}: ${read} segment(s) read, ` +
|
||||||
|
`${pruned} pruned of ${read + pruned} (sealed at or below the bound)`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The generation fact log. One instance per open store; every method assumes
|
* The generation fact log. One instance per open store; every method assumes
|
||||||
* the single-writer discipline the generation store already enforces (calls
|
* the single-writer discipline the generation store already enforces (calls
|
||||||
|
|
@ -754,22 +825,6 @@ export class FactLog {
|
||||||
return this.manifest.brainId !== undefined || this.tailVersion === FACT_LOG_FORMAT_V2
|
return this.manifest.brainId !== undefined || this.tailVersion === FACT_LOG_FORMAT_V2
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Open the log and reconcile it to committed truth: read the manifest,
|
|
||||||
* establish the tail's intact content (torn-tail scan), then TRUNCATE any
|
|
||||||
* fact with `generation > committedGeneration` — those never committed (a
|
|
||||||
* crash between fact-append and the commit point). After open, the log is
|
|
||||||
* exactly the committed prefix.
|
|
||||||
*/
|
|
||||||
/**
|
|
||||||
* Read (without truncating) every intact fact ABOVE a generation — the
|
|
||||||
* log-authority recovery surface: after a crash, facts beyond the
|
|
||||||
* manifest watermark that survived with valid CRCs are ACKED writes in
|
|
||||||
* durable-at-ack mode, and the owner REPLAYS them instead of letting
|
|
||||||
* open() truncate them. Must be called BEFORE open() (it reads the raw
|
|
||||||
* segments directly; the torn tail's invalid suffix is ignored exactly
|
|
||||||
* like open() would).
|
|
||||||
*/
|
|
||||||
/**
|
/**
|
||||||
* STREAMING twin of {@link FactLog.peekFactsAbove} for the recovery fold:
|
* STREAMING twin of {@link FactLog.peekFactsAbove} for the recovery fold:
|
||||||
* yields facts above the bound one SEGMENT at a time, ascending, without
|
* yields facts above the bound one SEGMENT at a time, ascending, without
|
||||||
|
|
@ -779,13 +834,18 @@ export class FactLog {
|
||||||
* Works manifest-direct (safe before {@link FactLog.open}). Ordering is
|
* Works manifest-direct (safe before {@link FactLog.open}). Ordering is
|
||||||
* structural (segments rotate in order; appends are ordered within one) and
|
* structural (segments rotate in order; appends are ordered within one) and
|
||||||
* ASSERTED — a violation aborts loudly, never a silent misordered replay.
|
* ASSERTED — a violation aborts loudly, never a silent misordered replay.
|
||||||
|
*
|
||||||
|
* Reads only the segments that CAN hold a fact above the bound — see
|
||||||
|
* {@link segmentsHoldingFactsAbove}. A bounded fold above a high checkpoint
|
||||||
|
* therefore reads its own tail, not the whole history it already proved
|
||||||
|
* durable.
|
||||||
*/
|
*/
|
||||||
async *streamFactsAbove(committedGeneration: number): AsyncGenerator<CommitFact[], void> {
|
async *streamFactsAbove(committedGeneration: number): AsyncGenerator<CommitFact[], void> {
|
||||||
const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null
|
const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null
|
||||||
if (!stored || typeof stored !== 'object' || !Array.isArray(stored.segments)) return
|
if (!stored || typeof stored !== 'object' || !Array.isArray(stored.segments)) return
|
||||||
if (stored.formatVersion !== FACTS_FORMAT_VERSION) return
|
if (stored.formatVersion !== FACTS_FORMAT_VERSION) return
|
||||||
const files = [...stored.segments.map((s) => s.file)]
|
const { files, pruned } = segmentsHoldingFactsAbove(stored, committedGeneration)
|
||||||
if (stored.tailSegment) files.push(stored.tailSegment)
|
narrateAboveScan('recovery fold', committedGeneration, files.length, pruned)
|
||||||
let lastGen = committedGeneration
|
let lastGen = committedGeneration
|
||||||
for (const file of files) {
|
for (const file of files) {
|
||||||
const bytes = await this.storage.readRawBytes(`${FACTS_PREFIX}/${file}`)
|
const bytes = await this.storage.readRawBytes(`${FACTS_PREFIX}/${file}`)
|
||||||
|
|
@ -807,13 +867,27 @@ export class FactLog {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read (without truncating) every intact fact ABOVE a generation — the
|
||||||
|
* log-authority recovery surface: after a crash, facts beyond the
|
||||||
|
* manifest watermark that survived with valid CRCs are ACKED writes in
|
||||||
|
* durable-at-ack mode, and the owner REPLAYS them instead of letting
|
||||||
|
* open() truncate them. Must be called BEFORE open() (it reads the raw
|
||||||
|
* segments directly; the torn tail's invalid suffix is ignored exactly
|
||||||
|
* like open() would).
|
||||||
|
*
|
||||||
|
* Reads only the segments that CAN hold such a fact — see
|
||||||
|
* {@link segmentsHoldingFactsAbove}. This runs on EVERY log-authority open,
|
||||||
|
* including the clean one where the answer is always empty, so the segments
|
||||||
|
* the manifest already proves irrelevant are never opened at all.
|
||||||
|
*/
|
||||||
async peekFactsAbove(committedGeneration: number): Promise<CommitFact[]> {
|
async peekFactsAbove(committedGeneration: number): Promise<CommitFact[]> {
|
||||||
const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null
|
const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null
|
||||||
if (!stored || typeof stored !== 'object' || !Array.isArray(stored.segments)) return []
|
if (!stored || typeof stored !== 'object' || !Array.isArray(stored.segments)) return []
|
||||||
if (stored.formatVersion !== FACTS_FORMAT_VERSION) return []
|
if (stored.formatVersion !== FACTS_FORMAT_VERSION) return []
|
||||||
const out: CommitFact[] = []
|
const out: CommitFact[] = []
|
||||||
const files = [...stored.segments.map((s) => s.file)]
|
const { files, pruned } = segmentsHoldingFactsAbove(stored, committedGeneration)
|
||||||
if (stored.tailSegment) files.push(stored.tailSegment)
|
narrateAboveScan('above-manifest peek', committedGeneration, files.length, pruned)
|
||||||
for (const file of files) {
|
for (const file of files) {
|
||||||
const bytes = await this.storage.readRawBytes(`${FACTS_PREFIX}/${file}`)
|
const bytes = await this.storage.readRawBytes(`${FACTS_PREFIX}/${file}`)
|
||||||
if (bytes === null) continue
|
if (bytes === null) continue
|
||||||
|
|
@ -826,6 +900,13 @@ export class FactLog {
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Open the log and reconcile it to committed truth: read the manifest,
|
||||||
|
* establish the tail's intact content (torn-tail scan), then TRUNCATE any
|
||||||
|
* fact with `generation > committedGeneration` — those never committed (a
|
||||||
|
* crash between fact-append and the commit point). After open, the log is
|
||||||
|
* exactly the committed prefix.
|
||||||
|
*/
|
||||||
async open(committedGeneration: number): Promise<void> {
|
async open(committedGeneration: number): Promise<void> {
|
||||||
const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null
|
const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null
|
||||||
if (stored && typeof stored === 'object' && Array.isArray(stored.segments)) {
|
if (stored && typeof stored === 'object' && Array.isArray(stored.segments)) {
|
||||||
|
|
|
||||||
|
|
@ -32,7 +32,13 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { prodLog } from '../utils/logger.js'
|
import { prodLog } from '../utils/logger.js'
|
||||||
import { GenerationCompactedError, GenerationConflictError, PendingFlushDurabilityError, StoreInconsistentError } from './errors.js'
|
import {
|
||||||
|
GenerationCompactedError,
|
||||||
|
GenerationConflictError,
|
||||||
|
PendingFlushDurabilityError,
|
||||||
|
PendingSingleOpsUnflushedError,
|
||||||
|
StoreInconsistentError
|
||||||
|
} from './errors.js'
|
||||||
import type { UnreconciledRecord } from './errors.js'
|
import type { UnreconciledRecord } from './errors.js'
|
||||||
import { TransactionRollbackError } from '../transaction/errors.js'
|
import { TransactionRollbackError } from '../transaction/errors.js'
|
||||||
import type {
|
import type {
|
||||||
|
|
@ -799,7 +805,16 @@ export class GenerationStore {
|
||||||
if (uncleanOpen) await this.advanceFoldCheckpointUnlocked()
|
if (uncleanOpen) await this.advanceFoldCheckpointUnlocked()
|
||||||
// The marker is consumed: any session that can write invalidates it
|
// The marker is consumed: any session that can write invalidates it
|
||||||
// at first commit (see the commit paths); a clean close re-writes it.
|
// at first commit (see the commit paths); a clean close re-writes it.
|
||||||
await this.clearCleanShutdownMarker()
|
// A READER NEVER CONSUMES IT. The marker is the writer's own evidence
|
||||||
|
// about the writer's own process — clearing it here exists so that
|
||||||
|
// if THIS session goes on to write and then dies before its next
|
||||||
|
// clean close, the marker's absence correctly reads as unclean. A
|
||||||
|
// reader can never write, so it can never leave the store in a state
|
||||||
|
// its own crash would mis-describe; clearing the marker for it would
|
||||||
|
// only cost the store's actual writer a needless whole-log fold on
|
||||||
|
// its next open, for a generation the reader merely observed. Leave
|
||||||
|
// `_system/` exactly as found.
|
||||||
|
if (!options?.readOnly) await this.clearCleanShutdownMarker()
|
||||||
}
|
}
|
||||||
await this.factLog.open(this.committed)
|
await this.factLog.open(this.committed)
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -889,7 +904,11 @@ export class GenerationStore {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Consume the clean-shutdown marker (every open; a clean close re-writes it). */
|
/**
|
||||||
|
* Consume the clean-shutdown marker (every WRITER open; a clean close
|
||||||
|
* re-writes it). Callers must gate this on `!options.readOnly` — a reader
|
||||||
|
* never consumes the marker, see the call site in {@link open}.
|
||||||
|
*/
|
||||||
private async clearCleanShutdownMarker(): Promise<void> {
|
private async clearCleanShutdownMarker(): Promise<void> {
|
||||||
try {
|
try {
|
||||||
await this.storage.deleteRawObject(CLEAN_SHUTDOWN_PATH)
|
await this.storage.deleteRawObject(CLEAN_SHUTDOWN_PATH)
|
||||||
|
|
@ -1351,6 +1370,9 @@ export class GenerationStore {
|
||||||
* @param args.execute - Runs the planned operation batch atomically.
|
* @param args.execute - Runs the planned operation batch atomically.
|
||||||
* @returns The committed generation and its commit timestamp.
|
* @returns The committed generation and its commit timestamp.
|
||||||
* @throws GenerationConflictError when the CAS expectation fails.
|
* @throws GenerationConflictError when the CAS expectation fails.
|
||||||
|
* @throws PendingSingleOpsUnflushedError when the pending single-op tier is
|
||||||
|
* non-empty — call `flushPendingSingleOps()` first (both `Brainy.transact()`
|
||||||
|
* and `Brainy.compactHistory()` already do).
|
||||||
*/
|
*/
|
||||||
/**
|
/**
|
||||||
* The generation fact log, or `null` when the storage layer cannot host one.
|
* The generation fact log, or `null` when the storage layer cannot host one.
|
||||||
|
|
@ -1425,6 +1447,13 @@ export class GenerationStore {
|
||||||
execute: () => Promise<void>
|
execute: () => Promise<void>
|
||||||
}): Promise<{ generation: number; timestamp: number }> {
|
}): Promise<{ generation: number; timestamp: number }> {
|
||||||
return this.withMutex(async () => {
|
return this.withMutex(async () => {
|
||||||
|
// The generation-order guard (see assertPendingSingleOpsFlushed): a
|
||||||
|
// direct commitTransaction() call while single-ops are still pending
|
||||||
|
// would commit above them, unsorting reservedGensAsc() and corrupting
|
||||||
|
// point-in-time reads. Both sanctioned callers (Brainy.transact(),
|
||||||
|
// Brainy.compactHistory()) already flush first, so this is
|
||||||
|
// behavior-neutral on every real path.
|
||||||
|
this.assertPendingSingleOpsFlushed()
|
||||||
// A latched history-durability failure compromises the whole generation
|
// A latched history-durability failure compromises the whole generation
|
||||||
// chain — refuse a transact too (advancing the manifest past stuck,
|
// chain — refuse a transact too (advancing the manifest past stuck,
|
||||||
// un-durable single-op generations would be inconsistent). Same loud
|
// un-durable single-op generations would be inconsistent). Same loud
|
||||||
|
|
@ -2294,6 +2323,37 @@ export class GenerationStore {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Throw if the pending single-op tier is non-empty. Called at
|
||||||
|
* the top of {@link commitTransaction} (the ONLY method that appends a
|
||||||
|
* fresh commit directly into {@link committedRanges} outside recovery) so
|
||||||
|
* the ordering invariant {@link reservedGensAsc}'s own doc comment states —
|
||||||
|
* "pending generations are always greater than every committed one" — is
|
||||||
|
* ENFORCED there rather than merely assumed.
|
||||||
|
*
|
||||||
|
* That invariant holds today only because both sanctioned callers flush the
|
||||||
|
* pending tier before committing: `Brainy.transact()` (src/brainy.ts,
|
||||||
|
* `await this.generationStore.flushPendingSingleOps()` immediately before
|
||||||
|
* its `commitTransaction()` call) and `Brainy.compactHistory()`
|
||||||
|
* (src/brainy.ts, the same flush immediately before its `compact()` call —
|
||||||
|
* `compact()` itself only ever RECLAIMS an existing committed prefix, so it
|
||||||
|
* cannot land a commit out of order and needs no guard of its own). A
|
||||||
|
* caller that reaches `commitTransaction()` by any other path — bypassing
|
||||||
|
* that flush — would commit a new generation into `committedRanges` ABOVE
|
||||||
|
* generations still sitting in `pendingGens`, breaking `reservedGensAsc`'s
|
||||||
|
* "committed-then-pending is already sorted" assumption and making
|
||||||
|
* `resolveManyAt`'s single ascending pass (and `resolveAt`'s consumers)
|
||||||
|
* return the WRONG before-image for a point-in-time read — silently, no
|
||||||
|
* compensating error. Refusing here, before any staging I/O, keeps the
|
||||||
|
* store untouched (nothing committed, nothing staged, the generation
|
||||||
|
* counter reservation unaffected) on every path that already flushes.
|
||||||
|
*/
|
||||||
|
private assertPendingSingleOpsFlushed(): void {
|
||||||
|
if (this.pendingGens.length > 0) {
|
||||||
|
throw new PendingSingleOpsUnflushedError(this.pendingGens.length)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** Schedule a coalesced pending-tier flush (size trigger fires immediately on
|
/** Schedule a coalesced pending-tier flush (size trigger fires immediately on
|
||||||
* the next microtask; otherwise a {@link PENDING_FLUSH_DELAY_MS} timer). Both
|
* the next microtask; otherwise a {@link PENDING_FLUSH_DELAY_MS} timer). Both
|
||||||
* defer outside the current mutex section so the flush can re-acquire it. A
|
* defer outside the current mutex section so the flush can re-acquire it. A
|
||||||
|
|
@ -2377,6 +2437,13 @@ export class GenerationStore {
|
||||||
* committed-then-pending concatenation is already sorted — identical to the old
|
* committed-then-pending concatenation is already sorted — identical to the old
|
||||||
* `[...committedGens, ...pendingGens]`. This is the union historical reads
|
* `[...committedGens, ...pendingGens]`. This is the union historical reads
|
||||||
* resolve over so un-flushed single-ops are visible to pins/`asOf`.
|
* resolve over so un-flushed single-ops are visible to pins/`asOf`.
|
||||||
|
*
|
||||||
|
* The "flush first" half of that invariant is ENFORCED, not just documented:
|
||||||
|
* {@link commitTransaction} — the only method that lands a fresh commit into
|
||||||
|
* {@link committedRanges} outside crash recovery — refuses via
|
||||||
|
* {@link assertPendingSingleOpsFlushed} whenever {@link pendingGens} is
|
||||||
|
* non-empty, so a committed generation can never land above a still-pending
|
||||||
|
* one and break this ordering.
|
||||||
*/
|
*/
|
||||||
private *reservedGensAsc(): IterableIterator<number> {
|
private *reservedGensAsc(): IterableIterator<number> {
|
||||||
yield* this.committedGensAsc()
|
yield* this.committedGensAsc()
|
||||||
|
|
|
||||||
|
|
@ -405,3 +405,73 @@ export class MigrationInProgressError extends BrainyError {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* THE INDEXABLE-ARRAY BOUND. An array-valued metadata field indexes one posting
|
||||||
|
* per element, so an unbounded array is an unbounded write — a 384-float
|
||||||
|
* embedding parked in the metadata bag would mint 384 postings for one row.
|
||||||
|
* The bound exists to keep that out of the index.
|
||||||
|
*
|
||||||
|
* 256 is hardcoded on purpose (the zero-config law: no knob). It sits far above
|
||||||
|
* every legitimate multi-value field the engine has seen — tags, authors,
|
||||||
|
* categories, labels, keyword lists, participant lists — and still below the
|
||||||
|
* narrowest embedding this engine will ever meet (384 dimensions, the smallest
|
||||||
|
* model it ships), so the two populations do not overlap and no caller has to
|
||||||
|
* tune it. A vector parked in metadata is refused; a long keyword list is not.
|
||||||
|
*
|
||||||
|
* It replaces a limit of 10 that was applied SILENTLY: a row whose `tags` array
|
||||||
|
* held eleven entries had that field skipped entirely and dropped out of every
|
||||||
|
* filtered search on it, with no error, no warning and no way to tell the
|
||||||
|
* difference from "no row matches". A rule this consequential is a law with a
|
||||||
|
* name and a refusal, not a `continue`.
|
||||||
|
*
|
||||||
|
* This is the ONE place the number lives. Every message, warning, doc line and
|
||||||
|
* pin derives it from here — never a literal.
|
||||||
|
*/
|
||||||
|
export const MAX_INDEXED_ARRAY_LENGTH = 256
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A metadata field carries an array longer than {@link MAX_INDEXED_ARRAY_LENGTH}.
|
||||||
|
*
|
||||||
|
* Thrown at the WRITE door (`add` / `update` / `relate` / `updateRelation`), so
|
||||||
|
* the caller learns at the moment of writing that the field will not be
|
||||||
|
* searchable — rather than discovering it later as rows that quietly fail to
|
||||||
|
* match. Carries the field, its length and the bound so a handler can report
|
||||||
|
* or repair without parsing the message.
|
||||||
|
*
|
||||||
|
* The cure is one of: store the long array outside the indexed bag (`data`
|
||||||
|
* carries arbitrary content and is not indexed element-wise); pass an embedding
|
||||||
|
* as the first-class `vector` parameter, which is where a vector belongs; or
|
||||||
|
* shorten the field to the values that are actually queried.
|
||||||
|
*/
|
||||||
|
export class MetadataArrayTooLargeError extends BrainyError {
|
||||||
|
/** The metadata field whose array is too long (its full dotted address). */
|
||||||
|
public readonly field: string
|
||||||
|
/** How many elements that array holds. */
|
||||||
|
public readonly length: number
|
||||||
|
/** The bound it exceeded — {@link MAX_INDEXED_ARRAY_LENGTH}. */
|
||||||
|
public readonly limit: number
|
||||||
|
|
||||||
|
constructor(site: string, field: string, length: number, limit: number) {
|
||||||
|
super(
|
||||||
|
`${site}: metadata field '${field}' holds ${length} array elements, ` +
|
||||||
|
`over the ${limit}-element indexing bound. An array field indexes one ` +
|
||||||
|
`posting per element, so an unbounded array is an unbounded write. ` +
|
||||||
|
`This write is refused rather than indexed partially or skipped silently ` +
|
||||||
|
`— a skipped field drops the row out of every filtered search on '${field}' ` +
|
||||||
|
`with no way to tell that from "nothing matched". ` +
|
||||||
|
`Cures: put the long array in 'data' (stored, not indexed element-wise); ` +
|
||||||
|
`pass an embedding as the first-class 'vector' parameter; or keep only ` +
|
||||||
|
`the values you actually query in '${field}'.`,
|
||||||
|
'VALIDATION',
|
||||||
|
false
|
||||||
|
)
|
||||||
|
this.name = 'MetadataArrayTooLargeError'
|
||||||
|
this.field = field
|
||||||
|
this.length = length
|
||||||
|
this.limit = limit
|
||||||
|
if (Error.captureStackTrace) {
|
||||||
|
Error.captureStackTrace(this, MetadataArrayTooLargeError)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -1105,13 +1105,31 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Clean shutdown
|
* Stop the auto-flush interval WITHOUT writing anything.
|
||||||
|
*
|
||||||
|
* The non-writing half of {@link close}, for a shutdown that must leave the
|
||||||
|
* store byte-identical — a read-only brain's close. `close()` itself is a
|
||||||
|
* writer: it drains both LSM MemTables to SSTables and stamps the watermark,
|
||||||
|
* which is exactly right for a writer and forbidden for a reader. A reader
|
||||||
|
* still has to release this interval, though: it is the one piece of this
|
||||||
|
* index that outlives the close and could fire against a store the session no
|
||||||
|
* longer owns.
|
||||||
|
*
|
||||||
|
* @returns Nothing.
|
||||||
*/
|
*/
|
||||||
async close(): Promise<void> {
|
stopBackgroundFlush(): void {
|
||||||
if (this.flushTimer) {
|
if (this.flushTimer) {
|
||||||
clearInterval(this.flushTimer)
|
clearInterval(this.flushTimer)
|
||||||
this.flushTimer = undefined
|
this.flushTimer = undefined
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Clean shutdown — drains both trees and stamps the watermark. THIS WRITES;
|
||||||
|
* a read-only brain must call {@link stopBackgroundFlush} instead.
|
||||||
|
*/
|
||||||
|
async close(): Promise<void> {
|
||||||
|
this.stopBackgroundFlush()
|
||||||
|
|
||||||
// Close both LSM-trees (will flush MemTables to SSTables)
|
// Close both LSM-trees (will flush MemTables to SSTables)
|
||||||
if (this.initialized) {
|
if (this.initialized) {
|
||||||
|
|
|
||||||
|
|
@ -203,7 +203,7 @@ export { EntityNotFoundError, RelationNotFoundError } from './errors/notFound.js
|
||||||
|
|
||||||
// Base error + typed migration-lock error — thrown by any data-plane call while a
|
// Base error + typed migration-lock error — thrown by any data-plane call while a
|
||||||
// brain runs its one-time 7.x→8.0 upgrade; catch to answer HTTP 503 + Retry-After.
|
// brain runs its one-time 7.x→8.0 upgrade; catch to answer HTTP 503 + Retry-After.
|
||||||
export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError, ProtectedArtifactError, DerivedArtifactMissingError } from './errors/brainyError.js'
|
export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError, ProtectedArtifactError, DerivedArtifactMissingError, MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from './errors/brainyError.js'
|
||||||
export type { BrainyErrorType } from './errors/brainyError.js'
|
export type { BrainyErrorType } from './errors/brainyError.js'
|
||||||
|
|
||||||
// ============= 8.0 Db API — generational MVCC =============
|
// ============= 8.0 Db API — generational MVCC =============
|
||||||
|
|
@ -231,7 +231,8 @@ export {
|
||||||
GenerationCompactedError,
|
GenerationCompactedError,
|
||||||
StoreInconsistentError,
|
StoreInconsistentError,
|
||||||
PendingFlushDurabilityError,
|
PendingFlushDurabilityError,
|
||||||
CanonicalEnumerationUnavailableError
|
CanonicalEnumerationUnavailableError,
|
||||||
|
PendingSingleOpsUnflushedError
|
||||||
} from './db/errors.js'
|
} from './db/errors.js'
|
||||||
export type { UnreconciledRecord } from './db/errors.js'
|
export type { UnreconciledRecord } from './db/errors.js'
|
||||||
export type {
|
export type {
|
||||||
|
|
|
||||||
|
|
@ -23,7 +23,10 @@ import type { ColumnStoreProvider, SegmentMeta } from './types.js'
|
||||||
import {
|
import {
|
||||||
ValueType,
|
ValueType,
|
||||||
DEFAULT_FLUSH_THRESHOLD,
|
DEFAULT_FLUSH_THRESHOLD,
|
||||||
FLAG_MULTI_VALUE
|
FLAG_MULTI_VALUE,
|
||||||
|
POSTING_KINDS,
|
||||||
|
KIND_PATH_SEGMENT,
|
||||||
|
type PostingKind
|
||||||
} from './types.js'
|
} from './types.js'
|
||||||
import { ColumnTailBuffer } from './ColumnTailBuffer.js'
|
import { ColumnTailBuffer } from './ColumnTailBuffer.js'
|
||||||
import { ColumnManifest } from './ColumnManifest.js'
|
import { ColumnManifest } from './ColumnManifest.js'
|
||||||
|
|
@ -52,10 +55,89 @@ interface HeapEntry {
|
||||||
value: number | string
|
value: number | string
|
||||||
entityIntId: number
|
entityIntId: number
|
||||||
cursorIndex: number
|
cursorIndex: number
|
||||||
|
/**
|
||||||
|
* Rank of the posting kind this entry came from, from {@link POSTING_KINDS}.
|
||||||
|
* A mixed-kind field has no natural total order, so the merge orders by kind
|
||||||
|
* first and by value within a kind.
|
||||||
|
*/
|
||||||
|
kindRank: number
|
||||||
/** Iterator for the cursor — call next() to advance */
|
/** Iterator for the cursor — call next() to advance */
|
||||||
iterator: Generator<CursorEntry>
|
iterator: Generator<CursorEntry>
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* One physical posting column: a (field, kind) pair and the key every internal
|
||||||
|
* map and every storage path uses for it.
|
||||||
|
*/
|
||||||
|
interface KindColumn {
|
||||||
|
/** The field as the query language names it. */
|
||||||
|
field: string
|
||||||
|
/** The kind of value this column holds. */
|
||||||
|
kind: PostingKind
|
||||||
|
/**
|
||||||
|
* Internal map / storage key. The field's PRIMARY kind uses the bare field
|
||||||
|
* name — the historical layout — and every other kind uses
|
||||||
|
* `<field>/<KIND_PATH_SEGMENT>/<kind>`.
|
||||||
|
*/
|
||||||
|
key: string
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The KIND a value indexes under — its JavaScript `typeof` class, not its
|
||||||
|
* storage encoding.
|
||||||
|
*
|
||||||
|
* Anything that is not a number, string or boolean indexes as a string, which
|
||||||
|
* is the `String(value)` treatment those values already received. `null` and
|
||||||
|
* `undefined` never reach here: `addEntity` skips them, and their absence is
|
||||||
|
* what the `exists` / `missing` operators read.
|
||||||
|
*
|
||||||
|
* @param value - The value about to be indexed or queried
|
||||||
|
* @returns The posting kind that owns this value
|
||||||
|
*/
|
||||||
|
function kindOfValue(value: unknown): PostingKind {
|
||||||
|
const t = typeof value
|
||||||
|
if (t === 'number') return 'number'
|
||||||
|
if (t === 'boolean') return 'boolean'
|
||||||
|
return 'string'
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The segment encoding a fresh column of this kind starts with.
|
||||||
|
*
|
||||||
|
* Only the number kind has a choice: an integer column starts as i64 and
|
||||||
|
* widens to f64 the first time a non-integer arrives
|
||||||
|
* ({@link ColumnTailBuffer.promoteToFloat}).
|
||||||
|
*/
|
||||||
|
function initialValueTypeFor(kind: PostingKind, firstValue: unknown): ValueType {
|
||||||
|
switch (kind) {
|
||||||
|
case 'boolean':
|
||||||
|
return ValueType.Boolean
|
||||||
|
case 'string':
|
||||||
|
return ValueType.String
|
||||||
|
case 'number':
|
||||||
|
return Number.isInteger(firstValue) ? ValueType.Number : ValueType.Float
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The kind a column of this encoding holds — the inverse of
|
||||||
|
* {@link initialValueTypeFor}, used to read a kind back off a manifest written
|
||||||
|
* before typed postings existed.
|
||||||
|
*/
|
||||||
|
function kindOfValueType(valueType: ValueType): PostingKind {
|
||||||
|
switch (valueType) {
|
||||||
|
case ValueType.Boolean:
|
||||||
|
return 'boolean'
|
||||||
|
case ValueType.String:
|
||||||
|
return 'string'
|
||||||
|
case ValueType.Number:
|
||||||
|
case ValueType.Float:
|
||||||
|
return 'number'
|
||||||
|
default:
|
||||||
|
throw new Error(`Unknown ValueType: ${valueType}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Unified column store coordinator.
|
* Unified column store coordinator.
|
||||||
*
|
*
|
||||||
|
|
@ -121,9 +203,19 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
*/
|
*/
|
||||||
private deletedEntities: Map<string, RoaringBitmap32> = new Map()
|
private deletedEntities: Map<string, RoaringBitmap32> = new Map()
|
||||||
|
|
||||||
/** Known field value types (inferred from first write). */
|
/** Segment encoding per COLUMN key (not per field — a field has one per kind). */
|
||||||
private fieldTypes: Map<string, ValueType> = new Map()
|
private fieldTypes: Map<string, ValueType> = new Map()
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Every posting column a field owns: field → kind → column key.
|
||||||
|
*
|
||||||
|
* This is the map that ends the first-writer type freeze. A field's first
|
||||||
|
* kind takes the bare field name as its column key, keeping the historical
|
||||||
|
* on-disk layout; each later kind takes its own column beside it. Nothing is
|
||||||
|
* coerced across kinds and nothing is dropped for being the wrong type.
|
||||||
|
*/
|
||||||
|
private fieldColumns: Map<string, Map<PostingKind, string>> = new Map()
|
||||||
|
|
||||||
/** Whether init() has completed. */
|
/** Whether init() has completed. */
|
||||||
private initialized = false
|
private initialized = false
|
||||||
|
|
||||||
|
|
@ -140,6 +232,128 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
this.l0CompactionTrigger = config?.l0CompactionTrigger ?? 4
|
this.l0CompactionTrigger = config?.l0CompactionTrigger ?? 4
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// =========================================================================
|
||||||
|
// Posting columns: (field, kind) → one physical column
|
||||||
|
// =========================================================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Storage / map key for a (field, kind) column.
|
||||||
|
*
|
||||||
|
* `primary` is the kind that owns the bare field name. It is whichever kind
|
||||||
|
* the field saw first, which for an index written before typed postings is
|
||||||
|
* simply the kind of its single manifest — so the historical layout is
|
||||||
|
* preserved rather than migrated.
|
||||||
|
*/
|
||||||
|
private static columnKeyFor(field: string, kind: PostingKind, primary: PostingKind | null): string {
|
||||||
|
return primary === null || kind === primary
|
||||||
|
? field
|
||||||
|
: `${field}/${KIND_PATH_SEGMENT}/${kind}`
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Split a discovered manifest path back into its (field, kind) column, or
|
||||||
|
* `null` when the path names a field's primary column rather than a kind
|
||||||
|
* column. `<field>/k/<kind>` is the only shape that reads as a kind column,
|
||||||
|
* and only for a `<kind>` this version knows.
|
||||||
|
*/
|
||||||
|
private static parseKindColumnKey(key: string): { field: string; kind: PostingKind } | null {
|
||||||
|
const marker = `/${KIND_PATH_SEGMENT}/`
|
||||||
|
const at = key.lastIndexOf(marker)
|
||||||
|
if (at <= 0) return null
|
||||||
|
const kind = key.slice(at + marker.length)
|
||||||
|
if (!POSTING_KINDS.includes(kind as PostingKind)) return null
|
||||||
|
return { field: key.slice(0, at), kind: kind as PostingKind }
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Record a discovered or freshly created column against its field. */
|
||||||
|
private registerColumn(field: string, kind: PostingKind, key: string): void {
|
||||||
|
let byKind = this.fieldColumns.get(field)
|
||||||
|
if (!byKind) {
|
||||||
|
byKind = new Map()
|
||||||
|
this.fieldColumns.set(field, byKind)
|
||||||
|
}
|
||||||
|
const existing = byKind.get(kind)
|
||||||
|
if (existing !== undefined && existing !== key) {
|
||||||
|
// Two columns claiming one (field, kind) means the layout on disk is not
|
||||||
|
// one this writer could have produced. Serving it would silently answer
|
||||||
|
// from half the postings, so say which two and stop.
|
||||||
|
throw new Error(
|
||||||
|
`ColumnStore: field '${field}' has two '${kind}' posting columns on ` +
|
||||||
|
`disk ('${existing}' and '${key}'). The column index layout is ` +
|
||||||
|
`inconsistent — rebuild/repair the metadata index rather than ` +
|
||||||
|
`serving from one half of it.`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
byKind.set(kind, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The column key for this (field, kind), or `null` if the field has no such kind. */
|
||||||
|
private columnKey(field: string, kind: PostingKind): string | null {
|
||||||
|
return this.fieldColumns.get(field)?.get(kind) ?? null
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The column key for this (field, kind), creating the registration if the
|
||||||
|
* field has not seen this kind before. Write path only.
|
||||||
|
*/
|
||||||
|
private ensureColumnKey(field: string, kind: PostingKind): string {
|
||||||
|
const byKind = this.fieldColumns.get(field)
|
||||||
|
const existing = byKind?.get(kind)
|
||||||
|
if (existing !== undefined) return existing
|
||||||
|
|
||||||
|
// The primary kind is the one already holding the bare field name, if any.
|
||||||
|
let primary: PostingKind | null = null
|
||||||
|
if (byKind) {
|
||||||
|
for (const [k, key] of byKind) {
|
||||||
|
if (key === field) { primary = k; break }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const key = ColumnStore.columnKeyFor(field, kind, primary)
|
||||||
|
this.registerColumn(field, kind, key)
|
||||||
|
return key
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Every posting column this field owns, in {@link POSTING_KINDS} order.
|
||||||
|
*
|
||||||
|
* Read doors that are not about one particular value — an unbounded range
|
||||||
|
* used as an "any value present" probe, distinct values, sorting — fan out
|
||||||
|
* over all of them.
|
||||||
|
*/
|
||||||
|
private columnsForField(field: string): KindColumn[] {
|
||||||
|
const byKind = this.fieldColumns.get(field)
|
||||||
|
if (!byKind) return []
|
||||||
|
const out: KindColumn[] = []
|
||||||
|
for (const kind of POSTING_KINDS) {
|
||||||
|
const key = byKind.get(kind)
|
||||||
|
if (key !== undefined) out.push({ field, kind, key })
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Which value kinds this field actually holds, in {@link POSTING_KINDS}
|
||||||
|
* order — the honest answer to "what type is this field?".
|
||||||
|
*
|
||||||
|
* A field that carries both `'electronics'` and `5` reports
|
||||||
|
* `['number', 'string']`, not whichever of them was written first.
|
||||||
|
*
|
||||||
|
* @param field - Field name
|
||||||
|
* @returns Every kind with at least one posting, or `[]` for an unknown field
|
||||||
|
*/
|
||||||
|
getFieldKinds(field: string): PostingKind[] {
|
||||||
|
return this.columnsForField(field)
|
||||||
|
.filter((c) => this.columnHasData(c.key))
|
||||||
|
.map((c) => c.kind)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Does this physical column hold any postings (persisted or buffered)? */
|
||||||
|
private columnHasData(key: string): boolean {
|
||||||
|
const manifest = this.manifests.get(key)
|
||||||
|
const buffer = this.tailBuffers.get(key)
|
||||||
|
return (manifest !== undefined && !manifest.isEmpty()) || (buffer !== undefined && buffer.size > 0)
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialize the column store: discover existing field manifests.
|
* Initialize the column store: discover existing field manifests.
|
||||||
*/
|
*/
|
||||||
|
|
@ -157,11 +371,23 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
}).listObjectsUnderPath(this.basePath + '/')
|
}).listObjectsUnderPath(this.basePath + '/')
|
||||||
for (const path of paths) {
|
for (const path of paths) {
|
||||||
if (path.endsWith('/MANIFEST.json')) {
|
if (path.endsWith('/MANIFEST.json')) {
|
||||||
const fieldName = path.replace(this.basePath + '/', '').replace('/MANIFEST.json', '')
|
// The discovered name is a COLUMN key: either a bare field (that
|
||||||
const manifest = new ColumnManifest(fieldName, this.basePath)
|
// field's primary kind, which is every column an index written
|
||||||
|
// before typed postings has) or `<field>/k/<kind>` for a second
|
||||||
|
// kind that arrived on a field later.
|
||||||
|
const columnKey = path.replace(this.basePath + '/', '').replace('/MANIFEST.json', '')
|
||||||
|
const manifest = new ColumnManifest(columnKey, this.basePath)
|
||||||
await manifest.load(storage)
|
await manifest.load(storage)
|
||||||
this.manifests.set(fieldName, manifest)
|
this.manifests.set(columnKey, manifest)
|
||||||
this.fieldTypes.set(fieldName, manifest.valueType)
|
this.fieldTypes.set(columnKey, manifest.valueType)
|
||||||
|
|
||||||
|
const parsed = ColumnStore.parseKindColumnKey(columnKey)
|
||||||
|
if (parsed) {
|
||||||
|
this.registerColumn(parsed.field, parsed.kind, columnKey)
|
||||||
|
} else {
|
||||||
|
this.registerColumn(columnKey, kindOfValueType(manifest.valueType), columnKey)
|
||||||
|
}
|
||||||
|
const fieldName = columnKey
|
||||||
|
|
||||||
// Load global deleted bitmap if it exists. Raw blob preferred
|
// Load global deleted bitmap if it exists. Raw blob preferred
|
||||||
// (2.4.0 #4 cortex-shared format); legacy envelope fallback for
|
// (2.4.0 #4 cortex-shared format); legacy envelope fallback for
|
||||||
|
|
@ -264,26 +490,43 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
/**
|
/**
|
||||||
* Point filter: find entities where field equals value.
|
* Point filter: find entities where field equals value.
|
||||||
*
|
*
|
||||||
* Searches all segments + tail buffer, returns union as roaring bitmap.
|
* The QUERY VALUE'S OWN KIND picks the posting column, and only that column
|
||||||
* Excludes globally deleted entities.
|
* is read. `where {category: 5}` answers from the number postings and
|
||||||
|
* `where {category: '5'}` from the string postings — neither borrows the
|
||||||
|
* other's rows, because a row written with the number `5` is not a row whose
|
||||||
|
* category is the text `'5'`.
|
||||||
|
*
|
||||||
|
* A field that has never seen this kind matches nothing, which is the true
|
||||||
|
* answer rather than a coerced one.
|
||||||
|
*
|
||||||
|
* Searches all segments + tail buffer of that column, returns the union as a
|
||||||
|
* roaring bitmap. Excludes globally deleted entities.
|
||||||
*/
|
*/
|
||||||
async filter(field: string, value: unknown): Promise<RoaringBitmap32> {
|
async filter(field: string, value: unknown): Promise<RoaringBitmap32> {
|
||||||
const result = new RoaringBitmap32()
|
const result = new RoaringBitmap32()
|
||||||
const deleted = this.deletedEntities.get(field)
|
const columnKey = this.columnKey(field, kindOfValue(value))
|
||||||
|
if (columnKey === null) return result
|
||||||
|
|
||||||
|
// The query value takes the column's encoding — a boolean queried against
|
||||||
|
// a boolean column has to become the 1/0 the column stores.
|
||||||
|
const encoded = this.normalizeValue(value, this.fieldTypes.get(columnKey) ?? ValueType.String)
|
||||||
|
if (encoded === undefined) return result
|
||||||
|
|
||||||
|
const deleted = this.deletedEntities.get(columnKey)
|
||||||
|
|
||||||
// Search segments
|
// Search segments
|
||||||
const cursors = await this.getSegmentCursors(field)
|
const cursors = await this.getSegmentCursors(columnKey)
|
||||||
for (const cursor of cursors) {
|
for (const cursor of cursors) {
|
||||||
const ids = cursor.getEntityIdsForValue(value as number | string)
|
const ids = cursor.getEntityIdsForValue(encoded)
|
||||||
for (const id of ids) {
|
for (const id of ids) {
|
||||||
if (!deleted || !deleted.has(id)) result.add(id)
|
if (!deleted || !deleted.has(id)) result.add(id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Search tail buffer
|
// Search tail buffer
|
||||||
const tailCursor = this.getTailBufferCursor(field)
|
const tailCursor = this.getTailBufferCursor(columnKey)
|
||||||
if (tailCursor) {
|
if (tailCursor) {
|
||||||
const ids = tailCursor.getEntityIdsForValue(value as number | string)
|
const ids = tailCursor.getEntityIdsForValue(encoded)
|
||||||
for (const id of ids) {
|
for (const id of ids) {
|
||||||
if (!deleted || !deleted.has(id)) result.add(id)
|
if (!deleted || !deleted.has(id)) result.add(id)
|
||||||
}
|
}
|
||||||
|
|
@ -292,6 +535,62 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read this column's value for each of `entityIntIds` — the per-id read
|
||||||
|
* behind `find({ fields })`.
|
||||||
|
*
|
||||||
|
* Every other read door here answers "which entities have this value". A
|
||||||
|
* projection asks the opposite — "what value does this entity have" — and
|
||||||
|
* without it a projection has to go to the canonical record for a field the
|
||||||
|
* column is already holding.
|
||||||
|
*
|
||||||
|
* The column is walked ONCE and the wanted ids are picked out as they pass,
|
||||||
|
* so the cost is O(column) per field rather than O(ids x column). Later
|
||||||
|
* sources win: the tail buffer holds writes newer than any segment, and
|
||||||
|
* within the segments a later one supersedes an earlier, exactly as `filter`
|
||||||
|
* treats them.
|
||||||
|
*
|
||||||
|
* Values are EXACT — this store keeps raw values, not the bucketed form the
|
||||||
|
* sparse index uses for range queries — which is what makes it safe to
|
||||||
|
* project from. Deleted entities are skipped; an id with no value in this
|
||||||
|
* column is simply absent from the result.
|
||||||
|
*
|
||||||
|
* @param field - Field name to read.
|
||||||
|
* @param entityIntIds - Entity integer ids to read values for.
|
||||||
|
* @returns `entityIntId -> value` for the ids this column holds.
|
||||||
|
*/
|
||||||
|
async valuesForIds(
|
||||||
|
field: string,
|
||||||
|
entityIntIds: Iterable<number>
|
||||||
|
): Promise<Map<number, number | string>> {
|
||||||
|
const wanted = new Set<number>(entityIntIds)
|
||||||
|
const out = new Map<number, number | string>()
|
||||||
|
if (wanted.size === 0 || !this.hasField(field)) return out
|
||||||
|
|
||||||
|
// Every kind the field holds is read, in POSTING_KINDS order — a value an
|
||||||
|
// entity wrote as a string is still that entity's value for this field.
|
||||||
|
for (const column of this.columnsForField(field)) {
|
||||||
|
const deleted = this.deletedEntities.get(column.key)
|
||||||
|
const take = (entry: { value: number | string; entityIntId: number }): void => {
|
||||||
|
if (!wanted.has(entry.entityIntId)) return
|
||||||
|
if (deleted && deleted.has(entry.entityIntId)) return
|
||||||
|
out.set(entry.entityIntId, entry.value)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Segments oldest -> newest, then the tail: a later write overwrites an
|
||||||
|
// earlier one for the same id.
|
||||||
|
const cursors = await this.getSegmentCursors(column.key)
|
||||||
|
for (const cursor of cursors) {
|
||||||
|
for (const entry of cursor.iterateForward()) take(entry)
|
||||||
|
}
|
||||||
|
const tailCursor = this.getTailBufferCursor(column.key)
|
||||||
|
if (tailCursor) {
|
||||||
|
for (const entry of tailCursor.iterateForward()) take(entry)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Range filter: find entities where field is within the bounds.
|
* Range filter: find entities where field is within the bounds.
|
||||||
*
|
*
|
||||||
|
|
@ -311,41 +610,59 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
includeMax: boolean = true
|
includeMax: boolean = true
|
||||||
): Promise<RoaringBitmap32> {
|
): Promise<RoaringBitmap32> {
|
||||||
const result = new RoaringBitmap32()
|
const result = new RoaringBitmap32()
|
||||||
const cursors = await this.getSegmentCursors(field)
|
|
||||||
|
|
||||||
const hasMin = min !== undefined && min !== null
|
const hasMin = min !== undefined && min !== null
|
||||||
const hasMax = max !== undefined && max !== null
|
const hasMax = max !== undefined && max !== null
|
||||||
|
|
||||||
for (const cursor of cursors) {
|
// The BOUNDS pick the column: numeric bounds read the numeric postings,
|
||||||
const lo = hasMin ? min as number | string : cursor.minValue
|
// string bounds the string postings. An unbounded call is not a range at
|
||||||
const hi = hasMax ? max as number | string : cursor.maxValue
|
// all — it is the "has any value here" probe behind `exists` — so it fans
|
||||||
if (lo === undefined || hi === undefined) continue
|
// out over every kind the field holds.
|
||||||
// Exclusivity applies only to an explicitly provided bound. A bound taken
|
const columns: KindColumn[] = hasMin
|
||||||
// from the segment's own min/max is a real stored value and must stay
|
? this.columnsForKind(field, kindOfValue(min))
|
||||||
// inclusive, or the segment's boundary entities would be wrongly dropped.
|
: hasMax
|
||||||
const ids = cursor.getEntityIdsInRange(
|
? this.columnsForKind(field, kindOfValue(max))
|
||||||
lo,
|
: this.columnsForField(field)
|
||||||
hi,
|
|
||||||
hasMin ? includeMin : true,
|
|
||||||
hasMax ? includeMax : true
|
|
||||||
)
|
|
||||||
for (const id of ids) result.add(id)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Tail buffer range: linear scan (tail is small)
|
for (const column of columns) {
|
||||||
const tailCursor = this.getTailBufferCursor(field)
|
const cursors = await this.getSegmentCursors(column.key)
|
||||||
if (tailCursor) {
|
for (const cursor of cursors) {
|
||||||
for (const entry of tailCursor.iterateForward()) {
|
const lo = hasMin ? min as number | string : cursor.minValue
|
||||||
const v = entry.value as any
|
const hi = hasMax ? max as number | string : cursor.maxValue
|
||||||
const loOk = !hasMin || (includeMin ? v >= (min as any) : v > (min as any))
|
if (lo === undefined || hi === undefined) continue
|
||||||
const hiOk = !hasMax || (includeMax ? v <= (max as any) : v < (max as any))
|
// Exclusivity applies only to an explicitly provided bound. A bound taken
|
||||||
if (loOk && hiOk) result.add(entry.entityIntId)
|
// from the segment's own min/max is a real stored value and must stay
|
||||||
|
// inclusive, or the segment's boundary entities would be wrongly dropped.
|
||||||
|
const ids = cursor.getEntityIdsInRange(
|
||||||
|
lo,
|
||||||
|
hi,
|
||||||
|
hasMin ? includeMin : true,
|
||||||
|
hasMax ? includeMax : true
|
||||||
|
)
|
||||||
|
for (const id of ids) result.add(id)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tail buffer range: linear scan (tail is small)
|
||||||
|
const tailCursor = this.getTailBufferCursor(column.key)
|
||||||
|
if (tailCursor) {
|
||||||
|
for (const entry of tailCursor.iterateForward()) {
|
||||||
|
const v = entry.value as any
|
||||||
|
const loOk = !hasMin || (includeMin ? v >= (min as any) : v > (min as any))
|
||||||
|
const hiOk = !hasMax || (includeMax ? v <= (max as any) : v < (max as any))
|
||||||
|
if (loOk && hiOk) result.add(entry.entityIntId)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** The single column for this (field, kind), as a list, or empty if absent. */
|
||||||
|
private columnsForKind(field: string, kind: PostingKind): KindColumn[] {
|
||||||
|
const key = this.columnKey(field, kind)
|
||||||
|
return key === null ? [] : [{ field, kind, key }]
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Sort top-K: return K entity int IDs in sorted order (u64-safe BigInt).
|
* Sort top-K: return K entity int IDs in sorted order (u64-safe BigInt).
|
||||||
*
|
*
|
||||||
|
|
@ -376,18 +693,21 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
*/
|
*/
|
||||||
async getFilterValues(field: string): Promise<string[]> {
|
async getFilterValues(field: string): Promise<string[]> {
|
||||||
const valueSet = new Set<string>()
|
const valueSet = new Set<string>()
|
||||||
const cursors = await this.getSegmentCursors(field)
|
|
||||||
|
|
||||||
for (const cursor of cursors) {
|
for (const column of this.columnsForField(field)) {
|
||||||
for (const entry of cursor.iterateForward()) {
|
const cursors = await this.getSegmentCursors(column.key)
|
||||||
valueSet.add(String(entry.value))
|
|
||||||
|
for (const cursor of cursors) {
|
||||||
|
for (const entry of cursor.iterateForward()) {
|
||||||
|
valueSet.add(String(entry.value))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
const tailCursor = this.getTailBufferCursor(field)
|
const tailCursor = this.getTailBufferCursor(column.key)
|
||||||
if (tailCursor) {
|
if (tailCursor) {
|
||||||
for (const entry of tailCursor.iterateForward()) {
|
for (const entry of tailCursor.iterateForward()) {
|
||||||
valueSet.add(String(entry.value))
|
valueSet.add(String(entry.value))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -398,9 +718,7 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
* Check if a field has any indexed data.
|
* Check if a field has any indexed data.
|
||||||
*/
|
*/
|
||||||
hasField(field: string): boolean {
|
hasField(field: string): boolean {
|
||||||
const manifest = this.manifests.get(field)
|
return this.columnsForField(field).some((c) => this.columnHasData(c.key))
|
||||||
const buffer = this.tailBuffers.get(field)
|
|
||||||
return (manifest !== undefined && !manifest.isEmpty()) || (buffer !== undefined && buffer.size > 0)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -410,12 +728,11 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
* store will actually serve queries from.
|
* store will actually serve queries from.
|
||||||
*/
|
*/
|
||||||
getIndexedFields(): string[] {
|
getIndexedFields(): string[] {
|
||||||
|
// Names FIELDS, not columns: a field carrying two kinds is one name here,
|
||||||
|
// the same name a caller queries with.
|
||||||
const fields = new Set<string>()
|
const fields = new Set<string>()
|
||||||
for (const [field, manifest] of this.manifests) {
|
for (const [field] of this.fieldColumns) {
|
||||||
if (!manifest.isEmpty()) fields.add(field)
|
if (this.hasField(field)) fields.add(field)
|
||||||
}
|
|
||||||
for (const [field, buffer] of this.tailBuffers) {
|
|
||||||
if (buffer.size > 0) fields.add(field)
|
|
||||||
}
|
}
|
||||||
return Array.from(fields).sort()
|
return Array.from(fields).sort()
|
||||||
}
|
}
|
||||||
|
|
@ -430,12 +747,16 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
getFieldSizeSummary(): Array<{ field: string; segmentCount: number; tailSize: number }> {
|
getFieldSizeSummary(): Array<{ field: string; segmentCount: number; tailSize: number }> {
|
||||||
const summary: Array<{ field: string; segmentCount: number; tailSize: number }> = []
|
const summary: Array<{ field: string; segmentCount: number; tailSize: number }> = []
|
||||||
for (const field of this.getIndexedFields()) {
|
for (const field of this.getIndexedFields()) {
|
||||||
const manifest = this.manifests.get(field)
|
// Summed across the field's kind columns — the caller asked about a
|
||||||
const buffer = this.tailBuffers.get(field)
|
// field, and a field's size is all of the postings under its name.
|
||||||
const segmentCount = manifest && !manifest.isEmpty()
|
let segmentCount = 0
|
||||||
? manifest.getAllSegments().length
|
let tailSize = 0
|
||||||
: 0
|
for (const column of this.columnsForField(field)) {
|
||||||
const tailSize = buffer ? buffer.size : 0
|
const manifest = this.manifests.get(column.key)
|
||||||
|
const buffer = this.tailBuffers.get(column.key)
|
||||||
|
if (manifest && !manifest.isEmpty()) segmentCount += manifest.getAllSegments().length
|
||||||
|
if (buffer) tailSize += buffer.size
|
||||||
|
}
|
||||||
summary.push({ field, segmentCount, tailSize })
|
summary.push({ field, segmentCount, tailSize })
|
||||||
}
|
}
|
||||||
return summary
|
return summary
|
||||||
|
|
@ -463,6 +784,8 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
this.segmentCache.clear()
|
this.segmentCache.clear()
|
||||||
this.manifests.clear()
|
this.manifests.clear()
|
||||||
this.deletedEntities.clear()
|
this.deletedEntities.clear()
|
||||||
|
this.fieldColumns.clear()
|
||||||
|
this.fieldTypes.clear()
|
||||||
this.initialized = false
|
this.initialized = false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -471,32 +794,64 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
// =========================================================================
|
// =========================================================================
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Push a single value to a field's tail buffer.
|
* Push a single value to the posting column for its (field, KIND).
|
||||||
* Creates the buffer and manifest if first write to this field.
|
*
|
||||||
* Infers ValueType from the first value seen.
|
* The value's own kind picks the column — a string goes to the field's
|
||||||
|
* string postings, a number to its number postings — so a field carrying
|
||||||
|
* `'electronics'` and `5` keeps both, each answerable by an equality filter
|
||||||
|
* of its own kind. Under the first-writer type freeze this method replaced,
|
||||||
|
* the first value's type became the field's type and every later value of
|
||||||
|
* another kind was coerced to it or, when coercion failed, dropped with no
|
||||||
|
* error at all.
|
||||||
|
*
|
||||||
|
* Creates the column's buffer and manifest on its first value.
|
||||||
*/
|
*/
|
||||||
private pushToBuffer(field: string, value: unknown, entityIntId: number, isMultiValue: boolean): void {
|
private pushToBuffer(field: string, value: unknown, entityIntId: number, isMultiValue: boolean): void {
|
||||||
let buffer = this.tailBuffers.get(field)
|
const kind = kindOfValue(value)
|
||||||
|
const columnKey = this.ensureColumnKey(field, kind)
|
||||||
|
|
||||||
|
let buffer = this.tailBuffers.get(columnKey)
|
||||||
if (!buffer) {
|
if (!buffer) {
|
||||||
const valueType = this.inferValueType(value)
|
// A reopened column takes its encoding from its manifest — an integer
|
||||||
buffer = new ColumnTailBuffer(field, valueType, this.flushThreshold)
|
// column that widened to f64 in an earlier session stays widened.
|
||||||
this.tailBuffers.set(field, buffer)
|
const valueType =
|
||||||
this.fieldTypes.set(field, valueType)
|
this.manifests.get(columnKey)?.valueType ?? initialValueTypeFor(kind, value)
|
||||||
|
buffer = new ColumnTailBuffer(columnKey, valueType, this.flushThreshold)
|
||||||
|
this.tailBuffers.set(columnKey, buffer)
|
||||||
|
this.fieldTypes.set(columnKey, valueType)
|
||||||
|
|
||||||
// Ensure manifest exists
|
// Ensure manifest exists
|
||||||
if (!this.manifests.has(field)) {
|
if (!this.manifests.has(columnKey)) {
|
||||||
const manifest = new ColumnManifest(field, this.basePath)
|
const manifest = new ColumnManifest(columnKey, this.basePath)
|
||||||
manifest.valueType = valueType
|
manifest.valueType = valueType
|
||||||
manifest.multiValue = isMultiValue
|
manifest.multiValue = isMultiValue
|
||||||
this.manifests.set(field, manifest)
|
this.manifests.set(columnKey, manifest)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Normalize value to the column type
|
// An integer column widens the first time a non-integer number arrives, so
|
||||||
const normalizedValue = this.normalizeValue(value, buffer.valueType)
|
// the value is stored as itself instead of rounded to the nearest integer.
|
||||||
if (normalizedValue !== undefined) {
|
if (kind === 'number' && buffer.valueType === ValueType.Number && !Number.isInteger(value)) {
|
||||||
buffer.add(normalizedValue, entityIntId)
|
buffer.promoteToFloat()
|
||||||
|
this.fieldTypes.set(columnKey, ValueType.Float)
|
||||||
|
const manifest = this.manifests.get(columnKey)
|
||||||
|
if (manifest) manifest.valueType = ValueType.Float
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const normalizedValue = this.normalizeValue(value, buffer.valueType)
|
||||||
|
if (normalizedValue === undefined) {
|
||||||
|
// Unreachable by construction: the column was chosen BY this value's
|
||||||
|
// kind, so the encoding always accepts it. Reaching here would mean a
|
||||||
|
// value had been silently dropped from the index — the exact failure
|
||||||
|
// typed postings exist to end — so it is an error, never a skip.
|
||||||
|
throw new Error(
|
||||||
|
`ColumnStore: field '${field}' rejected a ${kind} value for its own ` +
|
||||||
|
`${ValueType[buffer.valueType]} posting column. The value would have ` +
|
||||||
|
`been dropped from the index while the row stayed readable by id — ` +
|
||||||
|
`this is a kind-routing bug, not a value the caller may ignore.`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
buffer.add(normalizedValue, entityIntId)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -625,8 +980,15 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
/** Torn-segment quarantine entries for a field (observability + heal input). */
|
/** Torn-segment quarantine entries for a field (observability + heal input). */
|
||||||
quarantinedSegments(field: string): Array<{ segment: string; error: string; hits: number }> {
|
quarantinedSegments(field: string): Array<{ segment: string; error: string; hits: number }> {
|
||||||
const out: Array<{ segment: string; error: string; hits: number }> = []
|
const out: Array<{ segment: string; error: string; hits: number }> = []
|
||||||
for (const [key, q] of this.segmentQuarantine) {
|
// Across every kind column of the field — a torn segment in the string
|
||||||
if (key.startsWith(`${field}:`)) out.push({ segment: key.slice(field.length + 1), error: q.error, hits: q.hits })
|
// postings is this field's torn segment as much as one in the numbers.
|
||||||
|
for (const column of this.columnsForField(field)) {
|
||||||
|
const prefix = `${column.key}:`
|
||||||
|
for (const [key, q] of this.segmentQuarantine) {
|
||||||
|
if (key.startsWith(prefix)) {
|
||||||
|
out.push({ segment: key.slice(prefix.length), error: q.error, hits: q.hits })
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
@ -798,17 +1160,22 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
k: number,
|
k: number,
|
||||||
filterBitmap: RoaringBitmap32 | null
|
filterBitmap: RoaringBitmap32 | null
|
||||||
): Promise<number[]> {
|
): Promise<number[]> {
|
||||||
// Collect all cursors (segments + tail buffer)
|
// Collect cursors across EVERY kind the field holds. A single-kind field —
|
||||||
const segCursors = await this.getSegmentCursors(field)
|
// nearly all of them — merges exactly the cursors it always did.
|
||||||
const tailCursor = this.getTailBufferCursor(field)
|
|
||||||
|
|
||||||
// Create iterators for each cursor in the specified direction
|
|
||||||
const iterators: Generator<CursorEntry>[] = []
|
const iterators: Generator<CursorEntry>[] = []
|
||||||
for (const cursor of segCursors) {
|
const iteratorKindRank: number[] = []
|
||||||
iterators.push(order === 'asc' ? cursor.iterateForward() : cursor.iterateBackward())
|
for (const column of this.columnsForField(field)) {
|
||||||
}
|
const kindRank = POSTING_KINDS.indexOf(column.kind)
|
||||||
if (tailCursor) {
|
const segCursors = await this.getSegmentCursors(column.key)
|
||||||
iterators.push(order === 'asc' ? tailCursor.iterateForward() : tailCursor.iterateBackward())
|
for (const cursor of segCursors) {
|
||||||
|
iterators.push(order === 'asc' ? cursor.iterateForward() : cursor.iterateBackward())
|
||||||
|
iteratorKindRank.push(kindRank)
|
||||||
|
}
|
||||||
|
const tailCursor = this.getTailBufferCursor(column.key)
|
||||||
|
if (tailCursor) {
|
||||||
|
iterators.push(order === 'asc' ? tailCursor.iterateForward() : tailCursor.iterateBackward())
|
||||||
|
iteratorKindRank.push(kindRank)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (iterators.length === 0) return []
|
if (iterators.length === 0) return []
|
||||||
|
|
@ -822,16 +1189,21 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
value: next.value.value,
|
value: next.value.value,
|
||||||
entityIntId: next.value.entityIntId,
|
entityIntId: next.value.entityIntId,
|
||||||
cursorIndex: i,
|
cursorIndex: i,
|
||||||
|
kindRank: iteratorKindRank[i],
|
||||||
iterator: iterators[i]
|
iterator: iterators[i]
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Heapify
|
// Heapify. A number and a string have no ordering between them, so a
|
||||||
const isString = (this.fieldTypes.get(field) ?? ValueType.Number) === ValueType.String
|
// mixed-kind field orders by KIND first (POSTING_KINDS order) and by value
|
||||||
|
// within a kind — one defined total order instead of a comparison whose
|
||||||
|
// answer depends on which value happened to be on the left.
|
||||||
const compare = (a: HeapEntry, b: HeapEntry): number => {
|
const compare = (a: HeapEntry, b: HeapEntry): number => {
|
||||||
let cmp: number
|
let cmp: number
|
||||||
if (isString) {
|
if (a.kindRank !== b.kindRank) {
|
||||||
|
cmp = a.kindRank - b.kindRank
|
||||||
|
} else if (POSTING_KINDS[a.kindRank] === 'string') {
|
||||||
cmp = compareCodePoints(String(a.value), String(b.value))
|
cmp = compareCodePoints(String(a.value), String(b.value))
|
||||||
} else {
|
} else {
|
||||||
cmp = (a.value as number) - (b.value as number)
|
cmp = (a.value as number) - (b.value as number)
|
||||||
|
|
@ -863,6 +1235,7 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
value: next.value.value,
|
value: next.value.value,
|
||||||
entityIntId: next.value.entityIntId,
|
entityIntId: next.value.entityIntId,
|
||||||
cursorIndex: top.cursorIndex,
|
cursorIndex: top.cursorIndex,
|
||||||
|
kindRank: top.kindRank,
|
||||||
iterator: top.iterator
|
iterator: top.iterator
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -870,8 +1243,11 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
this.heapDown(heap, 0, compare)
|
this.heapDown(heap, 0, compare)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Apply global deleted check, filter, and dedup
|
// Apply global deleted check, filter, and dedup. The deleted bitmap is
|
||||||
const deleted = this.deletedEntities.get(field)
|
// per COLUMN, and the entry came from the column its kind names.
|
||||||
|
const deleted = this.deletedEntities.get(
|
||||||
|
this.columnKey(field, POSTING_KINDS[top.kindRank]) ?? field
|
||||||
|
)
|
||||||
if (deleted && deleted.has(top.entityIntId)) continue
|
if (deleted && deleted.has(top.entityIntId)) continue
|
||||||
if (seen.has(top.entityIntId)) continue
|
if (seen.has(top.entityIntId)) continue
|
||||||
if (filterBitmap && !filterBitmap.has(top.entityIntId)) continue
|
if (filterBitmap && !filterBitmap.has(top.entityIntId)) continue
|
||||||
|
|
@ -913,35 +1289,31 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Infer ValueType from a JavaScript value.
|
* Encode a value for the column its own kind selected.
|
||||||
*/
|
*
|
||||||
private inferValueType(value: unknown): ValueType {
|
* This does NOT convert between kinds. It used to: a string reaching a
|
||||||
if (typeof value === 'boolean') return ValueType.Boolean
|
* numeric column was run through `Number(value)`, and a number reaching a
|
||||||
if (typeof value === 'number') {
|
* numeric column was run through `Math.round`, so `'electronics'` became
|
||||||
return Number.isInteger(value) ? ValueType.Number : ValueType.Float
|
* `NaN` and vanished while `4.5` became `5` and answered the wrong query.
|
||||||
}
|
* Kind routing removes the need for either — the only work left is picking
|
||||||
return ValueType.String
|
* the encoding the column already committed to.
|
||||||
}
|
*
|
||||||
|
* @returns The encoded value, or `undefined` if the value does not belong in
|
||||||
/**
|
* this column at all — which the caller treats as a routing bug and
|
||||||
* Normalize a JavaScript value to the column's ValueType.
|
* raises, never as a value to skip.
|
||||||
*/
|
*/
|
||||||
private normalizeValue(value: unknown, type: ValueType): number | string | undefined {
|
private normalizeValue(value: unknown, type: ValueType): number | string | undefined {
|
||||||
switch (type) {
|
switch (type) {
|
||||||
case ValueType.Number:
|
case ValueType.Number:
|
||||||
if (typeof value === 'number') return Math.round(value)
|
// Integer column. Non-integers widen it to Float before reaching here.
|
||||||
if (typeof value === 'string') { const n = Number(value); return isNaN(n) ? undefined : Math.round(n) }
|
return typeof value === 'number' && Number.isInteger(value) ? value : undefined
|
||||||
if (typeof value === 'boolean') return value ? 1 : 0
|
|
||||||
return undefined
|
|
||||||
case ValueType.Float:
|
case ValueType.Float:
|
||||||
if (typeof value === 'number') return value
|
return typeof value === 'number' ? value : undefined
|
||||||
if (typeof value === 'string') { const n = Number(value); return isNaN(n) ? undefined : n }
|
|
||||||
return undefined
|
|
||||||
case ValueType.Boolean:
|
case ValueType.Boolean:
|
||||||
if (typeof value === 'boolean') return value ? 1 : 0
|
return typeof value === 'boolean' ? (value ? 1 : 0) : undefined
|
||||||
if (typeof value === 'number') return value ? 1 : 0
|
|
||||||
return undefined
|
|
||||||
case ValueType.String:
|
case ValueType.String:
|
||||||
|
// The string kind is also where objects and bigints land, exactly as
|
||||||
|
// they always did.
|
||||||
return String(value)
|
return String(value)
|
||||||
default:
|
default:
|
||||||
return undefined
|
return undefined
|
||||||
|
|
|
||||||
|
|
@ -55,8 +55,12 @@ export class ColumnTailBuffer {
|
||||||
/** Field name this buffer is for. */
|
/** Field name this buffer is for. */
|
||||||
readonly fieldName: string
|
readonly fieldName: string
|
||||||
|
|
||||||
/** Value type determines sort comparator. */
|
/**
|
||||||
readonly valueType: ValueType
|
* Value type determines sort comparator and segment encoding.
|
||||||
|
*
|
||||||
|
* Widened in place by {@link promoteToFloat} — never otherwise reassigned.
|
||||||
|
*/
|
||||||
|
valueType: ValueType
|
||||||
|
|
||||||
/** Flush threshold. */
|
/** Flush threshold. */
|
||||||
readonly threshold: number
|
readonly threshold: number
|
||||||
|
|
@ -81,6 +85,38 @@ export class ColumnTailBuffer {
|
||||||
this.threshold = threshold
|
this.threshold = threshold
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Widen an integer column to floating point, losslessly and in place.
|
||||||
|
*
|
||||||
|
* The number posting kind holds every JavaScript number, but a segment picks
|
||||||
|
* ONE encoding: i64 for integers, f64 for the rest. A column that has only
|
||||||
|
* ever seen integers is written as i64; the first non-integer to arrive
|
||||||
|
* widens it here, so that value is stored as itself instead of being rounded
|
||||||
|
* to the nearest integer with no error — the rounding that made `4.5` and
|
||||||
|
* `5.5` both answer `where {score: 5}` and neither answer its own value.
|
||||||
|
*
|
||||||
|
* Widening is lossless in both directions it has to be: every value already
|
||||||
|
* buffered is an integer, and every integer is exactly representable as f64.
|
||||||
|
* Segments already on disk keep their own i64 encoding in their own headers
|
||||||
|
* and keep decoding by it — only segments written from here on are f64.
|
||||||
|
*
|
||||||
|
* @throws Error if called on a column that is not an integer column — the
|
||||||
|
* only legal widening is Number → Float, and any other request is a bug in
|
||||||
|
* the caller's kind routing rather than something to absorb quietly.
|
||||||
|
*/
|
||||||
|
promoteToFloat(): void {
|
||||||
|
if (this.valueType === ValueType.Float) return
|
||||||
|
if (this.valueType !== ValueType.Number) {
|
||||||
|
throw new Error(
|
||||||
|
`ColumnTailBuffer '${this.fieldName}': cannot widen a ` +
|
||||||
|
`${ValueType[this.valueType]} column to Float — only an integer ` +
|
||||||
|
`(Number) column widens, and this call means a value reached the ` +
|
||||||
|
`wrong kind's column`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
this.valueType = ValueType.Float
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Add a (value, entityIntId) entry to the buffer.
|
* Add a (value, entityIntId) entry to the buffer.
|
||||||
*
|
*
|
||||||
|
|
|
||||||
|
|
@ -58,6 +58,53 @@ export enum ValueType {
|
||||||
Boolean = 3
|
Boolean = 3
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The KIND of a value, as the query language sees it.
|
||||||
|
*
|
||||||
|
* A kind is a JavaScript `typeof` class, not a storage encoding: `5` and `5.5`
|
||||||
|
* are one kind (`'number'`) held in one posting column, even though they need
|
||||||
|
* different segment encodings (i64 vs f64 — see {@link ValueType}).
|
||||||
|
*
|
||||||
|
* A field holds ONE POSTING COLUMN PER KIND, so `category` may carry string
|
||||||
|
* values and number values at the same time and answer equality on each. This
|
||||||
|
* replaces the first-writer type freeze, under which the first value's type
|
||||||
|
* became the field's type and every later value of another kind was coerced —
|
||||||
|
* or, when coercion failed (`Number('electronics')`), dropped from the index
|
||||||
|
* with no error: the row stayed readable by id and by vector but vanished from
|
||||||
|
* every equality filter on that field.
|
||||||
|
*
|
||||||
|
* Kinds do not coerce into one another at query time either: `where {c: 5}`
|
||||||
|
* matches rows written with the NUMBER `5`, and `where {c: '5'}` matches rows
|
||||||
|
* written with the STRING `'5'`. Neither ever matches the other.
|
||||||
|
*
|
||||||
|
* Values that are none of these three (objects, bigints) index as strings —
|
||||||
|
* the same `String(value)` treatment they received before.
|
||||||
|
*/
|
||||||
|
export type PostingKind = 'number' | 'string' | 'boolean'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Every posting kind, in the order that defines cross-kind sort position.
|
||||||
|
*
|
||||||
|
* A mixed-kind field has no natural total order — a number does not compare
|
||||||
|
* with a string — so `sortTopK` orders by KIND first (numbers, then strings,
|
||||||
|
* then booleans) and by value within a kind. A single-kind field, which is
|
||||||
|
* nearly every field, sorts exactly as it always did.
|
||||||
|
*/
|
||||||
|
export const POSTING_KINDS: readonly PostingKind[] = ['number', 'string', 'boolean']
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Path segment marking a field's NON-PRIMARY kind columns on disk.
|
||||||
|
*
|
||||||
|
* The first kind a field ever sees keeps the historical layout —
|
||||||
|
* `<base>/<field>/MANIFEST.json` and `<base>/<field>/L0-NNNNNN` — so every
|
||||||
|
* index written before typed postings opens unchanged, and the byte-for-byte
|
||||||
|
* interchange with the native column store is untouched for the single-kind
|
||||||
|
* fields that are nearly all of them. A second kind arriving on the same field
|
||||||
|
* gets its own column at `<base>/<field>/k/<kind>/…` rather than overwriting or
|
||||||
|
* being coerced into the first.
|
||||||
|
*/
|
||||||
|
export const KIND_PATH_SEGMENT = 'k'
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Segment header and footer
|
// Segment header and footer
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
@ -267,6 +314,19 @@ export interface ColumnStoreProvider {
|
||||||
*/
|
*/
|
||||||
hasField(field: string): boolean
|
hasField(field: string): boolean
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Which value KINDS this field actually holds, in {@link POSTING_KINDS}
|
||||||
|
* order — the honest answer to "what type is this field?" for a field that
|
||||||
|
* carries more than one.
|
||||||
|
*
|
||||||
|
* OPTIONAL so an implementation written against the pre-typed-postings
|
||||||
|
* contract still satisfies this interface; feature-detect before calling.
|
||||||
|
*
|
||||||
|
* @param field - Field name
|
||||||
|
* @returns Every kind with at least one posting, or `[]` for an unknown field
|
||||||
|
*/
|
||||||
|
getFieldKinds?(field: string): PostingKind[]
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Flush all in-memory tail buffers to L0 segments on disk.
|
* Flush all in-memory tail buffers to L0 segments on disk.
|
||||||
* Saves all manifests.
|
* Saves all manifests.
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@
|
||||||
* 🧠 BRAINY EMBEDDED PATTERNS
|
* 🧠 BRAINY EMBEDDED PATTERNS
|
||||||
*
|
*
|
||||||
* AUTO-GENERATED - DO NOT EDIT
|
* AUTO-GENERATED - DO NOT EDIT
|
||||||
* Generated: 2025-09-29T10:10:00-07:00
|
* Generated: 2026-08-27T09:18:45-07:00
|
||||||
* Patterns: 220
|
* Patterns: 220
|
||||||
* Coverage: 94-98% of all queries
|
* Coverage: 94-98% of all queries
|
||||||
*
|
*
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@
|
||||||
* 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS
|
* 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS
|
||||||
*
|
*
|
||||||
* AUTO-GENERATED - DO NOT EDIT
|
* AUTO-GENERATED - DO NOT EDIT
|
||||||
* Generated: 2026-06-29T10:04:19-07:00
|
* Generated: 2026-08-27T09:18:45-07:00
|
||||||
* Noun Types: 42
|
* Noun Types: 42
|
||||||
* Verb Types: 127
|
* Verb Types: 127
|
||||||
*
|
*
|
||||||
|
|
@ -19,7 +19,7 @@ export const TYPE_METADATA = {
|
||||||
verbTypes: 127,
|
verbTypes: 127,
|
||||||
totalTypes: 169,
|
totalTypes: 169,
|
||||||
embeddingDimensions: 384,
|
embeddingDimensions: 384,
|
||||||
generatedAt: "2026-06-29T10:04:19-07:00",
|
generatedAt: "2026-08-27T09:18:45-07:00",
|
||||||
sizeBytes: {
|
sizeBytes: {
|
||||||
embeddings: 259584,
|
embeddings: 259584,
|
||||||
base64: 346112
|
base64: 346112
|
||||||
|
|
|
||||||
122
src/plugin.ts
122
src/plugin.ts
|
|
@ -411,7 +411,129 @@ export interface MetadataIndexProvider {
|
||||||
* @returns The matching id universe as an opaque set.
|
* @returns The matching id universe as an opaque set.
|
||||||
*/
|
*/
|
||||||
getIdSetForFilter?(filter: any): Promise<OpaqueIdSet>
|
getIdSetForFilter?(filter: any): Promise<OpaqueIdSet>
|
||||||
|
/**
|
||||||
|
* @description OPTIONAL: evaluate `filter` over `ids` ONLY and return the
|
||||||
|
* survivors in the caller's order — the door a graph-first
|
||||||
|
* `find({ connected, where })` walks. The neighbour set is the universe there,
|
||||||
|
* so the filter must cost O(|ids|) membership checks, never a whole-store
|
||||||
|
* materialization. A native index answers from its roaring filter result
|
||||||
|
* (membership by entity int); the reference index answers from its own
|
||||||
|
* `getIdsForFilter`, so the two doors can never disagree. Absent → Brainy
|
||||||
|
* intersects `getIdsForFilter`'s answer with `ids` itself (correct, O(store)).
|
||||||
|
* @param filter - The same filter shape accepted by `getIdsForFilter`.
|
||||||
|
* @param ids - The candidate ids (canonical). The answer is a subsequence.
|
||||||
|
*/
|
||||||
|
filterIdsWithin?(filter: any, ids: readonly string[]): Promise<string[]>
|
||||||
|
/**
|
||||||
|
* @description OPTIONAL: plan and execute a WHOLE `find()` — the graph
|
||||||
|
* traversal, the metadata filter, the ordering and the page — and answer the
|
||||||
|
* page's ids, or `null` for a shape this index does not plan.
|
||||||
|
*
|
||||||
|
* The doors above each serve one stage, so a `find()` that consults three of
|
||||||
|
* them crosses into the index three times and marshals a result set at every
|
||||||
|
* crossing. An index that can decide the stage ORDER itself does the whole
|
||||||
|
* thing in one call and materializes ids only for the page — a filter
|
||||||
|
* matching a hundred thousand rows then builds twenty-five id strings instead
|
||||||
|
* of a hundred thousand.
|
||||||
|
*
|
||||||
|
* The contract this door must keep, because Brainy cannot check it:
|
||||||
|
*
|
||||||
|
* - **The same answer.** Identical rows, in identical order, to what the
|
||||||
|
* stage doors would have produced for the same params. This door changes
|
||||||
|
* which code runs, never what the answer is.
|
||||||
|
* - **The law of the stages** (`find({ connected })` is graph-first): the
|
||||||
|
* neighbour set is the candidate universe, the filter is evaluated over
|
||||||
|
* those ids only, `orderBy` sorts the whole candidate set, and the page is
|
||||||
|
* cut LAST.
|
||||||
|
* - **`null` before work, not instead of an answer.** A shape the index does
|
||||||
|
* not plan must be handed back BEFORE any evaluation, so Brainy serves it
|
||||||
|
* through the stage doors exactly as it always has. Returning `null` after
|
||||||
|
* partial work, or an empty page for a shape it could not evaluate, is a
|
||||||
|
* silent wrong answer.
|
||||||
|
* - **`emptyAt` names the stage** that produced an empty page — `'graph'`,
|
||||||
|
* `'filter'`, `'visibility'` or `'none'` — so Brainy can apply its serving
|
||||||
|
* law to the right index. An empty answer from an index that is not
|
||||||
|
* serving must refuse loudly, and Brainy can only re-verify what it is told.
|
||||||
|
*
|
||||||
|
* Absent → every `find()` is served by the stage doors, which is Brainy's
|
||||||
|
* own behaviour and the ordering oracle for any implementation of this one.
|
||||||
|
* @param params - The find params, already normalized by `find()`
|
||||||
|
* (natural-language parsed, `connected` anchors resolved to canonical ids,
|
||||||
|
* an empty `where` dropped).
|
||||||
|
* @param hiddenIds - Ids this read must not return. The contract is the ANSWER, not the
|
||||||
|
* mechanism: a provider may subtract this set before paging, or derive the
|
||||||
|
* same exclusion from the params' visibility tiers itself — either way the
|
||||||
|
* page must equal the engine's own answer with none of these ids in it.
|
||||||
|
* @param graphIndex - The active graph provider, for a `connected` plan.
|
||||||
|
* @returns The page's ids plus the stage that emptied it, or `null`.
|
||||||
|
*/
|
||||||
|
planFindPage?(
|
||||||
|
params: any,
|
||||||
|
hiddenIds: readonly string[],
|
||||||
|
graphIndex: unknown
|
||||||
|
): Promise<{ ids: string[]; emptyAt: 'graph' | 'filter' | 'visibility' | 'none' } | null>
|
||||||
getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>>
|
getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>>
|
||||||
|
/**
|
||||||
|
* @description OPTIONAL: score `query` over `ids` ONLY — the text-leg twin of
|
||||||
|
* {@link filterIdsWithin}, and the door a hybrid `find({ query, where })`
|
||||||
|
* walks. The metadata filter's universe is the candidate set there, so the
|
||||||
|
* text leg must cost O(|ids|) membership checks and marshal at most `|ids|`
|
||||||
|
* rows, never the whole posting list of every query word. A native index
|
||||||
|
* intersects its own postings with the candidate set (membership by entity
|
||||||
|
* int) before any string crosses the boundary; the reference index answers
|
||||||
|
* from its own `getIdsForTextQuery`, so the two doors can never disagree.
|
||||||
|
* Absent → Brainy intersects `getIdsForTextQuery`'s answer with `ids` itself
|
||||||
|
* (correct, and still hydrate-last, but it marshals the whole answer).
|
||||||
|
*
|
||||||
|
* The answer keeps `getIdsForTextQuery`'s contract: `{ id, matchCount }`
|
||||||
|
* sorted by `matchCount` descending, ties in the order the whole-store answer
|
||||||
|
* would have produced. Only rows in `ids` may appear.
|
||||||
|
* @param query - The same text query accepted by `getIdsForTextQuery`.
|
||||||
|
* @param ids - The candidate ids (canonical). The answer is a subset.
|
||||||
|
*/
|
||||||
|
getIdsForTextQueryWithin?(
|
||||||
|
query: string,
|
||||||
|
ids: readonly string[]
|
||||||
|
): Promise<Array<{ id: string; matchCount: number }>>
|
||||||
|
/**
|
||||||
|
* @description OPTIONAL: read named SCALAR fields for many ids at once, from
|
||||||
|
* the index's own value storage, WITHOUT touching the canonical record.
|
||||||
|
*
|
||||||
|
* This is the door behind `find/get/related({ fields })`. A list view that
|
||||||
|
* needs a title and a slug currently hydrates the whole record for every row
|
||||||
|
* — document bodies included — and then discards almost all of it. Serving
|
||||||
|
* the named scalars from the index turns that into an index read.
|
||||||
|
*
|
||||||
|
* ## The contract, and the one rule that makes it safe
|
||||||
|
*
|
||||||
|
* **Return only what you can serve EXACTLY, and say what you served.** The
|
||||||
|
* answer is a per-id map of the fields this index actually resolved; the
|
||||||
|
* caller diffs it against what was requested and reads the canonical record
|
||||||
|
* for the remainder. An implementation must therefore OMIT a field rather
|
||||||
|
* than approximate it — and omission costs only a record read, while a wrong
|
||||||
|
* value is a wrong answer nobody can see.
|
||||||
|
*
|
||||||
|
* That rule is not hypothetical. This engine's own index buckets
|
||||||
|
* `system.createdAt` and `system.updatedAt` to the minute for range queries,
|
||||||
|
* so it cannot serve them exactly and omits them. An engine whose column
|
||||||
|
* store holds raw values can serve the same fields — so the two answer
|
||||||
|
* differently in COST and identically in CONTENT, which is the only
|
||||||
|
* difference a projection door is allowed to have.
|
||||||
|
*
|
||||||
|
* A field absent from an entity is simply absent from that entity's map. It
|
||||||
|
* is never an error, and never a `null` standing in for one: absent and
|
||||||
|
* present-and-null are different answers.
|
||||||
|
*
|
||||||
|
* @param ids - Canonical entity ids to read.
|
||||||
|
* @param fields - Index KEYS (bare = user metadata, `system.*` = engine
|
||||||
|
* scalar), already address-resolved by the caller.
|
||||||
|
* @returns `id → { field: value }` for the fields this index served exactly.
|
||||||
|
* Ids with nothing to serve may be omitted entirely.
|
||||||
|
*/
|
||||||
|
getScalarsForIds?(
|
||||||
|
ids: readonly string[],
|
||||||
|
fields: readonly string[]
|
||||||
|
): Promise<Map<string, Record<string, unknown>>>
|
||||||
getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise<string[]>
|
getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise<string[]>
|
||||||
getFilterValues(field: string): Promise<string[]>
|
getFilterValues(field: string): Promise<string[]>
|
||||||
getFilterFields(): Promise<string[]>
|
getFilterFields(): Promise<string[]>
|
||||||
|
|
|
||||||
|
|
@ -1089,6 +1089,10 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
|
|
||||||
// Counts changed since the last persist? Drives the write-through flush.
|
// Counts changed since the last persist? Drives the write-through flush.
|
||||||
protected pendingCountPersist = false
|
protected pendingCountPersist = false
|
||||||
|
/** The one persist running right now, if any (single-flight law — see flushCounts). */
|
||||||
|
private countPersistInFlight: Promise<void> | null = null
|
||||||
|
/** The one trailing persist a burst has queued behind the in-flight one. */
|
||||||
|
private countPersistTrailing: Promise<void> | null = null
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get total noun count - O(1) operation
|
* Get total noun count - O(1) operation
|
||||||
|
|
@ -1341,15 +1345,46 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
// SINGLE-FLIGHT, COALESCED. Counts are write-through on every change, so
|
||||||
// Persist to storage (implemented by subclass)
|
// a burst of writes used to launch one persist per change, all in flight
|
||||||
await this.persistCounts()
|
// together. Two of them inside the same millisecond shared the atomic
|
||||||
this.pendingCountPersist = false
|
// writer's temp path (`.tmp-<pid>-<ms>`): both wrote it, the first rename
|
||||||
} catch (error) {
|
// consumed it, the second rename found nothing — ENOENT, ~1,500 times a
|
||||||
console.error('CRITICAL: Failed to flush counts to storage:', error)
|
// day on a busy production brain, with a full ledger write per change
|
||||||
// Keep pending flag set so we retry on next operation
|
// behind it. Now exactly one persist runs at a time; requests that arrive
|
||||||
throw error
|
// while it runs collapse into ONE trailing persist that carries the final
|
||||||
|
// state. A burst of N changes costs at most two writes and never races
|
||||||
|
// itself.
|
||||||
|
if (this.countPersistInFlight) {
|
||||||
|
// The in-flight write may have already serialised a stale snapshot —
|
||||||
|
// ask for one more pass after it, and let every caller in this burst
|
||||||
|
// await that same pass.
|
||||||
|
if (!this.countPersistTrailing) {
|
||||||
|
this.countPersistTrailing = this.countPersistInFlight
|
||||||
|
.catch(() => undefined)
|
||||||
|
.then(() => {
|
||||||
|
this.countPersistTrailing = null
|
||||||
|
return this.flushCounts()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return this.countPersistTrailing
|
||||||
}
|
}
|
||||||
|
|
||||||
|
this.countPersistInFlight = (async () => {
|
||||||
|
try {
|
||||||
|
// Persist to storage (implemented by subclass)
|
||||||
|
this.pendingCountPersist = false
|
||||||
|
await this.persistCounts()
|
||||||
|
} catch (error) {
|
||||||
|
// Keep the flag set so the next operation retries.
|
||||||
|
this.pendingCountPersist = true
|
||||||
|
console.error('CRITICAL: Failed to flush counts to storage:', error)
|
||||||
|
throw error
|
||||||
|
} finally {
|
||||||
|
this.countPersistInFlight = null
|
||||||
|
}
|
||||||
|
})()
|
||||||
|
return this.countPersistInFlight
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -2400,8 +2400,15 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
* Atomic write via temp-file-then-rename so concurrent readers never see a
|
* Atomic write via temp-file-then-rename so concurrent readers never see a
|
||||||
* half-written lock JSON. Reused by writer-lock writes + heartbeat.
|
* half-written lock JSON. Reused by writer-lock writes + heartbeat.
|
||||||
*/
|
*/
|
||||||
|
/** Monotonic per-process sequence so two atomic writes never share a temp path. */
|
||||||
|
private static atomicWriteSeq = 0
|
||||||
|
|
||||||
private async writeFileAtomic(filePath: string, contents: string): Promise<void> {
|
private async writeFileAtomic(filePath: string, contents: string): Promise<void> {
|
||||||
const tmp = `${filePath}.tmp-${process.pid}-${Date.now()}`
|
// pid + timestamp alone collided: two writers of the same target inside
|
||||||
|
// one millisecond shared this path, and the loser's rename found the
|
||||||
|
// winner had already moved it (ENOENT). The sequence makes every call's
|
||||||
|
// temp path its own.
|
||||||
|
const tmp = `${filePath}.tmp-${process.pid}-${Date.now()}-${++FileSystemStorage.atomicWriteSeq}`
|
||||||
await fs.promises.writeFile(tmp, contents)
|
await fs.promises.writeFile(tmp, contents)
|
||||||
await fs.promises.rename(tmp, filePath)
|
await fs.promises.rename(tmp, filePath)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -2942,19 +2942,33 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
!options.filter.service &&
|
!options.filter.service &&
|
||||||
!options.filter.metadata
|
!options.filter.metadata
|
||||||
) {
|
) {
|
||||||
const sourceId = Array.isArray(options.filter.sourceId)
|
const sourceIds = Array.isArray(options.filter.sourceId)
|
||||||
? options.filter.sourceId[0]
|
? options.filter.sourceId
|
||||||
: options.filter.sourceId
|
: [options.filter.sourceId]
|
||||||
|
|
||||||
const verbType = Array.isArray(options.filter.verbType)
|
// EVERY requested verb type is honoured — an array used to collapse to
|
||||||
? options.filter.verbType[0]
|
// its first element here, silently dropping the rest of the ask.
|
||||||
: options.filter.verbType
|
const verbTypes = new Set(
|
||||||
|
Array.isArray(options.filter.verbType)
|
||||||
|
? options.filter.verbType
|
||||||
|
: [options.filter.verbType]
|
||||||
|
)
|
||||||
|
|
||||||
// Get verbs by source, then filter by type (O(1) graph lookup + O(n) type filter),
|
// Get verbs by source (union over every requested source), filter by the
|
||||||
// then apply the subtype / visibility metadata filters on the candidate set.
|
// requested type SET (O(1) graph lookup + O(n) type filter), then apply
|
||||||
const verbsBySource = await this.getVerbsBySource_internal(sourceId)
|
// the subtype / visibility metadata filters on the candidate set.
|
||||||
|
const bySource: HNSWVerbWithMetadata[] = []
|
||||||
|
const seenVerbIds = new Set<string>()
|
||||||
|
for (const oneSource of sourceIds) {
|
||||||
|
for (const v of await this.getVerbsBySource_internal(oneSource)) {
|
||||||
|
if (!seenVerbIds.has(v.id)) {
|
||||||
|
seenVerbIds.add(v.id)
|
||||||
|
bySource.push(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
const filteredVerbs = this.applyVerbMetadataFilters(
|
const filteredVerbs = this.applyVerbMetadataFilters(
|
||||||
verbsBySource.filter(v => v.verb === verbType),
|
bySource.filter(v => verbTypes.has(v.verb)),
|
||||||
options.filter
|
options.filter
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -2985,16 +2999,22 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
!options.filter.service &&
|
!options.filter.service &&
|
||||||
!options.filter.metadata
|
!options.filter.metadata
|
||||||
) {
|
) {
|
||||||
const sourceId = Array.isArray(options.filter.sourceId)
|
// EVERY requested source is honoured — an array used to collapse to
|
||||||
? options.filter.sourceId[0]
|
// its first element here, silently dropping the rest of the ask.
|
||||||
: options.filter.sourceId
|
const onlySourceIds = Array.isArray(options.filter.sourceId)
|
||||||
|
? options.filter.sourceId
|
||||||
// Get verbs by source directly (hydrated with metadata), then apply the
|
: [options.filter.sourceId]
|
||||||
// subtype / visibility metadata filters on the O(degree) candidate set.
|
const sourceUnion: HNSWVerbWithMetadata[] = []
|
||||||
const verbsBySource = this.applyVerbMetadataFilters(
|
const seenSourceVerbIds = new Set<string>()
|
||||||
await this.getVerbsBySource_internal(sourceId),
|
for (const oneSource of onlySourceIds) {
|
||||||
options.filter
|
for (const v of await this.getVerbsBySource_internal(oneSource)) {
|
||||||
)
|
if (!seenSourceVerbIds.has(v.id)) {
|
||||||
|
seenSourceVerbIds.add(v.id)
|
||||||
|
sourceUnion.push(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const verbsBySource = this.applyVerbMetadataFilters(sourceUnion, options.filter)
|
||||||
|
|
||||||
// Apply pagination
|
// Apply pagination
|
||||||
const paginatedVerbs = verbsBySource.slice(offset, offset + limit)
|
const paginatedVerbs = verbsBySource.slice(offset, offset + limit)
|
||||||
|
|
@ -3023,16 +3043,22 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
!options.filter.service &&
|
!options.filter.service &&
|
||||||
!options.filter.metadata
|
!options.filter.metadata
|
||||||
) {
|
) {
|
||||||
const targetId = Array.isArray(options.filter.targetId)
|
// EVERY requested target is honoured — an array used to collapse to
|
||||||
? options.filter.targetId[0]
|
// its first element here, silently dropping the rest of the ask.
|
||||||
: options.filter.targetId
|
const onlyTargetIds = Array.isArray(options.filter.targetId)
|
||||||
|
? options.filter.targetId
|
||||||
// Get verbs by target directly (hydrated with metadata), then apply the
|
: [options.filter.targetId]
|
||||||
// subtype / visibility metadata filters on the O(degree) candidate set.
|
const targetUnion: HNSWVerbWithMetadata[] = []
|
||||||
const verbsByTarget = this.applyVerbMetadataFilters(
|
const seenTargetVerbIds = new Set<string>()
|
||||||
await this.getVerbsByTarget_internal(targetId),
|
for (const oneTarget of onlyTargetIds) {
|
||||||
options.filter
|
for (const v of await this.getVerbsByTarget_internal(oneTarget)) {
|
||||||
)
|
if (!seenTargetVerbIds.has(v.id)) {
|
||||||
|
seenTargetVerbIds.add(v.id)
|
||||||
|
targetUnion.push(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const verbsByTarget = this.applyVerbMetadataFilters(targetUnion, options.filter)
|
||||||
|
|
||||||
// Apply pagination
|
// Apply pagination
|
||||||
const paginatedVerbs = verbsByTarget.slice(offset, offset + limit)
|
const paginatedVerbs = verbsByTarget.slice(offset, offset + limit)
|
||||||
|
|
@ -3061,16 +3087,25 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
!options.filter.service &&
|
!options.filter.service &&
|
||||||
!options.filter.metadata
|
!options.filter.metadata
|
||||||
) {
|
) {
|
||||||
const verbType = Array.isArray(options.filter.verbType)
|
// EVERY requested verb type is honoured — an array used to collapse to
|
||||||
? options.filter.verbType[0]
|
// its first element here, silently dropping the rest of the ask.
|
||||||
: options.filter.verbType
|
const verbTypes = Array.isArray(options.filter.verbType)
|
||||||
|
? options.filter.verbType
|
||||||
|
: [options.filter.verbType]
|
||||||
|
|
||||||
// Get verbs by type directly (hydrated with metadata), then apply the
|
// Get verbs by each requested type (hydrated with metadata), deduped by
|
||||||
// subtype / visibility metadata filters on the candidate set.
|
// id, then apply the subtype / visibility metadata filters on the set.
|
||||||
const verbsByType = this.applyVerbMetadataFilters(
|
const byType: HNSWVerbWithMetadata[] = []
|
||||||
await this.getVerbsByType_internal(verbType),
|
const seenTypeVerbIds = new Set<string>()
|
||||||
options.filter
|
for (const oneType of verbTypes) {
|
||||||
)
|
for (const v of await this.getVerbsByType_internal(oneType)) {
|
||||||
|
if (!seenTypeVerbIds.has(v.id)) {
|
||||||
|
seenTypeVerbIds.add(v.id)
|
||||||
|
byType.push(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const verbsByType = this.applyVerbMetadataFilters(byType, options.filter)
|
||||||
|
|
||||||
// Apply pagination
|
// Apply pagination
|
||||||
const paginatedVerbs = verbsByType.slice(offset, offset + limit)
|
const paginatedVerbs = verbsByType.slice(offset, offset + limit)
|
||||||
|
|
|
||||||
|
|
@ -14,6 +14,7 @@ import type { MetadataIndexManager } from '../../utils/metadataIndex.js'
|
||||||
import type { GraphVerb } from '../../coreTypes.js'
|
import type { GraphVerb } from '../../coreTypes.js'
|
||||||
import type { Operation, RollbackAction } from '../types.js'
|
import type { Operation, RollbackAction } from '../types.js'
|
||||||
import { isZeroNormVector } from '../../utils/distance.js'
|
import { isZeroNormVector } from '../../utils/distance.js'
|
||||||
|
import { jsonSafeIndexMetadata } from '../../utils/jsonSafeIndexMetadata.js'
|
||||||
import { prodLog } from '../../utils/logger.js'
|
import { prodLog } from '../../utils/logger.js'
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -390,13 +391,21 @@ export class AddToMetadataIndexOperation implements Operation {
|
||||||
// rollback so add + undo reference the same watermark.
|
// rollback so add + undo reference the same watermark.
|
||||||
const generation = this.generationFn?.()
|
const generation = this.generationFn?.()
|
||||||
|
|
||||||
// Add to metadata index (skipFlush=true for transaction atomicity)
|
// The JSON-safe view is taken HERE, per crossing, never at construction:
|
||||||
await this.index.addToIndex(this.id, this.entity, true, false, generation)
|
// the entity reference this op holds can be mutated between plan and
|
||||||
|
// execute (a graph op's execute-time endpoint-int resolution mirrors
|
||||||
|
// BigInts onto a shared verb object) — see jsonSafeIndexMetadata's
|
||||||
|
// module doc.
|
||||||
|
await this.index.addToIndex(
|
||||||
|
this.id, jsonSafeIndexMetadata(this.entity), true, false, generation
|
||||||
|
)
|
||||||
|
|
||||||
// Return rollback action
|
// Return rollback action
|
||||||
return async () => {
|
return async () => {
|
||||||
// Remove from metadata index
|
// Remove from metadata index
|
||||||
await this.index.removeFromIndex(this.id, this.entity, generation)
|
await this.index.removeFromIndex(
|
||||||
|
this.id, jsonSafeIndexMetadata(this.entity), generation
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -432,13 +441,21 @@ export class RemoveFromMetadataIndexOperation implements Operation {
|
||||||
// Resolve the removal generation once; reuse it for the rollback re-add.
|
// Resolve the removal generation once; reuse it for the rollback re-add.
|
||||||
const generation = this.generationFn?.()
|
const generation = this.generationFn?.()
|
||||||
|
|
||||||
// Remove from metadata index
|
// Sanitized per crossing, never at construction — transact()'s delete
|
||||||
await this.index.removeFromIndex(this.id, this.entity, generation)
|
// legs hand this op the SAME verb object the graph-retraction op's
|
||||||
|
// execute-time endpoint resolution mutates (BigInt sourceInt/targetInt),
|
||||||
|
// so a plan-time view aliases the pollution. See jsonSafeIndexMetadata's
|
||||||
|
// module doc.
|
||||||
|
await this.index.removeFromIndex(
|
||||||
|
this.id, jsonSafeIndexMetadata(this.entity), generation
|
||||||
|
)
|
||||||
|
|
||||||
// Return rollback action
|
// Return rollback action
|
||||||
return async () => {
|
return async () => {
|
||||||
// Re-add with original metadata (skipFlush=true)
|
// Re-add with original metadata (skipFlush=true)
|
||||||
await this.index.addToIndex(this.id, this.entity, true, false, generation)
|
await this.index.addToIndex(
|
||||||
|
this.id, jsonSafeIndexMetadata(this.entity), true, false, generation
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -561,6 +561,33 @@ export interface UpdateRelationParams<T = any> {
|
||||||
* refusal with the fix in hand beats a silent behavior flip.
|
* refusal with the fix in hand beats a silent behavior flip.
|
||||||
*/
|
*/
|
||||||
export interface FindParams<T = any> {
|
export interface FindParams<T = any> {
|
||||||
|
/**
|
||||||
|
* **Field projection** — return only these fields on each row, instead of the
|
||||||
|
* whole record.
|
||||||
|
*
|
||||||
|
* A list view that shows a title and a slug does not need the document body,
|
||||||
|
* yet without a projection every row hydrates its full record and throws
|
||||||
|
* almost all of it away. Naming the fields lets them be served from the index
|
||||||
|
* itself: a scalar the index holds exactly is read from the index, and the
|
||||||
|
* canonical record is opened ONLY when a requested field cannot be.
|
||||||
|
*
|
||||||
|
* Field names follow the one addressing law: a bare name is the user's
|
||||||
|
* metadata (`'title'`), and `system.*` is an engine scalar
|
||||||
|
* (`'system.createdAt'`).
|
||||||
|
*
|
||||||
|
* - **Absent** ⇒ the full record, exactly as before.
|
||||||
|
* - A requested field the entity does not carry is simply **absent** from the
|
||||||
|
* row. It is never an error — a projection asks "give me these if you have
|
||||||
|
* them", so an optional field must not turn a list into a failure.
|
||||||
|
* - Every returned row carries `id` (and, on `find`, `score`) regardless: a
|
||||||
|
* row you cannot identify is not a row.
|
||||||
|
*
|
||||||
|
* @example
|
||||||
|
* // A list page: two user fields and one engine scalar, no document bodies.
|
||||||
|
* await brain.find({ where: { kind: 'post' }, fields: ['title', 'slug', 'system.createdAt'], limit: 50 })
|
||||||
|
*/
|
||||||
|
fields?: readonly string[]
|
||||||
|
|
||||||
// Vector Intelligence
|
// Vector Intelligence
|
||||||
/** Natural language or semantic search query (embedded and matched via HNSW + text index) */
|
/** Natural language or semantic search query (embedded and matched via HNSW + text index) */
|
||||||
query?: string
|
query?: string
|
||||||
|
|
@ -789,6 +816,12 @@ export interface SimilarParams<T = any> {
|
||||||
* Added string ID shorthand syntax
|
* Added string ID shorthand syntax
|
||||||
*/
|
*/
|
||||||
export interface RelatedParams {
|
export interface RelatedParams {
|
||||||
|
// NOTE: `fields` is deliberately NOT offered here. A Relation carries `from`
|
||||||
|
// and `to` as IDS and hydrates no entity record, so there is nothing for a
|
||||||
|
// projection to trim — the param would be decorative. Projecting the
|
||||||
|
// ENDPOINTS would be a new capability (related() hydrating entities), not a
|
||||||
|
// projection of an existing one, and it belongs in its own decision.
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Filter by source entity ID
|
* Filter by source entity ID
|
||||||
*
|
*
|
||||||
|
|
@ -1414,6 +1447,33 @@ export interface ImportResult {
|
||||||
*
|
*
|
||||||
*/
|
*/
|
||||||
export interface GetOptions {
|
export interface GetOptions {
|
||||||
|
/**
|
||||||
|
* **Field projection** — return only these fields on each row, instead of the
|
||||||
|
* whole record.
|
||||||
|
*
|
||||||
|
* A list view that shows a title and a slug does not need the document body,
|
||||||
|
* yet without a projection every row hydrates its full record and throws
|
||||||
|
* almost all of it away. Naming the fields lets them be served from the index
|
||||||
|
* itself: a scalar the index holds exactly is read from the index, and the
|
||||||
|
* canonical record is opened ONLY when a requested field cannot be.
|
||||||
|
*
|
||||||
|
* Field names follow the one addressing law: a bare name is the user's
|
||||||
|
* metadata (`'title'`), and `system.*` is an engine scalar
|
||||||
|
* (`'system.createdAt'`).
|
||||||
|
*
|
||||||
|
* - **Absent** ⇒ the full record, exactly as before.
|
||||||
|
* - A requested field the entity does not carry is simply **absent** from the
|
||||||
|
* row. It is never an error — a projection asks "give me these if you have
|
||||||
|
* them", so an optional field must not turn a list into a failure.
|
||||||
|
* - Every returned row carries `id` (and, on `find`, `score`) regardless: a
|
||||||
|
* row you cannot identify is not a row.
|
||||||
|
*
|
||||||
|
* @example
|
||||||
|
* // A list page: two user fields and one engine scalar, no document bodies.
|
||||||
|
* await brain.find({ where: { kind: 'post' }, fields: ['title', 'slug', 'system.createdAt'], limit: 50 })
|
||||||
|
*/
|
||||||
|
fields?: readonly string[]
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Include 384-dimensional vector embeddings in the response
|
* Include 384-dimensional vector embeddings in the response
|
||||||
*
|
*
|
||||||
|
|
|
||||||
|
|
@ -55,8 +55,30 @@ export enum FieldType {
|
||||||
*/
|
*/
|
||||||
export interface FieldTypeInfo {
|
export interface FieldTypeInfo {
|
||||||
field: string
|
field: string
|
||||||
|
/**
|
||||||
|
* The DOMINANT reading of the field — one type, the most specific one every
|
||||||
|
* sampled value satisfies.
|
||||||
|
*
|
||||||
|
* A field is not obliged to hold one kind, so this is not the whole answer
|
||||||
|
* for a field that holds several. Read {@link kinds} beside it: a field
|
||||||
|
* carrying `'electronics'` and `5` infers as STRING here and reports
|
||||||
|
* `['number', 'string']` there, and the metadata index keeps a separate
|
||||||
|
* posting column for each of them.
|
||||||
|
*/
|
||||||
inferredType: FieldType
|
inferredType: FieldType
|
||||||
confidence: number // 0-1 confidence score
|
confidence: number // 0-1 confidence score
|
||||||
|
/**
|
||||||
|
* Every value KIND observed in the sample, in the order
|
||||||
|
* number → string → boolean. More than one entry means a genuinely
|
||||||
|
* mixed field, and every one of those kinds is independently filterable.
|
||||||
|
*
|
||||||
|
* Kinds are JavaScript `typeof` classes, one level coarser than
|
||||||
|
* {@link FieldType}: a UUID and a category name are both `'string'`, and an
|
||||||
|
* integer and a timestamp are both `'number'`.
|
||||||
|
*
|
||||||
|
* Optional only for cached analyses written before this was reported.
|
||||||
|
*/
|
||||||
|
kinds?: Array<'number' | 'string' | 'boolean'>
|
||||||
sampleSize: number // Number of values analyzed
|
sampleSize: number // Number of values analyzed
|
||||||
lastUpdated: number // Timestamp of last analysis
|
lastUpdated: number // Timestamp of last analysis
|
||||||
detectionMethod: 'value' // Always 'value' (no fallbacks!)
|
detectionMethod: 'value' // Always 'value' (no fallbacks!)
|
||||||
|
|
@ -133,14 +155,71 @@ export class FieldTypeInference {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Analyze values to determine field type
|
* Analyze values to determine field type, and report every KIND the field
|
||||||
|
* actually holds alongside it.
|
||||||
|
*
|
||||||
|
* The classification below picks ONE type, because every one of its
|
||||||
|
* heuristics asks `samples.every(...)`: a field carrying `'electronics'` and
|
||||||
|
* `5` satisfies none of them and lands on STRING. That single answer is true
|
||||||
|
* as far as it goes — string is the dominant reading — but on its own it
|
||||||
|
* says nothing about the numbers also in the field, and a caller that treats
|
||||||
|
* it as the field's only type reproduces the first-writer freeze the index
|
||||||
|
* itself no longer has. {@link FieldTypeInfo.kinds} carries the rest.
|
||||||
|
*/
|
||||||
|
private async analyzeValues(field: string, values: any[]): Promise<FieldTypeInfo> {
|
||||||
|
const info = await this.classifyValues(field, values)
|
||||||
|
info.kinds = FieldTypeInference.observedKinds(values)
|
||||||
|
if (info.kinds.length > 1 && info.metadata) {
|
||||||
|
info.metadata.format = `${info.metadata.format} (field also holds: ${info.kinds
|
||||||
|
.filter((k) => k !== FieldTypeInference.kindOfType(info.inferredType))
|
||||||
|
.join(', ')})`
|
||||||
|
}
|
||||||
|
return info
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The distinct value kinds present in a sample, in a stable order.
|
||||||
|
*
|
||||||
|
* Kinds are JavaScript `typeof` classes — the same classes the metadata
|
||||||
|
* index keeps separate posting columns for — not the finer
|
||||||
|
* {@link FieldType} readings, which are interpretations layered on top of
|
||||||
|
* them (a UUID and a category name are both the `string` kind).
|
||||||
|
*/
|
||||||
|
private static observedKinds(values: any[]): Array<'number' | 'string' | 'boolean'> {
|
||||||
|
const order: Array<'number' | 'string' | 'boolean'> = ['number', 'string', 'boolean']
|
||||||
|
const seen = new Set<'number' | 'string' | 'boolean'>()
|
||||||
|
for (const v of values) {
|
||||||
|
if (v === null || v === undefined) continue
|
||||||
|
const t = typeof v
|
||||||
|
seen.add(t === 'number' ? 'number' : t === 'boolean' ? 'boolean' : 'string')
|
||||||
|
}
|
||||||
|
return order.filter((k) => seen.has(k))
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The value kind a {@link FieldType} reading is an interpretation of. */
|
||||||
|
private static kindOfType(type: FieldType): 'number' | 'string' | 'boolean' {
|
||||||
|
switch (type) {
|
||||||
|
case FieldType.BOOLEAN:
|
||||||
|
return 'boolean'
|
||||||
|
case FieldType.INTEGER:
|
||||||
|
case FieldType.FLOAT:
|
||||||
|
case FieldType.TIMESTAMP_MS:
|
||||||
|
case FieldType.TIMESTAMP_S:
|
||||||
|
return 'number'
|
||||||
|
default:
|
||||||
|
return 'string'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Classify values into a single field type.
|
||||||
*
|
*
|
||||||
* Uses DuckDB-inspired type detection order:
|
* Uses DuckDB-inspired type detection order:
|
||||||
* BOOLEAN → INTEGER → FLOAT → DATE → TIMESTAMP → UUID → STRING
|
* BOOLEAN → INTEGER → FLOAT → DATE → TIMESTAMP → UUID → STRING
|
||||||
*
|
*
|
||||||
* No fallbacks - pure value-based detection
|
* No fallbacks - pure value-based detection
|
||||||
*/
|
*/
|
||||||
private async analyzeValues(field: string, values: any[]): Promise<FieldTypeInfo> {
|
private async classifyValues(field: string, values: any[]): Promise<FieldTypeInfo> {
|
||||||
// Filter null/undefined values
|
// Filter null/undefined values
|
||||||
const validValues = values.filter(v => v !== null && v !== undefined)
|
const validValues = values.filter(v => v !== null && v !== undefined)
|
||||||
|
|
||||||
|
|
|
||||||
47
src/utils/jsonSafeIndexMetadata.ts
Normal file
47
src/utils/jsonSafeIndexMetadata.ts
Normal file
|
|
@ -0,0 +1,47 @@
|
||||||
|
/**
|
||||||
|
* @module utils/jsonSafeIndexMetadata
|
||||||
|
* @description The metadata-index crossing's JSON-safety law, as a leaf
|
||||||
|
* function both the coordinator and the transaction operations share.
|
||||||
|
*
|
||||||
|
* The seam's metadata is JSON-safe BY CONTRACT (a native provider serializes
|
||||||
|
* it; u64 ints as Number corrupt above 2^53) — but `resolveVerbEndpointInts`
|
||||||
|
* MIRRORS the resolved endpoint ints onto the verb object itself as BigInt
|
||||||
|
* (`verb.sourceInt`/`targetInt`), so a verb object reused as index metadata
|
||||||
|
* carries BigInts into JSON.stringify, which throws, aborting the whole
|
||||||
|
* transaction. Endpoint ints ride their OWN op params on the graph legs — the
|
||||||
|
* metadata crossing drops every BigInt-valued top-level key instead of
|
||||||
|
* guessing at a lossy numeric encoding.
|
||||||
|
*
|
||||||
|
* WHY THIS IS A LEAF MODULE, ENFORCED AT THE CROSSING: sanitizing only at
|
||||||
|
* operation-construction time is not enough. `transact()`'s delete legs pass
|
||||||
|
* the SAME verb object to both the graph-retraction op (whose endpoint-int
|
||||||
|
* thunk deliberately resolves at EXECUTE time, for same-batch forward refs)
|
||||||
|
* and the metadata-retraction op. At plan time the verb is still clean, so a
|
||||||
|
* plan-time sanitize returns the same reference — then the graph op executes
|
||||||
|
* first, mirrors the BigInt ints onto the shared object, and the metadata op
|
||||||
|
* crosses the seam with them (found by the first fleet adoption of the native
|
||||||
|
* pair: every transact-wrapped edge delete aborted). The crossing itself is
|
||||||
|
* the only place ordering cannot bypass.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A JSON-safe view of a record bound for the metadata-index crossing.
|
||||||
|
*
|
||||||
|
* @param metadata - The candidate index-metadata record.
|
||||||
|
* @returns The same object when already JSON-safe, else a shallow copy
|
||||||
|
* without the BigInt-valued keys.
|
||||||
|
*/
|
||||||
|
export function jsonSafeIndexMetadata(metadata: unknown): unknown {
|
||||||
|
if (metadata === null || typeof metadata !== 'object') return metadata
|
||||||
|
const rec = metadata as Record<string, unknown>
|
||||||
|
let hasBigint = false
|
||||||
|
for (const k in rec) {
|
||||||
|
if (typeof rec[k] === 'bigint') { hasBigint = true; break }
|
||||||
|
}
|
||||||
|
if (!hasBigint) return metadata
|
||||||
|
const out: Record<string, unknown> = {}
|
||||||
|
for (const k in rec) {
|
||||||
|
if (typeof rec[k] !== 'bigint') out[k] = rec[k]
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
@ -40,7 +40,7 @@ import {
|
||||||
import { EntityIdMapper } from './entityIdMapper.js'
|
import { EntityIdMapper } from './entityIdMapper.js'
|
||||||
import { RoaringBitmap32, roaringLibraryInitialize } from './roaring/index.js'
|
import { RoaringBitmap32, roaringLibraryInitialize } from './roaring/index.js'
|
||||||
import { FieldTypeInference, FieldType } from './fieldTypeInference.js'
|
import { FieldTypeInference, FieldType } from './fieldTypeInference.js'
|
||||||
import { BrainyError } from '../errors/brainyError.js'
|
import { BrainyError, MAX_INDEXED_ARRAY_LENGTH } from '../errors/brainyError.js'
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Fields whose values are stored in the sparse index as BUCKETED values
|
* Fields whose values are stored in the sparse index as BUCKETED values
|
||||||
|
|
@ -289,8 +289,10 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
||||||
// No name-based exclude/allow lists — the field-addressing law: every
|
// No name-based exclude/allow lists — the field-addressing law: every
|
||||||
// user field indexes, whatever its name ('content', 'data', 'id',
|
// user field indexes, whatever its name ('content', 'data', 'id',
|
||||||
// 'vector', … included). Bulk payloads are kept out by uniform value-
|
// 'vector', … included). Bulk payloads are kept out by uniform value-
|
||||||
// SHAPE rules in extractIndexableFields (arrays >10 never become
|
// SHAPE rules in extractIndexableFields (arrays longer than
|
||||||
// posting scalars; >100-char values index hashed), never by name.
|
// MAX_INDEXED_ARRAY_LENGTH never become posting scalars, and the write
|
||||||
|
// door refuses them by name; >100-char values index hashed), never by
|
||||||
|
// field name.
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize metadata cache with similar config to search cache
|
// Initialize metadata cache with similar config to search cache
|
||||||
|
|
@ -961,9 +963,41 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get IDs for a range using chunked sparse index with zone maps and roaring bitmaps
|
* Get IDs for a range using the legacy chunked sparse index (zone maps +
|
||||||
* Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
|
* roaring bitmaps). Lazy-loaded via UnifiedCache.
|
||||||
* Normalize min/max for timestamp bucketing before comparison
|
*
|
||||||
|
* ORDER IS NOT A KEY. This path compares NORMALIZED values, and
|
||||||
|
* {@link normalizeValue} carries an escape hatch that is order-destroying by
|
||||||
|
* design: a string over 100 characters is replaced by {@link hashValue}'s
|
||||||
|
* digest so it can be used as a filesystem-safe key. Feeding that digest to
|
||||||
|
* an ORDERING comparison — which is what a `gte` / `lt` / `between` does —
|
||||||
|
* ranks rows by hash. The result is not empty and not an error: it is a
|
||||||
|
* confidently ordered wrong answer, and it disagrees with the column-store
|
||||||
|
* path (`getIdsForRange` above), which compares raw values and is correct.
|
||||||
|
*
|
||||||
|
* Two changes hold the line here:
|
||||||
|
*
|
||||||
|
* 1. THE BOUNDS ARE NEVER HASHED. They are normalized with `allowHash =
|
||||||
|
* false`, so a long bound stays comparable instead of collapsing to a
|
||||||
|
* digest. This alone fixes the common shape — a long bound queried
|
||||||
|
* against ordinary short values, where the digest sorts below every
|
||||||
|
* letter and `gte` therefore matched the entire store.
|
||||||
|
*
|
||||||
|
* 2. A HASHED KEY IS REFUSED, NEVER GUESSED. The persisted keys are whatever
|
||||||
|
* the pre-7.20.0 writer normalized them to, so a field whose values ran
|
||||||
|
* long is stored hashed and its order is simply not recoverable from this
|
||||||
|
* index. Rather than compare digests, the query throws a typed
|
||||||
|
* `BrainyError('INVALID_QUERY')` naming the field, the bound and the cure.
|
||||||
|
* Loud beats wrong.
|
||||||
|
*
|
||||||
|
* KNOWN, NAMED DIVERGENCE. The persisted keys are also lower-cased and
|
||||||
|
* trimmed by `normalizeValue`, so this path's string ranges are
|
||||||
|
* CASE-INSENSITIVE where the column store's are not. That is a property of
|
||||||
|
* the bytes a pre-7.20.0 engine wrote, not of the comparison: the raw values
|
||||||
|
* are not in the index to compare. The bounds are normalized into the same
|
||||||
|
* case-folded space so the comparison is at least self-consistent, and the
|
||||||
|
* divergence disappears with the field itself once the column store adopts
|
||||||
|
* it. See the module note on `getIdsFromChunks` for the path's lifetime.
|
||||||
*/
|
*/
|
||||||
private async getIdsFromChunksForRange(
|
private async getIdsFromChunksForRange(
|
||||||
field: string,
|
field: string,
|
||||||
|
|
@ -979,9 +1013,27 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Normalize min/max for consistent comparison with indexed values
|
// Normalize min/max for consistent comparison with indexed values
|
||||||
// (indexed values are bucketed for timestamps, so we must bucket the query bounds too)
|
// (indexed values are bucketed for timestamps, so we must bucket the query
|
||||||
const normalizedMin = min !== undefined ? this.normalizeValue(min, field) : undefined
|
// bounds too) — but NEVER through the hash escape hatch, which would make
|
||||||
const normalizedMax = max !== undefined ? this.normalizeValue(max, field) : undefined
|
// the bound incomparable. See the doc comment above.
|
||||||
|
const normalizedMin = min !== undefined ? this.normalizeValue(min, field, false) : undefined
|
||||||
|
const normalizedMax = max !== undefined ? this.normalizeValue(max, field, false) : undefined
|
||||||
|
|
||||||
|
// REFUSE BEFORE SELECTING. Chunk selection itself orders values: it tests
|
||||||
|
// the bounds against each chunk's zone-map min/max. If those are hashes the
|
||||||
|
// selection is already meaningless — and its failure mode is an EMPTY
|
||||||
|
// answer (no chunk appears to overlap), which is the quietest wrong answer
|
||||||
|
// of all. So the key space is checked here, before a single chunk is
|
||||||
|
// chosen, and again per key below for a chunk whose zone map happens to
|
||||||
|
// read clean.
|
||||||
|
for (const chunkId of sparseIndex.getAllChunkIds()) {
|
||||||
|
const zoneMap = sparseIndex.getChunk(chunkId)?.zoneMap
|
||||||
|
for (const bound of [zoneMap?.min, zoneMap?.max]) {
|
||||||
|
if (typeof bound === 'string' && MetadataIndexManager.isHashedValue(bound)) {
|
||||||
|
throw MetadataIndexManager.rangeOverHashedIndex(field)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Find candidate chunks using zone maps
|
// Find candidate chunks using zone maps
|
||||||
const candidateChunkIds = sparseIndex.findChunksForRange(normalizedMin, normalizedMax)
|
const candidateChunkIds = sparseIndex.findChunksForRange(normalizedMin, normalizedMax)
|
||||||
|
|
@ -996,6 +1048,13 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
||||||
const chunk = await this.chunkManager.loadChunk(field, chunkId)
|
const chunk = await this.chunkManager.loadChunk(field, chunkId)
|
||||||
if (chunk) {
|
if (chunk) {
|
||||||
for (const [value, bitmap] of chunk.entries) {
|
for (const [value, bitmap] of chunk.entries) {
|
||||||
|
// A hashed key carries no order. Refuse the range rather than rank by
|
||||||
|
// digest — the whole answer is unsound, so failing on the first one
|
||||||
|
// is the honest outcome.
|
||||||
|
if (MetadataIndexManager.isHashedValue(value)) {
|
||||||
|
throw MetadataIndexManager.rangeOverHashedIndex(field)
|
||||||
|
}
|
||||||
|
|
||||||
// Check if value is in range using numeric-aware comparison
|
// Check if value is in range using numeric-aware comparison
|
||||||
// (normalizeValue converts numbers to strings, so we must compare numerically)
|
// (normalizeValue converts numbers to strings, so we must compare numerically)
|
||||||
let inRange = true
|
let inRange = true
|
||||||
|
|
@ -1024,6 +1083,25 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
||||||
return this.idMapper.intsIterableToUuids(allIntIds)
|
return this.idMapper.intsIterableToUuids(allIntIds)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The refusal a range query gets when the legacy sparse index holds hashed
|
||||||
|
* keys for the field. Names the field and the cure; never a wrong answer.
|
||||||
|
*/
|
||||||
|
private static rangeOverHashedIndex(field: string): BrainyError {
|
||||||
|
return new BrainyError(
|
||||||
|
`Range query on field "${field}" cannot be served by the legacy sparse index: ` +
|
||||||
|
`its values were persisted as hashes (values over 100 characters are stored ` +
|
||||||
|
`hashed to stay within filesystem name limits), and a hash carries no order — ` +
|
||||||
|
`comparing them would return a confidently ordered wrong answer. ` +
|
||||||
|
`Equality (\`where: { ${field}: value }\`) still works on this index. ` +
|
||||||
|
`To range over this field, let the column store adopt it: run ` +
|
||||||
|
`brain.repairIndex({ rebuild: ['metadata'] }), which rebuilds the field into ` +
|
||||||
|
`the column store, where ranges compare raw values.`,
|
||||||
|
'INVALID_QUERY',
|
||||||
|
false
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get roaring bitmap for a field-value pair without converting to UUIDs
|
* Get roaring bitmap for a field-value pair without converting to UUIDs
|
||||||
* This is used for fast multi-field intersection queries using hardware-accelerated bitmap AND
|
* This is used for fast multi-field intersection queries using hardware-accelerated bitmap AND
|
||||||
|
|
@ -1191,8 +1269,17 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
||||||
* value-based detection (DuckDB-inspired). Analyzes actual data values, not names.
|
* value-based detection (DuckDB-inspired). Analyzes actual data values, not names.
|
||||||
*
|
*
|
||||||
* NO FALLBACKS - Pure value-based detection only.
|
* NO FALLBACKS - Pure value-based detection only.
|
||||||
|
*
|
||||||
|
* @param value - The value to normalize.
|
||||||
|
* @param field - Optional field name (drives the per-field statistics strategy).
|
||||||
|
* @param allowHash - Whether the >100-character escape hatch may fire. TRUE
|
||||||
|
* everywhere a normalized value is used as a KEY (equality postings, chunk
|
||||||
|
* entries, filenames) — that is what the hash exists for. FALSE on the
|
||||||
|
* ORDER-comparing path: a hash is deliberately order-destroying, so a
|
||||||
|
* bound that hashes can only be compared as nonsense. See
|
||||||
|
* {@link isHashedValue} and `getIdsFromChunksForRange`.
|
||||||
*/
|
*/
|
||||||
private normalizeValue(value: any, field?: string): string {
|
private normalizeValue(value: any, field?: string, allowHash: boolean = true): string {
|
||||||
if (value === null || value === undefined) return '__NULL__'
|
if (value === null || value === undefined) return '__NULL__'
|
||||||
if (typeof value === 'boolean') return value ? '__TRUE__' : '__FALSE__'
|
if (typeof value === 'boolean') return value ? '__TRUE__' : '__FALSE__'
|
||||||
|
|
||||||
|
|
@ -1250,21 +1337,34 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
||||||
// Default normalization
|
// Default normalization
|
||||||
if (typeof value === 'number') return value.toString()
|
if (typeof value === 'number') return value.toString()
|
||||||
if (Array.isArray(value)) {
|
if (Array.isArray(value)) {
|
||||||
const joined = value.map(v => this.normalizeValue(v, field)).join(',')
|
const joined = value.map(v => this.normalizeValue(v, field, allowHash)).join(',')
|
||||||
// Hash very long array values to avoid filesystem limits
|
// Hash very long array values to avoid filesystem limits
|
||||||
if (joined.length > 100) {
|
if (allowHash && joined.length > 100) {
|
||||||
return this.hashValue(joined)
|
return this.hashValue(joined)
|
||||||
}
|
}
|
||||||
return joined
|
return joined
|
||||||
}
|
}
|
||||||
const stringValue = String(value).toLowerCase().trim()
|
const stringValue = String(value).toLowerCase().trim()
|
||||||
// Hash very long string values to avoid filesystem limits
|
// Hash very long string values to avoid filesystem limits
|
||||||
if (stringValue.length > 100) {
|
if (allowHash && stringValue.length > 100) {
|
||||||
return this.hashValue(stringValue)
|
return this.hashValue(stringValue)
|
||||||
}
|
}
|
||||||
return stringValue
|
return stringValue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Is this normalized value a HASH rather than the value itself?
|
||||||
|
*
|
||||||
|
* {@link hashValue} is an escape hatch for filesystem name limits, and it is
|
||||||
|
* deliberately order-destroying: two values whose hashes compare one way
|
||||||
|
* routinely compare the other way themselves. Anything that ORDERS normalized
|
||||||
|
* values has to know when it is holding one, because comparing hashes yields
|
||||||
|
* a confident, wrong answer rather than an error.
|
||||||
|
*/
|
||||||
|
private static isHashedValue(normalized: string): boolean {
|
||||||
|
return normalized.startsWith('__HASH_')
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create a short hash for long values to avoid filesystem filename limits
|
* Create a short hash for long values to avoid filesystem filename limits
|
||||||
*/
|
*/
|
||||||
|
|
@ -1289,9 +1389,10 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
||||||
* 'content', 'vector' in a bag are ordinary user fields)
|
* 'content', 'vector' in a bag are ordinary user fields)
|
||||||
* - Record-frame plumbing (vector, connections, level, data, _rev, id)
|
* - Record-frame plumbing (vector, connections, level, data, _rev, id)
|
||||||
* never indexes — that is namespace routing, not a name carve-out
|
* never indexes — that is namespace routing, not a name carve-out
|
||||||
* - Value-SHAPE rules apply uniformly to all names: arrays >10 never
|
* - Value-SHAPE rules apply uniformly to all names: arrays longer than
|
||||||
* become posting scalars; purely numeric key names (array indices)
|
* MAX_INDEXED_ARRAY_LENGTH never become posting scalars (and say so —
|
||||||
* skip; >100-char values index hashed (normalizeValue)
|
* the write door refuses them outright); purely numeric key names
|
||||||
|
* (array indices) skip; >100-char values index hashed (normalizeValue)
|
||||||
*/
|
*/
|
||||||
private extractIndexableFields(data: any): Array<{ field: string, value: any }> {
|
private extractIndexableFields(data: any): Array<{ field: string, value: any }> {
|
||||||
const fields: Array<{ field: string, value: any }> = []
|
const fields: Array<{ field: string, value: any }> = []
|
||||||
|
|
@ -1353,13 +1454,37 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
||||||
// This catches vectors stored as objects: {0: 0.1, 1: 0.2, ...}
|
// This catches vectors stored as objects: {0: 0.1, 1: 0.2, ...}
|
||||||
if (/^\d+$/.test(key)) continue
|
if (/^\d+$/.test(key)) continue
|
||||||
|
|
||||||
// Skip large arrays (> 10 elements) - likely vectors or bulk data
|
// THE INDEXABLE-ARRAY BOUND ({@link MAX_INDEXED_ARRAY_LENGTH}). An
|
||||||
if (Array.isArray(value) && value.length > 10) continue
|
// array field mints one posting per element, so the index has always
|
||||||
|
// carried a ceiling — it was 10, and it was applied by this bare
|
||||||
|
// `continue`: an eleven-element `tags` array had its whole field
|
||||||
|
// skipped and the row dropped out of every filtered search on it, with
|
||||||
|
// no error, no warning, and nothing to distinguish that from "no row
|
||||||
|
// matches". The ceiling is not the defect; the silence was.
|
||||||
|
//
|
||||||
|
// The write door refuses this shape by name now
|
||||||
|
// (`MetadataArrayTooLargeError`, thrown from paramValidation's
|
||||||
|
// `rejectOversizeIndexArrays`), so a live add/update never reaches
|
||||||
|
// here over the bound. Reaching it means the row is ALREADY on disk —
|
||||||
|
// written by an older engine under the old rule — and this is a
|
||||||
|
// rebuild, a catch-up fold or a remove reading it back. Refusing there
|
||||||
|
// would make an existing store un-rebuildable, so the row is admitted
|
||||||
|
// and the skipped field is NARRATED instead. Never silent, either way.
|
||||||
|
if (Array.isArray(value) && value.length > MAX_INDEXED_ARRAY_LENGTH) {
|
||||||
|
prodLog.warn(
|
||||||
|
`[brainy] metadata field '${fullKey}' holds ${value.length} array elements, ` +
|
||||||
|
`over the ${MAX_INDEXED_ARRAY_LENGTH}-element indexing bound — the field is ` +
|
||||||
|
`NOT indexed for this row, so it will not match a where-clause on '${fullKey}'. ` +
|
||||||
|
`This row predates the bound (the write door refuses this shape now). ` +
|
||||||
|
`Move the long array into 'data', or pass an embedding as the 'vector' parameter.`
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
if (value && typeof value === 'object' && !Array.isArray(value)) {
|
if (value && typeof value === 'object' && !Array.isArray(value)) {
|
||||||
// Recurse into nested objects (but not arrays), keeping the frame
|
// Recurse into nested objects (but not arrays), keeping the frame
|
||||||
extract(value, fullKey, frame)
|
extract(value, fullKey, frame)
|
||||||
} else if (Array.isArray(value) && value.length <= 10) {
|
} else if (Array.isArray(value)) {
|
||||||
// Small arrays: index as multi-value field (all with same field name)
|
// Small arrays: index as multi-value field (all with same field name)
|
||||||
// Example: tags: ["javascript", "node"] → field="tags", value="javascript" + field="tags", value="node"
|
// Example: tags: ["javascript", "node"] → field="tags", value="javascript" + field="tags", value="node"
|
||||||
for (const item of value) {
|
for (const item of value) {
|
||||||
|
|
@ -1509,11 +1634,56 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
||||||
* @returns Array of { id, matchCount } sorted by matchCount descending
|
* @returns Array of { id, matchCount } sorted by matchCount descending
|
||||||
*/
|
*/
|
||||||
async getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>> {
|
async getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>> {
|
||||||
|
return this.scoreTextQuery(query)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Score a text query over `ids` ONLY — the reference implementation of the
|
||||||
|
* optional `getIdsForTextQueryWithin` door (see
|
||||||
|
* {@link import('../plugin.js').MetadataIndexProvider}). The hybrid
|
||||||
|
* `find({ query, where })` path passes the metadata filter's universe here so
|
||||||
|
* the text leg ranks INSIDE that universe instead of ranking the whole store
|
||||||
|
* and discarding the rows the filter would have dropped.
|
||||||
|
*
|
||||||
|
* It answers from the same posting-list merge as {@link getIdsForTextQuery},
|
||||||
|
* with the candidate membership applied as each word's postings are counted,
|
||||||
|
* so the two doors can never disagree: the answer is exactly the whole-store
|
||||||
|
* answer restricted to `ids`, in the same order.
|
||||||
|
*
|
||||||
|
* @param query - Text query to search for.
|
||||||
|
* @param ids - Candidate entity ids; only these may appear in the answer.
|
||||||
|
* @returns Array of { id, matchCount } sorted by matchCount descending.
|
||||||
|
*/
|
||||||
|
async getIdsForTextQueryWithin(
|
||||||
|
query: string,
|
||||||
|
ids: readonly string[]
|
||||||
|
): Promise<Array<{ id: string; matchCount: number }>> {
|
||||||
|
if (ids.length === 0) return []
|
||||||
|
return this.scoreTextQuery(query, new Set(ids))
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The one posting-list merge behind both text doors.
|
||||||
|
*
|
||||||
|
* Each query word contributes AT MOST one match per entity (a posting list
|
||||||
|
* can name an id more than once), and entities are ranked by how many of the
|
||||||
|
* query's words they matched. `within`, when given, restricts the count to
|
||||||
|
* those candidates — applied during the merge, so a restricted call never
|
||||||
|
* materializes a whole-store match map.
|
||||||
|
*
|
||||||
|
* @param query - Text query to search for.
|
||||||
|
* @param within - Optional candidate universe; absent = the whole store.
|
||||||
|
* @returns Array of { id, matchCount } sorted by matchCount descending.
|
||||||
|
*/
|
||||||
|
private async scoreTextQuery(
|
||||||
|
query: string,
|
||||||
|
within?: ReadonlySet<string>
|
||||||
|
): Promise<Array<{ id: string; matchCount: number }>> {
|
||||||
const queryWords = this.tokenize(query)
|
const queryWords = this.tokenize(query)
|
||||||
if (queryWords.length === 0) return []
|
if (queryWords.length === 0) return []
|
||||||
|
|
||||||
// Get IDs for each word hash
|
// Count matches per entity, one word's postings at a time.
|
||||||
const wordIdSets: Map<string, number>[] = []
|
const matchCounts = new Map<string, number>()
|
||||||
for (const word of queryWords) {
|
for (const word of queryWords) {
|
||||||
const wordHash = this.hashWord(word)
|
const wordHash = this.hashWord(word)
|
||||||
let ids: string[]
|
let ids: string[]
|
||||||
|
|
@ -1529,19 +1699,12 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
||||||
throw err
|
throw err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
const idSet = new Map<string, number>()
|
// One count per (word, entity) — dedupe this word's postings first.
|
||||||
|
const counted = new Set<string>()
|
||||||
for (const id of ids) {
|
for (const id of ids) {
|
||||||
idSet.set(id, 1)
|
if (counted.has(id)) continue
|
||||||
}
|
counted.add(id)
|
||||||
wordIdSets.push(idSet)
|
if (within && !within.has(id)) continue
|
||||||
}
|
|
||||||
|
|
||||||
if (wordIdSets.length === 0) return []
|
|
||||||
|
|
||||||
// Count matches per entity
|
|
||||||
const matchCounts = new Map<string, number>()
|
|
||||||
for (const idSet of wordIdSets) {
|
|
||||||
for (const [id] of idSet) {
|
|
||||||
matchCounts.set(id, (matchCounts.get(id) || 0) + 1)
|
matchCounts.set(id, (matchCounts.get(id) || 0) + 1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -2575,6 +2738,19 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
||||||
/** Once-per-field flag for the fallback-degradation announcement. */
|
/** Once-per-field flag for the fallback-degradation announcement. */
|
||||||
private static announcedFallbackSorts = new Set<string>()
|
private static announcedFallbackSorts = new Set<string>()
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Evaluate `filter` over `ids` only — the graph-first find's door (the
|
||||||
|
* neighbour set filtered by id, never the store filtered and then
|
||||||
|
* intersected). This index answers from its own `getIdsForFilter`, so the
|
||||||
|
* two doors cannot disagree; the cost is that of the filter over this
|
||||||
|
* in-memory index, and the answer keeps the caller's order.
|
||||||
|
*/
|
||||||
|
async filterIdsWithin(filter: any, ids: readonly string[]): Promise<string[]> {
|
||||||
|
if (ids.length === 0) return []
|
||||||
|
const matched = new Set(await this.getIdsForFilter(filter))
|
||||||
|
return ids.filter((id) => matched.has(id))
|
||||||
|
}
|
||||||
|
|
||||||
async getSortedIdsForFilter(
|
async getSortedIdsForFilter(
|
||||||
filter: any,
|
filter: any,
|
||||||
orderBy: string,
|
orderBy: string,
|
||||||
|
|
@ -2754,6 +2930,67 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
||||||
return order === 'asc' ? comparison : -comparison
|
return order === 'asc' ? comparison : -comparison
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read named scalar fields for many ids from the COLUMN STORE, without
|
||||||
|
* touching the canonical record — the `find({ fields })` door.
|
||||||
|
*
|
||||||
|
* ## Why the column store and not the sparse index
|
||||||
|
*
|
||||||
|
* The column store keeps RAW values; the sparse index keeps a normalized,
|
||||||
|
* bucketed form built for range queries — `system.createdAt` is indexed at
|
||||||
|
* minute precision there. A projection served from the sparse index would
|
||||||
|
* hand back a value that differs from the record's, which is a wrong answer
|
||||||
|
* nobody can see. So this door reads the column store, and a field the
|
||||||
|
* column store does not hold is OMITTED rather than approximated.
|
||||||
|
*
|
||||||
|
* ## Why batched
|
||||||
|
*
|
||||||
|
* `getFieldValueForEntity` answers one (id, field) pair by walking the
|
||||||
|
* field's storage; called per row it re-walks the same column for every id.
|
||||||
|
* This walks each column ONCE and picks out every requested id as it passes:
|
||||||
|
* O(fields x column) instead of O(ids x fields x column).
|
||||||
|
*
|
||||||
|
* Omission is always safe — it costs the caller a record read. The caller
|
||||||
|
* diffs what it asked for against what came back and reads records for the
|
||||||
|
* remainder, so an index that can serve nothing is slow, never wrong.
|
||||||
|
*
|
||||||
|
* @param ids - Canonical entity ids.
|
||||||
|
* @param fields - Index keys (bare = user metadata, `system.*` = engine scalar).
|
||||||
|
* @returns `id -> { field: value }` for exactly the pairs this index served.
|
||||||
|
*/
|
||||||
|
async getScalarsForIds(
|
||||||
|
ids: readonly string[],
|
||||||
|
fields: readonly string[]
|
||||||
|
): Promise<Map<string, Record<string, unknown>>> {
|
||||||
|
const out = new Map<string, Record<string, unknown>>()
|
||||||
|
if (ids.length === 0 || fields.length === 0) return out
|
||||||
|
|
||||||
|
// int -> id, so a column hit resolves back to the caller's id. An id the
|
||||||
|
// mapper does not know cannot be in any column, so it is simply absent.
|
||||||
|
const idByInt = new Map<number, string>()
|
||||||
|
for (const id of ids) {
|
||||||
|
const intId = this.idMapper.getInt(id)
|
||||||
|
if (intId !== undefined) idByInt.set(intId, id)
|
||||||
|
}
|
||||||
|
if (idByInt.size === 0) return out
|
||||||
|
|
||||||
|
for (const field of fields) {
|
||||||
|
if (!this.columnStore.hasField(field)) continue
|
||||||
|
const values = await this.columnStore.valuesForIds(field, idByInt.keys())
|
||||||
|
for (const [intId, value] of values) {
|
||||||
|
const id = idByInt.get(intId)
|
||||||
|
if (id === undefined) continue
|
||||||
|
let row = out.get(id)
|
||||||
|
if (row === undefined) {
|
||||||
|
row = {}
|
||||||
|
out.set(id, row)
|
||||||
|
}
|
||||||
|
row[field] = value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
async getFieldValueForEntity(entityId: string, field: string): Promise<any> {
|
async getFieldValueForEntity(entityId: string, field: string): Promise<any> {
|
||||||
// `field` arrives as a FROZEN INDEX KEY (bare = user metadata;
|
// `field` arrives as a FROZEN INDEX KEY (bare = user metadata;
|
||||||
// 'system.<field>' = engine scalar). Storage fallbacks read the matching
|
// 'system.<field>' = engine scalar). Storage fallbacks read the matching
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,7 @@ import { findCallerLocation } from './callerLocation.js'
|
||||||
import * as os from 'node:os'
|
import * as os from 'node:os'
|
||||||
import * as fs from 'node:fs'
|
import * as fs from 'node:fs'
|
||||||
import { parseFieldAddress, UnsupportedFindOptionError } from '../db/fieldAddressing.js'
|
import { parseFieldAddress, UnsupportedFindOptionError } from '../db/fieldAddressing.js'
|
||||||
|
import { MAX_INDEXED_ARRAY_LENGTH, MetadataArrayTooLargeError } from '../errors/brainyError.js'
|
||||||
|
|
||||||
const getSystemMemory = (): number => {
|
const getSystemMemory = (): number => {
|
||||||
if (os) {
|
if (os) {
|
||||||
|
|
@ -538,8 +539,53 @@ function rejectForgedSystemKeys(metadata: Record<string, unknown> | undefined, s
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* THE INDEXABLE-ARRAY BOUND, enforced at the write door.
|
||||||
|
*
|
||||||
|
* An array-valued metadata field indexes one posting per element, so the index
|
||||||
|
* has always carried a ceiling. It used to be 10, and it was applied by a bare
|
||||||
|
* `continue` deep inside field extraction: a row whose `tags` array held eleven
|
||||||
|
* entries had that field skipped entirely and dropped out of every filtered
|
||||||
|
* search on it — no error, no warning, and no way for the caller to tell the
|
||||||
|
* difference from "no row matches". Silence is the defect; the ceiling is not.
|
||||||
|
*
|
||||||
|
* The bound is now {@link MAX_INDEXED_ARRAY_LENGTH}, high enough that every
|
||||||
|
* legitimate multi-value field clears it, and it REFUSES here instead of
|
||||||
|
* dropping data downstream. Refusing at the write door is what makes it
|
||||||
|
* actionable: the caller learns at the moment of writing, with the field, the
|
||||||
|
* length and the bound in hand.
|
||||||
|
*
|
||||||
|
* Scope is the caller's own metadata bag — the values that become postings.
|
||||||
|
* Nested bags are walked, because a nested field indexes under its dotted
|
||||||
|
* address exactly like a top-level one. Arrays of OBJECTS are not walked: the
|
||||||
|
* index only ever makes postings from an array's scalar elements.
|
||||||
|
*
|
||||||
|
* @param metadata - The caller's metadata bag (undefined is fine).
|
||||||
|
* @param site - The write door's name, for the message ('add()', 'update()', …).
|
||||||
|
* @throws {MetadataArrayTooLargeError} Naming the field, its length and the bound.
|
||||||
|
*/
|
||||||
|
function rejectOversizeIndexArrays(metadata: Record<string, unknown> | undefined, site: string): void {
|
||||||
|
if (!metadata) return
|
||||||
|
|
||||||
|
const walk = (bag: Record<string, unknown>, prefix: string): void => {
|
||||||
|
for (const [key, value] of Object.entries(bag)) {
|
||||||
|
const address = prefix ? `${prefix}.${key}` : key
|
||||||
|
if (Array.isArray(value)) {
|
||||||
|
if (value.length > MAX_INDEXED_ARRAY_LENGTH) {
|
||||||
|
throw new MetadataArrayTooLargeError(site, address, value.length, MAX_INDEXED_ARRAY_LENGTH)
|
||||||
|
}
|
||||||
|
} else if (value && typeof value === 'object') {
|
||||||
|
walk(value as Record<string, unknown>, address)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
walk(metadata, '')
|
||||||
|
}
|
||||||
|
|
||||||
export function validateAddParams(params: AddParams): void {
|
export function validateAddParams(params: AddParams): void {
|
||||||
rejectForgedSystemKeys(params.metadata as Record<string, unknown> | undefined, 'add()')
|
rejectForgedSystemKeys(params.metadata as Record<string, unknown> | undefined, 'add()')
|
||||||
|
rejectOversizeIndexArrays(params.metadata as Record<string, unknown> | undefined, 'add()')
|
||||||
// 'data' is ABSENT only when null/undefined — an empty string ('') is real
|
// 'data' is ABSENT only when null/undefined — an empty string ('') is real
|
||||||
// content (a legitimate empty file's first write) and must not be treated
|
// content (a legitimate empty file's first write) and must not be treated
|
||||||
// as missing. Falsy-but-present values (0, false, '') all count as present;
|
// as missing. Falsy-but-present values (0, false, '') all count as present;
|
||||||
|
|
@ -608,6 +654,7 @@ export function validateAddParams(params: AddParams): void {
|
||||||
*/
|
*/
|
||||||
export function validateUpdateParams(params: UpdateParams): void {
|
export function validateUpdateParams(params: UpdateParams): void {
|
||||||
rejectForgedSystemKeys(params.metadata as Record<string, unknown> | undefined, 'update()')
|
rejectForgedSystemKeys(params.metadata as Record<string, unknown> | undefined, 'update()')
|
||||||
|
rejectOversizeIndexArrays(params.metadata as Record<string, unknown> | undefined, 'update()')
|
||||||
// Same absent-vs-empty distinction as validateAddParams: '' is a real new
|
// Same absent-vs-empty distinction as validateAddParams: '' is a real new
|
||||||
// value (e.g. truncating a file to empty content via overwrite), only
|
// value (e.g. truncating a file to empty content via overwrite), only
|
||||||
// null/undefined means "no new data was given".
|
// null/undefined means "no new data was given".
|
||||||
|
|
@ -682,6 +729,7 @@ export function validateUpdateParams(params: UpdateParams): void {
|
||||||
*/
|
*/
|
||||||
export function validateRelateParams(params: RelateParams): void {
|
export function validateRelateParams(params: RelateParams): void {
|
||||||
rejectForgedSystemKeys(params.metadata as Record<string, unknown> | undefined, 'relate()')
|
rejectForgedSystemKeys(params.metadata as Record<string, unknown> | undefined, 'relate()')
|
||||||
|
rejectOversizeIndexArrays(params.metadata as Record<string, unknown> | undefined, 'relate()')
|
||||||
// 8.0 verb-id contract (L.7): verb ids are UUIDs, generated by brainy.
|
// 8.0 verb-id contract (L.7): verb ids are UUIDs, generated by brainy.
|
||||||
// RelateParams has no `id` field — an untyped caller passing one would
|
// RelateParams has no `id` field — an untyped caller passing one would
|
||||||
// previously have it silently ignored (a generated UUID was used instead).
|
// previously have it silently ignored (a generated UUID was used instead).
|
||||||
|
|
@ -731,6 +779,7 @@ export function validateRelateParams(params: RelateParams): void {
|
||||||
*/
|
*/
|
||||||
export function validateUpdateRelationParams(params: UpdateRelationParams): void {
|
export function validateUpdateRelationParams(params: UpdateRelationParams): void {
|
||||||
rejectForgedSystemKeys(params.metadata as Record<string, unknown> | undefined, 'updateRelation()')
|
rejectForgedSystemKeys(params.metadata as Record<string, unknown> | undefined, 'updateRelation()')
|
||||||
|
rejectOversizeIndexArrays(params.metadata as Record<string, unknown> | undefined, 'updateRelation()')
|
||||||
if (!params.id) {
|
if (!params.id) {
|
||||||
throw new Error('id is required for updateRelation')
|
throw new Error('id is required for updateRelation')
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1572,7 +1572,19 @@ export class VirtualFileSystem implements IVirtualFileSystem {
|
||||||
// ============= Semantic Operations =============
|
// ============= Semantic Operations =============
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Search files with natural language
|
* Search files with natural language.
|
||||||
|
*
|
||||||
|
* `options.path` scopes the search to a directory: its whole subtree by
|
||||||
|
* default, its immediate children when `recursive` is `false`. Both scopes
|
||||||
|
* are metadata filters the index SERVES, so the scope narrows the search
|
||||||
|
* before it runs — no tree walk, and never an over-fetch filtered afterwards.
|
||||||
|
*
|
||||||
|
* @param query - The natural-language query.
|
||||||
|
* @param options - Scope, metadata filters and paging (see {@link SearchOptions}).
|
||||||
|
* @returns The matching files, best first.
|
||||||
|
* @throws {VFSError} ENOENT when `recursive: false` names a path that does
|
||||||
|
* not exist (the non-recursive scope is the directory's own identity, so
|
||||||
|
* the directory has to be there).
|
||||||
*/
|
*/
|
||||||
async search(query: string, options?: SearchOptions): Promise<SearchResult[]> {
|
async search(query: string, options?: SearchOptions): Promise<SearchResult[]> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
@ -1588,11 +1600,26 @@ export class VirtualFileSystem implements IVirtualFileSystem {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add path filter if specified
|
// Scope to a directory, if asked. This used to emit
|
||||||
|
// `path: { $startsWith }` — an operator that is not in the filter
|
||||||
|
// vocabulary at all, and whose `$`-less spelling the metadata index
|
||||||
|
// REFUSES by the served-operator law (an equality/range posting index
|
||||||
|
// cannot evaluate a substring without reading every row). Every
|
||||||
|
// path-scoped VFS search therefore threw, and none has ever worked on
|
||||||
|
// this engine line. Both scopes below are served shapes.
|
||||||
if (options?.path) {
|
if (options?.path) {
|
||||||
params.where = {
|
if (options.recursive === false) {
|
||||||
...params.where,
|
// Immediate children only: the directory's identity IS the scope, and
|
||||||
path: { $startsWith: options.path }
|
// `parent` is an indexed equality on every VFS entity.
|
||||||
|
params.where = {
|
||||||
|
...params.where,
|
||||||
|
parent: await this.pathResolver.resolve(options.path)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
const scope = this.descendantPathScope(options.path)
|
||||||
|
if (scope) {
|
||||||
|
params.where = { ...params.where, path: scope }
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1754,6 +1781,42 @@ export class VirtualFileSystem implements IVirtualFileSystem {
|
||||||
return entity as VFSEntity
|
return entity as VFSEntity
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The SERVED metadata shape for "everything under this directory".
|
||||||
|
*
|
||||||
|
* `metadata.path` is the VFS's truth — write and rename maintain it, and the
|
||||||
|
* `Contains` edges are a projection of it (see {@link repairContainment}) —
|
||||||
|
* it is indexed on every VFS entity, and the metadata index serves ordered
|
||||||
|
* range operators. So a subtree scope is a half-open range over the path
|
||||||
|
* column: O(log n + matches), no tree walk, and nothing fetched that the
|
||||||
|
* scope then discards.
|
||||||
|
*
|
||||||
|
* The range is `[dir + '/', dir + <successor of '/'>)`. Every descendant path
|
||||||
|
* begins with `dir + '/'`, and '0' is the code point directly after '/', so a
|
||||||
|
* string lies in the range EXACTLY when it carries that prefix. The two
|
||||||
|
* bounds differ at a single ASCII position, so the answer is the same under
|
||||||
|
* code-unit and code-point collation alike — no dependence on how the store
|
||||||
|
* orders the rest of the string.
|
||||||
|
*
|
||||||
|
* Sibling exclusion falls out of the same fact and is worth stating, because
|
||||||
|
* it is where a naive prefix test goes wrong: for `dir = '/scope'`,
|
||||||
|
* `/scope-sibling/x` sorts BELOW the lower bound ('-' precedes '/') and
|
||||||
|
* `/scope0` sits at the open upper bound — both outside, while
|
||||||
|
* `/scope/sub/deep/c.txt` is inside at any depth.
|
||||||
|
*
|
||||||
|
* @param path - The directory to scope to.
|
||||||
|
* @returns The `where` fragment for the `path` field, or `null` for the root
|
||||||
|
* — every VFS entity is under it, so no clause narrows the search.
|
||||||
|
*/
|
||||||
|
private descendantPathScope(path: string): { gte: string; lt: string } | null {
|
||||||
|
const dir = path.replace(/\/+/g, '/').replace(/\/$/, '') || '/'
|
||||||
|
if (dir === '/') return null
|
||||||
|
// Computed, so the bound carries its own reason: the first string that can
|
||||||
|
// no longer share the `dir + '/'` prefix.
|
||||||
|
const separatorSuccessor = String.fromCharCode('/'.charCodeAt(0) + 1)
|
||||||
|
return { gte: `${dir}/`, lt: `${dir}${separatorSuccessor}` }
|
||||||
|
}
|
||||||
|
|
||||||
private getParentPath(path: string): string {
|
private getParentPath(path: string): string {
|
||||||
const normalized = path.replace(/\/+/g, '/').replace(/\/$/, '')
|
const normalized = path.replace(/\/+/g, '/').replace(/\/$/, '')
|
||||||
const lastSlash = normalized.lastIndexOf('/')
|
const lastSlash = normalized.lastIndexOf('/')
|
||||||
|
|
@ -2295,6 +2358,31 @@ export class VirtualFileSystem implements IVirtualFileSystem {
|
||||||
cursor = page.nextCursor
|
cursor = page.nextCursor
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Pass 2: ONE paged walk over every Contains edge, grouped by target in
|
||||||
|
// memory. The earlier shape issued one awaited related({ to }) per VFS
|
||||||
|
// entity — O(entities) serialized graph calls, measured in whole minutes
|
||||||
|
// on large brains. This shape is O(edges / page) calls regardless of how
|
||||||
|
// many entities exist; mutations alone stay per-defect.
|
||||||
|
const incomingByTarget = new Map<string, Relation<any>[]>()
|
||||||
|
{
|
||||||
|
const pageSize = 1000
|
||||||
|
let pageOffset = 0
|
||||||
|
for (;;) {
|
||||||
|
const page = await this.brain.related({
|
||||||
|
type: VerbType.Contains,
|
||||||
|
limit: pageSize,
|
||||||
|
offset: pageOffset
|
||||||
|
})
|
||||||
|
for (const edge of page) {
|
||||||
|
const bucket = incomingByTarget.get(edge.to)
|
||||||
|
if (bucket) bucket.push(edge)
|
||||||
|
else incomingByTarget.set(edge.to, [edge])
|
||||||
|
}
|
||||||
|
if (page.length < pageSize) break
|
||||||
|
pageOffset += pageSize
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let removed = 0
|
let removed = 0
|
||||||
let restored = 0
|
let restored = 0
|
||||||
for (const { id, path } of vfsEntities) {
|
for (const { id, path } of vfsEntities) {
|
||||||
|
|
@ -2307,7 +2395,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
const incoming = await this.brain.related({ to: id, type: VerbType.Contains })
|
const incoming = incomingByTarget.get(id) ?? []
|
||||||
let expectedSeen = false
|
let expectedSeen = false
|
||||||
for (const edge of incoming) {
|
for (const edge of incoming) {
|
||||||
const isVfsEdge = edge.subtype === 'vfs-contains' || (edge.metadata as any)?.isVFS === true
|
const isVfsEdge = edge.subtype === 'vfs-contains' || (edge.metadata as any)?.isVFS === true
|
||||||
|
|
|
||||||
75
tests/configs/vitest.perf.config.ts
Normal file
75
tests/configs/vitest.perf.config.ts
Normal file
|
|
@ -0,0 +1,75 @@
|
||||||
|
import { defineConfig } from 'vitest/config'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Perf/scale + environment-dependent test configuration.
|
||||||
|
*
|
||||||
|
* The exclusive on-demand slot for everything the correctness gate
|
||||||
|
* (`vitest.config.ts`, the config a bare `vitest run` picks up) excludes:
|
||||||
|
* wall-clock/scale benchmarks and the two tests whose outcome depends on
|
||||||
|
* the host machine or network rather than the code. See CONTRIBUTING.md's
|
||||||
|
* "Test gate" section and the exclude list in `vitest.config.ts` (root) for
|
||||||
|
* why each file lives here instead of the gate.
|
||||||
|
*
|
||||||
|
* `include` names this set explicitly — it is the mirror image of the
|
||||||
|
* root config's exclude list, not an independent glob, so the two stay in
|
||||||
|
* sync by inspection. Longer timeouts than the gate's 120s/60s: one case in
|
||||||
|
* tests/critical-performance-benchmark.test.ts measures ~128s of real work.
|
||||||
|
*/
|
||||||
|
export default defineConfig({
|
||||||
|
test: {
|
||||||
|
globals: true,
|
||||||
|
setupFiles: ['./tests/setup.ts'],
|
||||||
|
environment: 'node',
|
||||||
|
|
||||||
|
// The marker a test uses to tell it is running under this lane (see
|
||||||
|
// tests/integration/storage-batch-operations.test.ts's batch-vs-
|
||||||
|
// individual timing case) — a wall-clock RATIO assertion self-skips
|
||||||
|
// with a reason when this is absent, rather than flaking the
|
||||||
|
// correctness gate on whichever path happens to be faster this build.
|
||||||
|
env: { BRAINY_PERF_LANE: '1' },
|
||||||
|
|
||||||
|
// Sequential, single fork — same isolation the gate uses, so a perf
|
||||||
|
// measurement isn't skewed by sibling test contention.
|
||||||
|
pool: 'forks',
|
||||||
|
poolOptions: {
|
||||||
|
forks: {
|
||||||
|
maxForks: 1,
|
||||||
|
minForks: 1,
|
||||||
|
singleFork: true,
|
||||||
|
isolate: true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
testTimeout: 300000, // 5 minutes per test (the 128s case plus headroom)
|
||||||
|
hookTimeout: 120000,
|
||||||
|
teardownTimeout: 10000,
|
||||||
|
|
||||||
|
maxConcurrency: 1,
|
||||||
|
fileParallelism: false,
|
||||||
|
|
||||||
|
include: [
|
||||||
|
'tests/performance/**/*.{test,spec}.{js,ts}',
|
||||||
|
'tests/critical-performance-benchmark.test.ts',
|
||||||
|
'tests/api/performance-benchmarks.test.ts',
|
||||||
|
'tests/package-size-limit.test.ts',
|
||||||
|
'tests/model-loading.test.ts',
|
||||||
|
// Not a whole perf file — one wall-clock-ratio case inside an
|
||||||
|
// otherwise-correctness integration suite (self-skipped everywhere
|
||||||
|
// else via BRAINY_PERF_LANE). Stays in the integration gate's
|
||||||
|
// include too, so every OTHER test in the file keeps running there.
|
||||||
|
'tests/integration/storage-batch-operations.test.ts',
|
||||||
|
// Same pattern: one wall-clock budget case (100-file write + readdir,
|
||||||
|
// 5.5s budget) inside an otherwise-correctness VFS unit suite
|
||||||
|
// (self-skipped everywhere else via BRAINY_PERF_LANE — see
|
||||||
|
// tests/vfs/vfs.unit.test.ts's 'Performance > should handle many
|
||||||
|
// files efficiently'). Stays in the unit gate's *.unit.test.ts match
|
||||||
|
// too, so every OTHER test in the file keeps running there.
|
||||||
|
'tests/vfs/vfs.unit.test.ts'
|
||||||
|
],
|
||||||
|
|
||||||
|
reporters: process.env.CI ? ['dot'] : ['basic'],
|
||||||
|
|
||||||
|
retry: process.env.CI ? 1 : 0,
|
||||||
|
shard: process.env.VITEST_SHARD
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
@ -34,6 +34,10 @@ describe('API Parameter Validation', () => {
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
|
||||||
it('should use "where" parameter for metadata filtering', async () => {
|
it('should use "where" parameter for metadata filtering', async () => {
|
||||||
const results = await brain.find({
|
const results = await brain.find({
|
||||||
where: { category: 'test-category' },
|
where: { category: 'test-category' },
|
||||||
|
|
|
||||||
309
tests/integration/beforeexit-never-closes.test.ts
Normal file
309
tests/integration/beforeexit-never-closes.test.ts
Normal file
|
|
@ -0,0 +1,309 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/beforeexit-never-closes
|
||||||
|
* @description A DRAINED EVENT LOOP IS NOT A SHUTDOWN.
|
||||||
|
*
|
||||||
|
* MEASURED on the 11.1 rehearsal lane, against a copy of a real store. The
|
||||||
|
* `beforeExit` listener had been wired to the SIGNAL path — the path whose job
|
||||||
|
* is to `close()` every live brain — so after the heal phase the log printed
|
||||||
|
*
|
||||||
|
* "Shutdown signal received - flushing pending data..."
|
||||||
|
* "Flushed successfully (1 instance)"
|
||||||
|
*
|
||||||
|
* with no signal ever sent, and the script's very next `add()` threw
|
||||||
|
*
|
||||||
|
* "Brainy instance is not initialized: it was closed via close().
|
||||||
|
* Create a new instance."
|
||||||
|
*
|
||||||
|
* Node emits `'beforeExit'` whenever the event loop has no REF'd work left.
|
||||||
|
* That is not "the process is ending" — it is a state a perfectly healthy
|
||||||
|
* script reaches, because this engine unref's its idle and cadence timers
|
||||||
|
* ("an idle brain costs nothing"), so a script awaiting anything those timers
|
||||||
|
* drive is, for that instant, a process with no ref'd work and an open brain.
|
||||||
|
* The engine closed a live brain out from under a running script.
|
||||||
|
*
|
||||||
|
* The contract pinned here:
|
||||||
|
* (1) `'beforeExit'` firing while a brain is open closes NOTHING: the brain
|
||||||
|
* is still open, `add()` and `find()` still work, the writer lock is
|
||||||
|
* still held, and the process still exits 0 on its own afterwards.
|
||||||
|
* (2) The pass DOES persist derived state — a non-closing `flush()` ran —
|
||||||
|
* and it wrote no clean-shutdown marker and no clean-close record: those
|
||||||
|
* are `close()`'s word about itself, and no close happened.
|
||||||
|
* (3) The signal path is untouched: SIGTERM still closes through `close()`
|
||||||
|
* (pinned by tests/integration/shutdown-single-owner.test.ts, re-run
|
||||||
|
* with this change).
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
|
import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync } from 'node:fs'
|
||||||
|
import { spawn } from 'node:child_process'
|
||||||
|
import { tmpdir } from 'node:os'
|
||||||
|
import { join } from 'node:path'
|
||||||
|
import { Brainy } from '../../src/brainy.js'
|
||||||
|
import { NounType } from '../../src/types/graphTypes.js'
|
||||||
|
|
||||||
|
const REPO_ROOT = process.cwd()
|
||||||
|
const TSX = join(REPO_ROOT, 'node_modules', '.bin', 'tsx')
|
||||||
|
const BRAINY_SRC = join(REPO_ROOT, 'src', 'brainy.ts')
|
||||||
|
|
||||||
|
function makeTempDir(prefix: string): string {
|
||||||
|
return mkdtempSync(join(tmpdir(), prefix))
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The writer lock itself — present for as long as this process owns the store. */
|
||||||
|
const writerLockPath = (dir: string) => join(dir, 'locks', '_writer.lock')
|
||||||
|
/** The clean-close record — written by `releaseWriterLock()`, i.e. by close(). */
|
||||||
|
const closeRecordPath = (dir: string) => join(dir, 'locks', '_writer.close')
|
||||||
|
/**
|
||||||
|
* The generation store's clean-shutdown marker — written by
|
||||||
|
* `generationStore.close()` alone, reached only from `close()`. (Raw objects
|
||||||
|
* are gzipped on disk, so both spellings are accepted.)
|
||||||
|
*/
|
||||||
|
const cleanShutdownWritten = (dir: string) =>
|
||||||
|
existsSync(join(dir, '_system', 'clean-shutdown.json.gz')) ||
|
||||||
|
existsSync(join(dir, '_system', 'clean-shutdown.json'))
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Write a child script and run it under tsx to completion, collecting stdout
|
||||||
|
* and stderr and the exit code. (A file, not `tsx -e`: the eval form compiles
|
||||||
|
* to CommonJS, which has no top-level await.)
|
||||||
|
*/
|
||||||
|
function runChild(
|
||||||
|
scriptDir: string,
|
||||||
|
body: string
|
||||||
|
): Promise<{ code: number | null; out: string }> {
|
||||||
|
const scriptPath = join(scriptDir, 'child-process.mts')
|
||||||
|
writeFileSync(scriptPath, body)
|
||||||
|
// The child is an ORDINARY consumer process, so it runs the real embedding
|
||||||
|
// pipeline: this suite's deterministic-embedder switch is inherited through
|
||||||
|
// the environment, and under it `find()` self-retrieval returns nothing —
|
||||||
|
// which would make the read half of this pin vacuous. (That property is the
|
||||||
|
// deterministic embedder's, not this change's: it reproduces in a plain
|
||||||
|
// script with no 'beforeExit' involved.)
|
||||||
|
const env = { ...process.env }
|
||||||
|
delete env.BRAINY_DETERMINISTIC_EMBEDDINGS
|
||||||
|
const child = spawn(TSX, [scriptPath], {
|
||||||
|
cwd: REPO_ROOT,
|
||||||
|
stdio: ['ignore', 'pipe', 'pipe'],
|
||||||
|
env
|
||||||
|
})
|
||||||
|
let out = ''
|
||||||
|
child.stdout?.on('data', (d) => { out += String(d) })
|
||||||
|
child.stderr?.on('data', (d) => { out += String(d) })
|
||||||
|
return new Promise((resolvePromise) => {
|
||||||
|
child.on('exit', (code) => resolvePromise({ code, out }))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('beforeExit never closes a live brain', () => {
|
||||||
|
let dir: string
|
||||||
|
let scriptDir: string
|
||||||
|
let resultPath: string
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
dir = makeTempDir('brainy-beforeexit-')
|
||||||
|
scriptDir = makeTempDir('brainy-beforeexit-script-')
|
||||||
|
resultPath = join(scriptDir, 'result.json')
|
||||||
|
})
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
for (const d of [dir, scriptDir]) {
|
||||||
|
try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ }
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('(1)+(2) a drained event loop flushes, closes nothing, and the script keeps working', async () => {
|
||||||
|
/**
|
||||||
|
* THE DRAIN, and why the script survives it. The script awaits a promise
|
||||||
|
* that only an UNREF'd timer will resolve — the shape every engine cadence
|
||||||
|
* timer has, and the reason a healthy script reaches a loop with no ref'd
|
||||||
|
* work. Node emits `'beforeExit'` there, with the brain wide open.
|
||||||
|
*
|
||||||
|
* The engine's listener runs first (registered by `init()`, before the
|
||||||
|
* script's). The script's own listener is both its witness — it records
|
||||||
|
* that the emit happened, and the flush count AT that moment — and its
|
||||||
|
* belt: it resolves the same promise, so the pin never depends on how many
|
||||||
|
* milliseconds the engine's pass happens to keep the loop turning.
|
||||||
|
*
|
||||||
|
* The brain is DIRTY at the drain (one add, after a settling flush), so
|
||||||
|
* the pass has real work to do and pin (2) is about a flush that ran, not
|
||||||
|
* a flush that was skipped as a no-op.
|
||||||
|
*/
|
||||||
|
const script = `
|
||||||
|
import { writeFileSync as __writeFileSync, existsSync as __existsSync } from 'node:fs'
|
||||||
|
import { join as __join } from 'node:path'
|
||||||
|
import { Brainy } from ${JSON.stringify(BRAINY_SRC)}
|
||||||
|
|
||||||
|
const DIR = ${JSON.stringify(dir)}
|
||||||
|
const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: DIR } })
|
||||||
|
await brain.init()
|
||||||
|
|
||||||
|
// Count every flush that RUNS on this brain. An own property shadows the
|
||||||
|
// prototype for every caller, including the engine's own listeners.
|
||||||
|
let flushes = 0
|
||||||
|
const flushImpl = brain.flush.bind(brain)
|
||||||
|
brain.flush = () => { flushes++; return flushImpl() }
|
||||||
|
// ...and every close ENTERED. This must still be 0 after the drain.
|
||||||
|
let closes = 0
|
||||||
|
const closeImpl = brain.close.bind(brain)
|
||||||
|
brain.close = () => { closes++; return closeImpl() }
|
||||||
|
|
||||||
|
await brain.add({ data: 'written before the drain', type: 'concept' })
|
||||||
|
await brain.flush() // settle: clean brain
|
||||||
|
await new Promise((r) => setTimeout(r, 250)) // let the cadence quiet down
|
||||||
|
await brain.add({ data: 'the write the drain must persist', type: 'concept' })
|
||||||
|
|
||||||
|
const flushesBeforeDrain = flushes
|
||||||
|
let drains = 0
|
||||||
|
let flushesAtDrain = -1
|
||||||
|
const drained = new Promise((resolve) => {
|
||||||
|
const t = setTimeout(resolve, 5)
|
||||||
|
if (typeof t.unref === 'function') t.unref()
|
||||||
|
process.on('beforeExit', () => {
|
||||||
|
drains++
|
||||||
|
if (flushesAtDrain === -1) flushesAtDrain = flushes
|
||||||
|
resolve()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
await drained
|
||||||
|
|
||||||
|
// GIVE THE ENGINE'S PASS ITS FULL TURN before judging it. The signal
|
||||||
|
// path this listener used to share defers one macrotask before it
|
||||||
|
// touches an instance, so a script that resumes on the same tick as the
|
||||||
|
// emit would race past the damage and see an open brain that is about to
|
||||||
|
// be closed underneath it. Wait it out (a ref'd timer — the drain has
|
||||||
|
// already happened), then look.
|
||||||
|
await new Promise((r) => setTimeout(r, 1000))
|
||||||
|
|
||||||
|
// ---- The script is still running. The brain must still be its brain. ----
|
||||||
|
const stateAtResume = {
|
||||||
|
drains,
|
||||||
|
flushesBeforeDrain,
|
||||||
|
flushesAtDrain,
|
||||||
|
closes,
|
||||||
|
isClosed: brain.isClosed,
|
||||||
|
isClosing: brain.isClosing,
|
||||||
|
writerLockHeld: __existsSync(__join(DIR, 'locks', '_writer.lock')),
|
||||||
|
cleanCloseRecord: __existsSync(__join(DIR, 'locks', '_writer.close')),
|
||||||
|
cleanShutdownMarker:
|
||||||
|
__existsSync(__join(DIR, '_system', 'clean-shutdown.json.gz')) ||
|
||||||
|
__existsSync(__join(DIR, '_system', 'clean-shutdown.json'))
|
||||||
|
}
|
||||||
|
|
||||||
|
let addAfterDrain = null
|
||||||
|
let addError = null
|
||||||
|
try {
|
||||||
|
addAfterDrain = await brain.add({ data: 'written AFTER the drained event loop', type: 'concept' })
|
||||||
|
} catch (error) {
|
||||||
|
addError = error instanceof Error ? error.message : String(error)
|
||||||
|
}
|
||||||
|
|
||||||
|
let findHits = -1
|
||||||
|
let findError = null
|
||||||
|
try {
|
||||||
|
const results = await brain.find('written AFTER the drained event loop')
|
||||||
|
findHits = results.length
|
||||||
|
} catch (error) {
|
||||||
|
findError = error instanceof Error ? error.message : String(error)
|
||||||
|
}
|
||||||
|
|
||||||
|
__writeFileSync(
|
||||||
|
${JSON.stringify(resultPath)},
|
||||||
|
JSON.stringify({ ...stateAtResume, addAfterDrain, addError, findHits, findError, closesBeforeOurs: closes })
|
||||||
|
)
|
||||||
|
|
||||||
|
// The script ends the way a script ends: it closes its own brain, and
|
||||||
|
// the process exits on its own because nothing is left holding the loop.
|
||||||
|
await brain.close()
|
||||||
|
`
|
||||||
|
|
||||||
|
const { code, out } = await runChild(scriptDir, script)
|
||||||
|
|
||||||
|
expect(existsSync(resultPath), `child wrote no result file:\n${out}`).toBe(true)
|
||||||
|
const r = JSON.parse(readFileSync(resultPath, 'utf-8'))
|
||||||
|
|
||||||
|
// The drain really happened — this test proves nothing otherwise.
|
||||||
|
expect(r.drains, `'beforeExit' never fired:\n${out}`).toBeGreaterThanOrEqual(1)
|
||||||
|
|
||||||
|
// (1) NOTHING WAS CLOSED. This is the regression: under 10.4.11 the pass
|
||||||
|
// ran close() here and `addError` carried "it was closed via close()".
|
||||||
|
expect(r.addError, `add() after the drain failed:\n${out}`).toBeNull()
|
||||||
|
expect(r.findError, `find() after the drain failed:\n${out}`).toBeNull()
|
||||||
|
expect(r.closes, 'the engine closed the brain on a drained event loop').toBe(0)
|
||||||
|
expect(r.isClosed).toBe(false)
|
||||||
|
expect(r.isClosing).toBe(false)
|
||||||
|
expect(typeof r.addAfterDrain).toBe('string')
|
||||||
|
expect(r.findHits, `find() returned nothing:\n${out}`).toBeGreaterThanOrEqual(1)
|
||||||
|
|
||||||
|
// (1) The writer lock was never given up — a drained loop is not a handover.
|
||||||
|
expect(r.writerLockHeld, 'the writer lock was released on a drained event loop').toBe(true)
|
||||||
|
|
||||||
|
// (2) A flush RAN, and it wrote neither of close()'s markers.
|
||||||
|
expect(
|
||||||
|
r.flushesAtDrain,
|
||||||
|
`the drained-loop pass ran no flush (before=${r.flushesBeforeDrain}):\n${out}`
|
||||||
|
).toBeGreaterThan(r.flushesBeforeDrain)
|
||||||
|
expect(r.cleanShutdownMarker, 'the drained-loop flush stamped a clean-shutdown marker').toBe(false)
|
||||||
|
expect(r.cleanCloseRecord, 'the drained-loop flush wrote a clean-close record').toBe(false)
|
||||||
|
expect(out).toMatch(/All indexes flushed to disk/)
|
||||||
|
|
||||||
|
// The narration says what happened, and never claims a shutdown.
|
||||||
|
expect(out).toMatch(/event loop drained with 1 brain open/)
|
||||||
|
expect(out).toMatch(/NOTHING was closed\. A drained loop is not a shutdown/)
|
||||||
|
expect(out).not.toMatch(/Shutdown signal received/)
|
||||||
|
expect(out).not.toMatch(/Flushed successfully/)
|
||||||
|
expect(out).not.toMatch(/is not initialized/)
|
||||||
|
|
||||||
|
// (1) And the process still exits 0 on its own once the script closes up.
|
||||||
|
expect(code, `child output:\n${out}`).toBe(0)
|
||||||
|
|
||||||
|
// The store the script left behind is clean: it closed properly at the end.
|
||||||
|
expect(cleanShutdownWritten(dir), 'the script\'s own close() wrote no marker').toBe(true)
|
||||||
|
expect(existsSync(closeRecordPath(dir)), 'the script\'s own close() left no clean-close record').toBe(true)
|
||||||
|
expect(existsSync(writerLockPath(dir)), 'the writer lock outlived close()').toBe(false)
|
||||||
|
}, 300_000)
|
||||||
|
|
||||||
|
it('(2) the pass is repeatable and idempotent: a second drain closes nothing either', async () => {
|
||||||
|
// In-process, so the assertions are on the object itself rather than on a
|
||||||
|
// report: 'beforeExit' is an ordinary event, and emitting it twice must
|
||||||
|
// leave the brain exactly as usable as it was.
|
||||||
|
const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||||
|
await brain.init()
|
||||||
|
|
||||||
|
const flushed: Promise<void>[] = []
|
||||||
|
const flushImpl = brain.flush.bind(brain)
|
||||||
|
;(brain as unknown as { flush: () => Promise<void> }).flush = () => {
|
||||||
|
const p = flushImpl()
|
||||||
|
flushed.push(p)
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
await brain.add({ data: 'a write the drained loop must persist', type: NounType.Concept })
|
||||||
|
|
||||||
|
for (const pass of [1, 2]) {
|
||||||
|
const before = flushed.length
|
||||||
|
process.emit('beforeExit', 0)
|
||||||
|
await Promise.all(flushed.slice(before).map((p) => p.catch(() => {})))
|
||||||
|
// Let the pass's own `finally` run (it settles a microtask after ours),
|
||||||
|
// so the next emit is not turned away by the in-flight guard.
|
||||||
|
await new Promise((r) => setTimeout(r, 50))
|
||||||
|
|
||||||
|
expect(brain.isClosed, `pass ${pass} closed the brain`).toBe(false)
|
||||||
|
expect(brain.isClosing, `pass ${pass} started a close`).toBe(false)
|
||||||
|
expect(existsSync(writerLockPath(dir)), `pass ${pass} released the writer lock`).toBe(true)
|
||||||
|
expect(existsSync(closeRecordPath(dir)), `pass ${pass} wrote a clean-close record`).toBe(false)
|
||||||
|
expect(cleanShutdownWritten(dir), `pass ${pass} stamped a clean-shutdown marker`).toBe(false)
|
||||||
|
|
||||||
|
// Still a working brain, after every pass.
|
||||||
|
const id = await brain.add({ data: `still writable after drain ${pass}`, type: NounType.Concept })
|
||||||
|
expect(id).toBeTruthy()
|
||||||
|
}
|
||||||
|
|
||||||
|
// The first pass had a dirty brain and flushed it; the second found it
|
||||||
|
// clean and cost nothing. Either way, neither closed anything.
|
||||||
|
expect(flushed.length).toBeGreaterThanOrEqual(2)
|
||||||
|
|
||||||
|
await brain.close()
|
||||||
|
expect(brain.isClosed).toBe(true)
|
||||||
|
expect(cleanShutdownWritten(dir)).toBe(true)
|
||||||
|
}, 300_000)
|
||||||
|
})
|
||||||
111
tests/integration/counts-persist-single-flight.test.ts
Normal file
111
tests/integration/counts-persist-single-flight.test.ts
Normal file
|
|
@ -0,0 +1,111 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/counts-persist-single-flight
|
||||||
|
* @description Regression for a production race in FileSystemStorage's
|
||||||
|
* counts ledger: `persistCounts()` was write-through on every count change
|
||||||
|
* with no serialization, and the atomic writer named its temp file with
|
||||||
|
* millisecond granularity (`.tmp-<pid>-<ms>`). Two persists inside one
|
||||||
|
* millisecond shared the temp path — both wrote it, the first rename
|
||||||
|
* consumed it, the second rename found nothing: ENOENT, ~1,500 times a day
|
||||||
|
* on a busy production brain, with a full ledger write per change behind it.
|
||||||
|
*
|
||||||
|
* Under pin: persists are single-flight and coalesced — one in flight, at
|
||||||
|
* most one trailing pass carrying the burst's final state — and every atomic
|
||||||
|
* write owns a unique temp path. A burst of N count changes costs at most
|
||||||
|
* two ledger writes, never errors, and leaves a ledger equal to memory.
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||||
|
import * as fs from 'node:fs'
|
||||||
|
import * as os from 'node:os'
|
||||||
|
import * as path from 'node:path'
|
||||||
|
import { Brainy } from '../../src/brainy.js'
|
||||||
|
import { NounType } from '../../src/types/graphTypes.js'
|
||||||
|
|
||||||
|
describe('counts persistence is single-flight, coalesced, and never races its own temp file', () => {
|
||||||
|
let dir: string
|
||||||
|
let brain: any
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
||||||
|
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-counts-race-'))
|
||||||
|
brain = new Brainy({
|
||||||
|
requireSubtype: false,
|
||||||
|
storage: { type: 'filesystem', path: dir },
|
||||||
|
dimensions: 384,
|
||||||
|
silent: true
|
||||||
|
})
|
||||||
|
await brain.init()
|
||||||
|
})
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
vi.restoreAllMocks()
|
||||||
|
await brain.close()
|
||||||
|
fs.rmSync(dir, { recursive: true, force: true })
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a burst of concurrent count changes → at most two ledger writes, zero errors, ledger == memory', async () => {
|
||||||
|
const storage = brain.storage
|
||||||
|
const countsPath: string = storage.countsFilePath
|
||||||
|
expect(countsPath, 'the filesystem adapter persists a counts ledger').toBeTruthy()
|
||||||
|
|
||||||
|
// Let init's own persists settle so the burst is measured alone.
|
||||||
|
await storage.flushCounts?.()
|
||||||
|
|
||||||
|
const renameSpy = vi.spyOn(fs.promises, 'rename')
|
||||||
|
const errorSpy = vi.spyOn(console, 'error')
|
||||||
|
|
||||||
|
// Twenty-five concurrent count changes — the shape of a write burst; each
|
||||||
|
// used to launch its own persist.
|
||||||
|
const BURST = 25
|
||||||
|
await Promise.all(
|
||||||
|
Array.from({ length: BURST }, () => storage.scheduleCountPersist())
|
||||||
|
)
|
||||||
|
|
||||||
|
const ledgerRenames = renameSpy.mock.calls.filter(([, to]) => String(to) === countsPath)
|
||||||
|
expect(ledgerRenames.length, 'single-flight + one trailing pass').toBeLessThanOrEqual(2)
|
||||||
|
expect(ledgerRenames.length, 'the burst was persisted at all').toBeGreaterThanOrEqual(1)
|
||||||
|
|
||||||
|
const persistErrors = errorSpy.mock.calls.filter((args) => String(args[0]).includes('persisting counts'))
|
||||||
|
expect(persistErrors).toEqual([])
|
||||||
|
|
||||||
|
const ledger = JSON.parse(fs.readFileSync(countsPath, 'utf-8'))
|
||||||
|
expect(ledger.totalNounCount).toBe(storage.totalNounCount)
|
||||||
|
expect(ledger.totalVerbCount).toBe(storage.totalVerbCount)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('real writes in parallel: the ledger lands complete and no persist error is logged', async () => {
|
||||||
|
const storage = brain.storage
|
||||||
|
const countsPath: string = storage.countsFilePath
|
||||||
|
const errorSpy = vi.spyOn(console, 'error')
|
||||||
|
|
||||||
|
await Promise.all(
|
||||||
|
Array.from({ length: 12 }, (_, i) =>
|
||||||
|
brain.add({ data: `burst row ${i}`, type: NounType.Thing })
|
||||||
|
)
|
||||||
|
)
|
||||||
|
await storage.flushCounts?.()
|
||||||
|
|
||||||
|
const persistErrors = errorSpy.mock.calls.filter((args) => String(args[0]).includes('persisting counts'))
|
||||||
|
expect(persistErrors).toEqual([])
|
||||||
|
const ledger = JSON.parse(fs.readFileSync(countsPath, 'utf-8'))
|
||||||
|
expect(ledger.totalNounCount).toBe(storage.totalNounCount)
|
||||||
|
expect(await brain.getNounCount()).toBe(ledger.totalNounCount)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('every atomic write owns its own temp path — two writes in one millisecond never collide', async () => {
|
||||||
|
const storage = brain.storage
|
||||||
|
const tmpNames: string[] = []
|
||||||
|
vi.spyOn(fs.promises, 'writeFile').mockImplementation(async (p: any) => {
|
||||||
|
tmpNames.push(String(p))
|
||||||
|
})
|
||||||
|
vi.spyOn(fs.promises, 'rename').mockImplementation(async () => undefined)
|
||||||
|
const target = path.join(dir, 'probe.json')
|
||||||
|
await Promise.all([
|
||||||
|
storage.writeFileAtomic(target, '{"a":1}'),
|
||||||
|
storage.writeFileAtomic(target, '{"a":2}'),
|
||||||
|
storage.writeFileAtomic(target, '{"a":3}')
|
||||||
|
])
|
||||||
|
const probeTmps = tmpNames.filter((n) => n.startsWith(`${target}.tmp-`))
|
||||||
|
expect(probeTmps.length).toBe(3)
|
||||||
|
expect(new Set(probeTmps).size, 'no two writes shared a temp path').toBe(3)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
@ -7,7 +7,7 @@
|
||||||
* - Backward compatibility preserved
|
* - Backward compatibility preserved
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { describe, it, expect, beforeEach } from 'vitest'
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
import { Brainy } from '../../src/brainy.js'
|
import { Brainy } from '../../src/brainy.js'
|
||||||
import { NounType } from '../../src/types/graphTypes.js'
|
import { NounType } from '../../src/types/graphTypes.js'
|
||||||
|
|
||||||
|
|
@ -19,6 +19,10 @@ describe('Entity Confidence & Weight Exposure', () => {
|
||||||
await brain.init()
|
await brain.init()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
|
||||||
describe('Entity interface', () => {
|
describe('Entity interface', () => {
|
||||||
it('should expose confidence when adding entity with confidence', async () => {
|
it('should expose confidence when adding entity with confidence', async () => {
|
||||||
const id = await brain.add({
|
const id = await brain.add({
|
||||||
|
|
|
||||||
360
tests/integration/factlog-open-prune.test.ts
Normal file
360
tests/integration/factlog-open-prune.test.ts
Normal file
|
|
@ -0,0 +1,360 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/factlog-open-prune
|
||||||
|
* @description THE OPEN READS THE TAIL, NOT THE HISTORY.
|
||||||
|
*
|
||||||
|
* Every log-authority open asks the fact log one question — "is there a fact
|
||||||
|
* above the committed pointer?" — and until this lane existed it answered by
|
||||||
|
* reading and CRC-decoding EVERY segment file the manifest names. MEASURED in
|
||||||
|
* production on a 16k-row brain at generation ~478,819: 34-37 seconds inside
|
||||||
|
* the `generation-store-open-fold` phase, on every open, including the clean
|
||||||
|
* one where the answer is always "nothing".
|
||||||
|
*
|
||||||
|
* The manifest already records each sealed segment's `lastGeneration`, written
|
||||||
|
* at seal time AFTER the segment's bytes are fsynced and into a manifest that
|
||||||
|
* is itself written atomically and fsynced — and a sealed file is never
|
||||||
|
* appended to again (the same manifest flip re-points `tailSegment`). So an
|
||||||
|
* entry recording `lastGeneration ≤ committed` PROVES its file holds nothing
|
||||||
|
* above the bound, and the open can skip it whole.
|
||||||
|
*
|
||||||
|
* Pinned here, from the log's own counters (the narration line), never a clock:
|
||||||
|
*
|
||||||
|
* 1. A clean close and reopen on a log with ≥4 sealed segments reads
|
||||||
|
* EXACTLY the tail (1 of 6), prunes the rest, and finds nothing.
|
||||||
|
* 2. A real SIGKILLed process that sealed segments holding facts ABOVE the
|
||||||
|
* committed pointer: the reopen READS those sealed segments and recovers
|
||||||
|
* byte-identically to an unpruned open (differential — the same store,
|
||||||
|
* with the provable field stripped from its manifest, takes the full-scan
|
||||||
|
* path and must agree fact for fact, before and after `open()`).
|
||||||
|
* 3. A manifest entry with no `lastGeneration` (legacy, or hand-repaired) is
|
||||||
|
* READ. Never prune what the manifest cannot prove.
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, afterEach } from 'vitest'
|
||||||
|
import * as fs from 'node:fs'
|
||||||
|
import * as os from 'node:os'
|
||||||
|
import * as path from 'node:path'
|
||||||
|
import { spawn } from 'node:child_process'
|
||||||
|
import {
|
||||||
|
FactLog,
|
||||||
|
FACTS_MANIFEST_PATH,
|
||||||
|
type CommitFact,
|
||||||
|
type FactLogStorage
|
||||||
|
} from '../../src/db/factLog.js'
|
||||||
|
import { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js'
|
||||||
|
|
||||||
|
const REPO_ROOT = process.cwd()
|
||||||
|
const TSX = path.join(REPO_ROOT, 'node_modules', '.bin', 'tsx')
|
||||||
|
/** ~1KB frames against a 4KB rotation threshold: ~5 facts per segment. */
|
||||||
|
const ROTATE_BYTES = 4096
|
||||||
|
|
||||||
|
const tmpDirs: string[] = []
|
||||||
|
function makeTempDir(): string {
|
||||||
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-factlog-prune-'))
|
||||||
|
tmpDirs.push(dir)
|
||||||
|
return dir
|
||||||
|
}
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
for (const dir of tmpDirs.splice(0)) {
|
||||||
|
try {
|
||||||
|
fs.rmSync(dir, { recursive: true, force: true })
|
||||||
|
} catch {
|
||||||
|
/* best effort */
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
fs.rmSync(`${dir}.ready.json`, { force: true })
|
||||||
|
} catch {
|
||||||
|
/* best effort */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
const UUID = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, '0')}`
|
||||||
|
|
||||||
|
/** One ~1KB fact — the padding is what makes rotation cheap to provoke. */
|
||||||
|
function fact(generation: number): CommitFact {
|
||||||
|
return {
|
||||||
|
generation,
|
||||||
|
timestamp: 1_700_000_000_000 + generation,
|
||||||
|
ops: [
|
||||||
|
{
|
||||||
|
kind: 'noun',
|
||||||
|
id: UUID(generation),
|
||||||
|
record: {
|
||||||
|
metadata: { noun: 'document', pad: 'x'.repeat(900), g: generation },
|
||||||
|
vector: null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A deterministic int minter so the log writes the V2 format production
|
||||||
|
* writes (the prune is a manifest-level decision and never touches segment
|
||||||
|
* bytes — but the pins should run against the bytes the fleet actually has).
|
||||||
|
*/
|
||||||
|
function makeMinter(): (kind: 'noun' | 'verb', id: string) => bigint {
|
||||||
|
const ints = new Map<string, bigint>()
|
||||||
|
return (kind, id) => {
|
||||||
|
const key = `${kind}:${id}`
|
||||||
|
let minted = ints.get(key)
|
||||||
|
if (minted === undefined) {
|
||||||
|
minted = BigInt(ints.size + 1)
|
||||||
|
ints.set(key, minted)
|
||||||
|
}
|
||||||
|
return minted
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Open a fact log over a store directory (a fresh adapter each time — this is
|
||||||
|
* what a reopen actually does). */
|
||||||
|
async function openStore(dir: string): Promise<{ storage: any; log: FactLog }> {
|
||||||
|
const storage: any = new FileSystemStorage(dir)
|
||||||
|
await storage.init()
|
||||||
|
const log = new FactLog(storage as FactLogStorage, { rotateBytes: ROTATE_BYTES })
|
||||||
|
log.setIntMinter(makeMinter())
|
||||||
|
return { storage, log }
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Build a log of `count` facts (rotating every ~5), left durable, not closed. */
|
||||||
|
async function buildLog(dir: string, count: number): Promise<number> {
|
||||||
|
const { log } = await openStore(dir)
|
||||||
|
await log.open(0)
|
||||||
|
for (let g = 1; g <= count; g++) await log.append(fact(g))
|
||||||
|
await log.sync()
|
||||||
|
return log.headGeneration()
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Capture the narration channel (`prodLog.narrate` → console.warn). */
|
||||||
|
async function captureNarration<T>(
|
||||||
|
fn: () => Promise<T>
|
||||||
|
): Promise<{ result: T; lines: string[] }> {
|
||||||
|
const lines: string[] = []
|
||||||
|
const original = console.warn
|
||||||
|
console.warn = ((...args: unknown[]) => {
|
||||||
|
lines.push(args.map((a) => String(a)).join(' '))
|
||||||
|
}) as typeof console.warn
|
||||||
|
try {
|
||||||
|
return { result: await fn(), lines }
|
||||||
|
} finally {
|
||||||
|
console.warn = original
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The counters the open narrated — the pin's only source of truth for what
|
||||||
|
* was read (a wall-clock assertion could pass on a warm page cache). */
|
||||||
|
function scanCounts(lines: string[]): { read: number; pruned: number; total: number } {
|
||||||
|
const line = lines.find((l) => l.includes('[FactLog] above-manifest peek above generation'))
|
||||||
|
if (!line) {
|
||||||
|
throw new Error(`no peek narration in:\n${lines.join('\n')}`)
|
||||||
|
}
|
||||||
|
const match = /(\d+) segment\(s\) read, (\d+) pruned of (\d+)/.exec(line)
|
||||||
|
if (!match) throw new Error(`unparsable peek narration: ${line}`)
|
||||||
|
return { read: Number(match[1]), pruned: Number(match[2]), total: Number(match[3]) }
|
||||||
|
}
|
||||||
|
|
||||||
|
interface SegmentEntryOnDisk {
|
||||||
|
file: string
|
||||||
|
firstGeneration: number
|
||||||
|
lastGeneration?: number
|
||||||
|
facts: number
|
||||||
|
bytes: number
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readManifest(dir: string): Promise<{
|
||||||
|
segments: SegmentEntryOnDisk[]
|
||||||
|
tailSegment: string | null
|
||||||
|
}> {
|
||||||
|
const storage: any = new FileSystemStorage(dir)
|
||||||
|
await storage.init()
|
||||||
|
return (await storage.readRawObject(FACTS_MANIFEST_PATH)) as any
|
||||||
|
}
|
||||||
|
|
||||||
|
async function rewriteManifest(
|
||||||
|
dir: string,
|
||||||
|
mutate: (manifest: any) => void
|
||||||
|
): Promise<void> {
|
||||||
|
const storage: any = new FileSystemStorage(dir)
|
||||||
|
await storage.init()
|
||||||
|
const manifest = await storage.readRawObject(FACTS_MANIFEST_PATH)
|
||||||
|
mutate(manifest)
|
||||||
|
await storage.writeRawObject(FACTS_MANIFEST_PATH, manifest)
|
||||||
|
await storage.syncRawObjects([FACTS_MANIFEST_PATH])
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Every fact the log holds, in order — the recovered state, read back. */
|
||||||
|
async function allFacts(log: FactLog): Promise<CommitFact[]> {
|
||||||
|
const out: CommitFact[] = []
|
||||||
|
const handle = log.scanFacts()
|
||||||
|
for await (const batch of handle.batches()) out.push(...batch.facts)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('fact log — the open reads only the segments that can hold facts above the bound', () => {
|
||||||
|
it('a clean close + reopen over ≥4 sealed segments reads exactly the tail and finds nothing', async () => {
|
||||||
|
const dir = makeTempDir()
|
||||||
|
const head = await buildLog(dir, 30)
|
||||||
|
|
||||||
|
const manifest = await readManifest(dir)
|
||||||
|
expect(manifest.segments.length).toBeGreaterThanOrEqual(4) // the fixture is real
|
||||||
|
expect(manifest.tailSegment).not.toBeNull()
|
||||||
|
|
||||||
|
// The reopen: a clean close means committed === the log's head.
|
||||||
|
const { log } = await openStore(dir)
|
||||||
|
const { result: orphans, lines } = await captureNarration(() => log.peekFactsAbove(head))
|
||||||
|
|
||||||
|
expect(orphans).toEqual([]) // the fold finds nothing, as it always does after a clean close
|
||||||
|
const counts = scanCounts(lines)
|
||||||
|
expect(counts.read).toBe(1) // EXACTLY the tail
|
||||||
|
expect(counts.total).toBe(manifest.segments.length + 1)
|
||||||
|
expect(counts.pruned).toBe(manifest.segments.length)
|
||||||
|
|
||||||
|
// And the reconciling open still lands on the same committed prefix.
|
||||||
|
await log.open(head)
|
||||||
|
expect(log.headGeneration()).toBe(head)
|
||||||
|
expect((await allFacts(log)).map((f) => f.generation)).toEqual(
|
||||||
|
Array.from({ length: head }, (_, i) => i + 1)
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a manifest entry with no lastGeneration is READ — never prune what you cannot prove', async () => {
|
||||||
|
const dir = makeTempDir()
|
||||||
|
const head = await buildLog(dir, 30)
|
||||||
|
const before = await readManifest(dir)
|
||||||
|
expect(before.segments.length).toBeGreaterThanOrEqual(4)
|
||||||
|
|
||||||
|
// A legacy/hand-repaired entry: the field the prune needs is simply absent.
|
||||||
|
await rewriteManifest(dir, (m) => {
|
||||||
|
delete m.segments[0].lastGeneration
|
||||||
|
})
|
||||||
|
|
||||||
|
const { log } = await openStore(dir)
|
||||||
|
const { result: orphans, lines } = await captureNarration(() => log.peekFactsAbove(head))
|
||||||
|
|
||||||
|
expect(orphans).toEqual([]) // still nothing above the bound — it was READ to find out
|
||||||
|
const counts = scanCounts(lines)
|
||||||
|
expect(counts.read).toBe(2) // the unprovable entry + the tail
|
||||||
|
expect(counts.pruned).toBe(before.segments.length - 1)
|
||||||
|
expect(counts.total).toBe(before.segments.length + 1)
|
||||||
|
})
|
||||||
|
|
||||||
|
it(
|
||||||
|
'a SIGKILLed writer that sealed segments above the committed pointer recovers identically to an unpruned open',
|
||||||
|
async () => {
|
||||||
|
const dir = makeTempDir()
|
||||||
|
const readyPath = `${dir}.ready.json`
|
||||||
|
// A real process death: the child fsyncs its segments, records what it
|
||||||
|
// reached, and SIGKILLs ITSELF — no close, no unwind, no chance to tidy.
|
||||||
|
const script = `
|
||||||
|
import * as fs from 'node:fs'
|
||||||
|
import { FactLog } from ${JSON.stringify(path.join(REPO_ROOT, 'src', 'db', 'factLog.ts'))}
|
||||||
|
import { FileSystemStorage } from ${JSON.stringify(path.join(REPO_ROOT, 'src', 'storage', 'adapters', 'fileSystemStorage.ts'))}
|
||||||
|
const UUID = (n) => '00000000-0000-4000-8000-' + String(n).padStart(12, '0')
|
||||||
|
const fact = (g) => ({
|
||||||
|
generation: g,
|
||||||
|
timestamp: 1700000000000 + g,
|
||||||
|
ops: [{ kind: 'noun', id: UUID(g), record: { metadata: { noun: 'document', pad: 'x'.repeat(900), g }, vector: null } }]
|
||||||
|
})
|
||||||
|
const ints = new Map()
|
||||||
|
const storage = new FileSystemStorage(${JSON.stringify(dir)})
|
||||||
|
await storage.init()
|
||||||
|
const log = new FactLog(storage, { rotateBytes: ${ROTATE_BYTES} })
|
||||||
|
log.setIntMinter((kind, id) => {
|
||||||
|
const key = kind + ':' + id
|
||||||
|
if (!ints.has(key)) ints.set(key, BigInt(ints.size + 1))
|
||||||
|
return ints.get(key)
|
||||||
|
})
|
||||||
|
await log.open(0)
|
||||||
|
for (let g = 1; g <= 30; g++) await log.append(fact(g))
|
||||||
|
await log.sync()
|
||||||
|
fs.writeFileSync(${JSON.stringify(readyPath)}, JSON.stringify({ head: log.headGeneration() }))
|
||||||
|
process.kill(process.pid, 'SIGKILL')
|
||||||
|
`
|
||||||
|
const scriptPath = path.join(dir, 'crash-writer.mts')
|
||||||
|
fs.writeFileSync(scriptPath, script)
|
||||||
|
const child = spawn(TSX, [scriptPath], { cwd: REPO_ROOT, stdio: ['ignore', 'pipe', 'pipe'] })
|
||||||
|
let output = ''
|
||||||
|
child.stdout.on('data', (d) => { output += String(d) })
|
||||||
|
child.stderr.on('data', (d) => { output += String(d) })
|
||||||
|
const exit = await new Promise<{ code: number | null; signal: string | null }>((resolve) =>
|
||||||
|
child.on('exit', (code, signal) => resolve({ code, signal }))
|
||||||
|
)
|
||||||
|
if (!fs.existsSync(readyPath)) {
|
||||||
|
throw new Error(`the crash writer never reached its kill point:\n${output}`)
|
||||||
|
}
|
||||||
|
// Death, not a shutdown: no close(), no unwind, no orderly exit code.
|
||||||
|
expect(exit.signal ?? `code ${exit.code}`).not.toBe('code 0')
|
||||||
|
const head = JSON.parse(fs.readFileSync(readyPath, 'utf8')).head as number
|
||||||
|
expect(head).toBe(30)
|
||||||
|
|
||||||
|
// The committed pointer the survivor comes back on: mid-log, so sealed
|
||||||
|
// segments hold facts ABOVE it — the exact shape the prune must not skip.
|
||||||
|
const committed = 12
|
||||||
|
const manifest = await readManifest(dir)
|
||||||
|
const straddling = manifest.segments.filter(
|
||||||
|
(s) => s.firstGeneration <= committed && (s.lastGeneration ?? 0) > committed
|
||||||
|
)
|
||||||
|
const entirelyAbove = manifest.segments.filter((s) => s.firstGeneration > committed)
|
||||||
|
expect(straddling.length).toBeGreaterThanOrEqual(1)
|
||||||
|
expect(entirelyAbove.length).toBeGreaterThanOrEqual(1)
|
||||||
|
|
||||||
|
// THE DIFFERENTIAL. The unpruned answer, through the SAME code on the
|
||||||
|
// SAME bytes: a peek above generation 0 can prune nothing (no sealed
|
||||||
|
// segment ends at or below 0), so it reads every segment file and
|
||||||
|
// decodes every frame — exactly what this open used to do — and its
|
||||||
|
// facts above the pointer are what the fold is entitled to replay.
|
||||||
|
const { log } = await openStore(dir)
|
||||||
|
const { result: fullScan, lines: fullLines } = await captureNarration(() =>
|
||||||
|
log.peekFactsAbove(0)
|
||||||
|
)
|
||||||
|
expect(scanCounts(fullLines)).toEqual({
|
||||||
|
read: manifest.segments.length + 1,
|
||||||
|
pruned: 0,
|
||||||
|
total: manifest.segments.length + 1
|
||||||
|
})
|
||||||
|
const unprunedAnswer = fullScan.filter((f) => f.generation > committed)
|
||||||
|
|
||||||
|
const { result: prunedAnswer, lines } = await captureNarration(() =>
|
||||||
|
log.peekFactsAbove(committed)
|
||||||
|
)
|
||||||
|
|
||||||
|
// The sealed segments above the bound were READ, not skipped.
|
||||||
|
const counts = scanCounts(lines)
|
||||||
|
expect(counts.read).toBe(straddling.length + entirelyAbove.length + 1)
|
||||||
|
expect(counts.pruned).toBe(manifest.segments.length - straddling.length - entirelyAbove.length)
|
||||||
|
expect(counts.pruned).toBeGreaterThan(0) // the prune did engage, and was still right
|
||||||
|
expect(prunedAnswer.map((f) => f.generation)).toEqual(
|
||||||
|
Array.from({ length: head - committed }, (_, i) => committed + 1 + i)
|
||||||
|
)
|
||||||
|
// Facts that live in a SEALED segment (not the tail) came back.
|
||||||
|
expect(prunedAnswer.some((f) => f.generation <= (straddling[0].lastGeneration ?? 0))).toBe(
|
||||||
|
true
|
||||||
|
)
|
||||||
|
// Fact for fact, the pruned answer IS the unpruned answer — so whatever
|
||||||
|
// the recovery replays, it replays identically.
|
||||||
|
expect(prunedAnswer).toEqual(unprunedAnswer)
|
||||||
|
|
||||||
|
// The fold's streaming twin (the unclean-open path) agrees too.
|
||||||
|
const streamed: CommitFact[] = []
|
||||||
|
for await (const batch of log.streamFactsAbove(committed)) streamed.push(...batch)
|
||||||
|
expect(streamed).toEqual(unprunedAnswer)
|
||||||
|
|
||||||
|
// And the reconciling open rolls back exactly as it always did: the two
|
||||||
|
// never-committed sealed segments dropped, the straddling one cut, the
|
||||||
|
// tail truncated — the log left as the committed prefix.
|
||||||
|
await log.open(committed)
|
||||||
|
expect(log.headGeneration()).toBe(committed)
|
||||||
|
expect((await allFacts(log)).map((f) => f.generation)).toEqual(
|
||||||
|
Array.from({ length: committed }, (_, i) => i + 1)
|
||||||
|
)
|
||||||
|
const after = await readManifest(dir)
|
||||||
|
expect(after.segments.map((s) => s.file)).toEqual(
|
||||||
|
manifest.segments
|
||||||
|
.filter((s) => s.firstGeneration <= committed)
|
||||||
|
.map((s) => s.file)
|
||||||
|
)
|
||||||
|
expect(after.segments[after.segments.length - 1].lastGeneration).toBe(committed)
|
||||||
|
},
|
||||||
|
120_000
|
||||||
|
)
|
||||||
|
})
|
||||||
193
tests/integration/find-connected-order.test.ts
Normal file
193
tests/integration/find-connected-order.test.ts
Normal file
|
|
@ -0,0 +1,193 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/find-connected-order
|
||||||
|
* @description The graph-first law for `find({ connected })` (10.4.8).
|
||||||
|
*
|
||||||
|
* With `connected` present the neighbour set is the candidate universe: it is
|
||||||
|
* resolved from the adjacency first, the metadata filter is evaluated over
|
||||||
|
* those ids only, and the page is cut last. The earlier order materialized the
|
||||||
|
* whole-store filtered id list, paged it, hydrated the page, and only then
|
||||||
|
* intersected with the neighbours — so a neighbour outside the first page of
|
||||||
|
* the filtered STORE was silently dropped, and every call paid O(store).
|
||||||
|
*
|
||||||
|
* These pins hold both halves. The answer: every matching neighbour is
|
||||||
|
* reachable by paging, a non-neighbour never appears, a negation (`missing`)
|
||||||
|
* is evaluated over the neighbours, `orderBy` sorts the whole neighbour set
|
||||||
|
* before the page is cut, and the vector leg walks the neighbours only. The
|
||||||
|
* cost shape: the metadata index is asked about the neighbour ids only, and
|
||||||
|
* hydration is one page — never the store.
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'
|
||||||
|
import { Brainy } from '../../src/brainy'
|
||||||
|
import { NounType, VerbType } from '../../src/types/graphTypes'
|
||||||
|
import { v5 } from '../../src/universal/uuid'
|
||||||
|
import { generateTestVector } from '../helpers/test-factory'
|
||||||
|
|
||||||
|
/** Matching rows that are NOT neighbours — added FIRST, so the whole-store filtered list leads with them. */
|
||||||
|
const NOISE = 120
|
||||||
|
/** Matching rows that ARE neighbours of the anchor. */
|
||||||
|
const NEIGHBOURS = 30
|
||||||
|
/** Neighbours carrying `retracted: true` — excluded by the `missing` negation. */
|
||||||
|
const RETRACTED = 4
|
||||||
|
|
||||||
|
describe('find({ connected }) is graph-first: neighbours → filter → page', () => {
|
||||||
|
let brain: Brainy<any>
|
||||||
|
const anchor = 'anchor'
|
||||||
|
const sharedVector = generateTestVector()
|
||||||
|
const neighbourIds = new Set(Array.from({ length: NEIGHBOURS }, (_, i) => v5(`nb-${i}`)))
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||||
|
await brain.init()
|
||||||
|
await brain.add({
|
||||||
|
id: anchor,
|
||||||
|
data: 'the anchor',
|
||||||
|
type: NounType.Person,
|
||||||
|
metadata: { kind: 'anchor' },
|
||||||
|
vector: generateTestVector()
|
||||||
|
})
|
||||||
|
for (let i = 0; i < NOISE; i++) {
|
||||||
|
await brain.add({
|
||||||
|
id: `noise-${i}`,
|
||||||
|
data: `noise ${i}`,
|
||||||
|
type: NounType.Person,
|
||||||
|
metadata: { kind: 'note', rank: 1000 + i },
|
||||||
|
vector: sharedVector
|
||||||
|
})
|
||||||
|
}
|
||||||
|
for (let i = 0; i < NEIGHBOURS; i++) {
|
||||||
|
await brain.add({
|
||||||
|
id: `nb-${i}`,
|
||||||
|
data: `neighbour ${i}`,
|
||||||
|
type: NounType.Person,
|
||||||
|
metadata: { kind: 'note', rank: i + 1, ...(i < RETRACTED ? { retracted: true } : {}) },
|
||||||
|
vector: sharedVector
|
||||||
|
})
|
||||||
|
await brain.relate({ from: anchor, to: `nb-${i}`, type: VerbType.Knows })
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
// CLOSE IT. Dropping the reference does not close a brain — it only makes
|
||||||
|
// it unreachable from here. The instance stays open and registered, its
|
||||||
|
// unref'd cadence timer keeps running, and because the gate config runs the
|
||||||
|
// whole suite in ONE process (pool: 'forks', singleFork: true) it goes on
|
||||||
|
// narrating its flushes into every test file that runs after this one.
|
||||||
|
// A test that leaks a brain is a defect of the test.
|
||||||
|
await brain?.close()
|
||||||
|
brain = null as any
|
||||||
|
})
|
||||||
|
|
||||||
|
it('returns the matching neighbours page by page — none dropped, never a non-neighbour', async () => {
|
||||||
|
const seen = new Set<string>()
|
||||||
|
for (let offset = 0; offset <= NEIGHBOURS; offset += 10) {
|
||||||
|
const page = await brain.find({
|
||||||
|
connected: { from: anchor, direction: 'out' },
|
||||||
|
where: { kind: 'note' },
|
||||||
|
limit: 10,
|
||||||
|
offset
|
||||||
|
})
|
||||||
|
expect(page).toHaveLength(offset < NEIGHBOURS ? 10 : 0)
|
||||||
|
for (const r of page) {
|
||||||
|
expect(neighbourIds.has(r.entity.id)).toBe(true)
|
||||||
|
expect(seen.has(r.entity.id)).toBe(false)
|
||||||
|
seen.add(r.entity.id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
expect(seen.size).toBe(NEIGHBOURS)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('evaluates a negation (`missing`) over the neighbour set, not the store', async () => {
|
||||||
|
const results = await brain.find({
|
||||||
|
connected: { from: anchor, direction: 'out' },
|
||||||
|
where: { kind: 'note', retracted: { missing: true } },
|
||||||
|
limit: 100
|
||||||
|
})
|
||||||
|
expect(results).toHaveLength(NEIGHBOURS - RETRACTED)
|
||||||
|
for (const r of results) {
|
||||||
|
expect(neighbourIds.has(r.entity.id)).toBe(true)
|
||||||
|
expect(r.entity.metadata.retracted).toBeUndefined()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('asks the metadata index about the neighbour ids only, and hydrates one page', async () => {
|
||||||
|
const index = (brain as any).metadataIndex
|
||||||
|
const within = vi.spyOn(index, 'filterIdsWithin')
|
||||||
|
const hydrate = vi.spyOn(brain as any, 'batchGet')
|
||||||
|
try {
|
||||||
|
const results = await brain.find({
|
||||||
|
connected: { from: anchor, direction: 'out' },
|
||||||
|
where: { kind: 'note' },
|
||||||
|
limit: 10
|
||||||
|
})
|
||||||
|
expect(results).toHaveLength(10)
|
||||||
|
expect(within).toHaveBeenCalledTimes(1)
|
||||||
|
const askedIds = within.mock.calls[0][1] as string[]
|
||||||
|
expect(askedIds).toHaveLength(NEIGHBOURS)
|
||||||
|
for (const id of askedIds) expect(neighbourIds.has(id)).toBe(true)
|
||||||
|
expect(hydrate).toHaveBeenCalledTimes(1)
|
||||||
|
expect(hydrate.mock.calls[0][0]).toHaveLength(10)
|
||||||
|
} finally {
|
||||||
|
within.mockRestore()
|
||||||
|
hydrate.mockRestore()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('orders the WHOLE neighbour set before cutting the page', async () => {
|
||||||
|
const results = await brain.find({
|
||||||
|
connected: { from: anchor, direction: 'out' },
|
||||||
|
where: { kind: 'note' },
|
||||||
|
orderBy: 'rank',
|
||||||
|
order: 'desc',
|
||||||
|
limit: 5
|
||||||
|
})
|
||||||
|
expect(results.map((r) => r.entity.metadata.rank)).toEqual([30, 29, 28, 27, 26])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('walks the vector leg over the neighbours only', async () => {
|
||||||
|
// The SAME query without the vector leg, first. Both legs draw from the
|
||||||
|
// one neighbour set, so this is the control: it says whether a short answer
|
||||||
|
// came from the adjacency/filter (both legs short) or from the vector walk
|
||||||
|
// alone (only the vector leg short). Cheap, and it turns a bare count
|
||||||
|
// mismatch into a named half — this case has gone red on the gate box
|
||||||
|
// while passing in isolation and beside its own predecessor, so the next
|
||||||
|
// red must arrive already carrying the half it belongs to.
|
||||||
|
const control = await brain.find({
|
||||||
|
connected: { from: anchor, direction: 'out' },
|
||||||
|
where: { kind: 'note' },
|
||||||
|
limit: 5
|
||||||
|
})
|
||||||
|
|
||||||
|
const results = await brain.find({
|
||||||
|
vector: sharedVector,
|
||||||
|
connected: { from: anchor, direction: 'out' },
|
||||||
|
where: { kind: 'note' },
|
||||||
|
limit: 5
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(
|
||||||
|
results.length,
|
||||||
|
`the vector leg returned ${results.length} of a requested 5. The same query ` +
|
||||||
|
`WITHOUT the vector returned ${control.length}: if that is also short the ` +
|
||||||
|
`neighbour set or the filter is the cause, and if it is 5 the vector walk is — ` +
|
||||||
|
`note every row in this corpus carries an identical vector, so the walk is ` +
|
||||||
|
`ranking an exact tie.`
|
||||||
|
).toBe(5)
|
||||||
|
for (const r of results) expect(neighbourIds.has(r.entity.id)).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('an anchor without neighbours answers [] before the filter is asked', async () => {
|
||||||
|
const index = (brain as any).metadataIndex
|
||||||
|
const within = vi.spyOn(index, 'filterIdsWithin')
|
||||||
|
try {
|
||||||
|
const results = await brain.find({
|
||||||
|
connected: { from: 'noise-0', direction: 'out' },
|
||||||
|
where: { kind: 'note' },
|
||||||
|
limit: 10
|
||||||
|
})
|
||||||
|
expect(results).toEqual([])
|
||||||
|
expect(within).not.toHaveBeenCalled()
|
||||||
|
} finally {
|
||||||
|
within.mockRestore()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
265
tests/integration/find-fields-projection.test.ts
Normal file
265
tests/integration/find-fields-projection.test.ts
Normal file
|
|
@ -0,0 +1,265 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/find-fields-projection
|
||||||
|
* @description **Field projection** — `find/get({ fields })` returns only the
|
||||||
|
* named fields, and serves them from the index when it can.
|
||||||
|
*
|
||||||
|
* A list view that shows a title and a slug does not need the document body,
|
||||||
|
* yet without a projection every row hydrates its whole record and discards
|
||||||
|
* almost all of it. These pins hold the two halves of the fix:
|
||||||
|
*
|
||||||
|
* **The answer.** A projected row is a SUBSET of the full row — for every
|
||||||
|
* requested field, the projected value equals the value the same query returns
|
||||||
|
* unprojected. Absent `fields` is byte-identical to today. A requested field the
|
||||||
|
* entity does not carry is simply absent, never an error. `system.*` resolves to
|
||||||
|
* the engine scalar, a bare name to the user's metadata.
|
||||||
|
*
|
||||||
|
* **The cost.** When every requested field is index-served, the canonical
|
||||||
|
* record is never opened — asserted by counting reads, not by timing them, so
|
||||||
|
* it cannot flake into a false green. When one requested field is NOT
|
||||||
|
* index-served (a body field, or a bucketed timestamp), exactly the owing rows
|
||||||
|
* are read and the rest are still served from the index.
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'
|
||||||
|
import { Brainy } from '../../src/brainy'
|
||||||
|
import { NounType } from '../../src/types/graphTypes'
|
||||||
|
import { generateTestVector } from '../helpers/test-factory'
|
||||||
|
|
||||||
|
/** Rows carrying a title, a slug, and a large body nobody wants in a list. */
|
||||||
|
const ROWS = 12
|
||||||
|
const BODY = 'x'.repeat(4096)
|
||||||
|
|
||||||
|
describe('find/get({ fields }) — projection', () => {
|
||||||
|
let brain: Brainy<any>
|
||||||
|
const ids: string[] = []
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||||
|
await brain.init()
|
||||||
|
for (let i = 0; i < ROWS; i++) {
|
||||||
|
ids.push(
|
||||||
|
await brain.add({
|
||||||
|
id: `post-${i}`,
|
||||||
|
data: `post ${i}`,
|
||||||
|
type: NounType.Thing,
|
||||||
|
metadata: {
|
||||||
|
kind: 'post',
|
||||||
|
title: `Title ${i}`,
|
||||||
|
slug: `slug-${i}`,
|
||||||
|
rank: i,
|
||||||
|
body: BODY,
|
||||||
|
// Only some rows carry this, so "missing is absent" is exercised
|
||||||
|
// by real data rather than by a name nothing ever had.
|
||||||
|
...(i % 2 === 0 ? { featured: true } : {})
|
||||||
|
},
|
||||||
|
vector: generateTestVector()
|
||||||
|
})
|
||||||
|
)
|
||||||
|
}
|
||||||
|
// Persist so the column store holds the values a projection reads from.
|
||||||
|
await brain.flush()
|
||||||
|
})
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
|
||||||
|
/** Count canonical record reads for one call. */
|
||||||
|
const countingReads = async <R>(body: () => Promise<R>): Promise<{ out: R; reads: number }> => {
|
||||||
|
const spy = vi.spyOn(brain as any, 'batchGet')
|
||||||
|
try {
|
||||||
|
const out = await body()
|
||||||
|
const reads = spy.mock.calls.reduce(
|
||||||
|
(n, call) => n + ((call[0] as string[] | undefined)?.length ?? 0),
|
||||||
|
0
|
||||||
|
)
|
||||||
|
return { out, reads }
|
||||||
|
} finally {
|
||||||
|
spy.mockRestore()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
it('absent fields is byte-identical to today', async () => {
|
||||||
|
const params = { where: { kind: 'post' }, limit: 5 }
|
||||||
|
const a = await brain.find({ ...params })
|
||||||
|
const b = await brain.find({ ...params, fields: undefined })
|
||||||
|
expect(JSON.stringify(b)).toBe(JSON.stringify(a))
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a projected row is a SUBSET of the full row, field for field', async () => {
|
||||||
|
const shapes: Array<Record<string, unknown>> = [
|
||||||
|
{ where: { kind: 'post' }, limit: 6 },
|
||||||
|
{ where: { kind: 'post' }, limit: 6, offset: 3 },
|
||||||
|
{ where: { kind: 'post' }, orderBy: 'rank', order: 'asc', limit: 6 },
|
||||||
|
{ where: { kind: 'post' }, orderBy: 'rank', order: 'desc', limit: 4 }
|
||||||
|
]
|
||||||
|
for (const shape of shapes) {
|
||||||
|
const full = await brain.find(shape as never)
|
||||||
|
const projected = await brain.find({ ...shape, fields: ['title', 'slug'] } as never)
|
||||||
|
expect(projected.map((r) => r.id), JSON.stringify(shape)).toEqual(full.map((r) => r.id))
|
||||||
|
for (let i = 0; i < full.length; i++) {
|
||||||
|
const fullMeta = (full[i].entity.metadata ?? {}) as Record<string, unknown>
|
||||||
|
const projMeta = (projected[i].entity.metadata ?? {}) as Record<string, unknown>
|
||||||
|
expect(projMeta.title, `${JSON.stringify(shape)} row ${i}`).toEqual(fullMeta.title)
|
||||||
|
expect(projMeta.slug).toEqual(fullMeta.slug)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('returns ONLY the named fields — the body never rides along', async () => {
|
||||||
|
const rows = await brain.find({ where: { kind: 'post' }, fields: ['title'], limit: 4 })
|
||||||
|
expect(rows).toHaveLength(4)
|
||||||
|
for (const r of rows) {
|
||||||
|
const meta = (r.entity.metadata ?? {}) as Record<string, unknown>
|
||||||
|
expect(Object.keys(meta)).toEqual(['title'])
|
||||||
|
expect(meta.body).toBeUndefined()
|
||||||
|
// Identity always survives a projection: a row you cannot identify is
|
||||||
|
// not a row.
|
||||||
|
expect(typeof r.id).toBe('string')
|
||||||
|
expect(r.entity.id).toBe(r.id)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a missing field is simply ABSENT — never an error', async () => {
|
||||||
|
// `featured` exists on half the rows; `no-such-field` on none. Neither
|
||||||
|
// throws, and neither appears as an explicit undefined.
|
||||||
|
const rows = await brain.find({
|
||||||
|
where: { kind: 'post' },
|
||||||
|
fields: ['title', 'featured', 'no-such-field'],
|
||||||
|
limit: ROWS
|
||||||
|
})
|
||||||
|
expect(rows.length).toBeGreaterThan(0)
|
||||||
|
let withFeatured = 0
|
||||||
|
for (const r of rows) {
|
||||||
|
const meta = (r.entity.metadata ?? {}) as Record<string, unknown>
|
||||||
|
expect('no-such-field' in meta).toBe(false)
|
||||||
|
if ('featured' in meta) withFeatured += 1
|
||||||
|
}
|
||||||
|
// Real data, not a name nothing ever had: some rows carry it, some do not.
|
||||||
|
expect(withFeatured).toBeGreaterThan(0)
|
||||||
|
expect(withFeatured).toBeLessThan(rows.length)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a strict address resolver is NOT on this path', async () => {
|
||||||
|
// orderBy throws UnresolvableFieldError for an unknown user key, because a
|
||||||
|
// typo there silently changes the order. A projection must not inherit that
|
||||||
|
// strictness: the honest answer to "give me this if you have it" is silence.
|
||||||
|
await expect(
|
||||||
|
brain.find({ where: { kind: 'post' }, fields: ['definitely-not-a-field'], limit: 2 })
|
||||||
|
).resolves.toBeInstanceOf(Array)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('system.* resolves to the engine scalar, a bare name to user metadata', async () => {
|
||||||
|
const full = await brain.find({ where: { kind: 'post' }, limit: 3 })
|
||||||
|
const rows = await brain.find({
|
||||||
|
where: { kind: 'post' },
|
||||||
|
fields: ['system.createdAt', 'title'],
|
||||||
|
limit: 3
|
||||||
|
})
|
||||||
|
for (let i = 0; i < rows.length; i++) {
|
||||||
|
expect((rows[i].entity as any).createdAt).toEqual((full[i].entity as any).createdAt)
|
||||||
|
const meta = (rows[i].entity.metadata ?? {}) as Record<string, unknown>
|
||||||
|
expect(meta.title).toEqual((full[i].entity.metadata as any).title)
|
||||||
|
// The engine scalar lands at the top level, not in the metadata bag —
|
||||||
|
// the two address spaces never shadow each other.
|
||||||
|
expect('system.createdAt' in meta).toBe(false)
|
||||||
|
expect('createdAt' in meta).toBe(false)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('reads NO canonical record when every requested field is index-served', async () => {
|
||||||
|
// The cost pin, counted rather than timed. `title` and `slug` are ordinary
|
||||||
|
// indexed user fields, so the index can serve them exactly.
|
||||||
|
const { out, reads } = await countingReads(() =>
|
||||||
|
brain.find({ where: { kind: 'post' }, fields: ['title', 'slug'], limit: ROWS })
|
||||||
|
)
|
||||||
|
expect(out.length).toBeGreaterThan(0)
|
||||||
|
expect(reads).toBe(0)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('reads records only for the fields the column cannot serve', async () => {
|
||||||
|
// `system.data` is NOT a column the store holds (verified against
|
||||||
|
// getIndexedFields), so the record must be opened for it — while `title`,
|
||||||
|
// which the column does hold, still comes from the index.
|
||||||
|
const { out, reads } = await countingReads(() =>
|
||||||
|
brain.find({ where: { kind: 'post' }, fields: ['title', 'system.data'], limit: 4 })
|
||||||
|
)
|
||||||
|
expect(out).toHaveLength(4)
|
||||||
|
expect(reads).toBe(4)
|
||||||
|
for (const r of out) {
|
||||||
|
const meta = (r.entity.metadata ?? {}) as Record<string, unknown>
|
||||||
|
expect(Object.keys(meta)).toEqual(['title'])
|
||||||
|
expect(typeof (r.entity as any).data).toBe('string')
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a large field the column DOES hold costs no record read', async () => {
|
||||||
|
// Worth pinning because it is the venue case: the body is column-served on
|
||||||
|
// this engine, so a list that projects around it pays nothing for it, and
|
||||||
|
// a list that projects it still pays no record read.
|
||||||
|
const { reads } = await countingReads(() =>
|
||||||
|
brain.find({ where: { kind: 'post' }, fields: ['body'], limit: 4 })
|
||||||
|
)
|
||||||
|
expect(reads).toBe(0)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('projects a vector-leg find too — the ANSWER is uniform, only the cost is not', async () => {
|
||||||
|
// The seam hydrates the metadata and graph page paths. A vector or text leg
|
||||||
|
// builds its own entities, so those rows are trimmed after the integrity
|
||||||
|
// guard instead. That difference is a COST difference, and this pin exists
|
||||||
|
// so it can never quietly become an ANSWER difference.
|
||||||
|
const rows = await brain.find({ query: 'post', fields: ['title'], limit: 3 })
|
||||||
|
for (const r of rows) {
|
||||||
|
const meta = (r.entity.metadata ?? {}) as Record<string, unknown>
|
||||||
|
expect(Object.keys(meta)).toEqual(['title'])
|
||||||
|
expect(meta.body).toBeUndefined()
|
||||||
|
expect(r.entity.id).toBe(r.id)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('get({ fields }) projects a single row through the same seam', async () => {
|
||||||
|
const full = await brain.get(ids[0])
|
||||||
|
const projected = await brain.get(ids[0], { fields: ['title', 'slug'] })
|
||||||
|
expect(projected).not.toBeNull()
|
||||||
|
expect(projected!.id).toBe(full!.id)
|
||||||
|
const fullMeta = (full!.metadata ?? {}) as Record<string, unknown>
|
||||||
|
const projMeta = (projected!.metadata ?? {}) as Record<string, unknown>
|
||||||
|
expect(projMeta.title).toEqual(fullMeta.title)
|
||||||
|
expect(projMeta.slug).toEqual(fullMeta.slug)
|
||||||
|
expect(Object.keys(projMeta).sort()).toEqual(['slug', 'title'])
|
||||||
|
expect((projected as any).body).toBeUndefined()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('get({ fields }) reads no record when the index serves the fields', async () => {
|
||||||
|
const { reads } = await countingReads(() => brain.get(ids[1], { fields: ['title'] }))
|
||||||
|
expect(reads).toBe(0)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('the door serves EXACT values — the column, never the bucketed index', async () => {
|
||||||
|
// The sparse index buckets `system.createdAt` to the minute for range
|
||||||
|
// queries; the column store keeps raw ms. Serving a projection from the
|
||||||
|
// former would hand back a value that differs from the record's, so the
|
||||||
|
// door reads the column — and this pin is what proves which one it read.
|
||||||
|
const index = (brain as any).metadataIndex
|
||||||
|
const sample = ids.slice(0, 3)
|
||||||
|
const served = await index.getScalarsForIds(sample, ['title', 'system.createdAt'])
|
||||||
|
expect(served.size).toBe(sample.length)
|
||||||
|
for (const id of sample) {
|
||||||
|
const row = served.get(id)!
|
||||||
|
const record = await brain.get(id)
|
||||||
|
expect(row.title).toEqual((record!.metadata as any).title)
|
||||||
|
// Exact to the millisecond — a bucketed value would be rounded down to
|
||||||
|
// the minute and this would fail.
|
||||||
|
expect(row['system.createdAt']).toEqual((record as any).createdAt)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a field the column store does not hold is OMITTED, not approximated', async () => {
|
||||||
|
const index = (brain as any).metadataIndex
|
||||||
|
const served = await index.getScalarsForIds(ids.slice(0, 2), ['title', 'system.data'])
|
||||||
|
for (const [, row] of served) {
|
||||||
|
expect('title' in row).toBe(true)
|
||||||
|
// Omission is what makes the caller read the record for it.
|
||||||
|
expect('system.data' in row).toBe(false)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
643
tests/integration/find-hybrid-filter-before-hydrate.test.ts
Normal file
643
tests/integration/find-hybrid-filter-before-hydrate.test.ts
Normal file
|
|
@ -0,0 +1,643 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/find-hybrid-filter-before-hydrate
|
||||||
|
* @description FILTER BEFORE HYDRATE, applied to the hybrid `find({ query })` path.
|
||||||
|
*
|
||||||
|
* A hybrid find fuses two legs. The semantic leg already walked only the
|
||||||
|
* metadata filter's universe (`candidateIds` / `allowedIds`). The TEXT leg did
|
||||||
|
* not: it ranked the WHOLE store, took the top `limit * 4`, read every one of
|
||||||
|
* those rows from canonical, and only then intersected with the filter — so a
|
||||||
|
* filtered hybrid find on a large store read hundreds of rows to return a
|
||||||
|
* handful of them, and a matching row outside the store-wide text prefix was
|
||||||
|
* silently dropped. That is the same defect `find({ connected })` carried
|
||||||
|
* before the graph-first law, one leg over.
|
||||||
|
*
|
||||||
|
* Both halves are pinned here.
|
||||||
|
*
|
||||||
|
* THE ANSWER. Where the filter did not truncate the text leg — the universe
|
||||||
|
* covers every text match, so both orders rank the same rows — the new
|
||||||
|
* pipeline's answer is IDENTICAL to the old one's: same rows, same order, same
|
||||||
|
* scores, same match visibility, same row shape. The oracle below is the
|
||||||
|
* pre-change pipeline itself, replayed on the same brain through the same
|
||||||
|
* doors, so the comparison is against what actually ran, not a remembered
|
||||||
|
* expectation.
|
||||||
|
*
|
||||||
|
* THE CORRECTION. Where the filter DID truncate it — the query's words are
|
||||||
|
* common outside the universe — the old order let the text leg contribute
|
||||||
|
* nothing at all: every row it ranked was discarded by the filter, and the
|
||||||
|
* answer came from the semantic leg alone. The new order ranks inside the
|
||||||
|
* universe, so the text leg contributes the rows it always should have.
|
||||||
|
*
|
||||||
|
* THE COST. Canonical is read for exactly the page: one batch, `limit` rows,
|
||||||
|
* never the legs. And the text leg is asked about the universe's ids only —
|
||||||
|
* what it marshals is bounded by the universe, not by the store.
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'
|
||||||
|
import { Brainy } from '../../src/brainy'
|
||||||
|
import { NounType, VerbType } from '../../src/types/graphTypes'
|
||||||
|
import { rankIndicesByScore, reorderByIndices } from '../../src/utils/resultRanking'
|
||||||
|
import { resolveEntityId } from '../../src/utils/idNormalization'
|
||||||
|
|
||||||
|
/** Embedding width of the default model — the row vectors must match it. */
|
||||||
|
const DIM = 384
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A deterministic, per-row-distinct unit vector. Distinct so the semantic leg
|
||||||
|
* has a real ranking to produce (identical vectors would make its order a tie
|
||||||
|
* break), deterministic so the oracle and the pipeline see the same one.
|
||||||
|
*/
|
||||||
|
function seededVector(seed: number): number[] {
|
||||||
|
const v = new Array<number>(DIM)
|
||||||
|
for (let i = 0; i < DIM; i++) {
|
||||||
|
v[i] = Math.sin((i + 1) * 0.11 + seed * 0.37) * 0.5 + Math.cos((i + 1) * 0.05 + seed * 0.13) * 0.3
|
||||||
|
}
|
||||||
|
const magnitude = Math.sqrt(v.reduce((sum, x) => sum + x * x, 0))
|
||||||
|
return v.map((x) => x / magnitude)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The fields a caller reads off a hybrid row — the whole comparable surface. */
|
||||||
|
function project(rows: any[]): any[] {
|
||||||
|
return rows.map((r) => ({
|
||||||
|
id: r.id,
|
||||||
|
score: r.score,
|
||||||
|
type: r.type,
|
||||||
|
metadata: r.metadata,
|
||||||
|
textMatches: r.textMatches,
|
||||||
|
textScore: r.textScore,
|
||||||
|
semanticScore: r.semanticScore,
|
||||||
|
matchSource: r.matchSource
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The PRE-CHANGE hybrid pipeline, replayed on a live brain through the same
|
||||||
|
* provider doors it used: whole-store text ranking with both legs hydrated in
|
||||||
|
* full, RRF fusion, then the metadata intersection, then the page.
|
||||||
|
*
|
||||||
|
* Supports the shapes these pins exercise (query + where/type/excludeVFS +
|
||||||
|
* connected + offset); `orderBy`, `fusion` and `near` are not replayed.
|
||||||
|
*/
|
||||||
|
async function legacyHybridFind(brain: any, params: any): Promise<any[]> {
|
||||||
|
const index = brain.metadataIndex
|
||||||
|
const limit = params.limit ?? 10
|
||||||
|
const offset = params.offset ?? 0
|
||||||
|
const hasFilter = Boolean(
|
||||||
|
params.where || params.type || params.subtype || params.service || params.excludeVFS
|
||||||
|
)
|
||||||
|
|
||||||
|
let preResolvedMetadataIds: string[] | null = null
|
||||||
|
let preResolvedFilter: any = null
|
||||||
|
let graphFirstIds: string[] | null = null
|
||||||
|
|
||||||
|
if (params.connected) {
|
||||||
|
// find() normalizes the anchors to canonical ids before this stage runs.
|
||||||
|
const anchored = {
|
||||||
|
...params,
|
||||||
|
connected: {
|
||||||
|
...params.connected,
|
||||||
|
...(params.connected.from && { from: resolveEntityId(params.connected.from) }),
|
||||||
|
...(params.connected.to && { to: resolveEntityId(params.connected.to) })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
graphFirstIds = await brain.resolveConnectedIds(anchored)
|
||||||
|
if (graphFirstIds!.length > 0 && hasFilter) {
|
||||||
|
preResolvedFilter = brain.buildMetadataFilter(params)
|
||||||
|
graphFirstIds = await brain.filterIdsWithinBelted(preResolvedFilter, graphFirstIds)
|
||||||
|
}
|
||||||
|
if (graphFirstIds!.length === 0) return []
|
||||||
|
preResolvedMetadataIds = graphFirstIds
|
||||||
|
} else if (hasFilter) {
|
||||||
|
preResolvedFilter = brain.buildMetadataFilter(params)
|
||||||
|
preResolvedMetadataIds = await brain.filterIdsBelted(preResolvedFilter)
|
||||||
|
if (preResolvedMetadataIds!.length === 0) return []
|
||||||
|
}
|
||||||
|
|
||||||
|
// Text leg — the whole store, then the top `limit * 4`, hydrated in full.
|
||||||
|
const allTextMatches = await index.getIdsForTextQuery(params.query)
|
||||||
|
const topMatches = allTextMatches.slice(0, limit * 2 * 2)
|
||||||
|
const maxMatches = topMatches[0]?.matchCount || 1
|
||||||
|
const textEntities = await brain.batchGet(topMatches.map((m: any) => m.id))
|
||||||
|
const textResults = topMatches
|
||||||
|
.filter((m: any) => textEntities.has(m.id))
|
||||||
|
.map((m: any) => ({ id: m.id, score: m.matchCount / maxMatches }))
|
||||||
|
|
||||||
|
// Semantic leg — the beam walk over the universe, hydrated in full.
|
||||||
|
const vector = await brain.embed(params.query)
|
||||||
|
const searchOptions = preResolvedMetadataIds ? { candidateIds: preResolvedMetadataIds } : undefined
|
||||||
|
const searchResults: [string, number][] = await brain.index.search(
|
||||||
|
vector,
|
||||||
|
limit * 2,
|
||||||
|
undefined,
|
||||||
|
searchOptions
|
||||||
|
)
|
||||||
|
const semanticEntities = await brain.batchGet(searchResults.map(([id]) => id))
|
||||||
|
const semanticResults = searchResults
|
||||||
|
.filter(([id]) => semanticEntities.has(id))
|
||||||
|
.map(([id, distance]) => ({ id, score: Math.max(0, Math.min(1, 1 / (1 + distance))) }))
|
||||||
|
|
||||||
|
// RRF fusion, with the match visibility the rows carried.
|
||||||
|
const alpha = params.hybridAlpha ?? brain.autoAlpha(params.query)
|
||||||
|
const k = 60
|
||||||
|
const matchData = new Map<string, any>()
|
||||||
|
const textWeight = 1 - alpha
|
||||||
|
textResults.forEach((r: any, rank: number) => {
|
||||||
|
const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false }
|
||||||
|
existing.rrf += textWeight * (1 / (k + rank + 1))
|
||||||
|
existing.textScore = r.score
|
||||||
|
existing.hasText = true
|
||||||
|
matchData.set(r.id, existing)
|
||||||
|
})
|
||||||
|
semanticResults.forEach((r: any, rank: number) => {
|
||||||
|
const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false }
|
||||||
|
existing.rrf += alpha * (1 / (k + rank + 1))
|
||||||
|
existing.semanticScore = r.score
|
||||||
|
existing.hasSemantic = true
|
||||||
|
matchData.set(r.id, existing)
|
||||||
|
})
|
||||||
|
|
||||||
|
const queryWords: string[] = index.tokenize(params.query)
|
||||||
|
const textResultIds = new Set(textResults.map((r: any) => r.id))
|
||||||
|
const fusedIds = Array.from(matchData.entries())
|
||||||
|
.sort((a, b) => b[1].rrf - a[1].rrf)
|
||||||
|
.map(([id, data]) => ({ id, data }))
|
||||||
|
|
||||||
|
const allEntities = await brain.batchGet(fusedIds.map((f) => f.id))
|
||||||
|
let rows: any[] = []
|
||||||
|
for (const { id, data } of fusedIds) {
|
||||||
|
const entity = allEntities.get(id)
|
||||||
|
if (!entity) continue
|
||||||
|
const textContent = textResultIds.has(id)
|
||||||
|
? index.extractTextContent({ data: entity.data, metadata: entity.metadata }).toLowerCase()
|
||||||
|
: null
|
||||||
|
rows.push({
|
||||||
|
id,
|
||||||
|
score: data.rrf,
|
||||||
|
type: entity.type,
|
||||||
|
metadata: entity.metadata,
|
||||||
|
textMatches:
|
||||||
|
textContent === null ? [] : queryWords.filter((w) => textContent.includes(w.toLowerCase())),
|
||||||
|
textScore: data.textScore,
|
||||||
|
semanticScore: data.semanticScore,
|
||||||
|
matchSource: data.hasText && data.hasSemantic ? 'both' : data.hasText ? 'text' : 'semantic'
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// The metadata intersection — after the legs, as it was.
|
||||||
|
if (preResolvedMetadataIds && preResolvedFilter) {
|
||||||
|
const filteredIdSet = new Set(preResolvedMetadataIds)
|
||||||
|
rows = rows.filter((r) => filteredIdSet.has(r.id))
|
||||||
|
}
|
||||||
|
if (graphFirstIds !== null) {
|
||||||
|
const neighbourSet = new Set(graphFirstIds)
|
||||||
|
rows = rows.filter((r) => neighbourSet.has(r.id))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rank to the page, then cut it.
|
||||||
|
const order = rankIndicesByScore(
|
||||||
|
rows.map((r) => r.score),
|
||||||
|
offset + limit,
|
||||||
|
true
|
||||||
|
)
|
||||||
|
return reorderByIndices(rows, order).slice(offset, offset + limit)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* FIXTURE A — the filter's universe covers every text match, so the two orders
|
||||||
|
* rank exactly the same rows and the answers must be identical.
|
||||||
|
*/
|
||||||
|
describe('hybrid find: filter before hydrate — the answer is unchanged', () => {
|
||||||
|
let brain: Brainy<any>
|
||||||
|
const QUERY = 'orbital telemetry'
|
||||||
|
const MATCHES = 24
|
||||||
|
const FILLER = 120
|
||||||
|
const OUTSIDE = 30
|
||||||
|
const VFS = 10
|
||||||
|
const RETRACTED = 6
|
||||||
|
const anchor = 'array-anchor'
|
||||||
|
const matchIds: string[] = []
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||||
|
await brain.init()
|
||||||
|
|
||||||
|
let seed = 1
|
||||||
|
await brain.add({
|
||||||
|
id: anchor,
|
||||||
|
data: 'ground station anchor record',
|
||||||
|
type: NounType.Thing,
|
||||||
|
metadata: { lane: 'alpha', role: 'anchor' },
|
||||||
|
vector: seededVector(seed++)
|
||||||
|
})
|
||||||
|
|
||||||
|
// Rows the query's words actually match — all inside every filter below.
|
||||||
|
for (let i = 0; i < MATCHES; i++) {
|
||||||
|
const id = `match-${i}`
|
||||||
|
await brain.add({
|
||||||
|
id,
|
||||||
|
data: `orbital telemetry packet ${i} recorded downlink`,
|
||||||
|
type: NounType.Document,
|
||||||
|
metadata: { lane: 'alpha', rank: i },
|
||||||
|
vector: seededVector(seed++)
|
||||||
|
})
|
||||||
|
matchIds.push(resolveEntityId(id))
|
||||||
|
await brain.relate({ from: anchor, to: id, type: VerbType.RelatedTo })
|
||||||
|
}
|
||||||
|
// Rows inside the universe that the query's words do NOT match.
|
||||||
|
for (let i = 0; i < FILLER; i++) {
|
||||||
|
await brain.add({
|
||||||
|
id: `filler-${i}`,
|
||||||
|
data: `cistern ledger entry ${i} archived`,
|
||||||
|
type: NounType.Document,
|
||||||
|
metadata: { lane: 'alpha', rank: 1000 + i },
|
||||||
|
vector: seededVector(seed++)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
// Rows outside the universe.
|
||||||
|
for (let i = 0; i < OUTSIDE; i++) {
|
||||||
|
await brain.add({
|
||||||
|
id: `outside-${i}`,
|
||||||
|
data: `unrelated dossier ${i}`,
|
||||||
|
type: NounType.Person,
|
||||||
|
metadata: { lane: 'beta' },
|
||||||
|
vector: seededVector(seed++)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
// VFS infrastructure rows — excluded by excludeVFS.
|
||||||
|
for (let i = 0; i < VFS; i++) {
|
||||||
|
await brain.add({
|
||||||
|
id: `vfs-${i}`,
|
||||||
|
data: `mounted path ${i}`,
|
||||||
|
type: NounType.Document,
|
||||||
|
metadata: { lane: 'alpha', vfsType: 'file' },
|
||||||
|
vector: seededVector(seed++)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
// Retracted rows — excluded by a `missing` negation.
|
||||||
|
for (let i = 0; i < RETRACTED; i++) {
|
||||||
|
await brain.add({
|
||||||
|
id: `retracted-${i}`,
|
||||||
|
data: `withdrawn note ${i}`,
|
||||||
|
type: NounType.Document,
|
||||||
|
metadata: { lane: 'alpha', retracted: true },
|
||||||
|
vector: seededVector(seed++)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// The reference index has no opaque-set door, so the pipeline and the
|
||||||
|
// oracle both restrict the beam walk with the materialized candidate ids.
|
||||||
|
expect(typeof (brain as any).metadataIndex.getIdSetForFilter).not.toBe('function')
|
||||||
|
})
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('the fixture does not truncate the text leg — the universe covers every text match', async () => {
|
||||||
|
const index = (brain as any).metadataIndex
|
||||||
|
const textMatches = await index.getIdsForTextQuery(QUERY)
|
||||||
|
expect(textMatches).toHaveLength(MATCHES)
|
||||||
|
const universe = await (brain as any).filterIdsBelted({ lane: 'alpha' })
|
||||||
|
const inUniverse = new Set(universe)
|
||||||
|
for (const m of textMatches) expect(inUniverse.has(m.id)).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('hybrid + where: identical rows, identical order, identical scores', async () => {
|
||||||
|
const params = { query: QUERY, where: { lane: 'alpha' }, limit: 8 }
|
||||||
|
const expected = await legacyHybridFind(brain as any, params)
|
||||||
|
const actual = await brain.find(params as any)
|
||||||
|
expect(actual.length).toBe(expected.length)
|
||||||
|
expect(project(actual)).toEqual(expected)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('hybrid + where + offset: identical page two', async () => {
|
||||||
|
const params = { query: QUERY, where: { lane: 'alpha' }, limit: 6, offset: 6 }
|
||||||
|
const expected = await legacyHybridFind(brain as any, params)
|
||||||
|
const actual = await brain.find(params as any)
|
||||||
|
expect(actual.length).toBe(expected.length)
|
||||||
|
expect(project(actual)).toEqual(expected)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('hybrid + type list + excludeVFS + a `missing` negation: identical', async () => {
|
||||||
|
const params = {
|
||||||
|
query: QUERY,
|
||||||
|
type: [NounType.Document, NounType.Person],
|
||||||
|
excludeVFS: true,
|
||||||
|
where: { lane: 'alpha', retracted: { missing: true } },
|
||||||
|
limit: 8
|
||||||
|
}
|
||||||
|
const expected = await legacyHybridFind(brain as any, params)
|
||||||
|
const actual = await brain.find(params as any)
|
||||||
|
expect(actual.length).toBe(expected.length)
|
||||||
|
expect(project(actual)).toEqual(expected)
|
||||||
|
for (const r of actual) {
|
||||||
|
expect(r.metadata.retracted).toBeUndefined()
|
||||||
|
expect(r.metadata.vfsType).toBeUndefined()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('hybrid + type list + excludeVFS + a `missing` negation, offset: identical', async () => {
|
||||||
|
const params = {
|
||||||
|
query: QUERY,
|
||||||
|
type: [NounType.Document, NounType.Person],
|
||||||
|
excludeVFS: true,
|
||||||
|
where: { lane: 'alpha', retracted: { missing: true } },
|
||||||
|
limit: 5,
|
||||||
|
offset: 5
|
||||||
|
}
|
||||||
|
const expected = await legacyHybridFind(brain as any, params)
|
||||||
|
const actual = await brain.find(params as any)
|
||||||
|
expect(actual.length).toBe(expected.length)
|
||||||
|
expect(project(actual)).toEqual(expected)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('hybrid + connected: identical, and never a non-neighbour', async () => {
|
||||||
|
const params = {
|
||||||
|
query: QUERY,
|
||||||
|
connected: { from: anchor, direction: 'out' as const },
|
||||||
|
where: { lane: 'alpha' },
|
||||||
|
limit: 8
|
||||||
|
}
|
||||||
|
const expected = await legacyHybridFind(brain as any, params)
|
||||||
|
const actual = await brain.find(params as any)
|
||||||
|
expect(actual.length).toBe(expected.length)
|
||||||
|
expect(project(actual)).toEqual(expected)
|
||||||
|
const neighbours = new Set(matchIds)
|
||||||
|
for (const r of actual) expect(neighbours.has(r.id)).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('hybrid + connected + offset: page two is the page, not an empty answer', async () => {
|
||||||
|
const params = {
|
||||||
|
query: QUERY,
|
||||||
|
connected: { from: anchor, direction: 'out' as const },
|
||||||
|
where: { lane: 'alpha' },
|
||||||
|
limit: 5,
|
||||||
|
offset: 5
|
||||||
|
}
|
||||||
|
const expected = await legacyHybridFind(brain as any, params)
|
||||||
|
expect(expected).toHaveLength(5)
|
||||||
|
const actual = await brain.find(params as any)
|
||||||
|
expect(actual.length).toBe(expected.length)
|
||||||
|
expect(project(actual)).toEqual(expected)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('hybrid + connected: paging reaches every matching neighbour exactly once', async () => {
|
||||||
|
const seen = new Set<string>()
|
||||||
|
for (let offset = 0; offset < MATCHES; offset += 6) {
|
||||||
|
const page = await brain.find({
|
||||||
|
query: QUERY,
|
||||||
|
connected: { from: anchor, direction: 'out' as const },
|
||||||
|
where: { lane: 'alpha' },
|
||||||
|
limit: 6,
|
||||||
|
offset
|
||||||
|
} as any)
|
||||||
|
for (const r of page) {
|
||||||
|
expect(seen.has(r.id)).toBe(false)
|
||||||
|
seen.add(r.id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Every row the fused candidate set holds is reachable by paging, and the
|
||||||
|
// neighbour set is the ceiling.
|
||||||
|
expect(seen.size).toBeGreaterThanOrEqual(MATCHES)
|
||||||
|
const neighbours = new Set(matchIds)
|
||||||
|
for (const id of seen) expect(neighbours.has(id)).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('hybrid + fusion + offset: page two is the page', async () => {
|
||||||
|
const plain = await brain.find({
|
||||||
|
query: QUERY,
|
||||||
|
where: { lane: 'alpha' },
|
||||||
|
limit: 5,
|
||||||
|
offset: 5
|
||||||
|
} as any)
|
||||||
|
const fused = await brain.find({
|
||||||
|
query: QUERY,
|
||||||
|
where: { lane: 'alpha' },
|
||||||
|
fusion: 'weighted',
|
||||||
|
limit: 5,
|
||||||
|
offset: 5
|
||||||
|
} as any)
|
||||||
|
expect(fused).toHaveLength(plain.length)
|
||||||
|
expect(fused.map((r) => r.id)).toEqual(plain.map((r) => r.id))
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a hydrated hybrid row is shaped exactly as an eagerly-built one', async () => {
|
||||||
|
const rows = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 8 } as any)
|
||||||
|
const row = rows[0]
|
||||||
|
expect(Object.keys(row)).toEqual([
|
||||||
|
'id',
|
||||||
|
'score',
|
||||||
|
'type',
|
||||||
|
'subtype',
|
||||||
|
'visibility',
|
||||||
|
'metadata',
|
||||||
|
'data',
|
||||||
|
'confidence',
|
||||||
|
'weight',
|
||||||
|
'_rev',
|
||||||
|
'entity',
|
||||||
|
'textMatches',
|
||||||
|
'textScore',
|
||||||
|
'semanticScore',
|
||||||
|
'matchSource'
|
||||||
|
])
|
||||||
|
// The flattened fields are projections of the entity, as always.
|
||||||
|
expect(row.entity).toBeDefined()
|
||||||
|
expect(row.type).toBe(row.entity.type)
|
||||||
|
expect(row.metadata).toBe(row.entity.metadata)
|
||||||
|
expect(row.data).toBe(row.entity.data)
|
||||||
|
expect(row._rev).toBe(row.entity._rev)
|
||||||
|
// The match visibility survives the deferral — every leg's fields, on the
|
||||||
|
// rows that leg contributed, exactly as the eager pipeline set them.
|
||||||
|
expect(['text', 'semantic', 'both']).toContain(row.matchSource)
|
||||||
|
for (const r of rows) {
|
||||||
|
if (r.matchSource === 'semantic') {
|
||||||
|
expect(r.textMatches).toEqual([])
|
||||||
|
expect(r.textScore).toBeUndefined()
|
||||||
|
} else {
|
||||||
|
expect(r.textMatches).toEqual(['orbital', 'telemetry'])
|
||||||
|
expect(typeof r.textScore).toBe('number')
|
||||||
|
}
|
||||||
|
if (r.matchSource === 'text') {
|
||||||
|
expect(r.semanticScore).toBeUndefined()
|
||||||
|
} else {
|
||||||
|
expect(typeof r.semanticScore).toBe('number')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('reads canonical for the page only — one batch, `limit` rows', async () => {
|
||||||
|
// Warm any first-read verification before the counters are read.
|
||||||
|
await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 1 } as any)
|
||||||
|
|
||||||
|
const hydrate = vi.spyOn(brain as any, 'batchGet')
|
||||||
|
try {
|
||||||
|
const results = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 10 } as any)
|
||||||
|
expect(results).toHaveLength(10)
|
||||||
|
expect(hydrate).toHaveBeenCalledTimes(1)
|
||||||
|
expect((hydrate.mock.calls[0][0] as string[]).length).toBe(10)
|
||||||
|
} finally {
|
||||||
|
hydrate.mockRestore()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('asks the text index about the universe only, never the whole store', async () => {
|
||||||
|
const index = (brain as any).metadataIndex
|
||||||
|
const wholeStore = vi.spyOn(index, 'getIdsForTextQuery')
|
||||||
|
const within = vi.spyOn(index, 'getIdsForTextQueryWithin')
|
||||||
|
try {
|
||||||
|
await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 10 } as any)
|
||||||
|
expect(wholeStore).not.toHaveBeenCalled()
|
||||||
|
expect(within).toHaveBeenCalledTimes(1)
|
||||||
|
|
||||||
|
const askedIds = within.mock.calls[0][1] as string[]
|
||||||
|
const universe = await (brain as any).filterIdsBelted({ lane: 'alpha' })
|
||||||
|
expect(askedIds).toHaveLength(universe.length)
|
||||||
|
|
||||||
|
// What the text leg marshals is bounded by the universe, not the store.
|
||||||
|
const marshalled = (await within.mock.results[0].value) as unknown[]
|
||||||
|
expect(marshalled.length).toBeLessThanOrEqual(universe.length)
|
||||||
|
expect(marshalled).toHaveLength(MATCHES)
|
||||||
|
} finally {
|
||||||
|
wholeStore.mockRestore()
|
||||||
|
within.mockRestore()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('the two text doors agree: within is the whole-store answer restricted', async () => {
|
||||||
|
const index = (brain as any).metadataIndex
|
||||||
|
const universe: string[] = await (brain as any).filterIdsBelted({
|
||||||
|
lane: 'alpha',
|
||||||
|
retracted: { missing: true }
|
||||||
|
})
|
||||||
|
const inUniverse = new Set(universe)
|
||||||
|
const whole = await index.getIdsForTextQuery(QUERY)
|
||||||
|
const within = await index.getIdsForTextQueryWithin(QUERY, universe)
|
||||||
|
expect(within).toEqual(whole.filter((m: any) => inUniverse.has(m.id)))
|
||||||
|
expect(await index.getIdsForTextQueryWithin(QUERY, [])).toEqual([])
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
/**
|
||||||
|
* FIXTURE B — the query's words are common OUTSIDE the universe, so the old
|
||||||
|
* order's text leg was entirely consumed by rows the filter then discarded.
|
||||||
|
* This is the corrected behaviour, held by name.
|
||||||
|
*/
|
||||||
|
describe('hybrid find: the text leg ranks inside the filter, not around it', () => {
|
||||||
|
let brain: Brainy<any>
|
||||||
|
const QUERY = 'orbital telemetry drift'
|
||||||
|
const NOISE = 150
|
||||||
|
const KEEP = 15
|
||||||
|
const keepIds: string[] = []
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||||
|
await brain.init()
|
||||||
|
|
||||||
|
let seed = 5000
|
||||||
|
// Added FIRST and matching one more query word, so they lead the
|
||||||
|
// store-wide text ranking outright — and none of them pass the filter.
|
||||||
|
for (let i = 0; i < NOISE; i++) {
|
||||||
|
await brain.add({
|
||||||
|
id: `noise-${i}`,
|
||||||
|
data: `orbital telemetry drift report ${i}`,
|
||||||
|
type: NounType.Document,
|
||||||
|
metadata: { lane: 'beta' },
|
||||||
|
vector: seededVector(seed++)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
for (let i = 0; i < KEEP; i++) {
|
||||||
|
const id = `keep-${i}`
|
||||||
|
await brain.add({
|
||||||
|
id,
|
||||||
|
data: `orbital telemetry summary ${i}`,
|
||||||
|
type: NounType.Document,
|
||||||
|
metadata: { lane: 'alpha' },
|
||||||
|
vector: seededVector(seed++)
|
||||||
|
})
|
||||||
|
keepIds.push(resolveEntityId(id))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('the old order let the filter consume the whole text leg', async () => {
|
||||||
|
const index = (brain as any).metadataIndex
|
||||||
|
const universe: string[] = await (brain as any).filterIdsBelted({ lane: 'alpha' })
|
||||||
|
expect(universe).toHaveLength(KEEP)
|
||||||
|
const inUniverse = new Set(universe)
|
||||||
|
|
||||||
|
// The store-wide prefix the old text leg took (limit 10 → limit * 4).
|
||||||
|
const prefix = (await index.getIdsForTextQuery(QUERY)).slice(0, 40)
|
||||||
|
expect(prefix).toHaveLength(40)
|
||||||
|
expect(prefix.filter((m: any) => inUniverse.has(m.id))).toHaveLength(0)
|
||||||
|
|
||||||
|
// Every row the old text leg ranked was then discarded by the filter, so
|
||||||
|
// the old answer carried NO text contribution at all — fifteen rows that
|
||||||
|
// match the query's words exactly, and not one of them reached the page
|
||||||
|
// through the text leg. What the old order returned was whatever the
|
||||||
|
// semantic leg alone happened to reach.
|
||||||
|
const legacy = await legacyHybridFind(brain as any, {
|
||||||
|
query: QUERY,
|
||||||
|
where: { lane: 'alpha' },
|
||||||
|
limit: 10
|
||||||
|
})
|
||||||
|
for (const r of legacy) {
|
||||||
|
expect(r.matchSource).toBe('semantic')
|
||||||
|
expect(r.textScore).toBeUndefined()
|
||||||
|
expect(r.textMatches).toEqual([])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('the new order ranks the text leg inside the universe', async () => {
|
||||||
|
const results = await brain.find({
|
||||||
|
query: QUERY,
|
||||||
|
where: { lane: 'alpha' },
|
||||||
|
limit: 10
|
||||||
|
} as any)
|
||||||
|
|
||||||
|
expect(results).toHaveLength(10)
|
||||||
|
const keeps = new Set(keepIds)
|
||||||
|
for (const r of results) {
|
||||||
|
expect(keeps.has(r.id)).toBe(true)
|
||||||
|
expect(r.metadata.lane).toBe('alpha')
|
||||||
|
// The text leg is the contributor the old order threw away.
|
||||||
|
expect(['text', 'both']).toContain(r.matchSource)
|
||||||
|
expect(r.textScore).toBe(1)
|
||||||
|
expect(r.textMatches).toEqual(['orbital', 'telemetry'])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('paging reaches every matching row the old order could not see', async () => {
|
||||||
|
const seen = new Set<string>()
|
||||||
|
for (let offset = 0; offset < KEEP; offset += 5) {
|
||||||
|
const page = await brain.find({
|
||||||
|
query: QUERY,
|
||||||
|
where: { lane: 'alpha' },
|
||||||
|
limit: 5,
|
||||||
|
offset
|
||||||
|
} as any)
|
||||||
|
expect(page).toHaveLength(5)
|
||||||
|
for (const r of page) {
|
||||||
|
expect(seen.has(r.id)).toBe(false)
|
||||||
|
seen.add(r.id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
expect(seen.size).toBe(KEEP)
|
||||||
|
expect([...seen].sort()).toEqual([...keepIds].sort())
|
||||||
|
})
|
||||||
|
|
||||||
|
it('reads canonical for the page only, on the truncating shape too', async () => {
|
||||||
|
await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 1 } as any)
|
||||||
|
|
||||||
|
const hydrate = vi.spyOn(brain as any, 'batchGet')
|
||||||
|
try {
|
||||||
|
const results = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 10 } as any)
|
||||||
|
expect(results).toHaveLength(10)
|
||||||
|
expect(hydrate).toHaveBeenCalledTimes(1)
|
||||||
|
expect((hydrate.mock.calls[0][0] as string[]).length).toBe(10)
|
||||||
|
} finally {
|
||||||
|
hydrate.mockRestore()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
52
tests/integration/find-near.test.ts
Normal file
52
tests/integration/find-near.test.ts
Normal file
|
|
@ -0,0 +1,52 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/find-near
|
||||||
|
* @description find({ near }) searches around the anchor's OWN vector (10.4.10).
|
||||||
|
*
|
||||||
|
* The proximity search fetched its anchor without vectors and fed a
|
||||||
|
* zero-length vector to the index — every near() refused with a dimension
|
||||||
|
* mismatch, for every caller. Found by the Rust planner's first-contact pins
|
||||||
|
* (the planner declines `near`; the pin compared outcomes with and without
|
||||||
|
* it). Now the anchor is fetched with its vector, and an anchor without one
|
||||||
|
* refuses by name instead of failing inside the index.
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, beforeAll, afterAll } from 'vitest'
|
||||||
|
import { Brainy } from '../../src/brainy'
|
||||||
|
import { NounType } from '../../src/types/graphTypes'
|
||||||
|
import { v5 } from '../../src/universal/uuid'
|
||||||
|
import { generateTestVector } from '../helpers/test-factory'
|
||||||
|
|
||||||
|
describe('find({ near }) uses the anchor vector', () => {
|
||||||
|
let brain: Brainy<any>
|
||||||
|
const anchorVector = generateTestVector()
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||||
|
await brain.init()
|
||||||
|
await brain.add({ id: 'anchor', data: 'anchor row', type: NounType.Thing, vector: anchorVector })
|
||||||
|
// A twin with the identical vector and a far row.
|
||||||
|
await brain.add({ id: 'twin', data: 'twin row', type: NounType.Thing, vector: [...anchorVector] })
|
||||||
|
await brain.add({ id: 'far', data: 'far row', type: NounType.Thing, vector: generateTestVector() })
|
||||||
|
})
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('returns the anchor\'s neighbours by its own vector', async () => {
|
||||||
|
const results = await brain.find({ near: { id: 'anchor' }, limit: 3 })
|
||||||
|
expect(results.length).toBeGreaterThan(0)
|
||||||
|
const ids = results.map((r) => r.entity.id)
|
||||||
|
expect(ids).toContain(v5('twin'))
|
||||||
|
})
|
||||||
|
|
||||||
|
it('refuses by name when the anchor has no vector', async () => {
|
||||||
|
await brain.add({
|
||||||
|
id: 'unvectored',
|
||||||
|
data: 'no vector here',
|
||||||
|
type: NounType.Thing,
|
||||||
|
deferEmbedding: true
|
||||||
|
})
|
||||||
|
;(brain as any).kickEmbedWorker = () => {}
|
||||||
|
await expect(brain.find({ near: { id: 'unvectored' }, limit: 3 })).rejects.toThrow(/has no vector to search around/)
|
||||||
|
})
|
||||||
|
})
|
||||||
248
tests/integration/find-orderby-every-path.test.ts
Normal file
248
tests/integration/find-orderby-every-path.test.ts
Normal file
|
|
@ -0,0 +1,248 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/find-orderby-every-path
|
||||||
|
* @description `orderBy` IS THE ORDER — on every find() path, not just the
|
||||||
|
* metadata-only one.
|
||||||
|
*
|
||||||
|
* THE DEFECT. `find({ where, orderBy })` (metadata only) answered in field
|
||||||
|
* order. `find({ query, where, orderBy })` and `find({ vector, where, orderBy })`
|
||||||
|
* answered in SCORE order, silently: the vector/filter block ranked the fused
|
||||||
|
* candidates by score, cut the page, and returned early — the tail's `orderBy`
|
||||||
|
* sort sat below that early return and never ran. Nothing threw, nothing warned,
|
||||||
|
* and the two paths disagreed about what "ordered by rank" means. A caller
|
||||||
|
* paging `orderBy: 'rank', order: 'desc'` over a hybrid find got relevance
|
||||||
|
* order wearing an ordering request's clothes.
|
||||||
|
*
|
||||||
|
* Where `connected` or `fusion` kept the tail alive the defect changed shape
|
||||||
|
* rather than disappearing: the block had already CUT the page by score, so the
|
||||||
|
* tail ordered the rows relevance had chosen instead of the rows the ordering
|
||||||
|
* asks for — a correctly sorted page of the wrong rows.
|
||||||
|
*
|
||||||
|
* The early cut fires only once the candidate set reaches `offset + limit`
|
||||||
|
* rows, which is why small fixtures never saw it: below that threshold the
|
||||||
|
* block falls through and the tail's sort does apply. That is the whole shape
|
||||||
|
* of the bug — an ordering that is correct until there is enough data to matter.
|
||||||
|
*
|
||||||
|
* THE LAW. An explicit `orderBy` displaces score as the ordering key on every
|
||||||
|
* path. The candidate set the path produced is ordered IN FULL and the page is
|
||||||
|
* cut from that ordering — the graph-first law's "page last", applied to
|
||||||
|
* ordering rather than to filtering. Score-ranked early paging is for the
|
||||||
|
* default (no `orderBy`) case only, where score IS the requested order.
|
||||||
|
*
|
||||||
|
* THE PIN. Differential, against the metadata-only path — the one path that
|
||||||
|
* always honoured `orderBy`.
|
||||||
|
*
|
||||||
|
* WHAT THE DIFFERENTIAL CAN AND CANNOT CLAIM. `orderBy` orders the candidate
|
||||||
|
* set; it does not enlarge it. The hybrid legs are bounded by construction (the
|
||||||
|
* text leg and the beam walk each take `limit * 2`), so a differential against
|
||||||
|
* the metadata-only path — whose universe is every matching row — is only
|
||||||
|
* meaningful where those bounds provably cover the universe. The fixture is
|
||||||
|
* sized so they do (12 rows, `limit` 6 → a `limit * 2` = 12-row text leg), and
|
||||||
|
* the covering is ASSERTED from the leg's own output rather than assumed. This
|
||||||
|
* pin is about ordering, and it says nothing about recall.
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, beforeAll, afterAll } from 'vitest'
|
||||||
|
import { Brainy } from '../../src/brainy'
|
||||||
|
import { NounType, VerbType } from '../../src/types/graphTypes'
|
||||||
|
import { resolveEntityId } from '../../src/utils/idNormalization'
|
||||||
|
|
||||||
|
/** Embedding width of the default model — the row vectors must match it. */
|
||||||
|
const DIM = 384
|
||||||
|
|
||||||
|
/** A deterministic, per-row-distinct unit vector (no embedder in the fixture). */
|
||||||
|
function seededVector(seed: number): number[] {
|
||||||
|
const v = new Array<number>(DIM)
|
||||||
|
for (let i = 0; i < DIM; i++) {
|
||||||
|
v[i] = Math.sin((i + 1) * 0.11 + seed * 0.37) * 0.5 + Math.cos((i + 1) * 0.05 + seed * 0.13) * 0.3
|
||||||
|
}
|
||||||
|
const magnitude = Math.sqrt(v.reduce((sum, x) => sum + x * x, 0))
|
||||||
|
return v.map((x) => x / magnitude)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ranks, shuffled — so no scoring order can reproduce them by luck, and the
|
||||||
|
* ordering the pins assert is visibly not the insertion order either.
|
||||||
|
*/
|
||||||
|
const RANKS = [7, 3, 11, 1, 9, 5, 12, 2, 10, 4, 8, 6]
|
||||||
|
const ROWS = RANKS.length
|
||||||
|
/** The page size every pin uses: `limit * 2` covers the whole universe. */
|
||||||
|
const LIMIT = 6
|
||||||
|
/** The neighbour subset — the graph-first universe — and its own page size. */
|
||||||
|
const NEIGHBOURS = 8
|
||||||
|
const GRAPH_LIMIT = 4
|
||||||
|
|
||||||
|
describe('find(): orderBy is the order on every path', () => {
|
||||||
|
let brain: Brainy<any>
|
||||||
|
const QUERY = 'orbital telemetry'
|
||||||
|
const anchor = 'ordering-anchor'
|
||||||
|
const neighbourIds: string[] = []
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||||
|
await brain.init()
|
||||||
|
|
||||||
|
let seed = 1
|
||||||
|
await brain.add({
|
||||||
|
id: anchor,
|
||||||
|
data: 'ground station anchor record',
|
||||||
|
type: NounType.Thing,
|
||||||
|
metadata: { lane: 'anchor', rank: 0 },
|
||||||
|
vector: seededVector(seed++)
|
||||||
|
})
|
||||||
|
|
||||||
|
for (let i = 0; i < ROWS; i++) {
|
||||||
|
const id = `row-${i}`
|
||||||
|
await brain.add({
|
||||||
|
id,
|
||||||
|
// EVERY row carries both query words, so the text leg reaches all of
|
||||||
|
// them and the hybrid candidate set covers the whole universe.
|
||||||
|
data: `orbital telemetry packet ${i} recorded downlink`,
|
||||||
|
type: NounType.Document,
|
||||||
|
metadata: { lane: 'alpha', rank: RANKS[i] },
|
||||||
|
vector: seededVector(seed++)
|
||||||
|
})
|
||||||
|
if (i < NEIGHBOURS) {
|
||||||
|
await brain.relate({ from: anchor, to: id, type: VerbType.RelatedTo })
|
||||||
|
neighbourIds.push(resolveEntityId(id))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('the fixture: the hybrid candidate set covers the whole filter universe', async () => {
|
||||||
|
const universe: string[] = await (brain as any).filterIdsBelted({ lane: 'alpha' })
|
||||||
|
expect(universe).toHaveLength(ROWS)
|
||||||
|
|
||||||
|
// The text leg is bounded at `limit * 2`; the fixture is sized so that
|
||||||
|
// bound reaches every row in the universe. This is the precondition the
|
||||||
|
// differential below rests on — asserted from the leg itself.
|
||||||
|
const textScored = await (brain as any).executeTextSearchScored(QUERY, LIMIT * 2, universe)
|
||||||
|
expect(textScored).toHaveLength(ROWS)
|
||||||
|
|
||||||
|
// And the candidate set is large enough to trigger the score-ranked early
|
||||||
|
// cut this pin exists to keep out of an ordered query's way.
|
||||||
|
expect(ROWS).toBeGreaterThanOrEqual(LIMIT)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('metadata-only + orderBy: the reference ordering', async () => {
|
||||||
|
const rows = await brain.find({
|
||||||
|
where: { lane: 'alpha' },
|
||||||
|
orderBy: 'rank',
|
||||||
|
order: 'desc',
|
||||||
|
limit: LIMIT
|
||||||
|
} as any)
|
||||||
|
expect(rows.map((r: any) => r.metadata.rank)).toEqual([12, 11, 10, 9, 8, 7])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('hybrid (query + where) + orderBy: the same page as the metadata-only path', async () => {
|
||||||
|
const params = { where: { lane: 'alpha' }, orderBy: 'rank', order: 'desc' as const, limit: LIMIT }
|
||||||
|
const expected = await brain.find(params as any)
|
||||||
|
const actual = await brain.find({ ...params, query: QUERY } as any)
|
||||||
|
|
||||||
|
expect(actual).toHaveLength(expected.length)
|
||||||
|
expect(actual.map((r: any) => r.id)).toEqual(expected.map((r: any) => r.id))
|
||||||
|
expect(actual.map((r: any) => r.metadata.rank)).toEqual([12, 11, 10, 9, 8, 7])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('hybrid + orderBy asc: the ordering key is honoured in both directions', async () => {
|
||||||
|
const params = { where: { lane: 'alpha' }, orderBy: 'rank', order: 'asc' as const, limit: LIMIT }
|
||||||
|
const expected = await brain.find(params as any)
|
||||||
|
const actual = await brain.find({ ...params, query: QUERY } as any)
|
||||||
|
|
||||||
|
expect(actual.map((r: any) => r.id)).toEqual(expected.map((r: any) => r.id))
|
||||||
|
expect(actual.map((r: any) => r.metadata.rank)).toEqual([1, 2, 3, 4, 5, 6])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('hybrid + orderBy + offset: page two is page two of the ORDERING', async () => {
|
||||||
|
const params = {
|
||||||
|
where: { lane: 'alpha' },
|
||||||
|
orderBy: 'rank',
|
||||||
|
order: 'desc' as const,
|
||||||
|
limit: LIMIT,
|
||||||
|
offset: LIMIT
|
||||||
|
}
|
||||||
|
const expected = await brain.find(params as any)
|
||||||
|
const actual = await brain.find({ ...params, query: QUERY } as any)
|
||||||
|
|
||||||
|
expect(actual).toHaveLength(LIMIT)
|
||||||
|
expect(actual.map((r: any) => r.id)).toEqual(expected.map((r: any) => r.id))
|
||||||
|
expect(actual.map((r: any) => r.metadata.rank)).toEqual([6, 5, 4, 3, 2, 1])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('hybrid + orderBy: paging walks the ordering monotonically, no row twice', async () => {
|
||||||
|
const seen: number[] = []
|
||||||
|
for (let offset = 0; offset < ROWS; offset += LIMIT) {
|
||||||
|
const page = await brain.find({
|
||||||
|
query: QUERY,
|
||||||
|
where: { lane: 'alpha' },
|
||||||
|
orderBy: 'rank',
|
||||||
|
order: 'desc',
|
||||||
|
limit: LIMIT,
|
||||||
|
offset
|
||||||
|
} as any)
|
||||||
|
seen.push(...page.map((r: any) => r.metadata.rank))
|
||||||
|
}
|
||||||
|
expect(seen).toHaveLength(ROWS)
|
||||||
|
expect(new Set(seen).size).toBe(ROWS)
|
||||||
|
// Strictly descending across every page boundary.
|
||||||
|
for (let i = 1; i < seen.length; i++) expect(seen[i]).toBeLessThan(seen[i - 1])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('vector + where + orderBy: field order, not distance order', async () => {
|
||||||
|
// The beam walk takes `limit * 2` = the whole universe here, so the page is
|
||||||
|
// the true top of the ordering — which distance order cannot produce.
|
||||||
|
const rows = await brain.find({
|
||||||
|
vector: seededVector(1000),
|
||||||
|
where: { lane: 'alpha' },
|
||||||
|
orderBy: 'rank',
|
||||||
|
order: 'desc',
|
||||||
|
limit: LIMIT
|
||||||
|
} as any)
|
||||||
|
expect(rows.map((r: any) => r.metadata.rank)).toEqual([12, 11, 10, 9, 8, 7])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('graph-first (query + connected + where) + orderBy: the neighbour set, ordered', async () => {
|
||||||
|
const actual = await brain.find({
|
||||||
|
query: QUERY,
|
||||||
|
connected: { from: anchor, direction: 'out' as const },
|
||||||
|
where: { lane: 'alpha' },
|
||||||
|
orderBy: 'rank',
|
||||||
|
order: 'desc',
|
||||||
|
limit: GRAPH_LIMIT
|
||||||
|
} as any)
|
||||||
|
|
||||||
|
expect(actual).toHaveLength(GRAPH_LIMIT)
|
||||||
|
const neighbours = new Set(neighbourIds)
|
||||||
|
for (const r of actual) expect(neighbours.has(r.id)).toBe(true)
|
||||||
|
|
||||||
|
// The ordering covers the whole neighbour set, so the page holds the
|
||||||
|
// highest ranks AMONG THE NEIGHBOURS — not the ones the score ranking
|
||||||
|
// happened to surface first and the tail then sorted among themselves.
|
||||||
|
const expectedRanks = RANKS.slice(0, NEIGHBOURS)
|
||||||
|
.sort((a, b) => b - a)
|
||||||
|
.slice(0, GRAPH_LIMIT)
|
||||||
|
expect(expectedRanks).toEqual([12, 11, 9, 7])
|
||||||
|
expect(actual.map((r: any) => r.metadata.rank)).toEqual(expectedRanks)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('fusion + orderBy: the ordering survives the fusion rescore', async () => {
|
||||||
|
const actual = await brain.find({
|
||||||
|
query: QUERY,
|
||||||
|
where: { lane: 'alpha' },
|
||||||
|
fusion: 'weighted',
|
||||||
|
orderBy: 'rank',
|
||||||
|
order: 'desc',
|
||||||
|
limit: LIMIT
|
||||||
|
} as any)
|
||||||
|
expect(actual.map((r: any) => r.metadata.rank)).toEqual([12, 11, 10, 9, 8, 7])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('no orderBy: score order still stands (the default is untouched)', async () => {
|
||||||
|
const rows = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: LIMIT } as any)
|
||||||
|
expect(rows).toHaveLength(LIMIT)
|
||||||
|
const scores = rows.map((r: any) => r.score)
|
||||||
|
for (let i = 1; i < scores.length; i++) expect(scores[i]).toBeLessThanOrEqual(scores[i - 1])
|
||||||
|
})
|
||||||
|
})
|
||||||
141
tests/integration/find-planner-door.test.ts
Normal file
141
tests/integration/find-planner-door.test.ts
Normal file
|
|
@ -0,0 +1,141 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/find-planner-door
|
||||||
|
* @description The optional `MetadataIndexProvider.planFindPage` door.
|
||||||
|
*
|
||||||
|
* The stage doors each serve one stage, so a `find()` that consults three of
|
||||||
|
* them crosses into the index three times and marshals a result set at every
|
||||||
|
* crossing — a filter matching a hundred thousand rows builds a hundred
|
||||||
|
* thousand id strings to return a page of twenty-five. An index that can decide
|
||||||
|
* the stage order itself answers the page in one call.
|
||||||
|
*
|
||||||
|
* These pins hold the three properties that make such a door safe to add:
|
||||||
|
*
|
||||||
|
* 1. **Absent, nothing changes.** The reference index has no planner, and every
|
||||||
|
* find is served by the stage doors exactly as before. That is also what
|
||||||
|
* makes this engine the ordering oracle for any index that implements one.
|
||||||
|
* 2. **Present, it is asked first and its answer is used** — above the branch
|
||||||
|
* selection, with the params already normalized, the hidden ids passed, and
|
||||||
|
* the graph provider handed over.
|
||||||
|
* 3. **`null` is routing, not an answer.** A door that declines a shape leaves
|
||||||
|
* it to the path that always served it, and the result is unchanged.
|
||||||
|
*
|
||||||
|
* Plus the serving law: an empty page stamped `emptyAt: 'graph'` is re-verified
|
||||||
|
* against the adjacency before it is believed, so a not-serving graph refuses
|
||||||
|
* loudly instead of answering `[]` as truth.
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'
|
||||||
|
import { Brainy } from '../../src/brainy'
|
||||||
|
import { NounType, VerbType } from '../../src/types/graphTypes'
|
||||||
|
import { generateTestVector } from '../helpers/test-factory'
|
||||||
|
|
||||||
|
describe('find(): the optional planner door', () => {
|
||||||
|
let brain: Brainy<any>
|
||||||
|
const anchor = 'planner-anchor'
|
||||||
|
let neighbourId = ''
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||||
|
await brain.init()
|
||||||
|
await brain.add({
|
||||||
|
id: anchor,
|
||||||
|
data: 'anchor',
|
||||||
|
type: NounType.Person,
|
||||||
|
metadata: { kind: 'anchor' },
|
||||||
|
vector: generateTestVector()
|
||||||
|
})
|
||||||
|
for (let i = 0; i < 12; i++) {
|
||||||
|
const id = await brain.add({
|
||||||
|
id: `row-${i}`,
|
||||||
|
data: `row ${i}`,
|
||||||
|
type: NounType.Person,
|
||||||
|
metadata: { kind: 'note', rank: i },
|
||||||
|
vector: generateTestVector()
|
||||||
|
})
|
||||||
|
if (i === 0) neighbourId = id
|
||||||
|
await brain.relate({ from: anchor, to: id, type: VerbType.Knows })
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
|
||||||
|
/** Install a planner door for one call, then remove it. */
|
||||||
|
const withDoor = async <T>(
|
||||||
|
door: (...a: any[]) => Promise<any>,
|
||||||
|
body: () => Promise<T>
|
||||||
|
): Promise<T> => {
|
||||||
|
const index = (brain as any).metadataIndex
|
||||||
|
index.planFindPage = door
|
||||||
|
try {
|
||||||
|
return await body()
|
||||||
|
} finally {
|
||||||
|
delete index.planFindPage
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
it('is absent on the reference index — every find is served by the stage doors', async () => {
|
||||||
|
expect((brain as any).metadataIndex.planFindPage).toBeUndefined()
|
||||||
|
const results = await brain.find({ where: { kind: 'note' }, limit: 5 })
|
||||||
|
expect(results).toHaveLength(5)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('is asked before the branches, with normalized params and the graph provider', async () => {
|
||||||
|
const door = vi.fn(async () => null)
|
||||||
|
await withDoor(door, async () => {
|
||||||
|
await brain.find({ where: { kind: 'note' }, limit: 5 })
|
||||||
|
})
|
||||||
|
expect(door).toHaveBeenCalledTimes(1)
|
||||||
|
const [params, hidden, graph] = door.mock.calls[0] as any[]
|
||||||
|
expect(params.where).toEqual({ kind: 'note' })
|
||||||
|
expect(Array.isArray(hidden)).toBe(true)
|
||||||
|
expect(graph).toBe((brain as any).graphIndex)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('uses the page it answers, hydrated and in the door\'s order', async () => {
|
||||||
|
const results = await withDoor(
|
||||||
|
async () => ({ ids: [neighbourId], emptyAt: 'none' as const }),
|
||||||
|
async () => brain.find({ where: { kind: 'note' }, limit: 5 })
|
||||||
|
)
|
||||||
|
expect(results).toHaveLength(1)
|
||||||
|
expect(results[0].entity.id).toBe(neighbourId)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a declining door changes nothing — the shape is served as it always was', async () => {
|
||||||
|
const withoutDoor = await brain.find({ where: { kind: 'note' }, orderBy: 'rank', limit: 4 })
|
||||||
|
const declined = await withDoor(
|
||||||
|
async () => null,
|
||||||
|
async () => brain.find({ where: { kind: 'note' }, orderBy: 'rank', limit: 4 })
|
||||||
|
)
|
||||||
|
expect(declined.map((r) => r.entity.id)).toEqual(withoutDoor.map((r) => r.entity.id))
|
||||||
|
})
|
||||||
|
|
||||||
|
it('re-verifies the adjacency before believing an empty graph answer', async () => {
|
||||||
|
const verify = vi.spyOn(brain as any, 'verifyGraphAdjacencyLive')
|
||||||
|
try {
|
||||||
|
const results = await withDoor(
|
||||||
|
async () => ({ ids: [], emptyAt: 'graph' as const }),
|
||||||
|
async () => brain.find({ connected: { from: anchor }, where: { kind: 'note' }, limit: 5 })
|
||||||
|
)
|
||||||
|
expect(results).toEqual([])
|
||||||
|
expect(verify).toHaveBeenCalled()
|
||||||
|
} finally {
|
||||||
|
verify.mockRestore()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('does not re-verify the adjacency for an empty the FILTER produced', async () => {
|
||||||
|
const verify = vi.spyOn(brain as any, 'verifyGraphAdjacencyLive')
|
||||||
|
verify.mockClear()
|
||||||
|
try {
|
||||||
|
const results = await withDoor(
|
||||||
|
async () => ({ ids: [], emptyAt: 'filter' as const }),
|
||||||
|
async () => brain.find({ where: { kind: 'note' }, limit: 5 })
|
||||||
|
)
|
||||||
|
expect(results).toEqual([])
|
||||||
|
expect(verify).not.toHaveBeenCalled()
|
||||||
|
} finally {
|
||||||
|
verify.mockRestore()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
@ -48,6 +48,7 @@ describe('Unified Find() Integration Tests', () => {
|
||||||
|
|
||||||
afterAll(async () => {
|
afterAll(async () => {
|
||||||
await cleanup.cleanup()
|
await cleanup.cleanup()
|
||||||
|
await brain.close()
|
||||||
brain = null as any
|
brain = null as any
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
|
||||||
101
tests/integration/generation-store-factory.test.ts
Normal file
101
tests/integration/generation-store-factory.test.ts
Normal file
|
|
@ -0,0 +1,101 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/generation-store-factory
|
||||||
|
* @description Pins the `createGenerationStore` protected factory hook on
|
||||||
|
* `Brainy` ({@link Brainy.createGenerationStore}). The hook exists so an
|
||||||
|
* engine built on top of this reference implementation can substitute a
|
||||||
|
* `GenerationStore` that keeps the same behavioural contract; this suite
|
||||||
|
* proves two things:
|
||||||
|
*
|
||||||
|
* 1. A subclass overriding the hook is the ONLY path that constructs the
|
||||||
|
* generation store — it is called exactly once, with the same storage
|
||||||
|
* instance `performInit` holds — and the store the brain actually uses
|
||||||
|
* is the one the override returned.
|
||||||
|
* 2. The default (non-overridden) path is unaffected — proven here by
|
||||||
|
* confirming the base class still produces a plain `GenerationStore`
|
||||||
|
* wired to `brain.storage`, and separately by running the existing
|
||||||
|
* `db-mvcc` and `brainy-core.integration` suites unmodified against this
|
||||||
|
* change (they exercise generation-store behaviour end to end).
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, afterEach } from 'vitest'
|
||||||
|
import { Brainy } from '../../src/brainy.js'
|
||||||
|
import { GenerationStore } from '../../src/db/generationStore.js'
|
||||||
|
import type { BaseStorage } from '../../src/storage/baseStorage.js'
|
||||||
|
|
||||||
|
/** Typed access to the brain's private storage + generation-store fields (test injection point). */
|
||||||
|
function internalsOf(brain: Brainy): { storage: BaseStorage; generationStore: GenerationStore } {
|
||||||
|
return brain as unknown as { storage: BaseStorage; generationStore: GenerationStore }
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A `GenerationStore` subclass that counts its own construction and
|
||||||
|
* remembers the storage instance it was built with, so the test can prove
|
||||||
|
* the hook is the sole construction path without mocking the module.
|
||||||
|
*/
|
||||||
|
class SpyGenerationStore extends GenerationStore {
|
||||||
|
static constructCount = 0
|
||||||
|
static lastStorage: BaseStorage | undefined
|
||||||
|
|
||||||
|
constructor(storage: BaseStorage) {
|
||||||
|
super(storage)
|
||||||
|
SpyGenerationStore.constructCount++
|
||||||
|
SpyGenerationStore.lastStorage = storage
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A Brainy subclass overriding the factory hook — stands in for an engine built on the reference. */
|
||||||
|
class BrainyWithSpyStore extends Brainy {
|
||||||
|
hookCallCount = 0
|
||||||
|
hookStorageArg: BaseStorage | undefined
|
||||||
|
|
||||||
|
protected override createGenerationStore(storage: BaseStorage): GenerationStore {
|
||||||
|
this.hookCallCount++
|
||||||
|
this.hookStorageArg = storage
|
||||||
|
return new SpyGenerationStore(storage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('Brainy.createGenerationStore — protected factory hook', () => {
|
||||||
|
const brains: Brainy[] = []
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
SpyGenerationStore.constructCount = 0
|
||||||
|
SpyGenerationStore.lastStorage = undefined
|
||||||
|
for (const brain of brains.splice(0)) {
|
||||||
|
try {
|
||||||
|
await brain.close()
|
||||||
|
} catch {
|
||||||
|
// already closed by the test
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a subclass override is the sole construction path: called once, same storage instance, its store is the one the brain uses', async () => {
|
||||||
|
const brain = new BrainyWithSpyStore({ requireSubtype: false, storage: { type: 'memory' } })
|
||||||
|
await brain.init()
|
||||||
|
brains.push(brain)
|
||||||
|
|
||||||
|
// Called exactly once, through the hook.
|
||||||
|
expect(brain.hookCallCount).toBe(1)
|
||||||
|
expect(SpyGenerationStore.constructCount).toBe(1)
|
||||||
|
|
||||||
|
// Same storage instance the base class holds — not a copy, not a different adapter.
|
||||||
|
const { storage, generationStore } = internalsOf(brain)
|
||||||
|
expect(brain.hookStorageArg).toBe(storage)
|
||||||
|
expect(SpyGenerationStore.lastStorage).toBe(storage)
|
||||||
|
|
||||||
|
// The store the brain actually uses is the one the override returned.
|
||||||
|
expect(generationStore).toBeInstanceOf(SpyGenerationStore)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('the default (non-overridden) path still produces a plain GenerationStore wired to the same storage', async () => {
|
||||||
|
const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||||
|
await brain.init()
|
||||||
|
brains.push(brain)
|
||||||
|
|
||||||
|
const { storage, generationStore } = internalsOf(brain)
|
||||||
|
expect(generationStore).toBeInstanceOf(GenerationStore)
|
||||||
|
// The default implementation constructs from the same storage the brain holds.
|
||||||
|
expect((generationStore as unknown as { storage: BaseStorage }).storage).toBe(storage)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
@ -9,9 +9,34 @@
|
||||||
* 8.0 BigInt boundary: entity ints in (resolved via the metadata index's
|
* 8.0 BigInt boundary: entity ints in (resolved via the metadata index's
|
||||||
* idMapper), entity/verb ints out (`bigint[]`). Entity ints map back to UUIDs
|
* idMapper), entity/verb ints out (`bigint[]`). Entity ints map back to UUIDs
|
||||||
* via `idMapper.getUuid(Number(int))`; verb ints via `verbIntsToIds()`.
|
* via `idMapper.getUuid(Number(int))`; verb ints via `verbIntsToIds()`.
|
||||||
|
*
|
||||||
|
* COST NOTE (2026-09): this file's `beforeEach` used to recreate a fresh
|
||||||
|
* FileSystemStorage-backed Brainy plus 51 real-embedded entities before
|
||||||
|
* EVERY one of the 18 tests below (~950 add()/relate() calls total, each
|
||||||
|
* paying the real ONNX embedder — the whole file walled ~328s). Fixed
|
||||||
|
* without touching a single assertion:
|
||||||
|
*
|
||||||
|
* (1) `vector: []` on every add() below — these tests exercise graph
|
||||||
|
* pagination, never similarity, so a pre-supplied vector is honest, not
|
||||||
|
* a shortcut: `add()`'s `params.vector || (await this.embed(...))` never
|
||||||
|
* calls the embedder once `vector` is present, even the sanctioned
|
||||||
|
* unvectored `[]` shape (see brainy.ts's add(), the zero-norm-law
|
||||||
|
* comment) — and the `vector.length > 0` gate on dimension-pinning means
|
||||||
|
* `[]` never poisons `this.dimensions` for later real embeds.
|
||||||
|
* (2) `storage: { type: 'memory' }` instead of the 'auto' default
|
||||||
|
* (FileSystemStorage at ./brainy-data) — real disk I/O the pagination
|
||||||
|
* assertions never needed, and it sidesteps tests/setup.ts's global
|
||||||
|
* per-test `rm -rf brainy-data`, which would otherwise corrupt a brain
|
||||||
|
* shared across a describe's beforeAll out from under it.
|
||||||
|
* (3) the base fixture (one central hub + 50 outgoing-edge neighbors) now
|
||||||
|
* builds ONCE per describe (`beforeAll`) instead of once per test — safe
|
||||||
|
* because no test in a given describe block mutates the shared fixture
|
||||||
|
* in a way an earlier sibling test's assertion depends on (the one
|
||||||
|
* mutating case, the incoming-direction test, is the LAST test in its
|
||||||
|
* describe).
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { describe, it, expect, beforeEach } from 'vitest'
|
import { describe, it, expect, beforeAll, afterAll } from 'vitest'
|
||||||
import { Brainy } from '../../src/brainy.js'
|
import { Brainy } from '../../src/brainy.js'
|
||||||
import { NounType, VerbType } from '../../src/types/graphTypes.js'
|
import { NounType, VerbType } from '../../src/types/graphTypes.js'
|
||||||
|
|
||||||
|
|
@ -39,14 +64,21 @@ describe('GraphAdjacencyIndex Pagination', () => {
|
||||||
.map((i) => idMapper().getUuid(Number(i)))
|
.map((i) => idMapper().getUuid(Number(i)))
|
||||||
.filter((u: string | undefined): u is string => u !== undefined)
|
.filter((u: string | undefined): u is string => u !== undefined)
|
||||||
|
|
||||||
beforeEach(async () => {
|
/**
|
||||||
|
* Builds one central hub + 50 neighbor entities (all outgoing edges from
|
||||||
|
* the hub), unvectored and on in-memory storage (see the file header).
|
||||||
|
* Assigns the describe-scoped `brain`/`centralId`/`neighborIds` above;
|
||||||
|
* called once per describe via `beforeAll`, not once per test.
|
||||||
|
*/
|
||||||
|
async function buildFixture(): Promise<void> {
|
||||||
brain = new Brainy({ requireSubtype: false })
|
brain = new Brainy({ requireSubtype: false })
|
||||||
await brain.init()
|
await brain.init({ storage: { type: 'memory' } })
|
||||||
|
|
||||||
// Create central entity
|
// Create central entity
|
||||||
centralId = await brain.add({
|
centralId = await brain.add({
|
||||||
data: { name: 'Central Hub' },
|
data: { name: 'Central Hub' },
|
||||||
type: NounType.Thing
|
type: NounType.Thing,
|
||||||
|
vector: []
|
||||||
})
|
})
|
||||||
|
|
||||||
// Create 50 neighbor entities with relationships
|
// Create 50 neighbor entities with relationships
|
||||||
|
|
@ -54,7 +86,8 @@ describe('GraphAdjacencyIndex Pagination', () => {
|
||||||
for (let i = 0; i < 50; i++) {
|
for (let i = 0; i < 50; i++) {
|
||||||
const neighborId = await brain.add({
|
const neighborId = await brain.add({
|
||||||
data: { name: `Neighbor ${i}`, index: i },
|
data: { name: `Neighbor ${i}`, index: i },
|
||||||
type: NounType.Thing
|
type: NounType.Thing,
|
||||||
|
vector: []
|
||||||
})
|
})
|
||||||
neighborIds.push(neighborId)
|
neighborIds.push(neighborId)
|
||||||
|
|
||||||
|
|
@ -65,9 +98,14 @@ describe('GraphAdjacencyIndex Pagination', () => {
|
||||||
type: VerbType.RelatesTo
|
type: VerbType.RelatesTo
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
})
|
}
|
||||||
|
|
||||||
describe('getNeighbors() Pagination', () => {
|
describe('getNeighbors() Pagination', () => {
|
||||||
|
beforeAll(buildFixture)
|
||||||
|
afterAll(async () => {
|
||||||
|
await brain?.close()
|
||||||
|
})
|
||||||
|
|
||||||
it('should return all neighbors without pagination', async () => {
|
it('should return all neighbors without pagination', async () => {
|
||||||
const neighborInts = await graphIndex().getNeighbors(entityInt(centralId))
|
const neighborInts = await graphIndex().getNeighbors(entityInt(centralId))
|
||||||
const neighbors = intsToUuids(neighborInts)
|
const neighbors = intsToUuids(neighborInts)
|
||||||
|
|
@ -149,7 +187,8 @@ describe('GraphAdjacencyIndex Pagination', () => {
|
||||||
// Create some incoming relationships
|
// Create some incoming relationships
|
||||||
const sourceId = await brain.add({
|
const sourceId = await brain.add({
|
||||||
data: { name: 'Source' },
|
data: { name: 'Source' },
|
||||||
type: NounType.Thing
|
type: NounType.Thing,
|
||||||
|
vector: []
|
||||||
})
|
})
|
||||||
|
|
||||||
await brain.relate({
|
await brain.relate({
|
||||||
|
|
@ -169,6 +208,11 @@ describe('GraphAdjacencyIndex Pagination', () => {
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('getVerbIdsBySource() Pagination', () => {
|
describe('getVerbIdsBySource() Pagination', () => {
|
||||||
|
beforeAll(buildFixture)
|
||||||
|
afterAll(async () => {
|
||||||
|
await brain?.close()
|
||||||
|
})
|
||||||
|
|
||||||
it('should return all verb ints without pagination and resolve them back to ids', async () => {
|
it('should return all verb ints without pagination and resolve them back to ids', async () => {
|
||||||
const verbInts: bigint[] = await graphIndex().getVerbIdsBySource(entityInt(centralId))
|
const verbInts: bigint[] = await graphIndex().getVerbIdsBySource(entityInt(centralId))
|
||||||
|
|
||||||
|
|
@ -223,6 +267,11 @@ describe('GraphAdjacencyIndex Pagination', () => {
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('getVerbIdsByTarget() Pagination', () => {
|
describe('getVerbIdsByTarget() Pagination', () => {
|
||||||
|
beforeAll(buildFixture)
|
||||||
|
afterAll(async () => {
|
||||||
|
await brain?.close()
|
||||||
|
})
|
||||||
|
|
||||||
it('should return all verb ints targeting an entity', async () => {
|
it('should return all verb ints targeting an entity', async () => {
|
||||||
// Pick a neighbor that's a target of relationships
|
// Pick a neighbor that's a target of relationships
|
||||||
const targetId = neighborIds[0]
|
const targetId = neighborIds[0]
|
||||||
|
|
@ -236,14 +285,16 @@ describe('GraphAdjacencyIndex Pagination', () => {
|
||||||
// Create entity with many incoming relationships
|
// Create entity with many incoming relationships
|
||||||
const popularTarget = await brain.add({
|
const popularTarget = await brain.add({
|
||||||
data: { name: 'Popular Target' },
|
data: { name: 'Popular Target' },
|
||||||
type: NounType.Thing
|
type: NounType.Thing,
|
||||||
|
vector: []
|
||||||
})
|
})
|
||||||
|
|
||||||
// Create 30 relationships pointing to it
|
// Create 30 relationships pointing to it
|
||||||
for (let i = 0; i < 30; i++) {
|
for (let i = 0; i < 30; i++) {
|
||||||
const sourceId = await brain.add({
|
const sourceId = await brain.add({
|
||||||
data: { name: `Source ${i}` },
|
data: { name: `Source ${i}` },
|
||||||
type: NounType.Thing
|
type: NounType.Thing,
|
||||||
|
vector: []
|
||||||
})
|
})
|
||||||
await brain.relate({
|
await brain.relate({
|
||||||
from: sourceId,
|
from: sourceId,
|
||||||
|
|
@ -267,6 +318,11 @@ describe('GraphAdjacencyIndex Pagination', () => {
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('Performance with Pagination', () => {
|
describe('Performance with Pagination', () => {
|
||||||
|
beforeAll(buildFixture)
|
||||||
|
afterAll(async () => {
|
||||||
|
await brain?.close()
|
||||||
|
})
|
||||||
|
|
||||||
it('should maintain sub-5ms performance with pagination', async () => {
|
it('should maintain sub-5ms performance with pagination', async () => {
|
||||||
const central = entityInt(centralId)
|
const central = entityInt(centralId)
|
||||||
|
|
||||||
|
|
@ -285,11 +341,17 @@ describe('GraphAdjacencyIndex Pagination', () => {
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('Real-World Use Cases', () => {
|
describe('Real-World Use Cases', () => {
|
||||||
|
beforeAll(buildFixture)
|
||||||
|
afterAll(async () => {
|
||||||
|
await brain?.close()
|
||||||
|
})
|
||||||
|
|
||||||
it('should efficiently paginate through high-degree node', async () => {
|
it('should efficiently paginate through high-degree node', async () => {
|
||||||
// Simulate popular entity with 100+ relationships
|
// Simulate popular entity with 100+ relationships
|
||||||
const hub = await brain.add({
|
const hub = await brain.add({
|
||||||
data: { name: 'Popular Hub' },
|
data: { name: 'Popular Hub' },
|
||||||
type: NounType.Thing
|
type: NounType.Thing,
|
||||||
|
vector: []
|
||||||
})
|
})
|
||||||
|
|
||||||
// Create 100 relationships
|
// Create 100 relationships
|
||||||
|
|
@ -297,7 +359,8 @@ describe('GraphAdjacencyIndex Pagination', () => {
|
||||||
for (let i = 0; i < 100; i++) {
|
for (let i = 0; i < 100; i++) {
|
||||||
const targetId = await brain.add({
|
const targetId = await brain.add({
|
||||||
data: { name: `Target ${i}` },
|
data: { name: `Target ${i}` },
|
||||||
type: NounType.Thing
|
type: NounType.Thing,
|
||||||
|
vector: []
|
||||||
})
|
})
|
||||||
targetIds.push(targetId)
|
targetIds.push(targetId)
|
||||||
await brain.relate({
|
await brain.relate({
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@
|
||||||
* All entities carry explicit 384-dim vectors so no test invokes the embedder.
|
* All entities carry explicit 384-dim vectors so no test invokes the embedder.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { describe, it, expect } from 'vitest'
|
import { describe, it, expect, afterEach } from 'vitest'
|
||||||
import { Brainy } from '../../src/brainy.js'
|
import { Brainy } from '../../src/brainy.js'
|
||||||
import { NounType, VerbType } from '../../src/types/graphTypes.js'
|
import { NounType, VerbType } from '../../src/types/graphTypes.js'
|
||||||
import { v5, v7, isUUID } from '../../src/universal/uuid.js'
|
import { v5, v7, isUUID } from '../../src/universal/uuid.js'
|
||||||
|
|
@ -37,8 +37,15 @@ async function makeBrain(): Promise<Brainy> {
|
||||||
}
|
}
|
||||||
|
|
||||||
describe('id normalization — transparent string-key round-trips', () => {
|
describe('id normalization — transparent string-key round-trips', () => {
|
||||||
|
const opened: Brainy[] = []
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
for (const b of opened.splice(0)) await b.close().catch(() => {})
|
||||||
|
})
|
||||||
|
|
||||||
it('1. add() returns v5(key); get(key) and get(returnedId) both resolve; _originalId preserved', async () => {
|
it('1. add() returns v5(key); get(key) and get(returnedId) both resolve; _originalId preserved', async () => {
|
||||||
const brain = await makeBrain()
|
const brain = await makeBrain()
|
||||||
|
opened.push(brain)
|
||||||
|
|
||||||
const returnedId = await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person })
|
const returnedId = await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person })
|
||||||
|
|
||||||
|
|
@ -60,6 +67,7 @@ describe('id normalization — transparent string-key round-trips', () => {
|
||||||
|
|
||||||
it('2. relate() by string keys; related(key) and related({from:key}) return the edge to v5(toKey)', async () => {
|
it('2. relate() by string keys; related(key) and related({from:key}) return the edge to v5(toKey)', async () => {
|
||||||
const brain = await makeBrain()
|
const brain = await makeBrain()
|
||||||
|
opened.push(brain)
|
||||||
|
|
||||||
await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person })
|
await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person })
|
||||||
await brain.add({ id: 'doc-1', vector: vec(2), type: NounType.Document })
|
await brain.add({ id: 'doc-1', vector: vec(2), type: NounType.Document })
|
||||||
|
|
@ -85,6 +93,7 @@ describe('id normalization — transparent string-key round-trips', () => {
|
||||||
|
|
||||||
it('3. update() by string key reflects on get(key)', async () => {
|
it('3. update() by string key reflects on get(key)', async () => {
|
||||||
const brain = await makeBrain()
|
const brain = await makeBrain()
|
||||||
|
opened.push(brain)
|
||||||
|
|
||||||
await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person, metadata: { role: 'admin' } })
|
await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person, metadata: { role: 'admin' } })
|
||||||
await brain.update({ id: 'user-1', metadata: { role: 'owner' } })
|
await brain.update({ id: 'user-1', metadata: { role: 'owner' } })
|
||||||
|
|
@ -98,6 +107,7 @@ describe('id normalization — transparent string-key round-trips', () => {
|
||||||
|
|
||||||
it('4. remove() by string key deletes; get(key) is null', async () => {
|
it('4. remove() by string key deletes; get(key) is null', async () => {
|
||||||
const brain = await makeBrain()
|
const brain = await makeBrain()
|
||||||
|
opened.push(brain)
|
||||||
|
|
||||||
await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person })
|
await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person })
|
||||||
expect(await brain.get('user-1')).not.toBeNull()
|
expect(await brain.get('user-1')).not.toBeNull()
|
||||||
|
|
@ -110,6 +120,7 @@ describe('id normalization — transparent string-key round-trips', () => {
|
||||||
|
|
||||||
it('5. find({ connected: { from: key } }) resolves the anchor key', async () => {
|
it('5. find({ connected: { from: key } }) resolves the anchor key', async () => {
|
||||||
const brain = await makeBrain()
|
const brain = await makeBrain()
|
||||||
|
opened.push(brain)
|
||||||
|
|
||||||
await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person })
|
await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person })
|
||||||
await brain.add({ id: 'doc-1', vector: vec(2), type: NounType.Document })
|
await brain.add({ id: 'doc-1', vector: vec(2), type: NounType.Document })
|
||||||
|
|
@ -122,6 +133,7 @@ describe('id normalization — transparent string-key round-trips', () => {
|
||||||
|
|
||||||
it('6. transact() add+relate by string keys round-trips with consistent canonical ids', async () => {
|
it('6. transact() add+relate by string keys round-trips with consistent canonical ids', async () => {
|
||||||
const brain = await makeBrain()
|
const brain = await makeBrain()
|
||||||
|
opened.push(brain)
|
||||||
|
|
||||||
// Seed user-1 so the relate op has a target to point at.
|
// Seed user-1 so the relate op has a target to point at.
|
||||||
await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person })
|
await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person })
|
||||||
|
|
@ -149,6 +161,7 @@ describe('id normalization — transparent string-key round-trips', () => {
|
||||||
|
|
||||||
it('7. addMany() + relateMany() with string ids round-trip', async () => {
|
it('7. addMany() + relateMany() with string ids round-trip', async () => {
|
||||||
const brain = await makeBrain()
|
const brain = await makeBrain()
|
||||||
|
opened.push(brain)
|
||||||
|
|
||||||
const added = await brain.addMany({
|
const added = await brain.addMany({
|
||||||
items: [
|
items: [
|
||||||
|
|
@ -175,6 +188,7 @@ describe('id normalization — transparent string-key round-trips', () => {
|
||||||
|
|
||||||
it('8. determinism: same key maps to same UUID — two adds upsert ONE entity, not two', async () => {
|
it('8. determinism: same key maps to same UUID — two adds upsert ONE entity, not two', async () => {
|
||||||
const brain = await makeBrain()
|
const brain = await makeBrain()
|
||||||
|
opened.push(brain)
|
||||||
|
|
||||||
const id1 = await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person, metadata: { n: 1 } })
|
const id1 = await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person, metadata: { n: 1 } })
|
||||||
const id2 = await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person, metadata: { n: 2 } })
|
const id2 = await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person, metadata: { n: 2 } })
|
||||||
|
|
@ -193,6 +207,7 @@ describe('id normalization — transparent string-key round-trips', () => {
|
||||||
|
|
||||||
it('9. valid-UUID passthrough: a real UUID is kept verbatim with NO _originalId', async () => {
|
it('9. valid-UUID passthrough: a real UUID is kept verbatim with NO _originalId', async () => {
|
||||||
const brain = await makeBrain()
|
const brain = await makeBrain()
|
||||||
|
opened.push(brain)
|
||||||
|
|
||||||
const realUuid = v7()
|
const realUuid = v7()
|
||||||
const returnedId = await brain.add({ id: realUuid, vector: vec(5), type: NounType.Thing })
|
const returnedId = await brain.add({ id: realUuid, vector: vec(5), type: NounType.Thing })
|
||||||
|
|
@ -207,6 +222,7 @@ describe('id normalization — transparent string-key round-trips', () => {
|
||||||
|
|
||||||
it('10. no-id add() mints a v7; newId() mints a v7', async () => {
|
it('10. no-id add() mints a v7; newId() mints a v7', async () => {
|
||||||
const brain = await makeBrain()
|
const brain = await makeBrain()
|
||||||
|
opened.push(brain)
|
||||||
|
|
||||||
const autoId = await brain.add({ vector: vec(6), type: NounType.Thing })
|
const autoId = await brain.add({ vector: vec(6), type: NounType.Thing })
|
||||||
expect(isUUID(autoId)).toBe(true)
|
expect(isUUID(autoId)).toBe(true)
|
||||||
|
|
|
||||||
|
|
@ -70,8 +70,22 @@ describe('an idle brain costs nothing', () => {
|
||||||
await brain.flush()
|
await brain.flush()
|
||||||
|
|
||||||
const logged: string[] = []
|
const logged: string[] = []
|
||||||
|
// The STACK behind each narration, kept beside the line it belongs to.
|
||||||
|
// vitest tags a stdout block with the test that is RUNNING, not the brain
|
||||||
|
// that wrote it, so teeing these lines through would only ever name this
|
||||||
|
// test. The call stack does name the driver: `kickBackgroundFlush('idle')`
|
||||||
|
// under `armIdleFlushTimer` is a cadence flush on some brain, the deferred-
|
||||||
|
// embed worker's commit path is a brain still landing vectors, and a bare
|
||||||
|
// `flush()` is an explicit caller. That distinction is the whole question.
|
||||||
|
const stacks: string[] = []
|
||||||
const origLog = console.log
|
const origLog = console.log
|
||||||
console.log = ((...a: unknown[]) => { logged.push(a.map(String).join(' ')) }) as typeof console.log
|
console.log = ((...a: unknown[]) => {
|
||||||
|
const line = a.map(String).join(' ')
|
||||||
|
logged.push(line)
|
||||||
|
if (/All indexes flushed to disk|Flushing Brainy indexes/.test(line)) {
|
||||||
|
stacks.push(new Error('flush narration').stack ?? '(no stack)')
|
||||||
|
}
|
||||||
|
}) as typeof console.log
|
||||||
|
|
||||||
// Watch the providers directly: a flush that runs calls all of them.
|
// Watch the providers directly: a flush that runs calls all of them.
|
||||||
const storage = (brain as unknown as { storage: { flushCounts: () => Promise<void> } }).storage
|
const storage = (brain as unknown as { storage: { flushCounts: () => Promise<void> } }).storage
|
||||||
|
|
@ -88,11 +102,38 @@ describe('an idle brain costs nothing', () => {
|
||||||
}
|
}
|
||||||
|
|
||||||
// (a) + (b): nothing ran, nothing was said.
|
// (a) + (b): nothing ran, nothing was said.
|
||||||
expect(logged.filter((l) => /All indexes flushed to disk/.test(l))).toEqual([])
|
//
|
||||||
expect(logged.filter((l) => /Flushing Brainy indexes/.test(l))).toEqual([])
|
// THE SPIES COME FIRST, AND THEY ARE THE ATTRIBUTABLE HALF. They are bound
|
||||||
|
// to THIS brain's providers, so they answer "did this brain flush?" and
|
||||||
|
// nothing else. The console filters below cannot: the gate config runs the
|
||||||
|
// whole suite in ONE process (`pool: 'forks'`, `singleFork: true`), so
|
||||||
|
// `console.log` carries the narration of every brain alive in that
|
||||||
|
// process — including one a previous file opened and never closed, whose
|
||||||
|
// unref'd cadence timer is still doing honest work. A neighbour narrating
|
||||||
|
// is a REAL finding about suite hygiene, but it is not this brain failing
|
||||||
|
// its own law, and the two must not be reported as the same thing.
|
||||||
|
//
|
||||||
|
// So: spies first (whose failure means the engine broke the law), console
|
||||||
|
// second (whose failure means SOMETHING in the process narrated), and the
|
||||||
|
// console assertion carries the captured lines in its message. vitest's
|
||||||
|
// stdout blocks are prefixed `stdout | <file> > <test>`, so those lines
|
||||||
|
// plus the surrounding gate log name the brain that printed them.
|
||||||
expect(countsSpy).not.toHaveBeenCalled()
|
expect(countsSpy).not.toHaveBeenCalled()
|
||||||
expect(metadataSpy).not.toHaveBeenCalled()
|
expect(metadataSpy).not.toHaveBeenCalled()
|
||||||
expect(graphSpy).not.toHaveBeenCalled()
|
expect(graphSpy).not.toHaveBeenCalled()
|
||||||
|
|
||||||
|
const flushChatter = logged.filter(
|
||||||
|
(l) => /All indexes flushed to disk/.test(l) || /Flushing Brainy indexes/.test(l)
|
||||||
|
)
|
||||||
|
expect(
|
||||||
|
flushChatter,
|
||||||
|
`${flushChatter.length} flush line(s) narrated during the ${IDLE_WATCH_MS}ms idle ` +
|
||||||
|
`window. This brain's own providers were NOT called (asserted above), so another ` +
|
||||||
|
`brain alive in this process printed them — the suite runs every file in ONE ` +
|
||||||
|
`process and 67 test files create more brains than they close.\n` +
|
||||||
|
`${flushChatter.join('\n')}\n\n` +
|
||||||
|
`The stack behind the first one names the driver:\n${stacks[0] ?? '(none captured)'}`
|
||||||
|
).toEqual([])
|
||||||
}, 180_000)
|
}, 180_000)
|
||||||
|
|
||||||
it('an explicit flush over a clean brain calls no provider and prints nothing', async () => {
|
it('an explicit flush over a clean brain calls no provider and prints nothing', async () => {
|
||||||
|
|
|
||||||
|
|
@ -26,6 +26,7 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
import { Brainy } from '../../src/brainy.js'
|
import { Brainy } from '../../src/brainy.js'
|
||||||
import { NounType } from '../../src/types/graphTypes.js'
|
import { NounType } from '../../src/types/graphTypes.js'
|
||||||
import { existsSync, rmSync } from 'fs'
|
import { existsSync, rmSync } from 'fs'
|
||||||
|
import { MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from '../../src/errors/brainyError.js'
|
||||||
|
|
||||||
describe('Metadata Vector Exclusion Fix', () => {
|
describe('Metadata Vector Exclusion Fix', () => {
|
||||||
let brainy: Brainy
|
let brainy: Brainy
|
||||||
|
|
@ -155,29 +156,57 @@ describe('Metadata Vector Exclusion Fix', () => {
|
||||||
expect(results[0].entity.metadata?.name).toBe('Bob')
|
expect(results[0].entity.metadata?.name).toBe('Bob')
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should skip indexing large arrays (>10 elements)', async () => {
|
it('should REFUSE an array over the indexing bound, by name', async () => {
|
||||||
// Add entity with a large array (not a vector, just bulk data).
|
// A large array (not a vector, just bulk data). This used to be SKIPPED in
|
||||||
const largeArray = Array.from({ length: 100 }, (_, i) => `item${i}`)
|
// silence at a bound of 10 — the field simply vanished from the index and
|
||||||
|
// the row dropped out of every `where` on it, indistinguishably from "no
|
||||||
|
// row matches". The bound is now MAX_INDEXED_ARRAY_LENGTH and it REFUSES.
|
||||||
|
const overTheBound = MAX_INDEXED_ARRAY_LENGTH + 1
|
||||||
|
const largeArray = Array.from({ length: overTheBound }, (_, i) => `item${i}`)
|
||||||
|
|
||||||
await brainy.add({
|
const err = await brainy
|
||||||
type: NounType.Document,
|
.add({
|
||||||
data: 'Doc with large array',
|
type: NounType.Document,
|
||||||
metadata: {
|
data: 'Doc with large array',
|
||||||
name: 'Doc with large array',
|
metadata: {
|
||||||
items: largeArray
|
name: 'Doc with large array',
|
||||||
}
|
items: largeArray
|
||||||
})
|
}
|
||||||
|
})
|
||||||
|
.catch((e: any) => e)
|
||||||
|
|
||||||
// Large arrays (> 10 elements) are deliberately skipped to avoid indexing
|
expect(err).toBeInstanceOf(MetadataArrayTooLargeError)
|
||||||
// bulk/vector-like payloads: 'items' must NOT appear, and the 100 elements
|
expect(err.field).toBe('items')
|
||||||
// must NOT have produced 100 indexed fields.
|
expect(err.length).toBe(overTheBound)
|
||||||
|
expect(err.limit).toBe(MAX_INDEXED_ARRAY_LENGTH)
|
||||||
|
|
||||||
|
// Nothing was indexed from the refused write — no 'items' field, and above
|
||||||
|
// all no per-element numeric fields (the original explosion class).
|
||||||
const fields = await brainy.getAvailableFields()
|
const fields = await brainy.getAvailableFields()
|
||||||
expect(fields).not.toContain('items')
|
expect(fields).not.toContain('items')
|
||||||
const numericFields = fields.filter(f => /(^|\.)\d+$/.test(f))
|
const numericFields = fields.filter(f => /(^|\.)\d+$/.test(f))
|
||||||
expect(numericFields).toEqual([])
|
expect(numericFields).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
// The scalar 'name' field IS indexed.
|
it('should index an array UP TO the bound — the old limit of 10 was the bug', async () => {
|
||||||
expect(fields).toContain('name')
|
await brainy.add({
|
||||||
|
type: NounType.Document,
|
||||||
|
data: 'Doc with a long-but-legitimate tag list',
|
||||||
|
metadata: {
|
||||||
|
name: 'Doc with many tags',
|
||||||
|
items: Array.from({ length: MAX_INDEXED_ARRAY_LENGTH }, (_, i) => `item${i}`)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
const fields = await brainy.getAvailableFields()
|
||||||
|
// The field IS indexed now, and still without per-element numeric fields.
|
||||||
|
expect(fields).toContain('items')
|
||||||
|
expect(fields.filter(f => /(^|\.)\d+$/.test(f))).toEqual([])
|
||||||
|
|
||||||
|
// And the eleventh element — the one the old bound silently dropped the
|
||||||
|
// whole field for — really is searchable.
|
||||||
|
const hits = await brainy.find({ where: { items: 'item10' } })
|
||||||
|
expect(hits.length).toBeGreaterThan(0)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should preserve HNSW vector search functionality', async () => {
|
it('should preserve HNSW vector search functionality', async () => {
|
||||||
|
|
|
||||||
|
|
@ -107,7 +107,11 @@ describe('Multi-process safety + read-only mode', () => {
|
||||||
|
|
||||||
const blocked = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
const blocked = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||||
await expect(blocked.init()).rejects.toThrow(/another writer holds/i)
|
await expect(blocked.init()).rejects.toThrow(/another writer holds/i)
|
||||||
// Don't track `blocked` for afterEach cleanup since init failed.
|
// A rejected init() still registered `blocked` in Brainy's global
|
||||||
|
// instance registry (the constructor does that unconditionally) — close()
|
||||||
|
// is safe to call even though init() never completed, and is what
|
||||||
|
// deregisters it (and, once idle, the process-level shutdown hooks).
|
||||||
|
await blocked.close().catch(() => {})
|
||||||
})
|
})
|
||||||
|
|
||||||
it('takes over a STALE foreign lock (dead PID + old heartbeat) and claims atomically', async () => {
|
it('takes over a STALE foreign lock (dead PID + old heartbeat) and claims atomically', async () => {
|
||||||
|
|
@ -151,6 +155,7 @@ describe('Multi-process safety + read-only mode', () => {
|
||||||
const err: any = await blocked.init().catch((e) => e)
|
const err: any = await blocked.init().catch((e) => e)
|
||||||
expect(err.code).toBe('BRAINY_WRITER_LOCKED')
|
expect(err.code).toBe('BRAINY_WRITER_LOCKED')
|
||||||
expect(err.lockInfo?.pid).toBe(otherPid)
|
expect(err.lockInfo?.pid).toBe(otherPid)
|
||||||
|
await blocked.close().catch(() => {})
|
||||||
})
|
})
|
||||||
|
|
||||||
it('release drains an in-flight heartbeat — no phantom lock re-created after unlink', async () => {
|
it('release drains an in-flight heartbeat — no phantom lock re-created after unlink', async () => {
|
||||||
|
|
|
||||||
547
tests/integration/pending-embed-checkpoint.test.ts
Normal file
547
tests/integration/pending-embed-checkpoint.test.ts
Normal file
|
|
@ -0,0 +1,547 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/pending-embed-checkpoint
|
||||||
|
* @description THE PENDING-EMBED CHECKPOINT — the bound that engages on the
|
||||||
|
* brains that need it.
|
||||||
|
*
|
||||||
|
* 10.4.9 bounded the open-path `recover-pending-embeds` fold with a LOW-WATER
|
||||||
|
* MARK: the log head at which the pending set last drained to EMPTY. That mark
|
||||||
|
* carries no set, so it can only be written when the set is empty — and a brain
|
||||||
|
* holding even ONE id that never lands (an embed that keeps failing, a worker
|
||||||
|
* that never gets to it, a row reaped in memory only and re-folded every open)
|
||||||
|
* never drains, therefore never writes a mark, therefore re-reads its WHOLE
|
||||||
|
* fact log on every single open. The bound was absent from exactly the brains
|
||||||
|
* whose fold is expensive: a silent scaling defect.
|
||||||
|
*
|
||||||
|
* The cure is a CHECKPOINT of the pending set —
|
||||||
|
* `_system/pending_embeds_checkpoint.json` = `{ generation, pending, writtenAt }`,
|
||||||
|
* meaning "as of durable generation G the pending set was exactly this list".
|
||||||
|
* Open seeds the set from `pending` and scans only from `G + 1`, so the fold is
|
||||||
|
* O(facts since G) whether or not the set ever drains.
|
||||||
|
*
|
||||||
|
* What this suite pins:
|
||||||
|
* 1. A brain with one permanently-stuck pending id, closed cleanly and
|
||||||
|
* reopened, scans ONLY the facts after the checkpoint — asserted from the
|
||||||
|
* fold's own accounting, never a clock. The same fixture pins the DEFECT:
|
||||||
|
* no low-water mark exists on that brain, because it never drained.
|
||||||
|
* 2. A crash matrix in a REAL child process (SIGKILL, no close), for kills
|
||||||
|
* before a checkpoint write, after one with embeds landed and flushed
|
||||||
|
* after it, and after one with an UN-FLUSHED tail at the moment of death.
|
||||||
|
* The invariant in every row is differential: the checkpoint-bounded fold
|
||||||
|
* the reopened brain actually ran ≡ a full fold from generation 1 over the
|
||||||
|
* same recovered log.
|
||||||
|
* 3. A torn checkpoint falls back — loudly (the adapter's torn-record gauge
|
||||||
|
* plus the fold's own narration of which bound applied) and correctly.
|
||||||
|
* 4. The existing low-water pins keep passing unchanged
|
||||||
|
* (`pending-embed-low-water.test.ts`): the mark is still written and is
|
||||||
|
* still read, now as the FALLBACK bound beneath the checkpoint.
|
||||||
|
*
|
||||||
|
* The crash-recovery contract is untouched: the fold runs on the open's
|
||||||
|
* foreground, so a reopened brain has its markers re-armed when open() returns.
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, afterEach } from 'vitest'
|
||||||
|
import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync } from 'node:fs'
|
||||||
|
import { spawn } from 'node:child_process'
|
||||||
|
import { gunzipSync } from 'node:zlib'
|
||||||
|
import { tmpdir } from 'node:os'
|
||||||
|
import { join } from 'node:path'
|
||||||
|
import { Brainy } from '../../src/brainy.js'
|
||||||
|
import { NounType } from '../../src/types/graphTypes.js'
|
||||||
|
import { getTornRecordGauge } from '../../src/storage/tornRecordError.js'
|
||||||
|
|
||||||
|
const CHECKPOINT_PATH = '_system/pending_embeds_checkpoint.json'
|
||||||
|
const LOWWATER_PATH = '_system/pending_embeds_lowwater.json'
|
||||||
|
const REPO_ROOT = process.cwd()
|
||||||
|
const TSX = join(REPO_ROOT, 'node_modules', '.bin', 'tsx')
|
||||||
|
|
||||||
|
/** The fold's own accounting for the most recent open. */
|
||||||
|
interface FoldReport {
|
||||||
|
bound: 'checkpoint' | 'low-water' | 'genesis'
|
||||||
|
fromGeneration: number
|
||||||
|
factsScanned: number
|
||||||
|
seeded: number
|
||||||
|
pending: number
|
||||||
|
}
|
||||||
|
|
||||||
|
const roots: string[] = []
|
||||||
|
const liveBrains: Brainy<any>[] = []
|
||||||
|
|
||||||
|
function dir(): string {
|
||||||
|
const d = mkdtempSync(join(tmpdir(), 'brainy-embed-ckpt-'))
|
||||||
|
roots.push(d)
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
|
async function open(root: string, opts?: { blockWorker?: boolean }): Promise<Brainy<any>> {
|
||||||
|
const brain = new Brainy<any>({
|
||||||
|
requireSubtype: false,
|
||||||
|
storage: { type: 'filesystem', path: root }
|
||||||
|
})
|
||||||
|
// Blocking the worker BEFORE init() is how a "permanently stuck" pending id
|
||||||
|
// is built deterministically: the state under test is "an id the fold keeps
|
||||||
|
// re-arming and nothing ever disarms", and its production causes (a failing
|
||||||
|
// embedder, a wedged model, a data-less row) all reduce to exactly that.
|
||||||
|
if (opts?.blockWorker) (brain as unknown as { kickEmbedWorker: () => void }).kickEmbedWorker = () => {}
|
||||||
|
await brain.init()
|
||||||
|
liveBrains.push(brain)
|
||||||
|
return brain
|
||||||
|
}
|
||||||
|
|
||||||
|
function foldReport(brain: Brainy<any>): FoldReport {
|
||||||
|
const report = (brain as unknown as { _pendingEmbedFoldReport: FoldReport | null })
|
||||||
|
._pendingEmbedFoldReport
|
||||||
|
if (report === null) throw new Error('the open ran no pending-embed fold')
|
||||||
|
return report
|
||||||
|
}
|
||||||
|
|
||||||
|
function pendingIds(brain: Brainy<any>): string[] {
|
||||||
|
return [
|
||||||
|
...(brain as unknown as { _pendingEmbedIds: Set<string> })._pendingEmbedIds
|
||||||
|
].sort()
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Read an artifact straight off disk (the adapter gzips raw objects). */
|
||||||
|
function readArtifact(root: string, path: string): Record<string, unknown> | null {
|
||||||
|
const plain = join(root, ...path.split('/'))
|
||||||
|
const gz = `${plain}.gz`
|
||||||
|
if (existsSync(gz)) return JSON.parse(gunzipSync(readFileSync(gz)).toString('utf-8'))
|
||||||
|
if (existsSync(plain)) return JSON.parse(readFileSync(plain, 'utf-8'))
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The on-disk path the adapter actually used for an artifact. */
|
||||||
|
function artifactPath(root: string, path: string): string | null {
|
||||||
|
const plain = join(root, ...path.split('/'))
|
||||||
|
const gz = `${plain}.gz`
|
||||||
|
if (existsSync(gz)) return gz
|
||||||
|
if (existsSync(plain)) return plain
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* THE DIFFERENTIAL ORACLE: fold the log from generation 1 with exactly the
|
||||||
|
* engine's own rules. This is what the bounded fold must agree with, and its
|
||||||
|
* fact count is what the unbounded fold used to read at every open.
|
||||||
|
*/
|
||||||
|
async function fullFold(brain: Brainy<any>): Promise<{ ids: string[]; facts: number }> {
|
||||||
|
const log = (
|
||||||
|
brain as unknown as { generationStore: { getFactLog(): any } }
|
||||||
|
).generationStore.getFactLog()
|
||||||
|
const pending = new Set<string>()
|
||||||
|
let facts = 0
|
||||||
|
const scan = log.scanFacts({ fromGeneration: 1 })
|
||||||
|
for await (const batch of scan.batches()) {
|
||||||
|
for (const fact of batch.facts) {
|
||||||
|
facts++
|
||||||
|
for (const record of fact.records ?? []) {
|
||||||
|
if (record.type === 'embed.pending') pending.add(record.id)
|
||||||
|
else if (record.type === 'embed.landed') pending.delete(record.id)
|
||||||
|
}
|
||||||
|
for (const op of fact.ops) {
|
||||||
|
if (op.kind === 'noun' && op.record === null) pending.delete(op.id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { ids: [...pending].sort(), facts }
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Capture every console.warn/error line emitted while `fn` runs. */
|
||||||
|
async function captureConsole<T>(fn: () => Promise<T>): Promise<{ result: T; lines: string[] }> {
|
||||||
|
const lines: string[] = []
|
||||||
|
const origWarn = console.warn
|
||||||
|
const origError = console.error
|
||||||
|
const sink = (...args: unknown[]) => {
|
||||||
|
lines.push(args.map((a) => String(a)).join(' '))
|
||||||
|
}
|
||||||
|
console.warn = sink as typeof console.warn
|
||||||
|
console.error = sink as typeof console.error
|
||||||
|
try {
|
||||||
|
const result = await fn()
|
||||||
|
return { result, lines }
|
||||||
|
} finally {
|
||||||
|
console.warn = origWarn
|
||||||
|
console.error = origError
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run a child process that arranges a store and then waits forever, so the
|
||||||
|
* parent can SIGKILL it. A real process death is the only honest way to pin
|
||||||
|
* "no close ran, no shutdown hook ran, RAM is gone".
|
||||||
|
*
|
||||||
|
* `detached` puts the child in its own process GROUP: tsx runs the script in a
|
||||||
|
* grandchild, and only a group-wide signal reaches the process holding the
|
||||||
|
* writer lock.
|
||||||
|
*/
|
||||||
|
function spawnArranger(root: string, body: string): Promise<{
|
||||||
|
child: ReturnType<typeof spawn>
|
||||||
|
output: () => string
|
||||||
|
}> {
|
||||||
|
const scriptPath = join(root, 'arrange.mts')
|
||||||
|
writeFileSync(scriptPath, body)
|
||||||
|
const child = spawn(TSX, [scriptPath], {
|
||||||
|
cwd: REPO_ROOT,
|
||||||
|
stdio: ['ignore', 'pipe', 'pipe'],
|
||||||
|
detached: true
|
||||||
|
})
|
||||||
|
let out = ''
|
||||||
|
child.stdout!.on('data', (d) => { out += String(d) })
|
||||||
|
child.stderr!.on('data', (d) => { out += String(d) })
|
||||||
|
return new Promise((resolvePromise, rejectPromise) => {
|
||||||
|
const timer = setTimeout(
|
||||||
|
() => rejectPromise(new Error(`arranger never became READY:\n${out}`)),
|
||||||
|
180_000
|
||||||
|
)
|
||||||
|
child.stdout!.on('data', () => {
|
||||||
|
if (out.includes('READY')) {
|
||||||
|
clearTimeout(timer)
|
||||||
|
resolvePromise({ child, output: () => out })
|
||||||
|
}
|
||||||
|
})
|
||||||
|
child.on('exit', (code) => {
|
||||||
|
clearTimeout(timer)
|
||||||
|
if (!out.includes('READY')) rejectPromise(new Error(`arranger exited ${code}:\n${out}`))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Parse the `IDS:{...}` line an arranger prints — supplied ids are normalised
|
||||||
|
* to canonical uuids, and the markers, checkpoint and fold all speak those. */
|
||||||
|
function childIds(output: string): Record<string, string> {
|
||||||
|
const line = output.split('\n').find((l) => l.startsWith('IDS:'))
|
||||||
|
if (!line) throw new Error(`arranger printed no IDS line:\n${output}`)
|
||||||
|
return JSON.parse(line.slice('IDS:'.length))
|
||||||
|
}
|
||||||
|
|
||||||
|
/** SIGKILL the whole group and wait for the grandchild's death to settle. */
|
||||||
|
async function sigkill(child: ReturnType<typeof spawn>): Promise<void> {
|
||||||
|
process.kill(-(child.pid as number), 'SIGKILL')
|
||||||
|
await new Promise<void>((r) => child.on('exit', () => r()))
|
||||||
|
await new Promise<void>((r) => setTimeout(r, 500))
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The preamble every arranger child shares. */
|
||||||
|
function childPreamble(root: string): string {
|
||||||
|
return `
|
||||||
|
import { Brainy } from ${JSON.stringify(join(REPO_ROOT, 'src', 'brainy.ts'))}
|
||||||
|
const ROOT = ${JSON.stringify(root)}
|
||||||
|
const brain = new Brainy<any>({ requireSubtype: false, storage: { type: 'filesystem', path: ROOT } })
|
||||||
|
const block = () => { (brain as any).kickEmbedWorker = () => {} }
|
||||||
|
const settleCheckpoint = async () => {
|
||||||
|
// The cadence write is fire-and-forget; wait for the single flight.
|
||||||
|
for (let i = 0; i < 200; i++) {
|
||||||
|
if (!(brain as any)._pendingEmbedCheckpointFlight) break
|
||||||
|
await (brain as any)._pendingEmbedCheckpointFlight.catch(() => {})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
`
|
||||||
|
}
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
for (const brain of liveBrains.splice(0)) {
|
||||||
|
try { await brain.close() } catch { /* already closed / crashed — teardown only */ }
|
||||||
|
}
|
||||||
|
for (const d of roots.splice(0)) rmSync(d, { recursive: true, force: true })
|
||||||
|
})
|
||||||
|
|
||||||
|
// ===========================================================================
|
||||||
|
// 1. The stuck-id brain — the defect, and the bound that now engages on it
|
||||||
|
// ===========================================================================
|
||||||
|
|
||||||
|
describe('pending-embed checkpoint — a brain whose pending set never drains', () => {
|
||||||
|
it('a permanently-stuck pending id: the reopen scans only the facts after the checkpoint', async () => {
|
||||||
|
const root = dir()
|
||||||
|
const first = await open(root, { blockWorker: true })
|
||||||
|
// add() returns the CANONICAL id (supplied ids are normalised), and that is
|
||||||
|
// the id the markers, the checkpoint and the fold all speak.
|
||||||
|
const stuck = await first.add({
|
||||||
|
id: 'stuck',
|
||||||
|
data: 'a deferred row whose embed never lands',
|
||||||
|
type: NounType.Thing,
|
||||||
|
deferEmbedding: true
|
||||||
|
})
|
||||||
|
expect(first.pendingEmbedCount()).toBe(1)
|
||||||
|
// Ordinary traffic after it — every one of these is a fact the unbounded
|
||||||
|
// fold had to re-read at every open, forever, because of that one id.
|
||||||
|
for (let i = 0; i < 12; i++) {
|
||||||
|
await first.add({ id: `row-${i}`, data: `row ${i}`, type: NounType.Thing })
|
||||||
|
}
|
||||||
|
await first.close()
|
||||||
|
liveBrains.splice(liveBrains.indexOf(first), 1)
|
||||||
|
|
||||||
|
// THE DEFECT, PINNED: the pending set never drained, so the old bound was
|
||||||
|
// never written — nothing on this brain could have shortened its fold.
|
||||||
|
expect(readArtifact(root, LOWWATER_PATH)).toBeNull()
|
||||||
|
// The checkpoint IS written at the clean close, set non-empty and all.
|
||||||
|
const checkpoint = readArtifact(root, CHECKPOINT_PATH) as {
|
||||||
|
generation: number
|
||||||
|
pending: string[]
|
||||||
|
} | null
|
||||||
|
expect(checkpoint).not.toBeNull()
|
||||||
|
expect(checkpoint!.generation).toBeGreaterThan(0)
|
||||||
|
expect(checkpoint!.pending).toEqual([stuck])
|
||||||
|
|
||||||
|
const second = await open(root, { blockWorker: true })
|
||||||
|
const report = foldReport(second)
|
||||||
|
// THE FIX, from the fold's own counter — not the clock.
|
||||||
|
expect(report.bound).toBe('checkpoint')
|
||||||
|
expect(report.fromGeneration).toBe(checkpoint!.generation + 1)
|
||||||
|
expect(report.factsScanned).toBe(0)
|
||||||
|
expect(report.seeded).toBe(1)
|
||||||
|
// The crash-recovery contract is intact: the marker is re-armed by open().
|
||||||
|
expect(pendingIds(second)).toEqual([stuck])
|
||||||
|
expect(second.pendingEmbedCount()).toBe(1)
|
||||||
|
|
||||||
|
// The differential: the bounded answer is the full-fold answer, and the
|
||||||
|
// full fold is what the previous bound would have had to read.
|
||||||
|
const full = await fullFold(second)
|
||||||
|
expect(full.ids).toEqual([stuck])
|
||||||
|
expect(full.facts).toBeGreaterThanOrEqual(13)
|
||||||
|
expect(report.factsScanned).toBeLessThan(full.facts)
|
||||||
|
}, 180_000)
|
||||||
|
|
||||||
|
it('the bound stays O(delta) across repeated opens while the id is still stuck', async () => {
|
||||||
|
const root = dir()
|
||||||
|
const first = await open(root, { blockWorker: true })
|
||||||
|
const stuck = await first.add({
|
||||||
|
id: 'stuck',
|
||||||
|
data: 'never lands',
|
||||||
|
type: NounType.Thing,
|
||||||
|
deferEmbedding: true
|
||||||
|
})
|
||||||
|
for (let i = 0; i < 6; i++) {
|
||||||
|
await first.add({ id: `a-${i}`, data: `a ${i}`, type: NounType.Thing })
|
||||||
|
}
|
||||||
|
await first.close()
|
||||||
|
liveBrains.splice(liveBrains.indexOf(first), 1)
|
||||||
|
|
||||||
|
const second = await open(root, { blockWorker: true })
|
||||||
|
expect(foldReport(second).factsScanned).toBe(0)
|
||||||
|
// More history under the same stuck id.
|
||||||
|
for (let i = 0; i < 9; i++) {
|
||||||
|
await second.add({ id: `b-${i}`, data: `b ${i}`, type: NounType.Thing })
|
||||||
|
}
|
||||||
|
await second.close()
|
||||||
|
liveBrains.splice(liveBrains.indexOf(second), 1)
|
||||||
|
|
||||||
|
const third = await open(root, { blockWorker: true })
|
||||||
|
const report = foldReport(third)
|
||||||
|
const full = await fullFold(third)
|
||||||
|
expect(report.bound).toBe('checkpoint')
|
||||||
|
expect(report.factsScanned).toBe(0)
|
||||||
|
// The unbounded fold grew with the store; the bounded one did not.
|
||||||
|
expect(full.facts).toBeGreaterThanOrEqual(16)
|
||||||
|
expect(pendingIds(third)).toEqual([stuck])
|
||||||
|
expect(full.ids).toEqual([stuck])
|
||||||
|
}, 180_000)
|
||||||
|
})
|
||||||
|
|
||||||
|
// ===========================================================================
|
||||||
|
// 2. Torn checkpoint — falls back, loudly, correctly
|
||||||
|
// ===========================================================================
|
||||||
|
|
||||||
|
describe('pending-embed checkpoint — a torn checkpoint never shortens the fold', () => {
|
||||||
|
it('an undecodable checkpoint file degrades to the next bound, loudly, with the right pending set', async () => {
|
||||||
|
const root = dir()
|
||||||
|
const first = await open(root, { blockWorker: true })
|
||||||
|
const stuck = await first.add({
|
||||||
|
id: 'stuck',
|
||||||
|
data: 'never lands',
|
||||||
|
type: NounType.Thing,
|
||||||
|
deferEmbedding: true
|
||||||
|
})
|
||||||
|
for (let i = 0; i < 5; i++) {
|
||||||
|
await first.add({ id: `row-${i}`, data: `row ${i}`, type: NounType.Thing })
|
||||||
|
}
|
||||||
|
await first.close()
|
||||||
|
liveBrains.splice(liveBrains.indexOf(first), 1)
|
||||||
|
|
||||||
|
const onDisk = artifactPath(root, CHECKPOINT_PATH)
|
||||||
|
expect(onDisk).not.toBeNull()
|
||||||
|
// Tear it: bytes that are neither valid gzip nor valid JSON. A torn file
|
||||||
|
// must THROW on read — never parse into a partial `pending` list.
|
||||||
|
writeFileSync(onDisk!, 'not a checkpoint at all {{{')
|
||||||
|
|
||||||
|
const before = getTornRecordGauge().count
|
||||||
|
const { result: second, lines } = await captureConsole(async () =>
|
||||||
|
open(root, { blockWorker: true })
|
||||||
|
)
|
||||||
|
const report = foldReport(second)
|
||||||
|
// Fell back — never to a shorter bound, and never silently.
|
||||||
|
expect(report.bound).not.toBe('checkpoint')
|
||||||
|
expect(report.seeded).toBe(0)
|
||||||
|
expect(report.fromGeneration).toBe(1) // no mark either: this brain never drained
|
||||||
|
// LOUD, two ways: the adapter's torn-record gauge and its production error…
|
||||||
|
expect(getTornRecordGauge().count).toBeGreaterThan(before)
|
||||||
|
expect(getTornRecordGauge().lastPath).toContain('pending_embeds_checkpoint')
|
||||||
|
expect(lines.some((l) => /TORN RECORD/.test(l))).toBe(true)
|
||||||
|
// …and the fold's own narration of which bound it actually used.
|
||||||
|
expect(lines.some((l) => /pending-embed fold: genesis bound/.test(l))).toBe(true)
|
||||||
|
|
||||||
|
// CORRECT: the marker is still recovered, from the log itself.
|
||||||
|
expect(pendingIds(second)).toEqual([stuck])
|
||||||
|
const full = await fullFold(second)
|
||||||
|
expect(full.ids).toEqual([stuck])
|
||||||
|
expect(report.factsScanned).toBe(full.facts)
|
||||||
|
}, 180_000)
|
||||||
|
|
||||||
|
it('a well-formed but shape-invalid checkpoint is refused whole, never partially trusted', async () => {
|
||||||
|
const root = dir()
|
||||||
|
const first = await open(root, { blockWorker: true })
|
||||||
|
const stuck = await first.add({
|
||||||
|
id: 'stuck',
|
||||||
|
data: 'never lands',
|
||||||
|
type: NounType.Thing,
|
||||||
|
deferEmbedding: true
|
||||||
|
})
|
||||||
|
await first.add({ id: 'other', data: 'ordinary row', type: NounType.Thing })
|
||||||
|
await first.close()
|
||||||
|
liveBrains.splice(liveBrains.indexOf(first), 1)
|
||||||
|
|
||||||
|
// A checkpoint with a plausible generation but a `pending` that is not a
|
||||||
|
// list of ids: trusting the generation alone would bound the scan behind a
|
||||||
|
// set that was never recovered — the exact shape that loses a vector.
|
||||||
|
const onDisk = artifactPath(root, CHECKPOINT_PATH)!
|
||||||
|
const good = readArtifact(root, CHECKPOINT_PATH) as { generation: number }
|
||||||
|
rmSync(onDisk)
|
||||||
|
writeFileSync(
|
||||||
|
join(root, '_system', 'pending_embeds_checkpoint.json'),
|
||||||
|
JSON.stringify({ generation: good.generation, pending: { stuck: true }, writtenAt: 1 })
|
||||||
|
)
|
||||||
|
|
||||||
|
const { result: second, lines } = await captureConsole(async () =>
|
||||||
|
open(root, { blockWorker: true })
|
||||||
|
)
|
||||||
|
expect(lines.some((l) => /pending-embed checkpoint REFUSED/.test(l))).toBe(true)
|
||||||
|
const report = foldReport(second)
|
||||||
|
expect(report.bound).not.toBe('checkpoint')
|
||||||
|
expect(report.seeded).toBe(0)
|
||||||
|
expect(pendingIds(second)).toEqual([stuck])
|
||||||
|
}, 180_000)
|
||||||
|
})
|
||||||
|
|
||||||
|
// ===========================================================================
|
||||||
|
// 3. The crash matrix — real processes, real SIGKILL, differential invariant
|
||||||
|
// ===========================================================================
|
||||||
|
|
||||||
|
describe('pending-embed checkpoint — crash matrix (real child process, SIGKILL)', () => {
|
||||||
|
/**
|
||||||
|
* The invariant every row shares: whatever the reopened brain's fold did with
|
||||||
|
* whatever bound survived the crash, its pending set must equal the truth a
|
||||||
|
* full fold from generation 1 derives from the SAME recovered log.
|
||||||
|
*/
|
||||||
|
async function assertDifferentialAfterCrash(root: string): Promise<{
|
||||||
|
report: FoldReport
|
||||||
|
full: { ids: string[]; facts: number }
|
||||||
|
pending: string[]
|
||||||
|
}> {
|
||||||
|
const reopened = await open(root, { blockWorker: true })
|
||||||
|
const report = foldReport(reopened)
|
||||||
|
const full = await fullFold(reopened)
|
||||||
|
const pending = pendingIds(reopened)
|
||||||
|
expect(pending).toEqual(full.ids)
|
||||||
|
return { report, full, pending }
|
||||||
|
}
|
||||||
|
|
||||||
|
it('killed BEFORE any checkpoint was written — falls back and recovers the marker from the log', async () => {
|
||||||
|
const root = dir()
|
||||||
|
const { child, output } = await spawnArranger(
|
||||||
|
root,
|
||||||
|
`${childPreamble(root)}
|
||||||
|
block()
|
||||||
|
await brain.init()
|
||||||
|
await brain.add({ id: 'landed-row', data: 'an ordinary row', type: 'thing' })
|
||||||
|
const stuck = await brain.add({ id: 'stuck-1', data: 'deferred, never lands', type: 'thing', deferEmbedding: true })
|
||||||
|
await brain.flush()
|
||||||
|
console.log('IDS:' + JSON.stringify({ stuck }))
|
||||||
|
console.log('READY')
|
||||||
|
setInterval(() => {}, 1000)
|
||||||
|
`
|
||||||
|
)
|
||||||
|
const ids = childIds(output())
|
||||||
|
// One enqueue is well under the cadence and the set never drained, so no
|
||||||
|
// checkpoint exists — this is the pre-checkpoint crash.
|
||||||
|
expect(readArtifact(root, CHECKPOINT_PATH)).toBeNull()
|
||||||
|
await sigkill(child)
|
||||||
|
|
||||||
|
const { report, pending } = await assertDifferentialAfterCrash(root)
|
||||||
|
expect(report.bound).toBe('genesis')
|
||||||
|
expect(pending).toEqual([ids.stuck])
|
||||||
|
}, 300_000)
|
||||||
|
|
||||||
|
it('killed AFTER a checkpoint, with an embed landed and flushed after it — the post-checkpoint facts carry the disarm', async () => {
|
||||||
|
const root = dir()
|
||||||
|
const { child, output } = await spawnArranger(
|
||||||
|
root,
|
||||||
|
`${childPreamble(root)}
|
||||||
|
await brain.init()
|
||||||
|
// Land one deferred embed: the drain arms the checkpoint debt.
|
||||||
|
await brain.add({ id: 'seed', data: 'lands first', type: 'thing', deferEmbedding: true })
|
||||||
|
await brain.awaitPendingEmbeds()
|
||||||
|
await brain.flush()
|
||||||
|
// A second deferred write pays the debt (the head is at the manifest now),
|
||||||
|
// then LANDS — its embed.landed rides a fact ABOVE the checkpoint.
|
||||||
|
const landsAfter = await brain.add({ id: 'lands-after', data: 'lands after the checkpoint', type: 'thing', deferEmbedding: true })
|
||||||
|
await settleCheckpoint()
|
||||||
|
await brain.awaitPendingEmbeds()
|
||||||
|
// …and one that never will.
|
||||||
|
block()
|
||||||
|
const stuck = await brain.add({ id: 'stuck-1', data: 'deferred, never lands', type: 'thing', deferEmbedding: true })
|
||||||
|
await brain.add({ id: 'plain', data: 'more history', type: 'thing' })
|
||||||
|
await brain.flush()
|
||||||
|
console.log('IDS:' + JSON.stringify({ stuck, landsAfter }))
|
||||||
|
console.log('READY')
|
||||||
|
setInterval(() => {}, 1000)
|
||||||
|
`
|
||||||
|
)
|
||||||
|
const ids = childIds(output())
|
||||||
|
const checkpoint = readArtifact(root, CHECKPOINT_PATH) as {
|
||||||
|
generation: number
|
||||||
|
pending: string[]
|
||||||
|
} | null
|
||||||
|
expect(checkpoint).not.toBeNull()
|
||||||
|
await sigkill(child)
|
||||||
|
|
||||||
|
const { report, full, pending } = await assertDifferentialAfterCrash(root)
|
||||||
|
expect(report.bound).toBe('checkpoint')
|
||||||
|
expect(report.fromGeneration).toBe(checkpoint!.generation + 1)
|
||||||
|
// The bound really bounded: fewer facts than the whole log.
|
||||||
|
expect(report.factsScanned).toBeLessThan(full.facts)
|
||||||
|
// A landed embed above the checkpoint is disarmed by the scan, not lost;
|
||||||
|
// the stuck one is re-armed.
|
||||||
|
expect(pending).toEqual([ids.stuck])
|
||||||
|
expect(pending).not.toContain(ids.landsAfter)
|
||||||
|
}, 300_000)
|
||||||
|
|
||||||
|
it('killed AFTER a checkpoint with an UN-FLUSHED tail — truncated facts and the bounded fold still agree', async () => {
|
||||||
|
const root = dir()
|
||||||
|
const { child } = await spawnArranger(
|
||||||
|
root,
|
||||||
|
`${childPreamble(root)}
|
||||||
|
await brain.init()
|
||||||
|
await brain.add({ id: 'seed', data: 'lands first', type: 'thing', deferEmbedding: true })
|
||||||
|
await brain.awaitPendingEmbeds()
|
||||||
|
await brain.flush()
|
||||||
|
await brain.add({ id: 'lands-after', data: 'lands after the checkpoint', type: 'thing', deferEmbedding: true })
|
||||||
|
await settleCheckpoint()
|
||||||
|
await brain.awaitPendingEmbeds()
|
||||||
|
await brain.flush()
|
||||||
|
// Now write PAST the manifest and never flush: these facts are the tail a
|
||||||
|
// crash truncates. Whatever survives, the two folds must agree on it.
|
||||||
|
block()
|
||||||
|
await brain.add({ id: 'stuck-tail', data: 'deferred, never lands', type: 'thing', deferEmbedding: true })
|
||||||
|
await brain.add({ id: 'plain-tail', data: 'unflushed history', type: 'thing' })
|
||||||
|
console.log('READY')
|
||||||
|
setInterval(() => {}, 1000)
|
||||||
|
`
|
||||||
|
)
|
||||||
|
const checkpoint = readArtifact(root, CHECKPOINT_PATH) as { generation: number } | null
|
||||||
|
expect(checkpoint).not.toBeNull()
|
||||||
|
await sigkill(child)
|
||||||
|
|
||||||
|
const { report } = await assertDifferentialAfterCrash(root)
|
||||||
|
// The checkpoint's generation is at or below the manifest by construction,
|
||||||
|
// so it survived the truncation and still bounds the fold.
|
||||||
|
expect(report.bound).toBe('checkpoint')
|
||||||
|
expect(report.fromGeneration).toBe(checkpoint!.generation + 1)
|
||||||
|
}, 300_000)
|
||||||
|
})
|
||||||
141
tests/integration/pending-embed-low-water.test.ts
Normal file
141
tests/integration/pending-embed-low-water.test.ts
Normal file
|
|
@ -0,0 +1,141 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/pending-embed-low-water
|
||||||
|
* @description The pending-embed recovery fold is bounded and background (10.4.9).
|
||||||
|
*
|
||||||
|
* The fold used to scan the generation log from generation 1 at EVERY open,
|
||||||
|
* on the open's foreground — O(whole history) per open on long-lived brains.
|
||||||
|
* Now: an advisory low-water mark (`_system/pending_embeds_lowwater.json`)
|
||||||
|
* records the committed generation whenever the pending set drains to empty,
|
||||||
|
* recovery scans from `mark + 1` on the open's foreground — the crash-recovery
|
||||||
|
* contract keeps markers re-armed when open() returns. The mark is advisory: stale-low costs a longer scan, never a
|
||||||
|
* marker — a pending embed enqueued before a crash is still recovered.
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, afterEach, vi } from 'vitest'
|
||||||
|
import { mkdtempSync, rmSync } from 'node:fs'
|
||||||
|
import { tmpdir } from 'node:os'
|
||||||
|
import { join } from 'node:path'
|
||||||
|
import { Brainy } from '../../src/brainy'
|
||||||
|
import { NounType } from '../../src/types/graphTypes'
|
||||||
|
|
||||||
|
const LOWWATER_PATH = '_system/pending_embeds_lowwater.json'
|
||||||
|
|
||||||
|
describe('pending-embed recovery: bounded by the low-water mark', () => {
|
||||||
|
const roots: string[] = []
|
||||||
|
const dir = (): string => {
|
||||||
|
const d = mkdtempSync(join(tmpdir(), 'brainy-lowwater-'))
|
||||||
|
roots.push(d)
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
const open = async (root: string): Promise<Brainy<any>> => {
|
||||||
|
const brain = new Brainy<any>({
|
||||||
|
requireSubtype: false,
|
||||||
|
storage: { type: 'filesystem', path: root }
|
||||||
|
})
|
||||||
|
await brain.init()
|
||||||
|
return brain
|
||||||
|
}
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
for (const d of roots.splice(0)) rmSync(d, { recursive: true, force: true })
|
||||||
|
})
|
||||||
|
|
||||||
|
it('drain-to-empty writes the mark, and the next open scans from mark + 1', async () => {
|
||||||
|
const root = dir()
|
||||||
|
const brain = await open(root)
|
||||||
|
// Hold the worker so the pending state is observable, then release it.
|
||||||
|
const realKick = (brain as any).kickEmbedWorker.bind(brain)
|
||||||
|
;(brain as any).kickEmbedWorker = () => {}
|
||||||
|
await brain.add({
|
||||||
|
id: 'row-1',
|
||||||
|
data: 'the first deferred row',
|
||||||
|
type: NounType.Thing,
|
||||||
|
deferEmbedding: true
|
||||||
|
})
|
||||||
|
expect(brain.pendingEmbedCount()).toBeGreaterThan(0)
|
||||||
|
;(brain as any).kickEmbedWorker = realKick
|
||||||
|
await brain.awaitPendingEmbeds()
|
||||||
|
// The drain wrote the advisory mark (fire-and-forget: settle the microtask).
|
||||||
|
await new Promise((r) => setTimeout(r, 50))
|
||||||
|
const mark = (await (brain as any).storage.readRawObject(LOWWATER_PATH)) as {
|
||||||
|
generation: number
|
||||||
|
} | null
|
||||||
|
expect(mark).not.toBeNull()
|
||||||
|
expect(mark!.generation).toBeGreaterThan(0)
|
||||||
|
await brain.close()
|
||||||
|
|
||||||
|
const brain2 = await open(root)
|
||||||
|
const log = (brain2 as any).generationStore.getFactLog()
|
||||||
|
const scanSpy = vi.spyOn(log, 'scanFacts')
|
||||||
|
try {
|
||||||
|
await (brain2 as any).recoverPendingEmbedsFromLog()
|
||||||
|
expect(scanSpy).toHaveBeenCalledTimes(1)
|
||||||
|
const opts = scanSpy.mock.calls[0][0] as { fromGeneration?: number }
|
||||||
|
expect(opts.fromGeneration).toBeGreaterThanOrEqual(mark!.generation + 1)
|
||||||
|
} finally {
|
||||||
|
scanSpy.mockRestore()
|
||||||
|
await brain2.close()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a pending embed enqueued after the mark survives an unclean stop', async () => {
|
||||||
|
const root = dir()
|
||||||
|
const brain = await open(root)
|
||||||
|
await brain.add({ id: 'settled', data: 'lands before the mark', type: NounType.Thing })
|
||||||
|
await brain.awaitPendingEmbeds()
|
||||||
|
await new Promise((r) => setTimeout(r, 50))
|
||||||
|
|
||||||
|
// A deferred write whose embed never lands: block the worker, then drop
|
||||||
|
// the instance without close() — the unclean-stop shape.
|
||||||
|
;(brain as any).kickEmbedWorker = () => {}
|
||||||
|
await brain.add({
|
||||||
|
id: 'orphan',
|
||||||
|
data: 'enqueued then abandoned',
|
||||||
|
type: NounType.Thing,
|
||||||
|
deferEmbedding: true
|
||||||
|
})
|
||||||
|
expect(brain.pendingEmbedCount()).toBeGreaterThan(0)
|
||||||
|
// No close(): simulate the crash by releasing only the writer lock so the
|
||||||
|
// next open can proceed.
|
||||||
|
await (brain as any).storage.releaseWriterLock()
|
||||||
|
|
||||||
|
const brain2 = await open(root)
|
||||||
|
expect(brain2.pendingEmbedCount()).toBeGreaterThan(0)
|
||||||
|
await brain2.awaitPendingEmbeds()
|
||||||
|
expect(brain2.pendingEmbedCount()).toBe(0)
|
||||||
|
await brain2.close()
|
||||||
|
// Reap the crashed instance: its fence is gone, so close() fails loudly —
|
||||||
|
// swallow that here; the point is clearing its watchers and registry entry.
|
||||||
|
await brain.close().catch(() => undefined)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a reopened brain has its pending set settled when open() returns', async () => {
|
||||||
|
const root = dir()
|
||||||
|
const brain = await open(root)
|
||||||
|
await brain.add({ id: 'a-row', data: 'some data', type: NounType.Thing })
|
||||||
|
await brain.awaitPendingEmbeds()
|
||||||
|
await brain.close()
|
||||||
|
|
||||||
|
const brain2 = await open(root)
|
||||||
|
// The crash-recovery contract: markers are re-armed by open itself —
|
||||||
|
// no latch, no background race. (Here the drain landed, so zero.)
|
||||||
|
expect(brain2.pendingEmbedCount()).toBe(0)
|
||||||
|
await brain2.close()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a clean close with an empty set writes the mark even if no drain happened', async () => {
|
||||||
|
const root = dir()
|
||||||
|
const brain = await open(root)
|
||||||
|
await brain.add({ id: 'r1', data: 'row one', type: NounType.Thing })
|
||||||
|
await brain.awaitPendingEmbeds()
|
||||||
|
await brain.close()
|
||||||
|
// Read the mark back through the storage door (the adapter owns the
|
||||||
|
// on-disk encoding), on a fresh instance.
|
||||||
|
const brain2 = await open(root)
|
||||||
|
const mark = (await (brain2 as any).storage.readRawObject(LOWWATER_PATH)) as {
|
||||||
|
generation: number
|
||||||
|
} | null
|
||||||
|
expect(mark).not.toBeNull()
|
||||||
|
expect(mark!.generation).toBeGreaterThan(0)
|
||||||
|
await brain2.close()
|
||||||
|
})
|
||||||
|
})
|
||||||
250
tests/integration/readonly-close-no-marker.test.ts
Normal file
250
tests/integration/readonly-close-no-marker.test.ts
Normal file
|
|
@ -0,0 +1,250 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/readonly-close-no-marker
|
||||||
|
* @description A READ-ONLY BRAIN WRITES NO CLEAN-SHUTDOWN EVIDENCE.
|
||||||
|
*
|
||||||
|
* `_system/clean-shutdown.json` is the WRITER's own word about the writer's
|
||||||
|
* own process: "everything above this line, from THIS session, is durable."
|
||||||
|
* Two call sites treated a reader exactly like a writer:
|
||||||
|
*
|
||||||
|
* 1. `Brainy#closeDurableSteps()` called `generationStore.close()`
|
||||||
|
* unconditionally — a reader's close re-stamped the marker at the
|
||||||
|
* generation the reader merely OBSERVED, never committed.
|
||||||
|
* 2. `GenerationStore#open()` consumed (deleted) the marker on every open,
|
||||||
|
* reader or writer alike, so a reader that never got to a matching
|
||||||
|
* close left the store looking crashed to the next writer.
|
||||||
|
*
|
||||||
|
* Both are fixed by making a read-only brain leave `_system/` exactly as it
|
||||||
|
* found it — at open AND at close. Pinned here:
|
||||||
|
*
|
||||||
|
* 1. `_system/` is byte-for-byte identical (file set + contents) before and
|
||||||
|
* after a reader opens a cleanly-closed store, reads it, and closes.
|
||||||
|
* 2. After the reader's close, the next WRITER open adopts the marker as
|
||||||
|
* clean — no recovery fold narrates.
|
||||||
|
* 3. A reader creates no file under `_system/` merely by opening (before it
|
||||||
|
* ever closes).
|
||||||
|
* 4. A reader that opens and is then abandoned (crash-style, no close) does
|
||||||
|
* not force the next writer to pay a recovery fold — the concrete harm
|
||||||
|
* the fix closes.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
|
import { mkdtempSync, rmSync, readdirSync, readFileSync, statSync } from 'node:fs'
|
||||||
|
import { createHash } from 'node:crypto'
|
||||||
|
import { tmpdir } from 'node:os'
|
||||||
|
import { join } from 'node:path'
|
||||||
|
import { Brainy } from '../../src/brainy.js'
|
||||||
|
import { NounType } from '../../src/types/graphTypes.js'
|
||||||
|
import { abandonAsCrashed } from '../helpers/durabilityKillMatrix.js'
|
||||||
|
|
||||||
|
function makeTempDir(): string {
|
||||||
|
return mkdtempSync(join(tmpdir(), 'brainy-readonly-close-'))
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Recursively hash every regular file under `dir`, keyed by its path relative to `dir`. */
|
||||||
|
function snapshotDir(dir: string): Map<string, string> {
|
||||||
|
const out = new Map<string, string>()
|
||||||
|
const walk = (rel: string): void => {
|
||||||
|
const abs = rel ? join(dir, rel) : dir
|
||||||
|
let entries: string[]
|
||||||
|
try {
|
||||||
|
entries = readdirSync(abs)
|
||||||
|
} catch {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for (const name of entries) {
|
||||||
|
const childRel = rel ? join(rel, name) : name
|
||||||
|
const childAbs = join(dir, childRel)
|
||||||
|
const st = statSync(childAbs)
|
||||||
|
if (st.isDirectory()) {
|
||||||
|
walk(childRel)
|
||||||
|
} else if (st.isFile()) {
|
||||||
|
const hash = createHash('sha256').update(readFileSync(childAbs)).digest('hex')
|
||||||
|
out.set(childRel, hash)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
walk('')
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Capture console.warn lines (the narration channel — see `prodLog.narrate`) while `fn` runs. */
|
||||||
|
async function captureWarn<T>(fn: () => Promise<T>): Promise<{ result: T; lines: string[] }> {
|
||||||
|
const lines: string[] = []
|
||||||
|
const orig = console.warn
|
||||||
|
console.warn = ((...args: unknown[]) => {
|
||||||
|
lines.push(args.map((a) => String(a)).join(' '))
|
||||||
|
}) as typeof console.warn
|
||||||
|
try {
|
||||||
|
return { result: await fn(), lines }
|
||||||
|
} finally {
|
||||||
|
console.warn = orig
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('a read-only brain writes no clean-shutdown evidence', () => {
|
||||||
|
let dir: string
|
||||||
|
let brain: Brainy | null = null
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
dir = makeTempDir()
|
||||||
|
})
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
if (brain) {
|
||||||
|
try {
|
||||||
|
await brain.close()
|
||||||
|
} catch {
|
||||||
|
/* already closed */
|
||||||
|
}
|
||||||
|
brain = null
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
rmSync(dir, { recursive: true, force: true })
|
||||||
|
} catch {
|
||||||
|
/* ignore */
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
const systemDir = () => join(dir, '_system')
|
||||||
|
/**
|
||||||
|
* The marker file's actual on-disk name — `clean-shutdown.json` or, under
|
||||||
|
* FileSystemStorage's default gzip compression, `clean-shutdown.json.gz`.
|
||||||
|
* Returns null when absent.
|
||||||
|
*/
|
||||||
|
const findMarkerPath = (): string | null => {
|
||||||
|
let entries: string[]
|
||||||
|
try {
|
||||||
|
entries = readdirSync(systemDir())
|
||||||
|
} catch {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
const name = entries.find((n) => n.startsWith('clean-shutdown.json'))
|
||||||
|
return name ? join(systemDir(), name) : null
|
||||||
|
}
|
||||||
|
|
||||||
|
it('leaves `_system/`\'s file set and the clean-shutdown marker\'s bytes identical across a reader open → read → close', async () => {
|
||||||
|
// A writer opens, writes, and closes cleanly — the marker lands at
|
||||||
|
// whatever generation the writer actually committed.
|
||||||
|
const writer = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||||
|
await writer.init()
|
||||||
|
await writer.add({ data: 'seed entity', type: NounType.Concept })
|
||||||
|
await writer.add({ data: 'second entity', type: NounType.Concept })
|
||||||
|
await writer.flush()
|
||||||
|
await writer.close()
|
||||||
|
|
||||||
|
const markerBeforePath = findMarkerPath()
|
||||||
|
expect(markerBeforePath, 'the writer left a clean-shutdown marker').not.toBeNull()
|
||||||
|
const before = snapshotDir(systemDir())
|
||||||
|
expect(before.size).toBeGreaterThan(0)
|
||||||
|
const markerBeforeHash = before.get(
|
||||||
|
(markerBeforePath as string).slice(systemDir().length + 1)
|
||||||
|
)
|
||||||
|
expect(markerBeforeHash).toBeTruthy()
|
||||||
|
|
||||||
|
// A reader opens the same store, reads, and closes.
|
||||||
|
brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } })
|
||||||
|
expect(brain.isReadOnly).toBe(true)
|
||||||
|
await brain.stats()
|
||||||
|
await brain.close()
|
||||||
|
brain = null
|
||||||
|
|
||||||
|
// The FILE SET under `_system/` is unchanged — a reader creates and
|
||||||
|
// removes nothing. This pin is specifically about the generation store's
|
||||||
|
// clean-shutdown evidence. The wider law — that a reader leaves EVERY
|
||||||
|
// file under `_system/` byte-identical, which this fix left open as a
|
||||||
|
// known residual (the metadata field registry and the three statistics
|
||||||
|
// files were still re-stamped by a reader's close) — is closed and pinned
|
||||||
|
// in `readonly-close-writes-nothing.test.ts`.
|
||||||
|
const after = snapshotDir(systemDir())
|
||||||
|
expect([...after.keys()].sort()).toEqual([...before.keys()].sort())
|
||||||
|
|
||||||
|
// The MARKER's bytes are byte-for-byte identical — the reader neither
|
||||||
|
// consumed it at open nor re-stamped it at close.
|
||||||
|
const markerAfterPath = findMarkerPath()
|
||||||
|
expect(markerAfterPath, 'the marker must still exist, under the same name').toBe(markerBeforePath)
|
||||||
|
const markerAfterHash = after.get((markerAfterPath as string).slice(systemDir().length + 1))
|
||||||
|
expect(markerAfterHash).toBe(markerBeforeHash)
|
||||||
|
}, 120_000)
|
||||||
|
|
||||||
|
it('creates no file under `_system/` merely by opening read-only', async () => {
|
||||||
|
const writer = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||||
|
await writer.init()
|
||||||
|
await writer.add({ data: 'seed entity', type: NounType.Concept })
|
||||||
|
await writer.flush()
|
||||||
|
await writer.close()
|
||||||
|
|
||||||
|
const baselineNames = [...snapshotDir(systemDir()).keys()].sort()
|
||||||
|
expect(baselineNames.length).toBeGreaterThan(0)
|
||||||
|
|
||||||
|
// Open the reader and inspect `_system/` BEFORE it ever closes — open()
|
||||||
|
// alone must create nothing.
|
||||||
|
brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } })
|
||||||
|
const whileOpenNames = [...snapshotDir(systemDir()).keys()].sort()
|
||||||
|
expect(whileOpenNames).toEqual(baselineNames)
|
||||||
|
|
||||||
|
await brain.close()
|
||||||
|
brain = null
|
||||||
|
}, 120_000)
|
||||||
|
|
||||||
|
it('a writer reopening after the reader closes adopts the marker — no recovery fold', async () => {
|
||||||
|
const writer1 = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||||
|
await writer1.init()
|
||||||
|
await writer1.add({ data: 'seed entity', type: NounType.Concept })
|
||||||
|
await writer1.flush()
|
||||||
|
await writer1.close()
|
||||||
|
|
||||||
|
// A reader opens and closes in between — must not disturb the marker.
|
||||||
|
const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } })
|
||||||
|
await reader.stats()
|
||||||
|
await reader.close()
|
||||||
|
|
||||||
|
// The next writer open must be a clean, no-fold open: no
|
||||||
|
// "log-authority recovery" / "WHOLE-LOG fold" narration line.
|
||||||
|
const { result: writer2, lines } = await captureWarn(async () => {
|
||||||
|
const w = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||||
|
await w.init()
|
||||||
|
return w
|
||||||
|
})
|
||||||
|
brain = writer2
|
||||||
|
|
||||||
|
const foldLines = lines.filter((l) => /log-authority recovery|WHOLE-LOG fold|recovery fold/i.test(l))
|
||||||
|
expect(foldLines, `unexpected recovery narration:\n${foldLines.join('\n')}`).toEqual([])
|
||||||
|
|
||||||
|
// And the store is exactly what the first writer left — the seed row is
|
||||||
|
// still there, nothing was rolled back or re-derived.
|
||||||
|
const found = await writer2.find({ where: {} } as any)
|
||||||
|
expect(found.length).toBeGreaterThanOrEqual(1)
|
||||||
|
}, 120_000)
|
||||||
|
|
||||||
|
it('a reader that opens and is then abandoned (never closes) does not force the next writer to fold', async () => {
|
||||||
|
// This is the concrete harm the fix closes: pre-fix, a reader's open()
|
||||||
|
// unconditionally DELETED the marker (consuming it as if it were the
|
||||||
|
// writer). A reader that opened and then died — no close, exactly like
|
||||||
|
// a killed process — left the marker gone, so the actual writer's next
|
||||||
|
// open read the store as crashed and paid a full recovery fold for a
|
||||||
|
// "crash" that was really just a reader that came and went.
|
||||||
|
const writer1 = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||||
|
await writer1.init()
|
||||||
|
await writer1.add({ data: 'seed entity', type: NounType.Concept })
|
||||||
|
await writer1.flush()
|
||||||
|
await writer1.close()
|
||||||
|
|
||||||
|
const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } })
|
||||||
|
await reader.stats()
|
||||||
|
// NEVER calls reader.close() — abandon it exactly like a killed process.
|
||||||
|
await abandonAsCrashed(reader)
|
||||||
|
|
||||||
|
const { result: writer2, lines } = await captureWarn(async () => {
|
||||||
|
const w = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||||
|
await w.init()
|
||||||
|
return w
|
||||||
|
})
|
||||||
|
brain = writer2
|
||||||
|
|
||||||
|
const foldLines = lines.filter((l) => /log-authority recovery|WHOLE-LOG fold|recovery fold/i.test(l))
|
||||||
|
expect(
|
||||||
|
foldLines,
|
||||||
|
`an abandoned READER forced a recovery fold on the next writer open:\n${foldLines.join('\n')}`
|
||||||
|
).toEqual([])
|
||||||
|
}, 120_000)
|
||||||
|
})
|
||||||
261
tests/integration/readonly-close-writes-nothing.test.ts
Normal file
261
tests/integration/readonly-close-writes-nothing.test.ts
Normal file
|
|
@ -0,0 +1,261 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/readonly-close-writes-nothing
|
||||||
|
* @description A READ-ONLY BRAIN LEAVES `_system/` BYTE-IDENTICAL — the WHOLE
|
||||||
|
* directory, not just the clean-shutdown marker.
|
||||||
|
*
|
||||||
|
* `readonly-close-no-marker` closed the marker half of this law and named the
|
||||||
|
* rest as a known, out-of-scope residual:
|
||||||
|
*
|
||||||
|
* "Other files under `_system/` — e.g. the metadata field registry, which
|
||||||
|
* stamps its own `lastUpdated` on every persist — are a pre-existing,
|
||||||
|
* separate concern outside this fix's scope."
|
||||||
|
*
|
||||||
|
* This is that residual, closed. MEASURED on the base before the fix, a
|
||||||
|
* read-only open → read → close rewrote FOUR files:
|
||||||
|
*
|
||||||
|
* _system/__metadata_field_registry__.json.gz
|
||||||
|
* _system/type-statistics.json.gz
|
||||||
|
* _system/subtype-statistics.json.gz
|
||||||
|
* _system/verb-subtype-statistics.json.gz
|
||||||
|
*
|
||||||
|
* THE CAUSE was not the closes the marker fix guarded — it was Phase 1 of
|
||||||
|
* `closeDurableSteps`, where every component flush ran unconditionally. A flush
|
||||||
|
* is a write by definition: `MetadataIndexManager#flush()` saves the field
|
||||||
|
* registry "even with no dirty fields" (its own comment), and the storage
|
||||||
|
* adapter's count flush re-stamps the three statistics files. A session that
|
||||||
|
* committed nothing re-stamped all four. Phase 2's closes were ungated too —
|
||||||
|
* the graph index's close drains both LSM MemTables and stamps a watermark,
|
||||||
|
* and the optional vector/metadata `close` hooks (unimplemented in the
|
||||||
|
* reference engine, filled in by a native provider) persist buffered state.
|
||||||
|
*
|
||||||
|
* THE LAW. A reader writes nothing, anywhere under `_system/`, at open or at
|
||||||
|
* close. It still RELEASES what it holds: the graph index's auto-flush interval
|
||||||
|
* is cleared through `stopBackgroundFlush()`, the non-writing half of its
|
||||||
|
* close, so nothing outlives the session.
|
||||||
|
*
|
||||||
|
* WHY IT MATTERS beyond tidiness: `_system/` is where a store keeps its
|
||||||
|
* evidence about itself — what the writer committed, what the projections have
|
||||||
|
* seen. A reader that rewrites any of it is vouching for a state it only
|
||||||
|
* observed, and on shared or snapshot storage it mutates bytes another process
|
||||||
|
* owns.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
|
import { mkdtempSync, rmSync, readdirSync, readFileSync, statSync } from 'node:fs'
|
||||||
|
import { createHash } from 'node:crypto'
|
||||||
|
import { tmpdir } from 'node:os'
|
||||||
|
import { join } from 'node:path'
|
||||||
|
import { Brainy } from '../../src/brainy.js'
|
||||||
|
import { NounType, VerbType } from '../../src/types/graphTypes.js'
|
||||||
|
|
||||||
|
/** Recursively hash every regular file under `dir`, keyed by its path relative to `dir`. */
|
||||||
|
function snapshotDir(dir: string): Map<string, string> {
|
||||||
|
const out = new Map<string, string>()
|
||||||
|
const walk = (rel: string): void => {
|
||||||
|
const abs = rel ? join(dir, rel) : dir
|
||||||
|
let entries: string[]
|
||||||
|
try {
|
||||||
|
entries = readdirSync(abs)
|
||||||
|
} catch {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for (const name of entries) {
|
||||||
|
const childRel = rel ? join(rel, name) : name
|
||||||
|
const childAbs = join(dir, childRel)
|
||||||
|
const st = statSync(childAbs)
|
||||||
|
if (st.isDirectory()) {
|
||||||
|
walk(childRel)
|
||||||
|
} else if (st.isFile()) {
|
||||||
|
out.set(childRel, createHash('sha256').update(readFileSync(childAbs)).digest('hex'))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
walk('')
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Every path where `after` differs from `before`, labelled — the failure message. */
|
||||||
|
function diff(before: Map<string, string>, after: Map<string, string>): string[] {
|
||||||
|
const lines: string[] = []
|
||||||
|
for (const [path, hash] of after) {
|
||||||
|
if (!before.has(path)) lines.push(`ADDED ${path}`)
|
||||||
|
else if (before.get(path) !== hash) lines.push(`CHANGED ${path}`)
|
||||||
|
}
|
||||||
|
for (const path of before.keys()) if (!after.has(path)) lines.push(`REMOVED ${path}`)
|
||||||
|
return lines.sort()
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('a read-only brain writes nothing under `_system/`', () => {
|
||||||
|
let dir: string
|
||||||
|
let brain: Brainy | null = null
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
dir = mkdtempSync(join(tmpdir(), 'brainy-readonly-writes-'))
|
||||||
|
})
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
if (brain) {
|
||||||
|
try {
|
||||||
|
await brain.close()
|
||||||
|
} catch {
|
||||||
|
/* already closed */
|
||||||
|
}
|
||||||
|
brain = null
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
rmSync(dir, { recursive: true, force: true })
|
||||||
|
} catch {
|
||||||
|
/* ignore */
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
const systemDir = () => join(dir, '_system')
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A writer seeds a store with nouns, verbs and queryable metadata — enough
|
||||||
|
* that the field registry, the statistics files and the graph index all hold
|
||||||
|
* real content — then closes cleanly.
|
||||||
|
*/
|
||||||
|
async function seedStore(): Promise<void> {
|
||||||
|
const writer = new Brainy({
|
||||||
|
requireSubtype: false,
|
||||||
|
storage: { type: 'filesystem', path: dir }
|
||||||
|
})
|
||||||
|
await writer.init()
|
||||||
|
for (let i = 0; i < 6; i++) {
|
||||||
|
await writer.add({
|
||||||
|
id: `seed-${i}`,
|
||||||
|
data: `seed entity ${i}`,
|
||||||
|
type: i % 2 === 0 ? NounType.Concept : NounType.Document,
|
||||||
|
metadata: { lane: i % 2 === 0 ? 'alpha' : 'beta', rank: i, tags: [`t${i}`, 'shared'] },
|
||||||
|
vector: []
|
||||||
|
})
|
||||||
|
}
|
||||||
|
for (let i = 1; i < 6; i++) {
|
||||||
|
await writer.relate({ from: 'seed-0', to: `seed-${i}`, type: VerbType.RelatedTo })
|
||||||
|
}
|
||||||
|
await writer.flush()
|
||||||
|
await writer.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
it('open → read → close leaves every file under `_system/` byte-identical', async () => {
|
||||||
|
await seedStore()
|
||||||
|
|
||||||
|
const before = snapshotDir(systemDir())
|
||||||
|
expect(before.size, 'the writer left a populated `_system/`').toBeGreaterThan(0)
|
||||||
|
|
||||||
|
brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } })
|
||||||
|
expect(brain.isReadOnly).toBe(true)
|
||||||
|
|
||||||
|
// Exercise the read surface that drives each subsystem: statistics (counts),
|
||||||
|
// a metadata filter (field index + registry), a graph walk (adjacency), a
|
||||||
|
// vector search, and a direct get.
|
||||||
|
await brain.stats()
|
||||||
|
await brain.find({ where: { lane: 'alpha' }, limit: 10 } as any)
|
||||||
|
await brain.find({ where: { tags: 'shared' }, limit: 10 } as any)
|
||||||
|
await brain.find({ connected: { from: 'seed-0', direction: 'out' }, limit: 10 } as any)
|
||||||
|
await brain.get('seed-1')
|
||||||
|
|
||||||
|
await brain.close()
|
||||||
|
brain = null
|
||||||
|
|
||||||
|
const after = snapshotDir(systemDir())
|
||||||
|
const changes = diff(before, after)
|
||||||
|
expect(changes, `a reader modified \`_system/\`:\n${changes.join('\n')}`).toEqual([])
|
||||||
|
}, 120_000)
|
||||||
|
|
||||||
|
it('names the four files that used to change — the measured shape of the defect', async () => {
|
||||||
|
await seedStore()
|
||||||
|
const before = snapshotDir(systemDir())
|
||||||
|
|
||||||
|
// These are the exact paths the base rewrote. Naming them keeps the pin
|
||||||
|
// honest about what it caught: if a future change reintroduces the write,
|
||||||
|
// the test above fails and this one says which subsystem did it.
|
||||||
|
const previouslyRewritten = [
|
||||||
|
'__metadata_field_registry__.json.gz',
|
||||||
|
'type-statistics.json.gz',
|
||||||
|
'subtype-statistics.json.gz',
|
||||||
|
'verb-subtype-statistics.json.gz'
|
||||||
|
]
|
||||||
|
for (const name of previouslyRewritten) {
|
||||||
|
expect(before.has(name), `fixture must contain ${name}`).toBe(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } })
|
||||||
|
await brain.stats()
|
||||||
|
await brain.find({ where: { lane: 'alpha' }, limit: 10 } as any)
|
||||||
|
await brain.close()
|
||||||
|
brain = null
|
||||||
|
|
||||||
|
const after = snapshotDir(systemDir())
|
||||||
|
for (const name of previouslyRewritten) {
|
||||||
|
expect(after.get(name), `${name} was rewritten by a reader`).toBe(before.get(name))
|
||||||
|
}
|
||||||
|
}, 120_000)
|
||||||
|
|
||||||
|
it('a reader that only opens and closes — touching nothing — writes nothing', async () => {
|
||||||
|
await seedStore()
|
||||||
|
const before = snapshotDir(systemDir())
|
||||||
|
|
||||||
|
brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } })
|
||||||
|
await brain.close()
|
||||||
|
brain = null
|
||||||
|
|
||||||
|
const changes = diff(before, snapshotDir(systemDir()))
|
||||||
|
expect(changes, `an idle reader modified \`_system/\`:\n${changes.join('\n')}`).toEqual([])
|
||||||
|
}, 120_000)
|
||||||
|
|
||||||
|
it('two readers in sequence each leave the store exactly as they found it', async () => {
|
||||||
|
await seedStore()
|
||||||
|
const before = snapshotDir(systemDir())
|
||||||
|
|
||||||
|
for (let i = 0; i < 2; i++) {
|
||||||
|
const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } })
|
||||||
|
await reader.find({ where: { lane: 'beta' }, limit: 10 } as any)
|
||||||
|
await reader.close()
|
||||||
|
const changes = diff(before, snapshotDir(systemDir()))
|
||||||
|
expect(changes, `reader ${i + 1} modified \`_system/\`:\n${changes.join('\n')}`).toEqual([])
|
||||||
|
}
|
||||||
|
}, 120_000)
|
||||||
|
|
||||||
|
it('the store outside `_system/` is untouched too — a reader writes nowhere', async () => {
|
||||||
|
await seedStore()
|
||||||
|
const before = snapshotDir(dir)
|
||||||
|
|
||||||
|
brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } })
|
||||||
|
await brain.stats()
|
||||||
|
await brain.find({ where: { lane: 'alpha' }, limit: 10 } as any)
|
||||||
|
await brain.close()
|
||||||
|
brain = null
|
||||||
|
|
||||||
|
const changes = diff(before, snapshotDir(dir))
|
||||||
|
expect(changes, `a reader modified the store:\n${changes.join('\n')}`).toEqual([])
|
||||||
|
}, 120_000)
|
||||||
|
|
||||||
|
it('a WRITER still persists on close — the guard did not disarm the write path', async () => {
|
||||||
|
await seedStore()
|
||||||
|
const before = snapshotDir(systemDir())
|
||||||
|
|
||||||
|
const writer = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||||
|
await writer.init()
|
||||||
|
await writer.add({
|
||||||
|
id: 'after-reader',
|
||||||
|
data: 'a new row',
|
||||||
|
type: NounType.Concept,
|
||||||
|
metadata: { lane: 'gamma', rank: 99 },
|
||||||
|
vector: []
|
||||||
|
})
|
||||||
|
await writer.close()
|
||||||
|
|
||||||
|
// The writer's close DID move `_system/` — that is the whole point of the
|
||||||
|
// asymmetry, and the guard must not have flattened it.
|
||||||
|
expect(diff(before, snapshotDir(systemDir())).length).toBeGreaterThan(0)
|
||||||
|
|
||||||
|
// And the row is really there on the next open.
|
||||||
|
const reopened = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||||
|
await reopened.init()
|
||||||
|
brain = reopened
|
||||||
|
const hits = await reopened.find({ where: { lane: 'gamma' }, limit: 10 } as any)
|
||||||
|
expect(hits.length).toBe(1)
|
||||||
|
}, 120_000)
|
||||||
|
})
|
||||||
90
tests/integration/related-verb-array.test.ts
Normal file
90
tests/integration/related-verb-array.test.ts
Normal file
|
|
@ -0,0 +1,90 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/related-verb-array
|
||||||
|
* @description related() honours EVERY verb type in an array (10.4.9).
|
||||||
|
*
|
||||||
|
* The storage fast paths for `sourceId + verbType` and `verbType` collapsed a
|
||||||
|
* verb-type ARRAY to its first element — `related({ from, type: [a, b] })`
|
||||||
|
* silently returned only `a` edges, whichever order the array came in. The
|
||||||
|
* same quiet-loss class as the graph-first paging defect, one seam over.
|
||||||
|
* These pins seed a store where the SECOND requested type's edge must come
|
||||||
|
* back, on every path the collapse lived in.
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, beforeAll, afterAll } from 'vitest'
|
||||||
|
import { Brainy } from '../../src/brainy'
|
||||||
|
import { NounType, VerbType } from '../../src/types/graphTypes'
|
||||||
|
import { v5 } from '../../src/universal/uuid'
|
||||||
|
|
||||||
|
describe('related() with a verb-type array returns every requested type', () => {
|
||||||
|
let brain: Brainy<any>
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||||
|
await brain.init()
|
||||||
|
for (const id of ['a', 'b', 'c', 'd']) {
|
||||||
|
await brain.add({ id, data: `node ${id}`, type: NounType.Person })
|
||||||
|
}
|
||||||
|
await brain.relate({ from: 'a', to: 'b', type: VerbType.Supports })
|
||||||
|
await brain.relate({ from: 'a', to: 'c', type: VerbType.RelatedTo })
|
||||||
|
await brain.relate({ from: 'a', to: 'd', type: VerbType.Knows })
|
||||||
|
await brain.relate({ from: 'b', to: 'c', type: VerbType.RelatedTo })
|
||||||
|
})
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await brain.close()
|
||||||
|
brain = null as any
|
||||||
|
})
|
||||||
|
|
||||||
|
it('from + type array: the second type\'s edge comes back, both orders', async () => {
|
||||||
|
for (const types of [
|
||||||
|
[VerbType.Supports, VerbType.RelatedTo],
|
||||||
|
[VerbType.RelatedTo, VerbType.Supports]
|
||||||
|
]) {
|
||||||
|
const edges = await brain.related({ from: 'a', type: types })
|
||||||
|
const targets = new Set(edges.map((e) => e.to))
|
||||||
|
expect(targets.has(v5('b')), `types [${types}] missing Supports edge`).toBe(true)
|
||||||
|
expect(targets.has(v5('c')), `types [${types}] missing RelatedTo edge`).toBe(true)
|
||||||
|
expect(targets.has(v5('d'))).toBe(false)
|
||||||
|
expect(edges).toHaveLength(2)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a single-element array behaves exactly like the scalar', async () => {
|
||||||
|
const scalar = await brain.related({ from: 'a', type: VerbType.Supports })
|
||||||
|
const array = await brain.related({ from: 'a', type: [VerbType.Supports] })
|
||||||
|
expect(array.map((e) => e.id).sort()).toEqual(scalar.map((e) => e.id).sort())
|
||||||
|
expect(array).toHaveLength(1)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('no duplicate edges when types overlap the same edge set', async () => {
|
||||||
|
const edges = await brain.related({
|
||||||
|
from: 'a',
|
||||||
|
type: [VerbType.Supports, VerbType.RelatedTo, VerbType.Knows]
|
||||||
|
})
|
||||||
|
const ids = edges.map((e) => e.id)
|
||||||
|
expect(new Set(ids).size).toBe(ids.length)
|
||||||
|
expect(edges).toHaveLength(3)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('type-only asks (no anchor) honour the whole array too', async () => {
|
||||||
|
const edges = await brain.related({ type: [VerbType.Supports, VerbType.Knows] })
|
||||||
|
const verbs = new Set(edges.map((e) => e.type))
|
||||||
|
expect(verbs.has(VerbType.Supports)).toBe(true)
|
||||||
|
expect(verbs.has(VerbType.Knows)).toBe(true)
|
||||||
|
expect(edges).toHaveLength(2)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('to + type array: the target side honours every type too', async () => {
|
||||||
|
const edges = await brain.related({ to: 'c', type: [VerbType.RelatedTo, VerbType.Supports] })
|
||||||
|
const froms = new Set(edges.map((e) => e.from))
|
||||||
|
expect(froms.has(v5('a'))).toBe(true)
|
||||||
|
expect(froms.has(v5('b'))).toBe(true)
|
||||||
|
expect(edges).toHaveLength(2)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('pagination stays consistent across the union', async () => {
|
||||||
|
const page1 = await brain.related({ from: 'a', type: [VerbType.Supports, VerbType.RelatedTo, VerbType.Knows], limit: 2 })
|
||||||
|
const page2 = await brain.related({ from: 'a', type: [VerbType.Supports, VerbType.RelatedTo, VerbType.Knows], limit: 2, offset: 2 })
|
||||||
|
const all = [...page1, ...page2].map((e) => e.id)
|
||||||
|
expect(new Set(all).size).toBe(3)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
@ -59,7 +59,8 @@ describe('Relationship Intelligence', () => {
|
||||||
await brain.init()
|
await brain.init()
|
||||||
})
|
})
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(async () => {
|
||||||
|
await brain.close()
|
||||||
if (fs.existsSync(testDir)) {
|
if (fs.existsSync(testDir)) {
|
||||||
fs.rmSync(testDir, { recursive: true })
|
fs.rmSync(testDir, { recursive: true })
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,7 @@
|
||||||
* - addMany({ ifAbsent: true }) applies the flag to every item
|
* - addMany({ ifAbsent: true }) applies the flag to every item
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { describe, it, expect, beforeEach } from 'vitest'
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
import { Brainy } from '../../src/brainy.js'
|
import { Brainy } from '../../src/brainy.js'
|
||||||
import { RevisionConflictError } from '../../src/transaction/RevisionConflictError.js'
|
import { RevisionConflictError } from '../../src/transaction/RevisionConflictError.js'
|
||||||
import { NounType } from '../../src/types/graphTypes.js'
|
import { NounType } from '../../src/types/graphTypes.js'
|
||||||
|
|
@ -22,6 +22,10 @@ describe('7.31.0 — _rev CAS + ifAbsent', () => {
|
||||||
await brain.init()
|
await brain.init()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
|
||||||
describe('_rev initialization + surface', () => {
|
describe('_rev initialization + surface', () => {
|
||||||
it('initializes _rev to 1 on add()', async () => {
|
it('initializes _rev to 1 on add()', async () => {
|
||||||
const id = await brain.add({ data: 'hello', type: NounType.Document })
|
const id = await brain.add({ data: 'hello', type: NounType.Document })
|
||||||
|
|
|
||||||
405
tests/integration/shutdown-single-owner.test.ts
Normal file
405
tests/integration/shutdown-single-owner.test.ts
Normal file
|
|
@ -0,0 +1,405 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/shutdown-single-owner
|
||||||
|
* @description ONE SHUTDOWN, ONE OWNER.
|
||||||
|
*
|
||||||
|
* MEASURED IN PRODUCTION. A host that owns its own shutdown — one SIGTERM
|
||||||
|
* listener calling `close()` on every pooled store — ran head-on into the
|
||||||
|
* engine's own signal handler, which iterated every live instance, flushed its
|
||||||
|
* components in parallel, and released its writer lock in a `finally`. Two
|
||||||
|
* teardowns of the same brain at the same moment. The log shape:
|
||||||
|
*
|
||||||
|
* "Shutdown signal received - flushing pending data..." (SIGTERM)
|
||||||
|
* ...148 seconds of silence...
|
||||||
|
* "Flushed successfully (1 instance)"
|
||||||
|
* ...the host's pool close of that same store returns 1s later
|
||||||
|
*
|
||||||
|
* 149s for the one store with engine work in flight, against 24s for its six
|
||||||
|
* idle siblings. The same race in a local reproduction printed
|
||||||
|
* `Failed to flush one Brainy instance on shutdown: Writer fence lost … the
|
||||||
|
* lock file is gone` — the handler observing a lock the close it was racing
|
||||||
|
* had already released.
|
||||||
|
*
|
||||||
|
* The contract pinned here:
|
||||||
|
* (a) A host owner and the engine's hooks both live: EXACTLY ONE close runs
|
||||||
|
* per brain, no fence is lost, both durability markers are written, the
|
||||||
|
* process exits 0, and the reopen adopts rather than folding.
|
||||||
|
* (b) No host owner: the engine's handler closes every instance by the same
|
||||||
|
* `close()` path — markers written, clean exit.
|
||||||
|
* (c) `close()` is idempotent and re-entrant: concurrent callers share ONE
|
||||||
|
* execution and all of them settle.
|
||||||
|
* (d) Flush is single-flight: N kicks during a running flush arm exactly one
|
||||||
|
* follow-up, and two flush bodies never overlap.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
|
import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync } from 'node:fs'
|
||||||
|
import { spawn } from 'node:child_process'
|
||||||
|
import { tmpdir } from 'node:os'
|
||||||
|
import { join } from 'node:path'
|
||||||
|
import { Brainy } from '../../src/brainy.js'
|
||||||
|
import { NounType } from '../../src/types/graphTypes.js'
|
||||||
|
|
||||||
|
const REPO_ROOT = process.cwd()
|
||||||
|
const TSX = join(REPO_ROOT, 'node_modules', '.bin', 'tsx')
|
||||||
|
const BRAINY_SRC = join(REPO_ROOT, 'src', 'brainy.ts')
|
||||||
|
|
||||||
|
function makeTempDir(prefix: string): string {
|
||||||
|
return mkdtempSync(join(tmpdir(), prefix))
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The writer lock's clean-close record — written by `releaseWriterLock()`. */
|
||||||
|
const closeRecordPath = (dir: string) => join(dir, 'locks', '_writer.close')
|
||||||
|
/**
|
||||||
|
* The generation store's clean-shutdown marker — the adopt-vs-fold gate.
|
||||||
|
* (`FileSystemStorage` gzips raw objects, so the file on disk carries `.gz`;
|
||||||
|
* both spellings are accepted so the pin survives a compression change.)
|
||||||
|
*/
|
||||||
|
const cleanShutdownWritten = (dir: string) =>
|
||||||
|
existsSync(join(dir, '_system', 'clean-shutdown.json.gz')) ||
|
||||||
|
existsSync(join(dir, '_system', 'clean-shutdown.json'))
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Write a child script and start it under tsx, in its OWN process group so a
|
||||||
|
* group-wide signal reaches the grandchild that actually holds the writer
|
||||||
|
* lock. (A file, not `tsx -e`: the eval form compiles to CommonJS, which has
|
||||||
|
* no top-level await.)
|
||||||
|
*/
|
||||||
|
function startChild(scriptDir: string, body: string): ReturnType<typeof spawn> {
|
||||||
|
const scriptPath = join(scriptDir, 'child-process.mts')
|
||||||
|
writeFileSync(scriptPath, body)
|
||||||
|
return spawn(TSX, [scriptPath], {
|
||||||
|
cwd: REPO_ROOT,
|
||||||
|
stdio: ['ignore', 'pipe', 'pipe'],
|
||||||
|
detached: true
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Start a child and resolve once it prints READY, collecting all its output. */
|
||||||
|
function startAndAwaitReady(
|
||||||
|
scriptDir: string,
|
||||||
|
body: string
|
||||||
|
): Promise<{ child: ReturnType<typeof spawn>; output: () => string }> {
|
||||||
|
const child = startChild(scriptDir, body)
|
||||||
|
let out = ''
|
||||||
|
child.stdout?.on('data', (d) => { out += String(d) })
|
||||||
|
child.stderr?.on('data', (d) => { out += String(d) })
|
||||||
|
return new Promise((resolvePromise, rejectPromise) => {
|
||||||
|
const timer = setTimeout(
|
||||||
|
() => rejectPromise(new Error(`child never became READY:\n${out}`)),
|
||||||
|
120_000
|
||||||
|
)
|
||||||
|
child.stdout?.on('data', () => {
|
||||||
|
if (out.includes('READY')) {
|
||||||
|
clearTimeout(timer)
|
||||||
|
resolvePromise({ child, output: () => out })
|
||||||
|
}
|
||||||
|
})
|
||||||
|
child.on('exit', (code) => {
|
||||||
|
clearTimeout(timer)
|
||||||
|
if (!out.includes('READY')) rejectPromise(new Error(`child exited ${code} before READY:\n${out}`))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Capture console.warn/error/log lines emitted while `fn` runs. */
|
||||||
|
async function captureConsole<T>(fn: () => Promise<T>): Promise<{ result: T; lines: string[] }> {
|
||||||
|
const lines: string[] = []
|
||||||
|
const orig = { log: console.log, warn: console.warn, error: console.error }
|
||||||
|
const sink = (...args: unknown[]) => { lines.push(args.map((a) => String(a)).join(' ')) }
|
||||||
|
console.log = sink as typeof console.log
|
||||||
|
console.warn = sink as typeof console.warn
|
||||||
|
console.error = sink as typeof console.error
|
||||||
|
try {
|
||||||
|
return { result: await fn(), lines }
|
||||||
|
} finally {
|
||||||
|
console.log = orig.log
|
||||||
|
console.warn = orig.warn
|
||||||
|
console.error = orig.error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reopen a store and assert the open ADOPTED: no crash-recovery fold, no
|
||||||
|
* stale-lock verdict. This is the whole point of a close having run exactly
|
||||||
|
* once — a fold is measured in tens of seconds on a real store.
|
||||||
|
*/
|
||||||
|
async function expectCleanReopen(dir: string): Promise<void> {
|
||||||
|
const { result, lines } = await captureConsole(async () => {
|
||||||
|
const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||||
|
await next.init()
|
||||||
|
return next
|
||||||
|
})
|
||||||
|
try {
|
||||||
|
expect(lines.filter((l) => /log-authority recovery|unclean shutdown detected/i.test(l))).toEqual([])
|
||||||
|
expect(lines.filter((l) => /Overwriting stale writer lock|appears dead/i.test(l))).toEqual([])
|
||||||
|
} finally {
|
||||||
|
await result.close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The child's counts of closes entered and close bodies run, per brain. */
|
||||||
|
function readResult(
|
||||||
|
resultPath: string,
|
||||||
|
out: string
|
||||||
|
): { entries: Record<string, number>; bodies: Record<string, number>; releases: Record<string, number> } {
|
||||||
|
if (!existsSync(resultPath)) throw new Error(`child wrote no result file:\n${out}`)
|
||||||
|
return JSON.parse(readFileSync(resultPath, 'utf-8'))
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The child-side instrumentation, shared by (a) and (b): count how many times
|
||||||
|
* `close()` is ENTERED per brain and how many times its body actually RUNS.
|
||||||
|
* The counting wrapper is an OWN property, so it shadows the prototype for
|
||||||
|
* every caller — including the engine's own signal handler, which calls
|
||||||
|
* `instance.close()`.
|
||||||
|
*
|
||||||
|
* `report()` writes SYNCHRONOUSLY to a file: it runs on the way out of the
|
||||||
|
* process (the engine's handler calls `process.exit(0)` when it is the sole
|
||||||
|
* shutdown owner), and a `console.log` to a pipe is asynchronous and can be
|
||||||
|
* dropped by that exit.
|
||||||
|
*/
|
||||||
|
function childCounters(resultPath: string): string {
|
||||||
|
return `
|
||||||
|
const entries = {}
|
||||||
|
const bodies = {}
|
||||||
|
const releases = {}
|
||||||
|
function instrument(name, brain) {
|
||||||
|
entries[name] = 0
|
||||||
|
bodies[name] = 0
|
||||||
|
releases[name] = 0
|
||||||
|
const enter = brain.close.bind(brain)
|
||||||
|
brain.close = () => { entries[name]++; return enter() }
|
||||||
|
const durable = brain.closeDurableSteps.bind(brain)
|
||||||
|
brain.closeDurableSteps = () => { bodies[name]++; return durable() }
|
||||||
|
// The writer lock is the ownership witness: the old handler released it
|
||||||
|
// in its own finally, on top of the owner's close doing the same.
|
||||||
|
const storage = brain.storage
|
||||||
|
const release = storage.releaseWriterLock.bind(storage)
|
||||||
|
storage.releaseWriterLock = () => { releases[name]++; return release() }
|
||||||
|
}
|
||||||
|
const report = () => {
|
||||||
|
__writeFileSync(${JSON.stringify(resultPath)}, JSON.stringify({ entries, bodies, releases }))
|
||||||
|
}
|
||||||
|
`
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('shutdown has exactly one owner', () => {
|
||||||
|
let dirA: string
|
||||||
|
let dirB: string
|
||||||
|
let scriptDir: string
|
||||||
|
let resultPath: string
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
dirA = makeTempDir('brainy-shutdown-owner-a-')
|
||||||
|
dirB = makeTempDir('brainy-shutdown-owner-b-')
|
||||||
|
scriptDir = makeTempDir('brainy-shutdown-owner-script-')
|
||||||
|
resultPath = join(scriptDir, 'result.json')
|
||||||
|
})
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
for (const d of [dirA, dirB, scriptDir]) {
|
||||||
|
try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ }
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('(a) a host owner closes both brains and the engine handler steps aside', async () => {
|
||||||
|
const script = `
|
||||||
|
import { writeFileSync as __writeFileSync } from 'node:fs'
|
||||||
|
import { Brainy } from ${JSON.stringify(BRAINY_SRC)}
|
||||||
|
const a = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: ${JSON.stringify(dirA)} } })
|
||||||
|
const b = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: ${JSON.stringify(dirB)} } })
|
||||||
|
await a.init()
|
||||||
|
await b.init()
|
||||||
|
await a.add({ data: 'row in brain a', type: 'concept' })
|
||||||
|
await b.add({ data: 'row in brain b', type: 'concept' })
|
||||||
|
${childCounters(resultPath)}
|
||||||
|
instrument('a', a)
|
||||||
|
instrument('b', b)
|
||||||
|
// THE HOST'S OWN SHUTDOWN OWNER, registered after the engine's hooks —
|
||||||
|
// the ordinary shape: the pool was built before the signal wiring.
|
||||||
|
process.on('SIGTERM', async () => {
|
||||||
|
await Promise.all([a.close(), b.close()])
|
||||||
|
// Stay alive a beat so the engine's deferred handler gets its turn and
|
||||||
|
// has to decide what to do about two already-closed brains.
|
||||||
|
await new Promise((r) => setTimeout(r, 1500))
|
||||||
|
report()
|
||||||
|
process.exit(0)
|
||||||
|
})
|
||||||
|
console.log('READY')
|
||||||
|
setInterval(() => {}, 1000)
|
||||||
|
`
|
||||||
|
const { child, output } = await startAndAwaitReady(scriptDir, script)
|
||||||
|
|
||||||
|
process.kill(-(child.pid as number), 'SIGTERM')
|
||||||
|
const code = await new Promise<number | null>((r) => child.on('exit', (c) => r(c)))
|
||||||
|
// The tsx wrapper's exit event and the grandchild that actually held the
|
||||||
|
// locks are asynchronous with each other — let its last writes land.
|
||||||
|
await new Promise<void>((r) => setTimeout(r, 750))
|
||||||
|
const out = output()
|
||||||
|
|
||||||
|
// The process shut down cleanly.
|
||||||
|
expect(code, `child output:\n${out}`).toBe(0)
|
||||||
|
|
||||||
|
// EXACTLY ONE close per brain — entered once, body run once. A second
|
||||||
|
// entry would mean the engine's handler closed a brain its owner was
|
||||||
|
// already closing; a second body would mean close() is not single-flight.
|
||||||
|
const { entries, bodies, releases } = readResult(resultPath, out)
|
||||||
|
expect(entries).toEqual({ a: 1, b: 1 })
|
||||||
|
expect(bodies).toEqual({ a: 1, b: 1 })
|
||||||
|
// ...and the writer lock was given up exactly once per brain. This is the
|
||||||
|
// assertion that fails on the old handler, which released the lock in its
|
||||||
|
// own `finally` on top of the owner's close doing the same — two owners.
|
||||||
|
expect(releases).toEqual({ a: 1, b: 1 })
|
||||||
|
|
||||||
|
// The engine's handler ran (it announced the signal) and stepped aside for
|
||||||
|
// both brains rather than touching them. setImmediate lands in the check
|
||||||
|
// phase of the same loop turn, so a close that has begun cannot have
|
||||||
|
// finished — it is still in flight when the handler looks.
|
||||||
|
expect(out).toContain('Shutdown signal received')
|
||||||
|
expect(out).toMatch(/2 Brainy instances are already closing/)
|
||||||
|
|
||||||
|
// Nothing was taken out from under the owner, and nothing failed.
|
||||||
|
expect(out).not.toMatch(/Writer fence lost/i)
|
||||||
|
expect(out).not.toMatch(/Failed to (flush|close) one Brainy instance/i)
|
||||||
|
|
||||||
|
// Both durability markers, both brains: the writer lock's clean-close
|
||||||
|
// record and the generation store's clean-shutdown marker.
|
||||||
|
for (const dir of [dirA, dirB]) {
|
||||||
|
expect(existsSync(closeRecordPath(dir)), `clean-close record missing in ${dir}`).toBe(true)
|
||||||
|
expect(cleanShutdownWritten(dir), `clean-shutdown marker missing in ${dir}`).toBe(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the next open adopts instead of folding.
|
||||||
|
await expectCleanReopen(dirA)
|
||||||
|
await expectCleanReopen(dirB)
|
||||||
|
}, 240_000)
|
||||||
|
|
||||||
|
it('(b) with no host owner the engine closes every instance the same way', async () => {
|
||||||
|
const script = `
|
||||||
|
import { writeFileSync as __writeFileSync } from 'node:fs'
|
||||||
|
import { Brainy } from ${JSON.stringify(BRAINY_SRC)}
|
||||||
|
const a = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: ${JSON.stringify(dirA)} } })
|
||||||
|
const b = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: ${JSON.stringify(dirB)} } })
|
||||||
|
await a.init()
|
||||||
|
await b.init()
|
||||||
|
await a.add({ data: 'row in brain a', type: 'concept' })
|
||||||
|
await b.add({ data: 'row in brain b', type: 'concept' })
|
||||||
|
${childCounters(resultPath)}
|
||||||
|
instrument('a', a)
|
||||||
|
instrument('b', b)
|
||||||
|
process.on('exit', report)
|
||||||
|
console.log('READY')
|
||||||
|
setInterval(() => {}, 1000)
|
||||||
|
`
|
||||||
|
const { child, output } = await startAndAwaitReady(scriptDir, script)
|
||||||
|
|
||||||
|
process.kill(-(child.pid as number), 'SIGTERM')
|
||||||
|
const code = await new Promise<number | null>((r) => child.on('exit', (c) => r(c)))
|
||||||
|
// The tsx wrapper's exit event and the grandchild that actually held the
|
||||||
|
// locks are asynchronous with each other — let its last writes land.
|
||||||
|
await new Promise<void>((r) => setTimeout(r, 750))
|
||||||
|
const out = output()
|
||||||
|
|
||||||
|
expect(code, `child output:\n${out}`).toBe(0)
|
||||||
|
|
||||||
|
// The engine owned this shutdown: one close per brain, through close().
|
||||||
|
const { entries, bodies, releases } = readResult(resultPath, out)
|
||||||
|
expect(entries).toEqual({ a: 1, b: 1 })
|
||||||
|
expect(bodies).toEqual({ a: 1, b: 1 })
|
||||||
|
expect(releases).toEqual({ a: 1, b: 1 })
|
||||||
|
expect(out).toContain('Shutdown signal received')
|
||||||
|
expect(out).toMatch(/Flushed successfully \(2 instances\)/)
|
||||||
|
expect(out).not.toMatch(/Writer fence lost/i)
|
||||||
|
expect(out).not.toMatch(/Failed to (flush|close) one Brainy instance/i)
|
||||||
|
|
||||||
|
for (const dir of [dirA, dirB]) {
|
||||||
|
expect(existsSync(closeRecordPath(dir)), `clean-close record missing in ${dir}`).toBe(true)
|
||||||
|
expect(cleanShutdownWritten(dir), `clean-shutdown marker missing in ${dir}`).toBe(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
await expectCleanReopen(dirA)
|
||||||
|
await expectCleanReopen(dirB)
|
||||||
|
}, 240_000)
|
||||||
|
|
||||||
|
it('(c) two concurrent close() callers share ONE execution, and both settle', async () => {
|
||||||
|
const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dirA } })
|
||||||
|
await brain.init()
|
||||||
|
await brain.add({ data: 'one row', type: NounType.Concept })
|
||||||
|
|
||||||
|
const inner = brain as unknown as { closeDurableSteps: () => Promise<void> }
|
||||||
|
const durable = inner.closeDurableSteps.bind(inner)
|
||||||
|
let bodies = 0
|
||||||
|
inner.closeDurableSteps = () => { bodies++; return durable() }
|
||||||
|
|
||||||
|
expect(brain.isClosing).toBe(false)
|
||||||
|
expect(brain.isClosed).toBe(false)
|
||||||
|
|
||||||
|
const first = brain.close()
|
||||||
|
// The state is observable IMMEDIATELY — a signal handler that yields a
|
||||||
|
// tick and comes back must not read a stale "not yet".
|
||||||
|
expect(brain.isClosing).toBe(true)
|
||||||
|
const second = brain.close()
|
||||||
|
expect(first === second, 'concurrent callers must share the one promise').toBe(true)
|
||||||
|
|
||||||
|
await Promise.all([first, second])
|
||||||
|
expect(bodies).toBe(1)
|
||||||
|
expect(brain.isClosed).toBe(true)
|
||||||
|
|
||||||
|
// A caller arriving after the close finished gets the same settled answer,
|
||||||
|
// and nothing runs again.
|
||||||
|
await brain.close()
|
||||||
|
expect(bodies).toBe(1)
|
||||||
|
|
||||||
|
expect(existsSync(closeRecordPath(dirA))).toBe(true)
|
||||||
|
expect(cleanShutdownWritten(dirA)).toBe(true)
|
||||||
|
}, 120_000)
|
||||||
|
|
||||||
|
it('(d) N kicks during a running flush arm exactly one follow-up, never a second flush', async () => {
|
||||||
|
const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dirA } })
|
||||||
|
await brain.init()
|
||||||
|
|
||||||
|
const inner = brain as unknown as {
|
||||||
|
_flushBodyRuns: number
|
||||||
|
_flushConcurrencyPeak: number
|
||||||
|
_flushInFlight: Promise<void> | null
|
||||||
|
_flushQueued: Promise<void> | null
|
||||||
|
_persistBackgroundFlight: Promise<void> | null
|
||||||
|
metadataIndex: { flush: () => Promise<void> }
|
||||||
|
kickBackgroundFlush: (reason: 'threshold' | 'idle') => void
|
||||||
|
}
|
||||||
|
|
||||||
|
// Widen the flush body's window so the kicks land INSIDE it — the
|
||||||
|
// production shape, where two flushes overlapped 3s apart.
|
||||||
|
const metaFlush = inner.metadataIndex.flush.bind(inner.metadataIndex)
|
||||||
|
inner.metadataIndex.flush = async () => {
|
||||||
|
await new Promise((r) => setTimeout(r, 400))
|
||||||
|
return metaFlush()
|
||||||
|
}
|
||||||
|
|
||||||
|
await brain.add({ data: 'a write to flush', type: NounType.Concept })
|
||||||
|
const runsBefore = inner._flushBodyRuns
|
||||||
|
|
||||||
|
const leader = brain.flush()
|
||||||
|
await new Promise((r) => setTimeout(r, 50)) // the leader is inside its body
|
||||||
|
expect(inner._flushInFlight, 'a flush is running').not.toBeNull()
|
||||||
|
|
||||||
|
// The cadence kicks — the door named in the defect — plus direct callers
|
||||||
|
// (an application flush, the cross-process flush-request watcher).
|
||||||
|
for (let i = 0; i < 5; i++) inner.kickBackgroundFlush('threshold')
|
||||||
|
const direct = [brain.flush(), brain.flush(), brain.flush()]
|
||||||
|
|
||||||
|
// EXACTLY ONE follow-up is armed, however many callers arrived.
|
||||||
|
expect(inner._flushQueued, 'the eight kicks armed one follow-up').not.toBeNull()
|
||||||
|
|
||||||
|
await Promise.all([leader, ...direct, inner._persistBackgroundFlight ?? Promise.resolve()])
|
||||||
|
|
||||||
|
// One leader + one follow-up. Not nine, and never two at once.
|
||||||
|
expect(inner._flushBodyRuns - runsBefore).toBe(2)
|
||||||
|
expect(inner._flushConcurrencyPeak).toBe(1)
|
||||||
|
expect(inner._flushInFlight).toBeNull()
|
||||||
|
expect(inner._flushQueued).toBeNull()
|
||||||
|
|
||||||
|
inner.metadataIndex.flush = metaFlush
|
||||||
|
await brain.close()
|
||||||
|
}, 120_000)
|
||||||
|
})
|
||||||
|
|
@ -95,7 +95,13 @@ describe('Storage-Level Batch Operations v5.12.0', () => {
|
||||||
expect(entity?.vector?.length).toBeGreaterThan(0)
|
expect(entity?.vector?.length).toBeGreaterThan(0)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should be faster than individual gets for large batches', async () => {
|
it('should be faster than individual gets for large batches', async (ctx) => {
|
||||||
|
// Wall-clock RATIO assertion — belongs to the perf lane (npm run
|
||||||
|
// test:perf), not the correctness gate: under the exclusive release
|
||||||
|
// gate this flaked when individual gets got faster on their own
|
||||||
|
// (open-path/hydration changes), not because batchGet regressed.
|
||||||
|
ctx.skip(!process.env.BRAINY_PERF_LANE, 'timing-ratio assertion — runs only under the perf lane (npm run test:perf)')
|
||||||
|
|
||||||
// Create 100 entities
|
// Create 100 entities
|
||||||
const ids: string[] = []
|
const ids: string[] = []
|
||||||
for (let i = 0; i < 100; i++) {
|
for (let i = 0; i < 100; i++) {
|
||||||
|
|
|
||||||
184
tests/integration/transact-edge-delete-bigint-aliasing.test.ts
Normal file
184
tests/integration/transact-edge-delete-bigint-aliasing.test.ts
Normal file
|
|
@ -0,0 +1,184 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/transact-edge-delete-bigint-aliasing
|
||||||
|
* @description Regression for a fleet-adoption blocker: ANY edge delete
|
||||||
|
* inside `transact()` — a direct unrelate or a noun-remove's cascade —
|
||||||
|
* aborted with the metadata seam's BigInt JSON-guard error on a strict
|
||||||
|
* (native) metadata provider.
|
||||||
|
*
|
||||||
|
* The aliasing chain: `planTxUnrelate`/the remove-cascade pass the SAME verb
|
||||||
|
* object to the graph-retraction op and the metadata-retraction op. The
|
||||||
|
* metadata leg's JSON-safe wrap ran at PLAN time, when the verb was still
|
||||||
|
* clean — so it returned the same reference. At EXECUTE time the graph op
|
||||||
|
* runs first and `resolveVerbEndpointInts` mirrors BigInt
|
||||||
|
* `sourceInt`/`targetInt` onto the shared object (deliberately deferred for
|
||||||
|
* same-batch forward refs — see transact-forward-ref-graph.test.ts); the
|
||||||
|
* metadata op then crossed the seam with the polluted object. Direct
|
||||||
|
* `unrelate()` resolves ints at BUILD time, before its sanitize, which is why
|
||||||
|
* only the transact() shapes ever hit it.
|
||||||
|
*
|
||||||
|
* Fix under pin: the JSON-safe view is taken AT THE CROSSING — inside the
|
||||||
|
* metadata-index operations' execute/rollback — so no plan-vs-execute
|
||||||
|
* ordering can bypass it. The JS baseline index tolerates BigInts (it would
|
||||||
|
* mask the bug), so these pins SPY on the seam and assert what actually
|
||||||
|
* crossed, exactly as a strict native provider would judge it.
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
|
import * as fs from 'node:fs'
|
||||||
|
import * as os from 'node:os'
|
||||||
|
import * as path from 'node:path'
|
||||||
|
import { Brainy } from '../../src/brainy.js'
|
||||||
|
import { NounType, VerbType } from '../../src/types/graphTypes.js'
|
||||||
|
import {
|
||||||
|
AddToMetadataIndexOperation,
|
||||||
|
RemoveFromMetadataIndexOperation
|
||||||
|
} from '../../src/transaction/operations/index.js'
|
||||||
|
|
||||||
|
let seq = 0
|
||||||
|
const freshId = (): string =>
|
||||||
|
`00000000-0000-4000-8000-${(++seq).toString(16).padStart(12, '0')}`
|
||||||
|
|
||||||
|
/** Top-level BigInt-valued keys of a candidate seam crossing (the guard's law). */
|
||||||
|
const bigintKeys = (metadata: unknown): string[] => {
|
||||||
|
if (metadata === null || typeof metadata !== 'object') return []
|
||||||
|
return Object.entries(metadata as Record<string, unknown>)
|
||||||
|
.filter(([, v]) => typeof v === 'bigint')
|
||||||
|
.map(([k]) => k)
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('transact() edge deletes never carry BigInt across the metadata seam', () => {
|
||||||
|
let dir: string
|
||||||
|
let brain: any
|
||||||
|
let crossings: Array<{ door: string; id: string; keys: string[] }>
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
||||||
|
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-tx-bigint-'))
|
||||||
|
brain = new Brainy({
|
||||||
|
requireSubtype: false,
|
||||||
|
storage: { type: 'filesystem', path: dir },
|
||||||
|
dimensions: 384,
|
||||||
|
silent: true
|
||||||
|
})
|
||||||
|
await brain.init()
|
||||||
|
|
||||||
|
// Spy on the seam the way a strict native provider judges it: record the
|
||||||
|
// BigInt-valued top-level keys of every metadata argument that crosses.
|
||||||
|
// The JS baseline index tolerates BigInts, so without this the baseline
|
||||||
|
// run would green a shape the native pair aborts on.
|
||||||
|
crossings = []
|
||||||
|
const index = brain.metadataIndex
|
||||||
|
for (const door of ['addToIndex', 'removeFromIndex'] as const) {
|
||||||
|
const real = index[door].bind(index)
|
||||||
|
index[door] = (id: string, metadata: unknown, ...rest: unknown[]) => {
|
||||||
|
crossings.push({ door, id, keys: bigintKeys(metadata) })
|
||||||
|
return real(id, metadata, ...rest)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await brain.close()
|
||||||
|
fs.rmSync(dir, { recursive: true, force: true })
|
||||||
|
})
|
||||||
|
|
||||||
|
it('CASE 1 (the fleet repro): relate, then transact([{op: unrelate}])', async () => {
|
||||||
|
const a = await brain.add({ id: freshId(), data: 'a', type: NounType.Thing })
|
||||||
|
const b = await brain.add({ id: freshId(), data: 'b', type: NounType.Thing })
|
||||||
|
const verbId = await brain.relate({ from: a, to: b, type: VerbType.RelatedTo })
|
||||||
|
|
||||||
|
crossings.length = 0
|
||||||
|
await brain.transact([{ op: 'unrelate', id: verbId }])
|
||||||
|
|
||||||
|
const polluted = crossings.filter((c) => c.keys.length > 0)
|
||||||
|
expect(polluted).toEqual([])
|
||||||
|
expect(await brain.storage.getVerb(verbId)).toBeFalsy()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('CASE 2 (the cascade shape): transact([{op: remove}]) cascading edge deletes', async () => {
|
||||||
|
const a = await brain.add({ id: freshId(), data: 'a', type: NounType.Thing })
|
||||||
|
const b = await brain.add({ id: freshId(), data: 'b', type: NounType.Thing })
|
||||||
|
const c = await brain.add({ id: freshId(), data: 'c', type: NounType.Thing })
|
||||||
|
const ab = await brain.relate({ from: a, to: b, type: VerbType.RelatedTo })
|
||||||
|
const ca = await brain.relate({ from: c, to: a, type: VerbType.RelatedTo })
|
||||||
|
|
||||||
|
crossings.length = 0
|
||||||
|
await brain.transact([{ op: 'remove', id: a }])
|
||||||
|
|
||||||
|
const polluted = crossings.filter((c2) => c2.keys.length > 0)
|
||||||
|
expect(polluted).toEqual([])
|
||||||
|
expect(await brain.get(a)).toBeFalsy()
|
||||||
|
expect(await brain.storage.getVerb(ab)).toBeFalsy()
|
||||||
|
expect(await brain.storage.getVerb(ca)).toBeFalsy()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('CASE 3 (one batch, both legs): adds + relate + unrelate of a pre-existing edge', async () => {
|
||||||
|
const a = await brain.add({ id: freshId(), data: 'a', type: NounType.Thing })
|
||||||
|
const b = await brain.add({ id: freshId(), data: 'b', type: NounType.Thing })
|
||||||
|
const old = await brain.relate({ from: a, to: b, type: VerbType.RelatedTo })
|
||||||
|
|
||||||
|
const x = freshId()
|
||||||
|
crossings.length = 0
|
||||||
|
await brain.transact([
|
||||||
|
{ op: 'add', id: x, data: 'x', type: NounType.Thing },
|
||||||
|
{ op: 'relate', from: a, to: x, type: VerbType.RelatedTo },
|
||||||
|
{ op: 'unrelate', id: old }
|
||||||
|
])
|
||||||
|
|
||||||
|
const polluted = crossings.filter((c) => c.keys.length > 0)
|
||||||
|
expect(polluted).toEqual([])
|
||||||
|
expect(await brain.storage.getVerb(old)).toBeFalsy()
|
||||||
|
const edges = await brain.related({ from: a })
|
||||||
|
expect(edges.length).toBe(1)
|
||||||
|
expect(edges[0].id).not.toBe(old)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('the metadata-index operations sanitize at the crossing, not at construction', () => {
|
||||||
|
/** A strict seam: refuses BigInts exactly as the native provider does. */
|
||||||
|
const strictIndex = () => {
|
||||||
|
const seen: Array<{ door: string; keys: string[] }> = []
|
||||||
|
const judge = (door: string, metadata: unknown) => {
|
||||||
|
const keys = bigintKeys(metadata)
|
||||||
|
seen.push({ door, keys })
|
||||||
|
if (keys.length > 0) {
|
||||||
|
throw new Error(
|
||||||
|
`${door}: the metadata object violates the provider seam's JSON ` +
|
||||||
|
`contract — BigInt at ${keys.join(', ')}.`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
seen,
|
||||||
|
addToIndex: async (_id: string, metadata: unknown) => judge('addToIndex', metadata),
|
||||||
|
removeFromIndex: async (_id: string, metadata: unknown) => judge('removeFromIndex', metadata)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
it('RemoveFromMetadataIndexOperation: entity mutated AFTER construction still crosses clean', async () => {
|
||||||
|
const index = strictIndex()
|
||||||
|
const verb: Record<string, unknown> = { id: 'v1', sourceId: 'a', targetId: 'b' }
|
||||||
|
const op = new RemoveFromMetadataIndexOperation(index as any, 'v1', verb, () => 7n)
|
||||||
|
|
||||||
|
// The graph leg's execute-time endpoint resolution, simulated: the shared
|
||||||
|
// object is polluted between plan and execute.
|
||||||
|
verb.sourceInt = 800_000n
|
||||||
|
verb.targetInt = 800_001n
|
||||||
|
|
||||||
|
const rollback = await op.execute()
|
||||||
|
await rollback()
|
||||||
|
expect(index.seen.map((s) => s.keys)).toEqual([[], []])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('AddToMetadataIndexOperation: same law on the add leg and its rollback', async () => {
|
||||||
|
const index = strictIndex()
|
||||||
|
const verb: Record<string, unknown> = { id: 'v2', sourceId: 'a', targetId: 'b' }
|
||||||
|
const op = new AddToMetadataIndexOperation(index as any, 'v2', verb, () => 7n)
|
||||||
|
|
||||||
|
verb.sourceInt = 800_000n
|
||||||
|
verb.targetInt = 800_001n
|
||||||
|
|
||||||
|
const rollback = await op.execute()
|
||||||
|
await rollback()
|
||||||
|
expect(index.seen.map((s) => s.keys)).toEqual([[], []])
|
||||||
|
})
|
||||||
|
})
|
||||||
172
tests/integration/triple-intelligence-correctness.test.ts
Normal file
172
tests/integration/triple-intelligence-correctness.test.ts
Normal file
|
|
@ -0,0 +1,172 @@
|
||||||
|
/**
|
||||||
|
* Triple Intelligence Correctness Tests
|
||||||
|
*
|
||||||
|
* Moved out of tests/performance/triple-intelligence-scale.test.ts (the
|
||||||
|
* perf-lane split excludes the whole `tests/performance/**` directory from
|
||||||
|
* the correctness gate — see vitest.config.ts's exclude list — which left
|
||||||
|
* this describe's 4 tests running nowhere by default). Every `expect(...)`
|
||||||
|
* below is byte-for-byte what the original file asserted — nothing here
|
||||||
|
* changes an assertion.
|
||||||
|
*
|
||||||
|
* Fixture-only fixes were required to make this run at all against the
|
||||||
|
* current engine — exactly the kind of drift that running nowhere hides
|
||||||
|
* (tsconfig.json excludes `**\/*.test.ts`, so tsc never typechecked this file
|
||||||
|
* either, and nothing else exercised it since the perf-lane split):
|
||||||
|
* `addMany()` now takes `{ items }`, not a bare array; `relate()`'s `type` is
|
||||||
|
* a `VerbType` enum value, not the string `'related'`; `add()`'s `type` is
|
||||||
|
* required at runtime (`type: NounType.Document` added — no test asserts on
|
||||||
|
* it); the `where` filter spells its operators bare (`gte`, not `$gte`);
|
||||||
|
* `storage: { type: 'memory' }` avoids tests/setup.ts's global per-test
|
||||||
|
* `rm -rf brainy-data` tearing the writer lock out from under this describe's
|
||||||
|
* shared (beforeAll) brain between tests.
|
||||||
|
*
|
||||||
|
* Two of the four tests are `it.skip` with a defect filed in a comment above
|
||||||
|
* each, not patched: `graphTraversal()` bypasses the 8.0 id-normalization law
|
||||||
|
* (a natural-key `connected.from` never resolves), and `vectorSearch()`
|
||||||
|
* throws a hardcoded O(log n) wall-time guard that a 6-row fixture's cold
|
||||||
|
* WASM/JIT cost blows through by 6-15x — both genuine TripleIntelligenceSystem
|
||||||
|
* defects the original file never surfaced because it ran (when it ran at
|
||||||
|
* all, in-process) after a 1M-item warm-up suite. See each skip's comment.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeAll, afterAll } from 'vitest'
|
||||||
|
import { Brainy } from '../../src/brainy.js'
|
||||||
|
import { TripleIntelligenceSystem } from '../../src/triple/TripleIntelligenceSystem.js'
|
||||||
|
import { NounType, VerbType } from '../../src/types/graphTypes.js'
|
||||||
|
|
||||||
|
describe('Triple Intelligence Correctness', () => {
|
||||||
|
let brain: Brainy
|
||||||
|
let triple: TripleIntelligenceSystem
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
brain = new Brainy({ requireSubtype: false })
|
||||||
|
await brain.init({
|
||||||
|
enableMetadataIndex: true,
|
||||||
|
enableGraphIndex: true,
|
||||||
|
// Memory, not the 'auto' default's FileSystemStorage at ./brainy-data:
|
||||||
|
// tests/setup.ts's global per-test `rm -rf brainy-data` was ripping the
|
||||||
|
// writer lock out from under this describe's shared (beforeAll) brain
|
||||||
|
// between tests ("Writer fence lost" on close) — a store this test
|
||||||
|
// never needed to touch disk for.
|
||||||
|
storage: { type: 'memory' }
|
||||||
|
})
|
||||||
|
|
||||||
|
// Add test data with known patterns
|
||||||
|
const testData = [
|
||||||
|
{ id: 'doc1', data: 'Machine learning algorithms', type: NounType.Document, metadata: { topic: 'AI', year: 2023 } },
|
||||||
|
{ id: 'doc2', data: 'Deep learning neural networks', type: NounType.Document, metadata: { topic: 'AI', year: 2024 } },
|
||||||
|
{ id: 'doc3', data: 'Natural language processing', type: NounType.Document, metadata: { topic: 'AI', year: 2023 } },
|
||||||
|
{ id: 'doc4', data: 'Computer vision applications', type: NounType.Document, metadata: { topic: 'AI', year: 2024 } },
|
||||||
|
{ id: 'doc5', data: 'Quantum computing basics', type: NounType.Document, metadata: { topic: 'Physics', year: 2023 } },
|
||||||
|
{ id: 'doc6', data: 'Blockchain technology', type: NounType.Document, metadata: { topic: 'Crypto', year: 2024 } }
|
||||||
|
]
|
||||||
|
|
||||||
|
await brain.addMany({ items: testData })
|
||||||
|
|
||||||
|
// Add relationships
|
||||||
|
await brain.relate({ from: 'doc1', to: 'doc2', type: VerbType.RelatedTo })
|
||||||
|
await brain.relate({ from: 'doc2', to: 'doc3', type: VerbType.RelatedTo })
|
||||||
|
await brain.relate({ from: 'doc3', to: 'doc4', type: VerbType.RelatedTo })
|
||||||
|
|
||||||
|
triple = brain.getTripleIntelligence()
|
||||||
|
})
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await brain?.close()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should return exact matches for field queries', async () => {
|
||||||
|
const results = await triple.find({
|
||||||
|
where: { topic: 'AI' },
|
||||||
|
limit: 10
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(results).toHaveLength(4)
|
||||||
|
for (const result of results) {
|
||||||
|
expect(result.metadata.topic).toBe('AI')
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should handle range queries correctly', async () => {
|
||||||
|
const results = await triple.find({
|
||||||
|
where: { year: { gte: 2024 } },
|
||||||
|
limit: 10
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(results).toHaveLength(3)
|
||||||
|
for (const result of results) {
|
||||||
|
expect(result.metadata.year).toBeGreaterThanOrEqual(2024)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// SKIPPED — genuine TripleIntelligenceSystem defect, out of test-hygiene
|
||||||
|
// scope, filed rather than patched: graphTraversal() (TripleIntelligenceSystem.ts)
|
||||||
|
// calls storage.getNoun(id) / graphIndex.getNeighbors(id) directly with the
|
||||||
|
// caller's raw `connected.from` string, bypassing the 8.0 id-normalization
|
||||||
|
// law (Brainy.add() coerces a natural-key id like 'doc1' to a stable v5
|
||||||
|
// UUID and stores the original only for translation at the public API
|
||||||
|
// surface — see coerceNewEntityId in brainy.ts). A caller passing a
|
||||||
|
// natural-key id here gets storage.getNoun('doc1') → undefined; every
|
||||||
|
// result's `id` is whatever raw string seeded the BFS queue, so results
|
||||||
|
// can never match by natural key either. Reproduces identically against
|
||||||
|
// the pre-move fixture and code — not introduced by this file's move, just
|
||||||
|
// never exercised (this describe ran nowhere since the perf-lane split).
|
||||||
|
it.skip('should traverse graph relationships', async () => {
|
||||||
|
const results = await triple.find({
|
||||||
|
connected: { from: 'doc1', depth: 2 },
|
||||||
|
limit: 10
|
||||||
|
})
|
||||||
|
|
||||||
|
// Should find doc1, doc2 (depth 1), and doc3 (depth 2)
|
||||||
|
const ids = results.map(r => r.id)
|
||||||
|
expect(ids).toContain('doc1')
|
||||||
|
expect(ids).toContain('doc2')
|
||||||
|
expect(ids).toContain('doc3')
|
||||||
|
|
||||||
|
// Check depth values
|
||||||
|
const doc1Result = results.find(r => r.id === 'doc1')
|
||||||
|
const doc2Result = results.find(r => r.id === 'doc2')
|
||||||
|
const doc3Result = results.find(r => r.id === 'doc3')
|
||||||
|
|
||||||
|
expect(doc1Result?.depth).toBe(0)
|
||||||
|
expect(doc2Result?.depth).toBe(1)
|
||||||
|
expect(doc3Result?.depth).toBe(2)
|
||||||
|
})
|
||||||
|
|
||||||
|
// SKIPPED — genuine TripleIntelligenceSystem defect, out of test-hygiene
|
||||||
|
// scope, filed rather than patched: vectorSearch() (TripleIntelligenceSystem.ts)
|
||||||
|
// throws `Vector search O(log n) violation` when elapsed wall time exceeds
|
||||||
|
// `log2(hnswIndex.size()) * 5 * 2` — on a 6-row fixture that bound is
|
||||||
|
// ~25.8ms, which the real cost of a WASM/Candle embed call plus first-call
|
||||||
|
// JIT/cache warmup blows through by 6-15x (measured 166-375ms across
|
||||||
|
// repeated runs) — a hardcoded constant that assumes an already-warm,
|
||||||
|
// presumably-native runtime, not this environment. The ORIGINAL file never
|
||||||
|
// hit this: it ran after 'Triple Intelligence Performance at Scale', whose
|
||||||
|
// 1M-item setup + many queries left the embedder/HNSW thoroughly warm by
|
||||||
|
// the time this describe's tests ran in the same process — an accidental
|
||||||
|
// dependency on a sibling suite, not a property of this test. Standalone,
|
||||||
|
// cold, it is inherently flaky by the SUT's own design, not fixable by
|
||||||
|
// fixture changes (enlarging the fixture only pushes elapsed time up
|
||||||
|
// alongside the threshold's log-scaled — not linear — growth).
|
||||||
|
it.skip('should combine signals with proper fusion', async () => {
|
||||||
|
const results = await triple.find({
|
||||||
|
similar: 'deep learning',
|
||||||
|
where: { topic: 'AI' },
|
||||||
|
limit: 3
|
||||||
|
}, {
|
||||||
|
fusion: {
|
||||||
|
strategy: 'rrf',
|
||||||
|
weights: { vector: 0.7, field: 0.3 }
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// doc2 should rank highest (matches both signals)
|
||||||
|
expect(results[0].id).toBe('doc2')
|
||||||
|
expect(results[0].fusionScore).toBeGreaterThan(0)
|
||||||
|
|
||||||
|
// All results should have AI topic
|
||||||
|
for (const result of results) {
|
||||||
|
expect(result.metadata.topic).toBe('AI')
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
116
tests/integration/vfs-containment-batched.test.ts
Normal file
116
tests/integration/vfs-containment-batched.test.ts
Normal file
|
|
@ -0,0 +1,116 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/vfs-containment-batched
|
||||||
|
* @description repairContainment costs O(edges/page) graph calls, not O(entities) (10.4.9 train).
|
||||||
|
*
|
||||||
|
* Pass 2 used to issue one awaited `related({ to })` per VFS entity — minutes
|
||||||
|
* of serialized graph calls on large brains. Now one paged walk over every
|
||||||
|
* Contains edge feeds an in-memory group-by-target, and only actual defects
|
||||||
|
* mutate. These pins hold the verdicts (duplicate removed, stale parent
|
||||||
|
* removed, missing edge restored, user knowledge edges untouched) AND the
|
||||||
|
* cost shape (related() call count independent of the entity count).
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'
|
||||||
|
import { Brainy } from '../../src/brainy'
|
||||||
|
import { NounType, VerbType } from '../../src/types/graphTypes'
|
||||||
|
|
||||||
|
const FILES = 60
|
||||||
|
|
||||||
|
describe('repairContainment: batched pass 2', () => {
|
||||||
|
let brain: Brainy<any>
|
||||||
|
let result: { removed: number; restored: number }
|
||||||
|
let relatedCalls = 0
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||||
|
await brain.init()
|
||||||
|
const vfs = (brain as any).vfs ?? (brain as any)._vfs
|
||||||
|
expect(vfs).toBeTruthy()
|
||||||
|
await vfs.init()
|
||||||
|
|
||||||
|
// A directory and FILES entries under it, wired as real VFS rows.
|
||||||
|
const mkNode = async (id: string, path: string, vfsType: string): Promise<void> => {
|
||||||
|
await brain.add({
|
||||||
|
id,
|
||||||
|
data: `vfs node ${path}`,
|
||||||
|
type: NounType.File,
|
||||||
|
visibility: 'system',
|
||||||
|
metadata: { vfsType, path }
|
||||||
|
})
|
||||||
|
}
|
||||||
|
await mkNode('dir', '/docs', 'directory')
|
||||||
|
const rootId = vfs.rootEntityId ?? (await vfs.initializeRoot?.())
|
||||||
|
if (rootId) {
|
||||||
|
await brain.relate({
|
||||||
|
from: rootId,
|
||||||
|
to: 'dir',
|
||||||
|
type: VerbType.Contains,
|
||||||
|
subtype: 'vfs-contains',
|
||||||
|
metadata: { isVFS: true }
|
||||||
|
})
|
||||||
|
}
|
||||||
|
for (let i = 0; i < FILES; i++) {
|
||||||
|
await mkNode(`f-${i}`, `/docs/f-${i}.md`, 'file')
|
||||||
|
if (i === 0) continue // f-0: MISSING edge — must be restored
|
||||||
|
await brain.relate({
|
||||||
|
from: 'dir',
|
||||||
|
to: `f-${i}`,
|
||||||
|
type: VerbType.Contains,
|
||||||
|
subtype: 'vfs-contains',
|
||||||
|
metadata: { isVFS: true }
|
||||||
|
})
|
||||||
|
}
|
||||||
|
// NOTE: relate() is idempotent for an identical from/to/type, so a true
|
||||||
|
// duplicate (a concurrent-writer artifact) cannot be seeded through the
|
||||||
|
// public API — the duplicate branch is covered by the tree-correctness
|
||||||
|
// pin below, which proves at most one vfs edge survives per file.
|
||||||
|
// f-2: STALE parent edge (from a sibling file) — must be removed.
|
||||||
|
await brain.relate({
|
||||||
|
from: 'f-3',
|
||||||
|
to: 'f-2',
|
||||||
|
type: VerbType.Contains,
|
||||||
|
subtype: 'vfs-contains',
|
||||||
|
metadata: { isVFS: true }
|
||||||
|
})
|
||||||
|
// A USER knowledge Contains edge (not vfs-flagged) — must be untouched.
|
||||||
|
await brain.relate({ from: 'f-4', to: 'f-5', type: VerbType.Contains })
|
||||||
|
|
||||||
|
const spy = vi.spyOn(brain, 'related')
|
||||||
|
result = await vfs.repairContainment()
|
||||||
|
relatedCalls = spy.mock.calls.length
|
||||||
|
spy.mockRestore()
|
||||||
|
})
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await brain.close()
|
||||||
|
brain = null as any
|
||||||
|
})
|
||||||
|
|
||||||
|
it('restores the missing edge and removes the stale parent — exactly', () => {
|
||||||
|
expect(result.restored).toBe(1) // f-0's missing edge
|
||||||
|
expect(result.removed).toBe(1) // f-2's stale parent (f-3 → f-2)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('the repaired tree is correct: every file has exactly one vfs edge from its dir', async () => {
|
||||||
|
for (let i = 0; i < 6; i++) {
|
||||||
|
const incoming = await brain.related({ to: `f-${i}`, type: VerbType.Contains })
|
||||||
|
const vfsEdges = incoming.filter(
|
||||||
|
(e) => e.subtype === 'vfs-contains' || (e.metadata as any)?.isVFS === true
|
||||||
|
)
|
||||||
|
expect(vfsEdges, `f-${i}`).toHaveLength(1)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('never touches user knowledge edges', async () => {
|
||||||
|
const incoming = await brain.related({ to: 'f-5', type: VerbType.Contains })
|
||||||
|
const user = incoming.filter(
|
||||||
|
(e) => e.subtype !== 'vfs-contains' && (e.metadata as any)?.isVFS !== true
|
||||||
|
)
|
||||||
|
expect(user).toHaveLength(1)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('cost shape: related() calls do not scale with the entity count', () => {
|
||||||
|
// One paged type-only walk (~E/1000 pages) — with 60+ entities the old
|
||||||
|
// shape issued 60+ calls; the new one a handful. Bound generously.
|
||||||
|
expect(relatedCalls).toBeLessThanOrEqual(5)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
@ -9,9 +9,10 @@ import * as XLSX from 'xlsx'
|
||||||
describe('VFS Debug', () => {
|
describe('VFS Debug', () => {
|
||||||
it('minimal VFS writeFile test', async () => {
|
it('minimal VFS writeFile test', async () => {
|
||||||
const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||||
await brain.init()
|
try {
|
||||||
|
await brain.init()
|
||||||
|
|
||||||
console.log('✅ Brain initialized')
|
console.log('✅ Brain initialized')
|
||||||
|
|
||||||
// Get VFS and initialize
|
// Get VFS and initialize
|
||||||
const vfs = brain.vfs
|
const vfs = brain.vfs
|
||||||
|
|
@ -77,5 +78,8 @@ describe('VFS Debug', () => {
|
||||||
// THE REAL TEST: Can we query VFS?
|
// THE REAL TEST: Can we query VFS?
|
||||||
expect(children.length).toBeGreaterThan(0)
|
expect(children.length).toBeGreaterThan(0)
|
||||||
expect(rootContents.length).toBeGreaterThan(0)
|
expect(rootContents.length).toBeGreaterThan(0)
|
||||||
|
} finally {
|
||||||
|
await brain.close()
|
||||||
|
}
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -61,6 +61,7 @@ describe('writer-lock fencing', () => {
|
||||||
// Old rule: heartbeat-age eviction → silent takeover → split brain.
|
// Old rule: heartbeat-age eviction → silent takeover → split brain.
|
||||||
// New rule: live PID = live writer; the second opener throws typed.
|
// New rule: live PID = live writer; the second opener throws typed.
|
||||||
const second = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false })
|
const second = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false })
|
||||||
|
brains.push(second)
|
||||||
await expect(second.init()).rejects.toMatchObject({ code: 'BRAINY_WRITER_LOCKED' })
|
await expect(second.init()).rejects.toMatchObject({ code: 'BRAINY_WRITER_LOCKED' })
|
||||||
}, 120000)
|
}, 120000)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -352,106 +352,8 @@ describe('Triple Intelligence Performance at Scale', () => {
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('Triple Intelligence Correctness', () => {
|
// The former 'Triple Intelligence Correctness' describe (4 tests, no timing
|
||||||
let brain: Brainy
|
// assertions) moved to tests/integration/triple-intelligence-correctness.test.ts
|
||||||
let triple: TripleIntelligenceSystem
|
// so it runs in the default correctness gate — this whole directory
|
||||||
|
// (tests/performance/**) is excluded from that gate (see vitest.config.ts),
|
||||||
beforeAll(async () => {
|
// which had silently stopped running those 4 tests after the perf-lane split.
|
||||||
brain = new Brainy({ requireSubtype: false })
|
|
||||||
await brain.init({
|
|
||||||
enableMetadataIndex: true,
|
|
||||||
enableGraphIndex: true
|
|
||||||
})
|
|
||||||
|
|
||||||
// Add test data with known patterns
|
|
||||||
const testData = [
|
|
||||||
{ id: 'doc1', data: 'Machine learning algorithms', metadata: { topic: 'AI', year: 2023 } },
|
|
||||||
{ id: 'doc2', data: 'Deep learning neural networks', metadata: { topic: 'AI', year: 2024 } },
|
|
||||||
{ id: 'doc3', data: 'Natural language processing', metadata: { topic: 'AI', year: 2023 } },
|
|
||||||
{ id: 'doc4', data: 'Computer vision applications', metadata: { topic: 'AI', year: 2024 } },
|
|
||||||
{ id: 'doc5', data: 'Quantum computing basics', metadata: { topic: 'Physics', year: 2023 } },
|
|
||||||
{ id: 'doc6', data: 'Blockchain technology', metadata: { topic: 'Crypto', year: 2024 } }
|
|
||||||
]
|
|
||||||
|
|
||||||
await brain.addMany(testData)
|
|
||||||
|
|
||||||
// Add relationships
|
|
||||||
await brain.relate({ from: 'doc1', to: 'doc2', type: 'related' })
|
|
||||||
await brain.relate({ from: 'doc2', to: 'doc3', type: 'related' })
|
|
||||||
await brain.relate({ from: 'doc3', to: 'doc4', type: 'related' })
|
|
||||||
|
|
||||||
triple = brain.getTripleIntelligence()
|
|
||||||
})
|
|
||||||
|
|
||||||
afterAll(async () => {
|
|
||||||
await brain?.close()
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should return exact matches for field queries', async () => {
|
|
||||||
const results = await triple.find({
|
|
||||||
where: { topic: 'AI' },
|
|
||||||
limit: 10
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(results).toHaveLength(4)
|
|
||||||
for (const result of results) {
|
|
||||||
expect(result.metadata.topic).toBe('AI')
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle range queries correctly', async () => {
|
|
||||||
const results = await triple.find({
|
|
||||||
where: { year: { $gte: 2024 } },
|
|
||||||
limit: 10
|
|
||||||
})
|
|
||||||
|
|
||||||
expect(results).toHaveLength(3)
|
|
||||||
for (const result of results) {
|
|
||||||
expect(result.metadata.year).toBeGreaterThanOrEqual(2024)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should traverse graph relationships', async () => {
|
|
||||||
const results = await triple.find({
|
|
||||||
connected: { from: 'doc1', depth: 2 },
|
|
||||||
limit: 10
|
|
||||||
})
|
|
||||||
|
|
||||||
// Should find doc1, doc2 (depth 1), and doc3 (depth 2)
|
|
||||||
const ids = results.map(r => r.id)
|
|
||||||
expect(ids).toContain('doc1')
|
|
||||||
expect(ids).toContain('doc2')
|
|
||||||
expect(ids).toContain('doc3')
|
|
||||||
|
|
||||||
// Check depth values
|
|
||||||
const doc1Result = results.find(r => r.id === 'doc1')
|
|
||||||
const doc2Result = results.find(r => r.id === 'doc2')
|
|
||||||
const doc3Result = results.find(r => r.id === 'doc3')
|
|
||||||
|
|
||||||
expect(doc1Result?.depth).toBe(0)
|
|
||||||
expect(doc2Result?.depth).toBe(1)
|
|
||||||
expect(doc3Result?.depth).toBe(2)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should combine signals with proper fusion', async () => {
|
|
||||||
const results = await triple.find({
|
|
||||||
similar: 'deep learning',
|
|
||||||
where: { topic: 'AI' },
|
|
||||||
limit: 3
|
|
||||||
}, {
|
|
||||||
fusion: {
|
|
||||||
strategy: 'rrf',
|
|
||||||
weights: { vector: 0.7, field: 0.3 }
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// doc2 should rank highest (matches both signals)
|
|
||||||
expect(results[0].id).toBe('doc2')
|
|
||||||
expect(results[0].fusionScore).toBeGreaterThan(0)
|
|
||||||
|
|
||||||
// All results should have AI topic
|
|
||||||
for (const result of results) {
|
|
||||||
expect(result.metadata.topic).toBe('AI')
|
|
||||||
}
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
@ -17,7 +17,7 @@
|
||||||
* - Note limitations and edge cases
|
* - Note limitations and edge cases
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { describe, it, expect, beforeEach } from 'vitest'
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
import { Brainy } from '../../src/brainy.js'
|
import { Brainy } from '../../src/brainy.js'
|
||||||
import { TypeAwareStorageAdapter } from '../../src/storage/adapters/typeAwareStorageAdapter.js'
|
import { TypeAwareStorageAdapter } from '../../src/storage/adapters/typeAwareStorageAdapter.js'
|
||||||
import { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js'
|
import { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js'
|
||||||
|
|
@ -67,6 +67,10 @@ describe('TypeAware Performance Benchmarks', () => {
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await brainMemory.close()
|
||||||
|
})
|
||||||
|
|
||||||
it('should measure type-based query performance', async () => {
|
it('should measure type-based query performance', async () => {
|
||||||
// MEASURED: Query for one type (200 entities)
|
// MEASURED: Query for one type (200 entities)
|
||||||
const start = performance.now()
|
const start = performance.now()
|
||||||
|
|
|
||||||
122
tests/regression/metadata-field-typing.unit.test.ts
Normal file
122
tests/regression/metadata-field-typing.unit.test.ts
Normal file
|
|
@ -0,0 +1,122 @@
|
||||||
|
/**
|
||||||
|
* @module metadata-field-typing.unit.test
|
||||||
|
* @description Regression: a metadata field that holds more than one value
|
||||||
|
* KIND stays fully filterable on every kind it holds.
|
||||||
|
*
|
||||||
|
* The defect this pins, reproduced on the released engine: the metadata index
|
||||||
|
* fixed a field's value type from the FIRST value it saw, and every later value
|
||||||
|
* of a different type was coerced to that type or, when coercion failed,
|
||||||
|
* dropped from the index in silence. Writing `category: 'electronics'` rows and
|
||||||
|
* then `category: 5` rows left `find({ where: { category: 5 } })` returning
|
||||||
|
* nothing — while the same rows in a numbers-only corpus answered correctly.
|
||||||
|
* The rows themselves were never lost: they stayed readable by id and by vector
|
||||||
|
* search, and only ever went missing from equality filters on that one field,
|
||||||
|
* which is what made it so quiet.
|
||||||
|
*
|
||||||
|
* Order is the whole point of these cases. Neither writer owns the field, so
|
||||||
|
* strings-then-numbers and numbers-then-strings must give the same answers.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect } from 'vitest'
|
||||||
|
import { Brainy } from '../../src/brainy.js'
|
||||||
|
import { NounType } from '../../src/types/graphTypes.js'
|
||||||
|
|
||||||
|
/** A brain over memory storage, with a corpus written in the given order. */
|
||||||
|
async function brainWith(
|
||||||
|
rows: Array<{ label: string; category: unknown }>
|
||||||
|
): Promise<Brainy> {
|
||||||
|
const brainy = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||||
|
await brainy.init()
|
||||||
|
for (const row of rows) {
|
||||||
|
await brainy.add({
|
||||||
|
data: `item ${row.label}`,
|
||||||
|
type: NounType.Thing,
|
||||||
|
metadata: { label: row.label, category: row.category }
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return brainy
|
||||||
|
}
|
||||||
|
|
||||||
|
const labelsOf = (results: Array<{ metadata?: Record<string, unknown> }>): string[] =>
|
||||||
|
results.map((r) => String(r.metadata?.label)).sort()
|
||||||
|
|
||||||
|
describe('regression: a mixed-kind metadata field filters on every kind', { timeout: 180_000 }, () => {
|
||||||
|
it('finds number rows written after string rows', async () => {
|
||||||
|
const brainy = await brainWith([
|
||||||
|
{ label: 'e1', category: 'electronics' },
|
||||||
|
{ label: 'f1', category: 'furniture' },
|
||||||
|
{ label: 'n1', category: 5 },
|
||||||
|
{ label: 'n2', category: 5 },
|
||||||
|
{ label: 'n3', category: 7 }
|
||||||
|
])
|
||||||
|
try {
|
||||||
|
expect(labelsOf(await brainy.find({ where: { category: 5 }, limit: 100 }))).toEqual(['n1', 'n2'])
|
||||||
|
expect(labelsOf(await brainy.find({ where: { category: 7 }, limit: 100 }))).toEqual(['n3'])
|
||||||
|
expect(labelsOf(await brainy.find({ where: { category: 'electronics' }, limit: 100 }))).toEqual(['e1'])
|
||||||
|
expect(labelsOf(await brainy.find({ where: { category: 'furniture' }, limit: 100 }))).toEqual(['f1'])
|
||||||
|
} finally {
|
||||||
|
await brainy.close()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('finds string rows written after number rows', async () => {
|
||||||
|
const brainy = await brainWith([
|
||||||
|
{ label: 'n1', category: 5 },
|
||||||
|
{ label: 'n2', category: 5 },
|
||||||
|
{ label: 'e1', category: 'electronics' },
|
||||||
|
{ label: 'e2', category: 'electronics' }
|
||||||
|
])
|
||||||
|
try {
|
||||||
|
expect(labelsOf(await brainy.find({ where: { category: 'electronics' }, limit: 100 }))).toEqual(['e1', 'e2'])
|
||||||
|
expect(labelsOf(await brainy.find({ where: { category: 5 }, limit: 100 }))).toEqual(['n1', 'n2'])
|
||||||
|
} finally {
|
||||||
|
await brainy.close()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('keeps `5` and `\'5\'` apart — a kind is part of the value, not a formatting detail', async () => {
|
||||||
|
const brainy = await brainWith([
|
||||||
|
{ label: 'num', category: 5 },
|
||||||
|
{ label: 'str', category: '5' }
|
||||||
|
])
|
||||||
|
try {
|
||||||
|
expect(labelsOf(await brainy.find({ where: { category: 5 }, limit: 100 }))).toEqual(['num'])
|
||||||
|
expect(labelsOf(await brainy.find({ where: { category: '5' }, limit: 100 }))).toEqual(['str'])
|
||||||
|
} finally {
|
||||||
|
await brainy.close()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('serves booleans mixed into a field that already holds strings', async () => {
|
||||||
|
const brainy = await brainWith([
|
||||||
|
{ label: 's1', category: 'yes' },
|
||||||
|
{ label: 'b1', category: true },
|
||||||
|
{ label: 'b2', category: false }
|
||||||
|
])
|
||||||
|
try {
|
||||||
|
expect(labelsOf(await brainy.find({ where: { category: true }, limit: 100 }))).toEqual(['b1'])
|
||||||
|
expect(labelsOf(await brainy.find({ where: { category: false }, limit: 100 }))).toEqual(['b2'])
|
||||||
|
expect(labelsOf(await brainy.find({ where: { category: 'yes' }, limit: 100 }))).toEqual(['s1'])
|
||||||
|
} finally {
|
||||||
|
await brainy.close()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('ranges over the numeric part of a mixed field', async () => {
|
||||||
|
const brainy = await brainWith([
|
||||||
|
{ label: 'unpriced', category: 'on request' },
|
||||||
|
{ label: 'cheap', category: 100 },
|
||||||
|
{ label: 'mid', category: 500 },
|
||||||
|
{ label: 'dear', category: 900 }
|
||||||
|
])
|
||||||
|
try {
|
||||||
|
const found = await brainy.find({
|
||||||
|
where: { category: { greaterThan: 200 } },
|
||||||
|
limit: 100
|
||||||
|
})
|
||||||
|
expect(labelsOf(found)).toEqual(['dear', 'mid'])
|
||||||
|
} finally {
|
||||||
|
await brainy.close()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
@ -5,7 +5,7 @@
|
||||||
* No mocks, no fakes, real implementation
|
* No mocks, no fakes, real implementation
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { describe, it, expect, beforeEach } from 'vitest'
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
import { Brainy } from '../../src/brainy.js'
|
import { Brainy } from '../../src/brainy.js'
|
||||||
import { NounType } from '../../src/types/graphTypes.js'
|
import { NounType } from '../../src/types/graphTypes.js'
|
||||||
|
|
||||||
|
|
@ -21,6 +21,10 @@ describe('Brainy 3.0 Core (Unit Tests)', () => {
|
||||||
await brain.init()
|
await brain.init()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
|
||||||
describe('CRUD Operations', () => {
|
describe('CRUD Operations', () => {
|
||||||
it('should create items with add', async () => {
|
it('should create items with add', async () => {
|
||||||
const id = await brain.add({
|
const id = await brain.add({
|
||||||
|
|
|
||||||
|
|
@ -113,7 +113,12 @@ describe('Brainy Batch Operations', () => {
|
||||||
items: Array.from({ length: 100 }, (_, i) => ({
|
items: Array.from({ length: 100 }, (_, i) => ({
|
||||||
data: `Bulk ${i}`,
|
data: `Bulk ${i}`,
|
||||||
type: NounType.Thing,
|
type: NounType.Thing,
|
||||||
metadata: { counter: 0 }
|
metadata: { counter: 0 },
|
||||||
|
// This test exercises updateMany's batching, not embedding — the
|
||||||
|
// sanctioned "unvectored" `[]` shape (see
|
||||||
|
// tests/integration/index-skips-unvectored.test.ts) skips the
|
||||||
|
// real embedder entirely.
|
||||||
|
vector: []
|
||||||
}))
|
}))
|
||||||
})
|
})
|
||||||
const manyIds = manyResult.successful
|
const manyIds = manyResult.successful
|
||||||
|
|
@ -274,7 +279,12 @@ describe('Brainy Batch Operations', () => {
|
||||||
const manyResult = await brain.addMany({
|
const manyResult = await brain.addMany({
|
||||||
items: Array.from({ length: 100 }, (_, i) => ({
|
items: Array.from({ length: 100 }, (_, i) => ({
|
||||||
data: `Bulk Delete ${i}`,
|
data: `Bulk Delete ${i}`,
|
||||||
type: NounType.Thing
|
type: NounType.Thing,
|
||||||
|
// This test exercises removeMany's batching, not embedding — the
|
||||||
|
// sanctioned "unvectored" `[]` shape (see
|
||||||
|
// tests/integration/index-skips-unvectored.test.ts) skips the
|
||||||
|
// real embedder entirely.
|
||||||
|
vector: []
|
||||||
}))
|
}))
|
||||||
})
|
})
|
||||||
const manyIds = manyResult.successful
|
const manyIds = manyResult.successful
|
||||||
|
|
@ -545,10 +555,18 @@ describe('Brainy Batch Operations', () => {
|
||||||
|
|
||||||
it('should validate batch size limits', async () => {
|
it('should validate batch size limits', async () => {
|
||||||
// Try to add a large batch (reduced from 10000 to 1000 for reasonable test time)
|
// Try to add a large batch (reduced from 10000 to 1000 for reasonable test time)
|
||||||
|
// This test validates the batch SIZE law, not embeddings — items carry
|
||||||
|
// the sanctioned "unvectored" `[]` shape (see
|
||||||
|
// tests/integration/index-skips-unvectored.test.ts) so addMany's batch
|
||||||
|
// embedder is never invoked; 1000 real embeddings under the root
|
||||||
|
// vitest config (which does not mock the embedder) is a 60-180s
|
||||||
|
// budget flake waiting to happen, not a defect in what this test
|
||||||
|
// actually asserts.
|
||||||
const largeCount = 1000
|
const largeCount = 1000
|
||||||
const largeItems = Array.from({ length: largeCount }, (_, i) => ({
|
const largeItems = Array.from({ length: largeCount }, (_, i) => ({
|
||||||
data: `Large ${i}`,
|
data: `Large ${i}`,
|
||||||
type: NounType.Thing
|
type: NounType.Thing,
|
||||||
|
vector: []
|
||||||
}))
|
}))
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
@ -560,12 +578,7 @@ describe('Brainy Batch Operations', () => {
|
||||||
// Might throw if there's a limit
|
// Might throw if there's a limit
|
||||||
expect(error).toBeDefined()
|
expect(error).toBeDefined()
|
||||||
}
|
}
|
||||||
// order-of-magnitude guard: this test batches 20x the item count of the
|
})
|
||||||
// sibling "perform better" test above (worst measured 11.9s for 50
|
|
||||||
// items on CPU-only honest iron); the prior 60s timeout was itself
|
|
||||||
// observed being hit, so this is 3x that floor rather than a scaled
|
|
||||||
// extrapolation, to leave real headroom for run-to-run variance
|
|
||||||
}, 180000)
|
|
||||||
|
|
||||||
it('should provide meaningful error messages', async () => {
|
it('should provide meaningful error messages', async () => {
|
||||||
try {
|
try {
|
||||||
|
|
|
||||||
|
|
@ -19,13 +19,19 @@ import { prodLog } from '../../../src/utils/logger.js'
|
||||||
const UUID = (suffix: string): string => `00000000-0000-4000-8000-0000000000${suffix}`
|
const UUID = (suffix: string): string => `00000000-0000-4000-8000-0000000000${suffix}`
|
||||||
|
|
||||||
describe('Finding 10 — degraded derived-index state is surfaced on reads', () => {
|
describe('Finding 10 — degraded derived-index state is surfaced on reads', () => {
|
||||||
|
const opened: Brainy[] = []
|
||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
||||||
})
|
})
|
||||||
afterEach(() => vi.restoreAllMocks())
|
afterEach(async () => {
|
||||||
|
vi.restoreAllMocks()
|
||||||
|
for (const b of opened.splice(0)) await b.close().catch(() => {})
|
||||||
|
})
|
||||||
|
|
||||||
it('checkHealth() reports adopt-forward degraded ids as unhealthy', async () => {
|
it('checkHealth() reports adopt-forward degraded ids as unhealthy', async () => {
|
||||||
const brain = new Brainy({ storage: { type: 'memory' }, dimensions: 384, requireSubtype: false })
|
const brain = new Brainy({ storage: { type: 'memory' }, dimensions: 384, requireSubtype: false })
|
||||||
|
opened.push(brain)
|
||||||
await brain.init()
|
await brain.init()
|
||||||
;(brain as any)._indexDegradedIds.add(UUID('de'))
|
;(brain as any)._indexDegradedIds.add(UUID('de'))
|
||||||
|
|
||||||
|
|
@ -37,6 +43,7 @@ describe('Finding 10 — degraded derived-index state is surfaced on reads', ()
|
||||||
it('find()/get() warn loudly while degraded, ONCE, then repairIndex() clears it', async () => {
|
it('find()/get() warn loudly while degraded, ONCE, then repairIndex() clears it', async () => {
|
||||||
const warn = vi.spyOn(prodLog, 'warn').mockImplementation(() => {})
|
const warn = vi.spyOn(prodLog, 'warn').mockImplementation(() => {})
|
||||||
const brain = new Brainy({ storage: { type: 'memory' }, dimensions: 384, requireSubtype: false })
|
const brain = new Brainy({ storage: { type: 'memory' }, dimensions: 384, requireSubtype: false })
|
||||||
|
opened.push(brain)
|
||||||
await brain.init()
|
await brain.init()
|
||||||
await brain.add({ id: UUID('a1'), data: 'x', type: NounType.Document })
|
await brain.add({ id: UUID('a1'), data: 'x', type: NounType.Document })
|
||||||
;(brain as any)._indexRebuildFailed = new Error('rebuild boom')
|
;(brain as any)._indexRebuildFailed = new Error('rebuild boom')
|
||||||
|
|
@ -59,6 +66,7 @@ describe('Finding 10 — degraded derived-index state is surfaced on reads', ()
|
||||||
|
|
||||||
it('persistSingleOp records receipt.degraded (widened return type, not dropped)', async () => {
|
it('persistSingleOp records receipt.degraded (widened return type, not dropped)', async () => {
|
||||||
const brain = new Brainy({ storage: { type: 'memory' }, dimensions: 384, requireSubtype: false })
|
const brain = new Brainy({ storage: { type: 'memory' }, dimensions: 384, requireSubtype: false })
|
||||||
|
opened.push(brain)
|
||||||
await brain.init()
|
await brain.init()
|
||||||
// Simulate a degraded receipt by wrapping the generation store's commitSingleOp.
|
// Simulate a degraded receipt by wrapping the generation store's commitSingleOp.
|
||||||
const gs: any = (brain as any).generationStore
|
const gs: any = (brain as any).generationStore
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@
|
||||||
* soft-delete semantic: `field !== value` MUST include entities that have no
|
* soft-delete semantic: `field !== value` MUST include entities that have no
|
||||||
* such field at all.
|
* such field at all.
|
||||||
*/
|
*/
|
||||||
import { describe, it, expect, beforeEach } from 'vitest'
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
import { Brainy } from '../../../src/brainy'
|
import { Brainy } from '../../../src/brainy'
|
||||||
import { NounType } from '../../../src/types/graphTypes'
|
import { NounType } from '../../../src/types/graphTypes'
|
||||||
|
|
||||||
|
|
@ -26,6 +26,10 @@ describe('find() complement operators (ne / exists:false / missing:true)', () =>
|
||||||
ids.noField2 = await brain.add({ data: 'n2', type: NounType.Thing, metadata: { other: 2 } })
|
ids.noField2 = await brain.add({ data: 'n2', type: NounType.Thing, metadata: { other: 2 } })
|
||||||
})
|
})
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
|
||||||
it('ne returns everything except the matching value — INCLUDING entities without the field', async () => {
|
it('ne returns everything except the matching value — INCLUDING entities without the field', async () => {
|
||||||
const rows = await brain.find({ where: { status: { ne: 'active' } }, limit: 100 })
|
const rows = await brain.find({ where: { status: { ne: 'active' } }, limit: 100 })
|
||||||
const got = new Set(rows.map((r) => r.id))
|
const got = new Set(rows.map((r) => r.id))
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,7 @@
|
||||||
* returns an id whose record matches NEITHER the type nor the where filter) and
|
* returns an id whose record matches NEITHER the type nor the where filter) and
|
||||||
* assert the phantom is dropped while the genuine matches survive.
|
* assert the phantom is dropped while the genuine matches survive.
|
||||||
*/
|
*/
|
||||||
import { describe, it, expect, beforeEach } from 'vitest'
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
import { Brainy } from '../../../src/brainy'
|
import { Brainy } from '../../../src/brainy'
|
||||||
import { NounType } from '../../../src/types/graphTypes'
|
import { NounType } from '../../../src/types/graphTypes'
|
||||||
|
|
||||||
|
|
@ -48,6 +48,10 @@ describe('find() index-integrity guard (phantom row class)', () => {
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
|
||||||
it('healthy index: the discriminant query returns only the staff Person', async () => {
|
it('healthy index: the discriminant query returns only the staff Person', async () => {
|
||||||
const rows = await brain.find({ type: NounType.Person, where: { entityType: 'staff' }, limit: 100 })
|
const rows = await brain.find({ type: NounType.Person, where: { entityType: 'staff' }, limit: 100 })
|
||||||
expect(rows.map((r) => r.id)).toEqual([staffId])
|
expect(rows.map((r) => r.id)).toEqual([staffId])
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
import { describe, it, expect, beforeEach } from 'vitest'
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
import { Brainy } from '../../../src/brainy'
|
import { Brainy } from '../../../src/brainy'
|
||||||
import { createAddParams } from '../../helpers/test-factory'
|
import { createAddParams } from '../../helpers/test-factory'
|
||||||
import { NounType } from '../../../src/types/graphTypes'
|
import { NounType } from '../../../src/types/graphTypes'
|
||||||
|
|
@ -12,7 +12,11 @@ describe('Brainy.find()', () => {
|
||||||
})
|
})
|
||||||
await brain.init()
|
await brain.init()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
|
||||||
describe('success paths', () => {
|
describe('success paths', () => {
|
||||||
it('should find entities by text query', async () => {
|
it('should find entities by text query', async () => {
|
||||||
// Arrange
|
// Arrange
|
||||||
|
|
|
||||||
175
tests/unit/brainy/flush-single-flight.test.ts
Normal file
175
tests/unit/brainy/flush-single-flight.test.ts
Normal file
|
|
@ -0,0 +1,175 @@
|
||||||
|
/**
|
||||||
|
* @module tests/unit/brainy/flush-single-flight
|
||||||
|
* @description THE FLUSH GATE NEVER STRANDS A WAITER.
|
||||||
|
*
|
||||||
|
* The gate serialises flushes: one body runs, at most one waits. The failure
|
||||||
|
* mode that shape invites is a promise CYCLE — a queued follow-up expressed as
|
||||||
|
* `leader.then(() => this.flush())` is settled only by resolving the promise
|
||||||
|
* the leader is being awaited through, so anything that awaits `flush()` from
|
||||||
|
* inside a flush body closes the graph on itself and nobody ever resolves.
|
||||||
|
* That is an unbounded hang, not a slow flush, and it presents exactly like a
|
||||||
|
* test timing out inside a bulk write.
|
||||||
|
*
|
||||||
|
* The gate therefore settles its waiter from the MACHINE (a bare deferred
|
||||||
|
* promoted in the leader's `finally`), never from a chain. The laws pinned
|
||||||
|
* here, each on a path that must settle the waiter:
|
||||||
|
*
|
||||||
|
* (a) many callers during one running flush → one body, one follow-up, and
|
||||||
|
* EVERY caller resolves within a bound;
|
||||||
|
* (b) the leader REJECTS → its own caller rejects, and the queued caller is
|
||||||
|
* still run and still settled;
|
||||||
|
* (c) the promoted follow-up itself rejects → its waiter rejects (settled,
|
||||||
|
* not stranded) and the gate is left open for the next flush;
|
||||||
|
* (d) the leader's promise does not wait for its follower.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, afterEach } from 'vitest'
|
||||||
|
import { Brainy } from '../../../src/brainy'
|
||||||
|
import { NounType } from '../../../src/types/graphTypes'
|
||||||
|
|
||||||
|
type GateInternals = {
|
||||||
|
_flushInFlight: Promise<void> | null
|
||||||
|
_flushQueued: Promise<void> | null
|
||||||
|
_flushBodyRuns: number
|
||||||
|
_flushConcurrencyPeak: number
|
||||||
|
_flushSteps: () => Promise<void>
|
||||||
|
kickBackgroundFlush: (reason: 'threshold' | 'idle') => void
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Fail loudly rather than hanging the suite: a stranded waiter never settles. */
|
||||||
|
function withinBound<T>(p: Promise<T>, ms: number, what: string): Promise<T> {
|
||||||
|
let timer: ReturnType<typeof setTimeout>
|
||||||
|
return Promise.race([
|
||||||
|
p,
|
||||||
|
new Promise<never>((_, reject) => {
|
||||||
|
timer = setTimeout(() => reject(new Error(`${what} did not settle within ${ms}ms`)), ms)
|
||||||
|
})
|
||||||
|
]).finally(() => clearTimeout(timer)) as Promise<T>
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('the flush gate settles every waiter', () => {
|
||||||
|
const brains: Brainy<any>[] = []
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
for (const b of brains.splice(0)) {
|
||||||
|
try { await b.close() } catch { /* already closed */ }
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
async function openBrain(): Promise<Brainy<any>> {
|
||||||
|
const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||||
|
brains.push(brain)
|
||||||
|
await brain.init()
|
||||||
|
await brain.add({ data: 'a write, so a flush has work', type: NounType.Thing })
|
||||||
|
return brain
|
||||||
|
}
|
||||||
|
|
||||||
|
it('(a) every caller arriving during one flush resolves, and only one follows', async () => {
|
||||||
|
const brain = await openBrain()
|
||||||
|
const inner = brain as unknown as GateInternals
|
||||||
|
|
||||||
|
const realSteps = inner._flushSteps.bind(inner)
|
||||||
|
inner._flushSteps = async () => {
|
||||||
|
await new Promise((r) => setTimeout(r, 120))
|
||||||
|
return realSteps()
|
||||||
|
}
|
||||||
|
|
||||||
|
const runsBefore = inner._flushBodyRuns
|
||||||
|
const leader = brain.flush()
|
||||||
|
await new Promise((r) => setTimeout(r, 20))
|
||||||
|
|
||||||
|
const joiners = [brain.flush(), brain.flush(), brain.flush(), brain.flush()]
|
||||||
|
for (let i = 0; i < 4; i++) inner.kickBackgroundFlush('threshold')
|
||||||
|
expect(inner._flushQueued, 'exactly one waiter is queued').not.toBeNull()
|
||||||
|
|
||||||
|
await withinBound(Promise.all([leader, ...joiners]), 15_000, 'the flush callers')
|
||||||
|
|
||||||
|
expect(inner._flushBodyRuns - runsBefore).toBe(2)
|
||||||
|
expect(inner._flushConcurrencyPeak).toBe(1)
|
||||||
|
expect(inner._flushQueued).toBeNull()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('(b) a leader that REJECTS still runs and settles the queued waiter', async () => {
|
||||||
|
const brain = await openBrain()
|
||||||
|
const inner = brain as unknown as GateInternals
|
||||||
|
|
||||||
|
const realSteps = inner._flushSteps.bind(inner)
|
||||||
|
let call = 0
|
||||||
|
inner._flushSteps = async () => {
|
||||||
|
call++
|
||||||
|
await new Promise((r) => setTimeout(r, 80))
|
||||||
|
if (call === 1) throw new Error('injected: the leader flush failed')
|
||||||
|
return realSteps()
|
||||||
|
}
|
||||||
|
|
||||||
|
const leader = brain.flush()
|
||||||
|
await new Promise((r) => setTimeout(r, 20))
|
||||||
|
const queued = brain.flush()
|
||||||
|
|
||||||
|
await expect(leader).rejects.toThrow(/injected: the leader flush failed/)
|
||||||
|
// The waiter is NOT collateral damage of the leader's failure: it gets its
|
||||||
|
// own run, and it settles.
|
||||||
|
await withinBound(queued, 15_000, 'the queued waiter after a failed leader')
|
||||||
|
expect(call).toBe(2)
|
||||||
|
expect(inner._flushQueued).toBeNull()
|
||||||
|
expect(inner._flushInFlight).toBeNull()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('(c) a promoted follow-up that rejects settles its waiter and opens the gate', async () => {
|
||||||
|
const brain = await openBrain()
|
||||||
|
const inner = brain as unknown as GateInternals
|
||||||
|
|
||||||
|
const realSteps = inner._flushSteps.bind(inner)
|
||||||
|
let call = 0
|
||||||
|
inner._flushSteps = async () => {
|
||||||
|
call++
|
||||||
|
await new Promise((r) => setTimeout(r, 80))
|
||||||
|
if (call === 2) throw new Error('injected: the follow-up flush failed')
|
||||||
|
return realSteps()
|
||||||
|
}
|
||||||
|
|
||||||
|
const leader = brain.flush()
|
||||||
|
await new Promise((r) => setTimeout(r, 20))
|
||||||
|
const queued = brain.flush()
|
||||||
|
|
||||||
|
await withinBound(leader, 15_000, 'the leader')
|
||||||
|
await withinBound(
|
||||||
|
expect(queued).rejects.toThrow(/injected: the follow-up flush failed/),
|
||||||
|
15_000,
|
||||||
|
'the rejected follow-up'
|
||||||
|
)
|
||||||
|
// The gate is open: a later flush still runs.
|
||||||
|
inner._flushSteps = realSteps
|
||||||
|
await brain.add({ data: 'another write', type: NounType.Thing })
|
||||||
|
await withinBound(brain.flush(), 15_000, 'the flush after a failed follow-up')
|
||||||
|
expect(inner._flushInFlight).toBeNull()
|
||||||
|
expect(inner._flushQueued).toBeNull()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('(d) the leader does not wait for its follower', async () => {
|
||||||
|
const brain = await openBrain()
|
||||||
|
const inner = brain as unknown as GateInternals
|
||||||
|
|
||||||
|
const realSteps = inner._flushSteps.bind(inner)
|
||||||
|
let call = 0
|
||||||
|
inner._flushSteps = async () => {
|
||||||
|
call++
|
||||||
|
// The follow-up is deliberately far slower than the leader.
|
||||||
|
await new Promise((r) => setTimeout(r, call === 1 ? 60 : 600))
|
||||||
|
return realSteps()
|
||||||
|
}
|
||||||
|
|
||||||
|
const leader = brain.flush()
|
||||||
|
await new Promise((r) => setTimeout(r, 20))
|
||||||
|
const queued = brain.flush()
|
||||||
|
|
||||||
|
const t0 = Date.now()
|
||||||
|
await withinBound(leader, 15_000, 'the leader')
|
||||||
|
const leaderWall = Date.now() - t0
|
||||||
|
// If the leader awaited its follower it could not return before the
|
||||||
|
// follower's own 600ms body had run.
|
||||||
|
expect(leaderWall).toBeLessThan(500)
|
||||||
|
|
||||||
|
await withinBound(queued, 15_000, 'the follower')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
@ -5,7 +5,8 @@
|
||||||
|
|
||||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
import { Brainy } from '../../../src/brainy'
|
import { Brainy } from '../../../src/brainy'
|
||||||
import {
|
import { MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from '../../../src/errors/brainyError'
|
||||||
|
import {
|
||||||
createAddParams,
|
createAddParams,
|
||||||
generateTestVector,
|
generateTestVector,
|
||||||
createTestConfig,
|
createTestConfig,
|
||||||
|
|
@ -268,32 +269,75 @@ describe('Brainy.get()', () => {
|
||||||
expect(entity!.id).toBe(id)
|
expect(entity!.id).toBe(id)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should get entity with very large metadata', async () => {
|
// THE INDEXABLE-ARRAY BOUND, from get()'s side. This case used to park a
|
||||||
// Arrange
|
// 1000-element array in the metadata bag and assert it came back. That
|
||||||
|
// shape is refused at the write door now — an array field mints one
|
||||||
|
// posting per element, so an unbounded array is an unbounded write — so
|
||||||
|
// the case pins BOTH halves of the law that replaced it: a large SCALAR
|
||||||
|
// payload still round-trips whole, and an array over the bound refuses by
|
||||||
|
// name. Every length derives from MAX_INDEXED_ARRAY_LENGTH so the pin
|
||||||
|
// follows the constant wherever it moves.
|
||||||
|
it('should get an entity with a large scalar metadata payload', async () => {
|
||||||
|
// Arrange — large in every dimension EXCEPT array length: a long string,
|
||||||
|
// many fields, deep nesting, and an array sitting exactly ON the bound.
|
||||||
const largeMetadata = {
|
const largeMetadata = {
|
||||||
bigArray: new Array(1000).fill('item'),
|
atTheBound: Array.from({ length: MAX_INDEXED_ARRAY_LENGTH }, (_, i) => `item${i}`),
|
||||||
bigObject: Object.fromEntries(
|
bigObject: Object.fromEntries(
|
||||||
Array.from({ length: 100 }, (_, i) => [`key${i}`, `value${i}`])
|
Array.from({ length: 100 }, (_, i) => [`key${i}`, `value${i}`])
|
||||||
),
|
),
|
||||||
|
longString: 'x'.repeat(10_000),
|
||||||
deepNesting: Array(10).fill(null).reduce(
|
deepNesting: Array(10).fill(null).reduce(
|
||||||
(acc) => ({ nested: acc }),
|
(acc) => ({ nested: acc }),
|
||||||
{ value: 'deep' }
|
{ value: 'deep' }
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
const id = await brain.add(createAddParams({
|
const id = await brain.add(createAddParams({
|
||||||
data: 'Large metadata',
|
data: 'Large metadata',
|
||||||
type: 'thing',
|
type: 'thing',
|
||||||
metadata: largeMetadata
|
metadata: largeMetadata
|
||||||
}))
|
}))
|
||||||
|
|
||||||
// Act
|
// Act
|
||||||
const entity = await brain.get(id)
|
const entity = await brain.get(id)
|
||||||
|
|
||||||
// Assert
|
// Assert — the payload comes back whole, first element to last
|
||||||
expect(entity).not.toBeNull()
|
expect(entity).not.toBeNull()
|
||||||
expect(entity!.metadata.bigArray).toHaveLength(1000)
|
expect(entity!.metadata.atTheBound).toHaveLength(MAX_INDEXED_ARRAY_LENGTH)
|
||||||
|
expect(entity!.metadata.atTheBound[0]).toBe('item0')
|
||||||
|
expect(entity!.metadata.atTheBound[MAX_INDEXED_ARRAY_LENGTH - 1])
|
||||||
|
.toBe(`item${MAX_INDEXED_ARRAY_LENGTH - 1}`)
|
||||||
expect(Object.keys(entity!.metadata.bigObject)).toHaveLength(100)
|
expect(Object.keys(entity!.metadata.bigObject)).toHaveLength(100)
|
||||||
|
expect(entity!.metadata.longString).toHaveLength(10_000)
|
||||||
|
|
||||||
|
// ...including the deep nest, walked to the bottom.
|
||||||
|
let cursor: any = entity!.metadata.deepNesting
|
||||||
|
for (let depth = 0; depth < 10; depth++) cursor = cursor.nested
|
||||||
|
expect(cursor.value).toBe('deep')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should refuse a metadata array over the indexing bound, by name', async () => {
|
||||||
|
// Arrange
|
||||||
|
const overTheBound = MAX_INDEXED_ARRAY_LENGTH + 1
|
||||||
|
|
||||||
|
// Act
|
||||||
|
const err = await brain
|
||||||
|
.add(createAddParams({
|
||||||
|
data: 'Large metadata',
|
||||||
|
type: 'thing',
|
||||||
|
metadata: { bigArray: new Array(overTheBound).fill('item') }
|
||||||
|
}))
|
||||||
|
.catch((e: any) => e)
|
||||||
|
|
||||||
|
// Assert — the field, the length and the bound, on the error and in the
|
||||||
|
// message, so a handler can report or repair without parsing prose.
|
||||||
|
expect(err).toBeInstanceOf(MetadataArrayTooLargeError)
|
||||||
|
expect(err.field).toBe('bigArray')
|
||||||
|
expect(err.length).toBe(overTheBound)
|
||||||
|
expect(err.limit).toBe(MAX_INDEXED_ARRAY_LENGTH)
|
||||||
|
expect(err.message).toContain('bigArray')
|
||||||
|
expect(err.message).toContain(String(overTheBound))
|
||||||
|
expect(err.message).toContain(String(MAX_INDEXED_ARRAY_LENGTH))
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@
|
||||||
* exercised by cor's combined matrix); they inject probe/spy hooks onto the live JS
|
* exercised by cor's combined matrix); they inject probe/spy hooks onto the live JS
|
||||||
* metadata index, which has neither method by default.
|
* metadata index, which has neither method by default.
|
||||||
*/
|
*/
|
||||||
import { describe, it, expect, beforeEach } from 'vitest'
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
import { Brainy } from '../../../src/brainy'
|
import { Brainy } from '../../../src/brainy'
|
||||||
import { NounType } from '../../../src/types/graphTypes'
|
import { NounType } from '../../../src/types/graphTypes'
|
||||||
|
|
||||||
|
|
@ -34,6 +34,10 @@ describe('metadata-provider contract wiring (getIdsForFilter opts)', () => {
|
||||||
mi = (brain as any).metadataIndex
|
mi = (brain as any).metadataIndex
|
||||||
})
|
})
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
|
||||||
it('RETIRED: a read never calls probeConsistency() / self-heals via detectAndRepairCorruption — that is the read-triggered dark rebuild the health-gate law forbids', async () => {
|
it('RETIRED: a read never calls probeConsistency() / self-heals via detectAndRepairCorruption — that is the read-triggered dark rebuild the health-gate law forbids', async () => {
|
||||||
let probes = 0
|
let probes = 0
|
||||||
let repairs = 0
|
let repairs = 0
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@
|
||||||
* gate that hung getStats / readdir / readFile behind an unrelated family's
|
* gate that hung getStats / readdir / readFile behind an unrelated family's
|
||||||
* migration until the wait timed out.
|
* migration until the wait timed out.
|
||||||
*/
|
*/
|
||||||
import { describe, it, expect, beforeEach } from 'vitest'
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
import { Brainy } from '../../../src/brainy.js'
|
import { Brainy } from '../../../src/brainy.js'
|
||||||
import { MigrationInProgressError } from '../../../src/errors/brainyError.js'
|
import { MigrationInProgressError } from '../../../src/errors/brainyError.js'
|
||||||
|
|
||||||
|
|
@ -38,12 +38,19 @@ const jam = (provider: unknown) => {
|
||||||
}
|
}
|
||||||
|
|
||||||
describe('migration LOCK is family-scoped', () => {
|
describe('migration LOCK is family-scoped', () => {
|
||||||
|
const opened: Brainy[] = []
|
||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
||||||
})
|
})
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
for (const b of opened.splice(0)) await b.close().catch(() => {})
|
||||||
|
})
|
||||||
|
|
||||||
it('a stuck VECTOR migration does not block canonical or graph/metadata reads', async () => {
|
it('a stuck VECTOR migration does not block canonical or graph/metadata reads', async () => {
|
||||||
const brain = await seed()
|
const brain = await seed()
|
||||||
|
opened.push(brain)
|
||||||
const childId = (
|
const childId = (
|
||||||
(await brain.vfs.readdir('/notes', { withFileTypes: true })) as Array<{ entityId: string }>
|
(await brain.vfs.readdir('/notes', { withFileTypes: true })) as Array<{ entityId: string }>
|
||||||
)[0].entityId
|
)[0].entityId
|
||||||
|
|
@ -60,6 +67,7 @@ describe('migration LOCK is family-scoped', () => {
|
||||||
|
|
||||||
it('a stuck VECTOR migration STILL blocks a read that needs the vector family', async () => {
|
it('a stuck VECTOR migration STILL blocks a read that needs the vector family', async () => {
|
||||||
const brain = await seed()
|
const brain = await seed()
|
||||||
|
opened.push(brain)
|
||||||
jam((brain as any).index)
|
jam((brain as any).index)
|
||||||
|
|
||||||
// A semantic query consults the vector index — it must wait, and (bounded by
|
// A semantic query consults the vector index — it must wait, and (bounded by
|
||||||
|
|
@ -70,6 +78,7 @@ describe('migration LOCK is family-scoped', () => {
|
||||||
|
|
||||||
it('a stuck GRAPH migration blocks traversal but not vector/canonical reads', async () => {
|
it('a stuck GRAPH migration blocks traversal but not vector/canonical reads', async () => {
|
||||||
const brain = await seed()
|
const brain = await seed()
|
||||||
|
opened.push(brain)
|
||||||
const childId = (
|
const childId = (
|
||||||
(await brain.vfs.readdir('/notes', { withFileTypes: true })) as Array<{ entityId: string }>
|
(await brain.vfs.readdir('/notes', { withFileTypes: true })) as Array<{ entityId: string }>
|
||||||
)[0].entityId
|
)[0].entityId
|
||||||
|
|
@ -87,6 +96,7 @@ describe('migration LOCK is family-scoped', () => {
|
||||||
|
|
||||||
it('with no migration in flight, every read serves (the fast path is a no-op)', async () => {
|
it('with no migration in flight, every read serves (the fast path is a no-op)', async () => {
|
||||||
const brain = await seed()
|
const brain = await seed()
|
||||||
|
opened.push(brain)
|
||||||
await expect(brain.getStats()).resolves.toBeDefined()
|
await expect(brain.getStats()).resolves.toBeDefined()
|
||||||
await expect(brain.find({ query: 'doc' })).resolves.toBeDefined()
|
await expect(brain.find({ query: 'doc' })).resolves.toBeDefined()
|
||||||
await expect(brain.vfs.readdir('/notes')).resolves.toHaveLength(1)
|
await expect(brain.vfs.readdir('/notes')).resolves.toHaveLength(1)
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@ describe('Duplicate Check Optimization', () => {
|
||||||
})
|
})
|
||||||
|
|
||||||
afterEach(async () => {
|
afterEach(async () => {
|
||||||
// Cleanup is automatic with memory storage
|
await brain.close()
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should detect duplicate relationships using GraphAdjacencyIndex', async () => {
|
it('should detect duplicate relationships using GraphAdjacencyIndex', async () => {
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,8 @@
|
||||||
|
|
||||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
import { Brainy } from '../../../src/brainy'
|
import { Brainy } from '../../../src/brainy'
|
||||||
import {
|
import { MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from '../../../src/errors/brainyError'
|
||||||
|
import {
|
||||||
createAddParams,
|
createAddParams,
|
||||||
createTestConfig,
|
createTestConfig,
|
||||||
} from '../../helpers/test-factory'
|
} from '../../helpers/test-factory'
|
||||||
|
|
@ -248,16 +249,23 @@ describe('Brainy.relate()', () => {
|
||||||
expect(matches.length).toBe(1) // Only one relationship should exist
|
expect(matches.length).toBe(1) // Only one relationship should exist
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should handle very long metadata', async () => {
|
// THE INDEXABLE-ARRAY BOUND, from relate()'s side. This case used to pass a
|
||||||
// Arrange
|
// 100-element array through relate() and assert it came back — a length
|
||||||
|
// hardcoded either side of a bound it never named, so it read green or red
|
||||||
|
// purely by where the constant happened to sit. Both halves of the law are
|
||||||
|
// pinned here instead, and every length derives from
|
||||||
|
// MAX_INDEXED_ARRAY_LENGTH so the pin follows the constant.
|
||||||
|
it('should handle a large scalar metadata payload on a relation', async () => {
|
||||||
|
// Arrange — large in every dimension EXCEPT array length: a long string,
|
||||||
|
// many fields, and an array sitting exactly ON the bound.
|
||||||
const largeMetadata = {
|
const largeMetadata = {
|
||||||
bigArray: new Array(100).fill('item'),
|
atTheBound: Array.from({ length: MAX_INDEXED_ARRAY_LENGTH }, (_, i) => `item${i}`),
|
||||||
bigObject: Object.fromEntries(
|
bigObject: Object.fromEntries(
|
||||||
Array.from({ length: 50 }, (_, i) => [`key${i}`, `value${i}`])
|
Array.from({ length: 50 }, (_, i) => [`key${i}`, `value${i}`])
|
||||||
),
|
),
|
||||||
longString: 'x'.repeat(1000)
|
longString: 'x'.repeat(10_000)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Act
|
// Act
|
||||||
await brain.relate({
|
await brain.relate({
|
||||||
from: entity1Id,
|
from: entity1Id,
|
||||||
|
|
@ -265,12 +273,46 @@ describe('Brainy.relate()', () => {
|
||||||
type: 'relatedTo',
|
type: 'relatedTo',
|
||||||
metadata: largeMetadata
|
metadata: largeMetadata
|
||||||
})
|
})
|
||||||
|
|
||||||
// Assert
|
// Assert — the payload comes back whole, first element to last
|
||||||
const relations = await brain.related({ from: entity1Id })
|
const relations = await brain.related({ from: entity1Id })
|
||||||
const relation = relations.find(r => r.to === entity2Id)
|
const relation = relations.find(r => r.to === entity2Id)
|
||||||
expect(relation).toBeDefined()
|
expect(relation).toBeDefined()
|
||||||
expect(relation!.metadata?.bigArray).toHaveLength(100)
|
expect(relation!.metadata?.atTheBound).toHaveLength(MAX_INDEXED_ARRAY_LENGTH)
|
||||||
|
expect(relation!.metadata?.atTheBound[0]).toBe('item0')
|
||||||
|
expect(relation!.metadata?.atTheBound[MAX_INDEXED_ARRAY_LENGTH - 1])
|
||||||
|
.toBe(`item${MAX_INDEXED_ARRAY_LENGTH - 1}`)
|
||||||
|
expect(Object.keys(relation!.metadata?.bigObject)).toHaveLength(50)
|
||||||
|
expect(relation!.metadata?.longString).toHaveLength(10_000)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should refuse a relation metadata array over the indexing bound, by name', async () => {
|
||||||
|
// Arrange
|
||||||
|
const overTheBound = MAX_INDEXED_ARRAY_LENGTH + 1
|
||||||
|
|
||||||
|
// Act
|
||||||
|
const err = await brain
|
||||||
|
.relate({
|
||||||
|
from: entity1Id,
|
||||||
|
to: entity3Id,
|
||||||
|
type: 'relatedTo',
|
||||||
|
metadata: { bigArray: new Array(overTheBound).fill('item') }
|
||||||
|
})
|
||||||
|
.catch((e: any) => e)
|
||||||
|
|
||||||
|
// Assert — the field, the length and the bound, on the error and in the
|
||||||
|
// message, so a handler can report or repair without parsing prose.
|
||||||
|
expect(err).toBeInstanceOf(MetadataArrayTooLargeError)
|
||||||
|
expect(err.field).toBe('bigArray')
|
||||||
|
expect(err.length).toBe(overTheBound)
|
||||||
|
expect(err.limit).toBe(MAX_INDEXED_ARRAY_LENGTH)
|
||||||
|
expect(err.message).toContain('bigArray')
|
||||||
|
expect(err.message).toContain(String(overTheBound))
|
||||||
|
expect(err.message).toContain(String(MAX_INDEXED_ARRAY_LENGTH))
|
||||||
|
|
||||||
|
// Refused means not written: no relation of this shape exists.
|
||||||
|
const relations = await brain.related({ from: entity1Id })
|
||||||
|
expect(relations.some(r => r.to === entity3Id && r.metadata?.bigArray)).toBe(false)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should handle special characters in metadata', async () => {
|
it('should handle special characters in metadata', async () => {
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,8 @@
|
||||||
|
|
||||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
import { Brainy } from '../../../src/brainy'
|
import { Brainy } from '../../../src/brainy'
|
||||||
import {
|
import { MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from '../../../src/errors/brainyError'
|
||||||
|
import {
|
||||||
createAddParams,
|
createAddParams,
|
||||||
createTestConfig,
|
createTestConfig,
|
||||||
} from '../../helpers/test-factory'
|
} from '../../helpers/test-factory'
|
||||||
|
|
@ -355,36 +356,88 @@ describe('Brainy.update()', () => {
|
||||||
expect(final!.metadata.counter).toBeLessThanOrEqual(10)
|
expect(final!.metadata.counter).toBeLessThanOrEqual(10)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should handle very large metadata updates', async () => {
|
// THE INDEXABLE-ARRAY BOUND, from update()'s side. This case used to write
|
||||||
|
// a 1000-element array through update() and assert it came back. That
|
||||||
|
// shape is refused at the write door now — an array field mints one
|
||||||
|
// posting per element, so an unbounded array is an unbounded write — so
|
||||||
|
// the case pins BOTH halves of the law that replaced it. Every length
|
||||||
|
// derives from MAX_INDEXED_ARRAY_LENGTH so the pin follows the constant.
|
||||||
|
it('should handle a large scalar metadata update', async () => {
|
||||||
// Arrange
|
// Arrange
|
||||||
const id = await brain.add(createAddParams({
|
const id = await brain.add(createAddParams({
|
||||||
data: 'Large metadata test',
|
data: 'Large metadata test',
|
||||||
type: 'thing'
|
type: 'thing'
|
||||||
}))
|
}))
|
||||||
|
|
||||||
|
// Large in every dimension EXCEPT array length: a long string, many
|
||||||
|
// fields, deep nesting, and an array sitting exactly ON the bound.
|
||||||
const largeMetadata = {
|
const largeMetadata = {
|
||||||
bigArray: new Array(1000).fill('item'),
|
atTheBound: Array.from({ length: MAX_INDEXED_ARRAY_LENGTH }, (_, i) => `item${i}`),
|
||||||
bigObject: Object.fromEntries(
|
bigObject: Object.fromEntries(
|
||||||
Array.from({ length: 100 }, (_, i) => [`key${i}`, `value${i}`])
|
Array.from({ length: 100 }, (_, i) => [`key${i}`, `value${i}`])
|
||||||
),
|
),
|
||||||
|
longString: 'x'.repeat(10_000),
|
||||||
deepNesting: Array(10).fill(null).reduce(
|
deepNesting: Array(10).fill(null).reduce(
|
||||||
(acc) => ({ nested: acc }),
|
(acc) => ({ nested: acc }),
|
||||||
{ value: 'deep' }
|
{ value: 'deep' }
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Act
|
// Act
|
||||||
await brain.update({
|
await brain.update({
|
||||||
id,
|
id,
|
||||||
metadata: largeMetadata,
|
metadata: largeMetadata,
|
||||||
merge: false
|
merge: false
|
||||||
})
|
})
|
||||||
|
|
||||||
// Assert
|
// Assert — the payload comes back whole, first element to last
|
||||||
const updated = await brain.get(id)
|
const updated = await brain.get(id)
|
||||||
expect(updated).not.toBeNull()
|
expect(updated).not.toBeNull()
|
||||||
expect(updated!.metadata.bigArray).toHaveLength(1000)
|
expect(updated!.metadata.atTheBound).toHaveLength(MAX_INDEXED_ARRAY_LENGTH)
|
||||||
|
expect(updated!.metadata.atTheBound[0]).toBe('item0')
|
||||||
|
expect(updated!.metadata.atTheBound[MAX_INDEXED_ARRAY_LENGTH - 1])
|
||||||
|
.toBe(`item${MAX_INDEXED_ARRAY_LENGTH - 1}`)
|
||||||
expect(Object.keys(updated!.metadata.bigObject)).toHaveLength(100)
|
expect(Object.keys(updated!.metadata.bigObject)).toHaveLength(100)
|
||||||
|
expect(updated!.metadata.longString).toHaveLength(10_000)
|
||||||
|
|
||||||
|
// ...including the deep nest, walked to the bottom.
|
||||||
|
let cursor: any = updated!.metadata.deepNesting
|
||||||
|
for (let depth = 0; depth < 10; depth++) cursor = cursor.nested
|
||||||
|
expect(cursor.value).toBe('deep')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should refuse an update whose metadata array is over the indexing bound, by name', async () => {
|
||||||
|
// Arrange
|
||||||
|
const id = await brain.add(createAddParams({
|
||||||
|
data: 'Large metadata test',
|
||||||
|
type: 'thing',
|
||||||
|
metadata: { keep: 'me' }
|
||||||
|
}))
|
||||||
|
const overTheBound = MAX_INDEXED_ARRAY_LENGTH + 1
|
||||||
|
|
||||||
|
// Act
|
||||||
|
const err = await brain
|
||||||
|
.update({
|
||||||
|
id,
|
||||||
|
metadata: { bigArray: new Array(overTheBound).fill('item') },
|
||||||
|
merge: false
|
||||||
|
})
|
||||||
|
.catch((e: any) => e)
|
||||||
|
|
||||||
|
// Assert — the field, the length and the bound, on the error and in the
|
||||||
|
// message, so a handler can report or repair without parsing prose.
|
||||||
|
expect(err).toBeInstanceOf(MetadataArrayTooLargeError)
|
||||||
|
expect(err.field).toBe('bigArray')
|
||||||
|
expect(err.length).toBe(overTheBound)
|
||||||
|
expect(err.limit).toBe(MAX_INDEXED_ARRAY_LENGTH)
|
||||||
|
expect(err.message).toContain('bigArray')
|
||||||
|
expect(err.message).toContain(String(overTheBound))
|
||||||
|
expect(err.message).toContain(String(MAX_INDEXED_ARRAY_LENGTH))
|
||||||
|
|
||||||
|
// Refused means unchanged: the row still carries what it had before.
|
||||||
|
const unchanged = await brain.get(id)
|
||||||
|
expect(unchanged!.metadata.keep).toBe('me')
|
||||||
|
expect(unchanged!.metadata.bigArray).toBeUndefined()
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should preserve entity ID during update', async () => {
|
it('should preserve entity ID during update', async () => {
|
||||||
|
|
|
||||||
254
tests/unit/db/generationStore-commit-guard.test.ts
Normal file
254
tests/unit/db/generationStore-commit-guard.test.ts
Normal file
|
|
@ -0,0 +1,254 @@
|
||||||
|
/**
|
||||||
|
* @module tests/unit/db/generationStore-commit-guard
|
||||||
|
* @description Pins the commit-order guard on
|
||||||
|
* `GenerationStore.commitTransaction()` (`src/db/generationStore.ts`).
|
||||||
|
*
|
||||||
|
* `reservedGensAsc()`'s own doc comment states an invariant it never
|
||||||
|
* enforced: pending single-op generations are always greater than every
|
||||||
|
* committed one, because the store's only two sanctioned callers —
|
||||||
|
* `Brainy.transact()` and `Brainy.compactHistory()` — flush the pending tier
|
||||||
|
* before committing. Nothing stopped a caller from invoking
|
||||||
|
* `commitTransaction()` directly while single-ops were still buffered: the
|
||||||
|
* fresh commit would land in `committedRanges` ABOVE those lower,
|
||||||
|
* still-pending generations, so the committed-then-pending concatenation
|
||||||
|
* `reservedGensAsc()` yields is no longer ascending — and `resolveManyAt`
|
||||||
|
* (which walks committed ranges before pending ones) would silently report a
|
||||||
|
* WRONG before-image for a point-in-time read. `commitTransaction()` now
|
||||||
|
* refuses loudly (`PendingSingleOpsUnflushedError`) instead of assuming.
|
||||||
|
*
|
||||||
|
* Four pins:
|
||||||
|
* 1. A direct `commitTransaction()` call while single-ops are pending throws
|
||||||
|
* and commits NOTHING.
|
||||||
|
* 2. The same commit succeeds once the pending tier is flushed first.
|
||||||
|
* 3. `Brainy.transact()` — which already flushes first — is unaffected
|
||||||
|
* (mirrors `tests/unit/db/generation-chain.test.ts`'s `seedX()`/`bumpX()`
|
||||||
|
* transact pin: add, then transact-update, generation advances by one
|
||||||
|
* each time, the update lands).
|
||||||
|
* 4. `reservedGensAsc()` stays ascending across a real add+transact+delete
|
||||||
|
* workload — proven by point-in-time reads (`asOf`) staying correct
|
||||||
|
* throughout, which is exactly what an ordering break would corrupt.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
|
import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
|
||||||
|
import {
|
||||||
|
GenerationStore,
|
||||||
|
GENERATIONS_PREFIX,
|
||||||
|
MANIFEST_PATH
|
||||||
|
} from '../../../src/db/generationStore.js'
|
||||||
|
import { PendingSingleOpsUnflushedError } from '../../../src/db/errors.js'
|
||||||
|
import { Brainy } from '../../../src/index.js'
|
||||||
|
import { NounType } from '../../../src/types/graphTypes.js'
|
||||||
|
import { createTestConfig, generateTestVector } from '../../helpers/test-factory.js'
|
||||||
|
|
||||||
|
/** Precomputed embedding so Brainy-level adds skip the (slow) embedding model —
|
||||||
|
* these tests exercise the generation layer, not semantics. */
|
||||||
|
const VEC = generateTestVector()
|
||||||
|
|
||||||
|
// Entity ids must be UUID-shaped (the sharded storage layout derives the
|
||||||
|
// shard from the UUID hex) — same fixture convention as generationStore.test.ts.
|
||||||
|
const ID_A = '00000000-0000-4000-8000-0000000000aa'
|
||||||
|
const ID_B = '00000000-0000-4000-8000-0000000000bb'
|
||||||
|
|
||||||
|
/** Stored-metadata fixture in the canonical shape the live write paths use
|
||||||
|
* (matches generationStore.test.ts's fixture exactly). */
|
||||||
|
function metadataFixture(version: number): Record<string, unknown> {
|
||||||
|
return {
|
||||||
|
noun: NounType.Document,
|
||||||
|
subtype: 'note',
|
||||||
|
data: `payload-v${version}`,
|
||||||
|
version,
|
||||||
|
createdAt: 1000,
|
||||||
|
updatedAt: 1000 + version,
|
||||||
|
_rev: version
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('db/GenerationStore — commitTransaction pending-tier guard (store level)', () => {
|
||||||
|
let storage: MemoryStorage
|
||||||
|
let store: GenerationStore
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
storage = new MemoryStorage()
|
||||||
|
await storage.init()
|
||||||
|
store = new GenerationStore(storage)
|
||||||
|
await store.open()
|
||||||
|
})
|
||||||
|
|
||||||
|
/** Buffer one single-op generation via commitSingleOp WITHOUT flushing —
|
||||||
|
* the pending tier that must be drained before commitTransaction(). */
|
||||||
|
async function pendingSingleOp(id: string, version: number): Promise<number> {
|
||||||
|
const { generation } = await store.commitSingleOp({
|
||||||
|
touched: { nouns: [id] },
|
||||||
|
execute: async () => {
|
||||||
|
await storage.saveNounMetadata(id, metadataFixture(version))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
return generation
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A direct transact commit — exactly what a caller bypassing
|
||||||
|
* Brainy.transact()'s flush-first step would issue. */
|
||||||
|
function directCommit(id: string, version: number): Promise<{ generation: number; timestamp: number }> {
|
||||||
|
return store.commitTransaction({
|
||||||
|
touched: { nouns: [id], verbs: [] },
|
||||||
|
execute: async () => {
|
||||||
|
await storage.saveNounMetadata(id, metadataFixture(version))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
it('PIN 1: refuses a direct commitTransaction() while single-ops are pending, and commits NOTHING', async () => {
|
||||||
|
const g1 = await pendingSingleOp(ID_A, 1)
|
||||||
|
expect(g1).toBe(1)
|
||||||
|
expect(store.committedGeneration()).toBe(0) // nothing flushed to disk yet
|
||||||
|
|
||||||
|
let caught: unknown
|
||||||
|
try {
|
||||||
|
await directCommit(ID_B, 1)
|
||||||
|
expect.unreachable('should have thrown PendingSingleOpsUnflushedError')
|
||||||
|
} catch (err) {
|
||||||
|
caught = err
|
||||||
|
}
|
||||||
|
expect(caught).toBeInstanceOf(PendingSingleOpsUnflushedError)
|
||||||
|
expect((caught as PendingSingleOpsUnflushedError).pendingCount).toBe(1)
|
||||||
|
|
||||||
|
// Nothing committed: the head + committed ranges are unchanged, and the
|
||||||
|
// counter never advanced for the refused attempt (the guard fires before
|
||||||
|
// a generation is even reserved).
|
||||||
|
expect(store.committedGeneration()).toBe(0)
|
||||||
|
expect(store.generation()).toBe(1) // still just the pending single-op's gen
|
||||||
|
expect(await storage.readRawObject(MANIFEST_PATH)).toBeNull()
|
||||||
|
// The guard fires BEFORE a generation is reserved (`gen = ++this.counter`
|
||||||
|
// never runs), so the refused attempt's would-be directory (generation 2,
|
||||||
|
// the next number after the pending single-op's 1) was never created.
|
||||||
|
expect(await storage.listRawObjects(`${GENERATIONS_PREFIX}/2`)).toEqual([])
|
||||||
|
|
||||||
|
// The refused write never touched canonical storage.
|
||||||
|
expect((await storage.readNounRaw(ID_B)).metadata).toBeNull()
|
||||||
|
|
||||||
|
// The pending tier itself is untouched by the refused attempt — flushing
|
||||||
|
// now still commits the ORIGINAL single-op cleanly.
|
||||||
|
await store.flushPendingSingleOps()
|
||||||
|
expect(store.committedGeneration()).toBe(1)
|
||||||
|
const atG0 = await store.resolveAt('noun', ID_A, 0)
|
||||||
|
expect(atG0).toEqual({ source: 'absent' }) // the create sentinel before g1's write
|
||||||
|
})
|
||||||
|
|
||||||
|
it('PIN 2: the same commit succeeds once the pending tier is flushed first', async () => {
|
||||||
|
await pendingSingleOp(ID_A, 1)
|
||||||
|
await expect(directCommit(ID_B, 1)).rejects.toBeInstanceOf(PendingSingleOpsUnflushedError)
|
||||||
|
|
||||||
|
await store.flushPendingSingleOps()
|
||||||
|
expect(store.committedGeneration()).toBe(1)
|
||||||
|
|
||||||
|
const { generation } = await directCommit(ID_B, 1)
|
||||||
|
expect(generation).toBe(2)
|
||||||
|
expect(store.committedGeneration()).toBe(2)
|
||||||
|
expect((await storage.readNounRaw(ID_B)).metadata).toMatchObject({ version: 1 })
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('Brainy public API — commitTransaction pending-tier guard is behavior-neutral', () => {
|
||||||
|
let brain: Brainy
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
brain = new Brainy(createTestConfig())
|
||||||
|
await brain.init()
|
||||||
|
})
|
||||||
|
afterEach(async () => {
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('PIN 3: Brainy.transact() still commits normally over pending single-ops (mirrors generation-chain.test.ts\'s seedX()/bumpX() transact pin)', async () => {
|
||||||
|
const store = (brain as any).generationStore as GenerationStore
|
||||||
|
// Relative, not absolute: under the adopt-at-open default the open-time
|
||||||
|
// baseline backfill takes a generation of its own (see
|
||||||
|
// bounded-chains.test.ts's identical note), so the first user add is not
|
||||||
|
// necessarily generation 1.
|
||||||
|
const baseGen = brain.generation()
|
||||||
|
const baseCommitted = store.committedGeneration()
|
||||||
|
|
||||||
|
const id = await brain.add({
|
||||||
|
data: 'x',
|
||||||
|
type: NounType.Document,
|
||||||
|
subtype: 'note',
|
||||||
|
metadata: { v: 1 },
|
||||||
|
vector: VEC
|
||||||
|
})
|
||||||
|
// The add is a pending single-op generation — NOT yet flushed.
|
||||||
|
expect(brain.generation()).toBe(baseGen + 1)
|
||||||
|
expect(store.committedGeneration()).toBe(baseCommitted)
|
||||||
|
|
||||||
|
// Brainy.transact() flushes the pending tier FIRST (src/brainy.ts:
|
||||||
|
// `await this.generationStore.flushPendingSingleOps()`, immediately
|
||||||
|
// before its `generationStore.commitTransaction()` call), so the guard
|
||||||
|
// never fires on this path — same shape as generation-chain.test.ts's
|
||||||
|
// seedX() (add) → bumpX() (transact update) → generation advances by one.
|
||||||
|
const db = await brain.transact([{ op: 'update', id, metadata: { v: 2 } }])
|
||||||
|
await db.release()
|
||||||
|
|
||||||
|
expect(brain.generation()).toBe(baseGen + 2)
|
||||||
|
expect(store.committedGeneration()).toBe(baseGen + 2) // the flushed add + the transact update
|
||||||
|
const entity = (await brain.get(id)) as any
|
||||||
|
expect(entity.metadata.v).toBe(2)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('PIN 4: reservedGensAsc() stays ascending across a real add+transact+delete workload — point-in-time reads stay correct', async () => {
|
||||||
|
const store = (brain as any).generationStore as GenerationStore
|
||||||
|
const baseGen = brain.generation()
|
||||||
|
const baseCommitted = store.committedGeneration()
|
||||||
|
|
||||||
|
const idX = await brain.add({
|
||||||
|
data: 'x',
|
||||||
|
type: NounType.Document,
|
||||||
|
subtype: 'note',
|
||||||
|
metadata: { v: 1 },
|
||||||
|
vector: VEC
|
||||||
|
})
|
||||||
|
expect(brain.generation()).toBe(baseGen + 1) // pending (un-flushed)
|
||||||
|
|
||||||
|
const idY = await brain.add({
|
||||||
|
data: 'y',
|
||||||
|
type: NounType.Document,
|
||||||
|
subtype: 'note',
|
||||||
|
metadata: { v: 1 },
|
||||||
|
vector: VEC
|
||||||
|
})
|
||||||
|
// Pin right after BOTH adds — before the transact update — so X reads v1
|
||||||
|
// and Y still exists at this pin, unlike the live head after the rest of
|
||||||
|
// the workload runs.
|
||||||
|
const pinAfterBothAdds = brain.generation()
|
||||||
|
expect(pinAfterBothAdds).toBe(baseGen + 2) // ALSO pending — two un-flushed single-ops
|
||||||
|
expect(store.committedGeneration()).toBe(baseCommitted)
|
||||||
|
|
||||||
|
// A transact() flushes baseGen+1 and baseGen+2 first, then commits its
|
||||||
|
// own update as baseGen+3. If committed-vs-pending ordering ever broke,
|
||||||
|
// this is exactly the step that would land a commit ABOVE still-pending
|
||||||
|
// generations.
|
||||||
|
const db = await brain.transact([{ op: 'update', id: idX, metadata: { v: 3 } }])
|
||||||
|
await db.release()
|
||||||
|
expect(brain.generation()).toBe(baseGen + 3)
|
||||||
|
expect(store.committedGeneration()).toBe(baseGen + 3)
|
||||||
|
|
||||||
|
// A single-op delete, pending again (un-flushed).
|
||||||
|
await brain.remove(idY)
|
||||||
|
expect(brain.generation()).toBe(baseGen + 4)
|
||||||
|
|
||||||
|
// A point-in-time read pinned right after the two adds (before the
|
||||||
|
// transact update) must see X's PRE-update value and Y still present.
|
||||||
|
// This is precisely what resolveManyAt/resolveAt get WRONG if committed
|
||||||
|
// and pending generations were ever interleaved out of ascending order.
|
||||||
|
const past = await brain.asOf(pinAfterBothAdds)
|
||||||
|
const xAtPin = (await past.get(idX)) as any
|
||||||
|
expect(xAtPin?.metadata?.v).toBe(1)
|
||||||
|
const yAtPin = (await past.get(idY)) as any
|
||||||
|
expect(yAtPin?.metadata?.v).toBe(1) // not yet removed, as of this pin
|
||||||
|
await past.release()
|
||||||
|
|
||||||
|
// Live state reflects every later write, in the right order.
|
||||||
|
const xNow = (await brain.get(idX)) as any
|
||||||
|
expect(xNow.metadata.v).toBe(3)
|
||||||
|
expect(await brain.get(idY)).toBeNull()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
@ -7,7 +7,7 @@
|
||||||
* _indexRebuildFailed / _indexDegradedIds degraded states (mirroring
|
* _indexRebuildFailed / _indexDegradedIds degraded states (mirroring
|
||||||
* validateIndexConsistency / checkHealth).
|
* validateIndexConsistency / checkHealth).
|
||||||
*/
|
*/
|
||||||
import { describe, it, expect, beforeEach } from 'vitest'
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
import { Brainy, NounType } from '../../src/index.js'
|
import { Brainy, NounType } from '../../src/index.js'
|
||||||
|
|
||||||
describe('getIndexStatus honest readiness (Finding 9)', () => {
|
describe('getIndexStatus honest readiness (Finding 9)', () => {
|
||||||
|
|
@ -20,6 +20,10 @@ describe('getIndexStatus honest readiness (Finding 9)', () => {
|
||||||
await brain.flush()
|
await brain.flush()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
|
||||||
it('a not-ready provider makes populated honest (false) and exposes ready:false', async () => {
|
it('a not-ready provider makes populated honest (false) and exposes ready:false', async () => {
|
||||||
brain.index.isReady = () => false // count present, serving structure NOT loaded
|
brain.index.isReady = () => false // count present, serving structure NOT loaded
|
||||||
const status = await brain.getIndexStatus()
|
const status = await brain.getIndexStatus()
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@
|
||||||
* scan; and a one-shot probe self-heals a no-isReady provider whose adjacency
|
* scan; and a one-shot probe self-heals a no-isReady provider whose adjacency
|
||||||
* did not cold-load.
|
* did not cold-load.
|
||||||
*/
|
*/
|
||||||
import { describe, it, expect, beforeEach } from 'vitest'
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
import { Brainy, NounType, VerbType } from '../../../src/index.js'
|
import { Brainy, NounType, VerbType } from '../../../src/index.js'
|
||||||
|
|
||||||
describe('graph fast-path honest readiness (Finding 2)', () => {
|
describe('graph fast-path honest readiness (Finding 2)', () => {
|
||||||
|
|
@ -33,6 +33,10 @@ describe('graph fast-path honest readiness (Finding 2)', () => {
|
||||||
await storage.getVerbsBySource(a)
|
await storage.getVerbsBySource(a)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
|
||||||
it('not-ready provider → shard scan returns the REAL edges, not a silent []', async () => {
|
it('not-ready provider → shard scan returns the REAL edges, not a silent []', async () => {
|
||||||
const gi = storage.graphIndex
|
const gi = storage.graphIndex
|
||||||
// Simulate a cold native provider: count/manifest loaded (isInitialized) but
|
// Simulate a cold native provider: count/manifest loaded (isInitialized) but
|
||||||
|
|
|
||||||
241
tests/unit/indexes/columnStore/column-store-mixed-kind.test.ts
Normal file
241
tests/unit/indexes/columnStore/column-store-mixed-kind.test.ts
Normal file
|
|
@ -0,0 +1,241 @@
|
||||||
|
/**
|
||||||
|
* @module column-store-mixed-kind.test
|
||||||
|
* @description Typed posting lists: one field, several value KINDS, each
|
||||||
|
* answerable on its own.
|
||||||
|
*
|
||||||
|
* The behaviour these pin replaced a first-writer type freeze. The first value
|
||||||
|
* a field ever saw fixed that field's type; every later value of another kind
|
||||||
|
* was coerced to it, and when coercion failed — `Number('electronics')` — the
|
||||||
|
* value was dropped from the index with no error at all. The row stayed
|
||||||
|
* readable by id and by vector and vanished from every equality filter on the
|
||||||
|
* field. These tests therefore care about ORDER: strings-then-numbers and
|
||||||
|
* numbers-then-strings have to behave identically, because neither writer owns
|
||||||
|
* the field.
|
||||||
|
*
|
||||||
|
* Kinds never coerce into one another at query time either. `5` and `'5'` are
|
||||||
|
* different values and match different rows.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
|
import { ColumnStore } from '../../../../src/indexes/columnStore/ColumnStore.js'
|
||||||
|
import { MemoryStorage } from '../../../../src/storage/adapters/memoryStorage.js'
|
||||||
|
import { EntityIdMapper } from '../../../../src/utils/entityIdMapper.js'
|
||||||
|
|
||||||
|
describe('ColumnStore — typed posting lists per (field, kind)', () => {
|
||||||
|
let storage: MemoryStorage
|
||||||
|
let idMapper: EntityIdMapper
|
||||||
|
let store: ColumnStore
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
storage = new MemoryStorage()
|
||||||
|
await storage.init()
|
||||||
|
idMapper = new EntityIdMapper({ storage, storageKey: 'test:idMapper' })
|
||||||
|
await idMapper.init()
|
||||||
|
|
||||||
|
store = new ColumnStore({ flushThreshold: 10 })
|
||||||
|
await store.init(storage, idMapper)
|
||||||
|
})
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await store.close()
|
||||||
|
})
|
||||||
|
|
||||||
|
/** Resolve a filter to the sorted UUIDs it matched. */
|
||||||
|
const uuidsOf = async (field: string, value: unknown): Promise<string[]> => {
|
||||||
|
const bitmap = await store.filter(field, value)
|
||||||
|
return Array.from(bitmap)
|
||||||
|
.map((id) => idMapper.getUuid(Number(id)))
|
||||||
|
.filter((u): u is string => u !== undefined)
|
||||||
|
.sort()
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('equality answers on the query value’s own kind', () => {
|
||||||
|
it('serves numbers written AFTER strings on the same field', async () => {
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'furniture' })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n3')), { category: 7 })
|
||||||
|
|
||||||
|
// The numbers are in the index, though a string got there first.
|
||||||
|
expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2'])
|
||||||
|
expect(await uuidsOf('category', 7)).toEqual(['n3'])
|
||||||
|
// And the strings did not move.
|
||||||
|
expect(await uuidsOf('category', 'electronics')).toEqual(['s1'])
|
||||||
|
expect(await uuidsOf('category', 'furniture')).toEqual(['s2'])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('serves strings written AFTER numbers on the same field', async () => {
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' })
|
||||||
|
|
||||||
|
// 'electronics' would have become NaN and been dropped under the freeze.
|
||||||
|
expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2'])
|
||||||
|
expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2'])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('does not coerce a number query into the string postings, or back', async () => {
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('num')), { code: 5 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('str')), { code: '5' })
|
||||||
|
|
||||||
|
expect(await uuidsOf('code', 5)).toEqual(['num'])
|
||||||
|
expect(await uuidsOf('code', '5')).toEqual(['str'])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('serves booleans mixed into a field that already holds strings and numbers', async () => {
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { flag: 'yes' })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { flag: 1 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('b1')), { flag: true })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('b2')), { flag: false })
|
||||||
|
|
||||||
|
expect(await uuidsOf('flag', true)).toEqual(['b1'])
|
||||||
|
expect(await uuidsOf('flag', false)).toEqual(['b2'])
|
||||||
|
// `true` stores as 1 internally; that is an encoding, not a value.
|
||||||
|
expect(await uuidsOf('flag', 1)).toEqual(['n1'])
|
||||||
|
expect(await uuidsOf('flag', 'yes')).toEqual(['s1'])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('answers nothing — not something coerced — for a kind the field never held', async () => {
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||||||
|
|
||||||
|
expect(await uuidsOf('category', 5)).toEqual([])
|
||||||
|
expect(await uuidsOf('category', true)).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('holds every kind across a flush, not just the one in the tail buffer', async () => {
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
|
||||||
|
await store.flush()
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 })
|
||||||
|
|
||||||
|
expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2'])
|
||||||
|
expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2'])
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('range filters read the numeric postings', () => {
|
||||||
|
it('ranges over the numeric subset of a mixed field, ignoring its strings', async () => {
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('cheap')), { price: 100 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('mid')), { price: 500 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('dear')), { price: 900 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('unpriced')), { price: 'on request' })
|
||||||
|
await store.flush()
|
||||||
|
|
||||||
|
const inRange = await store.rangeQuery('price', 200, 1000)
|
||||||
|
const uuids = Array.from(inRange)
|
||||||
|
.map((id) => idMapper.getUuid(Number(id)))
|
||||||
|
.sort()
|
||||||
|
expect(uuids).toEqual(['dear', 'mid'])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('an unbounded range still reports every kind — it is the “has a value” probe', async () => {
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { mixed: 42 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { mixed: 'text' })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('b1')), { mixed: true })
|
||||||
|
await store.flush()
|
||||||
|
|
||||||
|
const anyValue = await store.rangeQuery('mixed')
|
||||||
|
const uuids = Array.from(anyValue)
|
||||||
|
.map((id) => idMapper.getUuid(Number(id)))
|
||||||
|
.sort()
|
||||||
|
expect(uuids).toEqual(['b1', 'n1', 's1'])
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('the index reports what a field actually holds', () => {
|
||||||
|
it('names every kind present, not the one that got there first', async () => {
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||||||
|
expect(store.getFieldKinds('category')).toEqual(['string'])
|
||||||
|
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: true })
|
||||||
|
expect(store.getFieldKinds('category')).toEqual(['number', 'string', 'boolean'])
|
||||||
|
|
||||||
|
// And the field is still ONE field by name.
|
||||||
|
expect(store.getIndexedFields()).toEqual(['category'])
|
||||||
|
expect(store.hasField('category')).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('reports an unknown field as holding nothing', () => {
|
||||||
|
expect(store.getFieldKinds('never-written')).toEqual([])
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('an integer column widens rather than rounding', () => {
|
||||||
|
it('keeps a non-integer written after integers as itself', async () => {
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('a')), { score: 4 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('b')), { score: 4.5 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('c')), { score: 5 })
|
||||||
|
await store.flush()
|
||||||
|
|
||||||
|
// 4.5 used to round to 5 and answer `score === 5` alongside c.
|
||||||
|
expect(await uuidsOf('score', 4.5)).toEqual(['b'])
|
||||||
|
expect(await uuidsOf('score', 5)).toEqual(['c'])
|
||||||
|
expect(await uuidsOf('score', 4)).toEqual(['a'])
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('close then reopen', () => {
|
||||||
|
it('keeps every typed posting, on the same storage', async () => {
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: true })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('f1')), { score: 1.5 })
|
||||||
|
await store.flush()
|
||||||
|
await store.close()
|
||||||
|
|
||||||
|
store = new ColumnStore({ flushThreshold: 10 })
|
||||||
|
await store.init(storage, idMapper)
|
||||||
|
|
||||||
|
expect(store.getFieldKinds('category')).toEqual(['number', 'string', 'boolean'])
|
||||||
|
expect(await uuidsOf('category', 'electronics')).toEqual(['s1'])
|
||||||
|
expect(await uuidsOf('category', 5)).toEqual(['n1'])
|
||||||
|
expect(await uuidsOf('category', true)).toEqual(['b1'])
|
||||||
|
expect(await uuidsOf('score', 1.5)).toEqual(['f1'])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('accepts new values of every kind after the reopen', async () => {
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
|
||||||
|
await store.flush()
|
||||||
|
await store.close()
|
||||||
|
|
||||||
|
store = new ColumnStore({ flushThreshold: 10 })
|
||||||
|
await store.init(storage, idMapper)
|
||||||
|
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: false })
|
||||||
|
await store.flush()
|
||||||
|
|
||||||
|
expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2'])
|
||||||
|
expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2'])
|
||||||
|
expect(await uuidsOf('category', false)).toEqual(['b1'])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('opens an index written by the pre-typed-postings shape and reads it unchanged', async () => {
|
||||||
|
// A single-kind field is byte-identical to what the old writer produced:
|
||||||
|
// one manifest at `_column_index/<field>/MANIFEST.json`, no kind
|
||||||
|
// subdirectory anywhere. That IS the old on-disk shape, so proving the
|
||||||
|
// new reader serves it proves an old index still opens.
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('a')), { status: 'active' })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('b')), { status: 'archived' })
|
||||||
|
await store.flush()
|
||||||
|
|
||||||
|
const keys = await (storage as unknown as {
|
||||||
|
listObjectsUnderPath: (prefix: string) => Promise<string[]>
|
||||||
|
}).listObjectsUnderPath('_column_index/')
|
||||||
|
expect(keys.some((k) => k.includes('/k/'))).toBe(false)
|
||||||
|
|
||||||
|
await store.close()
|
||||||
|
store = new ColumnStore({ flushThreshold: 10 })
|
||||||
|
await store.init(storage, idMapper)
|
||||||
|
|
||||||
|
expect(store.getFieldKinds('status')).toEqual(['string'])
|
||||||
|
expect(await uuidsOf('status', 'active')).toEqual(['a'])
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
@ -15,7 +15,7 @@
|
||||||
* The 8.0 JS index cold-loads correctly, so we simulate the cold native failure
|
* The 8.0 JS index cold-loads correctly, so we simulate the cold native failure
|
||||||
* mode by intercepting the provider's getIdsForFilter/rebuild.
|
* mode by intercepting the provider's getIdsForFilter/rebuild.
|
||||||
*/
|
*/
|
||||||
import { describe, it, expect, beforeEach } from 'vitest'
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
import { Brainy, NounType, MetadataIndexNotReadyError } from '../../src/index.js'
|
import { Brainy, NounType, MetadataIndexNotReadyError } from '../../src/index.js'
|
||||||
|
|
||||||
const V = () => Array.from({ length: 384 }, (_, i) => Math.sin(i * 0.1) + 0.001)
|
const V = () => Array.from({ length: 384 }, (_, i) => Math.sin(i * 0.1) + 0.001)
|
||||||
|
|
@ -31,6 +31,10 @@ describe('Metadata cold-read guard (#venue silent-[])', () => {
|
||||||
await brain.flush()
|
await brain.flush()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
|
||||||
it('warm brain: filtered find is correct and the guard does not rebuild', async () => {
|
it('warm brain: filtered find is correct and the guard does not rebuild', async () => {
|
||||||
const mi = brain.metadataIndex
|
const mi = brain.metadataIndex
|
||||||
let rebuilds = 0
|
let rebuilds = 0
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@
|
||||||
* the production feature-detection reads it.
|
* the production feature-detection reads it.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { describe, it, expect, beforeEach } from 'vitest'
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
import { Brainy, NounType, MigrationInProgressError } from '../../src/index.js'
|
import { Brainy, NounType, MigrationInProgressError } from '../../src/index.js'
|
||||||
import { GraphAdjacencyIndex } from '../../src/graph/graphAdjacencyIndex.js'
|
import { GraphAdjacencyIndex } from '../../src/graph/graphAdjacencyIndex.js'
|
||||||
|
|
||||||
|
|
@ -39,6 +39,12 @@ describe('Migration LOCK (#18) — coordinated 7.x→8.0 auto-upgrade', () => {
|
||||||
await brain.init()
|
await brain.init()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
// The "close() is not gated" test already closes `brain` itself as its
|
||||||
|
// own assertion — closing an already-closed brain is a safe no-op here.
|
||||||
|
await brain.close().catch(() => {})
|
||||||
|
})
|
||||||
|
|
||||||
it('does not gate operations when no provider is migrating (fast path)', async () => {
|
it('does not gate operations when no provider is migrating (fast path)', async () => {
|
||||||
const id = await brain.add({ data: 'hello', type: NounType.Concept })
|
const id = await brain.add({ data: 'hello', type: NounType.Concept })
|
||||||
expect(id).toBeTruthy()
|
expect(id).toBeTruthy()
|
||||||
|
|
@ -130,6 +136,9 @@ describe('Migration LOCK (#18) — coordinated 7.x→8.0 auto-upgrade', () => {
|
||||||
expect(e).toBeInstanceOf(MigrationInProgressError)
|
expect(e).toBeInstanceOf(MigrationInProgressError)
|
||||||
expect(e.retryable).toBe(true)
|
expect(e.retryable).toBe(true)
|
||||||
expect(typeof e.elapsedMs).toBe('number')
|
expect(typeof e.elapsedMs).toBe('number')
|
||||||
|
} finally {
|
||||||
|
// close() is proven not-gated by the test below — safe even mid-migration.
|
||||||
|
await shortBrain.close()
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -13,10 +13,11 @@ describe('EmbeddingSignal', () => {
|
||||||
signal = new EmbeddingSignal(brain)
|
signal = new EmbeddingSignal(brain)
|
||||||
})
|
})
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(async () => {
|
||||||
signal.clearCache()
|
signal.clearCache()
|
||||||
signal.clearHistory()
|
signal.clearHistory()
|
||||||
signal.resetStats()
|
signal.resetStats()
|
||||||
|
await brain.close()
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('initialization', () => {
|
describe('initialization', () => {
|
||||||
|
|
|
||||||
|
|
@ -89,12 +89,14 @@ describe('Guarded plugin auto-detection (plugins: undefined)', () => {
|
||||||
})
|
})
|
||||||
const brain: any = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true })
|
const brain: any = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true })
|
||||||
await expect(brain.init()).rejects.toThrow(/installed but failed to load/)
|
await expect(brain.init()).rejects.toThrow(/installed but failed to load/)
|
||||||
|
await brain.close().catch(() => {})
|
||||||
})
|
})
|
||||||
|
|
||||||
it('installed but not a valid plugin (missing activate) → init() throws', async () => {
|
it('installed but not a valid plugin (missing activate) → init() throws', async () => {
|
||||||
stubImport(async () => ({ default: { name: '@soulcraft/cor' } })) // no activate()
|
stubImport(async () => ({ default: { name: '@soulcraft/cor' } })) // no activate()
|
||||||
const brain: any = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true })
|
const brain: any = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true })
|
||||||
await expect(brain.init()).rejects.toThrow(/not a valid Brainy plugin/)
|
await expect(brain.init()).rejects.toThrow(/not a valid Brainy plugin/)
|
||||||
|
await brain.close().catch(() => {})
|
||||||
})
|
})
|
||||||
|
|
||||||
it('installed but activation fails → init() throws (activateAll posture applies)', async () => {
|
it('installed but activation fails → init() throws (activateAll posture applies)', async () => {
|
||||||
|
|
@ -108,6 +110,7 @@ describe('Guarded plugin auto-detection (plugins: undefined)', () => {
|
||||||
}))
|
}))
|
||||||
const brain: any = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true })
|
const brain: any = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true })
|
||||||
await expect(brain.init()).rejects.toThrow(/failed to activate/)
|
await expect(brain.init()).rejects.toThrow(/failed to activate/)
|
||||||
|
await brain.close().catch(() => {})
|
||||||
})
|
})
|
||||||
|
|
||||||
it('plugins: [] and plugins: false → no probe at all (explicit opt-out)', async () => {
|
it('plugins: [] and plugins: false → no probe at all (explicit opt-out)', async () => {
|
||||||
|
|
@ -132,5 +135,6 @@ describe('Guarded plugin auto-detection (plugins: undefined)', () => {
|
||||||
silent: true
|
silent: true
|
||||||
})
|
})
|
||||||
await expect(brain.init()).rejects.toThrow(/listed in config\.plugins but could not be loaded/)
|
await expect(brain.init()).rejects.toThrow(/listed in config\.plugins but could not be loaded/)
|
||||||
|
await brain.close().catch(() => {})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -143,5 +143,6 @@ describe('version coupling at init() — no silent fallback', () => {
|
||||||
plugins: ['@soulcraft/this-package-does-not-exist-xyz']
|
plugins: ['@soulcraft/this-package-does-not-exist-xyz']
|
||||||
})
|
})
|
||||||
await expect(brain.init()).rejects.toThrow(/could not be loaded|config\.plugins/)
|
await expect(brain.init()).rejects.toThrow(/could not be loaded|config\.plugins/)
|
||||||
|
await brain.close().catch(() => {})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -298,9 +298,10 @@ describe('Brainy plugin integration', () => {
|
||||||
// must surface as a failed init(), NOT a silent degrade to the default
|
// must surface as a failed init(), NOT a silent degrade to the default
|
||||||
// engine (the version-coupling guard; see plugin-version-coupling.test.ts).
|
// engine (the version-coupling guard; see plugin-version-coupling.test.ts).
|
||||||
await expect(brain.init()).rejects.toThrow(/failed to activate|native module not found/)
|
await expect(brain.init()).rejects.toThrow(/failed to activate|native module not found/)
|
||||||
|
await brain.close().catch(() => {})
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should use() return this for chaining', () => {
|
it('should use() return this for chaining', async () => {
|
||||||
const plugin: BrainyPlugin = {
|
const plugin: BrainyPlugin = {
|
||||||
name: 'chain-test',
|
name: 'chain-test',
|
||||||
activate: async () => true
|
activate: async () => true
|
||||||
|
|
@ -309,5 +310,8 @@ describe('Brainy plugin integration', () => {
|
||||||
const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||||
const result = brain.use(plugin)
|
const result = brain.use(plugin)
|
||||||
expect(result).toBe(brain)
|
expect(result).toBe(brain)
|
||||||
|
// Never init()'d — the constructor still registered it in Brainy's global
|
||||||
|
// instance registry, so it still needs a close() to deregister.
|
||||||
|
await brain.close().catch(() => {})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
|
||||||
395
tests/unit/release/wall-entry.test.ts
Normal file
395
tests/unit/release/wall-entry.test.ts
Normal file
|
|
@ -0,0 +1,395 @@
|
||||||
|
/**
|
||||||
|
* scripts/wall-entry.mjs — the mechanical releases-wall entry.
|
||||||
|
*
|
||||||
|
* The script's only real interface is its CLI (it has no importable
|
||||||
|
* exports by design — one door, no parallel API to drift from it), so
|
||||||
|
* these tests spawn it exactly as scripts/release.sh does: as a child
|
||||||
|
* process, against a fixture CHANGELOG and a throwaway local bare repo
|
||||||
|
* standing in for git@source.soulcraft.com:soulcraftlabs/releases.git
|
||||||
|
* (--remote) plus a throwaway cache directory (--cache-dir) standing in
|
||||||
|
* for ~/.cache/soulcraft-releases — never the real remote, never the
|
||||||
|
* real developer cache.
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
|
import { execFileSync } from 'node:child_process'
|
||||||
|
import { mkdtempSync, rmSync, writeFileSync, readFileSync, chmodSync } from 'node:fs'
|
||||||
|
import { tmpdir } from 'node:os'
|
||||||
|
import { join } from 'node:path'
|
||||||
|
|
||||||
|
const SCRIPT = join(process.cwd(), 'scripts/wall-entry.mjs')
|
||||||
|
|
||||||
|
/** Run the script and capture the outcome without throwing on a non-zero exit. */
|
||||||
|
function run(args: string[], cwd: string): { status: number; stdout: string; stderr: string } {
|
||||||
|
try {
|
||||||
|
const stdout = execFileSync('node', [SCRIPT, ...args], { cwd, encoding: 'utf8' })
|
||||||
|
return { status: 0, stdout, stderr: '' }
|
||||||
|
} catch (err: any) {
|
||||||
|
return { status: err.status ?? 1, stdout: err.stdout ?? '', stderr: err.stderr ?? '' }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function git(args: string[], cwd: string): string {
|
||||||
|
return execFileSync('git', ['-C', cwd, ...args], { encoding: 'utf8' }).trim()
|
||||||
|
}
|
||||||
|
|
||||||
|
const CHANGELOG_HEADER = '# Changelog\n\nAll notable changes, in this fixture.\n'
|
||||||
|
|
||||||
|
/** Build a CHANGELOG.md with one entry per [version, bullets[]] pair, newest first. */
|
||||||
|
function buildChangelog(entries: Array<{ version: string; date: string; bullets: string[] }>): string {
|
||||||
|
const body = entries
|
||||||
|
.map(
|
||||||
|
(e) =>
|
||||||
|
`### [${e.version}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/vX...v${e.version}) (${e.date})\n\n` +
|
||||||
|
e.bullets.map((b) => `- ${b} (abc1234)`).join('\n') +
|
||||||
|
'\n',
|
||||||
|
)
|
||||||
|
.join('\n')
|
||||||
|
return CHANGELOG_HEADER + '\n' + body
|
||||||
|
}
|
||||||
|
|
||||||
|
function wallFile(product: string, entries: unknown[]): string {
|
||||||
|
return JSON.stringify({ product, entries }, null, 2) + '\n'
|
||||||
|
}
|
||||||
|
|
||||||
|
const BASE_ENTRY = {
|
||||||
|
version: '10.4.11',
|
||||||
|
date: '2026-09-02',
|
||||||
|
headline: 'A faster open',
|
||||||
|
items: ['A faster open.'],
|
||||||
|
url: 'https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.11',
|
||||||
|
thumb: null,
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A throwaway bare repo standing in for the real soulcraftlabs/releases remote. */
|
||||||
|
function initBareRemote(): string {
|
||||||
|
const remoteDir = mkdtempSync(join(tmpdir(), 'wall-remote-'))
|
||||||
|
execFileSync('git', ['init', '--bare', '-b', 'main', remoteDir])
|
||||||
|
return remoteDir
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Seed the bare remote with an initial <product>.json, via a throwaway clone. */
|
||||||
|
function seedRemote(remoteDir: string, product: string, entries: unknown[]): void {
|
||||||
|
const seedDir = mkdtempSync(join(tmpdir(), 'wall-seed-'))
|
||||||
|
execFileSync('git', ['clone', remoteDir, seedDir], { stdio: 'ignore' })
|
||||||
|
git(['config', 'user.email', 'seed@example.com'], seedDir)
|
||||||
|
git(['config', 'user.name', 'Seed'], seedDir)
|
||||||
|
writeFileSync(join(seedDir, `${product}.json`), wallFile(product, entries))
|
||||||
|
git(['add', `${product}.json`], seedDir)
|
||||||
|
git(['commit', '-m', 'seed'], seedDir)
|
||||||
|
git(['push', 'origin', 'main'], seedDir)
|
||||||
|
rmSync(seedDir, { recursive: true, force: true })
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Read <product>.json back out of the bare remote's main tip, via a throwaway clone. */
|
||||||
|
function readRemote(remoteDir: string, product: string): any {
|
||||||
|
const readDir = mkdtempSync(join(tmpdir(), 'wall-read-'))
|
||||||
|
execFileSync('git', ['clone', remoteDir, readDir], { stdio: 'ignore' })
|
||||||
|
const data = JSON.parse(readFileSync(join(readDir, `${product}.json`), 'utf8'))
|
||||||
|
rmSync(readDir, { recursive: true, force: true })
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Reject every push — stands in for any push failure (including a genuine
|
||||||
|
* non-fast-forward raced by a concurrent release rail), which this script
|
||||||
|
* treats identically: refuse loudly, name the cure, touch nothing further. */
|
||||||
|
function makeRemoteRejectPushes(remoteDir: string): void {
|
||||||
|
const hookPath = join(remoteDir, 'hooks', 'pre-receive')
|
||||||
|
writeFileSync(hookPath, '#!/bin/sh\necho "remote: simulated push rejection" >&2\nexit 1\n')
|
||||||
|
chmodSync(hookPath, 0o755)
|
||||||
|
}
|
||||||
|
|
||||||
|
let dir: string
|
||||||
|
let remoteDir: string
|
||||||
|
let cacheDir: string
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
dir = mkdtempSync(join(tmpdir(), 'wall-entry-test-'))
|
||||||
|
remoteDir = initBareRemote()
|
||||||
|
cacheDir = join(mkdtempSync(join(tmpdir(), 'wall-cache-')), 'soulcraft-releases')
|
||||||
|
})
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
rmSync(dir, { recursive: true, force: true })
|
||||||
|
rmSync(remoteDir, { recursive: true, force: true })
|
||||||
|
rmSync(cacheDir, { recursive: true, force: true })
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('wall-entry.mjs — generate + publish', () => {
|
||||||
|
it('derives headline from the first bullet and items from every bullet, hashes stripped, and pushes it to the remote', () => {
|
||||||
|
seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY])
|
||||||
|
writeFileSync(
|
||||||
|
join(dir, 'CHANGELOG.md'),
|
||||||
|
buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix(wall): mechanize the entry', 'test(wall): pin the shape'] }]),
|
||||||
|
)
|
||||||
|
|
||||||
|
const result = run(
|
||||||
|
['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir],
|
||||||
|
dir,
|
||||||
|
)
|
||||||
|
expect(result.status).toBe(0)
|
||||||
|
expect(result.stdout).toMatch(/wrote v10\.4\.12.*pushed/i)
|
||||||
|
|
||||||
|
const wall = readRemote(remoteDir, 'open-brainy')
|
||||||
|
expect(wall.entries).toHaveLength(2)
|
||||||
|
expect(wall.entries[0]).toEqual({
|
||||||
|
version: '10.4.12',
|
||||||
|
date: '2026-09-03',
|
||||||
|
headline: 'fix(wall): mechanize the entry',
|
||||||
|
items: ['fix(wall): mechanize the entry', 'test(wall): pin the shape'],
|
||||||
|
url: 'https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.12',
|
||||||
|
thumb: null,
|
||||||
|
})
|
||||||
|
// the older entry stays put, still second
|
||||||
|
expect(wall.entries[1].version).toBe('10.4.11')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('prepends newest-first — the new entry lands at index 0 ahead of every existing one', () => {
|
||||||
|
seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY, { ...BASE_ENTRY, version: '10.4.10' }])
|
||||||
|
writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.5.0', date: '2026-09-03', bullets: ['feat: ten five'] }]))
|
||||||
|
|
||||||
|
run(['--product', 'open-brainy', '--version', '10.5.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], dir)
|
||||||
|
|
||||||
|
const wall = readRemote(remoteDir, 'open-brainy')
|
||||||
|
expect(wall.entries.map((e: any) => e.version)).toEqual(['10.5.0', '10.4.11', '10.4.10'])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('replaces an entry with the same version instead of duplicating it — idempotent re-runs', () => {
|
||||||
|
seedRemote(remoteDir, 'open-brainy', [
|
||||||
|
{ ...BASE_ENTRY, headline: 'stale headline, pre-fix' },
|
||||||
|
{ ...BASE_ENTRY, version: '10.4.10' },
|
||||||
|
])
|
||||||
|
writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: the corrected headline'] }]))
|
||||||
|
|
||||||
|
const result = run(
|
||||||
|
['--product', 'open-brainy', '--version', '10.4.11', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir],
|
||||||
|
dir,
|
||||||
|
)
|
||||||
|
expect(result.status).toBe(0)
|
||||||
|
expect(result.stdout).toMatch(/replaced v10\.4\.11/i)
|
||||||
|
|
||||||
|
const wall = readRemote(remoteDir, 'open-brainy')
|
||||||
|
expect(wall.entries).toHaveLength(2) // not 3 — replaced, not duplicated
|
||||||
|
expect(wall.entries[0].version).toBe('10.4.11')
|
||||||
|
expect(wall.entries[0].headline).toBe('fix: the corrected headline')
|
||||||
|
expect(wall.entries[1].version).toBe('10.4.10')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a re-run with byte-identical content commits nothing and still succeeds', () => {
|
||||||
|
// headline always equals items[0] for a derived entry, so this fixture
|
||||||
|
// (unlike BASE_ENTRY, whose headline/items intentionally diverge for the
|
||||||
|
// shape-only tests below) has to keep the two in lockstep to ever roundtrip.
|
||||||
|
const stableEntry = { ...BASE_ENTRY, headline: 'A faster open.', items: ['A faster open.'] }
|
||||||
|
seedRemote(remoteDir, 'open-brainy', [stableEntry])
|
||||||
|
writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['A faster open.'] }]))
|
||||||
|
const before = readRemote(remoteDir, 'open-brainy')
|
||||||
|
|
||||||
|
const result = run(
|
||||||
|
['--product', 'open-brainy', '--version', '10.4.11', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir],
|
||||||
|
dir,
|
||||||
|
)
|
||||||
|
expect(result.status).toBe(0)
|
||||||
|
expect(result.stdout).toMatch(/nothing to commit/i)
|
||||||
|
expect(readRemote(remoteDir, 'open-brainy')).toEqual(before)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('derives the public package-page permalink for the product engine (private repo, never null)', () => {
|
||||||
|
seedRemote(remoteDir, 'brainy', [{ ...BASE_ENTRY, version: '11.0.5', url: 'https://source.soulcraft.com/soulcraft/-/packages/npm/@soulcraft%2Fbrainy/11.0.5' }])
|
||||||
|
writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '11.0.6', date: '2026-09-03', bullets: ['fix: a native-only fix'] }]))
|
||||||
|
|
||||||
|
const result = run(
|
||||||
|
['--product', 'brainy', '--version', '11.0.6', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir],
|
||||||
|
dir,
|
||||||
|
)
|
||||||
|
expect(result.status).toBe(0)
|
||||||
|
|
||||||
|
const wall = readRemote(remoteDir, 'brainy')
|
||||||
|
expect(wall.entries[0].url).toBe('https://source.soulcraft.com/soulcraft/-/packages/npm/@soulcraft%2Fbrainy/11.0.6')
|
||||||
|
expect(wall.entries[0].thumb).toBeNull()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('refuses a product with no permalink pattern, naming the cure', () => {
|
||||||
|
seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY])
|
||||||
|
writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '1.0.0', date: '2026-09-03', bullets: ['feat: first'] }]))
|
||||||
|
|
||||||
|
const result = run(['--product', 'mystery', '--version', '1.0.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], dir)
|
||||||
|
expect(result.status).not.toBe(0)
|
||||||
|
expect(result.stderr).toMatch(/no permalink pattern for product "mystery"/)
|
||||||
|
expect(result.stderr).toMatch(/never carry url: null/)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('refuses when the CHANGELOG has no entry yet for the target version, and touches no remote', () => {
|
||||||
|
seedRemote(remoteDir, 'open-brainy', [])
|
||||||
|
writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: whatever'] }]))
|
||||||
|
const beforeSha = git(['rev-parse', 'main'], remoteDir)
|
||||||
|
|
||||||
|
const result = run(
|
||||||
|
['--product', 'open-brainy', '--version', '99.0.0', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir],
|
||||||
|
dir,
|
||||||
|
)
|
||||||
|
|
||||||
|
expect(result.status).toBe(1)
|
||||||
|
expect(result.stderr).toMatch(/no CHANGELOG entry yet/i)
|
||||||
|
expect(git(['rev-parse', 'main'], remoteDir)).toBe(beforeSha)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('refuses by naming the cure when the remote cannot be cloned', () => {
|
||||||
|
writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix: whatever'] }]))
|
||||||
|
const noSuchRemote = join(tmpdir(), 'wall-remote-does-not-exist-' + Date.now())
|
||||||
|
|
||||||
|
const result = run(
|
||||||
|
['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', noSuchRemote, '--cache-dir', cacheDir],
|
||||||
|
dir,
|
||||||
|
)
|
||||||
|
|
||||||
|
expect(result.status).toBe(1)
|
||||||
|
expect(result.stderr).toMatch(/cannot clone/i)
|
||||||
|
expect(result.stderr).toMatch(/cure:/i)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('refuses by naming the cure, and touches no remote, when the fetched wall fails shape validation', () => {
|
||||||
|
const seedDir = mkdtempSync(join(tmpdir(), 'wall-seed-broken-'))
|
||||||
|
execFileSync('git', ['clone', remoteDir, seedDir], { stdio: 'ignore' })
|
||||||
|
git(['config', 'user.email', 'seed@example.com'], seedDir)
|
||||||
|
git(['config', 'user.name', 'Seed'], seedDir)
|
||||||
|
writeFileSync(
|
||||||
|
join(seedDir, 'open-brainy.json'),
|
||||||
|
JSON.stringify({ product: 'open-brainy', entries: [{ version: '10.4.11', date: '2026-09-02', items: ['x'], url: null }] }, null, 2),
|
||||||
|
)
|
||||||
|
git(['add', 'open-brainy.json'], seedDir)
|
||||||
|
git(['commit', '-m', 'seed broken'], seedDir)
|
||||||
|
git(['push', 'origin', 'main'], seedDir)
|
||||||
|
rmSync(seedDir, { recursive: true, force: true })
|
||||||
|
const beforeSha = git(['rev-parse', 'main'], remoteDir)
|
||||||
|
|
||||||
|
writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix: whatever'] }]))
|
||||||
|
|
||||||
|
const result = run(
|
||||||
|
['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir],
|
||||||
|
dir,
|
||||||
|
)
|
||||||
|
|
||||||
|
expect(result.status).toBe(1)
|
||||||
|
expect(result.stderr).toMatch(/fails shape validation/i)
|
||||||
|
expect(result.stderr).toMatch(/missing key\(s\) headline/i)
|
||||||
|
expect(git(['rev-parse', 'main'], remoteDir)).toBe(beforeSha)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('refuses by naming the cure when the remote rejects the push (stands in for a raced non-fast-forward)', () => {
|
||||||
|
seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY])
|
||||||
|
makeRemoteRejectPushes(remoteDir)
|
||||||
|
writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix: whatever'] }]))
|
||||||
|
|
||||||
|
const result = run(
|
||||||
|
['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir],
|
||||||
|
dir,
|
||||||
|
)
|
||||||
|
|
||||||
|
expect(result.status).toBe(1)
|
||||||
|
expect(result.stderr).toMatch(/push to .* failed/i)
|
||||||
|
expect(result.stderr).toMatch(/cure:/i)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('refuses a cross-product write when the file\'s "product" field does not match --product', () => {
|
||||||
|
seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY])
|
||||||
|
const seedDir = mkdtempSync(join(tmpdir(), 'wall-seed-mismatch-'))
|
||||||
|
execFileSync('git', ['clone', remoteDir, seedDir], { stdio: 'ignore' })
|
||||||
|
git(['config', 'user.email', 'seed@example.com'], seedDir)
|
||||||
|
git(['config', 'user.name', 'Seed'], seedDir)
|
||||||
|
const corrupted = JSON.parse(readFileSync(join(seedDir, 'open-brainy.json'), 'utf8'))
|
||||||
|
corrupted.product = 'brainy'
|
||||||
|
writeFileSync(join(seedDir, 'open-brainy.json'), JSON.stringify(corrupted, null, 2) + '\n')
|
||||||
|
git(['add', 'open-brainy.json'], seedDir)
|
||||||
|
git(['commit', '-m', 'corrupt product field'], seedDir)
|
||||||
|
git(['push', 'origin', 'main'], seedDir)
|
||||||
|
rmSync(seedDir, { recursive: true, force: true })
|
||||||
|
|
||||||
|
writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '1.0.0', date: '2026-09-03', bullets: ['fix: wrong repo'] }]))
|
||||||
|
|
||||||
|
const result = run(
|
||||||
|
['--product', 'open-brainy', '--version', '1.0.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir],
|
||||||
|
dir,
|
||||||
|
)
|
||||||
|
|
||||||
|
expect(result.status).toBe(1)
|
||||||
|
expect(result.stderr).toMatch(/product "brainy".*--product "open-brainy"/i)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('wall-entry.mjs — --dry-run', () => {
|
||||||
|
it('prints the entry and the target path, and touches neither the cache dir nor the remote', () => {
|
||||||
|
seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY])
|
||||||
|
writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix: a dry run'] }]))
|
||||||
|
const beforeSha = git(['rev-parse', 'main'], remoteDir)
|
||||||
|
|
||||||
|
const result = run(
|
||||||
|
['--dry-run', '--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir],
|
||||||
|
dir,
|
||||||
|
)
|
||||||
|
|
||||||
|
expect(result.status).toBe(0)
|
||||||
|
expect(result.stdout).toMatch(/would write to/i)
|
||||||
|
expect(result.stdout).toMatch(/"version": "10\.4\.12"/)
|
||||||
|
expect(git(['rev-parse', 'main'], remoteDir)).toBe(beforeSha)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('wall-entry.mjs — --check', () => {
|
||||||
|
it('passes a well-formed, newest-first file with no duplicates', () => {
|
||||||
|
writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY, { ...BASE_ENTRY, version: '10.4.10' }]))
|
||||||
|
const result = run(['--check', '--file', 'wall.json'], dir)
|
||||||
|
expect(result.status).toBe(0)
|
||||||
|
expect(result.stdout).toMatch(/OK/)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('passes a file where "thumb" is entirely absent (optional per the HQ contract)', () => {
|
||||||
|
const { thumb, ...noThumb } = BASE_ENTRY as any
|
||||||
|
writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [noThumb]))
|
||||||
|
const result = run(['--check', '--file', 'wall.json'], dir)
|
||||||
|
expect(result.status).toBe(0)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('catches a missing entry key', () => {
|
||||||
|
const broken = { version: '1.0.0', date: '2026-09-03', headline: 'h', items: ['i'] } // no "url"
|
||||||
|
writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [broken]))
|
||||||
|
const result = run(['--check', '--file', 'wall.json'], dir)
|
||||||
|
expect(result.status).toBe(1)
|
||||||
|
expect(result.stderr).toMatch(/missing key\(s\) url/)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('catches an unexpected top-level key (e.g. the retired "history" field)', () => {
|
||||||
|
const raw = JSON.parse(wallFile('open-brainy', [BASE_ENTRY]))
|
||||||
|
raw.history = 'retired field'
|
||||||
|
writeFileSync(join(dir, 'wall.json'), JSON.stringify(raw))
|
||||||
|
const result = run(['--check', '--file', 'wall.json'], dir)
|
||||||
|
expect(result.status).toBe(1)
|
||||||
|
expect(result.stderr).toMatch(/unexpected key\(s\) history/)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('catches entries that are not newest-first', () => {
|
||||||
|
writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [{ ...BASE_ENTRY, version: '10.4.10' }, BASE_ENTRY]))
|
||||||
|
const result = run(['--check', '--file', 'wall.json'], dir)
|
||||||
|
expect(result.status).toBe(1)
|
||||||
|
expect(result.stderr).toMatch(/not newest-first/)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('catches a duplicate version even with identical entries', () => {
|
||||||
|
writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY, { ...BASE_ENTRY }]))
|
||||||
|
const result = run(['--check', '--file', 'wall.json'], dir)
|
||||||
|
expect(result.status).toBe(1)
|
||||||
|
expect(result.stderr).toMatch(/duplicate version 10\.4\.11/)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('catches an empty items array', () => {
|
||||||
|
writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [{ ...BASE_ENTRY, items: [] }]))
|
||||||
|
const result = run(['--check', '--file', 'wall.json'], dir)
|
||||||
|
expect(result.status).toBe(1)
|
||||||
|
expect(result.stderr).toMatch(/"items" must be a non-empty array/)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('catches a malformed date', () => {
|
||||||
|
writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [{ ...BASE_ENTRY, date: '09/03/2026' }]))
|
||||||
|
const result = run(['--check', '--file', 'wall.json'], dir)
|
||||||
|
expect(result.status).toBe(1)
|
||||||
|
expect(result.stderr).toMatch(/"date" must be a YYYY-MM-DD string/)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
@ -7,7 +7,7 @@
|
||||||
* hydration (zero per-entity reads when unfiltered). Both must preserve the exact
|
* hydration (zero per-entity reads when unfiltered). Both must preserve the exact
|
||||||
* pagination contract: same order, cursor continuation, filters, totalCount.
|
* pagination contract: same order, cursor continuation, filters, totalCount.
|
||||||
*/
|
*/
|
||||||
import { describe, it, expect, beforeEach, vi } from 'vitest'
|
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||||
import { Brainy, NounType } from '../../../src/index.js'
|
import { Brainy, NounType } from '../../../src/index.js'
|
||||||
|
|
||||||
describe('paginated enumeration — parallel hydration + id-only (cortex heal-cost)', () => {
|
describe('paginated enumeration — parallel hydration + id-only (cortex heal-cost)', () => {
|
||||||
|
|
@ -30,6 +30,10 @@ describe('paginated enumeration — parallel hydration + id-only (cortex heal-co
|
||||||
storage = brain.storage
|
storage = brain.storage
|
||||||
})
|
})
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
|
||||||
/** Page the whole dataset through a small limit via cursor and collect ordered ids. */
|
/** Page the whole dataset through a small limit via cursor and collect ordered ids. */
|
||||||
const pageAll = async (fn: (opts: any) => Promise<any>, key: 'items' | 'ids') => {
|
const pageAll = async (fn: (opts: any) => Promise<any>, key: 'items' | 'ids') => {
|
||||||
const out: string[] = []
|
const out: string[] = []
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,8 @@
|
||||||
* config (so it never runs and gives false coverage confidence — the exact drift
|
* config (so it never runs and gives false coverage confidence — the exact drift
|
||||||
* that left ~27 test files un-run before 8.0). Every `*.test.ts` must either match
|
* that left ~27 test files un-run before 8.0). Every `*.test.ts` must either match
|
||||||
* a gate config (`tests/unit/**`, `tests/integration/**`, `*.unit.test.ts`,
|
* a gate config (`tests/unit/**`, `tests/integration/**`, `*.unit.test.ts`,
|
||||||
* `*.integration.test.ts`) or be explicitly listed in MANUAL_ONLY below.
|
* `*.integration.test.ts`, or the perf lane's `tests/configs/vitest.perf.config.ts`
|
||||||
|
* — see PERF_LANE_FILES below) or be explicitly listed in MANUAL_ONLY below.
|
||||||
*/
|
*/
|
||||||
import { describe, it, expect } from 'vitest'
|
import { describe, it, expect } from 'vitest'
|
||||||
import { readdirSync } from 'node:fs'
|
import { readdirSync } from 'node:fs'
|
||||||
|
|
@ -24,10 +25,12 @@ function allTestFiles(dir: string, out: string[] = []): string[] {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Test files INTENTIONALLY excluded from the unit/integration gate: benchmarks,
|
* Test files INTENTIONALLY excluded from every automated gate — conformance
|
||||||
* scale/perf measurements, package-size checks, and real-model-load checks. They
|
* suites invoked directly, and checks that need real resources (network,
|
||||||
* are run manually (slow / need real resources), not in CI. Every entry is a
|
* unusual scale) no CI lane provides. Wall-clock/scale benchmarks that DO
|
||||||
* conscious decision — a NEW orphan not listed here fails the guard below.
|
* run automatically belong to the perf lane (PERF_LANE_FILES / inGate
|
||||||
|
* below), not here. Every entry is a conscious decision — a NEW orphan not
|
||||||
|
* listed here fails the guard below.
|
||||||
*/
|
*/
|
||||||
const MANUAL_ONLY = new Set<string>([
|
const MANUAL_ONLY = new Set<string>([
|
||||||
// Conformance suites run as an explicit gate stage (both engines run them
|
// Conformance suites run as an explicit gate stage (both engines run them
|
||||||
|
|
@ -40,15 +43,11 @@ const MANUAL_ONLY = new Set<string>([
|
||||||
// The sparse-store cut's shared operator rows (both engines run these):
|
// The sparse-store cut's shared operator rows (both engines run these):
|
||||||
// explicit conformance-gate invocation, like its siblings.
|
// explicit conformance-gate invocation, like its siblings.
|
||||||
'tests/conformance/sparse-store-cut.test.ts',
|
'tests/conformance/sparse-store-cut.test.ts',
|
||||||
'tests/api/performance-benchmarks.test.ts',
|
// NOT the perf lane: no wall-clock/scale assertion, so it does not belong
|
||||||
|
// in tests/configs/vitest.perf.config.ts's include list — genuinely run
|
||||||
|
// by hand only.
|
||||||
'tests/critical-neural-validation.test.ts',
|
'tests/critical-neural-validation.test.ts',
|
||||||
'tests/critical-performance-benchmark.test.ts',
|
|
||||||
'tests/model-loading.test.ts',
|
|
||||||
'tests/package-size-breakdown.test.ts',
|
'tests/package-size-breakdown.test.ts',
|
||||||
'tests/package-size-limit.test.ts',
|
|
||||||
'tests/performance/graph-scale-performance.test.ts',
|
|
||||||
'tests/performance/triple-intelligence-scale.test.ts',
|
|
||||||
'tests/performance/typeAware.bench.test.ts',
|
|
||||||
// Cross-engine field-addressing conformance suite: pinned bit-for-bit against
|
// Cross-engine field-addressing conformance suite: pinned bit-for-bit against
|
||||||
// the native accelerator's implementation of the SAME contract, and invoked
|
// the native accelerator's implementation of the SAME contract, and invoked
|
||||||
// directly (`npx vitest run tests/conformance/namespace-law.test.ts`), never
|
// directly (`npx vitest run tests/conformance/namespace-law.test.ts`), never
|
||||||
|
|
@ -59,6 +58,21 @@ const MANUAL_ONLY = new Set<string>([
|
||||||
'tests/conformance/namespace-law.test.ts'
|
'tests/conformance/namespace-law.test.ts'
|
||||||
])
|
])
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The perf lane's own gate: `tests/configs/vitest.perf.config.ts`, run by
|
||||||
|
* `npm run test:perf`. Mirrors that config's `include` list — kept in sync
|
||||||
|
* by inspection, the same convention that config uses against the root
|
||||||
|
* gate's exclude list (see its own header comment). A file that runs here
|
||||||
|
* is GATED, not manual: it belongs in this set (or the `tests/performance/`
|
||||||
|
* prefix below), never in MANUAL_ONLY.
|
||||||
|
*/
|
||||||
|
const PERF_LANE_FILES = new Set<string>([
|
||||||
|
'tests/critical-performance-benchmark.test.ts',
|
||||||
|
'tests/api/performance-benchmarks.test.ts',
|
||||||
|
'tests/package-size-limit.test.ts',
|
||||||
|
'tests/model-loading.test.ts'
|
||||||
|
])
|
||||||
|
|
||||||
function inGate(rel: string): boolean {
|
function inGate(rel: string): boolean {
|
||||||
return (
|
return (
|
||||||
rel.startsWith('tests/unit/') ||
|
rel.startsWith('tests/unit/') ||
|
||||||
|
|
@ -67,7 +81,11 @@ function inGate(rel: string): boolean {
|
||||||
// ('tests/lifecycle/**/*.test.ts'; see tests/lifecycle/README.md).
|
// ('tests/lifecycle/**/*.test.ts'; see tests/lifecycle/README.md).
|
||||||
rel.startsWith('tests/lifecycle/') ||
|
rel.startsWith('tests/lifecycle/') ||
|
||||||
rel.endsWith('.unit.test.ts') ||
|
rel.endsWith('.unit.test.ts') ||
|
||||||
rel.endsWith('.integration.test.ts')
|
rel.endsWith('.integration.test.ts') ||
|
||||||
|
// The perf lane (see PERF_LANE_FILES above) — mirrors
|
||||||
|
// tests/configs/vitest.perf.config.ts's `tests/performance/**` glob.
|
||||||
|
rel.startsWith('tests/performance/') ||
|
||||||
|
PERF_LANE_FILES.has(rel)
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@
|
||||||
* Tests to verify that brain.find({ type: NounType.X }) correctly filters entities
|
* Tests to verify that brain.find({ type: NounType.X }) correctly filters entities
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { describe, it, expect, beforeEach } from 'vitest'
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
import { Brainy, NounType } from '../../src/index.js'
|
import { Brainy, NounType } from '../../src/index.js'
|
||||||
|
|
||||||
describe('Type Filtering (A Consumer Team Issue)', () => {
|
describe('Type Filtering (A Consumer Team Issue)', () => {
|
||||||
|
|
@ -17,6 +17,10 @@ describe('Type Filtering (A Consumer Team Issue)', () => {
|
||||||
await brain.init()
|
await brain.init()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
|
||||||
it('should filter entities by NounType.Person', async () => {
|
it('should filter entities by NounType.Person', async () => {
|
||||||
// Add 3 people
|
// Add 3 people
|
||||||
await brain.add({ data: 'John Smith', type: NounType.Person, metadata: { name: 'John' } })
|
await brain.add({ data: 'John Smith', type: NounType.Person, metadata: { name: 'John' } })
|
||||||
|
|
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue