diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 49956718..cdb2ab14 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -34,5 +34,7 @@ jobs:
with:
bun-version: latest
- run: npm ci
+ # test:bun imports the built dist/, so build first.
+ - run: npm run build
# Bun as a runtime is the supported Bun story (`bun add` / `bun run`).
- run: npm run test:bun
diff --git a/.gitignore b/.gitignore
index d7d40cbb..64e235b3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -93,9 +93,14 @@ docs/internal/
# Rust/Cargo build artifacts
src/embeddings/candle-wasm/target/
src/embeddings/candle-wasm/Cargo.lock
-src/embeddings/wasm/pkg/
-# But keep the pre-built WASM (committed for users without Rust)
+# Ignore the wasm-pack output dir's CONTENTS (note the `/*`, not `/`, so the
+# re-includes below can take effect — git cannot re-include a file whose parent
+# DIRECTORY is excluded). Keep the pre-built WASM committed: it ships in the npm
+# package anyway, it lets consumers + CI build without a Rust/wasm-pack toolchain,
+# and versioning it makes the shipped artifact reproducible (not "whatever the
+# maintainer last built").
+src/embeddings/wasm/pkg/*
!src/embeddings/wasm/pkg/*.wasm
!src/embeddings/wasm/pkg/*.js
!src/embeddings/wasm/pkg/*.d.ts
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f37736a8..fd0de54e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,274 @@
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
+### [8.9.0](https://github.com/soulcraftlabs/brainy/compare/v8.8.2...v8.9.0) (2026-07-19)
+
+- docs: measured performance envelopes v1 (per-op p50/p95 at 1k and 10k, pure-JS floor) (5cabd78)
+- fix: release drains in-flight writer-lock heartbeat — no phantom lock after unlink (70e4bc8)
+- feat: flush() never compacts — history maintenance moves to close() with bounded passes (300d9f2)
+
+
+### [8.8.2](https://github.com/soulcraftlabs/brainy/compare/v8.8.1...v8.8.2) (2026-07-19)
+
+- fix: one field-resolution law across aggregation hooks, source.where, removeMany, and find() spellings (945d92d)
+- chore: push public docs to the soulcraft.com ingest door on release (42037d0)
+
+
+### [8.8.1](https://github.com/soulcraftlabs/brainy/compare/v8.8.0...v8.8.1) (2026-07-18)
+
+- fix: O(1) adaptive retention accounting + historyStats fleet audit (6207e48)
+- fix: import dedup off-switch honesty + brain-owned lifecycle for the background pass (4fcef7b)
+
+
+### [8.8.0](https://github.com/soulcraftlabs/brainy/compare/v8.7.1...v8.8.0) (2026-07-17)
+
+- feat: OS-limit detection for pool-scale deployments (16a73b8)
+
+
+### [8.7.1](https://github.com/soulcraftlabs/brainy/compare/v8.7.0...v8.7.1) (2026-07-17)
+
+- fix: race-proof writer-lock acquisition + machine-readable conflict through init (01a3b46)
+
+
+### [8.7.0](https://github.com/soulcraftlabs/brainy/compare/v8.6.0...v8.7.0) (2026-07-17)
+
+- feat: scaled transact budgets + labeled timeout diagnostics + envelope docs (6ef9fcb)
+
+
+### [8.6.0](https://github.com/soulcraftlabs/brainy/compare/v8.5.2...v8.6.0) (2026-07-17)
+
+- feat: brain.auditGraph() — read-only graph-truth audit (2a03fae)
+
+
+### [8.5.2](https://github.com/soulcraftlabs/brainy/compare/v8.5.1...v8.5.2) (2026-07-17)
+
+- fix: exception-safe aggregation backfill + generation-verified adoption + loud open-path guards (a77b064)
+
+
+### [8.5.1](https://github.com/soulcraftlabs/brainy/compare/v8.5.0...v8.5.1) (2026-07-17)
+
+- fix: aggregation state adoption on reopen + single-flight backfill + query-cap ratchet removal (da55be7)
+- docs: external-backups/sparse-storage guide + generation fact log concept (593bb8b)
+
+
+### [8.5.0](https://github.com/soulcraftlabs/brainy/compare/v8.4.0...v8.5.0) (2026-07-15)
+
+- test: tolerant timing assertion in the execution-time measure test (4dc0a92)
+- feat: committedGeneration capability + pinned durability/stability contracts (d1ecee1)
+- docs: RELEASES.md entry for 8.5.0 (provider fact-log access + shared verifier) (e4f37cd)
+- feat: provider access to the fact log + shared stamp verifier via internals (352e356)
+
+
+### [8.4.0](https://github.com/soulcraftlabs/brainy/compare/v8.3.3...v8.4.0) (2026-07-15)
+
+- docs: RELEASES.md entry for 8.4.0 (generation fact log + family stamp) (4a60b43)
+- feat: entity-tree family stamp — sourceGeneration + rollup coherence at open (2888ae6)
+- feat: generation fact log — after-image commit records, dual-written at every commit point (38b0041)
+
+
+### [8.3.3](https://github.com/soulcraftlabs/brainy/compare/v8.3.2...v8.3.3) (2026-07-15)
+
+- docs: RELEASES.md entry for 8.3.3 (rename containment fix + repair) (c3feafd)
+- test: lens-consistency regression — combined vs subtype-only vs canonical ground truth (4fb41f9)
+- fix: VFS rename moves the containment edge — no ghost in the old directory (af8c179)
+
+
+### [8.3.2](https://github.com/soulcraftlabs/brainy/compare/v8.3.1...v8.3.2) (2026-07-14)
+
+- docs: RELEASES.md entry for 8.3.2 (honest counters) (0932ecd)
+- fix: honest counters — removal never re-reads the removed record + repairIndex recounts and persists all rollups (2e2ba9c)
+
+
+### [8.3.1](https://github.com/soulcraftlabs/brainy/compare/v8.3.0...v8.3.1) (2026-07-14)
+
+- docs: RELEASES.md entry for 8.3.1 (full-removal deletes + family-scoped gate) (c0c68ac)
+- fix: full-removal canonical deletes + family-scoped migration gate (366f9a9)
+- docs: cite the cross-layer integrity contract generically in comments and notes (1d26988)
+
+
+### [8.3.0](https://github.com/soulcraftlabs/brainy/compare/v8.2.8...v8.3.0) (2026-07-13)
+
+- docs: RELEASES.md entry for 8.3.0 (heal-cost + cross-layer integrity contract) (7692c6f)
+- perf: parallel + id-only canonical enumeration (heal-cost dominant term) (ec5b933)
+- feat: registered-blob family contract — declared index blobs are undeletable (bfa1762)
+- feat: validateIndexConsistency delegates to provider invariants (6bcb54f)
+
+
+### [8.2.8](https://github.com/soulcraftlabs/brainy/compare/v8.2.7...v8.2.8) (2026-07-13)
+
+- fix: honest index readiness — no silently-empty queries on a cold index (d0f69c7)
+
+
+### [8.2.7](https://github.com/soulcraftlabs/brainy/compare/v8.2.6...v8.2.7) (2026-07-13)
+
+- fix: restore loadBinaryBlob fault-propagation (native column-store lockstep) (b6c7039)
+
+
+### [8.2.6](https://github.com/soulcraftlabs/brainy/compare/v8.2.5...v8.2.6) (2026-07-13)
+
+- docs: RELEASES.md entry for 8.2.6 (write/index-spine hardening) (a873852)
+- chore: hold loadBinaryBlob fault-propagation for the cortex column-store lockstep (36c10c1)
+- fix: aggregation surfaces materialize/state-load failures loudly (02eff64)
+- fix: surface a degraded derived index on reads instead of serving it silently (ba958d9)
+- fix: saveBinaryBlob never acks a durable write that stored nothing (7feba49)
+- fix: refuse writes when single-op history cannot be made durable (54c1836)
+- fix: clear() wipes the full native/derived footprint, not a subset (d8301f8)
+- fix: surface segment/entity read faults loudly instead of masking as absent (af5d2f3)
+- fix: spine hardening pass 1 (part) — count symmetry, honest partial-load, flush durability, read-fault propagation (119087a)
+- test: pin the read-your-writes contract under the single writer (eb9c4eb)
+
+
+### [8.2.5](https://github.com/soulcraftlabs/brainy/compare/v8.2.4...v8.2.5) (2026-07-12)
+
+- docs: RELEASES.md entry for 8.2.5 (honest rollback-failure response) (a7c7aa5)
+- fix: honest response when a transaction rollback cannot complete (711d2f0)
+
+
+### [8.2.4](https://github.com/soulcraftlabs/brainy/compare/v8.2.3...v8.2.4) (2026-07-12)
+
+- docs: RELEASES.md entry for 8.2.4 (non-destructive restore) (4574695)
+- fix: non-destructive, crash-resumable restore (a2f4f6a)
+
+
+### [8.2.3](https://github.com/soulcraftlabs/brainy/compare/v8.2.2...v8.2.3) (2026-07-12)
+
+- docs: RELEASES.md entry for 8.2.3 (transact durability barrier) (be5ce0b)
+- fix: transact durability barrier — committed transactions are durable on return (3b8fa51)
+
+
+### [8.2.2](https://github.com/soulcraftlabs/brainy/compare/v8.2.1...v8.2.2) (2026-07-11)
+
+- docs: RELEASES.md entry for 8.2.2 (transaction timeout rollback) (ed97006)
+- fix: transaction timeout rolls back applied operations (no torn state) (508a8e3)
+
+
+### [8.2.1](https://github.com/soulcraftlabs/brainy/compare/v8.2.0...v8.2.1) (2026-07-10)
+
+- test: update graph-index operation constructors to the VerbEndpointInts signature (62a449d)
+- docs: RELEASES.md entry for 8.2.1 (transact forward-ref parity fix) (7089782)
+- fix: transact forward references resolve graph endpoint ints at execute time (a175406)
+
+
+### [8.2.0](https://github.com/soulcraftlabs/brainy/compare/v8.1.0...v8.2.0) (2026-07-10)
+
+- docs: RELEASES.md entry for 8.2.0 (temporal VFS) (98ceadc)
+- feat: temporal VFS — file content joins the Model-B immutability model (a3467e1)
+- docs: pin the write-path invariant in the plugin contract (the onChange change-feed guarantee) (4af8fb3)
+
+
+### [8.1.0](https://github.com/soulcraftlabs/brainy/compare/v8.0.17...v8.1.0) (2026-07-10)
+
+- docs: RELEASES.md entry for 8.1.0 (brain.onChange change feed) (4e9be08)
+- feat: brain.onChange — the in-process change feed for every committed mutation (fd5edb5)
+
+
+### [8.0.17](https://github.com/soulcraftlabs/brainy/compare/v8.0.16...v8.0.17) (2026-07-08)
+
+- docs: RELEASES.md entry for 8.0.17 (canonical count recovery + dead-machinery sweep) (6b8b9cb)
+- fix: count recovery scans the canonical layout; remove the dead 7.x hnsw sharding machinery (352e2da)
+
+
+### [8.0.16](https://github.com/soulcraftlabs/brainy/compare/v8.0.15...v8.0.16) (2026-07-08)
+
+- docs: RELEASES.md entry for 8.0.16 (atomic ifAbsent/upsert + exact blob refCounts) (54e7c0e)
+- fix: atomic ifAbsent/upsert inserts + exact blob reference counts under concurrency (867939e)
+
+
+### [8.0.15](https://github.com/soulcraftlabs/brainy/compare/v8.0.14...v8.0.15) (2026-07-08)
+
+- docs: RELEASES.md entry for 8.0.15 (atomic ifRev CAS) (b1fe25a)
+- fix: ifRev CAS is atomic — the revision check now runs under the commit mutex (9a3d1bd)
+
+
+### [8.0.14](https://github.com/soulcraftlabs/brainy/compare/v8.0.13...v8.0.14) (2026-07-07)
+
+- docs: RELEASES.md entry for 8.0.14 (migration preserves branch-scoped non-entity state) (64188a3)
+- fix: 7→8 migration preserves branch-scoped non-entity state instead of deleting it (a93bb4e)
+
+
+### [8.0.13](https://github.com/soulcraftlabs/brainy/compare/v8.0.12...v8.0.13) (2026-07-07)
+
+- docs: RELEASES.md entry for 8.0.13 (accurate boot log for established stores) (38e8de5)
+- fix: an established store no longer boot-logs "New installation" (3086916)
+
+
+### [8.0.12](https://github.com/soulcraftlabs/brainy/compare/v8.0.11...v8.0.12) (2026-07-07)
+
+- docs: RELEASES.md entry for 8.0.12 (7→8 VFS recovery, zero-rebuild cold open, strict query operators) (d9017e7)
+- fix: recover VFS content blobs stranded by a 7→8 upgrade, in place on open (c0f6ccd)
+- fix: validate where-operators and align the in-memory matcher to the documented set (6821e19)
+- docs: correct rc-era time-travel staleness + record the embedding-model ordering constraint (68da660)
+- fix: cold-open no longer re-derives durable indexes — complete the readiness contract for all three providers (61c247c)
+- docs: RELEASES.md entry for 8.0.11 (exit-hang class closed for every op shape) (4fde94b)
+
+
+### [8.0.11](https://github.com/soulcraftlabs/brainy/compare/v8.0.10...v8.0.11) (2026-07-02)
+
+- fix: no script shape can hang on brainy's internals — unref every maintenance timer + one-shot beforeExit (30eacbd)
+- docs: RELEASES.md entry for 8.0.10 (clean process exit after close) (2da2736)
+
+
+### [8.0.10](https://github.com/soulcraftlabs/brainy/compare/v8.0.9...v8.0.10) (2026-07-02)
+
+- fix: a bare script now exits cleanly after close() — release every process keep-alive (c540d63)
+
+
+### [8.0.9](https://github.com/soulcraftlabs/brainy/compare/v8.0.8...v8.0.9) (2026-07-02)
+
+- feat: guarded plugin auto-detection — installing @soulcraft/cor is the opt-in (588267b)
+
+
+### [8.0.8](https://github.com/soulcraftlabs/brainy/compare/v8.0.7...v8.0.8) (2026-07-02)
+
+- docs: plugins are explicit opt-in — correct the README scale section and plugins config comment (e420369)
+
+
+### [8.0.7](https://github.com/soulcraftlabs/brainy/compare/v8.0.1...v8.0.7) (2026-07-02)
+
+- docs: GA version is 8.0.7 — npm retired 8.0.0-8.0.6 (January dev-cycle unpublishes) (5db2c41)
+- chore(release): 8.0.1 (48bea9e)
+- docs: flagship README for the 8.0 GA; GA version is 8.0.1 (e44620e)
+- docs: rename the native provider to @soulcraft/cor across public docs and JSDoc (bf4a333)
+- chore(release): 8.0.0 (a3c2717)
+- docs: RELEASES.md 8.0.0 GA entry (RC notes become history) (4584d0b)
+- feat: promote the 8.0 u64-id line to main for the 8.0.0 GA (55d57f8)
+- fix(8.0): byte-copy _id_mapper/* in the pre-upgrade backup (cor's write-new nuance) (a30ed72)
+- chore(release): 7.33.5 (29b9d5f)
+- fix: metadata cold-read guard — no more silent [] on cold find({where}) (7.33.5) (9dc4c5e)
+- fix(8.0): metadata cold-read guard — no more silent [] on cold find({where}) (79e8709)
+- docs(8.0): add module JSDoc to typeValidation.ts (the one file missing a module block) (ab53fa0)
+- feat(8.0): auto pre-upgrade backup — hard-link snapshot before the 7.x→8.0 migration (1aad1f6)
+- fix(ci): commit the prebuilt wasm pkg + build before test:bun (green CI on fresh clone) (ed178e2)
+- chore(release): 7.33.4 (2be3d0f)
+- fix: never serve a silent [] from find({connected}) on a cold-loaded graph (fd699d0)
+- chore(release): 7.33.3 (d1665bb)
+- fix: re-validate find() results against the predicate (index-integrity guard) (7b5db0d)
+- chore(release): 7.33.2 (9593a27)
+- fix: graph adjacency cold-load consistency guard — no more silent [] on connected (1694f68)
+- chore(release): 7.33.1 (811c7da)
+- fix: getNouns cursor pagination re-scanned the first page forever (permanent CPU loop) (6721c52)
+- chore(release): 7.33.0 (526aaad)
+- feat: visibility tier (public/internal/system) on nouns + verbs (3a62445)
+- chore(release): 7.32.2 (c53dd61)
+- refactor: rename BackupData → PortableGraph (the type is interchange, not a backup) (89036de)
+- chore(release): 7.32.1 (5e7379d)
+- fix: getNouns().totalCount reports true total, not page size; quiet benign mmap-vector log (edff637)
+- chore(release): 7.32.0 (adec0ba)
+- feat: portable graph export()/import() (BackupData v1) on brain.data() (a408d37)
+- chore(release): 7.31.8 (89c6d04)
+- fix: query-cap memory misread (MemAvailable + floor) + rootDirectory getter for native mmap fast-path (3f8e097)
+- chore(release): 7.31.7 (4f8159c)
+- fix: vfs.rename() issues a metadata-only update + rollback of fresh adds removes them (ac29b0e)
+- chore(release): 7.31.6 (9b52629)
+- fix: remap reserved fields from update() metadata patches to their canonical location (67e5fc8)
+- chore(release): 7.31.5 (e5ec658)
+- fix: feature-detect setVectorBackend before wiring the mmap-vector backend (a537b36)
+- chore(release): 7.31.4 (a8cbab6)
+- fix: feature-detect setConnectionsCodec before wiring the connections codec (747ab97)
+- chore(release): 7.31.3 (cfb051c)
+- fix: mmap-vector backend capacity NaN at the provider FFI boundary (eade6ff)
+
+
### [8.0.0-rc.9](https://github.com/soulcraftlabs/brainy/compare/v8.0.0-rc.8...v8.0.0-rc.9) (2026-07-01)
- docs(8.0): RELEASES.md — rc.9 (migration LOCK + 6x cosine + ES2023/Node22 floor) (3a33987)
diff --git a/README.md b/README.md
index ce7452a8..d1342f52 100644
--- a/README.md
+++ b/README.md
@@ -1,440 +1,217 @@
-# Brainy
-
- 
+
-[](https://www.npmjs.com/package/@soulcraft/brainy)
-[](https://www.npmjs.com/package/@soulcraft/brainy)
-[](https://soulcraft.com/docs)
-[](LICENSE)
-[](https://www.typescriptlang.org/)
+Brainy
-**Three database paradigms. One API. Zero configuration.**
+
+ Three database paradigms. One API. Zero configuration.
+ The in-process knowledge database for TypeScript — vector search, graph traversal,
+ and metadata filtering unified in a single query.
+
-Built because we were tired of stitching together Pinecone + Neo4j + MongoDB and spending weeks on configuration before writing a single line of business logic. Brainy unifies vector search, graph traversal, and metadata filtering so you don't have to choose.
+
+ 
+ 
+ 
+ 
+ 
+ 
+
-**New here?** → **[What is Brainy? — plain-language overview, no jargon](docs/eli5.md)**
+
+ Quick start ·
+ One query ·
+ Features ·
+ Scale with Cor ·
+ Docs
+
---
-## Install
+Built because we were tired of stitching a vector store to a graph database to a document store — and spending weeks on plumbing before writing a line of business logic. Brainy indexes every fact **three ways at once** and lets one call query them together:
+
+| You write | Brainy indexes it as | You query it with |
+|---|---|---|
+| `data: 'Ada wrote the first program'` | a **384-dim vector** (local embedding — no API key) | `find({ query: 'computing pioneers' })` |
+| `metadata: { field: 'CS', year: 1843 }` | **structured fields** (O(1) exact, O(log n) range) | `find({ where: { year: { lessThan: 1900 } } })` |
+| `relate({ from: ada, to: babbage })` | a **typed, directed graph edge** | `find({ connected: { to: babbage, depth: 2 } })` |
+
+It runs **inside your process** — no server, no Docker, nothing to operate — and persists to plain files you can snapshot with a hard link.
+
+**New here?** → **[What is Brainy? — plain-language overview, no jargon](docs/eli5.md)**
+
+## Quick start
```bash
-bun add @soulcraft/brainy # fastest — recommended
-npm install @soulcraft/brainy # Node.js — fully supported
+bun add @soulcraft/brainy # Bun ≥ 1.1 — recommended
+npm install @soulcraft/brainy # Node.js ≥ 22
```
-## Quick Start
-
```javascript
import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
-const brain = new Brainy()
+const brain = new Brainy() // in-memory; one line swaps to disk
await brain.init()
-// Add knowledge — text auto-embeds, metadata auto-indexes
-const reactId = await brain.add({
+// Text auto-embeds locally; metadata auto-indexes
+const react = await brain.add({
data: 'React is a JavaScript library for building user interfaces',
type: NounType.Concept,
+ subtype: 'library',
metadata: { category: 'frontend', year: 2013 }
})
-const nextId = await brain.add({
- data: 'Next.js framework for React with server-side rendering',
+const next = await brain.add({
+ data: 'Next.js is a React framework with server-side rendering',
type: NounType.Concept,
- metadata: { category: 'framework', year: 2016 }
+ subtype: 'framework',
+ metadata: { category: 'frontend', year: 2016 }
})
-// Create a relationship
-await brain.relate({ from: nextId, to: reactId, type: VerbType.BuiltOn })
-
-// Query all three paradigms at once
-const results = await brain.find({
- query: 'modern frontend frameworks', // Vector similarity
- where: { year: { greaterThan: 2015 } }, // Metadata filtering
- connected: { to: reactId, depth: 2 } // Graph traversal
-})
+await brain.relate({ from: next, to: react, type: VerbType.DependsOn, subtype: 'runtime' })
```
-**[Full API Reference](docs/api/README.md)** | **[soulcraft.com/docs](https://soulcraft.com/docs)**
-
----
-
-## Three Indexes, One Query
-
-Every piece of knowledge lives in three indexes simultaneously:
-
-- **`data`** → **Vector index** — Content for semantic search. Strings auto-embed into 384-dim vectors. Queried with `find({ query: '...' })`.
-- **`metadata`** → **Metadata index** — Structured fields for filtering. O(1) lookups. Queried with `find({ where: { ... } })`.
-- **`relate()`** → **Graph index** — Typed, directed relationships between entities. Traversed with `find({ connected: { ... } })`.
-
-```javascript
-// Data → vector index (semantic search)
-const articleId = await brain.add({
- data: 'A deep dive into transformer architectures',
- type: NounType.Document,
- metadata: { author: 'Dr. Chen', year: 2024, tags: ['AI'] } // → metadata index
-})
-
-// Relationships → graph index
-await brain.relate({ from: authorId, to: articleId, type: VerbType.Authored })
-
-// Query all three at once
-brain.find({
- query: 'attention mechanisms', // Vector similarity
- where: { year: { greaterThan: 2023 } }, // Metadata filter
- connected: { from: authorId, depth: 1 } // Graph traversal
-})
-```
-
-**[Data Model Reference](docs/DATA_MODEL.md)** | **[Query Operators](docs/QUERY_OPERATORS.md)**
-
----
-
-## Features
-
-### Triple Intelligence
-
-Vector search + graph traversal + metadata filtering in every query. No stitching services together — one `find()` call combines all three.
+## One query, three engines
```javascript
const results = await brain.find({
- query: 'machine learning',
- where: { department: 'engineering', level: 'senior' },
- connected: { from: teamLeadId, via: VerbType.WorksWith, depth: 2 }
+ query: 'modern frontend frameworks', // vector — what it means
+ where: { year: { greaterThan: 2015 } }, // metadata — what it is
+ connected: { to: react, depth: 2 } // graph — what it touches
})
```
-### Hybrid Search
+Every clause is optional; any combination composes. Under the hood Brainy plans the query across an HNSW vector index, a roaring-bitmap field index, and an adjacency graph index — and re-validates every result against your predicate before returning it, so a corrupt index can never hand you a wrong answer.
-Automatically combines keyword (text) and semantic (vector) search. No configuration needed.
+## Feature tour
+
+### The database is a value
+
+Pin it, rewind it, fork it. Snapshot isolation without a server.
```javascript
-await brain.find({ query: 'David Smith' }) // Auto: text + semantic
-await brain.find({ query: 'AI concepts', searchMode: 'semantic' }) // Semantic only
-await brain.find({ query: 'exact id', searchMode: 'text' }) // Text only
+const db = brain.now() // pin current state — O(1)
+
+await brain.transact([ // atomic all-or-nothing, CAS-guarded
+ { op: 'update', id: order, metadata: { status: 'paid' } },
+ { op: 'relate', from: invoice, to: order, type: VerbType.References, subtype: 'billing' }
+], { ifAtGeneration: db.generation })
+
+await db.get(order) // still 'pending' — pinned forever
+await brain.get(order) // 'paid' — live
+
+const lastWeek = await brain.asOf(Date.now() - 7 * 86_400_000) // full query surface, past state
+const whatIf = await db.with([{ op: 'remove', id: order }]) // speculative — never touches disk
+await brain.now().persist('/backups/today') // instant hard-link snapshot
```
-### Query Operators
+**[Consistency model](docs/concepts/consistency-model.md)** · **[Snapshots & time travel](docs/guides/snapshots-and-time-travel.md)**
-Filter metadata with equality, comparison, array, existence, pattern, and logical operators:
+### Local embeddings — no API keys
+
+Strings embed on-device with a bundled MiniLM model (WASM). Semantic search works offline, in CI, and on air-gapped machines, at zero cost per call. Hybrid keyword + semantic ranking is the default:
```javascript
-await brain.find({
- where: {
- status: 'active', // Exact match
- score: { greaterThan: 90 }, // Comparison
- tags: { contains: 'ai' }, // Array
- anyOf: [{ role: 'admin' }, { role: 'owner' }] // Logical OR
- }
-})
+await brain.find({ query: 'David Smith' }) // auto: text + semantic
+await brain.find({ query: 'AI concepts', searchMode: 'semantic' }) // semantic only
```
-**[Query Operators Reference](docs/QUERY_OPERATORS.md)** — all operators with indexed/in-memory matrix
+### A typed graph, not a bag of edges
-### Graph Relationships
-
-Typed, directed edges between entities. Traverse connections at any depth.
+42 entity types × 127 relationship types form a shared vocabulary for any domain — healthcare (`Patient → diagnoses → Condition`), finance (`Account → transfers → Transaction`), yours. Your own taxonomy layers on with `subtype`, enforced at write time:
```javascript
-await brain.relate({ from: personId, to: projectId, type: VerbType.WorksOn })
+await brain.add({ data: 'Avery Brooks', type: NounType.Person, subtype: 'employee' })
-const results = await brain.find({
- connected: { from: personId, via: VerbType.WorksOn, depth: 3 }
-})
+brain.counts.bySubtype(NounType.Person) // O(1) — { employee: 12, customer: 847 }
+brain.requireSubtype(NounType.Person, { values: ['employee', 'customer'], required: true })
```
-### Database as a Value
+**[Type system](docs/architecture/noun-verb-taxonomy.md)** · **[Subtypes & facets](docs/guides/subtypes-and-facets.md)**
-The whole database, pinned as an immutable value. Snapshot isolation, time travel, atomic transactions, instant hard-link snapshots.
+### Graph analytics built in
```javascript
-const db = brain.now() // Pin current state — O(1)
-
-// Atomic multi-write transaction (all-or-nothing, with CAS)
-await brain.transact([
- { op: 'update', id: orderId, metadata: { status: 'paid' } },
- { op: 'relate', from: invoiceId, to: orderId, type: VerbType.References, subtype: 'billing' }
-], { meta: { author: 'billing-service' }, ifAtGeneration: db.generation })
-
-await db.get(orderId) // Still 'pending' — pinned, forever
-await brain.get(orderId) // 'paid' — live
-
-// Time travel: full query surface at any past state
-const yesterday = await brain.asOf(new Date(Date.now() - 86_400_000))
-const past = await yesterday.find({ query: 'unpaid orders' })
-
-// What-if: speculative writes, nothing touches disk
-const whatIf = await db.with([{ op: 'remove', id: orderId }])
-
-// Instant backup: hard-link snapshot, opens read-only with Brainy.load()
-await brain.now().persist('/backups/today')
+await brain.graph.rank() // which entities matter most (centrality)
+await brain.graph.communities() // natural clusters
+await brain.graph.path(a, b) // how two things connect
+await brain.graph.subgraph([seed], { depth: 2 }) // bounded neighborhood → { nodes, edges }
+await brain.graph.export() // whole graph, one O(N+E) streaming pass
```
-**[Consistency Model](docs/concepts/consistency-model.md)** | **[Snapshots & Time Travel](docs/guides/snapshots-and-time-travel.md)**
+### Write-time aggregations
-### Virtual Filesystem
+`SUM` / `COUNT` / `AVG` / `MIN` / `MAX` with `GROUP BY` and time windows, maintained incrementally on every write — reads are O(1) lookups, not scans. **[Aggregation guide](docs/guides/aggregation.md)**
-File operations with semantic search built in.
-
-```javascript
-const vfs = brain.vfs
-
-await vfs.writeFile('/docs/readme.md', 'Project documentation')
-const content = await vfs.readFile('/docs/readme.md')
-const tree = await vfs.getTreeStructure('/docs', { maxDepth: 3 })
-
-// Semantic file search
-const matches = await vfs.search('React components with hooks')
-```
-
-**[VFS Quick Start](docs/vfs/QUICK_START.md)** | **[Common Patterns](docs/vfs/COMMON_PATTERNS.md)**
-
-### Import Anything
-
-CSV, Excel, PDF, URLs — auto-detected format, auto-classified entities.
+### Import anything
```javascript
await brain.import('customers.csv')
-await brain.import('sales-data.xlsx') // all sheets processed
-await brain.import('research-paper.pdf') // tables extracted automatically
+await brain.import('sales.xlsx') // every sheet
+await brain.import('research-paper.pdf') // tables extracted
await brain.import('https://api.example.com/data.json')
-await brain.import('./handbook.md', { vfsPath: '/imports/handbook' }) // preserve in VFS
```
-**[Import Guide](docs/guides/import-anything.md)**
+Entities auto-classify on the way in; `brain.extractEntities(text)` exposes the same NER ensemble directly. **[Import guide](docs/guides/import-anything.md)**
-### Entity Extraction
-
-AI-powered named entity recognition with 4-signal ensemble scoring.
+### A filesystem that understands content
```javascript
-const entities = await brain.extractEntities('John Smith founded Acme Corp in New York')
-// [
-// { text: 'John Smith', type: NounType.Person, confidence: 0.95 },
-// { text: 'Acme Corp', type: NounType.Organization, confidence: 0.92 },
-// { text: 'New York', type: NounType.Location, confidence: 0.88 }
-// ]
+await brain.vfs.writeFile('/docs/readme.md', 'Project documentation')
+await brain.vfs.search('React components with hooks') // semantic file search
```
-**[Neural Extraction Guide](docs/neural-extraction.md)**
+**[VFS quick start](docs/vfs/QUICK_START.md)**
-### Plugin System
+### Operations-grade by default
-Optional native acceleration via `@soulcraft/cortex` — SIMD distance calculations, CRoaring bitmaps, Candle ML embeddings.
+- **Single-writer, many-reader** — an exclusive lock protects the data directory; `Brainy.openReadOnly()` and the `brainy inspect` CLI examine a live brain from another process, safely.
+- **Self-upgrading data files** — a 7.x brain opens under 8.x and migrates itself behind an observable lock (`getIndexStatus().migration`), with an automatic pre-upgrade backup. No migration scripts.
+- **No silent wrong answers** — cold-open guards self-heal or throw typed errors (`MetadataIndexNotReadyError`, `GraphIndexNotReadyError`); they never return `[]` for data that exists.
+
+**[Multi-process model](docs/concepts/multi-process.md)** · **[Inspection guide](docs/guides/inspection.md)**
+
+## From laptop to hundreds of millions
+
+Brainy's TypeScript engines take you a long way. When you outgrow them, add the native engine — **the API doesn't change**:
+
+```bash
+npm install @soulcraft/cor
+```
```javascript
-const brain = new Brainy({ plugins: ['@soulcraft/cortex'] })
-await brain.init()
+const brain = new Brainy({ storage: { type: 'filesystem', path: './data' } })
+await brain.init() // @soulcraft/cor detected — same code, native engines underneath
```
-Plugins are opt-in. Brainy never auto-imports packages unless listed in `plugins`.
+Installing the package is the opt-in: if `@soulcraft/cor` is present, it loads and announces itself in the init log; if it's present but broken, `init()` **throws** — an installed accelerator never silently vanishes behind the JS engines. Opt out with `plugins: []`, or pin exactly what loads with `plugins: ['@soulcraft/cor']`. [`@soulcraft/cor`](https://www.npmjs.com/package/@soulcraft/cor) (Brainy 8.x ↔ Cor 3.x, version-matched) registers Rust implementations behind every provider seam: SIMD distance kernels, memory-mapped storage, a disk-native vector index that doesn't need your dataset in RAM, durable LSM field/graph indexes that serve cold opens instantly, and native aggregation. Recall@10 measured **0.99 / 0.96 / 0.96 at 1M / 10M / 100M vectors** in Cor's release gate.
-**[Plugin Documentation](docs/PLUGINS.md)**
+Open core, commercial accelerator: Brainy is MIT and complete on its own; Cor is licensed and funds both.
----
+## Performance
-## Type System
+- JS distance kernels: **~6× faster cosine, ~1.4× euclidean** than 7.x (measured: [`tests/benchmarks/distance-microbench.mjs`](tests/benchmarks/distance-microbench.mjs), 384-dim, median of 41).
+- Whole-graph reads are single **O(N + E)** cursor walks — a consumer-measured 19k-edge export dropped from ~27 s of per-node calls to one scan.
+- Full numbers and capacity planning: **[docs/PERFORMANCE.md](docs/PERFORMANCE.md)** · **[docs/SCALING.md](docs/SCALING.md)**
-42 noun types and 127 verb types form a universal knowledge protocol:
+## Use cases
-```
-42 Nouns × 127 Verbs = 5,334 base relationship combinations
-```
-
-Model any domain — healthcare (`Patient → diagnoses → Condition`), finance (`Account → transfers → Transaction`), education (`Student → completes → Course`), or your own.
-
-### Subtypes — sub-classification within a NounType *or* VerbType
-
-Both noun types and verb types are intentionally coarse. Use the top-level `subtype` field to sub-classify entities AND relationships within a type — flat string, no hierarchy, your choice of vocabulary:
-
-```javascript
-// Nouns: sub-classify entities
-await brain.add({
- data: 'Avery Brooks — runs the AI lab',
- type: NounType.Person,
- subtype: 'employee' // 'customer', 'vendor', 'contractor', …
-})
-
-// Verbs: sub-classify relationships
-await brain.relate({
- from: ceoId,
- to: vpId,
- type: VerbType.ReportsTo,
- subtype: 'direct' // 'dotted-line', 'matrix', …
-})
-
-// Filter on the fast path — column-store hit, not metadata fallback:
-const employees = await brain.find({ type: NounType.Person, subtype: 'employee' })
-const directReports = await brain.related({ from: ceoId, subtype: 'direct' })
-
-// O(1) counts via the persisted rollups:
-brain.counts.bySubtype(NounType.Person)
-// → { employee: 12, customer: 847, vendor: 34 }
-
-brain.counts.byRelationshipSubtype(VerbType.ReportsTo)
-// → { direct: 12, 'dotted-line': 3 }
-```
-
-**Enforce the pairing.** Register a vocabulary per type or turn on brain-wide strict mode to ensure every entity AND relationship has both `type` AND `subtype`:
-
-```javascript
-// Per-type rule with a closed vocabulary
-brain.requireSubtype(NounType.Person, { values: ['employee', 'customer'], required: true })
-
-// 8.0 default: every write requires a subtype. Exempt genuine catch-all types…
-const brain = new Brainy({ requireSubtype: { except: [NounType.Thing] } })
-
-// …or opt out while migrating pre-8.0 data, then audit and back-fill:
-const legacy = new Brainy({ requireSubtype: false })
-await legacy.audit() // gaps, grouped by type
-await legacy.fillSubtypes({ [NounType.Person]: 'unspecified' }) // close them
-```
-
-For other facets you want counted (`status`, `source`, `role`), register them with `brain.trackField(name)`. Renaming an existing convention to `subtype`? Use `brain.migrateField({from, to, entityKind: 'both'})` to walk nouns AND verbs in one pass. Full guide: **[Subtypes & Facets](docs/guides/subtypes-and-facets.md)**.
-
-**[Noun-Verb Taxonomy](docs/architecture/noun-verb-taxonomy.md)** | **[Stage 3 Canonical Reference](docs/STAGE3-CANONICAL-TAXONOMY.md)**
-
----
-
-## Storage: Memory and Filesystem
-
-The same API at every scale. Change one config line to go from prototype to production.
-
-### Development — Zero Config
-
-```javascript
-const brain = new Brainy()
-```
-
-### Production — Filesystem (gzip compression on by default)
-
-```javascript
-const brain = new Brainy({
- storage: { type: 'filesystem', path: './data' }
-})
-```
-
-### Backups and Portability — Snapshots
-
-```javascript
-const db = brain.now()
-await db.persist('/backups/2026-06-11') // instant hard-link snapshot
-await db.release()
-
-const snapshot = await Brainy.load('/backups/2026-06-11') // read-only Db
-const hits = await snapshot.find({ query: 'quarterly invoices' })
-await snapshot.release()
-```
-
-A snapshot directory is self-contained — copy it to another machine, open it with `Brainy.load()`, or restore it wholesale with `brain.restore(path, { confirm: true })`.
-
-Performance benchmarks and capacity planning in **[docs/PERFORMANCE.md](docs/PERFORMANCE.md)**.
-
-**[Capacity Planning](docs/operations/capacity-planning.md)**
-
----
-
-## Use Cases
-
-- **AI agents** — Persistent memory with semantic recall and relationship tracking
-- **Knowledge bases** — Auto-linking, semantic search, relationship-aware navigation
-- **Semantic search** — Find by meaning across codebases, documents, or media
-- **Enterprise knowledge** — CRM, product catalogs, institutional memory
-- **Interactive experiences** — Game worlds, NPCs, and characters that remember
-- **Content platforms** — Similarity-based discovery, intelligent tagging
-
----
+**AI agent memory** — persistent semantic recall with relationship tracking · **Knowledge bases** — auto-linking and meaning-aware navigation · **Semantic search** over codebases, documents, media · **Enterprise data** — CRM, catalogs, institutional memory · **Games & simulations** — worlds and characters that remember.
## Documentation
-### Start Here
-
-- **[Brainy explained simply](docs/eli5.md)** — Plain-language overview, no jargon, no code
-
-### Core
-
-- **[API Reference](docs/api/README.md)** — Every method with parameters, returns, and examples
-- **[Data Model](docs/DATA_MODEL.md)** — Entity structure, data vs metadata
-- **[Query Operators](docs/QUERY_OPERATORS.md)** — All BFO operators with examples
-- **[Find System](docs/FIND_SYSTEM.md)** — Natural language find() and hybrid search
-
-### Architecture
-
-- **[Architecture Overview](docs/architecture/overview.md)** — System design and components
-- **[Triple Intelligence](docs/architecture/triple-intelligence.md)** — Vector + graph + metadata unified query
-- **[Noun-Verb Taxonomy](docs/architecture/noun-verb-taxonomy.md)** — Universal type system
-- **[Data Storage Architecture](docs/architecture/data-storage-architecture.md)** — Type-aware indexing and HNSW
-
-### Virtual Filesystem
-
-- **[VFS Quick Start](docs/vfs/QUICK_START.md)** — Build file explorers that never crash
-- **[VFS Core](docs/vfs/VFS_CORE.md)** — Full VFS API reference
-- **[Semantic VFS](docs/vfs/SEMANTIC_VFS.md)** — AI-powered file navigation
-
-### Guides
-
-- **[Import Anything](docs/guides/import-anything.md)** — CSV, Excel, PDF, URLs
-- **[Framework Integration](docs/guides/framework-integration.md)** — React, Vue, Angular, Svelte
-- **[Natural Language Queries](docs/guides/natural-language.md)** — Master the find() method
-
-### Operations
-
-- **[Capacity Planning](docs/operations/capacity-planning.md)** — Memory, storage, and scaling
-- **[Performance](docs/PERFORMANCE.md)** — Benchmarks and architecture details
-
----
+| Start | Core | Going deeper |
+|---|---|---|
+| [Brainy explained simply](docs/eli5.md) | [API reference](docs/api/README.md) | [Architecture overview](docs/architecture/overview.md) |
+| [Installation](docs/guides/installation.md) | [Data model](docs/DATA_MODEL.md) | [Consistency model](docs/concepts/consistency-model.md) |
+| [Natural-language queries](docs/guides/natural-language.md) | [Query operators](docs/QUERY_OPERATORS.md) | [Multi-process model](docs/concepts/multi-process.md) |
+| | [Find system](docs/FIND_SYSTEM.md) | [Scaling](docs/SCALING.md) |
## Requirements
-**Bun 1.1+** (recommended) or **Node.js 22 LTS**
+**Bun ≥ 1.1** (recommended) or **Node.js ≥ 22**. Brainy 8.x is server-only; the 7.x line remains on npm for browser use.
-```bash
-bun add @soulcraft/brainy # Bun — best performance
-npm install @soulcraft/brainy # Node.js — fully supported
-```
+## Contributing & license
-> Brainy 8.0 is server-only. Browser support (OPFS storage, Web Workers, in-browser WASM embeddings) was removed in 8.0 — the 7.x line remains available on npm if you need it.
-
-## Single-Writer Model
-
-Brainy is **single-writer, many-reader** on filesystem storage. One writer
-holds an exclusive lock on the data directory; any number of readers can
-inspect it concurrently. Opening a second writer throws with the PID of the
-existing one.
-
-```typescript
-// Live application — writer mode is the default
-const brain = new Brainy({ storage: { type: 'filesystem', path: '/data/brain' } })
-await brain.init()
-
-// Out-of-band diagnostics from a separate process — safe to run while the
-// writer is live
-const reader = await Brainy.openReadOnly({
- storage: { type: 'filesystem', path: '/data/brain' }
-})
-await reader.requestFlush({ timeoutMs: 5000 })
-const stats = await reader.stats()
-```
-
-For incident debugging, use the `brainy inspect` CLI:
-
-```bash
-brainy inspect stats /data/brain
-brainy inspect find /data/brain --where '{"entityType":"booking"}'
-brainy inspect explain /data/brain --where '{"entityType":"booking"}'
-brainy inspect health /data/brain
-```
-
-See [the multi-process model](docs/concepts/multi-process.md) and the
-[inspection guide](docs/guides/inspection.md) for the full story, including
-stale-lock detection and the cross-process flush RPC.
-
-## Contributing
-
-We welcome contributions! See **[CONTRIBUTING.md](CONTRIBUTING.md)** for guidelines.
-
-## License
-
-MIT © Brainy Contributors
+Contributions welcome — see **[CONTRIBUTING.md](CONTRIBUTING.md)**. MIT © Brainy Contributors.
diff --git a/RELEASES.md b/RELEASES.md
index bb95511b..7799c6f6 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -8,9 +8,1021 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
- Debugging data, query, or storage behaviour
- A new Brainy feature is available that you want to adopt
+## Removed APIs — 7.x → 8.x (the complete ledger)
+
+Every public API removed at the 8.0 major, with its sanctioned replacement. If your code
+still calls a left-column name on 8.x it throws (or the config key is rejected) — the
+replacement is always a one-line change. (Standing contract from 8.9.0 forward: removals
+happen only at majors, after ≥1 minor of loud runtime deprecation naming the replacement.)
+
+| Removed (7.x) | Replacement (8.x) |
+|---|---|
+| `brain.search(query, k)` | `find({ query })` — semantic; `find({ query, searchMode })` for hybrid |
+| `brain.getRelations({...})` | `related(id, opts)` for adjacency; `find({ connected: {...} })` for scoped traversal |
+| `brain.neural()` clustering | `find({ vector })` + aggregation `GROUP BY` |
+| `Db.search()` | `db.find({ vector })` |
+| Pre-8.0 storage path aliases (`directory`, `basePath`, …) | one `storage.path` key (old aliases throw) |
+| Reserved keys inside `metadata` bags (silently remapped in 7.x) | top-level params (`subtype`, `visibility`, `confidence`, `weight`, …) — reserved-in-bag throws |
+| 7.x COW branches layout (`branches/main/`) | generational MVCC (`asOf()`, `now()`, `db.persist(path)`) — on-disk migration is automatic at first 8.x open |
+
+The fork/snapshot family (`brain.snapshot()`, `createSnapshot()`, `restoreSnapshot()`)
+is sometimes cited as a 7.x removal — those methods never existed on 7.x; the 8.0 Db API
+(`asOf`/`persist`/`restore({confirm})`) is their first real implementation.
+
---
-## v8.0.0 — release candidate (`8.0.0-rc.9` published 2026-07-01 on npm tag `rc`)
+## v8.9.0 — 2026-07-19 (flush is durability-only: history maintenance moves to close())
+
+The write path stops paying maintenance costs — the last structural piece of the
+flush-storm class (a production deployment measured single writes blocked 25–191s behind
+history reclaim running inline on flush under memory pressure):
+
+- **`flush()` never compacts history.** It persists the current window's deltas and
+ nothing else — its cost no longer depends on history backlog or retention mode, in any
+ configuration. **`close()` is the auto-compaction site** (time-bounded per pass, ~5s;
+ an early stop is a consistent prefix and the next pass resumes).
+- **`compactHistory()` gains `timeBudgetMs`** — bound your own maintenance windows; the
+ same resumable-prefix guarantee applies.
+- **The documented trade**: a long-lived writer that never closes accumulates history
+ until its next explicit `compactHistory()`. Predictable writes, explicit maintenance.
+ If you run bounded retention on an always-on service, schedule a periodic
+ `compactHistory({ ...caps, timeBudgetMs })` in your maintenance window.
+- **New public doc: `docs/performance-envelopes.md`** — measured per-op envelopes
+ (p50/p95 at stated scales, hardware, and backend, with the measuring script cited).
+ Refresh rule going forward: any release touching a measured path re-runs that op's
+ benchmark and updates the envelope in the same release.
+- **New in this file: the Removed APIs 7.x→8.x table** (top of this document) — every
+ removal with its sanctioned replacement, one place, per the engine-currency contract.
+ Standing from here: removals only at majors, after ≥1 minor of loud runtime deprecation.
+
+## v8.8.2 — 2026-07-19 (one field-resolution law: reserved-field aggregates stop drifting)
+
+Four fixes from a consumer conformance audit, all rooted in the same disease — two field-resolution
+regimes where there must be one:
+
+- **Aggregates grouped by a RESERVED field (`subtype`, `visibility`, …) now decrement on
+ delete.** The delete/update hooks fed the aggregation engine a partial entity view (type,
+ service, data, metadata only), so a reserved-field `groupBy` resolved to a nonexistent group
+ on the way DOWN — counts drifted upward forever after any delete, and updates that moved an
+ entity between reserved-field groups double-counted it. The hooks now pass the full-fidelity
+ entity view (every reserved field top-level, the same shape the add path uses). If your
+ deployment derives stats from reserved-field aggregates, re-define those aggregates once
+ after upgrading (a changed definition triggers one rescan) or run them fresh — the drifted
+ persisted counts do not self-heal retroactively.
+- **Aggregation `source.where` on reserved fields now filters** instead of silently matching
+ nothing: the matcher resolves fields through the same resolver `groupBy` uses (top-level
+ standard fields + custom metadata), so `where: { subtype: 'note' }` means what it says.
+- **`removeMany()` refuses empty/invalid selectors loudly.** A bare array passed positionally
+ (`removeMany([id])` instead of `removeMany({ ids: [id] })`), an empty params object, or
+ `ids: []` used to resolve successfully having deleted nothing. All three now throw.
+- **`find()` accepts both where-key spellings.** Metadata is flattened at index time
+ (`metadata.entry.title` indexes as `entry.title`); a `metadata.`-prefixed where key now
+ falls back to its flattened spelling when the prefixed one isn't indexed — the
+ "unindexed field(s), returning []" confusion for storage-shaped spellings is gone. (A
+ literal nested custom key named `metadata` still wins when indexed as spelled.)
+
+## v8.8.1 — 2026-07-18 (flush no longer walks the whole generation history + the import dedup off-switch is now honest)
+
+### The flush-storm fix (production incident, reported by a long-running deployment)
+
+Under the default adaptive retention, **every `flush()` re-walked the entire committed
+generation history** to compute total history bytes for the budget check — O(all
+generations) with disk re-reads past the 4,096-entry delta-cache bound. On a brain with
+70,000+ accumulated generations that turned every write into a full-tail scan (60-100s
+writes), even though the budget (free-RAM-based) never tripped and nothing was ever
+reclaimed. Fixed:
+
+- `historyBytes()` now maintains a **running total**: seeded by one walk on first use,
+ then updated incrementally at every commit and reclaim — the adaptive retention check
+ on every flush is O(1). Invariant regression-pinned (running total ≡ fresh walk through
+ both commit paths and compaction).
+- New **`brain.historyStats()`** (read-only, exported `HistoryStats`): generation count,
+ total on-disk bytes, generation/timestamp range, compaction horizon, retention mode,
+ and the effective adaptive budget — the one-call fleet-audit for sizing retention
+ exposure per brain.
+- Interim guidance for keep-everything deployments already affected: `retention: 'all'`
+ skips the adaptive accounting entirely (and is the correct policy if you never want
+ history reclaimed). The accumulated files are harmless at rest; this release removes
+ the per-write cost of their existence.
+
+### The import dedup off-switch (lifecycle honesty)
+
+The post-import background deduplication pass (a merge-DELETE writer that runs ~5 minutes
+after an import, merging entities judged duplicates by id / name / vector similarity) had
+three lifecycle defects, all fixed:
+
+- **`enableDeduplication: false` now actually disables it.** The background pass was
+ scheduled unconditionally — an import that explicitly opted out could still have
+ entities auto-removed 5 minutes later. The flag now gates BOTH the inline merge and
+ the background pass (regression-pinned).
+- **One deduplicator per brain, owned by the brain.** Each `import()` call constructed its
+ own coordinator + deduplicator, so the "debounced" timer never actually debounced across
+ imports (N imports = N delete timers). The brain now owns a single instance — the
+ debounce genuinely spans imports — and `close()` cancels pending work, so a delete pass
+ can never fire against a closed brain.
+- **The 5-minute timer is unref'd** — a pending pass no longer holds the process open
+ (the exit-hang class; this timer had escaped the earlier sweep).
+
+Retention note for keep-everything deployments: with `enableDeduplication: false` on
+import calls and `retention: 'all'` in config, no engine path removes records
+automatically.
+
+## v8.8.0 — 2026-07-17 (OS-limit detection for pool-scale deployments)
+
+Small minor: brains now detect the two OS limits that bite at pool scale and warn **before**
+the incident instead of during it.
+
+- At open (once per process, Linux-only, measurement-only), Brainy reads `RLIMIT_NOFILE`
+ (soft/hard, from `/proc/self/limits`) and `vm.max_map_count`, and warns loudly when either
+ sits below the pool-scale floors (soft NOFILE < 65 536; max_map_count < 262 144) — with the
+ exact raise commands (`ulimit -n` / `LimitNOFILE=` / `sysctl vm.max_map_count`). On stock
+ defaults the failure otherwise arrives as `EMFILE` or a failed mmap deep inside an index
+ open, long after the real cause stopped being visible. An unreadable limit produces **no**
+ warning — no measurement, no claim (non-Linux platforms stay silent).
+- Exported for ops doors: `checkOsLimits()` returns the full `OsLimitsReport`
+ (values + warnings) programmatically, with the floors exported as constants.
+
+## v8.7.1 — 2026-07-17 (writer-lock acquisition is race-proof + machine-readable through init)
+
+Two hardenings of the multi-process writer lock (the `locks/_writer.lock` lease that makes a
+second writer on the same brain directory fail loudly):
+
+- **Lock acquisition claims atomically.** The acquire path used read-then-write, leaving a
+ window where two processes racing an *absent* lock could both "succeed" — and the loser
+ kept running unlocked, silently. The claim is now an atomic create-exclusive write
+ (`O_EXCL`): exactly one racer wins; the loser re-evaluates and either fails loudly with
+ the winner's details or performs a verified stale-takeover. Bounded retries; contention
+ beyond them fails loudly rather than degrading into a lockless open.
+- **`BRAINY_WRITER_LOCKED` survives `init()`.** The conflict error documents a
+ machine-readable contract (`err.code`, `err.lockInfo` with the holder's pid/host/
+ heartbeat), but init's error wrapping silently stripped both, leaving consumers a message
+ to regex against. The error now passes through unwrapped.
+
+Measured while verifying (for operators sizing audits): `brain.auditGraph()` at a
+production-consumer scale of ~2,600 relationships / 800 entities costs ~0.1 s warm and
+~0.5 s cold, with exact scar counting across reopen.
+
+## v8.7.0 — 2026-07-17 (bulk-transact ergonomics: scaled budgets + timeout telemetry)
+
+The bulk-import ergonomics release, from a consumer's measured production incident (a serial
+import on network-attached storage at ~2 s/op met a flat 30 s transaction budget):
+
+- **The transact apply budget now scales with the batch** — `max(30 s, opCount × 2 s)` — or
+ is exactly what you pass as the new `TransactOptions.timeoutMs`. A flat 30 s cap silently
+ limited honest bulk work to ~15 operations on slow disks while looking generous for small
+ batches. Internal batch paths (e.g. `removeMany` chunks) get the same scaling.
+- **`TransactionTimeoutError` is a diagnosis, not just a failure**: it now reports the
+ operation it stopped at as `i/N` with the operation's name, elapsed vs budgeted time, and
+ states the batch rolled back atomically and is retryable. Its `context` carries the same
+ fields programmatically.
+- **The transact envelope is documented** — batch sizing, budget math, chunking with
+ `ifAbsent` idempotency, and the precompute pattern (`embedBatch` + per-op `vector`) that
+ keeps model inference out of the commit path. Guide: `docs/guides/optimistic-concurrency.md`.
+- Note: `brain.embed()` / `brain.embedBatch()` (the precompute APIs) already ship — public,
+ with native-provider passthrough, verified end-to-end (batch and single paths produce
+ bit-identical vectors; a vector-supplied `add` is fully searchable). Honest measurement:
+ on the default WASM engine, batch throughput ≈ sequential (~160 ms/text) — the precompute
+ win is keeping inference out of the budgeted commit path, not raw embedding speed.
+
+## v8.6.0 — 2026-07-17 (brain.auditGraph — the graph-truth verification instrument)
+
+A minor release adding one new public API, from the fleet's graph-trust program: a read-only
+audit that **proves whether relationship reads return stored truth** on a given brain.
+
+- **`brain.auditGraph(options?)`** walks every canonical relationship record, queries the same
+ read path applications use (`related()` / VFS `readdir`) with all visibility tiers included,
+ and classifies every discrepancy into its failure family: `missingFromReads` (records the
+ read path omits — a stale adjacency index), `danglingEndpoints` (relationships whose endpoint
+ entity no longer exists — the historical partial-delete scar class), and `readOnlyVerbIds`
+ (read-path edges with no stored record — ghosts). Design-hidden internal/system edges are
+ counted separately so intentional hiding is never misclassified as loss. Counts are exact;
+ example lists cap at `maxExamples` with an explicit `truncatedExamples` flag; the result is
+ narrated loudly on incoherence. Mutates nothing — safe on a live brain.
+- The operational pairing: audit → if incoherent, `repairIndex()` → audit again. A `coherent`
+ report after the repair is the verified all-clear. Run it after any engine upgrade, restore,
+ or migration. Guide: `docs/guides/inspection.md`.
+- Also: `getNounIds` pagination now refuses an undecodable resume cursor loudly (the same
+ contract `getNouns`/`getVerbs` gained in 8.5.2 — the third and final walk brought under it).
+- Types exported: `GraphAuditReport`, `GraphAuditDiscrepancy`.
+
+## v8.5.2 — 2026-07-17 (aggregation backfill: exception-safe, generation-verified, and loud)
+
+Hardening patch from a migration incident (a byte-copied store on new hardware; the service
+entered a silent full-CPU loop at boot). Four changes, all in the aggregation engine's
+backfill/adoption path:
+
+- **Backfill walks are exception-safe and non-destructive.** A rescan now builds into a
+ staging map and swaps in atomically on completion; a mid-walk failure drops the staging map,
+ keeps the previous live state serving, and surfaces the storage error to the failing query.
+ Previously the walk wiped live state *before* a scan that could throw, never cleared the
+ pending flag on failure, and re-ran a full walk on every subsequent query — a silent
+ wipe/walk/throw loop at the caller's retry rate.
+- **Failed walks are latched.** After a walk fails, retries within a 30-second cooldown rethrow
+ the recorded error instantly instead of re-walking — a tight caller-side retry loop now costs
+ one loud error per query, never a full store walk per query.
+- **Adoption is generation-verified.** Persisted aggregation state is stamped with the store's
+ committed generation at flush; reopen adoption requires the stamp to equal the current
+ watermark. Stale state (unclean shutdown) or over-counting state (a fact-log truncation on a
+ copied store pulled the watermark back) triggers exactly one loud rescan — never a silent
+ adopt. Pre-8.5.2 state on generation-aware stores rescans once after upgrade, then is stamped.
+- **The path narrates.** Adoption decisions, no-adoptable-state outcomes, walk start/finish
+ (entity count + duration), and walk failures all log by default; a non-advancing storage
+ pagination cursor aborts the walk loudly instead of looping forever.
+
+Plus three guards from a full audit of every loop in the open/init path:
+
+- **Invalid pagination cursors fail loudly.** A supplied-but-undecodable resume token to
+ `getNouns`/`getVerbs` used to silently restart the walk at offset 0 — to a `while(hasMore)`
+ caller that re-serves page 1 forever (an unbounded silent CPU loop). It now throws with a
+ clear message instead.
+- **The graph cold-load verb walk has a stall guard.** `hasMore=true` with a missing or
+ non-advancing cursor aborts loudly instead of re-reading the same page forever.
+- **A derived index AHEAD of the store is named at open.** Brainy already surfaced a provider
+ generation *behind* the committed watermark; the *ahead* direction (the signature of a
+ byte-copy of a live service, or a log truncation during crash recovery) now logs a loud
+ warning explaining what happened and that `brain.repairIndex()` forces a heal — instead of
+ passing unnamed into whatever the derived index does next.
+
+## v8.5.1 — 2026-07-17 (aggregation state survives restarts + the query-cap ratchet removed)
+
+Patch release from a production incident (aggregate/count paths taking 40–90 s on an idle box
+while vector search stayed fast, and every `find({ limit: 5000 })` suddenly failing against an
+"auto-configured query limit of 1000"). Three fixes, one cosmetic:
+
+- **Aggregation state is actually adopted on reopen.** The boot pattern `defineAggregate()` →
+ query raced the engine's async state load: the synchronous define always won, flagged a
+ backfill, and the first query then wiped the just-loaded persisted state and re-walked the
+ entire store — every restart, forever. Reopening with an unchanged definition now adopts the
+ persisted state directly (zero scans); a backfill runs only on a real definition change, a
+ missing/failed state load, or a write that landed before adoption (exactness wins). Apps that
+ rely on persisted definitions without re-defining at boot also no longer race a spurious
+ "Aggregate not defined".
+- **Backfills are single-flight and batched.** Concurrent queries on a cold aggregate used to
+ each wipe the others' partial state and start their own full store walk — under steady query
+ arrival the store never converged (the 40–90 s loop). Now all concurrent queries share one
+ walk, and one walk fills every aggregate pending backfill (M aggregates ≠ M scans).
+- **The query-cap "learning" ratchet is removed.** `maxLimit` is a memory-protection bound, but
+ a hidden tuner shrank it 20 % per recorded query while the lifetime-average query time
+ exceeded 1 s — down to a floor of 1000, below the documented 10 000 auto floor, with no
+ practical recovery, and the resulting error blamed "available free memory" (stale basis
+ label). The cap now comes from its construction-time basis (or your explicit
+ `maxQueryLimit` / `reservedQueryMemory`) alone and never changes at runtime; query timing is
+ recorded for diagnostics only.
+- **Cosmetic:** the engine's own persistence keys (`__aggregation_*`, `brainy:entityIdMapper`)
+ no longer log `[Storage] Unknown key format` at boot — they were always routed correctly;
+ they're now recognized before the warning fires.
+
+Operationally: if a host was bitten, upgrading and restarting is the whole fix — no repair
+ritual needed. Setting `maxQueryLimit` explicitly remains the valve that bypasses auto-detection
+entirely.
+
+## v8.5.0 — 2026-07-15 (provider access to the fact log + the shared stamp verifier)
+
+Small additive follow-up to 8.4.0, from the native accelerator's first consumption pass:
+
+- **Index providers can now reach the fact log through the storage adapter** — new optional
+ capability `storage.scanFacts()` / `storage.factLogHeadGeneration()` / `storage.factSegmentPaths()`,
+ wired by the host brain at init as a closure over its live log. Providers hold only `storage` and
+ must never construct their own fact-log reader (the log's open path is writer-side); `null` means
+ "no fact log here — use the enumeration walk."
+- **The family-stamp verifier is shared** via `@soulcraft/brainy/internals`
+ (`readFamilyStamp` / `writeFamilyStamp` / `verifyFamilyStamp` + types) so first-party native
+ providers run literally the same verification function, never a synchronized copy.
+- **Rollup stamp invariants accept strings** (`number | string`) — content fingerprints such as a
+ per-tree SHA-256 are valid invariant values; a type mismatch reads as incoherence, never a pass.
+- **`storage.committedGeneration()`** — the committed watermark as a capability, so a provider
+ compares its stamp's `sourceGeneration` against the store's truth without parsing the private
+ manifest format.
+- **Two durability/stability contracts pinned in the suite:** fsync-before-ack (holds for
+ `transact()` today; the single-op path is pinned as the documented future target — group commit
+ becomes latency batching, never durability skipping) and scan-stability-under-rotation (a scan
+ snapshot yields exactly its facts — no gaps, duplicates, or bleed-in — while segments rotate
+ beneath it).
+
+No behavior change for applications; all additions.
+
+## v8.4.0 — 2026-07-15 (the generation fact log — a sequential, self-verifying commit stream)
+
+A minor release, fully backward-compatible (all additions; no behavior change for existing APIs).
+This is infrastructure: it changes nothing about how you query today, and lays the substrate that
+makes index heals and incremental catch-up sequential-read problems instead of directory walks.
+
+- **Every committed write now also appends a "fact" — an after-image commit record.** Alongside the
+ existing before-image history, each committed generation (single-op and `transact()` alike) appends
+ what each touched entity/relationship *became* — or a body-less tombstone for a removal — to an
+ append-only, checksummed segment log under `_generations/facts/`. Crash-safe by construction: a
+ torn tail is detected and ignored; on open the log is reconciled to committed truth, so an absent
+ generation always means "never committed." `transact()` facts are durable when `transact()`
+ returns; single-op facts ride the same group-commit flush as their history.
+
+- **New: `brain.scanFacts()`** — stream committed facts in commit order, in batches, with heal-grade
+ telemetry (total scope up front; per-batch generation range, byte size, and segment; a summary
+ cross-check at the end; loud abort on any gap — never a silent skip). **`brain.factSegmentPaths()`**
+ hands zero-copy consumers the immutable sealed segment files directly. New exported types:
+ `CommitFact`, `FactOp`, `FactScanBatch`, `FactScanHandle`. Facts accumulate from the first write
+ after upgrading — pre-existing history is not retroactively converted (enumeration remains the
+ fallback for old data).
+
+- **New: the entity-tree family stamp.** At every flush/close, brainy stamps which committed
+ generation the canonical entity files reflect plus the rollup invariants (entity/relationship
+ counts) that verify the tree whole. At open, coherence is a comparison — a genuine divergence is
+ loud and names the failing invariant; `repairIndex()` recounts from canonical and re-stamps. New
+ exports: `readFamilyStamp`, `verifyFamilyStamp`, `ENTITY_TREE_STAMP_PATH`, `FamilyStamp` types.
+
+- **Storage adapters** gain optional binary raw-byte primitives (`appendRawBytes`, `readRawBytes`,
+ `writeRawBytes`, `rawByteSize`) — feature-detected; the filesystem and memory adapters implement
+ them; an adapter without them simply hosts no fact log. The fact-log namespace is registered as a
+ protected family: no sweeper or GC can delete under it.
+
+No API breaks. 24 new tests; the full commit-path regression suite is green.
+
+## v8.3.3 — 2026-07-15 (rename moves the containment edge — no ghost in the old directory)
+
+One production-reported fix plus a repair path, completing the delete/move hygiene arc (8.3.1 fixed
+deletes, 8.3.2 fixed counters, this fixes moves).
+
+- **A cross-directory `vfs.rename()` now MOVES the containment edge instead of accumulating one per
+ parent.** The old parent's `Contains` edge was never removed on a move, leaving the entity a child
+ of **both** directories: `readdir(oldDir)` kept listing it after the move, re-creating the old path
+ showed the same name twice, and any tree-walking consumer (sync engines, file browsers) saw the
+ file in two places. The old edge is now removed by edge id, resolved from the graph's own adjacency
+ — a removal never requires reading the thing being removed. Bonus fix in the same seam: a move **to
+ the root** now gets its containment edge (it was previously skipped, orphaning the file out of
+ `readdir('/')`).
+
+- **`repairIndex()` now also reconciles VFS containment** (new `vfs.repairContainment()`): every VFS
+ entity's containment edges are checked against its canonical `metadata.path` — stale old-parent
+ ghosts and duplicate edges are removed, a missing expected edge is restored, and user
+ knowledge-graph edges are never touched (only `vfs-contains` edges are candidates). Loud per
+ repair. Stores that performed cross-directory renames under ≤8.3.2 should run `brain.repairIndex()`
+ once after upgrading — the same single ritual now heals orphan directories, counters, **and**
+ containment edges.
+
+- Also ships a permanent lens-consistency regression suite (combined type+subtype vs subtype-only vs
+ canonical ground truth, id-for-id, warm and after a cold reopen), ported from the field
+ investigation that closed the historical lens-drop report.
+
+No API changes beyond the new optional `vfs.repairContainment()` (also invoked by `repairIndex()`).
+
+## v8.3.2 — 2026-07-14 (honest counters — the recount + removal-without-re-reading)
+
+Completes 8.3.1's delete-hygiene story at the counter layer, from a production proof chain reported
+by a downstream deployment: persisted entity totals were permanently **inflated** — deletes whose
+count decrement was silently skipped — and because paginated `totalCount` serves
+`Math.max(persistedTotal, scanned)`, the inflated number always won and **no disk cleanup could ever
+lower it**.
+
+- **A removal's count decrement no longer requires re-reading the record being removed.** The
+ decrement was sourced from re-reading the entity's metadata inside the delete; if that read
+ returned `null` (a replace race, or a ghost left by a pre-8.3.1 partial delete) the decrement was
+ silently skipped while the paired add had counted — minting drift on every write→delete→re-create
+ cycle. The caller's pre-delete read now rides through the whole delete path
+ (`remove()`/`removeMany()` → the delete operation → `deleteNoun`/`deleteVerb` →
+ `deleteNounMetadata`/`deleteVerbMetadata`, both sides symmetric): a null internal read falls back
+ to the known prior record instead of skipping. The `StorageAdapter` signatures gain an optional
+ `priorMetadata` parameter (additive; existing adapters unaffected).
+
+- **`repairIndex()` is the sanctioned counter recount — unconditional, and it actually persists.**
+ `rebuildTypeCounts()` previously rebuilt only the type-statistics arrays and computed the total
+ *just to log it* — the persisted scalar (`counts.json`) survived every "rebuild" untouched, so an
+ already-inflated brain could never be corrected. One canonical walk now rebuilds **every** counter
+ rollup — scalar totals, per-type maps, and type statistics — and persists them, and `repairIndex()`
+ runs it unconditionally (not only when orphan directories are found: counters can be inflated over
+ perfectly clean shelves). Brains with delete history should run `brain.repairIndex()` once after
+ upgrading; the correction survives reopen.
+
+No API breaks (optional-parameter additions only). Regression tests cover the drift cycle, the
+null-read decrement fallback, and the persisted recount across reopen.
+
+## v8.3.1 — 2026-07-14 (full-removal deletes + family-scoped migration gate)
+
+Two production-reported fixes in the write/index spine, plus an operator repair path. No API changes;
+all behavior changes make previously-wrong states honest.
+
+- **Deleting an entity now removes it completely — no more "ghost" leftovers on disk.** A canonical
+ noun delete removed the metadata (content) leg but left the entity's `vectors.json` and its `/`
+ directory behind. Consequences observed in a production deployment: deleted rows were
+ indistinguishable on disk from damage scars, enumerated counts inflated monotonically with every
+ delete (the leftovers were counted forever), and locator-style reads hit unreadable ghost rows.
+ `remove()`/`removeMany()`/`deleteNoun`/`deleteVerb` now remove **both legs and the entity
+ container**, with a full two-leg before-image rollback inside the transaction. The generation log
+ still holds the delete's before-image, so `asOf()` time-travel reconstructs deleted entities exactly
+ as before — this is live-HEAD hygiene, not a history change. This also fixes **duplicate `readdir`
+ entries for re-created VFS paths** at the root: with no ghost state, a delete always unposts its
+ index rows, so a delete→recreate cycle lists the path exactly once (regression-tested across
+ repeated cycles).
+
+- **`repairIndex()` prunes ghost/scar directories left by earlier versions.** Stores that deleted
+ entities under ≤8.3.0 may hold orphaned entity directories (a vector-only leg, or an empty dir).
+ `brain.repairIndex()` now sweeps them: it removes only containers with **no metadata content leg**
+ (never a directory that still holds content), logs every removal, and recomputes type/subtype
+ counts afterward so totals stop counting ghosts.
+
+- **Reads no longer hang behind an unrelated index migration (family-scoped gate).** During a native
+ provider's one-time background migration, *every* read — including plain `get()`, VFS
+ `readdir`/`readFile`, and metadata-only `find({ where })` — blocked on the whole-brain migration
+ lock until timeout, even when the migrating index was irrelevant to the read. The gate is now
+ scoped to the index families a read actually consults: canonical reads (`get`, `batchGet`, VFS
+ content) never wait; a `find` waits only on the families its query shape needs (vector for
+ semantic, metadata for `where`/type, graph for `connected`); graph traversals wait only on the
+ graph family. Writes and unclassified operations keep the conservative whole-brain wait. A read
+ that *does* need the migrating family still blocks (bounded by `migrationWaitTimeoutMs`) and
+ surfaces the retryable `MigrationInProgressError` — never a partial result.
+
+No breaking API change. Each fix ships with regression tests.
+
+## v8.3.0 — 2026-07-13 (faster index heals + the cross-layer integrity contract)
+
+Three additive changes. The first is an immediate, standalone performance win; the other two are the
+brainy side of the write/index-spine integrity contract, inert until a native accelerator
+that implements the matching hooks is present — so this release changes nothing for a JS-only brain
+beyond the speedup.
+
+- **Canonical enumeration is up to ~16× faster — the dominant term in an index heal.** The paginated
+ entity walk (`getNounsWithPagination`) hydrated each entity's vector + metadata one-at-a-time; since
+ every index rebuild enumerates canonical storage, that serial per-item latency dominated multi-minute
+ heals. Hydration is now 16-way bounded-concurrency, with the pagination contract (order, cursor
+ resume, filters, totalCount) byte-identical to before. New **`getNounIdsWithPagination()`** returns
+ ids without hydrating anything (zero per-entity reads when unfiltered) for callers that own their own
+ IO schedule.
+
+- **Cross-layer integrity — `validateIndexConsistency()` is no longer blind to native providers.** It
+ only ran the JS metadata index's own check, so a native provider whose manifest/segments/counts had
+ diverged still read as "healthy". It now feature-detects and aggregates each provider's optional
+ `validateInvariants()` self-report, names every failing invariant with its numbers, and exposes the
+ per-provider reports. `repairIndex()` now reconciles native derived state from canonical too
+ (rebuilding any provider whose failing invariant asks for it). New exported types
+ `ProviderInvariantReport` / `InvariantResult` / `InvariantHeal`.
+
+- **Registered-blob families — declared index files are undeletable through the storage layer.** A
+ provider can declare a derived-index blob *family* (a set of members that are load-bearing together);
+ once declared, `deleteBinaryBlob` / `removeRawPrefix` refuse to remove a member (new exported
+ `ProtectedArtifactError`), so a stray in-process sweeper cannot delete a load-bearing index file. The
+ declaration persists across reopen; `checkDerivedFamiliesPresent()` names any member missing on open.
+ New optional `StorageAdapter` surface (`registerDerivedFamily` / `unregisterDerivedFamily` /
+ `listDerivedFamilies` + `DerivedFamilyDeclaration`) and exported `DerivedArtifactMissingError`.
+
+No breaking API change (all additions are optional/new). Each change ships with regression tests.
+
+## v8.2.8 — 2026-07-13 (honest index readiness — no more silently-empty queries on a cold index)
+
+Closes the last of the three spine anti-patterns: the "dishonest readiness proxy," where `size() > 0`
+was treated as "this index actually serves queries." On a cold open (fresh boot, restart, crash
+recovery) a native index can load its **count** before its **serving structure** — so for a brief
+window it has data but cannot answer, and a query returned a silent empty result indistinguishable
+from "no such data." Every fix here asks the index whether it can *actually serve*, and if not,
+self-heals or fails loudly instead of returning `[]`.
+
+- **Semantic search no longer returns a silent `[]` on a cold vector index.** A pure semantic
+ `find({ query })` has no filter, so nothing previously guarded the vector index. A new one-shot
+ guard verifies the vector index serves a known persisted vector on the first semantic/proximity
+ search — preferring the provider's honest `isReady()` signal, else a known-vector self-match probe.
+ It rebuilds from canonical records if the serving structure did not load, and throws the new
+ **`VectorIndexNotReadyError`** only if a rebuild still cannot serve — never a silent empty result.
+
+- **Relationship reads fall back to the canonical scan instead of an empty result on a cold graph
+ index.** `getVerbsBySource`/`getVerbsByTarget` (used by relationship queries and virtual-filesystem
+ traversal) skipped the fast path only on `isInitialized` — which reads true once the manifest loaded
+ even if the source→target adjacency did not. They now consult the honest readiness signal and, when
+ the adjacency is not serving, take the correct-but-slower canonical shard scan. A one-shot self-heal
+ probe covers providers that expose no readiness signal.
+
+- **`getIndexStatus()` tells the truth for readiness probes.** It reported `populated: size>0` only, so
+ a Kubernetes readiness check could route traffic to a brain still warming up. It now folds in the
+ honest per-index `ready` signal (making `populated` honest), plus the degraded states already
+ surfaced by `checkHealth()`/`validateIndexConsistency()` (`rebuildFailed`/`rebuildError` and a
+ `degradedIds` count) — so a probe never reports 200-ready over a known-degraded index.
+
+This completes the write/index-spine hardening end to end. New export: **`VectorIndexNotReadyError`**;
+`getIndexStatus()` gains additive fields (`rebuildFailed`, `rebuildError?`, `degradedIds`, per-index
+`ready?`). No breaking API change. Each fix ships with a dedicated regression test.
+
+## v8.2.7 — 2026-07-13 (loadBinaryBlob fault-propagation — the lockstep completion of 8.2.6)
+
+Completes the Pass-1 spine hardening. `loadBinaryBlob` (the raw-blob read a native accelerator mmaps
+for its index files) previously returned `null` on ANY read error, so a real IO fault
+(EIO/EACCES/EMFILE) on a present-but-unreadable index blob masqueraded as "the blob is absent" —
+driving a needless full rebuild or an empty read. It now distinguishes genuine absence (ENOENT →
+`null`, the documented contract) from a real fault (→ throw), so a transient disk fault surfaces
+loudly instead of silently degrading the index.
+
+This one change was deliberately held out of 8.2.6 and ships now, in lockstep with the native
+accelerator release that hardened its two column-store read sites to handle the throw (a faulted
+segment read marks the field unavailable and throws a named error, instead of relying on
+null-on-error). A consumer on new brainy + an older accelerator was never at risk: 8.2.6 kept the
+prior swallow-on-fault behavior for this method until the accelerator was ready.
+
+No API change. Regression: `tests/unit/storage/blob-save-durability.test.ts` gains the loadBinaryBlob
+leg (absent → null; present → bytes; real fault → throws).
+
+## v8.2.6 — 2026-07-13 (write/index-spine hardening — loud errors, never quiet losses)
+
+Durability + integrity hardening across the write and index paths. Every fix converts a place that
+could *silently* lose data, serve a partial result, or acknowledge a write that did not land into a
+loud, observable failure. No breaking API changes; one new exported error.
+
+- **A durable blob write that stored nothing is no longer acknowledged.** The atomic blob write
+ (`saveBinaryBlob`, used for vector/graph index segments and the native index files) uses a unique
+ per-writer temp file, so a rename `ENOENT` can only mean *our* temp vanished before the rename —
+ the bytes never landed. It previously returned success on that path; it now retries once with a
+ fresh temp and, if the temp vanishes again, throws instead of acknowledging a write that persisted
+ nothing. A downstream deployment that mmaps these files no longer finds a "successfully written"
+ blob missing on the next open.
+
+- **Single-op history durability is enforced, not assumed.** The asynchronous group-commit flush
+ that persists single-op generation history previously swallowed a persist failure as a warning
+ while writes kept succeeding and their history piled up, undurable, in memory. It now tolerates a
+ transient blip (retry with capped backoff) and, after repeated failures, **refuses further writes**
+ with the new **`PendingFlushDurabilityError`** rather than promise a durability it cannot deliver.
+ Live canonical data is untouched; the latch self-heals the moment a flush succeeds. Callers needing
+ hard per-write durability should keep using `transact()` (or `flush()` after a single-op).
+
+- **A degraded derived index is surfaced on reads, not served silently.** When a non-fatal index
+ rebuild fails at open, or a write commits via adopt-forward recovery with an incomplete derived
+ index, `find()` and `get()` now emit one loud warning per degraded window (reads still return —
+ canonical is the source of truth), the state folds into `checkHealth()` /
+ `validateIndexConsistency()`, and `repairIndex()` reconciles and clears it.
+
+- **`clear()` removes the full derived footprint.** It previously left raw index blobs, the native
+ id-mapper, and column-index manifests on disk, so a cleared brain could re-read stale native state
+ on reopen. It now wipes them together as a set.
+
+- **Counts stay honest across deletes.** A delete decremented the per-type breakdown but not the
+ scalar total, so the total inflated permanently (and won pagination). Delete now decrements both;
+ the invariant `total === Σ per-type` holds across any interleaving of add / visibility-flip /
+ delete and across a reopen.
+
+- **Index-maintenance and aggregation failures are loud.** A partial LSM/segment load no longer
+ publishes the manifest's full count as if healthy; an HNSW flush that can't persist a node throws
+ (`HnswFlushError`) instead of returning a lying count and dropping the node; a corrupt or missing
+ manifest-listed column segment throws (`ColumnSegmentLoadError`) instead of silently dropping its
+ entities from every query; and aggregation materialization / state-load failures now warn instead
+ of vanishing into an empty catch.
+
+New export: **`PendingFlushDurabilityError`** (with `.cause` and `.failedAttempts`). No other public
+API change. Each fix ships with a dedicated regression test.
+
+## v8.2.5 — 2026-07-12 (honest response when a transaction rollback can't complete)
+
+Data-integrity fix. When a transaction failed and its rollback then *also* failed to undo a
+canonical write (retries exhausted), the old behavior logged, continued, and threw
+`TransactionRollbackError` — while the record it couldn't undo stayed durable on disk, and the
+transaction even reported its state as `'rolled_back'`. The caller got an error implying the write
+was undone; a read-back showed the record. A failed rollback had no truthful response.
+
+Rollback now tells the truth, with a two-branch contract:
+
+- **Adopt-forward (safe case).** A single-op write (`add`/`update`/…) whose only damage is a
+ durably-present record — the write the caller asked for — is **adopted**: its generation is
+ committed, the write returns success, and a loud warning records that the derived index may be
+ incomplete for that id until the next rebuild/`repairIndex()`. No error, no double-write; the
+ record is immediately durable and retrievable by `get()`.
+- **Fail loud + quarantine (unsafe case).** A multi-operation batch, or *any* case where a record
+ was lost (a remove/update whose restore-undo failed), throws the new **`StoreInconsistentError`**
+ naming every unreconciled record and its disposition (`orphan` vs `loss`), and puts the brain into
+ **write-quarantine**: reads keep working, but writes are refused until `repairIndex()` reconciles
+ the derived indexes against canonical storage and lifts the quarantine. The generation counter is
+ not advanced, and a transaction whose rollback failed is now `'inconsistent'`, never the
+ `'rolled_back'` lie.
+
+The decision is made by *observation*, not guesswork: both commit paths already hold byte-identical
+before-images, so after a failed rollback the store compares current canonical state to them to
+classify exactly which records are orphaned or lost.
+
+New export: `StoreInconsistentError` (with `.records` and `.cause`) and the `UnreconciledRecord` type.
+No other API change; `repairIndex()` gains the quarantine-lift behavior. Regression
+(`tests/integration/rollback-trapdoor.test.ts`) injects the exact failure (index add throws →
+canonical delete-undo fails) and pins adopt-forward (durable, get-able, not quarantined), fail-loud
+(`StoreInconsistentError` + quarantine + reads work + `repairIndex()` lifts it), and the error's
+record naming.
+
+## v8.2.4 — 2026-07-12 (restore can no longer destroy the store it's recovering)
+
+Recovery-safety fix. `restore()` removed the entire live brain directory and THEN copied the
+snapshot in — so any copy failure left the store destroyed with only a partial copy. The sharpest
+edge: the copy (`fs.cp`) materialized the holes of sparse mmap blob files, so a snapshot that fits
+on disk could balloon and `ENOSPC` mid-copy, and the recovery tool would have just destroyed the
+brain it was asked to recover. Reported from a downstream incident's recovery forensics.
+
+Restore is now **non-destructive and crash-resumable**:
+
+- The snapshot is copied into a staging area (`_restore_staging/`) **before any live data is
+ touched**. The copy is **sparse-aware** — all-zero regions are left as holes, so a store of
+ mostly-hole blobs restores at its true allocated size instead of its apparent size.
+- A copy failure (including `ENOSPC`) removes only the half-written staging area and throws; the
+ live store is left **exactly as it was**.
+- Only after the copy succeeds and a completion marker is `fsync`'d does an **atomic per-entry
+ swap** move the staged data into place — same-filesystem renames that cannot fail for disk space.
+- A crash mid-swap is finished **forward** on the next open: startup resumes a committed-but-
+ incomplete swap, or discards an uncommitted staging area (live data still authoritative).
+
+No API change — `restore(path, { confirm: true })` is unchanged. `persist()` was already safe
+(hard-link snapshot). Regression (`tests/integration/restore-nondestructive.test.ts`): a forced
+copy failure leaves live data fully intact, a normal restore round-trips, an interrupted-but-
+committed restore completes on reopen, an uncommitted staging area is discarded, and the sparse
+copy is byte-identical with allocation far below apparent size.
+
+## v8.2.3 — 2026-07-12 (a committed transaction is durable on return)
+
+Durability fix. A `transact()` reported "committed" while its canonical entity writes were still
+only in the OS page cache (written via tmp+rename, not yet `fsync`'d), even though the generation
+counter and manifest WERE fsync'd. A hard kill (power loss, SIGKILL) in that window could leave the
+durable generation counter **ahead of** the persisted entity bytes — so a consumer resuming from the
+counter would see "phantom progress": a generation that claims writes the disk never kept. Reported
+from a downstream migration's crash-lifecycle forensics.
+
+`commitTransaction` now runs a **durability barrier**: it records every canonical write and delete
+the batch's operations make, then `fsync`s that entire footprint (file contents, the rename
+directory entries, and the parent directories of any deletes) **before** advancing the generation
+counter and manifest. A committed transaction is therefore durable the moment `transact()` returns —
+the counter can never outrun the entity bytes.
+
+**Durability contract, now explicit.** `transact()` is durable-on-return (above). A **single-op**
+write (`add`/`update`/`remove`/`relate`/…) is Model-B group-commit: its live bytes are written but
+become durable at the next `flush()` or `close()`, not the instant the call resolves — the counter
+is buffered alongside the data, so a crash loses both together (never a torn counter-ahead-of-state
+store). Need per-write durability? Use `transact()` (even for one op), or `flush()` after the write.
+This deliberately trades single-op fsync latency (a 3-5x write regression) for throughput.
+
+No API change; no accelerator involvement (the barrier is in the filesystem storage adapter). In-memory
+and durable-per-call (cloud object-PUT) adapters treat the barrier as a no-op. Regression:
+`tests/integration/transact-durability-barrier.test.ts` proves entity writes fsync in an earlier
+batch than the manifest, for single-op and multi-op (add+relate) transactions, and that a
+precommit-rejected batch opens no barrier and advances nothing.
+
+## v8.2.2 — 2026-07-11 (P0: a timed-out transaction now rolls back — no torn state)
+
+Data-integrity fix. A transaction that exceeded its time budget **mid-flight** (e.g. a bulk
+`transact()` on slower hardware crossing the 30s ceiling) threw a timeout error WITHOUT rolling
+back the operations it had already applied. The budget check sat outside the per-operation
+rollback path, so only per-operation *failures* rolled back — a timeout stranded the partial
+writes in canonical storage while the generation was never stamped, leaving torn, generation-less
+state. This was caught by a downstream migration that crossed the ceiling on a large batch.
+
+`Transaction.execute()` now has a **single rollback point**: any error that escapes the operation
+loop — an operation failure OR a mid-flight timeout — rolls back every applied operation in
+reverse order, then surfaces the original error (a rollback failure supersedes it, loudly, as
+before). This restores the invariant the generation-store commit path already depended on: a throw
+from execute means the applied operations were undone byte-identically, and the aborted
+transaction leaves `generation()` unchanged and storage byte-identical to its pre-transaction
+state.
+
+No API change. Regression pins the reported requirement — after a mid-flight timeout,
+storage is byte-identical and the transaction ends in the `rolled_back` terminal state
+(`tests/unit/transaction/timeout-rollback.test.ts`), plus the operation-failure and
+rollback-failure paths through the same single rollback point.
+
+**Note on the 30s ceiling itself** (configurable/scaled timeout, batched embedding precompute,
+timeout telemetry) — that ergonomics work is tracked separately; this release fixes only the
+correctness bug (a timeout must never leave partial state), independent of where the ceiling sits.
+
+## v8.2.1 — 2026-07-11 (transact forward references work on the native accelerator)
+
+Parity fix. `transact()` has always promised atomic forward references — `add` an entity and
+`relate` to it in one batch — but the transaction planner resolved relationship endpoint ids at
+**plan** time, before the batch's adds had applied. Asking the id mapper about an entity that
+doesn't exist yet made the accelerator's (correctly strict) native mapper throw in
+`EntityIdMapper.getOrAssign`, so `transact([{op:'add', id:X}, {op:'relate', to:X}])` failed on
+native deployments while the permissive JS mapper masked the bug — and silently leaked an id
+assignment whenever a batch was later rejected by a commit precondition.
+
+Endpoint resolution is now **lazy** — evaluated when the graph operation executes inside the
+commit, after the batch's adds have applied (the same lazy pattern the operation's generation
+stamp already used). Fixed across all transact-planned graph operations: relate (including the
+bidirectional reverse edge), remove's relationship cascade, and unrelate. Single-operation writes
+are unchanged. Also fixed as a byproduct: a rejected batch no longer pollutes the id mapper.
+
+Regression suite covers the exact reported shape, both-endpoints-in-batch, bidirectional,
+add+relate+remove in one batch, and the mapper-cleanliness proof on rejected batches
+(`tests/integration/transact-forward-ref-graph.test.ts`). No API changes; no accelerator version
+pairing required.
+
+## v8.2.0 — 2026-07-10 (temporal VFS — file content joins the immutability model)
+
+Time travel now covers Virtual Filesystem **content**. Previously the temporal model had a hole
+exactly where files were concerned: every entity write was an immutable generation, but the
+content *bytes* lived under an eager reference-count GC — deleting a file could physically destroy
+bytes that in-window history still referenced, while overwriting never released the old content at
+all (an unbounded silent leak that only *accidentally* preserved history). A production consumer's
+recovery from a bad deploy succeeded only because a stale field happened to hold the good value —
+luck, not a guarantee. Both directions are now fixed by making blob reclamation a **history**
+decision instead of a **liveness** decision:
+
+- **Content blobs are retention-protected.** Each blob tracks live references AND history
+ references (one per persisted generation record that carries its hash). Deleting/overwriting a
+ file drops only the live reference; bytes are physically reclaimed in exactly one place —
+ history compaction — once no live reference and no retained generation references the hash.
+ Pinned views are exempt automatically (compaction already respects pins). Crash ordering is
+ over-count-only (never under-count), so a crash can leak until the built-in scrub recounts, but
+ can never reclaim bytes history still needs. Existing stores get a one-time, marker-gated
+ backfill on open; cross-file content dedup is handled exactly.
+- **`vfs.readFile(path, { asOf })`** — the file's exact bytes as of a generation or `Date`,
+ guaranteed present within the retention window.
+- **`vfs.history(path)`** — the file's versions (`{ generation, timestamp, hash, size,
+ mimeType? }`, ascending). Restore = read the old bytes `asOf` and write them back (a new write;
+ history is never rewritten).
+- **Overwrites now refresh the file entity's `data`/embedding text** — previously semantic search
+ and the `data` field served the FIRST version's text forever.
+
+Lifecycle note: deleting a file no longer frees its bytes immediately — old content lives until
+compaction reclaims its generations, under the same `retention` budget as all Model-B history
+(`retention: 'all'` keeps every version forever). Guide: the new "Time travel for files" section in
+`docs/guides/snapshots-and-time-travel.md`. Native accelerator: unaffected (canonical write path
+only) — no version pairing required.
+
+## v8.1.0 — 2026-07-10 (`brain.onChange` — the in-process change feed)
+
+New public API: subscribe to every committed mutation with
+`brain.onChange(cb) → unsubscribe`. One event per affected record, for **every**
+canonical write regardless of origin — direct calls, batch methods, `transact()`,
+imports, and Virtual Filesystem writes all funnel through the same commit point the
+feed is emitted from. This is the authoritative in-process signal for live UIs,
+cache invalidation, and realtime sync layers (downstream SDKs can forward it over
+their own transports to make local and remote brains uniform).
+
+Event shape (`BrainyChangeEvent`, exported): `kind` (`entity`/`relation`/`store`),
+`op` (`add`/`update`/`remove`/`relate`/`unrelate`/`updateRelation`/`clear`/`restore`),
+the post-commit `entity` or `relation` view (type + full custom metadata), the
+committed `generation`, and `timestamp`. Notable properties:
+
+- **Post-commit only** — an aborted write (a losing `ifRev` CAS, a rejected
+ transaction) never emits. If you got the event, the write is durable.
+- **Deletes fully described** — `remove`/`unrelate` carry the record's last
+ committed state (from the commit's own history record), not just an id; batch
+ deletes included.
+- **Batches emit per item**; a `transact()` batch's events share its single
+ generation. Entity deletes also emit `unrelate` for each cascaded relationship.
+- **Commit-ordered, asynchronous, isolated** — events arrive in commit order,
+ dispatched in a microtask so a slow listener never delays a write and a throwing
+ listener never affects the write or other listeners. Zero overhead with no
+ subscribers.
+- **`kind: 'store'`** events for `clear()`/`restore()` mean "refetch everything".
+- Fire-and-forget by design; each event's `generation` composes with
+ `asOf()`/`transactionLog()` for catch-up after a gap.
+
+Guide: `docs/guides/reacting-to-changes.md`. No behavior changes for existing code.
+
+## v8.0.17 — 2026-07-08 (count recovery scans the real layout · ~1,100 lines of dead 7.x machinery removed)
+
+A cleanup of the vestigial 7.x "hnsw sharding" machinery that turned up two real fixes.
+
+**1 — Count recovery now scans the layout the database actually uses.** When `counts.json` is
+lost or corrupted (container restarts, partial copies), the recovery scan rebuilt the entity/
+relationship counters by counting files in `entities/*/hnsw/` — directories the 8.0 write path
+never populates — so an established store recovered to **zero counts** on real data: wrong
+`getNounCount()`/stats, a "New installation"-style boot log, and a mis-sized rebuild-strategy
+decision at open. The scan now counts the canonical `entities////` tree.
+Regression-tested: delete `counts.json` from a populated store → reopen → exact counts.
+
+**2 — A latent init-time deadlock removed.** The recovery scan's type-distribution sampler called
+a guarded accessor (`getNounMetadata` → `ensureInitialized`) from **inside** `init()`, which
+re-enters `init()` and hangs the open. It was unreachable before only because the old scan never
+found anything to sample; the fix reads the sampled metadata files directly.
+
+**3 — The dead machinery itself is gone (−1,138 lines).** Twenty-three methods with zero callers:
+the 7.x hnsw-layout entity/edge CRUD, the sharding-depth prober + depth-migration engine, and an
+orphaned streaming paginator. Verified by reachability analysis before deletion; no public API
+touched; the full suite is green. New stores no longer carry the always-empty legacy directories'
+scan cost, and the boot log keeps the truthful new-vs-established wording from v8.0.13.
+
+## v8.0.16 — 2026-07-08 (concurrency fast-follow: atomic `ifAbsent`/`upsert` + exact blob reference counts)
+
+Closes the two remaining check-then-act races found in the sweep that followed v8.0.15's CAS fix
+(disclosed in that release's notes). Same bug class — a check performed before the serialization
+point that guards the apply — same cure.
+
+**1 — `add({ ifAbsent })` and `add({ upsert })` are now atomic.** The absence check ran before the
+commit mutex, so N concurrent same-id creates could all pass it and all write — the second
+silently overwriting the first, violating ifAbsent's "returns the existing id **without writing**"
+contract and upsert's "merge, never clobber" contract. The insert leg now carries a must-be-absent
+precondition (the same conditional-commit primitive as `ifRev`), verified under the commit mutex:
+exactly one concurrent create wins; every other caller takes its documented resolution — ifAbsent
+returns the existing id with zero writes, upsert merges into the now-existing entity (with a
+bounded retry if a concurrent delete intervenes). Verified: 8 concurrent `ifAbsent` creates
+advance the store by exactly ONE generation; 8 concurrent upserts on an absent id produce one
+create + seven merges (`_rev` lands at exactly 8). Note: `transact()`'s batch-level
+ifAbsent/upsert keep planning-time semantics (the batch converges to a valid entity; the window is
+documented, not silent).
+
+**2 — Blob reference counts are exact under concurrency.** The content-addressed blob store's
+`write()` (dedup: exists → add a reference, absent → create) and `delete()` (decrement → remove at
+zero) were unserialized read-modify-writes over the blob's metadata. Concurrent writes of
+identical content could lose references — making a later delete remove bytes **another file still
+referenced** (data loss), or leak unreferenced blobs. All reference-count-bearing mutations are
+now serialized per content hash (distinct content never contends): N concurrent identical writes
+yield exactly N references, and a blob is physically removed only when the true last reference
+drops. Verified with concurrent write/delete storms.
+
+Both fixes are in-process complete by construction — storage enforces single-writer-per-directory,
+so the process is the whole concurrency domain. No API changes.
+
+## v8.0.15 — 2026-07-08 (`ifRev` CAS is now atomic — exactly one winner under concurrency)
+
+Correctness fix for optimistic concurrency, reported from a production cutover rehearsal. N
+**concurrent** `update({ ifRev })` calls carrying the same expected revision ALL succeeded — zero
+`RevisionConflictError`s, last-writer-wins, the other N−1 writes silently lost. (Sequential calls
+conflicted correctly.) The revision check ran before the commit mutex, so interleaved callers all
+passed it before any apply landed — breaking the exactly-one-winner semantics the
+[optimistic-concurrency guide](docs/guides/optimistic-concurrency.md) promises, which advisory
+locks and per-entity ledgers/counters build on.
+
+The fix makes the check-and-apply a **conditional commit**: the generation store's commit paths
+accept a precondition that runs under the commit mutex against the just-read authoritative
+before-images — the per-record analogue of `ifAtGeneration`, which always ran there. `update()`
+and `transact()` per-op `ifRev` both re-verify at that point (the earlier check remains as a cheap
+fast-fail). A conflict aborts atomically: nothing staged, nothing applied, the same
+`RevisionConflictError` as before. Two adjacent behaviors also became honest:
+
+- **`_rev` is now monotonic under concurrency.** The winner's stamp derives from the
+ authoritative before-image, so N concurrent plain updates advance `_rev` by N (previously they
+ could all stamp the same stale value).
+- **CAS against a concurrently-deleted entity** now throws `EntityNotFoundError` instead of
+ silently resurrecting it (plain updates keep their last-writer-wins re-create semantics).
+
+Verified with a concurrency regression suite: 8 parallel same-rev updates → exactly 1 winner + 7
+conflicts; the documented read→CAS→retry ledger loop converges exactly (8 workers, 8 decrements,
+0 lost) — `tests/integration/ifrev-concurrent-cas.test.ts`. If you serialized writes in your own
+adapter as a workaround, it can come out after this upgrade.
+
+## v8.0.14 — 2026-07-07 (7→8 migration preserves branch-scoped non-entity state instead of deleting it)
+
+Defense-in-depth for the one-time 7→8 layout migration. The migration rescues the head branch's
+entities (`branches//entities/*` → `entities/*`) and then drained the whole `branches//`
+directory. If a 7.x engine had written durable state under the head branch *outside* `entities/`
+(a branch-scoped index, blob area, or field registry), that drain would have silently deleted it —
+the same failure class as the VFS content blobs a 7.x store kept in `_cow/`. The migration now drains
+the branch only when nothing but the moved entities remains; if any non-entity object survives, it
+**preserves the branch** (no delete) and logs a loud warning naming the leftover keys, so the state is
+recoverable rather than lost. No effect on a normal migration (a clean branch still drains); purely a
+guard against silent data loss.
+
+Cosmetic boot-log fix. Every persisted 8.0 store logged `📁 New installation: using depth 1
+sharding` on **every** open — even established brains holding thousands of entities — which is
+alarming to read during a restart or incident. Cause: 8.0 stores nouns in the canonical
+`entities/nouns///vectors.json` layout, but the legacy sharding probe inspected the
+`entities/nouns/hnsw/` directory, which the 8.0 write path never populates, so it always concluded
+"new". The new-vs-existing decision now consults the layout the database actually reads and writes
+(plus the known noun count), so an established store logs `📁 Using depth 1 sharding (N entities)`
+and only a genuinely empty store reports a new installation. No behavior change beyond the log line —
+it never triggered a rebuild or migration; entities were always read correctly.
+
+## v8.0.12 — 2026-07-07 (7→8 VFS-content recovery · zero-rebuild cold open · strict query operators)
+
+Three consumer-facing fixes.
+
+**1 — A 7→8 upgrade recovers Virtual Filesystem content automatically (data-integrity).**
+7.x stored VFS file content as blobs in the branch system's copy-on-write area (`_cow/`). 8.0
+removed that system and stores content blobs in the content-addressed store (`_cas/`), but the
+one-time layout migration only moved entities — it never adopted the `_cow/` content blobs. On a
+store that used the VFS (for example, a CMS with published pages), that left every VFS-backed read
+throwing `Blob metadata not found` and the pages 500ing. **8.0.12 adds an on-open recovery pass**
+that adopts every orphaned `_cow/` blob into `_cas/` — copying both the bytes and the metadata,
+idempotently and **non-destructively** (the `_cow/` originals are never deleted). It heals both a
+fresh 7→8 upgrade and a store **already** upgraded by an earlier 8.0.x that stranded them, with no
+operator action: just open the store under 8.0.12. An explicit force path is exposed as
+`brain.vfs.adoptOrphanedBlobs()` (returns `{ cowBlobs, adopted, alreadyPresent, incomplete }`). The
+automatic pre-upgrade backup is now **retained** if any blob can't be fully adopted
+(`incomplete > 0`) instead of being removed on entity-migration "success" alone. Affects any 7.x
+store that used the VFS; native-8.0 and non-VFS stores no-op on a cheap existence check. Full guide:
+`docs/guides/upgrading-7-to-8.md`.
+
+**2 — A cold open no longer re-derives durable indexes it can just load.**
+Opening a persisted store rebuilt the vector, graph, and metadata indexes from the canonical
+records even when the durable index state was present and loadable — an O(N) cost paid on every
+boot (measured at ~48 s on an ~11k-entity store). The rebuild gate now consults an honest
+per-provider durability signal (`init()` / `isReady()`) instead of an in-memory size heuristic, so
+a provider that has loaded (or can cheaply demand-load) its persisted index is not rebuilt. The
+built-in JS indexes keep today's behavior (a rebuild *is* their load path); the live query-time
+guards that self-heal a genuinely lost index are unchanged. **The zero-rebuild boot activates when
+the accelerator exposes the durability signal (`@soulcraft/cor@3.0.5`);** on earlier accelerator
+versions 8.0.12 falls back to the previous behavior safely — no regression, just no speedup yet.
+
+**3 — Unknown query operators now throw instead of silently returning nothing.**
+`find({ where })` had two gaps: the in-memory matcher (used for egress re-validation and historical
+reads) was missing several documented operators (`in`, `greaterThanOrEqual`, `lessThanOrEqual`), so
+it silently disagreed with the index path; and an unknown operator key (a typo, or `notIn` written
+where `not: { in }` was meant) was treated as a nested-object field and silently matched nothing.
+8.0.12 aligns the matcher to the full documented operator set and **validates the `where` filter
+up front**, throwing a typed `BrainyError('INVALID_QUERY')` naming the bad operator. Dotted paths
+remain the supported form for nested fields. If you relied on an unknown key silently returning `[]`,
+it now throws — the fix is to use the documented operator or dot-notation.
+
+## v8.0.11 — 2026-07-02 (no script shape can hang on brainy's internals)
+
+Completes v8.0.10's exit fix for every operation class. Two further mechanisms found and fixed:
+the `beforeExit` auto-flush hook looped forever on any script that never reaches `close()` (Node
+re-emits `beforeExit` after each event-loop drain and the async flush schedules new work — it now
+self-deregisters before its single flush, which still lands your buffered data before exit), and
+every background-maintenance interval (graph auto-flush, LSM compaction, metadata write-buffer,
+VFS/path-cache maintenance, statistics debounce) is now unref'd at creation. Verified against the
+published package: an `add + relate` script exits cleanly both with `close()` (~0.5 s) and with no
+teardown at all — with the data confirmed durable on reopen. A per-operation-class sweep test now
+asserts no ref'd timer survives `close()`, so this bug class stays closed.
+
+## v8.0.10 — 2026-07-02 (a bare script exits cleanly after `close()`)
+
+A minimal `init → add → close` script used to hang forever after `close()` returned — brainy held
+four process keep-alives it never released: the global SIGTERM/SIGINT shutdown hooks (never
+removed; now deregistered when the last live instance closes), an internal cache-fairness interval
+(unclearable by construction; now unref'd), and the VFS/PathResolver maintenance intervals
+(`close()` never shut the VFS down; now wired). Verified against the published package: the same
+script now exits ~1 ms after `close()` resolves. No API change; servers and long-running processes
+are unaffected.
+
+## v8.0.9 — 2026-07-02 (guarded plugin auto-detection — the "install it and it's on" contract)
+
+With the default config (`plugins` unset), brainy now **auto-detects the first-party accelerator**:
+installing `@soulcraft/cor` is the opt-in. The detection is guarded — everything except "not
+installed" fails loud:
+
+- Not installed → plain brainy, silently (the free path — zero noise, zero cost).
+- Installed and healthy → it loads and announces itself (`[brainy] Plugin activated`).
+- Installed but broken (unresolvable, invalid shape, failed activation, version mismatch) →
+ **`init()` throws.** An installed accelerator never silently vanishes behind the JS engines.
+- `plugins: []` / `false` = explicit opt-out; `plugins: ['@soulcraft/cor']` pins the exact list
+ (unchanged semantics).
+
+Also new: if a plugin activates but registers **zero** native providers (e.g. a licensing gate
+declining to engage), brainy warns loudly instead of leaving you to discover every query is running
+on the JS engines.
+
+> This supersedes v8.0.8's "plugins are explicit opt-in" wording, which documented the pre-GA
+> loader accurately but contradicted the published product contract ("add the package and it
+> activates"). 8.0.9 makes the contract true — with the loud-failure guarantees intact.
+
+## v8.0.8 — 2026-07-02 (docs-only patch on the GA)
+
+Corrects the README's scale-up section: the native provider is **not** auto-detected — plugins are
+**explicit opt-in** (`new Brainy({ plugins: ['@soulcraft/cor'] })`; brainy never auto-imports a
+package you didn't list, and a listed plugin that fails to load throws rather than silently falling
+back to the JS engines). Also fixes the stale `plugins` config comment in the public types and one
+phrase in the plugin-author guide. No code change.
+
+## v8.0.7 — 2026-07-02 (GA, npm tag `latest`)
+
+> **Why 8.0.7:** npm permanently retires unpublished version numbers, and `8.0.0`–`8.0.6` were
+> consumed by a January development cycle (published and immediately unpublished). `8.0.7` is
+> the first stable release of the 8.x line; there are no earlier stable 8.0.x releases.
+
+**8.0.7 is the first stable major on the u64-id core.** It ships in lockstep with the optional
+native provider's `3.0` (the billion-scale path). Everything below works standalone on the open-core
+JS engine — a native provider is feature-detected and only changes the scale ceiling, never the API.
+
+**Upgrading from 7.x: nothing to script.** The first time 8.0 opens a 7.x brain it upgrades itself —
+an automatic, observable, **coordinated migration lock** rebuilds every derived index from your
+canonical records while Brainy **blocks and queues** reads and writes, so no operation ever touches a
+half-built index and no write is ever lost. Budget seconds-to-minutes per brain at large sizes; small
+brains rebuild inline on open. A pre-upgrade hard-link **backup** is taken automatically (removed on
+success, kept on failure for rollback). Observe it via `getIndexStatus().migration`; a caller that
+hits the window gets a typed, retryable **`MigrationInProgressError`** (catch → HTTP 503 +
+`Retry-After`). Bounded by `migrationWaitTimeoutMs` (default 30 s — it bounds *your wait*, not the
+rebuild). Opt out of the backup with `migrationBackup: false`.
+
+### Headline changes
+
+- **The cold-open silent-`[]` class is gone.** On a cold reopen, filtered finds (`find({ where })`)
+ and graph reads (`find({ connected })` / `neighbors()` / `related()`) never silently return `[]`
+ for data that is actually present. With a native provider the durable indexes cold-serve every
+ filter and edge with zero rebuild; the open-core engine adds belt-and-suspenders guards that
+ self-heal from canonical data or throw a loud, typed error — never a silent empty result. Two new
+ exported errors: **`MetadataIndexNotReadyError`**, **`GraphIndexNotReadyError`**.
+
+- **Faster vector search (open-core).** The JS distance functions are allocation-free loops instead
+ of `reduce`: **~6× cosine, ~1.4× euclidean** (MEASURED — `tests/benchmarks/distance-microbench.mjs`,
+ 384-dim, median of 41). Numerically identical, so recall is unchanged. Exact metadata-filter
+ pushdown and scale-aware search width keep `find()` recall-correct as data grows.
+
+- **Per-write immutable history.** Every write is generation-stamped; `now()` / `asOf()` /
+ `transact()` read a consistent point in time, and a retention knob bounds history growth. Single
+ writes participate in the same immutable timeline as transactions.
+
+- **Billion-scale resident memory.** Nothing is O(N)-resident on the write or time-travel path —
+ per-id caches removed, the committed-generation ledger is an interval set, per-id history chains
+ are bounded (hot-window + LRU). Resident memory is independent of entity count.
+
+- **Deterministic, round-trip-free writes.** Supply your own ids, forward-reference not-yet-written
+ entities inside a `transact()`, `ifAbsent` upsert, and `brain.newId()` (uuidv7) — write graphs
+ without a write→wait→get round-trip.
+
+### Breaking changes (summary — the full migration guide follows below)
+
+- **Runtime floor: Node ≥ 22 / Bun ≥ 1.1** (Bun recommended). Compiler target ES2023; the DOM lib is
+ dropped — 8.0 is a Node/Bun/Deno engine with no browser path.
+- **`neural()` removed** and **`Db.search` removed** — use `find()` on the brain.
+- **Storage config is one `path` key.** Removed aliases now throw with a message pointing at `path`.
+- **`get()` omits the vector by default** — pass `{ includeVectors: true }` when you need it.
+- **Export/import type is `PortableGraph`** (was `BackupData`; wire tag `brainy-backup` →
+ `brainy-portable-graph`). Re-export any snapshots you version outside Brainy.
+- **Reserved `visibility` field** (`public` / `internal` / `system`) on nouns and verbs; `internal`
+ and `system` are excluded from normal reads by default.
+- **4 deprecated query operators removed** (`is`/`isNot`/`greaterEqual`/`lessEqual`) and reserved
+ keys in a `metadata` bag now **throw** by default — details in the rc notes below.
+
+> Post-rc.9 hardening folded into GA (no on-disk or provider-contract change vs `8.0.0-rc.9`):
+> the metadata cold-read guard, the auto pre-upgrade backup (with an `_id_mapper/*` mmap byte-copy
+> correctness fix), and a CI fix so a fresh clone builds green.
+
+### RC history (rc.1 → rc.9, npm tag `rc`)
> **rc.9 changes (2026-07-01):**
> - **Whole-brain auto-upgrade is now a coordinated, observable LOCK — supersedes rc.8's no-freeze
@@ -74,8 +1086,8 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
surfaces, hard renames (no aliases, no deprecation period), and one flipped
default. This entry **is** the migration guide: the find/replace pairs, the
sed snippet, and the upgrade checklist below are the complete story — there is
-no separate migration doc. **`8.0.0-rc.6` is live** — install with
-`npm i @soulcraft/brainy@rc` to try it; the `latest` tag stays on 7.x until GA.
+no separate migration doc. **`8.0.7` is GA on `latest`** — install with
+`npm i @soulcraft/brainy@latest`.
> **rc.3–rc.5 additions (2026-06-24):**
> - **Showcase-quality GA hardening (rc.5).** A full readiness audit closed a cold-init
diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md
index dc9244d9..248a2c70 100644
--- a/docs/PERFORMANCE.md
+++ b/docs/PERFORMANCE.md
@@ -417,7 +417,7 @@ const brain = new Brainy({
})
```
-The default JS index is `JsHnswVectorIndex`. An optional native acceleration package (`@soulcraft/cortex`) can replace it with a higher-performing implementation; the public knobs stay the same.
+The default JS index is `JsHnswVectorIndex`. An optional native acceleration package (`@soulcraft/cor`) can replace it with a higher-performing implementation; the public knobs stay the same.
### Scale Scenarios
diff --git a/docs/PLUGINS.md b/docs/PLUGINS.md
index 8e82b646..d9a4d3e7 100644
--- a/docs/PLUGINS.md
+++ b/docs/PLUGINS.md
@@ -12,7 +12,7 @@ next:
# Plugin Development Guide
-Brainy has a plugin system that allows third-party packages to replace internal subsystems with custom implementations. This is how `@soulcraft/cortex` provides optional native acceleration, and it's the same system available to any developer.
+Brainy has a plugin system that allows third-party packages to replace internal subsystems with custom implementations. This is how `@soulcraft/cor` provides optional native acceleration, and it's the same system available to any developer.
## Architecture Overview
@@ -23,20 +23,19 @@ Brainy's plugin system uses **named providers** — string keys mapped to implem
3. The plugin calls `context.registerProvider(key, implementation)` for each subsystem it provides
4. Brainy checks each provider key and wires the implementation into its internal pipeline
-Plugins are **opt-in** — brainy never auto-imports packages. You must explicitly list plugins in the config:
+Installing the first-party accelerator is the opt-in: with the default config, brainy probes for `@soulcraft/cor` and loads it when present. Everything except "not installed" fails **loud** — a present-but-broken accelerator makes `init()` throw rather than silently degrading to the JS engines.
```typescript
-const brain = new Brainy({
- plugins: ['@soulcraft/cortex'] // explicitly load the native acceleration package
-})
+const brain = new Brainy() // @soulcraft/cor auto-detected when installed
+const pinned = new Brainy({ plugins: ['@soulcraft/cor'] }) // or pin exactly what loads
+const plain = new Brainy({ plugins: [] }) // or opt out of detection entirely
```
| `plugins` value | Behavior |
|---|---|
-| `undefined` (default) | No plugins loaded |
-| `false` | No plugins loaded |
-| `[]` | No plugins loaded |
-| `['@soulcraft/cortex']` | Load only the listed packages |
+| `undefined` (default) | Guarded auto-detection of `@soulcraft/cor`: not installed → no plugins, silently; installed → loads + announces; installed-but-broken → `init()` throws |
+| `false` / `[]` | No plugins, no detection (explicit opt-out) |
+| `['@soulcraft/cor']` | Load only the listed packages; a listed plugin that fails to load throws |
Plugins registered programmatically via `brain.use(plugin)` are always activated regardless of the `plugins` config.
@@ -70,7 +69,7 @@ export default myPlugin
### 2. Package exports
-Your package must export the plugin as the default export so brainy's auto-detection works:
+Your package must export the plugin as the default export so brainy's plugin loader can resolve it:
```typescript
// index.ts
@@ -150,6 +149,17 @@ context.registerProvider('embedBatch', async (texts: string[]) => {
### Index Providers
+> **Write-path invariant (the change-feed contract).** Every canonical
+> mutation flows through Brainy's generation-store commit points — index
+> providers are invoked *inside* that commit and never originate canonical
+> writes of their own. The `brain.onChange` change feed is emitted from those
+> commit points and relies on this: **a plugin must never introduce a write
+> path that bypasses the generation-store commit.** If a future provider ever
+> needs a direct native ingest path, it must either route through the commit
+> or emit equivalent change events — otherwise every `onChange` consumer
+> (live UIs, cache invalidation, realtime sync) silently develops a blind
+> spot.
+
#### `vector`
**Type:** `(config: object, distanceFunction: Function, options: object) => VectorIndexProvider-compatible`
@@ -182,6 +192,27 @@ context.registerProvider('vector', (config, distanceFn, options) => {
})
```
+#### The readiness contract (all three index providers)
+
+A provider that **persists its derived index** should implement the optional readiness
+members so a warm reopen never pays a redundant rebuild-from-canonical:
+
+- **`init?(): Promise`** — eager cold-load. Brainy awaits it once during
+ `brain.init()`, after the metadata provider's `init()` (the id-mapper hydrates first)
+ and **before the rebuild gate**.
+- **`isReady?(): boolean`** — honest durability signal. `true` ⇔ the persisted index is
+ loaded (or cheaply demand-loadable) and consistent with what was last persisted. When
+ exposed, the rebuild gate defers to this signal **instead of** the `size() === 0` /
+ `totalEntries === 0` heuristics — a disk-native index may report 0 resident entries
+ while fully durable. Never return `true` if the durable state failed to load: the
+ signal is honest in both directions, and a not-ready provider gets its rebuild even
+ when `size() > 0`.
+- **`isMigrating?(): boolean`** — while `true`, the provider owns its index (background
+ migration); brainy skips its rebuild entirely.
+
+Providers that implement none of these keep the size/count heuristics — correct for
+engines whose `rebuild()` *is* their load path (like brainy's built-in JS vector index).
+
#### `metadataIndex`
**Type:** `(storage: StorageAdapter) => MetadataIndexManager-compatible`
@@ -215,7 +246,7 @@ context.registerProvider('aggregation', (storage) => {
})
```
-When provided by an optional native acceleration plugin (such as `@soulcraft/cortex`), this enables:
+When provided by an optional native acceleration plugin (such as `@soulcraft/cor`), this enables:
- Compiled source filters (vs per-entity JS object traversal)
- Precise MIN/MAX via sorted data structures (vs lazy recompute)
- Parallel aggregate rebuild across CPU cores
@@ -251,7 +282,7 @@ Native msgpack encode/decode for SSTable serialization.
### Analytics Providers (Native-Only)
-These provider keys have **no JavaScript fallback** — they represent capabilities that require native code (SIMD, mmap, sub-microsecond latency). They are available when an optional native acceleration plugin (such as `@soulcraft/cortex`) is installed.
+These provider keys have **no JavaScript fallback** — they represent capabilities that require native code (SIMD, mmap, sub-microsecond latency). They are available when an optional native acceleration plugin (such as `@soulcraft/cor`) is installed.
Use `brain.getProvider('analytics:hyperloglog')` to check availability. Returns `undefined` if no plugin provides it.
@@ -359,8 +390,8 @@ brainy diagnostics
When a plugin is active, brainy automatically logs a provider summary after `init()`:
```
-[brainy] Plugin activated: @soulcraft/cortex
-[brainy] Providers: 8/10 native (@soulcraft/cortex) | default: vector, cache
+[brainy] Plugin activated: @soulcraft/cor
+[brainy] Providers: 8/10 native (@soulcraft/cor) | default: vector, cache
```
This tells you at a glance how many subsystems are accelerated and which ones are falling back to JavaScript. The log respects `config.silent`.
@@ -381,7 +412,7 @@ If a required provider is missing, the error message tells you exactly what's wr
```
[brainy] Required providers using JS fallback: graphIndex.
-Active plugins: @soulcraft/cortex.
+Active plugins: @soulcraft/cor.
These providers must be supplied by a plugin for this deployment.
Check plugin installation, license, and native module availability.
```
diff --git a/docs/QUERY_OPERATORS.md b/docs/QUERY_OPERATORS.md
index b1db7850..f4ac04e6 100644
--- a/docs/QUERY_OPERATORS.md
+++ b/docs/QUERY_OPERATORS.md
@@ -234,7 +234,7 @@ const all = await brain.related({
})
// Graph traversal — subtype filters traversal edges (depth-1 in 7.30 JS path;
-// multi-hop subtype filtering lands on Cortex native)
+// multi-hop subtype filtering lands on Cor native)
const reports = await brain.find({
connected: {
from: ceoId,
diff --git a/docs/RELEASE-GUIDE.md b/docs/RELEASE-GUIDE.md
index ce7d32c5..94c6a6cb 100644
--- a/docs/RELEASE-GUIDE.md
+++ b/docs/RELEASE-GUIDE.md
@@ -119,4 +119,13 @@ gh release create vY.Y.Y --generate-notes
- **Most releases should be MINOR or PATCH**
- **Major versions should be RARE**
- **When in doubt, it's probably MINOR**
-- **NEVER use "BREAKING CHANGE" for internal changes**
\ No newline at end of file
+- **NEVER use "BREAKING CHANGE" for internal changes**
+## Hard Ordering Constraints (check before EVERY release)
+
+- **Embedding model changes are SEQUENCED, not free.** No release may change the
+ embedding model (or its quantization/dimensions) before **vector model-version
+ stamping + hard-error-on-mismatch** ships. Stored vectors carry no model version
+ today; mixing vectors from two models silently corrupts every similarity
+ comparison. If a model bump is ever proposed, the stamping work moves ahead of
+ it in the schedule — coordinate with the native provider so both engines stamp
+ and enforce identically. (Registered with the native-provider team 2026-07-07.)
diff --git a/docs/SCALING.md b/docs/SCALING.md
index c1e00987..e9ae1136 100644
--- a/docs/SCALING.md
+++ b/docs/SCALING.md
@@ -37,7 +37,7 @@ The three knobs that matter most:
1. **`config.vector.recall`** — `'fast'`, `'balanced'`, or `'accurate'` (default `'balanced'`)
2. **`config.vector.persistMode`** — `'immediate'` for durability, `'deferred'` for throughput
-The native vector provider (via the optional `@soulcraft/cortex` package) extends this with a higher-performing index — and its own at-scale acceleration such as on-disk compressed indexing — when installed.
+The native vector provider (via the optional `@soulcraft/cor` package) extends this with a higher-performing index — and its own at-scale acceleration such as on-disk compressed indexing — when installed.
## Measured Performance
@@ -77,7 +77,7 @@ graph, metadata index, and 100k edges).
**Scale ceiling (open-core).** The pure-JS HNSW *build* cost (~100 inserts/s at 384-dim on
one core) makes the in-process open-core path most appropriate up to ~10⁵–10⁶ entities.
*Query* latency stays low well beyond that, but for the 10⁸–10¹⁰ regime install the native
-provider (`@soulcraft/cortex`, on-disk DiskANN) — same API, no code change. _Projected from
+provider (`@soulcraft/cor`, on-disk DiskANN) — same API, no code change. _Projected from
the two measured points, vector p50 at 1M is ~2 ms; metadata-heavy composition grows with
match-set size and is the path to move onto the native provider first._
@@ -213,12 +213,12 @@ const stats = await brain.stats()
### Issue: Slow queries
1. Switch to `vector.recall: 'fast'`
2. Increase the read cache (`storage.cache.maxSize`)
-3. Consider the optional native vector provider via `@soulcraft/cortex`
+3. Consider the optional native vector provider via `@soulcraft/cor`
### Issue: Memory pressure
1. Reduce `storage.cache.maxSize`
2. Move to `vector.persistMode: 'deferred'` to batch writes
-3. Consider the optional native vector provider via `@soulcraft/cortex` for at-scale index acceleration
+3. Consider the optional native vector provider via `@soulcraft/cor` for at-scale index acceleration
### Issue: Slow startup after a crash
1. Use `vector.persistMode: 'immediate'` so the index file stays in sync with storage
diff --git a/docs/architecture/aggregation.md b/docs/architecture/aggregation.md
index 10b55689..443ee727 100644
--- a/docs/architecture/aggregation.md
+++ b/docs/architecture/aggregation.md
@@ -33,7 +33,7 @@
│ ▼ │
│ ┌──────────────────────┐ │
│ │ AggregationProvider │ (optional, registered by │
-│ │ ├─ incrementalUpdate│ plugin like @soulcraft/cortex)│
+│ │ ├─ incrementalUpdate│ plugin like @soulcraft/cor) │
│ │ ├─ rebuildAggregate │ │
│ │ ├─ queryAggregate │ │
│ │ └─ serialize/restore│ │
@@ -135,7 +135,7 @@ M2 is clamped to zero on remove to prevent floating-point drift from producing n
The TypeScript engine uses simple comparison for add operations and marks MIN/MAX as potentially stale on delete (since removing the current min/max value requires a rescan). Stale values are lazily recomputed on the next query.
-The Cortex native engine uses a `BTreeMap, u64>` that tracks the exact frequency of every value, providing precise MIN/MAX after any sequence of operations without rescanning.
+The Cor native engine uses a `BTreeMap, u64>` that tracks the exact frequency of every value, providing precise MIN/MAX after any sequence of operations without rescanning.
### Time Window Bucketing
diff --git a/docs/architecture/data-storage-architecture.md b/docs/architecture/data-storage-architecture.md
index a51ebf05..48064757 100644
--- a/docs/architecture/data-storage-architecture.md
+++ b/docs/architecture/data-storage-architecture.md
@@ -369,7 +369,10 @@ await brain.restore('/backups/today', { confirm: true }) // replace store stat
through the metadata index, not the filesystem.
- **`_system/`** holds singletons plus 256 hash buckets of index state.
- **`_generations/` + `manifest.json` + `tx-log.jsonl`** implement
- generational MVCC; `transact()` is the unit of history.
+ generational MVCC. History is per-write: every `add()`/`update()`/`remove()`/
+ `relate()` gets its own generation, and `transact()` groups several ops into
+ one atomic generation. (Single-op retention has been the model since 8.0;
+ you never need to route a write through `transact()` just to keep its history.)
- **`_column_index/` + `_blobs/`** hold the columnar metadata runs and binary
blobs (VFS content included).
- **`locks/`** coordinates the single writer and reader flush requests, and
diff --git a/docs/architecture/index-architecture.md b/docs/architecture/index-architecture.md
index 4d9f37fe..8b3dc540 100644
--- a/docs/architecture/index-architecture.md
+++ b/docs/architecture/index-architecture.md
@@ -402,7 +402,7 @@ const DEFAULT_EXCLUDE_FIELDS = [
**Purpose**: O(log n) semantic similarity search using vector embeddings.
-The default JS implementation is `JsHnswVectorIndex`; an optional native acceleration package (`@soulcraft/cortex`) can register a higher-performing `VectorIndexProvider` through the plugin system. The public API stays the same either way.
+The default JS implementation is `JsHnswVectorIndex`; an optional native acceleration package (`@soulcraft/cor`) can register a higher-performing `VectorIndexProvider` through the plugin system. The public API stays the same either way.
### Internal Architecture
diff --git a/docs/architecture/multiprocess-storage-mixin.md b/docs/architecture/multiprocess-storage-mixin.md
index 7b53fa33..46f98398 100644
--- a/docs/architecture/multiprocess-storage-mixin.md
+++ b/docs/architecture/multiprocess-storage-mixin.md
@@ -22,7 +22,7 @@ requestFlushOverFilesystem(timeoutMs)
They live on `BaseStorage` as no-op defaults and are overridden on
`FileSystemStorage` with real implementations. Any adapter extending
-`FileSystemStorage` (e.g. Cortex's `MmapFileSystemStorage`) inherits the
+`FileSystemStorage` (e.g. Cor's `MmapFileSystemStorage`) inherits the
real ones for free.
This works correctly today. The question is whether the methods *belong*
@@ -83,7 +83,7 @@ Benefits:
## The case against doing it now
- Breaking change for any adapter that already overrides these methods.
- `FileSystemStorage` is the only one in-tree, but Cortex's
+ `FileSystemStorage` is the only one in-tree, but Cor's
`MmapFileSystemStorage` inherits from it — interface relocation would
ripple through the plugin ecosystem.
- The current state works. The real failure modes seen in the field
diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md
index 3f77e133..1bc6e66c 100644
--- a/docs/architecture/overview.md
+++ b/docs/architecture/overview.md
@@ -48,7 +48,7 @@ Pluggable vector index (`VectorIndexProvider`) for efficient nearest-neighbor se
- **Configurable recall**: `fast` / `balanced` / `accurate` presets trade recall for latency
- **Scalable**: Handles millions of vectors per process
- **Persistent**: Serializable to storage
-- **Swappable**: Replace with a native implementation (such as `@soulcraft/cortex`) via the plugin system without changing application code
+- **Swappable**: Replace with a native implementation (such as `@soulcraft/cor`) via the plugin system without changing application code
### Metadata Index Manager
High-performance field indexing system:
diff --git a/docs/architecture/zero-config.md b/docs/architecture/zero-config.md
index 64a10461..d42d6784 100644
--- a/docs/architecture/zero-config.md
+++ b/docs/architecture/zero-config.md
@@ -84,7 +84,7 @@ const brain = new Brainy({
```
The default JS index is `JsHnswVectorIndex`. An optional native acceleration
-provider (the `@soulcraft/cortex` package) can replace it with a
+provider (the `@soulcraft/cor` package) can replace it with a
higher-performing implementation; the public knobs stay the same. Quantization
and other index-internal acceleration are the native provider's concern, not a
Brainy configuration option.
diff --git a/docs/concepts/generation-fact-log.md b/docs/concepts/generation-fact-log.md
new file mode 100644
index 00000000..f9b3e974
--- /dev/null
+++ b/docs/concepts/generation-fact-log.md
@@ -0,0 +1,117 @@
+---
+title: The Generation Fact Log
+slug: concepts/generation-fact-log
+public: true
+category: concepts
+template: concept
+order: 6
+description: Every committed write also appends a self-verifying "fact" — an after-image commit record — to an append-only log. What facts are, the crash-safety model, the scanFacts() streaming surface, family stamps, and how index providers consume the log for sequential heals.
+next:
+ - concepts/consistency-model
+ - guides/snapshots-and-time-travel
+---
+
+# The Generation Fact Log
+
+Since 8.4.0, every committed generation also appends a **fact** — a compact record of what each
+touched entity or relationship *became* — to an append-only, checksummed log under
+`_generations/facts/`. Where the generational history answers *"what did things look like
+before?"* (before-images, powering `asOf()` and rollback), the fact log answers *"what happened,
+in order?"* — one sequential, self-verifying stream of the store's present being written.
+
+Nothing about querying changes. The fact log exists for three consumers:
+
+1. **Index heals and rebuilds** — one sequential read in commit order replaces a per-entity
+ directory walk over millions of files.
+2. **Incremental catch-up** — a derived index that knows which generation it reflects reads *just
+ the gap*, instead of rebuilding from scratch.
+3. **Replay and audit tooling** — anything that wants the store's committed timeline as a stream.
+
+## What a fact is
+
+One fact per committed generation:
+
+- **`generation`** and **`timestamp`** — which commit, when.
+- **`ops`** — every write in that commit: `{ kind: 'noun' | 'verb', id, record }` where `record`
+ holds the entity's full after-image (both stored legs), or **`null` for a tombstone** — a
+ removal carries no body, by design.
+- **`meta`** — the transaction metadata `transact()` was submitted with, when present.
+- **`blobHashes`** — content-blob references, for exact reclamation accounting.
+
+Facts accumulate **from the first write after upgrading** — pre-existing history is not
+retroactively converted, and consumers fall back to the enumeration walk when no log exists.
+
+## Crash safety, in one paragraph
+
+Facts are appended and fsynced **inside the same durability window as the commit itself**, before
+the commit point — so after a crash, the log can only ever be *ahead* of committed truth, never
+behind it with a hole. On open, the store reconciles the log back to the committed watermark:
+torn tails are detected by per-record checksums and cut; whole records beyond the watermark are
+truncated. The invariant every reader can rely on: **an absent generation was never committed; a
+present fact was.** `transact()` facts are durable the moment `transact()` returns; single-op
+facts share the same group-commit flush as the rest of their generation, so a hard kill loses the
+fact and the generation *together* — never a torn state.
+
+## Reading the log
+
+```typescript
+const scan = brain.scanFacts({ fromGeneration: 1 })
+if (scan) {
+ // Telemetry up front — progress bars get a denominator from second zero.
+ console.log(scan.headGeneration, scan.segmentCount, scan.approxFactCount)
+
+ for await (const batch of scan.batches()) {
+ // Each batch: { facts, firstGeneration, lastGeneration, factCount, byteSize, segmentId }
+ for (const fact of batch.facts) {
+ for (const op of fact.ops) {
+ if (op.record === null) {
+ // a tombstone: op.id was removed in this generation
+ }
+ }
+ }
+ }
+
+ console.log(scan.summary()) // { factsYielded, segmentsRead } — the cross-check
+}
+```
+
+- `scanFacts()` returns `null` when the store hosts no fact log (older store, or a storage adapter
+ without binary append support) — fall back to enumerating entities.
+- Scans run against a **snapshot**: facts appended after the scan opens never bleed in, each fact
+ is yielded exactly once, and a detected gap aborts loudly — never a silent skip.
+- `brain.factSegmentPaths()` returns the immutable, *sealed* segment files for zero-copy consumers
+ (the append-mutable tail is excluded — read it through `scanFacts()`).
+
+## Family stamps: how a projection proves it's current
+
+Anything derived from the store — an index, the entity file tree itself — carries a **family
+stamp**: a small JSON record of *which committed generation the projection reflects*
+(`sourceGeneration`) plus the invariants that verify it whole (exact per-file byte sizes for
+bounded families; rollup invariants like entity counts for unbounded ones). At open, coherence is
+a **comparison**, not a walk:
+
+- stamp equals the committed watermark and invariants hold → serve;
+- stamp is behind → the projection reads just the gap from the fact log;
+- invariants fail → loud, named divergence — `brain.repairIndex()` rebuilds from canonical and
+ re-stamps.
+
+The verifier is exported (`verifyFamilyStamp`) so every projection — TypeScript or native — runs
+literally the same check.
+
+## For plugin authors: the storage capability
+
+Index providers receive the storage adapter, not the brain — so the host wires the log onto it.
+Feature-detect and prefer the stream; fall back to enumeration:
+
+```typescript
+const scan = storage.scanFacts?.({ fromGeneration: stamp.sourceGeneration + 1 })
+if (scan) {
+ // sequential catch-up from the log
+} else {
+ // enumeration walk (older store or adapter)
+}
+const committed = storage.committedGeneration?.() // the watermark stamps compare against
+```
+
+Providers must never construct their own reader over the log's files — the open path belongs to
+the single writer (it reconciles the log at open); the capability is the sanctioned seam.
diff --git a/docs/concepts/multi-process.md b/docs/concepts/multi-process.md
index cd65594b..8fda315f 100644
--- a/docs/concepts/multi-process.md
+++ b/docs/concepts/multi-process.md
@@ -55,20 +55,20 @@ process opening the same directory:
The fix is the lock: refuse to open a second writer. SQLite has done the same
since the late 1990s (`SQLITE_BUSY`).
-## What about Cortex?
+## What about Cor?
-Brainy + Cortex compose cleanly under this model:
+Brainy + Cor compose cleanly under this model:
-- Cortex stores its column-index segments inside the same `rootDir` (under
+- Cor stores its column-index segments inside the same `rootDir` (under
`indexes/_column_index/{field}/`).
-- Segments (`*.cidx` files) are **immutable** once written. Cortex mmaps them
+- Segments (`*.cidx` files) are **immutable** once written. Cor mmaps them
read-only.
- The `MANIFEST.json` per field is updated via atomic rename — readers see
either the old or new manifest, never a torn file.
-A reader process can safely mmap Cortex segments alongside a live writer
+A reader process can safely mmap Cor segments alongside a live writer
without coordination. The single Brainy writer lock at
-`/locks/_writer.lock` covers Cortex too, because Cortex segment
+`/locks/_writer.lock` covers Cor too, because Cor segment
writes happen on the writer's side.
## Stale-lock detection
@@ -141,7 +141,7 @@ The CLI `brainy inspect` subcommands all do this for you by default
processes can both succeed at `init()` in writer mode and clobber each
other's writes. A best-effort warning is logged in writer mode against a
non-filesystem backend.
-- **Long-running readers** do not automatically pick up new Cortex segments
+- **Long-running readers** do not automatically pick up new Cor segments
the writer publishes. One-shot inspector calls re-open the store and see
fresh segments; a reader that stays open for hours sees its column store
as-of the time it opened.
@@ -150,4 +150,4 @@ The CLI `brainy inspect` subcommands all do this for you by default
- `Brainy.openReadOnly()` — [API reference](../api/brainy.md)
- `brainy inspect` — [inspection guide](../guides/inspection.md)
-- Cortex columnar storage — see `node_modules/@soulcraft/cortex/README.md`
+- Cor columnar storage — see `node_modules/@soulcraft/cor/README.md`
diff --git a/docs/concepts/storage-adapters.md b/docs/concepts/storage-adapters.md
index e0f9b70d..af6d068f 100644
--- a/docs/concepts/storage-adapters.md
+++ b/docs/concepts/storage-adapters.md
@@ -33,7 +33,7 @@ BaseStorage (type-statistics, lifecycle helpers, generation
FileSystemStorage (real filesystem I/O, writer-lock implementation,
flush-request watcher, atomic writes)
↑
- (Cortex's MmapFileSystemStorage, etc.)
+ (Cor's MmapFileSystemStorage, etc.)
```
When you `extend FileSystemStorage`, your adapter inherits every method on
diff --git a/docs/eli5.md b/docs/eli5.md
index 303de040..e0bb9a19 100644
--- a/docs/eli5.md
+++ b/docs/eli5.md
@@ -11,7 +11,7 @@ next:
- getting-started/quick-start
---
-# Brainy and Cortex — Explained Simply
+# Brainy and Cor — Explained Simply
*A plain-language guide for anyone who wants to understand what this thing actually does.*
@@ -65,13 +65,13 @@ Brainy can narrow any result set down by exact labels or ranges in the same brea
---
-## What is Cortex?
+## What is Cor?
-Cortex is a turbocharger for Brainy.
+Cor is a turbocharger for Brainy.
Same car. Same controls. Same fuel. You just swap in a faster engine under the hood, and everything that used to take a moment now happens instantly.
-Technically, Cortex is an optional plugin written in Rust — a lower-level language that runs much closer to the raw metal of your processor. It plugs into Brainy and takes over the most compute-intensive work: the distance calculations that power meaning search, the number-crunching behind live aggregates, and the set operations that drive label filtering.
+Technically, Cor is an optional plugin written in Rust — a lower-level language that runs much closer to the raw metal of your processor. It plugs into Brainy and takes over the most compute-intensive work: the distance calculations that power meaning search, the number-crunching behind live aggregates, and the set operations that drive label filtering.
You install it with one line, register it with one call, and Brainy automatically uses it everywhere it can help.
@@ -83,9 +83,9 @@ Plain language:
- **Searches** go from "the blink of an eye" to "faster than a blink." The overall speedup is **5.2× on average** across all operations.
- **Live aggregates** are rebuilt using all CPU cores in parallel, so re-indexing large datasets takes a fraction of the time.
-- **Analytics** that aren't even possible in pure JavaScript — real-time anomaly detection, streaming percentile estimates, approximate unique counts — become available because Cortex brings the native capabilities required to run them efficiently.
+- **Analytics** that aren't even possible in pure JavaScript — real-time anomaly detection, streaming percentile estimates, approximate unique counts — become available because Cor brings the native capabilities required to run them efficiently.
-If Brainy is what makes knowledge fast, Cortex is what makes Brainy feel instant.
+If Brainy is what makes knowledge fast, Cor is what makes Brainy feel instant.
---
@@ -135,7 +135,7 @@ Brainy is the only row with every box checked. And it runs all of them in a sing
Brainy scales from a quick experiment to serious production datasets without changing a line of code. Small datasets live entirely in memory. Larger ones spill to disk, where Brainy shards and compresses everything automatically. Need a backup or a copy? Snapshots are instant — the same API the whole way.
-Add Cortex and you also unlock memory-mapped storage — aggregate state lives directly in the operating system's memory with zero serialization overhead, as fast as the hardware allows.
+Add Cor and you also unlock memory-mapped storage — aggregate state lives directly in the operating system's memory with zero serialization overhead, as fast as the hardware allows.
---
diff --git a/docs/guides/aggregation.md b/docs/guides/aggregation.md
index 1ae2c98e..11d86ec8 100644
--- a/docs/guides/aggregation.md
+++ b/docs/guides/aggregation.md
@@ -11,7 +11,13 @@ No batch jobs. No scheduled recalculations. Aggregates stay current with every w
**Defining over existing data:** if you define an aggregate on a store that already holds
matching entities, Brainy backfills it from those entities on the first query (a one-time scan,
then purely incremental). So `defineAggregate()` behaves the same whether you define it before
-or after the data exists — including when a persisted brain reopens already populated.
+or after the data exists.
+
+**Reopening a persisted brain:** aggregate state persists across restarts. Re-defining the
+same aggregate at boot (the normal declarative pattern) adopts the persisted state directly —
+no rescan. A backfill scan runs only when the definition actually changed, when no persisted
+state exists, or when the state failed to load; and however many aggregates need backfilling,
+they share a single scan.
## Quick Start
@@ -487,7 +493,7 @@ Aggregate definitions and running state are automatically persisted:
## Native Acceleration
-When [Cortex](https://github.com/soulcraftlabs/cortex) is installed as a plugin, the aggregation engine automatically uses Rust-accelerated computation:
+When [Cor](https://www.npmjs.com/package/@soulcraft/cor) is installed as a plugin, the aggregation engine automatically uses Rust-accelerated computation:
- Incremental updates run in Rust with BTreeMap-backed precise MIN/MAX
- Welford's online stddev/variance computed natively
@@ -496,7 +502,7 @@ When [Cortex](https://github.com/soulcraftlabs/cortex) is installed as a plugin,
```typescript
const brain = new Brainy({
- plugins: ['@soulcraft/cortex']
+ plugins: ['@soulcraft/cor']
})
await brain.init()
@@ -577,7 +583,7 @@ brain.defineAggregate({
Aggregation complexity per write is O(A x G x M) where A = matching aggregates, G = groupBy dimensions, M = metrics. For typical configurations (2-5 aggregates, 1-3 dimensions, 3-5 metrics), this is effectively O(1).
-With Cortex native acceleration:
+With Cor native acceleration:
| Operation | Throughput | Latency |
|-----------|-----------|---------|
diff --git a/docs/guides/enterprise-for-everyone.md b/docs/guides/enterprise-for-everyone.md
index de099d04..b79844e9 100644
--- a/docs/guides/enterprise-for-everyone.md
+++ b/docs/guides/enterprise-for-everyone.md
@@ -174,13 +174,13 @@ await brain.syncWith({
const brain = new Brainy()
// 1 → ~1M vectors: pure-JS HNSW, zero extra setup
-// 1M → 10B+ vectors: install @soulcraft/cortex for the native DiskANN provider
+// 1M → 10B+ vectors: install @soulcraft/cor for the native DiskANN provider
```
**Scaling model:**
- **Single process, no cluster**: Brainy runs in one process — no coordinator,
no peer discovery, no consensus to operate
-- **Optional native provider**: install `@soulcraft/cortex` to back the index with
+- **Optional native provider**: install `@soulcraft/cor` to back the index with
on-disk DiskANN that scales to 10B+ vectors on one machine
- **Per-tenant pools**: isolate tenants by giving each its own Brainy instance and
storage directory
diff --git a/docs/guides/external-backups-and-sparse-storage.md b/docs/guides/external-backups-and-sparse-storage.md
new file mode 100644
index 00000000..f28dcfd7
--- /dev/null
+++ b/docs/guides/external-backups-and-sparse-storage.md
@@ -0,0 +1,99 @@
+---
+title: External Backups & Sparse Storage
+slug: guides/external-backups
+public: true
+category: guides
+template: guide
+order: 10
+description: How to back up a brain directory with external tools (tar, rsync, cp) without exploding sparse files — why a store can show 100+ GB "apparent" size on a small disk, which files are sparse, and how persist()/restore() handle it for you.
+next:
+ - guides/snapshots-and-time-travel
+ - concepts/storage-adapters
+---
+
+# External Backups & Sparse Storage
+
+The built-in snapshot path — [`db.persist()` and `brain.restore()`](/docs/guides/snapshots-and-time-travel) —
+already handles everything on this page for you. Read this when you back up a brain directory with
+**external tools**: `tar`, `rsync`, `cp`, `scp`, or a filesystem-level backup agent.
+
+## The one-sentence rule
+
+> **Always use the sparse-aware flag**: `tar czSf` (capital `S`), `rsync --sparse`,
+> `cp --sparse=always`. A naive copy can turn a 2 GB store into a 100+ GB one — or fail
+> the disk entirely.
+
+## Why: some files are sparse
+
+When a native accelerator plugin is active, parts of the index live in **memory-mapped files**
+created at a large fixed virtual size — the file's *apparent* size — while the filesystem only
+allocates blocks that were actually written. A brand-new id-mapper file can report tens of
+gigabytes in `ls -l` while occupying a few megabytes on disk.
+
+Check the difference yourself:
+
+```bash
+ls -lh brain-data/_id_mapper/ # APPARENT size (can be huge)
+du -sh brain-data/ # ALLOCATED size (the real footprint)
+```
+
+The sparse candidates in a brain directory:
+
+| Path | What it is |
+|---|---|
+| `_id_mapper/` | The native id-mapper's mmap files (large fixed virtual size) |
+| `_blobs/` | Native index files (vector base, segments) — may be mmap-backed |
+
+Everything else (entities, `_system`, `_generations`, `_cas` content blobs) is ordinary dense data.
+
+## Doing it right
+
+**tar** — the `S` flag detects holes and stores only real data:
+
+```bash
+tar czSf brain-backup.tgz /data/brain
+# restore preserves the holes:
+tar xzSf brain-backup.tgz -C /data/
+```
+
+**rsync**:
+
+```bash
+rsync -a --sparse /data/brain/ backup-host:/backups/brain/
+```
+
+**cp**:
+
+```bash
+cp -a --sparse=always /data/brain /backups/brain
+```
+
+**What goes wrong without the flag:** the copy *materializes* every hole as real zero bytes.
+A store whose apparent size exceeds the target disk fails with `ENOSPC` partway through — and a
+copy that *does* fit silently costs the full apparent size in storage and transfer time.
+
+## What the built-in paths do (so you don't have to)
+
+- **`db.persist(path)`** snapshots via **hard links** — instant and space-shared, since every data
+ file is immutable-by-rename. The handful of append-in-place files (the transaction log, the
+ commit fact log's tail segment) and mmap-mutated directories (`_id_mapper/`) are **byte-copied**
+ instead, so a post-snapshot write can never reach through a shared inode into your backup.
+- **`brain.restore(path, { confirm: true })`** is **non-destructive and sparse-aware**: the snapshot
+ is copied into a staging area *before* any live data is touched (all-zero blocks stay holes), and
+ only after the copy fully succeeds does an atomic swap move it into place. A failed copy —
+ including `ENOSPC` — leaves the live store exactly as it was. A crash mid-swap completes forward
+ on the next open.
+
+## Live-store caveats for external tools
+
+1. **Prefer snapshotting a `persist()` output, not the live directory.** `persist()` produces a
+ crash-consistent, immutable snapshot; running `tar` against a live, actively-written directory
+ can capture a torn mid-write state. If you must archive live, stop writes first (or accept that
+ the archive is only as consistent as the moment's flush state).
+2. **Never prune or "clean up" files inside a brain directory.** Index files that look stale or
+ redundant are load-bearing; the store protects its declared index families from in-process
+ deletion, but an external `rm` bypasses that fence. If space is the concern, `du -sh` first —
+ the allocated size is usually far smaller than it looks.
+3. **Verify restores by opening them.** `Brainy.load(path)` opens any snapshot or restored
+ directory read-only — the store verifies its own coherence at open and reports loudly if
+ anything is missing or torn.
diff --git a/docs/guides/find-limits.md b/docs/guides/find-limits.md
index 66418b80..4c7fd252 100644
--- a/docs/guides/find-limits.md
+++ b/docs/guides/find-limits.md
@@ -40,6 +40,10 @@ Brainy picks `maxLimit` from the first of these that's available:
Worked example: a 4 GB Cloud Run container picks priority 3 → `floor(4 GB × 0.25 / 25 KB) = floor(40 960) = 40 000` results. A 900 MB free-memory box on priority 4 gets `floor(900 MB / 25 KB) = ~36 000`.
+The cap is fixed at construction and never changes at runtime. Query timing is recorded
+for diagnostics only — a burst of slow queries cannot silently shrink the cap, and the
+auto-detected tiers (3 and 4) never go below a floor of 10 000.
+
> **Calibration note.** Pre-7.30.2 used 100 KB per result instead of 25 KB, which produced caps that were 4× too tight for typical workloads (an 8 KB / result reality). 7.30.2 recalibrated to match observed entity sizes; existing `limit: 10_000` safety patterns now pass silently on any reasonably-sized box.
## What happens when you exceed the cap
diff --git a/docs/guides/import-anything.md b/docs/guides/import-anything.md
index e0cd94ed..b1bb15ef 100644
--- a/docs/guides/import-anything.md
+++ b/docs/guides/import-anything.md
@@ -300,7 +300,10 @@ await brain.import(data, {
// Deduplication
enableDeduplication: true, // Check for duplicate entities (default: true)
deduplicationThreshold: 0.85, // Similarity threshold for duplicates (0-1, default: 0.85)
- // Note: Auto-disabled for imports >100 entities
+ // Notes: false disables BOTH the inline merge and the background pass that
+ // runs ~5 min after the last import (merged duplicates are deleted).
+ // The inline pass auto-disables for imports >100 entities (O(n²) cost);
+ // the background pass still covers those unless the flag is false.
// Performance
chunkSize: 100, // Batch size for processing (default: varies by operation)
diff --git a/docs/guides/import-quick-reference.md b/docs/guides/import-quick-reference.md
index 5a850a86..7837d49e 100644
--- a/docs/guides/import-quick-reference.md
+++ b/docs/guides/import-quick-reference.md
@@ -86,11 +86,22 @@ await brain.import(file, {
```typescript
await brain.import(file, {
- enableDeduplication: true, // Check for duplicates (default: false)
+ enableDeduplication: true, // Check for duplicates (default: true)
deduplicationThreshold: 0.85 // Similarity threshold (default: 0.85)
})
```
+Deduplication merges entities judged duplicates — the non-primary records are
+**deleted**. Set `enableDeduplication: false` to disable it entirely: the flag
+gates both the inline merge during import and the background pass that runs
+about 5 minutes after the last import.
+
+```typescript
+await brain.import(file, {
+ enableDeduplication: false // No merging, inline or background
+})
+```
+
### Import Tracking
Track and organize imports by project:
diff --git a/docs/guides/inspection.md b/docs/guides/inspection.md
index 015c5118..240e81ae 100644
--- a/docs/guides/inspection.md
+++ b/docs/guides/inspection.md
@@ -166,6 +166,34 @@ brainy inspect diff /data/brain-prod /data/brain-staging
Sample-based — for a full diff, dump both with `inspect dump` and compare
the JSONL.
+## Auditing graph-read truth
+
+`brain.auditGraph()` (8.6.0+) proves — or disproves — that relationship reads
+return canonical truth on a given brain, without mutating anything. It walks
+every stored relationship record, asks the same read path your application
+uses (`related()`, VFS `readdir`) with every visibility tier included, and
+classifies every discrepancy:
+
+```typescript
+const report = await brain.auditGraph()
+
+report.coherent // true = related()/readdir can be trusted on this brain
+report.missingFromReadsCount // records the read path omits — stale index
+report.danglingEndpointsCount // relationships whose endpoint entity is gone
+report.readOnlyCount // read-path edges with NO stored record — ghosts
+report.visibilityHiddenCount // internal/system edges hidden by design (not a fault)
+```
+
+Counts are always exact; the example lists (`missingFromReads`,
+`danglingEndpoints`, `readOnlyVerbIds`) are capped at `maxExamples`
+(default 100) and `truncatedExamples` says so when they are.
+
+Run it after any engine upgrade, restore, or migration. If it reports
+discrepancies, run `brain.repairIndex()` and audit again — a `coherent`
+report after the repair is the verified statement that the heal worked.
+Cost: one relationship-record walk plus one indexed read per distinct
+source entity — safe on a live brain.
+
## Repairing a corrupted store
If invariants fail and you suspect index corruption, `inspect repair`
diff --git a/docs/guides/installation.md b/docs/guides/installation.md
index e1bfb2b6..0a36f632 100644
--- a/docs/guides/installation.md
+++ b/docs/guides/installation.md
@@ -45,20 +45,20 @@ console.log('Brainy ready.')
## Native Acceleration (Optional)
-For production workloads, add Cortex for Rust-accelerated SIMD distance calculations and native embeddings:
+For production workloads, add Cor for Rust-accelerated SIMD distance calculations and native embeddings:
```bash
-npm install @soulcraft/cortex
+npm install @soulcraft/cor
```
```typescript
import { Brainy } from '@soulcraft/brainy'
-const brain = new Brainy({ plugins: ['@soulcraft/cortex'] })
+const brain = new Brainy({ plugins: ['@soulcraft/cor'] })
await brain.init() // native providers registered during init
```
-Cortex registers native (Rust/SIMD) vector, metadata, and graph engines behind the same Brainy `find()` API — no code changes, an optional dependency for production-scale workloads.
+Cor registers native (Rust/SIMD) vector, metadata, and graph engines behind the same Brainy `find()` API — no code changes, an optional dependency for production-scale workloads.
## Server-only since 8.0
diff --git a/docs/guides/optimistic-concurrency.md b/docs/guides/optimistic-concurrency.md
index a21a8af0..268bc5fa 100644
--- a/docs/guides/optimistic-concurrency.md
+++ b/docs/guides/optimistic-concurrency.md
@@ -180,3 +180,35 @@ Brainy 8.0 has exactly two write-coordination counters, at two granularities:
They compose: a `transact()` batch can carry per-entity `ifRev` checks *and* a whole-store `ifAtGeneration`; any failed check rejects the entire batch before anything is staged. Generations also power snapshots and time travel (`brain.now()`, `brain.asOf()`, `db.persist()`) — see the [consistency model](../concepts/consistency-model.md) and [Snapshots & Time Travel](./snapshots-and-time-travel.md).
A snapshot or historical view captures each entity *including* its `_rev` at that moment, so reading the past and writing back with `ifRev` against the live state works exactly as you'd hope: the write fails if the entity moved since the state you copied from.
+
+## The transact envelope: batch size, budget, and bulk imports
+
+`transact()` applies its batch atomically under one commit — which means the whole batch
+shares one **apply budget**. Since 8.7.0 the budget scales with the batch:
+`max(30 s, opCount × 2 s)`, or exactly what you pass as `timeoutMs`. A tripped budget rolls
+the entire batch back (nothing partial survives) and throws a retryable
+`TransactionTimeoutError` that names the operation it stopped at, the batch size, and the
+elapsed vs budgeted time — a diagnosis, not just a failure:
+
+```
+Transaction timed out at operation 41/120 ('add') — 246012ms elapsed, budget 240000ms.
+The batch rolled back atomically; retry with a higher timeoutMs or a smaller batch.
+```
+
+Practical envelope guidance for bulk work:
+
+1. **Precompute embeddings outside the commit path.** Embedding inside `transact()` spends
+ the budget on model inference. Use `brain.embedBatch(texts)` and pass each vector via
+ the op's `vector` field — the commit then pays only storage costs, and a retried batch
+ never re-pays inference. (The win is *where* the inference happens, not raw embedding
+ throughput: on the default WASM engine, batch and sequential embedding measure
+ comparably, ~160 ms/text; native embedding providers may batch faster.)
+2. **Chunk very large imports** into batches of a few hundred ops with one `transact()`
+ each. You lose whole-import atomicity but keep per-chunk atomicity, bounded memory, and
+ resumability — pair with `ifAbsent` upserts so a retried chunk is idempotent.
+3. **Slow disks change the math, not the contract.** On network-attached storage a single
+ op can cost ~2 s (canonical write + fsync + index maintenance). The scaled default
+ absorbs that; pass an explicit `timeoutMs` only when you know better than the scale.
+4. **`addMany`/`relateMany` are the convenience tier** — they chunk and batch-embed for
+ you, with per-item error reporting instead of batch atomicity. Choose by what you need:
+ atomic-all-or-nothing → `transact()`; resilient bulk load → `addMany`.
diff --git a/docs/guides/reacting-to-changes.md b/docs/guides/reacting-to-changes.md
new file mode 100644
index 00000000..7367a5e3
--- /dev/null
+++ b/docs/guides/reacting-to-changes.md
@@ -0,0 +1,117 @@
+---
+title: Reacting to Changes
+slug: guides/reacting-to-changes
+public: true
+category: guides
+template: guide
+order: 11
+description: Subscribe to every committed mutation with `brain.onChange` — the in-process change feed behind live UIs, cache invalidation, and realtime sync. Covers the event shape, delivery guarantees, and catch-up patterns.
+next:
+ - guides/optimistic-concurrency
+ - guides/snapshots-and-time-travel
+---
+
+# Reacting to Changes
+
+`brain.onChange(cb)` is Brainy's in-process change feed: subscribe once and
+receive one event per committed mutation — **every** mutation, regardless of
+how it happened. Direct calls, batch methods, `transact()`, imports, and
+Virtual Filesystem writes all funnel through the same commit point the feed is
+emitted from, so nothing slips past it.
+
+```ts
+const off = brain.onChange((e) => {
+ if (e.kind === 'entity') {
+ console.log(`${e.op} ${e.entity?.type} ${e.id} @ generation ${e.generation}`)
+ }
+})
+
+await brain.add({ data: 'Ada Lovelace', type: 'person' })
+// → "add person 0198... @ generation 42"
+
+off() // unsubscribe when done
+```
+
+## The event
+
+```ts
+interface BrainyChangeEvent {
+ kind: 'entity' | 'relation' | 'store'
+ op: 'add' | 'update' | 'remove' | 'relate' | 'unrelate' | 'updateRelation'
+ | 'clear' | 'restore'
+ id?: string
+ entity?: { id: string; type: string; subtype?: string;
+ metadata: Record; service?: string }
+ relation?: { id: string; from: string; to: string; type: string;
+ metadata?: Record }
+ generation?: number
+ timestamp: number
+}
+```
+
+- **Entity events** (`add` / `update` / `remove`) carry the post-commit indexed
+ view — `type`, `subtype`, and the full custom `metadata`, so you can match
+ your own `where`-style filters against events without a read.
+- **Deletes are fully described.** A `remove` or `unrelate` event carries the
+ record's *last committed state* (sourced from the commit's own history
+ record), not just an id.
+- **Batches emit per item.** `addMany` / `updateMany` / `relateMany` /
+ `removeMany` emit one event per affected record; a `transact()` batch emits
+ one event per item, all sharing the batch's single `generation`.
+- **Cascades are visible.** Removing an entity also emits `unrelate` for each
+ relationship the delete cascaded to.
+- **Store-level events** (`kind: 'store'`) fire for the two wholesale
+ operations — `clear()` and `restore()` — and mean *"everything may have
+ changed; refetch what you care about."*
+
+## Delivery guarantees
+
+- **Post-commit only.** An aborted write — a losing
+ [`ifRev` compare-and-swap](optimistic-concurrency.md), a rejected
+ transaction — never emits. If you received the event, the write is durable.
+- **Commit-ordered.** Events arrive in the order writes committed;
+ `generation` is monotonic.
+- **Asynchronous, never blocking.** Delivery happens in a microtask after the
+ write completes. A slow listener cannot delay a write; a throwing listener
+ is logged and isolated from other listeners.
+- **Zero overhead when unused.** With no subscribers, the write path does no
+ event work at all.
+- **Fire-and-forget.** There is no replay or backpressure. For catch-up after
+ a disconnect, use the `generation` on each event together with
+ [`asOf()` / the transaction log](snapshots-and-time-travel.md): record the
+ last generation you processed, and on reconnect diff from there. For file
+ content specifically, `vfs.readFile(path, { asOf })` and
+ `vfs.history(path)` are the temporal read — see
+ [Snapshots & Time Travel](snapshots-and-time-travel.md).
+
+## Patterns
+
+**Cache invalidation** — drop cached reads for whatever changed:
+
+```ts
+brain.onChange((e) => {
+ if (e.kind === 'store') return cache.clear()
+ if (e.id) cache.delete(e.id)
+})
+```
+
+**Live queries (notify-and-refetch)** — re-run a query when a relevant change
+lands, rather than diffing incrementally:
+
+```ts
+brain.onChange((e) => {
+ if (e.kind === 'entity' && e.entity?.type === 'order') {
+ refreshOpenOrdersView() // debounce as needed
+ }
+})
+```
+
+**Forwarding to other processes** — the feed is in-process by design. To push
+changes to browsers or other services, forward events through your own
+transport (WebSocket, SSE) from the process that owns the brain.
+
+## Lifecycle
+
+`onChange` returns an unsubscribe function — call it when tearing down a
+subscriber (for example, when evicting a pooled instance). `brain.close()`
+drops all listeners; no events are delivered for or after `close()`.
diff --git a/docs/guides/snapshots-and-time-travel.md b/docs/guides/snapshots-and-time-travel.md
index d9f0e264..490aecab 100644
--- a/docs/guides/snapshots-and-time-travel.md
+++ b/docs/guides/snapshots-and-time-travel.md
@@ -9,6 +9,7 @@ description: Recipes for the Db API — instant backups with persist(), restore,
next:
- concepts/consistency-model
- guides/optimistic-concurrency
+ - guides/external-backups
---
# Snapshots & Time Travel
@@ -41,6 +42,10 @@ bytes. Cross-device targets fall back to per-file byte copies, and
persisting an in-memory brain serializes it to the same directory layout —
a real, durable store.
+> Archiving a brain directory with **external tools** (`tar`, `rsync`, `cp`)?
+> Some index files are sparse and can explode to their apparent size under a
+> naive copy — see [External Backups & Sparse Storage](/docs/guides/external-backups).
+
Two things to know:
- `persist()` requires the view to still be the store's **latest**
@@ -339,8 +344,12 @@ For per-entity write coordination (rather than whole-store history), the
## Keeping history bounded
Under Model-B every write is a generation, so history can grow quickly —
-Brainy auto-compacts on every `flush()`/`close()` under the **`retention`**
-knob (configured on the constructor):
+Brainy auto-compacts at `close()` (time-bounded per pass) under the
+**`retention`** knob (configured on the constructor). Since 8.9.0, `flush()`
+never compacts: flushing is durability work and costs only what the current
+window's writes cost, regardless of history backlog. A long-lived writer that
+never closes keeps its history until its next explicit `compactHistory()` —
+schedule one in your maintenance window if you run bounded retention:
```typescript
// Zero-config: ADAPTIVE — keep as much history as free disk/RAM allows,
@@ -354,10 +363,13 @@ new Brainy({ retention: 'all' })
new Brainy({ retention: { maxGenerations: 1000, maxAge: 7 * 86_400_000, maxBytes: 512 * 1024 ** 2 } })
```
-Reclaim manually at any time (the same caps):
+Reclaim manually at any time (the same caps, plus an optional per-pass time
+budget for maintenance windows — an early stop is a consistent prefix and the
+next pass resumes):
```typescript
await brain.compactHistory({ maxGenerations: 100, maxAge: 7 * 24 * 60 * 60 * 1000 })
+await brain.compactHistory({ maxBytes: 512 * 1024 ** 2, timeBudgetMs: 10_000 })
```
Compaction never breaks a pinned read — record-sets are reclaimed only when
@@ -366,6 +378,61 @@ are done with (including the ones `transact()` returns), and `persist()` any
generation you want to keep beyond the retention window: snapshots are
self-contained and unaffected by compaction.
+## Time travel for files (the VFS)
+
+Since 8.2.0, time travel covers Virtual Filesystem **content**, not just
+entity records. File bytes are retention-protected: a content blob referenced
+by any generation inside the retention window is never reclaimed, so reading
+the past always returns the exact bytes — never a stale field or a
+dangling hash.
+
+**`vfs.readFile(path, { asOf })`** takes a generation number or a `Date` and
+returns the file's exact bytes as they stood then. It resolves the path's
+current entity, then materializes its state at the target generation — so it
+answers *"what did the file at this path hold at that point?"* It bypasses
+the content cache; the `encoding` option still applies. Asking about a
+generation before the file existed throws the usual not-found error, and
+asking past the retention window's compaction horizon throws a
+compacted-generation error.
+
+**`vfs.history(path)`** returns the file's versions inside the retention
+window, oldest first — one `FileVersion` per generation that wrote the file,
+the newest entry being the current state:
+
+```typescript
+// A CMS page evolves…
+await brain.vfs.writeFile('/pages/home.json', '{"title":"Launch"}')
+await brain.vfs.writeFile('/pages/home.json', '{"title":"Launch v2"}')
+await brain.vfs.writeFile('/pages/home.json', '{"title":""}') // bad deploy!
+
+// Every version is listed and readable:
+const versions = await brain.vfs.history('/pages/home.json')
+// → [{ generation, timestamp, hash, size, mimeType? }, …] ascending
+
+const good = versions[versions.length - 2]
+const bytes = await brain.vfs.readFile('/pages/home.json', {
+ asOf: good.generation
+})
+
+// Restore = write the old bytes back. This is a NEW write (a new
+// generation) — history is never rewritten, so the bad version stays
+// visible in the audit trail.
+await brain.vfs.writeFile('/pages/home.json', bytes)
+```
+
+Two lifecycle consequences worth stating plainly:
+
+- **Deleting or overwriting a file no longer frees its bytes immediately.**
+ Old content lives until history compaction reclaims the generations that
+ reference it — the same `retention` budget that bounds all Model-B history
+ (and pinned views are exempt, exactly as above). Size your `retention` for
+ the file-version depth you want; `retention: 'all'` keeps every version of
+ every file forever.
+- **After `compactHistory()` reclaims a generation, its file versions are
+ gone** and their bytes are physically reclaimed. (This also fixed a
+ pre-8.2.0 defect where overwritten content was never reclaimed at all — an
+ unbounded silent leak.)
+
## From branches to values
If you used the pre-8.0 `fork`/`checkout`/`commit`/`versions` surface, every
diff --git a/docs/guides/subtypes-and-facets.md b/docs/guides/subtypes-and-facets.md
index 0b38b146..ff5de320 100644
--- a/docs/guides/subtypes-and-facets.md
+++ b/docs/guides/subtypes-and-facets.md
@@ -332,7 +332,7 @@ await brain.updateRelation({ id: relationId, weight: 0.5, confidence: 0.9 })
`find({ connected, subtype })` filters traversal edges by their subtype. Composes with `via` (verb-type filter):
```typescript
-// All direct reports two hops deep (will be supported in Cortex; for now depth-1)
+// All direct reports two hops deep (will be supported in Cor; for now depth-1)
const directChain = await brain.find({
connected: {
from: ceoId,
@@ -343,7 +343,7 @@ const directChain = await brain.find({
})
```
-Multi-hop subtype filtering (`depth > 1`) lights up on the Cortex native path; the JS path throws today rather than return incorrect partial results.
+Multi-hop subtype filtering (`depth > 1`) lights up on the Cor native path; the JS path throws today rather than return incorrect partial results.
### Counts
diff --git a/docs/guides/upgrading-7-to-8.md b/docs/guides/upgrading-7-to-8.md
new file mode 100644
index 00000000..a3c64fb9
--- /dev/null
+++ b/docs/guides/upgrading-7-to-8.md
@@ -0,0 +1,186 @@
+---
+title: Upgrading from 7.x to 8.0
+slug: guides/upgrading-7-to-8
+public: true
+category: guides
+template: guide
+order: 10
+description: What the one-time 7→8 on-disk migration does, how 8.0 automatically recovers Virtual Filesystem content that older layouts stored in the removed copy-on-write area, and how to verify (or force) that recovery.
+next:
+ - guides/storage-adapters
+ - guides/inspection
+---
+
+# Upgrading from 7.x to 8.0
+
+Opening a 7.x on-disk store with Brainy 8.0 runs a **one-time, in-place layout
+migration** the first time the store is opened. It is automatic (`autoMigrate`
+defaults to `true`), it runs once, and it stamps a marker so every later open is
+a no-op.
+
+Almost everything about the upgrade is transparent: your entities, relationships,
+metadata, and indexes migrate and rebuild without any action on your part. This
+guide covers the **one case that needs attention** — Virtual Filesystem (VFS)
+content — and how 8.0 recovers it for you.
+
+## TL;DR
+
+- **Just upgrade to `@soulcraft/brainy@8.0.12` (or later) and open the store.**
+ If a previous upgrade left VFS content stranded, 8.0.12 **heals it on open**,
+ with no operator action.
+- Want to force or script it? Call **`await brain.vfs.adoptOrphanedBlobs()`**.
+- The recovery is **non-destructive and idempotent** — it copies, never moves,
+ and running it twice is a no-op.
+
+## What the migration does
+
+The 7.x on-disk layout stored entities under a per-branch path
+(`branches//entities/...`). 8.0 uses a flat layout (`entities/...`). On
+first open, Brainy:
+
+1. Collapses `branches//entities/*` into the flat `entities/*` layout.
+2. Rebuilds the derived indexes (vector, graph, metadata) and count rollups from
+ the canonical entities.
+3. Stamps `_system/migration-layout.json` so re-opening is a no-op.
+4. Takes an automatic **pre-upgrade backup** of the directory before it starts,
+ and removes it once the upgrade is verified complete (see
+ [The safety net](#the-safety-net-pre-upgrade-backup) below).
+
+The log line to expect (once per store):
+
+```
+[brainy] Migrating a 7.x branch layout (branches/main) to the 8.0 flat layout
+ in place — 13175 entity files. This runs once; back up the directory first if
+ you need a rollback (8.0 does not keep the old layout).
+```
+
+## The one thing that needs recovery: VFS content
+
+If your application uses the Virtual Filesystem (VFS) to store file content (for
+example, a CMS that keeps page documents at paths like
+`/pages/homepage/page.json`), that content is held as **content blobs**.
+
+7.x kept those blobs in the branch system's **copy-on-write area** (`_cow/`).
+8.0 removed the branch/copy-on-write system, and its content blobs live in the
+content-addressed store (`_cas/`). The layout migration moves entities — it does
+**not** move the VFS content blobs. Left alone, a 7.x store's VFS blobs would
+remain in `_cow/`, and a read of one would throw:
+
+```
+VFS: Cannot read blob for /pages/homepage/page.json:
+ Blob metadata not found: 6b87cfb71ad2f04602a5c157214dc42000...
+```
+
+### 8.0.12 recovers them automatically
+
+Brainy 8.0.12 adds an **on-open recovery pass** that runs right after the layout
+migration. It scans `_cow/`, and for every content blob whose `blob:` +
+`blob-meta:` pair is not already in `_cas/`, it copies **both** across into the
+8.0 store. It:
+
+- **Heals a fresh 7→8 upgrade** (where `_cow/` still holds the blobs), **and**
+- **Heals a store already upgraded by an earlier 8.0.x** that stranded them —
+ the recovery is gated on the presence of `_cow/` and its own marker, not on
+ the layout-migration marker, so an already-migrated store is still healed.
+- Is a **cheap no-op** on a native-8.0 or fresh store (there is no `_cow/`), and
+ on a store already healed (a marker records completion so later opens skip the
+ scan).
+
+So the operator action for a stranded store is simply: **upgrade to 8.0.12 and
+open it.**
+
+```ts
+import { Brainy } from '@soulcraft/brainy'
+
+// Opening the store is all that is required — recovery runs during init().
+const brain = new Brainy({ storage: { type: 'filesystem', path: '/data/my-store' } })
+await brain.init()
+
+// The previously-failing read now succeeds.
+const page = await brain.vfs.readFile('/pages/homepage/page.json')
+```
+
+On a store that needed recovery you will see:
+
+```
+[brainy] Recovered 337 VFS content blob(s) stranded by a 7→8 upgrade
+ (adopted _cow/ → _cas/ in place).
+```
+
+### Forcing recovery explicitly
+
+If you would rather run the recovery deliberately (for example, in an upgrade
+script that asserts a clean result before flipping traffic), call it directly:
+
+```ts
+const result = await brain.vfs.adoptOrphanedBlobs()
+// → { cowBlobs, adopted, alreadyPresent, incomplete }
+console.log(`adopted ${result.adopted}, already present ${result.alreadyPresent}`)
+if (result.incomplete > 0) {
+ // One or more _cow/ blobs are missing their bytes or metadata — investigate
+ // _cow/ before discarding your own backup. See "The safety net" below.
+}
+```
+
+`adoptOrphanedBlobs()` self-initializes the brain, so it is safe to call
+immediately after construction. It returns:
+
+| Field | Meaning |
+| ---------------- | ---------------------------------------------------------------- |
+| `cowBlobs` | Distinct content-blob hashes found in `_cow/`. |
+| `adopted` | Newly copied into `_cas/` on this call. |
+| `alreadyPresent` | Already in `_cas/` (a prior open or run adopted them). |
+| `incomplete` | A `_cow/` blob missing its bytes *or* its metadata — **skipped** rather than half-adopted. |
+
+## Verifying recovery
+
+After opening under 8.0.12, confirm a previously-failing path reads:
+
+```ts
+const content = await brain.vfs.readFile('/pages/homepage/page.json')
+console.log(content.toString().slice(0, 80))
+```
+
+If you scripted it with `adoptOrphanedBlobs()`, a clean result is
+`incomplete === 0` and `adopted + alreadyPresent === cowBlobs`.
+
+## The safety net: pre-upgrade backup
+
+Brainy takes an automatic pre-upgrade backup and removes it once the upgrade is
+**verified complete**. In 8.0.12 that verification includes VFS content: if the
+blob recovery reports `incomplete > 0`, the completion marker is **not** stamped
+(the next open retries) and the **pre-upgrade backup is retained** so you still
+have a rollback while blobs remain unaccounted for. You will see:
+
+```
+[brainy] VFS blob recovery adopted N blob(s) but M orphaned _cow/ blob(s) are
+ missing their bytes or metadata and were left in place. The pre-upgrade backup
+ is being retained; inspect _cow/ before discarding it.
+```
+
+The recovery never deletes anything from `_cow/`, so the original blobs stay in
+place for inspection or a manual rollback regardless.
+
+## Who is affected
+
+**Any 7.x store that used the VFS to store file content** (so it has a `_cow/`
+area) is a candidate for stranded blobs on a 7→8 upgrade. Stores that never used
+the VFS have no `_cow/` content blobs and are unaffected.
+
+Because the recovery is **gated on the presence of `_cow/`**, it is
+self-selecting and safe to roll out everywhere:
+
+- On a store with stranded blobs → it adopts them.
+- On a native-8.0, fresh, or non-VFS store → it is a no-op on the existence
+ check.
+
+There is no configuration to set and nothing to opt into. Upgrading to 8.0.12
+and opening each store is sufficient.
+
+## Rollback
+
+The recovery is copy-only, so no rollback of the recovery itself is ever needed.
+If you need to roll back the **whole** 7→8 upgrade, restore the directory from
+your pre-upgrade backup (retained automatically while recovery is incomplete, or
+your own snapshot) and pin `@soulcraft/brainy@7.x`. 8.0 does not keep the old
+branch layout in place, so a directory-level restore is the rollback path.
diff --git a/docs/performance-envelopes.md b/docs/performance-envelopes.md
new file mode 100644
index 00000000..d29677e3
--- /dev/null
+++ b/docs/performance-envelopes.md
@@ -0,0 +1,83 @@
+---
+title: Performance Envelopes
+slug: guides/performance-envelopes
+public: true
+category: guides
+template: guide
+order: 40
+description: Measured per-operation latency envelopes at stated scales — what to expect, on what hardware, and exactly how each number was produced.
+next:
+ - guides/find-limits
+---
+
+# Performance Envelopes
+
+Every number on this page is **measured, never projected** — produced by the script
+cited at the bottom, against the built package (the artifact you install), on the stated
+hardware. Each entry says what was measured, at what scale, on which storage backend.
+When a release touches a measured path, that operation is re-measured and this page
+updates in the same release.
+
+Two scopes to keep straight:
+
+- **These envelopes are the pure-JS engine** (no native accelerator registered) on
+ filesystem storage. This is the floor every deployment gets from `npm install` alone.
+- **Accelerated deployments** (the optional native provider) publish their own numbers —
+ this page never claims them.
+
+## Read operations
+
+Reads are where the architecture pays off: after the write path has done its indexing
+work, queries answer from purpose-built indexes without scanning.
+
+| Operation | 1,000 entities | 10,000 entities | Notes |
+|---|---|---|---|
+| `get(id)` (warm) | p50 < 0.1ms | p50 < 0.1ms | served from cache/metadata index |
+| `find` (metadata: indexed equality + range, limit 100) | p50 1.0ms · p95 1.8ms | p50 7.0ms · p95 8.9ms | column-store bitmap paths |
+| `related(id)` (per-node adjacency) | p50 < 0.1ms · p95 0.2ms | p50 < 0.1ms | LSM adjacency index — O(degree), scale-independent |
+| `find` (semantic: embed + HNSW, 1k docs) | p50 178ms · p95 393ms | — | dominated by WASM query embedding (measured on a machine under concurrent load — treat the p95 as an upper bound); the vector search itself is single-digit ms |
+
+## Write operations
+
+Under Model-B **every write is its own durable generation** — a single-op `add` pays
+serialization, before-image staging, and fsync before it acks. That durability is priced
+into the write path visibly, by design:
+
+| Operation | 1,000 entities | 10,000 entities | Notes |
+|---|---|---|---|
+| `add` (single-op) | p50 167ms · p95 171ms | p50 165ms · p95 172ms | full durable generation per write — flat across scale |
+| `addMany` (bulk) | ~163ms/entity | ~187ms/entity | **currently per-item commits** — see the honest note below |
+| `relateMany` | ~0.8ms/edge | ~0.9ms/edge | edges batch efficiently today |
+| `flush` (steady-state, 1 pending write) | p50 8ms · p95 10ms | p50 45ms · p95 52ms | durability-only since 8.9.0 — cost no longer depends on history backlog or retention mode |
+
+**The honest note on bulk writes:** `addMany` today commits each item as its own
+generation (the same durability as single-op `add`, serialized by the single-writer
+lock), so bulk-load cost is N × single-op cost. Batched chunk commits (one generation
+and one fsync window per chunk, as `removeMany` already does) are designed into the
+unified-commit work on the current roadmap. Until that ships, size bulk imports
+accordingly — 10k entities is minutes, not seconds, on filesystem storage.
+
+## Open / close
+
+| Operation | 1,000 entities | 10,000 entities | Notes |
+|---|---|---|---|
+| `open` (empty store) | ~560ms | ~190ms | includes embedder initialization |
+| `open` (warm, populated, clean shutdown) | 763ms | 4.9s | pure-JS vector index load dominates and grows with entity count; the native accelerator exists precisely to remove this |
+| `close` | bounded | bounded | auto-compaction pass is time-bounded (~5s max) since 8.9.0 |
+
+A store that was NOT cleanly closed pays index rebuilds on top of the warm-open
+number (tens of seconds at 10k) — clean shutdown is worth engineering for.
+
+## How these were produced
+
+- **Hardware**: Intel Core i9-14900HX (32 threads), 62GB RAM, NVMe, Linux, Node v22.
+- **Backend**: `storage: { type: 'filesystem' }`, pure JS (no native providers).
+- **Embeddings**: deterministic stub for non-semantic ops (isolates engine cost);
+ the real WASM embedder for the semantic row (that's what you'll run).
+- **Method**: p50/p95 over 50–200 samples per op against the built `dist/`;
+ the measuring script ships in the repo history and re-runs per release.
+
+Numbers on different hardware will differ; the *shape* (sub-2ms indexed reads,
+~160ms embedding-bound semantic queries, durability-priced writes) is the envelope
+you should hold your deployment against. If your measurements diverge from these
+shapes by an order of magnitude, something is wrong — file it.
diff --git a/package-lock.json b/package-lock.json
index d31b11f0..fb9262e9 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "@soulcraft/brainy",
- "version": "8.0.0-rc.9",
+ "version": "8.9.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@soulcraft/brainy",
- "version": "8.0.0-rc.9",
+ "version": "8.9.0",
"license": "MIT",
"dependencies": {
"@msgpack/msgpack": "^3.1.2",
diff --git a/package.json b/package.json
index 5c6c1868..7366ce98 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "@soulcraft/brainy",
- "version": "8.0.0-rc.9",
+ "version": "8.9.0",
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
"main": "dist/index.js",
"module": "dist/index.js",
diff --git a/scripts/push-docs.js b/scripts/push-docs.js
new file mode 100644
index 00000000..699d332b
--- /dev/null
+++ b/scripts/push-docs.js
@@ -0,0 +1,116 @@
+#!/usr/bin/env node
+/**
+ * @module scripts/push-docs
+ * @description Push this repo's PUBLIC docs to the soulcraft.com docs ingest
+ * door after an npm publish (VENUE-DOCS-RELEASE-PUSH — retires the old
+ * build-time docs sync).
+ *
+ * Contract (mirrors the reference implementation on the serving side):
+ * POST {base}/api/docs/ingest
+ * headers: x-service-secret: $DOCS_INGEST_SECRET, Content-Type: application/json
+ * body: { docs: [{ slug, title, markdown, nav: { order, section } }] }
+ * batches of 10, idempotent per slug.
+ *
+ * A doc is public iff its frontmatter has `public: true` AND a `slug`. The
+ * frontmatter is stripped; `category` → nav.section, `order` → nav.order.
+ *
+ * Deliberately NOT pushed: the combined /docs landing index. It spans BOTH
+ * engine corpora (this repo's and the native accelerator's), so a per-repo
+ * push would clobber the union — the index is authored on the serving side.
+ *
+ * Env: DOCS_INGEST_SECRET (required), DOCS_INGEST_BASE (default
+ * https://soulcraft.com). Exits 0 with a LOUD warning when the secret is
+ * absent (the npm publish has already happened; the serving side runs its
+ * interim sync on request) and exits 1 when a push actually fails — the docs
+ * site would silently trail npm otherwise, and that must be visible.
+ */
+import * as fs from 'node:fs'
+import * as path from 'node:path'
+
+const BASE = (process.env.DOCS_INGEST_BASE || 'https://soulcraft.com').replace(/\/+$/, '')
+const SECRET = process.env.DOCS_INGEST_SECRET
+const DOCS_DIR = path.join(path.dirname(new URL(import.meta.url).pathname), '..', 'docs')
+const BATCH = 10
+
+if (!SECRET) {
+ console.warn(
+ '⚠️ DOCS PUSH SKIPPED: DOCS_INGEST_SECRET is not set.\n' +
+ ' soulcraft.com/docs now TRAILS this npm release until docs are pushed.\n' +
+ ' Either export DOCS_INGEST_SECRET and re-run `node scripts/push-docs.js`,\n' +
+ ' or ping venue on VENUE-DOCS-RELEASE-PUSH for the interim sync.'
+ )
+ process.exit(0)
+}
+
+/** Minimal frontmatter split — returns [meta, body] or [null, raw]. */
+function parseFrontmatter(raw) {
+ const m = raw.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/)
+ if (!m) return [null, raw]
+ const meta = {}
+ for (const line of m[1].split('\n')) {
+ const kv = line.match(/^(\w[\w-]*):\s*(.*)$/)
+ if (kv) meta[kv[1]] = kv[2].trim().replace(/^["']|["']$/g, '')
+ }
+ return [meta, m[2]]
+}
+
+const docs = []
+;(function walk(dir) {
+ for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+ const full = path.join(dir, entry.name)
+ if (entry.isDirectory()) walk(full)
+ else if (entry.name.endsWith('.md')) {
+ const [meta, body] = parseFrontmatter(fs.readFileSync(full, 'utf-8'))
+ if (!meta || meta.public !== 'true' || !meta.slug) continue
+ docs.push({
+ slug: meta.slug,
+ title: meta.title || meta.slug,
+ markdown: body.trim(),
+ nav: {
+ order: Number.parseInt(meta.order || '99', 10) || 99,
+ section: meta.category || 'guides'
+ }
+ })
+ }
+ }
+})(DOCS_DIR)
+
+if (docs.length === 0) {
+ console.error('❌ DOCS PUSH FAILED: zero public docs collected — refusing to push an empty corpus.')
+ process.exit(1)
+}
+docs.sort((a, b) => a.slug.localeCompare(b.slug))
+console.log(`Pushing ${docs.length} public docs to ${BASE}/api/docs/ingest …`)
+
+let failed = false
+for (let i = 0; i < docs.length; i += BATCH) {
+ const batch = docs.slice(i, i + BATCH)
+ try {
+ const res = await fetch(`${BASE}/api/docs/ingest`, {
+ method: 'POST',
+ headers: {
+ 'x-service-secret': SECRET,
+ 'Content-Type': 'application/json',
+ 'User-Agent': 'brainy-docs-push/1.0'
+ },
+ body: JSON.stringify({ docs: batch }),
+ signal: AbortSignal.timeout(120_000)
+ })
+ if (!res.ok) {
+ throw new Error(`HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)
+ }
+ console.log(` batch ${i / BATCH + 1}: ${batch.map((d) => d.slug).join(', ')} → ok`)
+ } catch (err) {
+ failed = true
+ console.error(` batch ${i / BATCH + 1} FAILED: ${err instanceof Error ? err.message : err}`)
+ }
+}
+
+if (failed) {
+ console.error(
+ '❌ DOCS PUSH INCOMPLETE — soulcraft.com/docs may trail npm. ' +
+ 'Re-run `node scripts/push-docs.js` or ping venue on VENUE-DOCS-RELEASE-PUSH.'
+ )
+ process.exit(1)
+}
+console.log('✅ Docs pushed.')
diff --git a/scripts/release.sh b/scripts/release.sh
index 0e6a9c43..7d860564 100755
--- a/scripts/release.sh
+++ b/scripts/release.sh
@@ -196,6 +196,18 @@ else
fi
echo -e "${GREEN}✅ GitHub release created${NC}\n"
+# Step 12: Push public docs to the soulcraft.com docs ingest door
+# (VENUE-DOCS-RELEASE-PUSH). Skips with a loud warning when
+# DOCS_INGEST_SECRET is unset; fails loudly (without undoing the publish —
+# that already happened) when a push errors, so the docs site never
+# silently trails npm.
+echo -e "${BLUE}1️⃣2️⃣ Pushing public docs to soulcraft.com/docs...${NC}"
+if node scripts/push-docs.js; then
+ echo -e "${GREEN}✅ Docs push step done${NC}\n"
+else
+ echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n"
+fi
+
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}"
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
diff --git a/src/aggregation/AggregationIndex.ts b/src/aggregation/AggregationIndex.ts
index bf9dc5de..f9382218 100644
--- a/src/aggregation/AggregationIndex.ts
+++ b/src/aggregation/AggregationIndex.ts
@@ -29,6 +29,7 @@ import { matchesMetadataFilter } from '../utils/metadataFilter.js'
import { compareCodePoints } from '../utils/collation.js'
import { bucketTimestamp } from './timeWindows.js'
import { NounType } from '../types/graphTypes.js'
+import { prodLog } from '../utils/logger.js'
/** Persistence key for aggregate definitions */
const DEFINITIONS_KEY = '__aggregation_definitions__'
@@ -87,10 +88,18 @@ function matchesSource(entity: Record, source: AggregateDefinit
if (entity.service !== source.service) return false
}
- // Metadata where filter — match against the entity's metadata sub-object
+ // Where filter — resolve each filtered field through resolveEntityField,
+ // the SAME single source of truth groupBy uses (top-level standard fields
+ // + custom metadata). Matching only the metadata sub-object made
+ // where:{subtype}/{visibility}/… a silent no-op: reserved fields never
+ // live in the custom bag, so those filters could never match anything.
if (source.where && Object.keys(source.where).length > 0) {
- const metadata = (entity.metadata ?? entity) as Record
- if (!matchesMetadataFilter(metadata, source.where)) return false
+ const e = entity as unknown as HNSWNounWithMetadata
+ const resolved: Record = {}
+ for (const key of Object.keys(source.where)) {
+ resolved[key] = resolveEntityField(e, key)
+ }
+ if (!matchesMetadataFilter(resolved, source.where)) return false
}
return true
@@ -269,7 +278,7 @@ function recomputeMinMaxFromCounts(state: MetricState): void {
/**
* Exact percentile over a value multiset, using linear interpolation between closest ranks
* (numpy 'linear' / type-7): `rank = p·(n−1)`, interpolating between the floor and ceil
- * positions of the sorted expansion. Mirrors Cortex's `MetricState::percentile` bit-for-bit.
+ * positions of the sorted expansion. Mirrors Cor's `MetricState::percentile` bit-for-bit.
*/
function computePercentile(valueCounts: Record, count: number, p: number): number {
if (count === 0) return 0
@@ -327,6 +336,30 @@ export class AggregationIndex {
/** Track aggregates with stale MIN/MAX (need lazy recompute) */
private staleMinMax = new Map>()
+ /** Resolves when init() has finished loading persisted definitions/state. */
+ private initPromise: Promise | null = null
+
+ /** True once init() has settled (success or failure). */
+ private initDone = false
+
+ /**
+ * Aggregates registered by the app before init() finished loading persisted
+ * state, awaiting reconciliation: init() adopts the persisted state when the
+ * definition hash matches; anything left unadopted when init settles resolves
+ * to a backfill. Deciding backfill eagerly at define time was the boot-order
+ * bug that wiped valid persisted state on every restart — the synchronous
+ * defineAggregate() always beats the async init().
+ */
+ private pendingAdopt = new Set()
+
+ /**
+ * In-flight rescan targets. While a name has a staging map, ALL
+ * contributions (the walk's and concurrent write hooks') land there instead
+ * of the live map; the live map keeps serving until {@link finishBackfill}
+ * swaps the staging map in atomically.
+ */
+ private backfillStaging = new Map>()
+
constructor(storage: StorageAdapter, nativeProvider?: AggregationProvider) {
this.storage = storage
this.nativeProvider = nativeProvider
@@ -336,21 +369,128 @@ export class AggregationIndex {
/**
* Initialize: load persisted definitions and state, detect changes, rebuild stale.
+ *
+ * Idempotent — repeated calls return the same promise. Definitions registered
+ * *before* this completes (the normal boot order: `defineAggregate()` is
+ * synchronous and always beats this async load) are reconciled rather than
+ * clobbered: the app's definition wins, and its persisted state is adopted
+ * when the definition hash matches — backfill happens only on a real change.
*/
- async init(): Promise {
+ init(): Promise {
+ if (!this.initPromise) {
+ this.initPromise = this.loadPersisted().finally(() => {
+ this.resolvePendingAdoptToBackfill()
+ this.initDone = true
+ })
+ }
+ return this.initPromise
+ }
+
+ /**
+ * Await the persisted-state load (if one was started) and settle every
+ * pending adoption decision. After this resolves, `getPendingBackfills()`
+ * is authoritative: a name is listed iff it genuinely needs a rescan.
+ * Query paths must await this before consulting backfill state.
+ */
+ async ready(): Promise {
+ if (this.initPromise) {
+ try {
+ await this.initPromise
+ } catch {
+ // The owner already surfaced the load failure loudly; backfill covers.
+ }
+ }
+ this.resolvePendingAdoptToBackfill()
+ }
+
+ /**
+ * Any definition still awaiting state adoption has no persisted state to
+ * adopt (or init never ran / failed) — it must backfill.
+ */
+ private resolvePendingAdoptToBackfill(): void {
+ if (this.pendingAdopt.size > 0) {
+ prodLog.info(
+ `[Aggregation] no adoptable persisted state for: ${Array.from(this.pendingAdopt).join(', ')} — flagged for backfill`
+ )
+ }
+ for (const name of this.pendingAdopt) this.needsBackfill.add(name)
+ this.pendingAdopt.clear()
+ }
+
+ /**
+ * May this persisted state be ADOPTED? When the store exposes its committed
+ * watermark, the state's `sourceGeneration` must EQUAL it: behind means
+ * later writes are missing from the state (unclean shutdown); ahead means
+ * it counts writes that no longer exist (e.g. a fact-log truncation on a
+ * copied store pulled the watermark back). Either way: one exact rescan,
+ * said out loud — never a silent adopt. Stores without the capability (and
+ * pre-stamp state on them) fall back to hash-only adoption.
+ */
+ private stateGenerationAdoptable(name: string, stateData: unknown): boolean {
+ const committed = this.storage.committedGeneration?.() ?? null
+ if (committed === null) return true
+ const raw = (stateData as Record).sourceGeneration
+ const stamped = typeof raw === 'number' ? raw : null
+ if (stamped === committed) return true
+ prodLog.warn(
+ `[Aggregation] '${name}': persisted state is at generation ${stamped ?? 'unstamped'} ` +
+ `but the store's committed generation is ${committed} — rescanning instead of adopting`
+ )
+ return false
+ }
+
+ private async loadPersisted(): Promise {
// Load persisted definitions
const savedDefs = await this.storage.getMetadata(DEFINITIONS_KEY)
if (savedDefs && typeof savedDefs === 'object' && savedDefs.definitions) {
const defs = savedDefs.definitions as Array
for (const def of defs) {
- this.definitions.set(def.name, def)
- const currentHash = hashDefinition(def)
const savedHash = def._hash || ''
- // Load persisted state
+ if (this.definitions.has(def.name)) {
+ // The app re-registered this aggregate before the load finished.
+ // The app's definition wins — never clobber it with the persisted
+ // copy. Adopt the persisted state when the definition is unchanged
+ // AND no write has landed for it yet (a landed write would be lost
+ // by adoption; the hook flips such names to backfill).
+ const appHash = this.definitionHashes.get(def.name) || ''
+ if (appHash === savedHash && this.pendingAdopt.has(def.name)) {
+ const stateData = await this.storage.getMetadata(`${STATE_KEY_PREFIX}${def.name}__`)
+ if (
+ stateData &&
+ stateData.groups &&
+ this.stateGenerationAdoptable(def.name, stateData)
+ ) {
+ const groupMap = new Map()
+ for (const group of stateData.groups as AggregateGroupState[]) {
+ groupMap.set(serializeGroupKey(group.groupKey), group)
+ }
+ this.states.set(def.name, groupMap)
+ this.pendingAdopt.delete(def.name)
+ this.needsBackfill.delete(def.name)
+ prodLog.info(
+ `[Aggregation] '${def.name}': adopted persisted state (${groupMap.size} groups) — no rescan`
+ )
+ }
+ // No/invalid persisted state: stays in pendingAdopt and resolves
+ // to backfill when init settles.
+ }
+ continue
+ }
+
+ // Not registered this session — restore definition + state from
+ // persistence.
+ this.definitions.set(def.name, def)
+ const currentHash = hashDefinition(def)
+
const stateData = await this.storage.getMetadata(`${STATE_KEY_PREFIX}${def.name}__`)
- if (stateData && stateData.groups && savedHash === currentHash) {
+ if (
+ stateData &&
+ stateData.groups &&
+ savedHash === currentHash &&
+ this.stateGenerationAdoptable(def.name, stateData)
+ ) {
// Definition unchanged — load state
const groupMap = new Map()
for (const group of stateData.groups as AggregateGroupState[]) {
@@ -358,6 +498,10 @@ export class AggregationIndex {
groupMap.set(serialized, group)
}
this.states.set(def.name, groupMap)
+ this.needsBackfill.delete(def.name)
+ prodLog.info(
+ `[Aggregation] '${def.name}': restored definition + adopted persisted state (${groupMap.size} groups)`
+ )
} else {
// Definition changed or no saved state — start fresh and backfill from
// existing entities (the owner drains needsBackfill on first query).
@@ -398,14 +542,22 @@ export class AggregationIndex {
}))
await this.storage.saveMetadata(DEFINITIONS_KEY, { definitions: defsToSave })
- // Persist dirty states
+ // Persist dirty states, stamped with the committed generation they
+ // reflect. The stamp is what makes reopen-adoption verifiable: state at a
+ // different generation than the store's committed watermark is stale (an
+ // unclean shutdown after later writes) or over-counts (a fact-log
+ // truncation on a copied store pulled the watermark BACK below the
+ // stamp) — either way the answer is one exact rescan, never a silent
+ // adopt. Read the generation after collecting groups so any racing
+ // commit resolves toward rescan, not wrong-adopt.
for (const name of this.dirty) {
const stateMap = this.states.get(name)
if (stateMap) {
const groups = Array.from(stateMap.values())
+ const sourceGeneration = this.storage.committedGeneration?.() ?? null
await this.storage.saveMetadata(
`${STATE_KEY_PREFIX}${name}__`,
- { groups }
+ sourceGeneration === null ? { groups } : { groups, sourceGeneration }
)
}
}
@@ -452,10 +604,19 @@ export class AggregationIndex {
this.definitions.set(def.name, def)
this.definitionHashes.set(def.name, newHash)
+ // First sight this session, before init() settled: defer the backfill
+ // decision — init() adopts the persisted state on hash match, and anything
+ // left unadopted resolves to backfill. Deciding eagerly here wiped valid
+ // persisted state on every restart.
+ if (!this.states.has(def.name) && !this.initDone) {
+ this.states.set(def.name, new Map())
+ this.pendingAdopt.add(def.name)
+ }
// Reset state if definition changed or doesn't exist yet, and flag it for
// backfill so already-stored entities are counted (write-time hooks only see
// future writes). The owner drains this on the next query via getPendingBackfills().
- if (!this.states.has(def.name) || (oldHash && oldHash !== newHash)) {
+ else if (!this.states.has(def.name) || (oldHash && oldHash !== newHash)) {
+ this.pendingAdopt.delete(def.name)
this.states.set(def.name, new Map())
this.needsBackfill.add(def.name)
}
@@ -476,6 +637,8 @@ export class AggregationIndex {
this.definitionHashes.delete(name)
this.states.delete(name)
this.staleMinMax.delete(name)
+ this.pendingAdopt.delete(name)
+ this.needsBackfill.delete(name)
// Notify native provider
if (this.nativeProvider?.removeAggregate) {
@@ -513,9 +676,17 @@ export class AggregationIndex {
return Array.from(this.needsBackfill)
}
- /** Clear an aggregate's state so a full rescan cannot double-count. */
+ /**
+ * Begin a rescan into a STAGING map. The live state is not touched — it
+ * keeps serving (possibly stale, but flagged pending) until the rescan
+ * completes and swaps in atomically. A mid-walk failure drops the staging
+ * map via {@link abortBackfill} and loses nothing: wiping live state before
+ * a scan that could throw was the destructive-before-durable defect.
+ * Contributions (walk + concurrent write hooks) land in staging while it
+ * exists, so the swapped-in result reflects writes that raced the walk.
+ */
beginBackfill(name: string): void {
- this.states.set(name, new Map())
+ this.backfillStaging.set(name, new Map())
// Reset native provider state for this aggregate too, if present.
const def = this.definitions.get(name)
if (def && this.nativeProvider?.removeAggregate && this.nativeProvider?.defineAggregate) {
@@ -524,6 +695,15 @@ export class AggregationIndex {
}
}
+ /**
+ * Abandon an in-flight rescan after a failure: drop the staging map, keep
+ * the live state serving, leave the aggregate flagged as pending so a later
+ * attempt rescans. The failure itself must be surfaced loudly by the owner.
+ */
+ abortBackfill(name: string): void {
+ this.backfillStaging.delete(name)
+ }
+
/** Feed one already-stored entity into a single aggregate during backfill. */
backfillEntity(name: string, entity: Record): void {
if (isAggregateEntity(entity)) return
@@ -537,14 +717,33 @@ export class AggregationIndex {
}
}
- /** Mark an aggregate's backfill complete; rebuilt state persists on next flush(). */
+ /** Swap the rebuilt staging state in atomically; persists on next flush(). */
finishBackfill(name: string): void {
+ const staged = this.backfillStaging.get(name)
+ if (staged) {
+ this.states.set(name, staged)
+ this.backfillStaging.delete(name)
+ }
this.needsBackfill.delete(name)
this.dirty.add(name)
}
// ============= Write-Time Hooks =============
+ /**
+ * A write is landing for an aggregate whose persisted-state adoption is still
+ * pending — adopting after this write would lose its contribution. Settle the
+ * decision now: an exact rescan instead of adoption. The window is the few
+ * milliseconds between a boot-time defineAggregate() and init() completing,
+ * so this rarely fires; when it does, correctness wins over the walk.
+ */
+ private resolveAdoptOnWrite(name: string): void {
+ if (this.pendingAdopt.has(name)) {
+ this.pendingAdopt.delete(name)
+ this.needsBackfill.add(name)
+ }
+ }
+
/**
* Called when an entity is added. Updates all matching aggregates.
*/
@@ -553,6 +752,7 @@ export class AggregationIndex {
for (const [name, def] of this.definitions) {
if (!matchesSource(entity, def.source)) continue
+ this.resolveAdoptOnWrite(name)
if (this.nativeProvider) {
const results = this.nativeProvider.incrementalUpdate(name, def, entity, 'add')
@@ -579,6 +779,10 @@ export class AggregationIndex {
const oldMatches = matchesSource(oldEntity, def.source)
const newMatches = matchesSource(newEntity, def.source)
+ if (oldMatches || newMatches) {
+ this.resolveAdoptOnWrite(name)
+ }
+
if (this.nativeProvider && (oldMatches || newMatches)) {
const results = this.nativeProvider.incrementalUpdate(name, def, newEntity, 'update', oldEntity)
this.applyNativeResults(name, results)
@@ -605,6 +809,7 @@ export class AggregationIndex {
for (const [name, def] of this.definitions) {
if (!matchesSource(entity, def.source)) continue
+ this.resolveAdoptOnWrite(name)
if (this.nativeProvider) {
const results = this.nativeProvider.incrementalUpdate(name, def, entity, 'delete')
@@ -757,7 +962,7 @@ export class AggregationIndex {
def: AggregateDefinition,
entity: Record
): void {
- const stateMap = this.states.get(aggName)!
+ const stateMap = (this.backfillStaging.get(aggName) ?? this.states.get(aggName))!
// Fan out: an unnest dimension makes one entity contribute to several groups.
for (const groupKey of computeGroupKeys(entity, def.groupBy)) {
@@ -815,7 +1020,7 @@ export class AggregationIndex {
def: AggregateDefinition,
entity: Record
): void {
- const stateMap = this.states.get(aggName)!
+ const stateMap = (this.backfillStaging.get(aggName) ?? this.states.get(aggName))!
// Fan out: reverse the entity's contribution from every group it joined.
for (const groupKey of computeGroupKeys(entity, def.groupBy)) {
@@ -871,7 +1076,7 @@ export class AggregationIndex {
* Apply results from native provider back into the state maps.
*/
private applyNativeResults(aggName: string, results: AggregateGroupState[]): void {
- const stateMap = this.states.get(aggName)!
+ const stateMap = (this.backfillStaging.get(aggName) ?? this.states.get(aggName))!
for (const group of results) {
const serialized = serializeGroupKey(group.groupKey)
stateMap.set(serialized, group)
diff --git a/src/aggregation/materializer.ts b/src/aggregation/materializer.ts
index 3a8dcb3f..f752648a 100644
--- a/src/aggregation/materializer.ts
+++ b/src/aggregation/materializer.ts
@@ -14,6 +14,7 @@ import type {
AggregateMetricDef
} from '../types/brainy.types.js'
import { serializeGroupKey } from './AggregationIndex.js'
+import { prodLog } from '../utils/logger.js'
/**
* Callback interface for the materializer to create/update Brainy entities.
@@ -86,8 +87,15 @@ export class AggregateMaterializer {
if (existing) clearTimeout(existing)
this.debounceTimers.set(key, setTimeout(() => {
- this.materializeOne(key).catch(() => {
- // Non-fatal — materialization is derived data
+ this.materializeOne(key).catch((err) => {
+ // Materialization is derived data (rebuildable via backfill-on-query), so
+ // a failure is non-fatal — but it leaves the materialized Measurement
+ // entity STALE. Surface it loudly rather than swallow it silently.
+ prodLog.warn(
+ `[Aggregation] Failed to materialize aggregate group '${key}': ` +
+ `${(err as Error).message}. The materialized value is stale until the ` +
+ `next successful materialization or a backfill-on-query.`
+ )
})
}, debounceMs))
}
diff --git a/src/brainy.ts b/src/brainy.ts
index efa0516f..9ab3acee 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -32,7 +32,7 @@ import {
getBrainyVersion
} from './utils/index.js'
import { embeddingManager } from './embeddings/EmbeddingManager.js'
-import { matchesMetadataFilter } from './utils/metadataFilter.js'
+import { matchesMetadataFilter, validateWhereFilter } from './utils/metadataFilter.js'
import { NaturalLanguageProcessor } from './neural/naturalLanguageProcessor.js'
import { NeuralEntityExtractor, ExtractedEntity } from './neural/entityExtractor.js'
import { TripleIntelligenceSystem } from './triple/TripleIntelligenceSystem.js'
@@ -46,8 +46,10 @@ import {
pageRank,
MinHeap
} from './graph/analyticsFallback.js'
+import { runGraphAudit, type GraphAuditReport } from './graph/graphAudit.js'
import { createPipeline } from './streaming/pipeline.js'
import { configureLogger, LogLevel, prodLog } from './utils/logger.js'
+import { warnOnLowOsLimits } from './utils/osLimits.js'
import { setGlobalCache } from './utils/unifiedCache.js'
import type { UnifiedCache } from './utils/unifiedCache.js'
import { rankIndicesByScore, reorderByIndices } from './utils/resultRanking.js'
@@ -70,6 +72,7 @@ import type {
} from './plugin.js'
import { ConnectionsCodec } from './hnsw/connectionsCodec.js'
import { TransactionManager } from './transaction/TransactionManager.js'
+import { transactTimeoutBudget } from './transaction/Transaction.js'
import { RevisionConflictError } from './transaction/RevisionConflictError.js'
import { EntityNotFoundError, RelationNotFoundError } from './errors/notFound.js'
import {
@@ -166,14 +169,30 @@ import {
type ImportOptions,
type ImportResult
} from './db/portableGraph.js'
-import { GenerationStore } from './db/generationStore.js'
+import { GenerationStore, type CommitBeforeImages } from './db/generationStore.js'
+import type { FactScanHandle } from './db/factLog.js'
+import {
+ ENTITY_TREE_STAMP_PATH,
+ readFamilyStamp,
+ verifyFamilyStamp,
+ writeFamilyStamp,
+ type FamilyStamp
+} from './db/familyStamp.js'
+import {
+ ChangeFeed,
+ type BrainyChangeEvent,
+ type ChangeListener,
+ type PendingChangeEvent
+} from './events/changeFeed.js'
import { isDeterministicEmbedMode } from './embeddings/deterministicEmbedMode.js'
-import { GenerationConflictError } from './db/errors.js'
-import { BrainyError, GraphIndexNotReadyError, MigrationInProgressError } from './errors/brainyError.js'
+import { GenerationConflictError, StoreInconsistentError } from './db/errors.js'
+import { BrainyError, GraphIndexNotReadyError, MetadataIndexNotReadyError, MigrationInProgressError, VectorIndexNotReadyError } from './errors/brainyError.js'
+import { assessIndexReadiness } from './utils/indexReadiness.js'
import { MemoryStorage } from './storage/adapters/memoryStorage.js'
import type {
CompactHistoryOptions,
CompactHistoryResult,
+ HistoryStats,
TransactOptions,
TransactReceipt,
TxLogEntry,
@@ -183,7 +202,7 @@ import type {
HistoryVersion
} from './db/types.js'
import { stableDeepEqual } from './db/stableEqual.js'
-import type { VersionedIndexProvider } from './plugin.js'
+import type { VersionedIndexProvider, ProviderInvariantReport } from './plugin.js'
import type { Operation, TransactionFunction } from './transaction/types.js'
/**
@@ -314,8 +333,78 @@ interface PlannedTransact {
touchedVerbs: string[]
/** Aggregation-index hooks to run after the commit point. */
postCommit: Array<() => void>
+ /**
+ * One entry per staged `{ op: 'update' }`, re-verified UNDER the commit
+ * mutex (the generation store's `precommit`) so per-op `ifRev` CAS is
+ * atomic with the apply — planning-time checks can interleave with
+ * concurrent writers. `updatedMetadata` is the staged metadata object (held
+ * by reference by the staged operation), re-stamped there with the
+ * authoritative `_rev`. A conflict rejects the WHOLE batch.
+ */
+ casUpdates: Array<{
+ id: string
+ ifRev?: number
+ updatedMetadata: { _rev?: number }
+ }>
+ /**
+ * Ids whose revision baseline THIS batch resets via an `add` op (a fresh
+ * create, or add's overwrite semantics which restamp `_rev: 1`). Updates to
+ * these ids sequence against in-batch state no concurrent writer can touch,
+ * so their plan-time CAS checks and rev stamps are already exact — the
+ * commit precondition leaves them untouched.
+ */
+ createdNouns: Set
+ /**
+ * Change-feed events, one per affected record in op order, populated by the
+ * planners ONLY when a listener is subscribed. Stamped with the batch's
+ * committed generation and emitted after `commitTransaction` returns — a
+ * rejected batch (CAS conflict, failed apply) emits nothing.
+ */
+ changeEvents: PendingChangeEvent[]
}
+/**
+ * Internal control-flow signal: an insert guarded by a must-be-absent
+ * precondition (`add({ ifAbsent })` / `add({ upsert })`) found the entity
+ * already present in the authoritative before-image under the commit mutex —
+ * a concurrent writer created it after the planning-time absence check.
+ * Never escapes `add()`: the caller converts it into the documented
+ * resolution (ifAbsent → return the existing id without writing; upsert →
+ * merge into the now-existing entity via `update()`).
+ */
+class InsertPreconditionExistsSignal extends Error {
+ constructor(readonly id: string) {
+ super(`insert precondition: entity ${id} already exists`)
+ this.name = 'InsertPreconditionExistsSignal'
+ }
+}
+
+/**
+ * @description The derived-index families a read may depend on. A read that
+ * consults none of them (a canonical-storage read: `get`, an entity
+ * enumeration, a VFS content/dir read) is index-independent and must never
+ * block on another family's one-time migration. Used by the family-scoped
+ * migration gate ({@link Brainy.awaitMigrationLock}).
+ */
+export type IndexFamily = 'vector' | 'metadata' | 'graph'
+
+/**
+ * How long a failed aggregation-backfill walk suppresses fresh walk attempts.
+ * Within the window, queries rethrow the recorded failure instantly (loud,
+ * cheap); after it, one new attempt is allowed. Bounds the damage of a
+ * caller-side tight retry loop against a deterministically-failing store.
+ */
+const AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS = 30_000
+
+/**
+ * Time budget for the auto-compaction pass at close() (8.9.0). Bounds how long
+ * a clean shutdown spends reclaiming history backlog — an early stop is a
+ * consistent prefix and the next close/explicit pass resumes. Explicit
+ * `compactHistory()` calls are unbounded unless the caller passes their own
+ * `timeBudgetMs` (maintenance windows choose their own budgets).
+ */
+const CLOSE_COMPACTION_BUDGET_MS = 5_000
+
/**
* The main Brainy class - Clean, Beautiful, Powerful
* REAL IMPLEMENTATION - No stubs, no mocks
@@ -327,10 +416,23 @@ export class Brainy implements BrainyInterface {
private static shutdownHooksRegisteredGlobally = false
private static instances: Brainy[] = []
+ /** The globally-registered shutdown listeners, kept so the LAST close() can
+ * deregister them. A process.on('SIGINT'/'SIGTERM') listener holds a ref'd
+ * signal handle that keeps the Node event loop alive — a library that never
+ * removes its listeners makes every bare script hang after close().
+ * See {@link registerShutdownHooks} / {@link deregisterShutdownHooksIfIdle}. */
+ private static sigtermListener?: () => void
+ private static sigintListener?: () => void
+ private static beforeExitListener?: () => void
+
/** Poll cadence (ms) for the migration LOCK when a provider exposes no
* event-driven `whenMigrationComplete()` signal. See {@link awaitMigrationLock}. */
private static readonly MIGRATION_POLL_INTERVAL_MS = 250
+ /** First-party accelerator packages probed by guarded auto-detection when
+ * `plugins` is undefined (installing one IS the opt-in). See {@link loadPlugins}. */
+ private static readonly AUTO_DETECT_PLUGIN_PACKAGES = ['@soulcraft/cor']
+
// Core components
private index!: JsHnswVectorIndex
private storage!: BaseStorage
@@ -415,12 +517,32 @@ export class Brainy implements BrainyInterface {
* during rebuild is NOT recorded here — it re-throws and aborts init() loudly.
*/
private _indexRebuildFailed: Error | null = null
+ /**
+ * Ids of records that committed via an adopt-forward failed-rollback recovery
+ * ({@link GenerationStore.commitSingleOp} `degraded`): the canonical record is
+ * durable but its derived index entry may be incomplete until
+ * {@link repairIndex}. Non-empty = a queryable degraded state, folded into
+ * {@link checkHealth}/{@link validateIndexConsistency} and warned on the read
+ * paths. Cleared by {@link repairIndex}.
+ */
+ private _indexDegradedIds: Set = new Set()
+ /** One-shot guard so the degraded-reads warning fires once per degraded window
+ * (reset when the degraded state clears). See {@link warnIfReadsDegraded}. */
+ private _degradedReadWarned = false
/** One-shot guard so the metadata cold-open consistency probe runs once per brain. */
private _metadataConsistencyProbed = false
/** Graph-adjacency cold-load consistency: verified-live this session (one-shot). */
private _graphAdjacencyVerified = false
/** Re-entrancy guard: a verify (rebuild → reads) is in flight. */
private _graphAdjacencyVerifying = false
+ /** Metadata field-index cold-read guard: verified-serving this session (one-shot). */
+ private _metadataVerified = false
+ /** Re-entrancy guard for {@link verifyMetadataLive}. */
+ private _metadataVerifying = false
+ /** Vector-index cold-read guard: verified-serving this session (one-shot). */
+ private _vectorVerified = false
+ /** Re-entrancy guard for {@link verifyVectorLive}. */
+ private _vectorVerifying = false
/**
* Coordinated migration LOCK (#18): dedup guards so the "upgrading, blocking"
* and "upgrade complete, resumed" lines each log once per migration window,
@@ -432,6 +554,26 @@ export class Brainy implements BrainyInterface {
* not migrating — the fallback `elapsedMs` a provider that omits `migrationStatus()`
* timing still gets in `getIndexStatus().migration`. See {@link migrationSnapshot}. */
private _migrationObservedAt: number | null = null
+ /** Path of the pre-upgrade backup taken this open (default-on, opt-out via
+ * `migrationBackup: false`), or `null` if none. Removed once the 7.x→8.0
+ * upgrade verifies + stamps; retained on failure. See {@link createMigrationBackupIfNeeded}. */
+ private _migrationBackupPath: string | null = null
+
+ /**
+ * The in-process change feed behind {@link onChange}. Emitted from the
+ * commit seam ({@link persistSingleOp} / {@link transact}), so every
+ * canonical mutation — any origin — produces exactly one post-commit event
+ * per affected record. See src/events/changeFeed.ts for the delivery
+ * contract.
+ */
+ private readonly _changeFeed = new ChangeFeed()
+
+ /** Set when the on-open VFS-blob adoption left one or more `_cow/` blobs it
+ * could not fully adopt (bytes or metadata missing). While true, the
+ * pre-upgrade backup is NOT auto-removed — the upgrade is not verifiably
+ * complete for the VFS. See {@link autoAdoptLegacyVfsBlobsIfNeeded}. */
+ private _vfsBlobAdoptionIncomplete = false
+
/**
* 8.0 ⇄ native-provider version handshake — the on-disk {@link BrainFormat}
* marker (`_system/brain-format.json`) as read during the store-open phase,
@@ -477,6 +619,11 @@ export class Brainy implements BrainyInterface {
private _hub?: IntegrationHub // Integration Hub for external tools
private _pendingMigrationRunner?: MigrationRunner // Deferred migration runner for large datasets
private _aggregationIndex?: AggregationIndex // Incremental aggregation engine
+ private _aggregationBackfillFlight: Promise | null = null // Single-flight backfill walk
+ // A failed walk latches its error: retries within the cooldown rethrow it
+ // instantly instead of re-walking, so a tight caller-side retry loop costs
+ // one loud error per query, never a full store walk per query.
+ private _aggregationBackfillFailure: { at: number; error: Error } | null = null
private _materializer?: AggregateMaterializer // Debounced materialization of aggregate results
/**
* Fields registered via `brain.trackField()` — drives optional value validation on
@@ -506,6 +653,13 @@ export class Brainy implements BrainyInterface {
// set this to ReaderMode so historical instances are protected the same way.
private operationalMode: BaseOperationalMode
+ // Write-quarantine after a failed transaction rollback left the store
+ // inconsistent (see StoreInconsistentError). Set at the moment the failure is
+ // surfaced; every subsequent mutation is refused via assertWritable until
+ // repairIndex() reconciles canonical vs derived state and clears it. Reads are
+ // never affected. null = healthy.
+ private storeInconsistency: StoreInconsistentError | null = null
+
// Ready Promise state (Unified readiness API)
// Allows consumers to await brain.ready for initialization completion
private _readyPromise: Promise | null = null
@@ -627,13 +781,13 @@ export class Brainy implements BrainyInterface {
* Whether the active storage adapter (anywhere in its prototype chain)
* implements a given optional method.
*
- * Storage-adapter plugins (e.g. `@soulcraft/cortex`'s `MmapFileSystemStorage
+ * Storage-adapter plugins (e.g. `@soulcraft/cor`'s `MmapFileSystemStorage
* extends FileSystemStorage`) inherit new methods Brainy adds to
* `FileSystemStorage` / `BaseStorage` automatically — `typeof` walks the
* prototype chain, so there's no in-package version skew to worry about as
* long as the plugin's own dist resolves `@soulcraft/brainy` dynamically
* (which Cortex 2.2.x onward does — see
- * `node_modules/@soulcraft/cortex/dist/storage/mmapFileSystemStorage.js`).
+ * `node_modules/@soulcraft/cor/dist/storage/mmapFileSystemStorage.js`).
*
* This helper exists for the **build/install** failure modes the import
* resolution can't catch:
@@ -671,6 +825,17 @@ export class Brainy implements BrainyInterface {
`Open in writer mode to modify data.`
)
}
+ // Write-quarantine: a prior transaction's rollback failed and left the store
+ // inconsistent. Refuse further writes (which would compound the damage) until
+ // repairIndex() reconciles and lifts the quarantine. Reads still work.
+ if (this.storeInconsistency) {
+ throw new Error(
+ `Cannot call ${method}() — the store is WRITE-QUARANTINED after a failed ` +
+ `transaction rollback left it inconsistent. Reads still work; run repairIndex() ` +
+ `to reconcile the derived indexes against canonical storage and lift the ` +
+ `quarantine. Original inconsistency: ${this.storeInconsistency.message}`
+ )
+ }
}
/**
@@ -768,7 +933,7 @@ export class Brainy implements BrainyInterface {
try {
// Auto-detect and activate plugins BEFORE storage setup
- // so plugin-provided storage factories (e.g., filesystem override from cortex) are available
+ // so plugin-provided storage factories (e.g., filesystem override from cor) are available
await this.loadPlugins()
// 7.x → 8.0 layout migration, BEFORE storage setup: collapse a legacy
@@ -781,11 +946,18 @@ export class Brainy implements BrainyInterface {
this.storage = await this.setupStorage()
await this.storage.init()
+ // OS-limit detection (once per process, Linux-only, measurement-only):
+ // warn NOW about RLIMIT_NOFILE / vm.max_map_count values that will bite
+ // at pool scale, instead of letting the operator meet them as EMFILE or
+ // a failed mmap deep inside an index open. Fire-and-forget — the check
+ // never affects open.
+ void warnOnLowOsLimits()
+
// Acquire the writer lock for filesystem (and other locking-capable) backends.
// Skipped in reader mode and on backends that don't support multi-process locking.
// Throws if another live writer holds the directory (unless force: true).
//
- // Defensive call: older storage adapters (e.g. `@soulcraft/cortex@2.2.0`
+ // Defensive call: older storage adapters (e.g. `@soulcraft/cor@2.2.0`
// and earlier) bundle a pre-7.21 `BaseStorage` that doesn't define these
// methods. We feature-detect each one rather than fail boot.
if (this.config.mode !== 'reader') {
@@ -842,6 +1014,44 @@ export class Brainy implements BrainyInterface {
readOnly: this.config.mode === 'reader'
})
+ // The generation fact log is CANONICAL state, not a derived index — no
+ // sweeper, GC, or blob-lifecycle path may ever delete under it. Declare
+ // its namespace as a protected family (rebuildable: false — a lost fact
+ // segment is NOT reconstructable) so the storage layer REFUSES such
+ // deletes; refusal beats trust. Feature-detected + idempotent per name.
+ if (
+ this.config.mode !== 'reader' &&
+ this.generationStore.getFactLog() &&
+ typeof this.storage.registerDerivedFamily === 'function'
+ ) {
+ await this.storage.registerDerivedFamily({
+ name: 'generation-facts',
+ members: ['_generations/facts/'],
+ namespace: true,
+ rebuildable: false
+ })
+ }
+
+ // Fact-scan capability: wire the storage seam through which index
+ // providers (which hold only `storage`) reach the fact log. A closure
+ // over the LIVE log — restore/reopen swaps the instance transparently —
+ // so a provider's heal can switch from the enumeration walk to one
+ // sequential fact scan whenever the log exists.
+ if (typeof (this.storage as BaseStorage).setFactScanSource === 'function') {
+ ;(this.storage as BaseStorage).setFactScanSource({
+ factLog: () => this.generationStore?.getFactLog() ?? null,
+ // The committed watermark, exposed as a capability so providers
+ // never parse the store's private manifest format.
+ committedGeneration: () => this.generationStore?.committedGeneration() ?? 0
+ })
+ }
+
+ // Entity-tree stamp coherence: compare the stamped sourceGeneration +
+ // rollup invariants against the log head + live counters. Loud on
+ // genuine incoherence (repairIndex heals), silent on absent/coherent,
+ // benign-behind refreshes at the next flush. Never blocks open.
+ await this.verifyEntityTreeStamp()
+
// 8.0 ⇄ native-provider version handshake: load the on-disk brain-format
// marker (`_system/brain-format.json`) into an in-memory field NOW —
// after the store-open phase, but BEFORE any derived index or native
@@ -856,6 +1066,16 @@ export class Brainy implements BrainyInterface {
this._indexEpochStale =
this._brainFormat === null || this._brainFormat.indexEpoch !== EXPECTED_INDEX_EPOCH
+ // Pre-upgrade backup (default-on, opt-out via `migrationBackup: false`): the
+ // on-disk format is stale, so a one-time 7.x → 8.0 rebuild will run below.
+ // Snapshot the brain dir NOW — before any provider (native or JS) rebuilds a
+ // derived index — so a migration bug can be rolled back. Removed once the
+ // upgrade verifies + stamps; retained on failure. No-op for a reader, for
+ // non-filesystem storage, or for a brain with no persisted data.
+ if (this._indexEpochStale && this.config.migrationBackup && !this.isReadOnly) {
+ await this.createMigrationBackupIfNeeded()
+ }
+
// Provider: embeddings (reassign embedder if plugin provides one)
const embeddingProvider = this.pluginRegistry.getProvider('embeddings')
if (embeddingProvider) {
@@ -882,7 +1102,7 @@ export class Brainy implements BrainyInterface {
setMsgpackImplementation(msgpackProvider)
}
- // Provider: sort:topK (e.g. cortex's native partial-sort / heap-select) — swaps the
+ // Provider: sort:topK (e.g. cor's native partial-sort / heap-select) — swaps the
// JS result-ranking used by find() to pick the top `offset + limit` rows. The provider
// returns indices into a scores array, ordered descending with stable ties, identical
// to the JS sortTopKIndicesJs. rankIndicesByScore validates the provider's output and
@@ -909,7 +1129,7 @@ export class Brainy implements BrainyInterface {
if (metadataFactory) {
this.metadataIndex = metadataFactory(this.storage)
} else {
- // JS fallback — inject native EntityIdMapper if cortex provides one
+ // JS fallback — inject native EntityIdMapper if cor provides one
const entityIdMapperFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('entityIdMapper')
this.metadataIndex = new MetadataIndexManager(this.storage, {}, {
entityIdMapper: entityIdMapperFactory ? entityIdMapperFactory(this.storage) : undefined,
@@ -930,14 +1150,21 @@ export class Brainy implements BrainyInterface {
this.graphIndex = graphIndex
}
- // Eager graph cold-load (cor contract). The native graph provider lazily
- // defers loading its source→target adjacency, so isReady() would report
- // false at the rebuild gate below and force a spurious rebuild (failing the
- // §7.1 rebuild()==0 acceptance). Trigger the eager cold-load now — AFTER
- // metadataIndex.init() above (the id-mapper is hydrated, so a native int
- // adjacency resolves endpoints through it: the CTX-BR-RESTORE-REBUILD order)
- // and BEFORE rebuildIndexesIfNeeded. Optional: the JS graph has no init()
- // and self-loads its adjacency on demand.
+ // Eager cold-load (readiness contract). A provider that persists its
+ // derived state exposes init?(): trigger the load NOW — AFTER
+ // metadataIndex.init() above (the id-mapper is hydrated first, so a
+ // native int-keyed index resolves endpoints/slots through it — the
+ // CTX-BR-RESTORE-REBUILD order) and BEFORE the rebuild gate — so a
+ // durable index reports its real size()/isReady() at the gate instead
+ // of eating a spurious rebuild-from-canonical on every open (§7.1
+ // rebuild()==0). Vector first, then graph. JS engines: the JS vector
+ // index has no init() (rebuild() IS its load path); the JS graph's
+ // init() already cold-loaded its LSM inside storage.getGraphIndex()
+ // (idempotent here).
+ const vectorWithInit = this.index as { init?: () => Promise }
+ if (typeof vectorWithInit.init === 'function') {
+ await vectorWithInit.init()
+ }
const graphWithInit = this.graphIndex as { init?: () => Promise }
if (typeof graphWithInit.init === 'function') {
await graphWithInit.init()
@@ -987,9 +1214,46 @@ export class Brainy implements BrainyInterface {
`(storage committed: ${committed}) — provider replays the gap per ` +
`the post-commit applier contract`
)
+ } else if (providerGen > committed) {
+ // The AHEAD direction is incoherence, not a replay gap: the provider's
+ // persisted index claims writes the store no longer has — the signature
+ // of a torn copy or a log truncation that pulled the committed
+ // watermark back (crash recovery, byte-copy of a live store). A replay
+ // can never converge on it and index answers may reference vanished
+ // writes. Name it loudly at open so it is never diagnosed from a
+ // silent journal; the provider's own coherence check / heal walk (or
+ // brain.repairIndex()) is the cure.
+ prodLog.warn(
+ `[Brainy] Versioned index provider is AHEAD of the store: provider ` +
+ `generation ${providerGen} vs committed ${committed}. This store was ` +
+ `likely copied from a live service or truncated during crash recovery. ` +
+ `Derived-index answers may reference rolled-back writes until the ` +
+ `provider heals from canonical (brain.repairIndex() forces it).`
+ )
}
}
+ // Recover VFS content blobs a 7→8 upgrade left stranded in the removed
+ // branch system's copy-on-write area (`_cow/`). Runs BEFORE the rebuild
+ // (whose success stamp removes the pre-upgrade backup) so the backup still
+ // covers this step, and works on the raw object primitives (no blob store
+ // needed yet). A cheap no-op on native-8.0 / fresh brains and on a brain
+ // already healed. See adoptLegacyCowBlobs.
+ await this.autoAdoptLegacyVfsBlobsIfNeeded()
+
+ // Temporal-blob contract: one-time (marker-gated) backfill of blob
+ // history reference counts for stores whose generation history predates
+ // the contract — after it, compaction reclaims blob bytes exactly (zero
+ // live AND zero history references). On a scrub failure the store runs
+ // leak-safe (no blob reclamation) rather than risk a premature delete.
+ if (typeof (this.storage as unknown as {
+ backfillBlobHistoryRefCountsIfNeeded?: () => Promise
+ }).backfillBlobHistoryRefCountsIfNeeded === 'function') {
+ await (this.storage as unknown as {
+ backfillBlobHistoryRefCountsIfNeeded: () => Promise
+ }).backfillBlobHistoryRefCountsIfNeeded()
+ }
+
// Rebuild indexes if needed for existing data
await this.rebuildIndexesIfNeeded()
@@ -1023,6 +1287,17 @@ export class Brainy implements BrainyInterface {
} else {
console.log(`[brainy] Providers: ${native.length}/${wellKnownKeys.length} native (${plugins}) | default: ${fallback.join(', ')}`)
}
+ // An activated accelerator that registered NOTHING is running as pure
+ // decoration — every query silently serves from the JS engines. Most
+ // often a licensing gate declining to engage. Say so, loudly, so an
+ // evaluator never concludes the accelerator "does nothing".
+ if (native.length === 0) {
+ console.warn(
+ `[brainy] ⚠ ${plugins} activated but registered 0 native providers — all queries are ` +
+ `running on the default JS engines. Check the plugin's requirements (e.g. a license ` +
+ `key) and its logs; see docs/PLUGINS.md.`
+ )
+ }
}
// Mark as initialized BEFORE VFS init
@@ -1116,6 +1391,13 @@ export class Brainy implements BrainyInterface {
if (this._readyReject) {
this._readyReject(error instanceof Error ? error : new Error(String(error)))
}
+ // Machine-readable init failures pass through UNWRAPPED — the writer-lock
+ // conflict documents an err.code/err.lockInfo contract ("callers detect
+ // this case via err.code"), and wrapping in a fresh Error silently
+ // stripped both, leaving consumers only a message to regex against.
+ if (error instanceof Error && (error as Error & { code?: string }).code === 'BRAINY_WRITER_LOCKED') {
+ throw error
+ }
throw new Error(`Failed to initialize Brainy: ${error}`)
}
}
@@ -1207,20 +1489,52 @@ export class Brainy implements BrainyInterface {
}
}
- // Graceful shutdown signals (registered once globally)
- process.on('SIGTERM', async () => {
+ // Graceful shutdown signals (registered once globally). The listeners are
+ // kept as statics so the last live instance's close() can deregister them
+ // — the signal handles they hold are ref'd and would otherwise keep the
+ // process alive forever after every brain is closed.
+ Brainy.sigtermListener = async () => {
await flushOnShutdown()
process.exit(0)
- })
-
- process.on('SIGINT', async () => {
+ }
+ Brainy.sigintListener = async () => {
await flushOnShutdown()
process.exit(0)
- })
-
- process.on('beforeExit', async () => {
+ }
+ Brainy.beforeExitListener = async () => {
+ // Self-deregister FIRST: Node re-emits 'beforeExit' after every event-
+ // loop drain, and this flush schedules new async work — with the
+ // listener still attached, a script that never calls close() would spin
+ // flush → drain → flush forever and never exit. One flush, then the
+ // next drain finds no listener and the process exits.
+ if (Brainy.beforeExitListener) {
+ process.off('beforeExit', Brainy.beforeExitListener)
+ Brainy.beforeExitListener = undefined
+ }
await flushOnShutdown()
- })
+ }
+ process.on('SIGTERM', Brainy.sigtermListener)
+ process.on('SIGINT', Brainy.sigintListener)
+ process.on('beforeExit', Brainy.beforeExitListener)
+ }
+
+ /**
+ * Deregister the global shutdown hooks once NO live instance remains, so a
+ * script that closed every brain exits on its own — a library must never
+ * keep its host process alive. Re-initializing later re-registers them
+ * (the `shutdownHooksRegisteredGlobally` flag resets here).
+ */
+ private static deregisterShutdownHooksIfIdle(): void {
+ if (Brainy.instances.length > 0 || !Brainy.shutdownHooksRegisteredGlobally) {
+ return
+ }
+ if (Brainy.sigtermListener) process.off('SIGTERM', Brainy.sigtermListener)
+ if (Brainy.sigintListener) process.off('SIGINT', Brainy.sigintListener)
+ if (Brainy.beforeExitListener) process.off('beforeExit', Brainy.beforeExitListener)
+ Brainy.sigtermListener = undefined
+ Brainy.sigintListener = undefined
+ Brainy.beforeExitListener = undefined
+ Brainy.shutdownHooksRegisteredGlobally = false
}
/**
@@ -1236,7 +1550,10 @@ export class Brainy implements BrainyInterface {
* re-initializing — using a closed Brainy is a consumer bug, not a lazy-init
* opportunity.
*/
- private async ensureInitialized(opts?: { bypassMigrationLock?: boolean }): Promise {
+ private async ensureInitialized(opts?: {
+ bypassMigrationLock?: boolean
+ needs?: IndexFamily[]
+ }): Promise {
if (this.closed) {
throw new Error('Brainy instance is not initialized: it was closed via close(). Create a new instance.')
}
@@ -1246,12 +1563,17 @@ export class Brainy implements BrainyInterface {
// Coordinated migration LOCK (#18): every data-plane read and write funnels
// through here, so this is the single choke point that holds operations while
// a native provider runs its one-time 7.x → 8.0 rebuild-from-canonical — no
- // op touches a half-built index. Observability (`health`/`checkHealth`) and
- // the lock-clearing path (`stampBrainFormat`, which does not route through
- // here) opt out so an operator can always watch progress and cor can stamp.
+ // op touches a half-built index. The gate is FAMILY-SCOPED: `needs` names the
+ // derived-index families this operation actually consults, so a read served
+ // entirely from canonical storage (`needs: []`) or from a healthy family
+ // never blocks on an UNRELATED family's migration. `needs` omitted = the
+ // conservative whole-brain wait (writes, and any read not yet classified).
+ // Observability (`health`/`checkHealth`) and the lock-clearing path
+ // (`stampBrainFormat`, which does not route through here) opt out entirely so
+ // an operator can always watch progress and a native provider can stamp.
// A brain that never migrates pays one boolean check (see awaitMigrationLock).
if (!opts?.bypassMigrationLock) {
- await this.awaitMigrationLock()
+ await this.awaitMigrationLock(opts?.needs)
}
}
@@ -1419,21 +1741,178 @@ export class Brainy implements BrainyInterface {
*/
private async persistSingleOp(
touched: { nouns?: string[]; verbs?: string[] },
- run: TransactionFunction
- ): Promise {
+ run: TransactionFunction,
+ precommit?: (before: CommitBeforeImages) => void,
+ pendingEvents?: PendingChangeEvent[]
+ ): Promise<{ generation?: number; timestamp: number; degraded?: string[] }> {
+ // Change-feed capture: when this write will emit, hold a reference to the
+ // commit's before-images so `remove` events can carry the record's last
+ // committed state (free — the commit reads them anyway for Model B).
+ let capturedBefore: CommitBeforeImages | undefined
+ const captureAndCheck =
+ pendingEvents && pendingEvents.length > 0
+ ? (before: CommitBeforeImages): void => {
+ capturedBefore = before
+ precommit?.(before)
+ }
+ : precommit
+
if (!this._generationStampingActive) {
// Init-time / infrastructure baseline write (e.g. the VFS root): apply
// WITHOUT creating a generation. Generation 0 is the freshly-materialized
// brain (bootstrap included); the first USER write is generation 1.
+ // Bootstrap writes are single-threaded, so a supplied precondition is
+ // honored against directly-read before-images (no commit mutex exists
+ // on this path — nothing to race).
+ if (captureAndCheck) {
+ const nouns = new Map()
+ for (const id of touched.nouns ?? []) {
+ const prev = await this.storage.readNounRaw(id)
+ nouns.set(id, { kind: 'noun', metadata: prev.metadata, vector: prev.vector })
+ }
+ const verbs = new Map()
+ for (const id of touched.verbs ?? []) {
+ const prev = await this.storage.readVerbRaw(id)
+ verbs.set(id, { kind: 'verb', metadata: prev.metadata, vector: prev.vector })
+ }
+ captureAndCheck({ nouns, verbs } as CommitBeforeImages)
+ }
await this.generationStore.runWithoutGeneration(() =>
- this.transactionManager.executeTransaction(run)
+ this.transactionManager.executeTransaction(run, {
+ timeout: transactTimeoutBudget(
+ (touched.nouns?.length ?? 0) + (touched.verbs?.length ?? 0)
+ )
+ })
)
- return
+ const timestamp = Date.now()
+ // Bootstrap writes are not generation-stamped; emit without one.
+ this.emitCommitted(pendingEvents, capturedBefore, undefined, timestamp)
+ return { timestamp }
}
- await this.generationStore.commitSingleOp({
- touched,
- execute: () => this.transactionManager.executeTransaction(run)
+ let receipt
+ try {
+ receipt = await this.generationStore.commitSingleOp({
+ touched,
+ precommit: captureAndCheck,
+ execute: () =>
+ this.transactionManager.executeTransaction(run, {
+ timeout: transactTimeoutBudget(
+ (touched.nouns?.length ?? 0) + (touched.verbs?.length ?? 0)
+ )
+ })
+ })
+ } catch (err) {
+ // A failed rollback that left the store inconsistent (a remove/update
+ // whose restore-undo failed) quarantines writes until repairIndex().
+ if (err instanceof StoreInconsistentError) this.storeInconsistency = err
+ throw err
+ }
+ // POST-COMMIT ONLY: an aborted commit (CAS conflict, failed apply) throws
+ // above and never reaches this line — the feed cannot announce a write
+ // that did not become durable. (A degraded adopt-forward write DID commit —
+ // it carries a generation and emits normally.)
+ this.emitCommitted(pendingEvents, capturedBefore, receipt.generation, receipt.timestamp)
+ // An adopt-forward failed-rollback recovery committed the record but may have
+ // left its derived index incomplete (generationStore already warned once).
+ // Record the ids so reads and checkHealth() surface the incompleteness rather
+ // than silently returning partial data; repairIndex() clears them.
+ if (receipt.degraded && receipt.degraded.length > 0) {
+ for (const degradedId of receipt.degraded) this._indexDegradedIds.add(degradedId)
+ this._degradedReadWarned = false // re-arm the read-path warning
+ if (!this.config.silent) {
+ prodLog.warn(
+ `[Brainy] A single-op write committed in a DEGRADED state — the derived ` +
+ `index may be incomplete for id(s) ${receipt.degraded.join(', ')}. ` +
+ `Reads may return partial results until repairIndex() reconciles them.`
+ )
+ }
+ }
+ return receipt
+ }
+
+ /**
+ * @description Stamp pending change events with their commit receipt,
+ * enrich entity `remove` events with the record's last committed state
+ * (from the commit's before-images), and hand them to the change feed for
+ * post-mutex dispatch. No-op when the write produced no events.
+ * @param pending - Events the mutation method constructed pre-commit
+ * (only when a listener is subscribed — see {@link ChangeFeed.hasListeners}).
+ * @param before - The commit's before-images (delete-payload source).
+ * @param generation - The committed generation (absent for bootstrap writes).
+ * @param timestamp - The commit timestamp.
+ */
+ private emitCommitted(
+ pending: PendingChangeEvent[] | undefined,
+ before: CommitBeforeImages | undefined,
+ generation: number | undefined,
+ timestamp: number
+ ): void {
+ if (!pending || pending.length === 0) return
+ const events: BrainyChangeEvent[] = pending.map((p) => {
+ let entity = p.entity
+ if (!entity && p.kind === 'entity' && p.op === 'remove' && p.id && before) {
+ const record = before.nouns.get(p.id)?.metadata as
+ | Record
+ | null
+ | undefined
+ if (record) {
+ entity = this.entityViewFromRawRecord(p.id, record)
+ }
+ }
+ return {
+ ...p,
+ ...(entity && { entity }),
+ ...(generation !== undefined && { generation }),
+ timestamp
+ }
})
+ this._changeFeed.emit(events)
+ }
+
+ /**
+ * @description Build the change-event entity view from a stored flat
+ * metadata record (the shape before-images and pre-delete reads carry),
+ * using THE canonical reserved/custom split so the view can never drift
+ * from the live read paths.
+ * @param id - The entity's canonical UUID.
+ * @param record - The stored flat metadata record.
+ * @returns The `ChangeEventEntity` view (type, subtype, service, custom metadata).
+ */
+ private entityViewFromRawRecord(
+ id: string,
+ record: Record
+ ): NonNullable {
+ const { reserved, custom } = splitNounMetadataRecord(record)
+ return {
+ id,
+ type: String(reserved.noun ?? 'unknown'),
+ ...(reserved.subtype !== undefined && { subtype: String(reserved.subtype) }),
+ metadata: custom,
+ ...(reserved.service !== undefined && { service: String(reserved.service) })
+ }
+ }
+
+ /**
+ * @description Build the AGGREGATION view of an entity from a stored flat
+ * metadata record — EVERY reserved field mapped to its top-level entity
+ * name (stored `noun` → `type`), custom metadata in `metadata`. This must
+ * mirror the add-path `entityForIndexing` shape exactly: the aggregation
+ * engine resolves groupBy/where fields via `resolveEntityField`
+ * (top-level standard fields + custom metadata), so a view that drops a
+ * reserved field makes every aggregate grouped by that field decrement a
+ * group that does not exist — counts then drift upward forever after
+ * deletes (SELF-AGGREGATE-DELETE-DRIFT). Do not hand-roll subsets of this.
+ * @param record - The stored flat metadata record (before-image or pre-delete read).
+ * @returns The full-fidelity entity view for aggregation hooks.
+ */
+ private entityForAggFromRawRecord(record: Record): Record {
+ const { reserved, custom } = splitNounMetadataRecord(record)
+ const { noun, ...rest } = reserved
+ return {
+ type: noun,
+ ...rest,
+ metadata: custom
+ }
}
/**
@@ -1504,7 +1983,10 @@ export class Brainy implements BrainyInterface {
// ifAbsent (7.31.0) — by-ID idempotent insert. Only meaningful when a custom id
// is supplied; a freshly generated UUID can never collide. Returns the existing
// (canonical) id without writing if the entity is already present (no throw,
- // no overwrite).
+ // no overwrite). This check is a FAST-FAIL (skips the embedding cost); the
+ // authoritative absence check runs in the insert's commit precondition
+ // below, atomic with the apply, so a concurrent same-id create cannot make
+ // both callers write.
if (params.id && params.ifAbsent) {
const existing = await this.storage.getNounMetadata(id)
if (existing) return id
@@ -1515,21 +1997,12 @@ export class Brainy implements BrainyInterface {
// delegate to update() so the supplied fields MERGE into the existing entity
// (metadata merge, re-embed on changed data, _rev bump, createdAt preserved)
// instead of the default destructive overwrite. When the id is absent, fall
- // through to the normal insert below.
+ // through to the guarded insert below (whose commit precondition converts a
+ // concurrent same-id create into this same merge — never an overwrite).
if (params.id && params.upsert) {
const existing = await this.storage.getNounMetadata(id)
if (existing) {
- await this.update({
- id,
- ...(params.data !== undefined && { data: params.data }),
- ...(params.type !== undefined && { type: params.type }),
- ...(params.subtype !== undefined && { subtype: params.subtype }),
- ...(params.visibility !== undefined && { visibility: params.visibility }),
- ...(params.metadata !== undefined && { metadata: params.metadata }),
- ...(params.vector !== undefined && { vector: params.vector }),
- ...(params.confidence !== undefined && { confidence: params.confidence }),
- ...(params.weight !== undefined && { weight: params.weight })
- })
+ await this.update(this.upsertMergeParams(id, params))
return id
}
}
@@ -1601,7 +2074,22 @@ export class Brainy implements BrainyInterface {
// Execute atomically with transaction system, generation-stamped as one
// immutable Model-B generation (before-image = the absent/create sentinel).
// All operations succeed or all rollback - prevents partial failures
- await this.persistSingleOp({ nouns: [id] }, async (tx) => {
+ // ifAbsent/upsert insert leg: must-be-absent commit precondition, run
+ // under the commit mutex against the authoritative before-image — the
+ // planning-time absence checks above are fast-fails that can interleave
+ // with a concurrent same-id create. Exactly one concurrent create wins;
+ // every other caller takes its documented resolution instead of
+ // silently overwriting the winner.
+ const requireAbsent = Boolean(params.id && (params.ifAbsent || params.upsert))
+ const insertPrecommit = requireAbsent
+ ? (before: CommitBeforeImages): void => {
+ if (before.nouns.get(id)?.metadata) {
+ throw new InsertPreconditionExistsSignal(id)
+ }
+ }
+ : undefined
+
+ const runInsert: TransactionFunction = async (tx) => {
// Operation 1: Save metadata FIRST (TypeAwareStorage caching)
// isNew=true: skip pre-read for rollback (entity doesn't exist yet)
tx.addOperation(
@@ -1628,7 +2116,62 @@ export class Brainy implements BrainyInterface {
tx.addOperation(
new AddToMetadataIndexOperation(this.metadataIndex, id, entityForIndexing)
)
- })
+ }
+
+ // Bounded retry closes the remaining interleavings without ever holding
+ // a lock across the loop: a lost insert race resolves to skip (ifAbsent)
+ // or merge (upsert); a merge that then hits a concurrent delete retries
+ // the insert. Each persistSingleOp call constructs fresh operations, so
+ // re-running the callback is safe.
+ // Change feed: built only when someone is listening (zero-cost gate).
+ const addEvents: PendingChangeEvent[] | undefined = this._changeFeed.hasListeners
+ ? [
+ {
+ kind: 'entity',
+ op: 'add',
+ id,
+ entity: {
+ id,
+ type: String(params.type),
+ ...(params.subtype !== undefined && { subtype: String(params.subtype) }),
+ metadata: (params.metadata as Record) ?? {},
+ ...(params.service !== undefined && { service: String(params.service) })
+ }
+ }
+ ]
+ : undefined
+
+ const MAX_UPSERT_ATTEMPTS = 10
+ for (let attempt = 0; ; attempt++) {
+ try {
+ await this.persistSingleOp({ nouns: [id] }, runInsert, insertPrecommit, addEvents)
+ break
+ } catch (err) {
+ if (!(err instanceof InsertPreconditionExistsSignal)) {
+ throw err
+ }
+ // A concurrent writer created the entity between our absence check
+ // and the commit.
+ if (params.ifAbsent) {
+ // Documented contract: return the existing id without writing.
+ return id
+ }
+ // upsert: merge into the now-existing entity. A concurrent delete
+ // between this conflict and the merge throws EntityNotFoundError —
+ // loop back and retry the insert.
+ try {
+ await this.update(this.upsertMergeParams(id, params))
+ return id
+ } catch (mergeErr) {
+ if (
+ !(mergeErr instanceof EntityNotFoundError) ||
+ attempt >= MAX_UPSERT_ATTEMPTS
+ ) {
+ throw mergeErr
+ }
+ }
+ }
+ }
// Aggregation hook (outside transaction — derived data, can be reconstructed)
if (this._aggregationIndex) {
@@ -1638,6 +2181,31 @@ export class Brainy implements BrainyInterface {
return id
}
+ /**
+ * @description The `update()` param mapping `add({ upsert })` delegates to
+ * when the target id already exists: every supplied field merges into the
+ * existing entity (metadata merge, re-embed on changed data, `_rev` bump,
+ * `createdAt` preserved); absent fields are left untouched. Shared by the
+ * planning-time upsert branch and the commit-conflict retry so the two
+ * paths can never drift.
+ * @param id - The canonical entity id the upsert resolved to.
+ * @param params - The original `add()` params carrying the fields to merge.
+ * @returns The `UpdateParams` for the merging `update()` call.
+ */
+ private upsertMergeParams(id: string, params: AddParams): UpdateParams {
+ return {
+ id,
+ ...(params.data !== undefined && { data: params.data }),
+ ...(params.type !== undefined && { type: params.type }),
+ ...(params.subtype !== undefined && { subtype: params.subtype }),
+ ...(params.visibility !== undefined && { visibility: params.visibility }),
+ ...(params.metadata !== undefined && { metadata: params.metadata }),
+ ...(params.vector !== undefined && { vector: params.vector }),
+ ...(params.confidence !== undefined && { confidence: params.confidence }),
+ ...(params.weight !== undefined && { weight: params.weight })
+ } as UpdateParams
+ }
+
/**
* Get an entity by ID
*
@@ -1771,7 +2339,10 @@ export class Brainy implements BrainyInterface {
*
*/
async get(id: string, options?: GetOptions): Promise | null> {
- await this.ensureInitialized()
+ // Canonical read: a get resolves an entity by id straight from storage and
+ // consults no derived index — it must not wait on any family's migration.
+ await this.ensureInitialized({ needs: [] })
+ this.warnIfReadsDegraded('get')
// Id normalization (8.0): a caller may read by their natural key — resolve
// it to the same canonical UUID add() stored. A real UUID passes through.
@@ -1829,7 +2400,8 @@ export class Brainy implements BrainyInterface {
* ```
*/
async batchGet(ids: string[], options?: GetOptions): Promise>> {
- await this.ensureInitialized()
+ // Canonical read (see get): resolves by id from storage, no derived index.
+ await this.ensureInitialized({ needs: [] })
// Id normalization (8.0): resolve each id to its canonical UUID so callers
// can batch-read by natural key. resolveEntityId is idempotent on real
@@ -2396,7 +2968,11 @@ export class Brainy implements BrainyInterface {
// ifRev (7.31.0) — optimistic concurrency. If caller supplied ifRev, the persisted
// _rev must match exactly. `_rev` is reserved: every read path surfaces it ONLY
// top-level (entities without one are read as rev 1), so that is the one place
- // to look.
+ // to look. This check is purely a FAST-FAIL (avoids paying the embedding
+ // cost on an obviously stale expectation) — the authoritative check runs
+ // in the commit precondition below, under the commit mutex, where it is
+ // atomic with the apply. Concurrent same-rev updates all pass HERE but
+ // exactly one survives THERE.
const currentRev = typeof existing._rev === 'number' ? existing._rev : 1
if (typeof params.ifRev === 'number' && params.ifRev !== currentRev) {
throw new RevisionConflictError(params.id, params.ifRev, currentRev)
@@ -2472,6 +3048,35 @@ export class Brainy implements BrainyInterface {
metadata: newMetadata
}
+ // Authoritative CAS + honest rev stamp, run by the generation store
+ // UNDER the commit mutex against the just-read before-image — the one
+ // point where check-and-apply is atomic (the fast-fail above can
+ // interleave with concurrent writers; this cannot). Also re-stamps
+ // `_rev` from the authoritative base so the counter stays monotonic
+ // even for concurrent non-CAS updates. The staged operations below
+ // capture `updatedMetadata` by reference, so the re-stamp lands.
+ const casPrecommit = (before: CommitBeforeImages): void => {
+ const beforeMeta = before.nouns.get(params.id)?.metadata as
+ | { _rev?: unknown }
+ | null
+ | undefined
+ if (!beforeMeta) {
+ // A concurrent remove() deleted the entity after our read. A CAS
+ // caller gets the truth; a plain update keeps its long-standing
+ // last-writer-wins semantics (the staged write re-creates it).
+ if (typeof params.ifRev === 'number') {
+ throw new EntityNotFoundError(params.id)
+ }
+ return
+ }
+ const authoritativeRev =
+ typeof beforeMeta._rev === 'number' ? beforeMeta._rev : 1
+ if (typeof params.ifRev === 'number' && params.ifRev !== authoritativeRev) {
+ throw new RevisionConflictError(params.id, params.ifRev, authoritativeRev)
+ }
+ updatedMetadata._rev = authoritativeRev + 1
+ }
+
// Execute atomically with transaction system, generation-stamped as one
// immutable Model-B generation (before-image = the entity's prior state).
await this.persistSingleOp({ nouns: [params.id] }, async (tx) => {
@@ -2531,17 +3136,37 @@ export class Brainy implements BrainyInterface {
tx.addOperation(
new AddToMetadataIndexOperation(this.metadataIndex, params.id, entityForIndexing)
)
- })
+ }, casPrecommit, this._changeFeed.hasListeners
+ ? [
+ {
+ kind: 'entity',
+ op: 'update',
+ id: params.id,
+ entity: {
+ id: params.id,
+ type: String(entityForIndexing.type),
+ ...(entityForIndexing.subtype !== undefined && {
+ subtype: String(entityForIndexing.subtype)
+ }),
+ metadata: (newMetadata as Record) ?? {},
+ ...(entityForIndexing.service !== undefined && {
+ service: String(entityForIndexing.service)
+ })
+ }
+ }
+ ]
+ : undefined)
- // Aggregation hook (outside transaction — derived data)
+ // Aggregation hook (outside transaction — derived data). `existing` is
+ // the full get() view — every reserved field top-level — and must be
+ // passed whole: a subset view makes the old-side decrement miss any
+ // reserved-field group (update would then double-count it).
if (this._aggregationIndex) {
- const oldEntityForAgg = {
- type: existing.type,
- service: existing.service,
- data: existing.data,
- metadata: existing.metadata
- }
- this._aggregationIndex.onEntityUpdated(params.id, entityForIndexing, oldEntityForAgg)
+ this._aggregationIndex.onEntityUpdated(
+ params.id,
+ entityForIndexing,
+ existing as unknown as Record
+ )
}
}
@@ -2603,9 +3228,11 @@ export class Brainy implements BrainyInterface {
)
}
- // Operation 3: Delete noun metadata
+ // Operation 3: Delete noun (full removal). The pre-read metadata rides
+ // along so the count decrement never depends on re-reading the record
+ // being removed (a null re-read must not silently skip it).
tx.addOperation(
- new DeleteNounMetadataOperation(this.storage, id)
+ new DeleteNounMetadataOperation(this.storage, id, metadata)
)
// Operations 4+: Delete all related verbs atomically
@@ -2614,28 +3241,52 @@ export class Brainy implements BrainyInterface {
// rollback can re-add through the BigInt addVerb contract)
const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb)
tx.addOperation(
- new RemoveFromGraphIndexOperation(this.graphIndex, verb, sourceInt, targetInt, this.graphWriteGeneration)
+ new RemoveFromGraphIndexOperation(this.graphIndex, verb, { sourceInt, targetInt }, this.graphWriteGeneration)
)
// Delete verb metadata
tx.addOperation(
new DeleteVerbMetadataOperation(this.storage, verb.id)
)
}
- })
+ },
+ undefined,
+ this._changeFeed.hasListeners
+ ? [
+ // The entity delete (payload = last state, from the pre-delete read)
+ // plus one unrelate per cascade-deleted relationship.
+ {
+ kind: 'entity',
+ op: 'remove',
+ id,
+ ...(metadata && {
+ entity: this.entityViewFromRawRecord(id, metadata as Record)
+ })
+ },
+ ...allVerbs.map(
+ (v): PendingChangeEvent => ({
+ kind: 'relation',
+ op: 'unrelate',
+ id: v.id,
+ relation: {
+ id: v.id,
+ from: v.sourceId,
+ to: v.targetId,
+ type: String(v.verb)
+ }
+ })
+ )
+ ]
+ : undefined)
- // Aggregation hook (outside transaction — derived data)
+ // Aggregation hook (outside transaction — derived data). The view must
+ // carry EVERY reserved field top-level (not a subset): a groupBy on
+ // subtype/visibility/etc. otherwise decrements a nonexistent group and
+ // the real count never comes down.
if (this._aggregationIndex && metadata) {
- // Reconstruct entity-like object from stored metadata via the
- // canonical reserved/custom split (the hand-rolled destructure here
- // missed subtype/_rev, leaking them into the aggregation view).
- const { reserved, custom } = splitNounMetadataRecord(metadata)
- const entityForAgg = {
- type: reserved.noun,
- service: reserved.service,
- data: reserved.data,
- metadata: custom
- }
- this._aggregationIndex.onEntityDeleted(id, entityForAgg)
+ this._aggregationIndex.onEntityDeleted(
+ id,
+ this.entityForAggFromRawRecord(metadata as Record)
+ )
}
}
@@ -2914,6 +3565,258 @@ export class Brainy implements BrainyInterface {
}
}
+ /**
+ * @description The metadata field-index counterpart of {@link verifyGraphAdjacencyLive}.
+ * On a cold open a native metadata provider can report data yet not serve its
+ * `where` postings, so `find({ where })` silently returns `[]` — the exact
+ * failure a downstream deployment reported (cold reads blanking filtered pages
+ * after every restart). This one-shot guard, run on the first FILTERED `find()`,
+ * closes that: it takes a KNOWN persisted entity + one of its plain field values
+ * and asks the index to resolve it. If the index returns the known id the field
+ * postings are live (the common case, and the ONLY cost on a warm brain — one
+ * O(1) probe). If it does not, the postings did not load: brainy rebuilds the
+ * index from the canonical records and re-probes; if it STILL cannot serve the
+ * known value it throws a loud {@link MetadataIndexNotReadyError} rather than
+ * let a silent `[]` stand. Inconclusive cases (empty store, no plain field to
+ * probe, a shared store surfacing a foreign entity) are treated as live — never
+ * a false rebuild. A migrating provider is skipped (it owns its locked rebuild).
+ * @returns `'live'` when the index serves, `'rebuilt'` when a rebuild restored it.
+ */
+ private async verifyMetadataLive(): Promise<'live' | 'rebuilt'> {
+ if (this._metadataVerified) return 'live'
+ // Migration LOCK (#18): a migrating provider owns its in-place rebuild — do
+ // not race it. Defensive; the data-plane lock already gates callers upstream.
+ if (this.providerIsMigrating(this.metadataIndex)) return 'live'
+ // Re-entrancy: rebuild() can trigger reads that call back into this guard.
+ if (this._metadataVerifying) return 'live'
+ this._metadataVerifying = true
+ try {
+ // A KNOWN persisted entity + one plain field to probe. Sample a few so a
+ // system-only entity (e.g. the VFS root) doesn't make every open inconclusive.
+ const sample = await this.storage.getNouns({ pagination: { limit: 5, offset: 0 } })
+ let probe: { field: string; value: string | number | boolean; id: string } | null = null
+ for (const noun of sample.items ?? []) {
+ probe = this.pickMetadataProbe(noun as { id?: string; metadata?: Record })
+ if (probe) break
+ }
+ if (!probe) {
+ // Empty store, or nothing with a plain user field to probe — inconclusive.
+ this._metadataVerified = true
+ return 'live'
+ }
+ const p = probe
+
+ const probeServes = async (): Promise => {
+ try {
+ const ids = await this.metadataIndex.getIdsForFilter({ [p.field]: p.value })
+ return ids.includes(p.id)
+ } catch {
+ // FIELD_NOT_INDEXED for a field a persisted entity actually holds is
+ // itself the cold/broken signal — treat as not-serving (→ rebuild).
+ return false
+ }
+ }
+
+ if (await probeServes()) {
+ this._metadataVerified = true
+ return 'live' // field postings are live — the common case
+ }
+
+ if (!this.config.silent) {
+ console.warn(
+ `[Brainy] Metadata field index returns no match for a known persisted value of ` +
+ `'${p.field}' — the field postings did not load on open. Rebuilding from storage…`
+ )
+ }
+ await this.metadataIndex.rebuild()
+
+ if (await probeServes()) {
+ this._metadataVerified = true
+ return 'rebuilt'
+ }
+ throw new MetadataIndexNotReadyError(
+ `Metadata field index cannot serve a known persisted value of '${p.field}' even after ` +
+ `a rebuild — find({ where }) and other filtered reads cannot be served reliably for ` +
+ `this brain (a silent empty result would misrepresent existing data).`
+ )
+ } catch (err) {
+ if (err instanceof MetadataIndexNotReadyError) throw err
+ // A transient probe/rebuild failure must not break the query NOR mask as
+ // "no data". Allow a re-check on the next filtered read and fall through.
+ this._metadataVerified = false
+ if (!this.config.silent) {
+ console.warn(`[Brainy] Metadata consistency check skipped (transient): ${err}`)
+ }
+ return 'live'
+ } finally {
+ this._metadataVerifying = false
+ }
+ }
+
+ /**
+ * @description Choose one plain (scalar, user-written) field from an entity's
+ * metadata to probe the field index with — skipping internal / system fields
+ * (`__words__` text hash, VFS markers, reserved `visibility`/`subtype`/`service`,
+ * the `noun` type alias, timestamps) that use different index paths, and any
+ * non-scalar value. Returns `null` when the entity has no probeable field.
+ */
+ private pickMetadataProbe(
+ noun: { id?: string; metadata?: Record }
+ ): { field: string; value: string | number | boolean; id: string } | null {
+ const id = noun?.id
+ const metadata = noun?.metadata
+ if (!id || !metadata || typeof metadata !== 'object') return null
+ const skip = new Set([
+ 'noun', 'type', 'subtype', 'service', 'visibility', 'id', 'vector',
+ 'isVFSEntity', 'vfsType', 'vfsPath', 'vfsName', 'createdAt', 'updatedAt'
+ ])
+ for (const [field, value] of Object.entries(metadata)) {
+ if (field.startsWith('_') || skip.has(field)) continue
+ if (value === null || value === undefined) continue
+ const t = typeof value
+ if (t === 'string' || t === 'number' || t === 'boolean') {
+ return { field, value: value as string | number | boolean, id }
+ }
+ }
+ return null
+ }
+
+ /**
+ * @description The vector-index counterpart of {@link verifyGraphAdjacencyLive}
+ * / {@link verifyMetadataLive}. On a cold open a native vector provider can
+ * report a non-zero `size()` (its persisted COUNT loaded) yet not have loaded
+ * its serving structure (the mmap/DiskANN graph) — so a pure semantic
+ * `find({ query })` silently returns `[]`. A pure semantic query has
+ * `hasFilterCriteria === false`, so the metadata guard never fires; this guard
+ * closes that gap. Run one-shot on the first vector/proximity search:
+ * - **Preferred (honest signal):** the provider exposes `isReady()`. `false`
+ * → rebuild from storage, re-check; if still `false`, throw
+ * {@link VectorIndexNotReadyError} rather than serving `[]`.
+ * - **Fallback (no `isReady()`):** a KNOWN persisted vector (sampled +
+ * hydrated) is searched against the index; if it does not self-match, the
+ * serving structure did not load — rebuild + re-probe, else throw.
+ * Inconclusive cases (empty store, no probeable vector, `size()===0` — where
+ * the JS baseline's cold load is `ensureIndexesLoaded`'s job) are treated as
+ * live: never a false rebuild. A migrating provider is skipped (it owns its
+ * locked rebuild).
+ * @returns `'live'` when the index serves, `'rebuilt'` when a rebuild restored it.
+ */
+ private async verifyVectorLive(): Promise<'live' | 'rebuilt'> {
+ if (this._vectorVerified) return 'live'
+ // Migration LOCK (#18): a migrating provider owns its in-place rebuild.
+ if (this.providerIsMigrating(this.index)) return 'live'
+ // Re-entrancy: rebuild() can trigger reads that call back into this guard.
+ if (this._vectorVerifying) return 'live'
+ this._vectorVerifying = true
+ try {
+ // ── Strategy 1: honest isReady() signal (native provider) ──────────────
+ const readiness = assessIndexReadiness(this.index)
+ if (readiness !== 'unknown') {
+ if (readiness === 'ready') {
+ this._vectorVerified = true
+ return 'live'
+ }
+ // Not ready: the serving structure did not load on open. Rebuild.
+ if (!this.config.silent) {
+ console.warn(
+ `[Brainy] Vector index reports not-ready (isReady() === false) — the persisted ` +
+ `vector index did not load on open. Rebuilding from storage…`
+ )
+ }
+ await this.index.rebuild()
+ if (assessIndexReadiness(this.index) === 'ready') {
+ this._vectorVerified = true
+ return 'rebuilt'
+ }
+ throw new VectorIndexNotReadyError(
+ `Vector index reports not-ready even after a rebuild — semantic find({ query }) and ` +
+ `proximity search cannot be served reliably for this brain (a silent empty result ` +
+ `would misrepresent existing data).`
+ )
+ }
+
+ // ── Strategy 2: known-vector probe (providers without isReady()) ───────
+ const claimed = this.index.size()
+ if (!claimed || claimed <= 0) return 'live' // JS cold path is ensureIndexesLoaded's job
+
+ const probe = await this.pickVectorProbe()
+ if (!probe) {
+ // Empty store, or nothing with a probeable vector — inconclusive.
+ this._vectorVerified = true
+ return 'live'
+ }
+ const p = probe
+
+ const probeServes = async (): Promise => {
+ // The failure mode we guard is the SILENT EMPTY result: a cold index that
+ // loaded its COUNT but not its serving structure returns `[]` for a
+ // known-present vector, while a warm index returns at least one hit. We
+ // check for a NON-EMPTY result, NOT an exact self-match — HNSW is
+ // approximate and `get()` may return a re-hydrated/normalized vector, so
+ // demanding the exact self as top-1 would false-positive on a perfectly
+ // healthy index (and wrongly rebuild → throw).
+ const hits = await this.index.search(p.vector, 1)
+ return hits.length > 0
+ }
+ void p.id // probe keyed on the vector; id retained for diagnostics only
+
+ if (await probeServes()) {
+ this._vectorVerified = true
+ return 'live' // serving structure is live — the common case
+ }
+
+ if (!this.config.silent) {
+ console.warn(
+ `[Brainy] Vector index reports ${claimed} vector(s) but a known persisted vector ` +
+ `returns no results — the serving structure did not load on open. Rebuilding…`
+ )
+ }
+ await this.index.rebuild()
+
+ if (await probeServes()) {
+ this._vectorVerified = true
+ return 'rebuilt'
+ }
+ throw new VectorIndexNotReadyError(
+ `Vector index reports ${claimed} vector(s) but a known persisted vector returns no ` +
+ `results even after a rebuild — semantic find({ query }) cannot be served reliably ` +
+ `for this brain (a silent empty result would misrepresent existing data).`
+ )
+ } catch (err) {
+ if (err instanceof VectorIndexNotReadyError) throw err
+ // A transient probe/rebuild failure must not break the query NOR mask as
+ // "no data". Allow a re-check on the next vector read and fall through.
+ this._vectorVerified = false
+ if (!this.config.silent) {
+ console.warn(`[Brainy] Vector consistency check skipped (transient): ${err}`)
+ }
+ return 'live'
+ } finally {
+ this._vectorVerifying = false
+ }
+ }
+
+ /**
+ * @description Sample a KNOWN persisted noun and hydrate its vector, to probe
+ * the vector index with. `get()` omits vectors by default, so this passes
+ * `{ includeVectors: true }`. Samples a few (a system-only / vectorless entity
+ * must not make every open inconclusive). Returns `null` when nothing has a
+ * probeable vector.
+ */
+ private async pickVectorProbe(): Promise<{ id: string; vector: number[] } | null> {
+ const sample = await this.storage.getNouns({ pagination: { limit: 5, offset: 0 } })
+ for (const noun of sample.items ?? []) {
+ const id = (noun as { id?: string }).id
+ if (!id) continue
+ const full = await this.get(id, { includeVectors: true })
+ const vector = (full as { vector?: number[] } | null)?.vector
+ if (Array.isArray(vector) && vector.length > 0) {
+ return { id, vector }
+ }
+ }
+ return null
+ }
+
// -------------------------------------------------------------------------
/**
@@ -3180,7 +4083,7 @@ export class Brainy implements BrainyInterface {
// Operation 3: Add to graph index for O(1) lookups
tx.addOperation(
new AddToGraphIndexOperation(
- this.graphIndex, verb, sourceInt, targetInt,
+ this.graphIndex, verb, { sourceInt, targetInt },
this.graphWriteGeneration,
(verbInt) => this.cacheVerbInt(verbInt, id)
)
@@ -3218,13 +4121,50 @@ export class Brainy implements BrainyInterface {
// Operation 6: Add reverse relationship to graph index
tx.addOperation(
new AddToGraphIndexOperation(
- this.graphIndex, reverseVerb, targetInt, sourceInt,
+ this.graphIndex, reverseVerb, { sourceInt: targetInt, targetInt: sourceInt },
this.graphWriteGeneration,
(verbInt) => this.cacheVerbInt(verbInt, reverseId)
)
)
}
- })
+ },
+ undefined,
+ this._changeFeed.hasListeners
+ ? [
+ {
+ kind: 'relation',
+ op: 'relate',
+ id,
+ relation: {
+ id,
+ from: params.from,
+ to: params.to,
+ type: String(params.type),
+ ...(params.metadata && {
+ metadata: params.metadata as Record
+ })
+ }
+ },
+ ...(reverseId
+ ? [
+ {
+ kind: 'relation',
+ op: 'relate',
+ id: reverseId,
+ relation: {
+ id: reverseId,
+ from: params.to,
+ to: params.from,
+ type: String(params.type),
+ ...(params.metadata && {
+ metadata: params.metadata as Record
+ })
+ }
+ } as PendingChangeEvent
+ ]
+ : [])
+ ]
+ : undefined)
return id
}
@@ -3263,7 +4203,7 @@ export class Brainy implements BrainyInterface {
if (verb && endpointInts) {
tx.addOperation(
new RemoveFromGraphIndexOperation(
- this.graphIndex, verb, endpointInts.sourceInt, endpointInts.targetInt, this.graphWriteGeneration
+ this.graphIndex, verb, endpointInts, this.graphWriteGeneration
)
)
}
@@ -3272,7 +4212,26 @@ export class Brainy implements BrainyInterface {
tx.addOperation(
new DeleteVerbMetadataOperation(this.storage, id)
)
- })
+ },
+ undefined,
+ this._changeFeed.hasListeners && verb
+ ? [
+ {
+ kind: 'relation',
+ op: 'unrelate',
+ id,
+ relation: {
+ id,
+ from: verb.sourceId,
+ to: verb.targetId,
+ type: String(verb.verb),
+ ...(verb.metadata && {
+ metadata: verb.metadata as Record
+ })
+ }
+ }
+ ]
+ : undefined)
}
/**
@@ -3396,18 +4355,37 @@ export class Brainy implements BrainyInterface {
if (typeChanged && reindexInts) {
tx.addOperation(
new RemoveFromGraphIndexOperation(
- this.graphIndex, existing, reindexInts.sourceInt, reindexInts.targetInt, this.graphWriteGeneration
+ this.graphIndex, existing, reindexInts, this.graphWriteGeneration
)
)
tx.addOperation(
new AddToGraphIndexOperation(
- this.graphIndex, verbForIndex, reindexInts.sourceInt, reindexInts.targetInt,
+ this.graphIndex, verbForIndex, reindexInts,
this.graphWriteGeneration,
(verbInt) => this.cacheVerbInt(verbInt, params.id)
)
)
}
- })
+ },
+ undefined,
+ this._changeFeed.hasListeners
+ ? [
+ {
+ kind: 'relation',
+ op: 'updateRelation',
+ id: params.id,
+ relation: {
+ id: params.id,
+ from: verbForIndex.sourceId,
+ to: verbForIndex.targetId,
+ type: String(verbForIndex.verb ?? verbForIndex.type),
+ ...(verbForIndex.metadata && {
+ metadata: verbForIndex.metadata as Record
+ })
+ }
+ }
+ ]
+ : undefined)
}
/**
@@ -3446,7 +4424,9 @@ export class Brainy implements BrainyInterface {
async related(
paramsOrId?: string | RelatedParams
): Promise[]> {
- await this.ensureInitialized()
+ // Graph read: relationship traversal consults only the graph adjacency
+ // family, so it waits on a graph migration but not on vector/metadata.
+ await this.ensureInitialized({ needs: ['graph'] })
await this.verifyGraphAdjacencyLive()
// Handle string ID shorthand: related(id) -> related({ from: id })
@@ -4888,6 +5868,10 @@ export class Brainy implements BrainyInterface {
): Promise {
await this.ensureInitialized()
this.ensureAggregationIndex()
+ // Persisted definitions load asynchronously — wait for them before deciding
+ // the aggregate doesn't exist (an app that relies on persisted definitions
+ // without re-defining at boot would otherwise race a spurious throw here).
+ await this._aggregationIndex!.ready()
if (!this._aggregationIndex!.hasAggregate(name)) {
throw new Error(`Aggregate '${name}' is not defined. Call defineAggregate() first.`)
}
@@ -4906,8 +5890,17 @@ export class Brainy implements BrainyInterface {
this._aggregationIndex = new AggregationIndex(this.storage, nativeProvider)
// Note: init() is async but definitions can be registered synchronously.
// State loading happens lazily on first query.
- this._aggregationIndex.init().catch(() => {
- // Non-fatal — aggregation state will be empty but definitions still work
+ this._aggregationIndex.init().catch((err) => {
+ // Non-fatal — definitions still work and state backfills on first query —
+ // but a failed state load means aggregates read empty/stale until then, so
+ // surface it loudly rather than swallow it.
+ if (!this.config.silent) {
+ prodLog.warn(
+ `[Brainy] Aggregation index state failed to load at init ` +
+ `(${(err as Error).message}). Aggregates may read empty until a ` +
+ `backfill-on-query repopulates them.`
+ )
+ }
})
}
@@ -4976,7 +5969,11 @@ export class Brainy implements BrainyInterface {
* })
*/
async find(query: string | FindParams): Promise[]> {
- await this.ensureInitialized()
+ // Init only here — the migration gate is applied family-scoped below, once
+ // the query params reveal which index families this read actually consults
+ // (a string query may parse to a where / connected / vector shape). The lazy
+ // loader and cold-read probes below already defer to a migrating provider.
+ await this.ensureInitialized({ needs: [] })
// Ensure indexes are loaded (lazy loading when disableAutoRebuild: true)
// This is a production-safe, concurrency-controlled lazy load
@@ -4987,6 +5984,12 @@ export class Brainy implements BrainyInterface {
// metadata counterpart of the graph cold-load guard. No-op for the JS index.
await this.ensureMetadataConsistencyProbed()
+ // Loudly flag a degraded derived index (failed init rebuild, or an
+ // adopt-forward degraded commit) so a partial result is never mistaken for
+ // authoritative. The streaming search() generator delegates to find(), so it
+ // is covered here.
+ this.warnIfReadsDegraded('find')
+
// Parse natural language queries
let params: FindParams =
typeof query === 'string' ? await this.parseNaturalQuery(query) : query
@@ -5009,6 +6012,13 @@ export class Brainy implements BrainyInterface {
// Zero-config validation (static import for performance)
validateFindParams(params)
+ // Family-scoped migration gate: wait only on the index families THIS query
+ // consults, so a filter or graph query served from a healthy family never
+ // blocks on an unrelated family's one-time 7.x → 8.0 migration. Placed once
+ // params are final (after natural-language parse + connected-id resolution)
+ // and before the aggregate path, which carries no gate of its own.
+ await this.awaitMigrationLock(this.queryIndexFamilies(params))
+
// Aggregate query path — early return when params.aggregate is set
if (params.aggregate) {
return this.findAggregate(params)
@@ -5031,6 +6041,24 @@ export class Brainy implements BrainyInterface {
const hasFilterCriteria = params.where || params.type || params.subtype || params.service
const hasGraphCriteria = params.connected
+ // Validate the raw where clause up front: an unrecognized operator (a typo
+ // like `notIn`, or any non-operator key on a field's object value) throws a
+ // typed BrainyError('INVALID_QUERY') instead of silently matching nothing.
+ // Done before the index/vector/graph paths so it fires even on an empty
+ // result set, and before brainy injects its own internal filter fields.
+ if (params.where) {
+ validateWhereFilter(params.where)
+ }
+
+ // Metadata cold-read guard: before trusting a filter result, verify the
+ // field index actually serves a known persisted value (one-shot per brain).
+ // A cold native index that has not loaded its `where` postings self-heals
+ // here (rebuild) or throws MetadataIndexNotReadyError — never a silent [].
+ // The graph counterpart (verifyGraphAdjacencyLive) covers `connected`.
+ if (hasFilterCriteria) {
+ await this.verifyMetadataLive()
+ }
+
// Handle metadata-only queries (no vector search needed)
if (!hasVectorSearchCriteria && !hasGraphCriteria && hasFilterCriteria) {
// Build filter for metadata index
@@ -5682,10 +6710,13 @@ export class Brainy implements BrainyInterface {
/**
* Add multiple entities in a single batch operation
*
- * Uses batch embedding (embedBatch) to pre-compute all vectors in a single
- * WASM forward pass instead of N individual embed() calls, providing 5-10x
- * speedup on bulk inserts. Automatically adapts batch size and parallelism
- * to the storage adapter (e.g., smaller batches for cloud storage).
+ * Uses batch embedding (embedBatch) to pre-compute all vectors before any
+ * storage write, keeping model inference out of the per-item write path.
+ * (On the default WASM engine, batch throughput is comparable to sequential
+ * embed() calls — measured ~160 ms/text either way; a native embedding
+ * provider may batch faster.) Automatically adapts batch size and
+ * parallelism to the storage adapter (e.g., smaller batches for cloud
+ * storage).
*
* @param params - Batch add parameters
* @param params.items - Array of AddParams (same shape as brain.add())
@@ -5883,6 +6914,30 @@ export class Brainy implements BrainyInterface {
this.assertWritable('removeMany')
await this.ensureInitialized()
+ // Loud selector validation: a call with no usable selector used to
+ // resolve successfully having deleted NOTHING (total: 0) — the classic
+ // silent no-op being a bare array passed positionally
+ // (removeMany([id]) instead of removeMany({ ids: [id] })). The caller
+ // believes the delete happened; every count derived afterwards is "wrong"
+ // while the engine was never even asked. Refuse instead.
+ if (Array.isArray(params)) {
+ throw new Error(
+ `removeMany() takes a params object, not a bare array — use removeMany({ ids: [...] })`
+ )
+ }
+ if (!params || (!params.ids && !params.type && !params.where)) {
+ throw new Error(
+ `removeMany() requires a selector: { ids } and/or { type, where }. ` +
+ `An empty selector would silently delete nothing — refusing.`
+ )
+ }
+ if (params.ids && params.ids.length === 0) {
+ throw new Error(
+ `removeMany() received ids: [] — an empty id list deletes nothing. ` +
+ `Pass the ids to delete, or omit ids and select by { type, where }.`
+ )
+ }
+
// Determine what to delete
let idsToDelete: string[] = []
@@ -5946,6 +7001,16 @@ export class Brainy implements BrainyInterface {
}
try {
+ // Change feed: the BUILDER populates this (per id actually staged), so
+ // an id whose builder failed never emits — the array is read only
+ // after the chunk's commit succeeds. Verb events dedupe within the
+ // chunk (two related chunk-members see the same cascade verb twice).
+ const chunkEvents: PendingChangeEvent[] | undefined = this._changeFeed
+ .hasListeners
+ ? []
+ : undefined
+ const seenVerbEvents = new Set()
+
// Process chunk as ONE atomic Model-B generation (entities + cascade verbs).
await this.persistSingleOp({ nouns: touchedNouns, verbs: touchedVerbs }, async (tx) => {
for (const id of chunk) {
@@ -5970,20 +7035,51 @@ export class Brainy implements BrainyInterface {
)
}
+ // Pre-read metadata rides along: the count decrement must not
+ // depend on re-reading the record being removed (see remove()).
tx.addOperation(
- new DeleteNounMetadataOperation(this.storage, id)
+ new DeleteNounMetadataOperation(this.storage, id, metadata)
)
for (const verb of allVerbs) {
const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb)
tx.addOperation(
- new RemoveFromGraphIndexOperation(this.graphIndex, verb, sourceInt, targetInt, this.graphWriteGeneration)
+ new RemoveFromGraphIndexOperation(this.graphIndex, verb, { sourceInt, targetInt }, this.graphWriteGeneration)
)
tx.addOperation(
new DeleteVerbMetadataOperation(this.storage, verb.id)
)
}
+ if (chunkEvents) {
+ chunkEvents.push({
+ kind: 'entity',
+ op: 'remove',
+ id,
+ ...(metadata && {
+ entity: this.entityViewFromRawRecord(
+ id,
+ metadata as Record
+ )
+ })
+ })
+ for (const verb of allVerbs) {
+ if (seenVerbEvents.has(verb.id)) continue
+ seenVerbEvents.add(verb.id)
+ chunkEvents.push({
+ kind: 'relation',
+ op: 'unrelate',
+ id: verb.id,
+ relation: {
+ id: verb.id,
+ from: verb.sourceId,
+ to: verb.targetId,
+ type: String(verb.verb)
+ }
+ })
+ }
+ }
+
chunkQueued.push(id)
} catch (error) {
chunkBuilderFailed.push({
@@ -5995,7 +7091,7 @@ export class Brainy implements BrainyInterface {
}
}
}
- })
+ }, undefined, chunkEvents)
// Transaction committed — queued IDs were actually deleted
result.successful.push(...chunkQueued)
@@ -6301,6 +7397,12 @@ export class Brainy implements BrainyInterface {
// VFS was never used, reset flag for clean state
this._vfsInitialized = false
}
+
+ // Change feed: clear() wipes the store wholesale (raw, not per-record) —
+ // one store-level event tells subscribers everything they held is gone.
+ this._changeFeed.emit([
+ { kind: 'store', op: 'clear', timestamp: Date.now() }
+ ])
}
// ─── Migration API ───────────────────────────────────────────────
@@ -6489,6 +7591,62 @@ export class Brainy implements BrainyInterface {
await writeBrainFormat(this.storage)
this._brainFormat = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH }
this._indexEpochStale = false
+ // The upgrade has verified + stamped — drop the pre-upgrade backup (no-op if
+ // none was taken, e.g. a non-migrating stamp of a fresh brain's initial marker).
+ await this.removeMigrationBackupSafe()
+ }
+
+ /**
+ * @description Open a sequential scan over the generation FACT LOG — the
+ * append-only record of every committed generation as an AFTER-IMAGE fact
+ * (what each touched entity/relationship became, or a body-less tombstone
+ * for a removal). The scan is the streaming substrate for index heals and
+ * incremental replays: one sequential read in commit order replaces a
+ * per-entity directory walk. The handle carries heal-narration telemetry
+ * (`headGeneration` / `segmentCount` / `approxFactCount` up front; ordered
+ * batches each stamped with their generation range, byte size, and segment;
+ * a `summary()` cross-check after iteration). A detected gap or damaged
+ * segment ABORTS the scan loudly — never a silent skip.
+ *
+ * Returns `null` when this store hosts no fact log: the storage adapter
+ * lacks the binary append primitives, or the brain predates the fact log
+ * (its history began before dual-write — facts exist only from the first
+ * write after upgrade; callers fall back to the canonical enumeration walk).
+ *
+ * @param options - `fromGeneration`/`toGeneration` bound the scan (inclusive
+ * both ends); `kinds` filters ops to `'noun'`/`'verb'`; `batchSize` caps
+ * facts per yielded batch (default 256).
+ * @returns The scan handle, or `null` when no fact log exists.
+ * @example
+ * const scan = brain.scanFacts({ fromGeneration: 1 })
+ * if (scan) {
+ * for await (const batch of scan.batches()) {
+ * for (const fact of batch.facts) {
+ * // fact.ops: [{ kind, id, record | null (tombstone) }, ...]
+ * }
+ * }
+ * }
+ */
+ scanFacts(options?: {
+ fromGeneration?: number
+ toGeneration?: number
+ kinds?: Array<'noun' | 'verb'>
+ batchSize?: number
+ }): FactScanHandle | null {
+ const factLog = this.generationStore?.getFactLog()
+ return factLog ? factLog.scanFacts(options) : null
+ }
+
+ /**
+ * @description The immutable, sealed fact-log segment files covering
+ * `fromGeneration` — the zero-copy handoff for consumers that map segment
+ * files directly instead of streaming {@link scanFacts} batches. The
+ * append-mutable TAIL segment is deliberately excluded (read it via
+ * `scanFacts`). Paths are storage-root-relative. Empty when no fact log
+ * exists or nothing is sealed yet.
+ */
+ factSegmentPaths(options?: { fromGeneration?: number }): string[] {
+ return this.generationStore?.getFactLog()?.segmentPaths(options) ?? []
}
/**
@@ -6668,18 +7826,78 @@ export class Brainy implements BrainyInterface {
const plan = await this.planTransact(ops)
- const { generation, timestamp } = await this.generationStore.commitTransaction({
- touched: { nouns: plan.touchedNouns, verbs: plan.touchedVerbs },
- meta: options?.meta,
- ifAtGeneration: options?.ifAtGeneration,
- execute: async () => {
- await this.transactionManager.executeTransaction(async (tx) => {
- for (const operation of plan.operations) {
- tx.addOperation(operation)
+ // Authoritative per-op ifRev CAS, run by the generation store UNDER the
+ // commit mutex against the just-read before-images — atomic with the
+ // apply (the planning-time checks above are fast-fails that can
+ // interleave with concurrent writers). Any conflict rejects the WHOLE
+ // batch before anything is staged or applied. Sequencing rules:
+ // - runningRev tracks multiple updates to the SAME entity in one batch
+ // (each op CASes against — and bumps — its predecessor's result).
+ // - An entity the batch itself creates (null before-image, forward-ref
+ // add+update) keeps its plan-sequenced stamps: no concurrent writer
+ // can have moved a rev that did not exist, so plan-time was exact.
+ const casPrecommit = (before: CommitBeforeImages): void => {
+ const runningRev = new Map()
+ for (const cas of plan.casUpdates) {
+ // An id this batch adds owns its revision baseline (the add restamps
+ // `_rev: 1` even on overwrite) — plan-time sequencing is exact, no
+ // concurrent writer can move a baseline the batch itself sets.
+ if (plan.createdNouns.has(cas.id)) {
+ continue
+ }
+ const beforeMeta = before.nouns.get(cas.id)?.metadata as
+ | { _rev?: unknown }
+ | null
+ | undefined
+ if (!beforeMeta && !runningRev.has(cas.id)) {
+ // A pre-existing entity a concurrent remove() deleted after
+ // planning: a CAS caller gets the truth; a plain update keeps its
+ // last-writer-wins re-create semantics (plan-stamped rev).
+ if (typeof cas.ifRev === 'number') {
+ throw new EntityNotFoundError(cas.id)
}
- })
+ continue
+ }
+ const base =
+ runningRev.get(cas.id) ??
+ (typeof beforeMeta?._rev === 'number' ? beforeMeta._rev : 1)
+ if (typeof cas.ifRev === 'number' && cas.ifRev !== base) {
+ throw new RevisionConflictError(cas.id, cas.ifRev, base)
+ }
+ const next = base + 1
+ cas.updatedMetadata._rev = next
+ runningRev.set(cas.id, next)
}
- })
+ }
+
+ let generation: number
+ let timestamp: number
+ try {
+ ;({ generation, timestamp } = await this.generationStore.commitTransaction({
+ touched: { nouns: plan.touchedNouns, verbs: plan.touchedVerbs },
+ meta: options?.meta,
+ ifAtGeneration: options?.ifAtGeneration,
+ precommit: casPrecommit,
+ execute: async () => {
+ await this.transactionManager.executeTransaction(
+ async (tx) => {
+ for (const operation of plan.operations) {
+ tx.addOperation(operation)
+ }
+ },
+ // Budget scales with the batch (or the caller's explicit
+ // timeoutMs): a flat 30s cap silently limited honest bulk work
+ // to ~15 ops on network disks (~2s/op measured in the field).
+ { timeout: transactTimeoutBudget(plan.operations.length, options?.timeoutMs) }
+ )
+ }
+ }))
+ } catch (err) {
+ // A batch whose rollback failed and left canonical records unreconciled
+ // quarantines writes until repairIndex() reconciles and lifts it.
+ if (err instanceof StoreInconsistentError) this.storeInconsistency = err
+ throw err
+ }
// Aggregation-index maintenance: derived data, applied after the commit
// point — exactly where the single-operation methods apply it.
@@ -6687,6 +7905,10 @@ export class Brainy implements BrainyInterface {
hook()
}
+ // Change feed: the batch's events share its single committed generation.
+ // A rejected batch throws at commitTransaction and never reaches here.
+ this.emitCommitted(plan.changeEvents, undefined, generation, timestamp)
+
const receipt: TransactReceipt = { generation, timestamp, ids: plan.ids }
return this.createPinnedDb({ generation, timestamp, receipt })
}
@@ -7043,6 +8265,72 @@ export class Brainy implements BrainyInterface {
return this.generationStore.compact(options)
}
+ /**
+ * @description Repack cold generation history into sealed segments —
+ * re-representation, never deletion: every record and delta stays readable
+ * (`asOf()` unchanged); the physical file count drops by orders of
+ * magnitude. Runs automatically (time-bounded) at `close()`; call this for
+ * explicit maintenance windows on long-lived writers. The ONLY history
+ * transform permitted under the archival profile (`retention: 'all'`).
+ * @param options - `timeBudgetMs` bounds the pass (early stop = consistent
+ * prefix, next pass resumes); `batchGenerations` sizes each fold.
+ * @returns Folded generation count and segments created.
+ */
+ async repackHistory(options?: {
+ timeBudgetMs?: number
+ batchGenerations?: number
+ }): Promise<{ foldedGenerations: number; segmentsCreated: number }> {
+ this.assertWritable('repackHistory')
+ await this.ensureInitialized()
+ await this.generationStore.flushPendingSingleOps()
+ return this.generationStore.repackHistory(options)
+ }
+
+ /**
+ * @description Read-only generational-history footprint for fleet audits:
+ * generation count, total on-disk bytes, generation/timestamp range, the
+ * compaction horizon, and the retention policy in force. Touches no data and
+ * changes nothing. First call pays one walk over committed deltas to seed
+ * the running byte total (subsequent calls — and every adaptive retention
+ * check — are then O(1)).
+ *
+ * A pool operator's exposure check is one call per brain:
+ * @example
+ * const stats = await brain.historyStats()
+ * console.log(`${stats.generations} generations, ${stats.bytes} bytes, mode=${stats.retentionMode}`)
+ */
+ async historyStats(): Promise {
+ await this.ensureInitialized()
+ const stats = await this.generationStore.historyStats()
+ const policy = this.resolveRetentionPolicy()
+ return {
+ ...stats,
+ retentionMode: policy.mode,
+ effectiveBudgetBytes:
+ policy.mode === 'adaptive'
+ ? this.adaptiveHistoryBudgetBytes(policy.budgetBytes)
+ : null
+ }
+ }
+
+ /**
+ * @description A deterministic content digest of the generation log through
+ * `g` (D8 — gate-to-generation provenance): identical history produces the
+ * identical digest on any machine; divergence produces a different one.
+ * Release gates and suite verdicts pin `{generation, digest}` and verify
+ * both at execution time instead of pinning a git commit. O(segments +
+ * live-tier window), never O(all generations). Throws `RangeError` out of
+ * range and `GenerationCompactedError` below the horizon — a gate can
+ * never silently pin reclaimed history.
+ * @example
+ * const gate = { generation: brain.generation(), digest: await brain.generationDigest(brain.generation()) }
+ */
+ async generationDigest(g: number): Promise {
+ await this.ensureInitialized()
+ await this.generationStore.flushPendingSingleOps()
+ return this.generationStore.generationDigest(g)
+ }
+
/**
* @description Drive the adaptive retention byte budget at runtime — the
* settable input a machine-level coordinator (e.g. cor's `ResourceManager`,
@@ -7122,19 +8410,24 @@ export class Brainy implements BrainyInterface {
/**
* @description Run history compaction under the resolved `retention` policy
- * when `autoCompact` is on (the default). Invoked from `flush()` and
- * `close()` so generational record-sets cannot accumulate unbounded across a
- * long-lived writer's lifetime.
+ * when `autoCompact` is on (the default). Invoked from `close()` ONLY
+ * (8.9.0) — flush() is durability work and never pays maintenance costs; a
+ * production deployment measured reclaim-on-flush blocking single writes
+ * for 25-191s under memory pressure. Long-lived writers that never close
+ * accumulate history until their next explicit `compactHistory()` — the
+ * documented trade: predictable writes, explicit maintenance.
*
* - `'all'` → returns without reclaiming (index compaction for speed still
* runs elsewhere; history is decoupled and kept).
* - `'adaptive'` → reclaim oldest-unpinned history down to the byte budget.
* - `'explicit'` → apply the supplied `maxGenerations`/`maxAge`/`maxBytes` caps.
*
+ * Every auto pass is TIME-BOUNDED ({@link CLOSE_COMPACTION_BUDGET_MS}) so a
+ * large backlog can never stall a clean shutdown; the next pass resumes.
* Read-only instances and an explicit `autoCompact: false` skip silently.
* Pinned generations are never reclaimed ({@link GenerationStore.compact}).
* Failures are logged and swallowed — compaction is housekeeping and must
- * never fail a flush or a clean shutdown.
+ * never fail a clean shutdown.
*/
private async autoCompactHistory(): Promise {
// Nothing to compact on a read-only instance or before init wired up the
@@ -7150,12 +8443,16 @@ export class Brainy implements BrainyInterface {
if (policy.mode === 'adaptive') {
const budget = this.adaptiveHistoryBudgetBytes(policy.budgetBytes)
if (budget === Infinity) return // no pressure signal → keep everything this pass
- await this.generationStore.compact({ maxBytes: budget })
+ await this.generationStore.compact({
+ maxBytes: budget,
+ timeBudgetMs: CLOSE_COMPACTION_BUDGET_MS
+ })
} else {
await this.generationStore.compact({
maxGenerations: policy.maxGenerations,
maxAge: policy.maxAge,
- maxBytes: policy.maxBytes
+ maxBytes: policy.maxBytes,
+ timeBudgetMs: CLOSE_COMPACTION_BUDGET_MS
})
}
} catch (error) {
@@ -7180,6 +8477,14 @@ export class Brainy implements BrainyInterface {
* replacement. Release them first; a warning is logged when live pins
* exist.
*
+ * NON-DESTRUCTIVE STAGING + SPARSE-AWARE: the snapshot is copied into a
+ * staging area BEFORE any live data is touched (all-zero blocks stay
+ * holes, so a sparse mmap store restores at its true allocated size — not
+ * its apparent size). Any copy failure, ENOSPC included, removes only the
+ * staging and throws; the live store is untouched. Only after the copy
+ * succeeds does an atomic per-entry swap move it into place; a crash
+ * mid-swap completes FORWARD on the next open.
+ *
* @param path - Snapshot directory to restore from.
* @param options - Must be `{ confirm: true }` — an explicit acknowledgment
* that current state is destroyed.
@@ -7230,6 +8535,12 @@ export class Brainy implements BrainyInterface {
this.index.rebuild(),
this.graphIndex.rebuild()
])
+
+ // Change feed: a restore is a wholesale raw-state replacement, not a
+ // per-record commit — one store-level event tells subscribers to refetch.
+ this._changeFeed.emit([
+ { kind: 'store', op: 'restore', timestamp: Date.now() }
+ ])
}
/**
@@ -7812,7 +9123,10 @@ export class Brainy implements BrainyInterface {
ids: [],
touchedNouns: [],
touchedVerbs: [],
- postCommit: []
+ postCommit: [],
+ casUpdates: [],
+ createdNouns: new Set(),
+ changeEvents: []
}
for (const op of ops) {
@@ -7947,6 +9261,11 @@ export class Brainy implements BrainyInterface {
? !state.nouns.has(id) && (await this.storage.getNounMetadata(id)) === null
: true
+ // Either way (fresh create OR overwrite) this add resets the id's revision
+ // baseline (`_rev: 1` below), so later in-batch updates to it sequence
+ // against batch-local state — see PlannedTransact.createdNouns.
+ plan.createdNouns.add(id)
+
const now = Date.now()
const storageMetadata = {
...params.metadata,
@@ -8001,6 +9320,20 @@ export class Brainy implements BrainyInterface {
this._aggregationIndex.onEntityAdded(id, entityForIndexing)
}
})
+ if (this._changeFeed.hasListeners) {
+ plan.changeEvents.push({
+ kind: 'entity',
+ op: 'add',
+ id,
+ entity: {
+ id,
+ type: String(params.type),
+ ...(params.subtype !== undefined && { subtype: String(params.subtype) }),
+ metadata: (params.metadata as Record) ?? {},
+ ...(params.service !== undefined && { service: String(params.service) })
+ }
+ })
+ }
state.nouns.set(id, { metadata: storageMetadata, vector })
state.removedNouns.delete(id)
@@ -8040,8 +9373,10 @@ export class Brainy implements BrainyInterface {
}
// ifRev CAS — identical resolution to update(); a conflict rejects the
- // WHOLE batch before anything is staged or applied. `_rev` is reserved:
- // every read path surfaces it ONLY top-level.
+ // WHOLE batch. This planning-time check is purely a FAST-FAIL (it can
+ // interleave with concurrent writers); the authoritative re-verify runs in
+ // transact()'s commit precondition under the commit mutex, via the
+ // plan.casUpdates entry registered below.
const currentRev = typeof existing._rev === 'number' ? existing._rev : 1
if (typeof params.ifRev === 'number' && params.ifRev !== currentRev) {
throw new RevisionConflictError(params.id, params.ifRev, currentRev)
@@ -8091,6 +9426,17 @@ export class Brainy implements BrainyInterface {
visibility: params.visibility ?? existing.visibility
})
}
+
+ // Register for the authoritative under-mutex CAS re-verify + rev re-stamp
+ // (see PlannedTransact.casUpdates). The staged UpdateNounMetadataOperation
+ // holds `updatedMetadata` by reference, so the precommit re-stamp lands in
+ // what execute() writes.
+ plan.casUpdates.push({
+ id: params.id,
+ ...(typeof params.ifRev === 'number' && { ifRev: params.ifRev }),
+ updatedMetadata
+ })
+
const entityForIndexing = {
id: params.id,
vector,
@@ -8143,17 +9489,32 @@ export class Brainy implements BrainyInterface {
)
plan.touchedNouns.push(params.id)
- const oldEntityForAgg = {
- type: existing.type,
- service: existing.service,
- data: existing.data,
- metadata: existing.metadata
- }
+ // The full planGetEntity view, passed whole — a subset view makes the
+ // old-side decrement miss reserved-field groups (double-count on update).
+ const oldEntityForAgg = existing as unknown as Record
plan.postCommit.push(() => {
if (this._aggregationIndex) {
this._aggregationIndex.onEntityUpdated(params.id, entityForIndexing, oldEntityForAgg)
}
})
+ if (this._changeFeed.hasListeners) {
+ plan.changeEvents.push({
+ kind: 'entity',
+ op: 'update',
+ id: params.id,
+ entity: {
+ id: params.id,
+ type: String(entityForIndexing.type),
+ ...(entityForIndexing.subtype !== undefined && {
+ subtype: String(entityForIndexing.subtype)
+ }),
+ metadata: (newMetadata as Record) ?? {},
+ ...(entityForIndexing.service !== undefined && {
+ service: String(entityForIndexing.service)
+ })
+ }
+ })
+ }
state.nouns.set(params.id, { metadata: updatedMetadata, vector })
return params.id
@@ -8209,28 +9570,51 @@ export class Brainy implements BrainyInterface {
if (metadata) {
plan.operations.push(new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata))
}
- plan.operations.push(new DeleteNounMetadataOperation(this.storage, id))
+ // Pre-read metadata rides along: the count decrement must not depend on
+ // re-reading the record being removed (see remove()).
+ plan.operations.push(new DeleteNounMetadataOperation(this.storage, id, metadata))
for (const verb of cascade.values()) {
- const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb)
plan.operations.push(
- new RemoveFromGraphIndexOperation(this.graphIndex, verb, sourceInt, targetInt, this.graphWriteGeneration),
+ // Endpoint ints resolve at EXECUTE time — a cascade verb (or its
+ // endpoints) may have been created earlier in this same batch, so a
+ // plan-time resolution would ask the id mapper about entities that do
+ // not exist yet (the native mapper rightly refuses).
+ new RemoveFromGraphIndexOperation(this.graphIndex, verb, () => this.resolveVerbEndpointInts(verb), this.graphWriteGeneration),
new DeleteVerbMetadataOperation(this.storage, verb.id)
)
plan.touchedVerbs.push(verb.id)
state.verbs.delete(verb.id)
state.removedVerbs.add(verb.id)
+ if (this._changeFeed.hasListeners) {
+ plan.changeEvents.push({
+ kind: 'relation',
+ op: 'unrelate',
+ id: verb.id,
+ relation: {
+ id: verb.id,
+ from: verb.sourceId,
+ to: verb.targetId,
+ type: String(verb.verb)
+ }
+ })
+ }
}
plan.touchedNouns.push(id)
+ if (this._changeFeed.hasListeners) {
+ plan.changeEvents.push({
+ kind: 'entity',
+ op: 'remove',
+ id,
+ ...(metadata && {
+ entity: this.entityViewFromRawRecord(id, metadata as Record)
+ })
+ })
+ }
if (metadata) {
- // Canonical reserved/custom split — mirror of remove()'s aggregation hook.
- const { reserved, custom } = splitNounMetadataRecord(metadata)
- const entityForAgg = {
- type: reserved.noun,
- service: reserved.service,
- data: reserved.data,
- metadata: custom
- }
+ // Mirror of remove()'s aggregation hook — the FULL reserved view, so
+ // reserved-field groupBy decrements find their group.
+ const entityForAgg = this.entityForAggFromRawRecord(metadata as Record)
plan.postCommit.push(() => {
if (this._aggregationIndex) {
this._aggregationIndex.onEntityDeleted(id, entityForAgg)
@@ -8337,8 +9721,6 @@ export class Brainy implements BrainyInterface {
data: params.data,
createdAt: now
}
- const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb)
-
plan.operations.push(
new SaveVerbOperation(this.storage, {
id,
@@ -8349,13 +9731,31 @@ export class Brainy implements BrainyInterface {
targetId: params.to
}),
new SaveVerbMetadataOperation(this.storage, id, verbMetadata),
- new AddToGraphIndexOperation(this.graphIndex, verb, sourceInt, targetInt, this.graphWriteGeneration, (verbInt) =>
+ // Endpoint ints resolve at EXECUTE time — after any same-batch add of
+ // an endpoint has applied. Plan-time resolution was a consumer-reported
+ // native/JS parity bug: transact([add X, relate →X]) asked the native
+ // id mapper to assign an int for an entity that did not exist yet.
+ new AddToGraphIndexOperation(this.graphIndex, verb, () => this.resolveVerbEndpointInts(verb), this.graphWriteGeneration, (verbInt) =>
this.cacheVerbInt(verbInt, id)
)
)
plan.touchedVerbs.push(id)
state.verbs.set(id, verb)
state.removedVerbs.delete(id)
+ if (this._changeFeed.hasListeners) {
+ plan.changeEvents.push({
+ kind: 'relation',
+ op: 'relate',
+ id,
+ relation: {
+ id,
+ from: params.from,
+ to: params.to,
+ type: String(params.type),
+ ...(params.metadata && { metadata: params.metadata as Record })
+ }
+ })
+ }
if (params.bidirectional) {
const reverseId = uuidv4()
@@ -8363,9 +9763,9 @@ export class Brainy implements BrainyInterface {
...verb,
id: reverseId,
sourceId: params.to,
- targetId: params.from,
- sourceInt: targetInt,
- targetInt: sourceInt
+ targetId: params.from
+ // sourceInt/targetInt are mirrored onto this object when the graph
+ // operation resolves endpoints at execute time.
}
plan.operations.push(
new SaveVerbOperation(this.storage, {
@@ -8377,13 +9777,27 @@ export class Brainy implements BrainyInterface {
targetId: params.from
}),
new SaveVerbMetadataOperation(this.storage, reverseId, verbMetadata),
- new AddToGraphIndexOperation(this.graphIndex, reverseVerb, targetInt, sourceInt, this.graphWriteGeneration, (verbInt) =>
+ new AddToGraphIndexOperation(this.graphIndex, reverseVerb, () => this.resolveVerbEndpointInts(reverseVerb), this.graphWriteGeneration, (verbInt) =>
this.cacheVerbInt(verbInt, reverseId)
)
)
plan.touchedVerbs.push(reverseId)
state.verbs.set(reverseId, reverseVerb)
state.removedVerbs.delete(reverseId)
+ if (this._changeFeed.hasListeners) {
+ plan.changeEvents.push({
+ kind: 'relation',
+ op: 'relate',
+ id: reverseId,
+ relation: {
+ id: reverseId,
+ from: params.to,
+ to: params.from,
+ type: String(params.type),
+ ...(params.metadata && { metadata: params.metadata as Record })
+ }
+ })
+ }
}
return id
@@ -8405,13 +9819,29 @@ export class Brainy implements BrainyInterface {
: (state.verbs.get(id) ?? (await this.storage.getVerb(id)))
if (verb) {
- const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb)
plan.operations.push(
- new RemoveFromGraphIndexOperation(this.graphIndex, verb, sourceInt, targetInt, this.graphWriteGeneration)
+ // Endpoint ints resolve at EXECUTE time — the verb (or its endpoints)
+ // may have been created earlier in this same batch (forward refs).
+ new RemoveFromGraphIndexOperation(this.graphIndex, verb, () => this.resolveVerbEndpointInts(verb), this.graphWriteGeneration)
)
}
plan.operations.push(new DeleteVerbMetadataOperation(this.storage, id))
plan.touchedVerbs.push(id)
+ if (this._changeFeed.hasListeners) {
+ plan.changeEvents.push({
+ kind: 'relation',
+ op: 'unrelate',
+ id,
+ ...(verb && {
+ relation: {
+ id,
+ from: verb.sourceId,
+ to: verb.targetId,
+ type: String(verb.verb)
+ }
+ })
+ })
+ }
state.verbs.delete(id)
state.removedVerbs.add(id)
@@ -8819,6 +10249,30 @@ export class Brainy implements BrainyInterface {
return await coordinator.import(source as Buffer | string | object, options)
}
+ /** Brain-owned background deduplicator (lazy; see getBackgroundDeduplicator). */
+ private _backgroundDedup?: import('./import/BackgroundDeduplicator.js').BackgroundDeduplicator
+
+ /**
+ * The single brain-owned BackgroundDeduplicator, lazily constructed.
+ *
+ * Ownership matters here: the post-import dedup timer must outlive the
+ * per-call ImportCoordinator but never the brain. One instance per brain
+ * restores the intended cross-import debounce (per-coordinator instances
+ * each armed their own timer, so the "debounce" never spanned imports) and
+ * gives close() a handle to cancel pending work — a delete pass must never
+ * fire against a closed brain.
+ * @internal
+ */
+ async getBackgroundDeduplicator(): Promise<
+ import('./import/BackgroundDeduplicator.js').BackgroundDeduplicator
+ > {
+ if (!this._backgroundDedup) {
+ const { BackgroundDeduplicator } = await import('./import/BackgroundDeduplicator.js')
+ this._backgroundDedup = new BackgroundDeduplicator(this)
+ }
+ return this._backgroundDedup
+ }
+
/**
* Virtual File System API - Knowledge Operating System
*
@@ -8901,6 +10355,43 @@ export class Brainy implements BrainyInterface {
return this._hub
}
+ /**
+ * @description Subscribe to this brain's committed mutations — the
+ * authoritative in-process change feed. Fires exactly once per affected
+ * record for EVERY canonical write, regardless of origin: direct API calls,
+ * batch methods, `transact()`, imports, the Virtual Filesystem, and
+ * native-accelerated deployments all funnel through the same commit point
+ * this feed is emitted from.
+ *
+ * Delivery contract (see {@link BrainyChangeEvent}):
+ * - **Post-commit only** — an aborted write (e.g. a losing `ifRev`
+ * compare-and-swap) never emits.
+ * - **Commit-ordered, asynchronous** — events arrive in the order writes
+ * became durable, dispatched in a microtask so a slow listener never
+ * delays a write. A throwing listener is isolated (logged, others
+ * unaffected).
+ * - **Fully described** — `remove`/`unrelate` events carry the record's
+ * LAST committed state, and batch operations emit one event per item.
+ * - **Fire-and-forget** — no replay or backpressure. Each event carries its
+ * committed `generation`, so catch-up after a gap can be built on
+ * {@link transactionLog} / {@link asOf}.
+ * - **Zero overhead when unused** — with no subscribers the write path does
+ * no event work at all.
+ *
+ * @param listener - Called once per committed mutation.
+ * @returns An unsubscribe function — call it when done (e.g. on teardown of
+ * a pooled instance) to stop delivery.
+ * @example
+ * const off = brain.onChange((e) => {
+ * if (e.kind === 'entity') console.log(e.op, e.entity?.type, e.id)
+ * })
+ * await brain.add({ data: 'hello', type: 'document' }) // → "add document "
+ * off()
+ */
+ onChange(listener: ChangeListener): () => void {
+ return this._changeFeed.subscribe(listener)
+ }
+
/**
* Get Triple Intelligence System
* Advanced pattern recognition and relationship analysis
@@ -9106,16 +10597,101 @@ export class Brainy implements BrainyInterface {
this.generationStore.persistCounterNow()
])
- // 6. Auto-compact generational history per config.retention (default on).
- // Runs after the flush so durable state is in place; respects live
- // Db pins and an explicit autoCompact: false.
- await this.autoCompactHistory()
+ // NOTE (8.9.0): flush() no longer compacts history. Flush is DURABILITY
+ // work — it must cost what this window's deltas cost, never what the
+ // history backlog costs. Auto-compaction (a MAINTENANCE concern) runs at
+ // close() and via explicit compactHistory(); a production deployment
+ // measured reclaim-on-flush stalling single writes for 25-191s under
+ // memory pressure, which is exactly the class this separation ends.
+
+ // 6. Stamp the entity tree: which source generation the canonical tree
+ // reflects + the rollup invariants that verify it whole (the counters
+ // persisted in step 1). Written at flush boundaries — the tree tracks
+ // every commit by construction, so the stamp is a durable checkpoint,
+ // not a per-commit cost. Open compares stamp vs log head + rollups.
+ await this.stampEntityTree()
const elapsed = Date.now() - startTime
console.log(`All indexes flushed to disk in ${elapsed}ms`)
}
+ /**
+ * @description Write the entity tree's FAMILY STAMP: `sourceGeneration` (the
+ * committed generation the canonical tree reflects — equal by construction,
+ * since the tree is written by the commit itself) plus the rollup invariants
+ * (entity/relationship counts) that verify the tree whole where per-file
+ * checks cannot scale. Verified at open by {@link verifyEntityTreeStamp};
+ * healed by `repairIndex()`, whose unconditional recount rebuilds the
+ * rollups from a canonical walk and re-stamps. Best-effort: a stamp-write
+ * fault warns loudly but never fails the flush that carried real data.
+ */
+ private async stampEntityTree(): Promise {
+ if (this.isReadOnly) return
+ try {
+ const [nounCount, verbCount] = await Promise.all([
+ this.storage.getNounCount(),
+ this.storage.getVerbCount()
+ ])
+ await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, {
+ family: 'entity-tree',
+ sourceGeneration: this.generationStore.generation(),
+ members: { mode: 'rollup', invariants: { nounCount, verbCount } }
+ })
+ } catch (error) {
+ prodLog.warn(
+ `[Brainy] entity-tree stamp write failed (coherence checking degrades until the ` +
+ `next successful flush): ${(error as Error).message}`
+ )
+ }
+ }
+
+ /**
+ * @description Open-time coherence check for the entity tree's family stamp:
+ * compare `sourceGeneration` against the log head and the stamped rollup
+ * invariants against the live counters. Verdicts:
+ * - `coherent` / `absent` (legacy store; first flush stamps) → silent.
+ * - `behind` → benign for the tree (it is written BY the commit; only the
+ * stamp is stale — a crash landed between commit and flush). Refreshed at
+ * the next flush.
+ * - `incoherent` → LOUD: the tree or its counters diverged from what was
+ * stamped — `repairIndex()` recounts from canonical and re-stamps.
+ * Never blocks open; a fault reading the stamp is surfaced as unverifiable,
+ * never conflated with absence.
+ */
+ private async verifyEntityTreeStamp(): Promise {
+ let stamp: FamilyStamp | null
+ try {
+ stamp = await readFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH)
+ } catch (error) {
+ prodLog.warn(
+ `[Brainy] entity-tree stamp is UNVERIFIABLE (read fault, not absence): ` +
+ `${(error as Error).message}`
+ )
+ return
+ }
+ const [nounCount, verbCount] = await Promise.all([
+ this.storage.getNounCount(),
+ this.storage.getVerbCount()
+ ])
+ const verdict = verifyFamilyStamp(stamp, this.generationStore.generation(), {
+ nounCount,
+ verbCount
+ })
+ if (verdict.state === 'incoherent') {
+ prodLog.warn(
+ `[Brainy] entity-tree stamp INCOHERENT at open: ${verdict.failures.join('; ')}. ` +
+ `The canonical tree or its counters diverged from the stamped state — run ` +
+ `brain.repairIndex() to recount from canonical and re-stamp.`
+ )
+ } else if (verdict.state === 'behind') {
+ prodLog.debug(
+ `[Brainy] entity-tree stamp is behind the log head (${verdict.stampSource} < ` +
+ `${verdict.head}) — benign (stamped at last flush); refreshes at the next flush.`
+ )
+ }
+ }
+
/**
* Ask the writer process serving this data directory to flush its in-memory
* indexes to disk, so a read-only inspector can observe fresh state.
@@ -9184,17 +10760,32 @@ export class Brainy implements BrainyInterface {
migrating: boolean
/** Structured progress while {@link migrating}; absent otherwise. */
migration?: MigrationProgress
+ /** `true` when a non-fatal index rebuild failed at init() (degraded — queries
+ * may be incomplete). Folds in the same `_indexRebuildFailed` signal that
+ * {@link validateIndexConsistency} / {@link checkHealth} already expose. */
+ rebuildFailed: boolean
+ /** The rebuild failure message when {@link rebuildFailed}; absent otherwise. */
+ rebuildError?: string
+ /** Count of records committed via adopt-forward failed-rollback recovery whose
+ * derived index may be incomplete until repairIndex() (the 8.2.6 degraded
+ * set). Non-zero = degraded — a readiness probe should not report 200-ready. */
+ degradedIds: number
hnswIndex: {
size: number
+ /** Honest "serving" — the provider's `isReady()` when exposed, else `size>0`. */
populated: boolean
+ /** The provider's honest `isReady()` signal; absent when unexposed. */
+ ready?: boolean
}
metadataIndex: {
entries: number
populated: boolean
+ ready?: boolean
}
graphIndex: {
relationships: number
populated: boolean
+ ready?: boolean
}
storage: {
totalEntities: number
@@ -9209,6 +10800,9 @@ export class Brainy implements BrainyInterface {
lazyRebuildCompleted: this.lazyRebuildCompleted,
disableAutoRebuild: this.config.disableAutoRebuild || false,
migrating: false,
+ rebuildFailed: this._indexRebuildFailed != null,
+ ...(this._indexRebuildFailed ? { rebuildError: this._indexRebuildFailed.message } : {}),
+ degradedIds: this._indexDegradedIds.size,
hnswIndex: { size: 0, populated: false },
metadataIndex: { entries: 0, populated: false },
graphIndex: { relationships: 0, populated: false },
@@ -9220,6 +10814,20 @@ export class Brainy implements BrainyInterface {
const hnswSize = this.index.size()
const graphSize = await this.graphIndex.size()
+ // Honest readiness: when a provider exposes isReady(), it is the truth of
+ // whether the index actually SERVES (a native index can report a non-zero
+ // size/count yet not have loaded its serving structure). `populated` reflects
+ // that honest signal when present, falling back to size>0 only for providers
+ // that do not expose isReady(). Mirrors validateIndexConsistency/checkHealth,
+ // which already fold in the same signals.
+ const readyOf = (p: unknown): boolean | undefined => {
+ const r = assessIndexReadiness(p)
+ return r === 'unknown' ? undefined : r === 'ready'
+ }
+ const hnswReady = readyOf(this.index)
+ const metadataReady = readyOf(this.metadataIndex)
+ const graphReady = readyOf(this.graphIndex)
+
// Check storage entity count
let storageEntityCount = 0
try {
@@ -9233,18 +10841,29 @@ export class Brainy implements BrainyInterface