diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml new file mode 100644 index 00000000..cdb2ab14 --- /dev/null +++ b/.forgejo/workflows/ci.yml @@ -0,0 +1,40 @@ +name: CI + +on: + push: + pull_request: + +jobs: + node: + name: Node ${{ matrix.node-version }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + node-version: ['22', '24'] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: npm + - run: npm ci + - run: npm run test:unit + + bun: + name: Bun (latest) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + - uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + - run: npm ci + # test:bun imports the built dist/, so build first. + - run: npm run build + # Bun as a runtime is the supported Bun story (`bun add` / `bun run`). + - run: npm run test:bun diff --git a/CHANGELOG.md b/CHANGELOG.md index 295891a2..b6dd91fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,108 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [8.10.0](https://github.com/soulcraftlabs/brainy/compare/v8.9.0...v8.10.0) (2026-07-23) + +- docs: adoption storefront — contributing guide, security policy, README support + cor section (9a99a7b) +- fix(release): push the public mirror explicitly and verify the tag lands at the right commit before publishing (6ba94c8) +- docs: project guide version line points at npm instead of a hardcoded stale number (3a1efc9) +- feat: vector provider identity is a required name field (hnsw-js), rendered [vector-index:] (3be4ba9) +- feat: warm contract (warm/warmOnOpen/provider warm hook), configurable transact budget floor, backend-neutral vector index op names (55b867c) + + +### [8.9.0](https://github.com/soulcraftlabs/brainy/compare/v8.8.2...v8.9.0) (2026-07-19) + +- docs: measured performance envelopes v1 (per-op p50/p95 at 1k and 10k, pure-JS floor) (5cabd78) +- fix: release drains in-flight writer-lock heartbeat — no phantom lock after unlink (70e4bc8) +- feat: flush() never compacts — history maintenance moves to close() with bounded passes (300d9f2) + + +### [8.8.2](https://github.com/soulcraftlabs/brainy/compare/v8.8.1...v8.8.2) (2026-07-19) + +- fix: one field-resolution law across aggregation hooks, source.where, removeMany, and find() spellings (945d92d) +- chore: push public docs to the soulcraft.com ingest door on release (42037d0) + + +### [8.8.1](https://github.com/soulcraftlabs/brainy/compare/v8.8.0...v8.8.1) (2026-07-18) + +- fix: O(1) adaptive retention accounting + historyStats fleet audit (6207e48) +- fix: import dedup off-switch honesty + brain-owned lifecycle for the background pass (4fcef7b) + + +### [8.8.0](https://github.com/soulcraftlabs/brainy/compare/v8.7.1...v8.8.0) (2026-07-17) + +- feat: OS-limit detection for pool-scale deployments (16a73b8) + + +### [8.7.1](https://github.com/soulcraftlabs/brainy/compare/v8.7.0...v8.7.1) (2026-07-17) + +- fix: race-proof writer-lock acquisition + machine-readable conflict through init (01a3b46) + + +### [8.7.0](https://github.com/soulcraftlabs/brainy/compare/v8.6.0...v8.7.0) (2026-07-17) + +- feat: scaled transact budgets + labeled timeout diagnostics + envelope docs (6ef9fcb) + + +### [8.6.0](https://github.com/soulcraftlabs/brainy/compare/v8.5.2...v8.6.0) (2026-07-17) + +- feat: brain.auditGraph() — read-only graph-truth audit (2a03fae) + + +### [8.5.2](https://github.com/soulcraftlabs/brainy/compare/v8.5.1...v8.5.2) (2026-07-17) + +- fix: exception-safe aggregation backfill + generation-verified adoption + loud open-path guards (a77b064) + + +### [8.5.1](https://github.com/soulcraftlabs/brainy/compare/v8.5.0...v8.5.1) (2026-07-17) + +- fix: aggregation state adoption on reopen + single-flight backfill + query-cap ratchet removal (da55be7) +- docs: external-backups/sparse-storage guide + generation fact log concept (593bb8b) + + +### [8.5.0](https://github.com/soulcraftlabs/brainy/compare/v8.4.0...v8.5.0) (2026-07-15) + +- test: tolerant timing assertion in the execution-time measure test (4dc0a92) +- feat: committedGeneration capability + pinned durability/stability contracts (d1ecee1) +- docs: RELEASES.md entry for 8.5.0 (provider fact-log access + shared verifier) (e4f37cd) +- feat: provider access to the fact log + shared stamp verifier via internals (352e356) + + +### [8.4.0](https://github.com/soulcraftlabs/brainy/compare/v8.3.3...v8.4.0) (2026-07-15) + +- docs: RELEASES.md entry for 8.4.0 (generation fact log + family stamp) (4a60b43) +- feat: entity-tree family stamp — sourceGeneration + rollup coherence at open (2888ae6) +- feat: generation fact log — after-image commit records, dual-written at every commit point (38b0041) + + +### [8.3.3](https://github.com/soulcraftlabs/brainy/compare/v8.3.2...v8.3.3) (2026-07-15) + +- docs: RELEASES.md entry for 8.3.3 (rename containment fix + repair) (c3feafd) +- test: lens-consistency regression — combined vs subtype-only vs canonical ground truth (4fb41f9) +- fix: VFS rename moves the containment edge — no ghost in the old directory (af8c179) + + +### [8.3.2](https://github.com/soulcraftlabs/brainy/compare/v8.3.1...v8.3.2) (2026-07-14) + +- docs: RELEASES.md entry for 8.3.2 (honest counters) (0932ecd) +- fix: honest counters — removal never re-reads the removed record + repairIndex recounts and persists all rollups (2e2ba9c) + + +### [8.3.1](https://github.com/soulcraftlabs/brainy/compare/v8.3.0...v8.3.1) (2026-07-14) + +- docs: RELEASES.md entry for 8.3.1 (full-removal deletes + family-scoped gate) (c0c68ac) +- fix: full-removal canonical deletes + family-scoped migration gate (366f9a9) +- docs: cite the cross-layer integrity contract generically in comments and notes (1d26988) + + +### [8.3.0](https://github.com/soulcraftlabs/brainy/compare/v8.2.8...v8.3.0) (2026-07-13) + +- docs: RELEASES.md entry for 8.3.0 (heal-cost + cross-layer integrity contract) (7692c6f) +- perf: parallel + id-only canonical enumeration (heal-cost dominant term) (ec5b933) +- feat: registered-blob family contract — declared index blobs are undeletable (bfa1762) +- feat: validateIndexConsistency delegates to provider invariants (6bcb54f) + + ### [8.2.8](https://github.com/soulcraftlabs/brainy/compare/v8.2.7...v8.2.8) (2026-07-13) - fix: honest index readiness — no silently-empty queries on a cold index (d0f69c7) diff --git a/CLAUDE.md b/CLAUDE.md index 568c10db..c7336a18 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,7 +12,7 @@ Handoff file: `/home/dpsifr/.strategy/PLATFORM-HANDOFF.md` **Brainy's current open actions:** None. MIT open-source — no platform-specific actions. -**Current version:** `@soulcraft/brainy@7.31.5` (latest published; 8.0.0 release candidate on `feat/8.0-u64-ids`) +**Current version:** run `npm view @soulcraft/brainy version` (never trust a hardcoded number here — this line went stale for months); consumer-facing changes tracked in `RELEASES.md` --- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ab8a0246..ef9c4a51 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,298 +1,70 @@ # Contributing to Brainy -Thank you for your interest in contributing to Brainy! This document provides guidelines and instructions for contributing to the project. +Brainy is MIT-licensed and genuinely open to outside contributions. This page +is the honest, current path — please don't rely on older instructions you +may find elsewhere in the repo's history. -## Code of Conduct +## Where the project lives -By participating in this project, you agree to abide by our Code of Conduct: -- Be respectful and inclusive -- Welcome newcomers and help them get started -- Focus on constructive criticism -- Respect differing viewpoints and experiences +The source of truth is a self-hosted forge: **source.soulcraft.com/soulcraft/brainy**. +It's anonymously readable and cloneable — no account needed to browse, clone, +or build. -## How to Contribute +**github.com/soulcraftlabs/brainy** is a public read-only mirror. It's a fine +place to read code or star the project, but issues and pull requests opened +there won't be picked up — please use one of the paths below instead. -### Reporting Issues +## How to contribute -Before creating an issue, please check existing issues to avoid duplicates. +**Found a bug, or have an idea?** Email **brainy@soulcraft.com**. No account, +no ceremony — you'll get a receipt, and it goes to a human. -When creating an issue, include: -- Clear, descriptive title -- Detailed description of the problem -- Steps to reproduce -- Expected vs actual behavior -- System information (OS, Node version, Brainy version) -- Code examples if applicable +**Want to send a patch?** Two ways, both first-class: -### Suggesting Features +- **Email a patch.** Run `git format-patch` against your change and email the + output to **brainy@soulcraft.com**. This is a genuinely supported path, not + a fallback — plenty of good contributions arrive this way. +- **Open a pull request on the forge.** Request an account at + **source.soulcraft.com** (registration is request-with-approval, so allow + a little lag), clone, push a branch, and open a PR there. Maintainers + review and land it. -Feature requests are welcome! Please provide: -- Clear use case -- Proposed API/interface -- Examples of how it would work -- Any potential challenges or considerations +Either way, for anything beyond a small fix, opening an issue first (email is +fine) to talk through the approach saves everyone rework. -### Pull Requests +## Development setup -#### Before Starting - -1. Check existing issues and PRs -2. Open an issue to discuss significant changes -3. Fork the repository -4. Create a feature branch from `main` - -#### Development Setup - -**Quick Setup (Recommended):** ```bash -# Clone your fork -git clone https://github.com/your-username/brainy.git +git clone https://source.soulcraft.com/soulcraft/brainy.git cd brainy - -# Run setup script (installs all dependencies including Rust) -./scripts/setup-dev.sh -``` - -**Manual Setup:** -```bash -# Clone your fork -git clone https://github.com/your-username/brainy.git -cd brainy - -# Install system dependencies (Ubuntu/Debian) -sudo apt-get install -y build-essential pkg-config libssl-dev - -# Install Rust (for WASM embedding engine) -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -source ~/.cargo/env -rustup target add wasm32-unknown-unknown -cargo install wasm-pack - -# Install Node.js dependencies npm install - -# Build Candle WASM embedding engine -npm run build:candle - -# Build TypeScript npm run build - -# Run tests npm test ``` -#### Making Changes +Tests run on [Vitest](https://vitest.dev/). `npm test` runs the unit suite; +see `package.json` for `test:integration`, `test:coverage`, and friends. -1. **Follow the code style** - - TypeScript for all source code - - Clear variable and function names - - Comments for complex logic - - JSDoc for public APIs +## Standards -2. **Write tests** - - Add tests for new features - - Update tests for changes - - Ensure all tests pass +- **Strict TypeScript.** No `any` escape hatches to dodge the type checker. +- **Tests exercise real behavior.** No mocking away the thing you're supposed + to be testing. +- **No stubs, no TODO-code.** If something can't be finished, say so and + leave it out — don't merge a placeholder. +- **JSDoc on every exported function, class, and type.** +- **[Conventional Commits](https://www.conventionalcommits.org/).** `feat:`, + `fix:`, `docs:`, `perf:`, `refactor:`, `test:`, `chore:`. Never + `BREAKING CHANGE` in a commit message — major version bumps are a separate, + deliberate decision. +- **Performance claims are measured or labeled projected.** If a PR or its + description states a number, cite the benchmark that produced it (see + [docs/performance-envelopes.md](docs/performance-envelopes.md) for the + pattern). Don't state an estimate as if it were measured. -3. **Update documentation** - - Update README if needed - - Add/update API documentation - - Include examples +## License -#### Commit Guidelines +Brainy is [MIT licensed](LICENSE). Contributions are accepted under the same +license — there's no CLA to sign. -Follow conventional commits format: - -``` -type(scope): description - -[optional body] - -[optional footer] -``` - -Types: -- `feat`: New feature -- `fix`: Bug fix -- `docs`: Documentation changes -- `style`: Code style changes -- `refactor`: Code refactoring -- `perf`: Performance improvements -- `test`: Test changes -- `chore`: Build/tooling changes - -Examples: -```bash -feat(triple): add graph traversal depth limit -fix(storage): handle concurrent write conflicts -docs(api): update search method documentation -``` - -#### Submitting PR - -1. Push to your fork -2. Create PR against `main` branch -3. Fill out PR template -4. Ensure CI checks pass -5. Wait for review - -### Testing - -#### Running Tests - -```bash -# Run all tests -npm test - -# Run specific test file -npm test tests/core.test.ts - -# Run with coverage -npm run test:coverage - -# Watch mode -npm run test:watch -``` - -#### Writing Tests - -```typescript -import { describe, it, expect } from 'vitest' -import { Brainy } from '../src' - -describe('Feature Name', () => { - it('should do something specific', async () => { - const brain = new Brainy() - await brain.init() - - // Test implementation - const result = await brain.search("test") - - expect(result).toBeDefined() - expect(result.length).toBeGreaterThan(0) - }) -}) -``` - -## Architecture Guidelines - -### Adding New Features - -1. **Check existing functionality** - - Review `ARCHITECTURE.md` - - Check if similar features exist - - Consider if it should be an augmentation - -2. **Design considerations** - - Maintain backward compatibility - - Consider performance impact - - Think about all storage adapters - - Plan for extensibility - -3. **Implementation checklist** - - [ ] Core functionality - - [ ] Tests (unit and integration) - - [ ] Documentation - - [ ] TypeScript types - - [ ] Examples - - [ ] Performance benchmarks (if applicable) - -### Creating Augmentations - -Augmentations extend Brainy's functionality: - -```typescript -import { BrainyAugmentation } from '../types' - -export class MyAugmentation extends BrainyAugmentation { - name = 'MyAugmentation' - - async onInit(brain: Brainy): Promise { - // Initialize augmentation - } - - async onAdd(item: any, brain: Brainy): Promise { - // Process before adding - return item - } - - async onSearch(query: any, results: any[], brain: Brainy): Promise { - // Process search results - return results - } -} -``` - -### Performance Considerations - -- Use batch operations where possible -- Implement caching strategically -- Consider memory usage -- Profile performance impacts -- Add benchmarks for critical paths - -## Documentation - -### API Documentation - -Use JSDoc for all public APIs: - -```typescript -/** - * Searches for similar items using vector similarity - * @param query - Search query (text or vector) - * @param options - Search options - * @returns Array of search results with scores - * @example - * ```typescript - * const results = await brain.search("machine learning", { limit: 10 }) - * ``` - */ -async search(query: string | Vector, options?: SearchOptions): Promise { - // Implementation -} -``` - -### Examples - -Add examples for new features: - -```typescript -// examples/feature-name.ts -import { Brainy } from 'brainy' - -async function exampleUsage() { - const brain = new Brainy() - await brain.init() - - // Show feature usage - // Include comments explaining what's happening - // Handle errors appropriately -} - -exampleUsage().catch(console.error) -``` - -## Release Process - -1. **Version bump**: Follow semantic versioning -2. **Update CHANGELOG**: Document all changes -3. **Run tests**: Ensure all tests pass -4. **Build**: Generate distribution files -5. **Tag**: Create git tag for version -6. **Publish**: Release to npm - -## Getting Help - -- **Discord**: Join our community -- **Issues**: Ask questions on GitHub -- **Discussions**: Share ideas and get feedback - -## Recognition - -Contributors will be recognized in: -- CHANGELOG.md for their contributions -- README.md contributors section -- GitHub contributors page - -Thank you for contributing to Brainy! 🧠 \ No newline at end of file +Thank you for considering a contribution. diff --git a/README.md b/README.md index d1342f52..2fc42060 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,9 @@ Quick start · One query · Features · - Scale with Cor · - Docs + Scale with Cor · + Docs · + Support

--- @@ -172,9 +173,11 @@ await brain.vfs.search('React components with hooks') // semantic file **[Multi-process model](docs/concepts/multi-process.md)** · **[Inspection guide](docs/guides/inspection.md)** -## From laptop to hundreds of millions +## When you outgrow Brainy -Brainy's TypeScript engines take you a long way. When you outgrow them, add the native engine — **the API doesn't change**: +Brainy's pure-TypeScript engines carry real workloads a long way on their own — see the measured, per-operation numbers (not marketing figures) in **[docs/performance-envelopes.md](docs/performance-envelopes.md)** for what to expect, unaccelerated, on plain filesystem storage. + +When a deployment needs native-scale vector/graph performance — memory-mapped indexes that don't need your dataset in RAM, billion-scale ambitions — add the native engine. **The API doesn't change:** ```bash npm install @soulcraft/cor @@ -187,13 +190,14 @@ await brain.init() // @soulcraft/cor detected — same code, native engines un Installing the package is the opt-in: if `@soulcraft/cor` is present, it loads and announces itself in the init log; if it's present but broken, `init()` **throws** — an installed accelerator never silently vanishes behind the JS engines. Opt out with `plugins: []`, or pin exactly what loads with `plugins: ['@soulcraft/cor']`. [`@soulcraft/cor`](https://www.npmjs.com/package/@soulcraft/cor) (Brainy 8.x ↔ Cor 3.x, version-matched) registers Rust implementations behind every provider seam: SIMD distance kernels, memory-mapped storage, a disk-native vector index that doesn't need your dataset in RAM, durable LSM field/graph indexes that serve cold opens instantly, and native aggregation. Recall@10 measured **0.99 / 0.96 / 0.96 at 1M / 10M / 100M vectors** in Cor's release gate. -Open core, commercial accelerator: Brainy is MIT and complete on its own; Cor is licensed and funds both. +Open core, commercial accelerator: Brainy is MIT and complete on its own — Cor is more headroom for when you need it, not capability held back to sell you later. Licensing and support: **cor@soulcraft.com**. ## Performance +- Per-operation p50/p95 at 1k and 10k entities, pure-JS floor, measured and re-run every release that touches a measured path: **[docs/performance-envelopes.md](docs/performance-envelopes.md)**. - JS distance kernels: **~6× faster cosine, ~1.4× euclidean** than 7.x (measured: [`tests/benchmarks/distance-microbench.mjs`](tests/benchmarks/distance-microbench.mjs), 384-dim, median of 41). - Whole-graph reads are single **O(N + E)** cursor walks — a consumer-measured 19k-edge export dropped from ~27 s of per-node calls to one scan. -- Full numbers and capacity planning: **[docs/PERFORMANCE.md](docs/PERFORMANCE.md)** · **[docs/SCALING.md](docs/SCALING.md)** +- Capacity planning and architecture: **[docs/PERFORMANCE.md](docs/PERFORMANCE.md)** · **[docs/SCALING.md](docs/SCALING.md)** ## Use cases @@ -212,6 +216,10 @@ Open core, commercial accelerator: Brainy is MIT and complete on its own; Cor is **Bun ≥ 1.1** (recommended) or **Node.js ≥ 22**. Brainy 8.x is server-only; the 7.x line remains on npm for browser use. -## Contributing & license +## Support & community -Contributions welcome — see **[CONTRIBUTING.md](CONTRIBUTING.md)**. MIT © Brainy Contributors. +- **Bugs and ideas** → **brainy@soulcraft.com** — no account needed, you'll get a receipt. +- **Security reports** → **security@soulcraft.com** — see **[SECURITY.md](SECURITY.md)**. +- **Contributing** → see **[CONTRIBUTING.md](CONTRIBUTING.md)**. + +MIT © Brainy Contributors. diff --git a/RELEASES.md b/RELEASES.md index a90dcce4..89bf38e7 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -8,8 +8,528 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so - Debugging data, query, or storage behaviour - A new Brainy feature is available that you want to adopt +## Removed APIs — 7.x → 8.x (the complete ledger) + +Every public API removed at the 8.0 major, with its sanctioned replacement. If your code +still calls a left-column name on 8.x it throws (or the config key is rejected) — the +replacement is always a one-line change. (Standing contract from 8.9.0 forward: removals +happen only at majors, after ≥1 minor of loud runtime deprecation naming the replacement.) + +| Removed (7.x) | Replacement (8.x) | +|---|---| +| `brain.search(query, k)` | `find({ query })` — semantic; `find({ query, searchMode })` for hybrid | +| `brain.getRelations({...})` | `related(id, opts)` for adjacency; `find({ connected: {...} })` for scoped traversal | +| `brain.neural()` clustering | `find({ vector })` + aggregation `GROUP BY` | +| `Db.search()` | `db.find({ vector })` | +| Pre-8.0 storage path aliases (`directory`, `basePath`, …) | one `storage.path` key (old aliases throw) | +| Reserved keys inside `metadata` bags (silently remapped in 7.x) | top-level params (`subtype`, `visibility`, `confidence`, `weight`, …) — reserved-in-bag throws | +| 7.x COW branches layout (`branches/main/`) | generational MVCC (`asOf()`, `now()`, `db.persist(path)`) — on-disk migration is automatic at first 8.x open | + +The fork/snapshot family (`brain.snapshot()`, `createSnapshot()`, `restoreSnapshot()`) +is sometimes cited as a 7.x removal — those methods never existed on 7.x; the 8.0 Db API +(`asOf`/`persist`/`restore({confirm})`) is their first real implementation. + --- +## Unreleased (the warm contract: cold-restart writes stop paying demand-load latency) + +From a production deployment's cold-restart incident: the FIRST writes after every +restart on a large brain measured 33–35s each (page cache cold) against the transact +apply budget — Brainy's own op-count-scaled budget, `max(30s, opCount × 2s)`, e.g. +32,000ms for a 16-op batch. There is no external deadline in this story, and no +post-completion veto either: every write is itself a multi-operation transaction (a +single `add()` applies several operations — canonical writes, the vector-index insert, +the metadata-index update), so ONE cold operation that legitimately runs ~33s consumes +the whole budget, the gate before the NEXT operation trips, and the write rolls back +atomically (zero loss, by design) — refused, retried, refused again, until the page +cache warms passively (~30 minutes). The cure is not weaker atomicity; it is the two +new knobs below — a budget floor sized for cold stores, and a warm contract that pays +demand-load cost OFF the transaction path. + +- **The budget's start-gating contract is now explicit, documented, and pinned by + regression tests.** The budget gates STARTING the next operation — completed work is + never rolled back for elapsed time — and a transaction's first operation now + unconditionally starts by code, not merely because elapsed time happens to be ~0 when + it is checked. This has been the shipped schedule since 8.7.0 (no behavior change for + existing integrations); it is now stated in `Transaction.execute()`'s contract JSDoc + and enforced by tests so it cannot silently regress. Mid-batch atomicity is unchanged: + a trip before operation `i+1` still rolls back `0..i` and throws a retryable + `TransactionTimeoutError`. +- **The budget's 30s floor is now configurable**: `new Brainy({ transactionBudgetFloorMs })` + raises (or lowers) the floor of `max(transactionBudgetFloorMs, opCount × 2000)` for every + internal transact batch. Useful for a store whose cold writes legitimately run past 30s + per operation, so a bulk batch gets a proportionally larger runway instead of tripping + mid-batch on cold-cache latency. +- **New: `brain.warm()`** — eagerly loads/faults-in the vector index, metadata index, and + graph adjacency so the first real operation after a cold restart runs at steady-state + cost instead of paying demand-load latency on the critical path. Returns a `WarmReport` + with one honest outcome per surface — never conflate the first two: + - `'warmed'` — the surface's own provider `warm()` hook ran, or a full-hydration seam + loaded every shard/field/segment from storage. Steady-state cost is paid. + - `'probed'` — no `warm()` hook was available, so a best-effort read (one `search()` call + for the vector index) faulted in *some* backing storage as a side effect — real work, + but never reported as `'warmed'`. + - `'unavailable'` — nothing ran (no hook, no hydration seam, or nothing to probe). +- **New config: `warmOnOpen: true`** makes `init()` await `brain.warm()` before it resolves + — a deliberate blocking trade-off: startup takes longer, the first request doesn't. + Default `false` (unchanged lazy behavior). +- **New optional provider hook: `warm?(): Promise`** on the vector and graph + acceleration provider contracts (`src/plugin.ts`) — a native provider can implement it to + eagerly pretouch its own backing storage (e.g. mmap pretouch); absence means brainy falls + back to the probe/hydration behavior above. +- **Transaction op-name strings changed in journals/timings**: the vector-index + transaction operations were renamed from `AddToHNSW`/`RemoveFromHNSW` to backend-neutral + `AddToVectorIndex(...)`/`RemoveFromVectorIndex(...)` — the old names hard-coded an + algorithm that may not be the one actually running (a non-HNSW native vector provider + emitting `"RemoveFromHNSW"` has sent an operator hunting an index that doesn't exist). + The parenthesized suffix names the ACTIVE backend: `hnsw-js` for the built-in engine, or + the native provider's own identity when it self-identifies. **If you parse these op-name + strings** (log processors, journal tooling), update your matcher from + `AddToHNSW`/`RemoveFromHNSW` to `AddToVectorIndex(`/`RemoveFromVectorIndex(`. +- **Provider identity is now a REQUIRED `name` field** on the vector provider contract + (`VectorIndexProvider.name`, `src/plugin.ts`) — every implementation self-reports its own + identity truthfully (its algorithm/engine), never inheriting a default. It renders as the + op-name suffix above and, wherever the vector index identifies itself in prose log lines, + as the tag `[vector-index:]`. **Native provider adoption is a one-line change**: + declare `readonly name = ''`. A provider instance that still lacks + `name` at runtime (an older native build compiled against the previous, optional field) is + never crashed on and never silently mislabeled: it stamps `unknown-provider` and emits one + loud warning naming the missing field, so the gap is discoverable instead of a permanent + fossil label in every journal line. + +## v8.9.0 — 2026-07-19 (flush is durability-only: history maintenance moves to close()) + +The write path stops paying maintenance costs — the last structural piece of the +flush-storm class (a production deployment measured single writes blocked 25–191s behind +history reclaim running inline on flush under memory pressure): + +- **`flush()` never compacts history.** It persists the current window's deltas and + nothing else — its cost no longer depends on history backlog or retention mode, in any + configuration. **`close()` is the auto-compaction site** (time-bounded per pass, ~5s; + an early stop is a consistent prefix and the next pass resumes). +- **`compactHistory()` gains `timeBudgetMs`** — bound your own maintenance windows; the + same resumable-prefix guarantee applies. +- **The documented trade**: a long-lived writer that never closes accumulates history + until its next explicit `compactHistory()`. Predictable writes, explicit maintenance. + If you run bounded retention on an always-on service, schedule a periodic + `compactHistory({ ...caps, timeBudgetMs })` in your maintenance window. +- **New public doc: `docs/performance-envelopes.md`** — measured per-op envelopes + (p50/p95 at stated scales, hardware, and backend, with the measuring script cited). + Refresh rule going forward: any release touching a measured path re-runs that op's + benchmark and updates the envelope in the same release. +- **New in this file: the Removed APIs 7.x→8.x table** (top of this document) — every + removal with its sanctioned replacement, one place, per the engine-currency contract. + Standing from here: removals only at majors, after ≥1 minor of loud runtime deprecation. + +## v8.8.2 — 2026-07-19 (one field-resolution law: reserved-field aggregates stop drifting) + +Four fixes from a consumer conformance audit, all rooted in the same disease — two field-resolution +regimes where there must be one: + +- **Aggregates grouped by a RESERVED field (`subtype`, `visibility`, …) now decrement on + delete.** The delete/update hooks fed the aggregation engine a partial entity view (type, + service, data, metadata only), so a reserved-field `groupBy` resolved to a nonexistent group + on the way DOWN — counts drifted upward forever after any delete, and updates that moved an + entity between reserved-field groups double-counted it. The hooks now pass the full-fidelity + entity view (every reserved field top-level, the same shape the add path uses). If your + deployment derives stats from reserved-field aggregates, re-define those aggregates once + after upgrading (a changed definition triggers one rescan) or run them fresh — the drifted + persisted counts do not self-heal retroactively. +- **Aggregation `source.where` on reserved fields now filters** instead of silently matching + nothing: the matcher resolves fields through the same resolver `groupBy` uses (top-level + standard fields + custom metadata), so `where: { subtype: 'note' }` means what it says. +- **`removeMany()` refuses empty/invalid selectors loudly.** A bare array passed positionally + (`removeMany([id])` instead of `removeMany({ ids: [id] })`), an empty params object, or + `ids: []` used to resolve successfully having deleted nothing. All three now throw. +- **`find()` accepts both where-key spellings.** Metadata is flattened at index time + (`metadata.entry.title` indexes as `entry.title`); a `metadata.`-prefixed where key now + falls back to its flattened spelling when the prefixed one isn't indexed — the + "unindexed field(s), returning []" confusion for storage-shaped spellings is gone. (A + literal nested custom key named `metadata` still wins when indexed as spelled.) + +## v8.8.1 — 2026-07-18 (flush no longer walks the whole generation history + the import dedup off-switch is now honest) + +### The flush-storm fix (production incident, reported by a long-running deployment) + +Under the default adaptive retention, **every `flush()` re-walked the entire committed +generation history** to compute total history bytes for the budget check — O(all +generations) with disk re-reads past the 4,096-entry delta-cache bound. On a brain with +70,000+ accumulated generations that turned every write into a full-tail scan (60-100s +writes), even though the budget (free-RAM-based) never tripped and nothing was ever +reclaimed. Fixed: + +- `historyBytes()` now maintains a **running total**: seeded by one walk on first use, + then updated incrementally at every commit and reclaim — the adaptive retention check + on every flush is O(1). Invariant regression-pinned (running total ≡ fresh walk through + both commit paths and compaction). +- New **`brain.historyStats()`** (read-only, exported `HistoryStats`): generation count, + total on-disk bytes, generation/timestamp range, compaction horizon, retention mode, + and the effective adaptive budget — the one-call fleet-audit for sizing retention + exposure per brain. +- Interim guidance for keep-everything deployments already affected: `retention: 'all'` + skips the adaptive accounting entirely (and is the correct policy if you never want + history reclaimed). The accumulated files are harmless at rest; this release removes + the per-write cost of their existence. + +### The import dedup off-switch (lifecycle honesty) + +The post-import background deduplication pass (a merge-DELETE writer that runs ~5 minutes +after an import, merging entities judged duplicates by id / name / vector similarity) had +three lifecycle defects, all fixed: + +- **`enableDeduplication: false` now actually disables it.** The background pass was + scheduled unconditionally — an import that explicitly opted out could still have + entities auto-removed 5 minutes later. The flag now gates BOTH the inline merge and + the background pass (regression-pinned). +- **One deduplicator per brain, owned by the brain.** Each `import()` call constructed its + own coordinator + deduplicator, so the "debounced" timer never actually debounced across + imports (N imports = N delete timers). The brain now owns a single instance — the + debounce genuinely spans imports — and `close()` cancels pending work, so a delete pass + can never fire against a closed brain. +- **The 5-minute timer is unref'd** — a pending pass no longer holds the process open + (the exit-hang class; this timer had escaped the earlier sweep). + +Retention note for keep-everything deployments: with `enableDeduplication: false` on +import calls and `retention: 'all'` in config, no engine path removes records +automatically. + +## v8.8.0 — 2026-07-17 (OS-limit detection for pool-scale deployments) + +Small minor: brains now detect the two OS limits that bite at pool scale and warn **before** +the incident instead of during it. + +- At open (once per process, Linux-only, measurement-only), Brainy reads `RLIMIT_NOFILE` + (soft/hard, from `/proc/self/limits`) and `vm.max_map_count`, and warns loudly when either + sits below the pool-scale floors (soft NOFILE < 65 536; max_map_count < 262 144) — with the + exact raise commands (`ulimit -n` / `LimitNOFILE=` / `sysctl vm.max_map_count`). On stock + defaults the failure otherwise arrives as `EMFILE` or a failed mmap deep inside an index + open, long after the real cause stopped being visible. An unreadable limit produces **no** + warning — no measurement, no claim (non-Linux platforms stay silent). +- Exported for ops doors: `checkOsLimits()` returns the full `OsLimitsReport` + (values + warnings) programmatically, with the floors exported as constants. + +## v8.7.1 — 2026-07-17 (writer-lock acquisition is race-proof + machine-readable through init) + +Two hardenings of the multi-process writer lock (the `locks/_writer.lock` lease that makes a +second writer on the same brain directory fail loudly): + +- **Lock acquisition claims atomically.** The acquire path used read-then-write, leaving a + window where two processes racing an *absent* lock could both "succeed" — and the loser + kept running unlocked, silently. The claim is now an atomic create-exclusive write + (`O_EXCL`): exactly one racer wins; the loser re-evaluates and either fails loudly with + the winner's details or performs a verified stale-takeover. Bounded retries; contention + beyond them fails loudly rather than degrading into a lockless open. +- **`BRAINY_WRITER_LOCKED` survives `init()`.** The conflict error documents a + machine-readable contract (`err.code`, `err.lockInfo` with the holder's pid/host/ + heartbeat), but init's error wrapping silently stripped both, leaving consumers a message + to regex against. The error now passes through unwrapped. + +Measured while verifying (for operators sizing audits): `brain.auditGraph()` at a +production-consumer scale of ~2,600 relationships / 800 entities costs ~0.1 s warm and +~0.5 s cold, with exact scar counting across reopen. + +## v8.7.0 — 2026-07-17 (bulk-transact ergonomics: scaled budgets + timeout telemetry) + +The bulk-import ergonomics release, from a consumer's measured production incident (a serial +import on network-attached storage at ~2 s/op met a flat 30 s transaction budget): + +- **The transact apply budget now scales with the batch** — `max(30 s, opCount × 2 s)` — or + is exactly what you pass as the new `TransactOptions.timeoutMs`. A flat 30 s cap silently + limited honest bulk work to ~15 operations on slow disks while looking generous for small + batches. Internal batch paths (e.g. `removeMany` chunks) get the same scaling. +- **`TransactionTimeoutError` is a diagnosis, not just a failure**: it now reports the + operation it stopped at as `i/N` with the operation's name, elapsed vs budgeted time, and + states the batch rolled back atomically and is retryable. Its `context` carries the same + fields programmatically. +- **The transact envelope is documented** — batch sizing, budget math, chunking with + `ifAbsent` idempotency, and the precompute pattern (`embedBatch` + per-op `vector`) that + keeps model inference out of the commit path. Guide: `docs/guides/optimistic-concurrency.md`. +- Note: `brain.embed()` / `brain.embedBatch()` (the precompute APIs) already ship — public, + with native-provider passthrough, verified end-to-end (batch and single paths produce + bit-identical vectors; a vector-supplied `add` is fully searchable). Honest measurement: + on the default WASM engine, batch throughput ≈ sequential (~160 ms/text) — the precompute + win is keeping inference out of the budgeted commit path, not raw embedding speed. + +## v8.6.0 — 2026-07-17 (brain.auditGraph — the graph-truth verification instrument) + +A minor release adding one new public API, from the fleet's graph-trust program: a read-only +audit that **proves whether relationship reads return stored truth** on a given brain. + +- **`brain.auditGraph(options?)`** walks every canonical relationship record, queries the same + read path applications use (`related()` / VFS `readdir`) with all visibility tiers included, + and classifies every discrepancy into its failure family: `missingFromReads` (records the + read path omits — a stale adjacency index), `danglingEndpoints` (relationships whose endpoint + entity no longer exists — the historical partial-delete scar class), and `readOnlyVerbIds` + (read-path edges with no stored record — ghosts). Design-hidden internal/system edges are + counted separately so intentional hiding is never misclassified as loss. Counts are exact; + example lists cap at `maxExamples` with an explicit `truncatedExamples` flag; the result is + narrated loudly on incoherence. Mutates nothing — safe on a live brain. +- The operational pairing: audit → if incoherent, `repairIndex()` → audit again. A `coherent` + report after the repair is the verified all-clear. Run it after any engine upgrade, restore, + or migration. Guide: `docs/guides/inspection.md`. +- Also: `getNounIds` pagination now refuses an undecodable resume cursor loudly (the same + contract `getNouns`/`getVerbs` gained in 8.5.2 — the third and final walk brought under it). +- Types exported: `GraphAuditReport`, `GraphAuditDiscrepancy`. + +## v8.5.2 — 2026-07-17 (aggregation backfill: exception-safe, generation-verified, and loud) + +Hardening patch from a migration incident (a byte-copied store on new hardware; the service +entered a silent full-CPU loop at boot). Four changes, all in the aggregation engine's +backfill/adoption path: + +- **Backfill walks are exception-safe and non-destructive.** A rescan now builds into a + staging map and swaps in atomically on completion; a mid-walk failure drops the staging map, + keeps the previous live state serving, and surfaces the storage error to the failing query. + Previously the walk wiped live state *before* a scan that could throw, never cleared the + pending flag on failure, and re-ran a full walk on every subsequent query — a silent + wipe/walk/throw loop at the caller's retry rate. +- **Failed walks are latched.** After a walk fails, retries within a 30-second cooldown rethrow + the recorded error instantly instead of re-walking — a tight caller-side retry loop now costs + one loud error per query, never a full store walk per query. +- **Adoption is generation-verified.** Persisted aggregation state is stamped with the store's + committed generation at flush; reopen adoption requires the stamp to equal the current + watermark. Stale state (unclean shutdown) or over-counting state (a fact-log truncation on a + copied store pulled the watermark back) triggers exactly one loud rescan — never a silent + adopt. Pre-8.5.2 state on generation-aware stores rescans once after upgrade, then is stamped. +- **The path narrates.** Adoption decisions, no-adoptable-state outcomes, walk start/finish + (entity count + duration), and walk failures all log by default; a non-advancing storage + pagination cursor aborts the walk loudly instead of looping forever. + +Plus three guards from a full audit of every loop in the open/init path: + +- **Invalid pagination cursors fail loudly.** A supplied-but-undecodable resume token to + `getNouns`/`getVerbs` used to silently restart the walk at offset 0 — to a `while(hasMore)` + caller that re-serves page 1 forever (an unbounded silent CPU loop). It now throws with a + clear message instead. +- **The graph cold-load verb walk has a stall guard.** `hasMore=true` with a missing or + non-advancing cursor aborts loudly instead of re-reading the same page forever. +- **A derived index AHEAD of the store is named at open.** Brainy already surfaced a provider + generation *behind* the committed watermark; the *ahead* direction (the signature of a + byte-copy of a live service, or a log truncation during crash recovery) now logs a loud + warning explaining what happened and that `brain.repairIndex()` forces a heal — instead of + passing unnamed into whatever the derived index does next. + +## v8.5.1 — 2026-07-17 (aggregation state survives restarts + the query-cap ratchet removed) + +Patch release from a production incident (aggregate/count paths taking 40–90 s on an idle box +while vector search stayed fast, and every `find({ limit: 5000 })` suddenly failing against an +"auto-configured query limit of 1000"). Three fixes, one cosmetic: + +- **Aggregation state is actually adopted on reopen.** The boot pattern `defineAggregate()` → + query raced the engine's async state load: the synchronous define always won, flagged a + backfill, and the first query then wiped the just-loaded persisted state and re-walked the + entire store — every restart, forever. Reopening with an unchanged definition now adopts the + persisted state directly (zero scans); a backfill runs only on a real definition change, a + missing/failed state load, or a write that landed before adoption (exactness wins). Apps that + rely on persisted definitions without re-defining at boot also no longer race a spurious + "Aggregate not defined". +- **Backfills are single-flight and batched.** Concurrent queries on a cold aggregate used to + each wipe the others' partial state and start their own full store walk — under steady query + arrival the store never converged (the 40–90 s loop). Now all concurrent queries share one + walk, and one walk fills every aggregate pending backfill (M aggregates ≠ M scans). +- **The query-cap "learning" ratchet is removed.** `maxLimit` is a memory-protection bound, but + a hidden tuner shrank it 20 % per recorded query while the lifetime-average query time + exceeded 1 s — down to a floor of 1000, below the documented 10 000 auto floor, with no + practical recovery, and the resulting error blamed "available free memory" (stale basis + label). The cap now comes from its construction-time basis (or your explicit + `maxQueryLimit` / `reservedQueryMemory`) alone and never changes at runtime; query timing is + recorded for diagnostics only. +- **Cosmetic:** the engine's own persistence keys (`__aggregation_*`, `brainy:entityIdMapper`) + no longer log `[Storage] Unknown key format` at boot — they were always routed correctly; + they're now recognized before the warning fires. + +Operationally: if a host was bitten, upgrading and restarting is the whole fix — no repair +ritual needed. Setting `maxQueryLimit` explicitly remains the valve that bypasses auto-detection +entirely. + +## v8.5.0 — 2026-07-15 (provider access to the fact log + the shared stamp verifier) + +Small additive follow-up to 8.4.0, from the native accelerator's first consumption pass: + +- **Index providers can now reach the fact log through the storage adapter** — new optional + capability `storage.scanFacts()` / `storage.factLogHeadGeneration()` / `storage.factSegmentPaths()`, + wired by the host brain at init as a closure over its live log. Providers hold only `storage` and + must never construct their own fact-log reader (the log's open path is writer-side); `null` means + "no fact log here — use the enumeration walk." +- **The family-stamp verifier is shared** via `@soulcraft/brainy/internals` + (`readFamilyStamp` / `writeFamilyStamp` / `verifyFamilyStamp` + types) so first-party native + providers run literally the same verification function, never a synchronized copy. +- **Rollup stamp invariants accept strings** (`number | string`) — content fingerprints such as a + per-tree SHA-256 are valid invariant values; a type mismatch reads as incoherence, never a pass. +- **`storage.committedGeneration()`** — the committed watermark as a capability, so a provider + compares its stamp's `sourceGeneration` against the store's truth without parsing the private + manifest format. +- **Two durability/stability contracts pinned in the suite:** fsync-before-ack (holds for + `transact()` today; the single-op path is pinned as the documented future target — group commit + becomes latency batching, never durability skipping) and scan-stability-under-rotation (a scan + snapshot yields exactly its facts — no gaps, duplicates, or bleed-in — while segments rotate + beneath it). + +No behavior change for applications; all additions. + +## v8.4.0 — 2026-07-15 (the generation fact log — a sequential, self-verifying commit stream) + +A minor release, fully backward-compatible (all additions; no behavior change for existing APIs). +This is infrastructure: it changes nothing about how you query today, and lays the substrate that +makes index heals and incremental catch-up sequential-read problems instead of directory walks. + +- **Every committed write now also appends a "fact" — an after-image commit record.** Alongside the + existing before-image history, each committed generation (single-op and `transact()` alike) appends + what each touched entity/relationship *became* — or a body-less tombstone for a removal — to an + append-only, checksummed segment log under `_generations/facts/`. Crash-safe by construction: a + torn tail is detected and ignored; on open the log is reconciled to committed truth, so an absent + generation always means "never committed." `transact()` facts are durable when `transact()` + returns; single-op facts ride the same group-commit flush as their history. + +- **New: `brain.scanFacts()`** — stream committed facts in commit order, in batches, with heal-grade + telemetry (total scope up front; per-batch generation range, byte size, and segment; a summary + cross-check at the end; loud abort on any gap — never a silent skip). **`brain.factSegmentPaths()`** + hands zero-copy consumers the immutable sealed segment files directly. New exported types: + `CommitFact`, `FactOp`, `FactScanBatch`, `FactScanHandle`. Facts accumulate from the first write + after upgrading — pre-existing history is not retroactively converted (enumeration remains the + fallback for old data). + +- **New: the entity-tree family stamp.** At every flush/close, brainy stamps which committed + generation the canonical entity files reflect plus the rollup invariants (entity/relationship + counts) that verify the tree whole. At open, coherence is a comparison — a genuine divergence is + loud and names the failing invariant; `repairIndex()` recounts from canonical and re-stamps. New + exports: `readFamilyStamp`, `verifyFamilyStamp`, `ENTITY_TREE_STAMP_PATH`, `FamilyStamp` types. + +- **Storage adapters** gain optional binary raw-byte primitives (`appendRawBytes`, `readRawBytes`, + `writeRawBytes`, `rawByteSize`) — feature-detected; the filesystem and memory adapters implement + them; an adapter without them simply hosts no fact log. The fact-log namespace is registered as a + protected family: no sweeper or GC can delete under it. + +No API breaks. 24 new tests; the full commit-path regression suite is green. + +## v8.3.3 — 2026-07-15 (rename moves the containment edge — no ghost in the old directory) + +One production-reported fix plus a repair path, completing the delete/move hygiene arc (8.3.1 fixed +deletes, 8.3.2 fixed counters, this fixes moves). + +- **A cross-directory `vfs.rename()` now MOVES the containment edge instead of accumulating one per + parent.** The old parent's `Contains` edge was never removed on a move, leaving the entity a child + of **both** directories: `readdir(oldDir)` kept listing it after the move, re-creating the old path + showed the same name twice, and any tree-walking consumer (sync engines, file browsers) saw the + file in two places. The old edge is now removed by edge id, resolved from the graph's own adjacency + — a removal never requires reading the thing being removed. Bonus fix in the same seam: a move **to + the root** now gets its containment edge (it was previously skipped, orphaning the file out of + `readdir('/')`). + +- **`repairIndex()` now also reconciles VFS containment** (new `vfs.repairContainment()`): every VFS + entity's containment edges are checked against its canonical `metadata.path` — stale old-parent + ghosts and duplicate edges are removed, a missing expected edge is restored, and user + knowledge-graph edges are never touched (only `vfs-contains` edges are candidates). Loud per + repair. Stores that performed cross-directory renames under ≤8.3.2 should run `brain.repairIndex()` + once after upgrading — the same single ritual now heals orphan directories, counters, **and** + containment edges. + +- Also ships a permanent lens-consistency regression suite (combined type+subtype vs subtype-only vs + canonical ground truth, id-for-id, warm and after a cold reopen), ported from the field + investigation that closed the historical lens-drop report. + +No API changes beyond the new optional `vfs.repairContainment()` (also invoked by `repairIndex()`). + +## v8.3.2 — 2026-07-14 (honest counters — the recount + removal-without-re-reading) + +Completes 8.3.1's delete-hygiene story at the counter layer, from a production proof chain reported +by a downstream deployment: persisted entity totals were permanently **inflated** — deletes whose +count decrement was silently skipped — and because paginated `totalCount` serves +`Math.max(persistedTotal, scanned)`, the inflated number always won and **no disk cleanup could ever +lower it**. + +- **A removal's count decrement no longer requires re-reading the record being removed.** The + decrement was sourced from re-reading the entity's metadata inside the delete; if that read + returned `null` (a replace race, or a ghost left by a pre-8.3.1 partial delete) the decrement was + silently skipped while the paired add had counted — minting drift on every write→delete→re-create + cycle. The caller's pre-delete read now rides through the whole delete path + (`remove()`/`removeMany()` → the delete operation → `deleteNoun`/`deleteVerb` → + `deleteNounMetadata`/`deleteVerbMetadata`, both sides symmetric): a null internal read falls back + to the known prior record instead of skipping. The `StorageAdapter` signatures gain an optional + `priorMetadata` parameter (additive; existing adapters unaffected). + +- **`repairIndex()` is the sanctioned counter recount — unconditional, and it actually persists.** + `rebuildTypeCounts()` previously rebuilt only the type-statistics arrays and computed the total + *just to log it* — the persisted scalar (`counts.json`) survived every "rebuild" untouched, so an + already-inflated brain could never be corrected. One canonical walk now rebuilds **every** counter + rollup — scalar totals, per-type maps, and type statistics — and persists them, and `repairIndex()` + runs it unconditionally (not only when orphan directories are found: counters can be inflated over + perfectly clean shelves). Brains with delete history should run `brain.repairIndex()` once after + upgrading; the correction survives reopen. + +No API breaks (optional-parameter additions only). Regression tests cover the drift cycle, the +null-read decrement fallback, and the persisted recount across reopen. + +## v8.3.1 — 2026-07-14 (full-removal deletes + family-scoped migration gate) + +Two production-reported fixes in the write/index spine, plus an operator repair path. No API changes; +all behavior changes make previously-wrong states honest. + +- **Deleting an entity now removes it completely — no more "ghost" leftovers on disk.** A canonical + noun delete removed the metadata (content) leg but left the entity's `vectors.json` and its `/` + directory behind. Consequences observed in a production deployment: deleted rows were + indistinguishable on disk from damage scars, enumerated counts inflated monotonically with every + delete (the leftovers were counted forever), and locator-style reads hit unreadable ghost rows. + `remove()`/`removeMany()`/`deleteNoun`/`deleteVerb` now remove **both legs and the entity + container**, with a full two-leg before-image rollback inside the transaction. The generation log + still holds the delete's before-image, so `asOf()` time-travel reconstructs deleted entities exactly + as before — this is live-HEAD hygiene, not a history change. This also fixes **duplicate `readdir` + entries for re-created VFS paths** at the root: with no ghost state, a delete always unposts its + index rows, so a delete→recreate cycle lists the path exactly once (regression-tested across + repeated cycles). + +- **`repairIndex()` prunes ghost/scar directories left by earlier versions.** Stores that deleted + entities under ≤8.3.0 may hold orphaned entity directories (a vector-only leg, or an empty dir). + `brain.repairIndex()` now sweeps them: it removes only containers with **no metadata content leg** + (never a directory that still holds content), logs every removal, and recomputes type/subtype + counts afterward so totals stop counting ghosts. + +- **Reads no longer hang behind an unrelated index migration (family-scoped gate).** During a native + provider's one-time background migration, *every* read — including plain `get()`, VFS + `readdir`/`readFile`, and metadata-only `find({ where })` — blocked on the whole-brain migration + lock until timeout, even when the migrating index was irrelevant to the read. The gate is now + scoped to the index families a read actually consults: canonical reads (`get`, `batchGet`, VFS + content) never wait; a `find` waits only on the families its query shape needs (vector for + semantic, metadata for `where`/type, graph for `connected`); graph traversals wait only on the + graph family. Writes and unclassified operations keep the conservative whole-brain wait. A read + that *does* need the migrating family still blocks (bounded by `migrationWaitTimeoutMs`) and + surfaces the retryable `MigrationInProgressError` — never a partial result. + +No breaking API change. Each fix ships with regression tests. + +## v8.3.0 — 2026-07-13 (faster index heals + the cross-layer integrity contract) + +Three additive changes. The first is an immediate, standalone performance win; the other two are the +brainy side of the write/index-spine integrity contract, inert until a native accelerator +that implements the matching hooks is present — so this release changes nothing for a JS-only brain +beyond the speedup. + +- **Canonical enumeration is up to ~16× faster — the dominant term in an index heal.** The paginated + entity walk (`getNounsWithPagination`) hydrated each entity's vector + metadata one-at-a-time; since + every index rebuild enumerates canonical storage, that serial per-item latency dominated multi-minute + heals. Hydration is now 16-way bounded-concurrency, with the pagination contract (order, cursor + resume, filters, totalCount) byte-identical to before. New **`getNounIdsWithPagination()`** returns + ids without hydrating anything (zero per-entity reads when unfiltered) for callers that own their own + IO schedule. + +- **Cross-layer integrity — `validateIndexConsistency()` is no longer blind to native providers.** It + only ran the JS metadata index's own check, so a native provider whose manifest/segments/counts had + diverged still read as "healthy". It now feature-detects and aggregates each provider's optional + `validateInvariants()` self-report, names every failing invariant with its numbers, and exposes the + per-provider reports. `repairIndex()` now reconciles native derived state from canonical too + (rebuilding any provider whose failing invariant asks for it). New exported types + `ProviderInvariantReport` / `InvariantResult` / `InvariantHeal`. + +- **Registered-blob families — declared index files are undeletable through the storage layer.** A + provider can declare a derived-index blob *family* (a set of members that are load-bearing together); + once declared, `deleteBinaryBlob` / `removeRawPrefix` refuse to remove a member (new exported + `ProtectedArtifactError`), so a stray in-process sweeper cannot delete a load-bearing index file. The + declaration persists across reopen; `checkDerivedFamiliesPresent()` names any member missing on open. + New optional `StorageAdapter` surface (`registerDerivedFamily` / `unregisterDerivedFamily` / + `listDerivedFamilies` + `DerivedFamilyDeclaration`) and exported `DerivedArtifactMissingError`. + +No breaking API change (all additions are optional/new). Each change ships with regression tests. + ## v8.2.8 — 2026-07-13 (honest index readiness — no more silently-empty queries on a cold index) Closes the last of the three spine anti-patterns: the "dishonest readiness proxy," where `size() > 0` diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..1f3c4732 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,36 @@ +# Security Policy + +## Reporting a vulnerability + +Email **security@soulcraft.com**. That's the one door for security reports +across the company, and it works the same way for Brainy: every report is +read by a human, you'll get a private receipt, and we'll work with you on +coordinated disclosure — please don't open a public issue for anything +that isn't already public. + +Include what you'd want if you were on the other end: affected version, +how to reproduce, and what you think the impact is. If you have a patch or +a suggested fix, send it along — it's welcome but not required. + +There is no bounty program today. We're saying that plainly so you know +what to expect going in. + +## Response time + +We respond as fast as truth allows. That means: no fixed SLA, no promise of +a reply within a specific number of hours — but a real report from a real +person gets read promptly and taken seriously. If you haven't heard anything +in a reasonable stretch, a follow-up email is completely fine. + +## Supported versions + +The latest `8.x` minor release line receives security fixes. If you're +running an older major version, please upgrade before reporting — we can't +commit to backporting fixes to unsupported lines. + +## Scope + +This policy covers the `@soulcraft/brainy` package itself — the code in +this repository. If you're evaluating a deployment that also uses +`@soulcraft/cor`, report issues in that package the same way, to the same +address; we'll route internally. diff --git a/docs/concepts/generation-fact-log.md b/docs/concepts/generation-fact-log.md new file mode 100644 index 00000000..f9b3e974 --- /dev/null +++ b/docs/concepts/generation-fact-log.md @@ -0,0 +1,117 @@ +--- +title: The Generation Fact Log +slug: concepts/generation-fact-log +public: true +category: concepts +template: concept +order: 6 +description: Every committed write also appends a self-verifying "fact" — an after-image commit record — to an append-only log. What facts are, the crash-safety model, the scanFacts() streaming surface, family stamps, and how index providers consume the log for sequential heals. +next: + - concepts/consistency-model + - guides/snapshots-and-time-travel +--- + +# The Generation Fact Log + +Since 8.4.0, every committed generation also appends a **fact** — a compact record of what each +touched entity or relationship *became* — to an append-only, checksummed log under +`_generations/facts/`. Where the generational history answers *"what did things look like +before?"* (before-images, powering `asOf()` and rollback), the fact log answers *"what happened, +in order?"* — one sequential, self-verifying stream of the store's present being written. + +Nothing about querying changes. The fact log exists for three consumers: + +1. **Index heals and rebuilds** — one sequential read in commit order replaces a per-entity + directory walk over millions of files. +2. **Incremental catch-up** — a derived index that knows which generation it reflects reads *just + the gap*, instead of rebuilding from scratch. +3. **Replay and audit tooling** — anything that wants the store's committed timeline as a stream. + +## What a fact is + +One fact per committed generation: + +- **`generation`** and **`timestamp`** — which commit, when. +- **`ops`** — every write in that commit: `{ kind: 'noun' | 'verb', id, record }` where `record` + holds the entity's full after-image (both stored legs), or **`null` for a tombstone** — a + removal carries no body, by design. +- **`meta`** — the transaction metadata `transact()` was submitted with, when present. +- **`blobHashes`** — content-blob references, for exact reclamation accounting. + +Facts accumulate **from the first write after upgrading** — pre-existing history is not +retroactively converted, and consumers fall back to the enumeration walk when no log exists. + +## Crash safety, in one paragraph + +Facts are appended and fsynced **inside the same durability window as the commit itself**, before +the commit point — so after a crash, the log can only ever be *ahead* of committed truth, never +behind it with a hole. On open, the store reconciles the log back to the committed watermark: +torn tails are detected by per-record checksums and cut; whole records beyond the watermark are +truncated. The invariant every reader can rely on: **an absent generation was never committed; a +present fact was.** `transact()` facts are durable the moment `transact()` returns; single-op +facts share the same group-commit flush as the rest of their generation, so a hard kill loses the +fact and the generation *together* — never a torn state. + +## Reading the log + +```typescript +const scan = brain.scanFacts({ fromGeneration: 1 }) +if (scan) { + // Telemetry up front — progress bars get a denominator from second zero. + console.log(scan.headGeneration, scan.segmentCount, scan.approxFactCount) + + for await (const batch of scan.batches()) { + // Each batch: { facts, firstGeneration, lastGeneration, factCount, byteSize, segmentId } + for (const fact of batch.facts) { + for (const op of fact.ops) { + if (op.record === null) { + // a tombstone: op.id was removed in this generation + } + } + } + } + + console.log(scan.summary()) // { factsYielded, segmentsRead } — the cross-check +} +``` + +- `scanFacts()` returns `null` when the store hosts no fact log (older store, or a storage adapter + without binary append support) — fall back to enumerating entities. +- Scans run against a **snapshot**: facts appended after the scan opens never bleed in, each fact + is yielded exactly once, and a detected gap aborts loudly — never a silent skip. +- `brain.factSegmentPaths()` returns the immutable, *sealed* segment files for zero-copy consumers + (the append-mutable tail is excluded — read it through `scanFacts()`). + +## Family stamps: how a projection proves it's current + +Anything derived from the store — an index, the entity file tree itself — carries a **family +stamp**: a small JSON record of *which committed generation the projection reflects* +(`sourceGeneration`) plus the invariants that verify it whole (exact per-file byte sizes for +bounded families; rollup invariants like entity counts for unbounded ones). At open, coherence is +a **comparison**, not a walk: + +- stamp equals the committed watermark and invariants hold → serve; +- stamp is behind → the projection reads just the gap from the fact log; +- invariants fail → loud, named divergence — `brain.repairIndex()` rebuilds from canonical and + re-stamps. + +The verifier is exported (`verifyFamilyStamp`) so every projection — TypeScript or native — runs +literally the same check. + +## For plugin authors: the storage capability + +Index providers receive the storage adapter, not the brain — so the host wires the log onto it. +Feature-detect and prefer the stream; fall back to enumeration: + +```typescript +const scan = storage.scanFacts?.({ fromGeneration: stamp.sourceGeneration + 1 }) +if (scan) { + // sequential catch-up from the log +} else { + // enumeration walk (older store or adapter) +} +const committed = storage.committedGeneration?.() // the watermark stamps compare against +``` + +Providers must never construct their own reader over the log's files — the open path belongs to +the single writer (it reconciles the log at open); the capability is the sanctioned seam. diff --git a/docs/guides/aggregation.md b/docs/guides/aggregation.md index e58eb2ce..11d86ec8 100644 --- a/docs/guides/aggregation.md +++ b/docs/guides/aggregation.md @@ -11,7 +11,13 @@ No batch jobs. No scheduled recalculations. Aggregates stay current with every w **Defining over existing data:** if you define an aggregate on a store that already holds matching entities, Brainy backfills it from those entities on the first query (a one-time scan, then purely incremental). So `defineAggregate()` behaves the same whether you define it before -or after the data exists — including when a persisted brain reopens already populated. +or after the data exists. + +**Reopening a persisted brain:** aggregate state persists across restarts. Re-defining the +same aggregate at boot (the normal declarative pattern) adopts the persisted state directly — +no rescan. A backfill scan runs only when the definition actually changed, when no persisted +state exists, or when the state failed to load; and however many aggregates need backfilling, +they share a single scan. ## Quick Start diff --git a/docs/guides/external-backups-and-sparse-storage.md b/docs/guides/external-backups-and-sparse-storage.md new file mode 100644 index 00000000..f28dcfd7 --- /dev/null +++ b/docs/guides/external-backups-and-sparse-storage.md @@ -0,0 +1,99 @@ +--- +title: External Backups & Sparse Storage +slug: guides/external-backups +public: true +category: guides +template: guide +order: 10 +description: How to back up a brain directory with external tools (tar, rsync, cp) without exploding sparse files — why a store can show 100+ GB "apparent" size on a small disk, which files are sparse, and how persist()/restore() handle it for you. +next: + - guides/snapshots-and-time-travel + - concepts/storage-adapters +--- + +# External Backups & Sparse Storage + +The built-in snapshot path — [`db.persist()` and `brain.restore()`](/docs/guides/snapshots-and-time-travel) — +already handles everything on this page for you. Read this when you back up a brain directory with +**external tools**: `tar`, `rsync`, `cp`, `scp`, or a filesystem-level backup agent. + +## The one-sentence rule + +> **Always use the sparse-aware flag**: `tar czSf` (capital `S`), `rsync --sparse`, +> `cp --sparse=always`. A naive copy can turn a 2 GB store into a 100+ GB one — or fail +> the disk entirely. + +## Why: some files are sparse + +When a native accelerator plugin is active, parts of the index live in **memory-mapped files** +created at a large fixed virtual size — the file's *apparent* size — while the filesystem only +allocates blocks that were actually written. A brand-new id-mapper file can report tens of +gigabytes in `ls -l` while occupying a few megabytes on disk. + +Check the difference yourself: + +```bash +ls -lh brain-data/_id_mapper/ # APPARENT size (can be huge) +du -sh brain-data/ # ALLOCATED size (the real footprint) +``` + +The sparse candidates in a brain directory: + +| Path | What it is | +|---|---| +| `_id_mapper/` | The native id-mapper's mmap files (large fixed virtual size) | +| `_blobs/` | Native index files (vector base, segments) — may be mmap-backed | + +Everything else (entities, `_system`, `_generations`, `_cas` content blobs) is ordinary dense data. + +## Doing it right + +**tar** — the `S` flag detects holes and stores only real data: + +```bash +tar czSf brain-backup.tgz /data/brain +# restore preserves the holes: +tar xzSf brain-backup.tgz -C /data/ +``` + +**rsync**: + +```bash +rsync -a --sparse /data/brain/ backup-host:/backups/brain/ +``` + +**cp**: + +```bash +cp -a --sparse=always /data/brain /backups/brain +``` + +**What goes wrong without the flag:** the copy *materializes* every hole as real zero bytes. +A store whose apparent size exceeds the target disk fails with `ENOSPC` partway through — and a +copy that *does* fit silently costs the full apparent size in storage and transfer time. + +## What the built-in paths do (so you don't have to) + +- **`db.persist(path)`** snapshots via **hard links** — instant and space-shared, since every data + file is immutable-by-rename. The handful of append-in-place files (the transaction log, the + commit fact log's tail segment) and mmap-mutated directories (`_id_mapper/`) are **byte-copied** + instead, so a post-snapshot write can never reach through a shared inode into your backup. +- **`brain.restore(path, { confirm: true })`** is **non-destructive and sparse-aware**: the snapshot + is copied into a staging area *before* any live data is touched (all-zero blocks stay holes), and + only after the copy fully succeeds does an atomic swap move it into place. A failed copy — + including `ENOSPC` — leaves the live store exactly as it was. A crash mid-swap completes forward + on the next open. + +## Live-store caveats for external tools + +1. **Prefer snapshotting a `persist()` output, not the live directory.** `persist()` produces a + crash-consistent, immutable snapshot; running `tar` against a live, actively-written directory + can capture a torn mid-write state. If you must archive live, stop writes first (or accept that + the archive is only as consistent as the moment's flush state). +2. **Never prune or "clean up" files inside a brain directory.** Index files that look stale or + redundant are load-bearing; the store protects its declared index families from in-process + deletion, but an external `rm` bypasses that fence. If space is the concern, `du -sh` first — + the allocated size is usually far smaller than it looks. +3. **Verify restores by opening them.** `Brainy.load(path)` opens any snapshot or restored + directory read-only — the store verifies its own coherence at open and reports loudly if + anything is missing or torn. diff --git a/docs/guides/find-limits.md b/docs/guides/find-limits.md index 66418b80..4c7fd252 100644 --- a/docs/guides/find-limits.md +++ b/docs/guides/find-limits.md @@ -40,6 +40,10 @@ Brainy picks `maxLimit` from the first of these that's available: Worked example: a 4 GB Cloud Run container picks priority 3 → `floor(4 GB × 0.25 / 25 KB) = floor(40 960) = 40 000` results. A 900 MB free-memory box on priority 4 gets `floor(900 MB / 25 KB) = ~36 000`. +The cap is fixed at construction and never changes at runtime. Query timing is recorded +for diagnostics only — a burst of slow queries cannot silently shrink the cap, and the +auto-detected tiers (3 and 4) never go below a floor of 10 000. + > **Calibration note.** Pre-7.30.2 used 100 KB per result instead of 25 KB, which produced caps that were 4× too tight for typical workloads (an 8 KB / result reality). 7.30.2 recalibrated to match observed entity sizes; existing `limit: 10_000` safety patterns now pass silently on any reasonably-sized box. ## What happens when you exceed the cap diff --git a/docs/guides/import-anything.md b/docs/guides/import-anything.md index e0cd94ed..b1bb15ef 100644 --- a/docs/guides/import-anything.md +++ b/docs/guides/import-anything.md @@ -300,7 +300,10 @@ await brain.import(data, { // Deduplication enableDeduplication: true, // Check for duplicate entities (default: true) deduplicationThreshold: 0.85, // Similarity threshold for duplicates (0-1, default: 0.85) - // Note: Auto-disabled for imports >100 entities + // Notes: false disables BOTH the inline merge and the background pass that + // runs ~5 min after the last import (merged duplicates are deleted). + // The inline pass auto-disables for imports >100 entities (O(n²) cost); + // the background pass still covers those unless the flag is false. // Performance chunkSize: 100, // Batch size for processing (default: varies by operation) diff --git a/docs/guides/import-quick-reference.md b/docs/guides/import-quick-reference.md index 5a850a86..7837d49e 100644 --- a/docs/guides/import-quick-reference.md +++ b/docs/guides/import-quick-reference.md @@ -86,11 +86,22 @@ await brain.import(file, { ```typescript await brain.import(file, { - enableDeduplication: true, // Check for duplicates (default: false) + enableDeduplication: true, // Check for duplicates (default: true) deduplicationThreshold: 0.85 // Similarity threshold (default: 0.85) }) ``` +Deduplication merges entities judged duplicates — the non-primary records are +**deleted**. Set `enableDeduplication: false` to disable it entirely: the flag +gates both the inline merge during import and the background pass that runs +about 5 minutes after the last import. + +```typescript +await brain.import(file, { + enableDeduplication: false // No merging, inline or background +}) +``` + ### Import Tracking Track and organize imports by project: diff --git a/docs/guides/inspection.md b/docs/guides/inspection.md index 015c5118..240e81ae 100644 --- a/docs/guides/inspection.md +++ b/docs/guides/inspection.md @@ -166,6 +166,34 @@ brainy inspect diff /data/brain-prod /data/brain-staging Sample-based — for a full diff, dump both with `inspect dump` and compare the JSONL. +## Auditing graph-read truth + +`brain.auditGraph()` (8.6.0+) proves — or disproves — that relationship reads +return canonical truth on a given brain, without mutating anything. It walks +every stored relationship record, asks the same read path your application +uses (`related()`, VFS `readdir`) with every visibility tier included, and +classifies every discrepancy: + +```typescript +const report = await brain.auditGraph() + +report.coherent // true = related()/readdir can be trusted on this brain +report.missingFromReadsCount // records the read path omits — stale index +report.danglingEndpointsCount // relationships whose endpoint entity is gone +report.readOnlyCount // read-path edges with NO stored record — ghosts +report.visibilityHiddenCount // internal/system edges hidden by design (not a fault) +``` + +Counts are always exact; the example lists (`missingFromReads`, +`danglingEndpoints`, `readOnlyVerbIds`) are capped at `maxExamples` +(default 100) and `truncatedExamples` says so when they are. + +Run it after any engine upgrade, restore, or migration. If it reports +discrepancies, run `brain.repairIndex()` and audit again — a `coherent` +report after the repair is the verified statement that the heal worked. +Cost: one relationship-record walk plus one indexed read per distinct +source entity — safe on a live brain. + ## Repairing a corrupted store If invariants fail and you suspect index corruption, `inspect repair` diff --git a/docs/guides/optimistic-concurrency.md b/docs/guides/optimistic-concurrency.md index a21a8af0..268bc5fa 100644 --- a/docs/guides/optimistic-concurrency.md +++ b/docs/guides/optimistic-concurrency.md @@ -180,3 +180,35 @@ Brainy 8.0 has exactly two write-coordination counters, at two granularities: They compose: a `transact()` batch can carry per-entity `ifRev` checks *and* a whole-store `ifAtGeneration`; any failed check rejects the entire batch before anything is staged. Generations also power snapshots and time travel (`brain.now()`, `brain.asOf()`, `db.persist()`) — see the [consistency model](../concepts/consistency-model.md) and [Snapshots & Time Travel](./snapshots-and-time-travel.md). A snapshot or historical view captures each entity *including* its `_rev` at that moment, so reading the past and writing back with `ifRev` against the live state works exactly as you'd hope: the write fails if the entity moved since the state you copied from. + +## The transact envelope: batch size, budget, and bulk imports + +`transact()` applies its batch atomically under one commit — which means the whole batch +shares one **apply budget**. Since 8.7.0 the budget scales with the batch: +`max(30 s, opCount × 2 s)`, or exactly what you pass as `timeoutMs`. A tripped budget rolls +the entire batch back (nothing partial survives) and throws a retryable +`TransactionTimeoutError` that names the operation it stopped at, the batch size, and the +elapsed vs budgeted time — a diagnosis, not just a failure: + +``` +Transaction timed out at operation 41/120 ('add') — 246012ms elapsed, budget 240000ms. +The batch rolled back atomically; retry with a higher timeoutMs or a smaller batch. +``` + +Practical envelope guidance for bulk work: + +1. **Precompute embeddings outside the commit path.** Embedding inside `transact()` spends + the budget on model inference. Use `brain.embedBatch(texts)` and pass each vector via + the op's `vector` field — the commit then pays only storage costs, and a retried batch + never re-pays inference. (The win is *where* the inference happens, not raw embedding + throughput: on the default WASM engine, batch and sequential embedding measure + comparably, ~160 ms/text; native embedding providers may batch faster.) +2. **Chunk very large imports** into batches of a few hundred ops with one `transact()` + each. You lose whole-import atomicity but keep per-chunk atomicity, bounded memory, and + resumability — pair with `ifAbsent` upserts so a retried chunk is idempotent. +3. **Slow disks change the math, not the contract.** On network-attached storage a single + op can cost ~2 s (canonical write + fsync + index maintenance). The scaled default + absorbs that; pass an explicit `timeoutMs` only when you know better than the scale. +4. **`addMany`/`relateMany` are the convenience tier** — they chunk and batch-embed for + you, with per-item error reporting instead of batch atomicity. Choose by what you need: + atomic-all-or-nothing → `transact()`; resilient bulk load → `addMany`. diff --git a/docs/guides/snapshots-and-time-travel.md b/docs/guides/snapshots-and-time-travel.md index 6f7b150a..490aecab 100644 --- a/docs/guides/snapshots-and-time-travel.md +++ b/docs/guides/snapshots-and-time-travel.md @@ -9,6 +9,7 @@ description: Recipes for the Db API — instant backups with persist(), restore, next: - concepts/consistency-model - guides/optimistic-concurrency + - guides/external-backups --- # Snapshots & Time Travel @@ -41,6 +42,10 @@ bytes. Cross-device targets fall back to per-file byte copies, and persisting an in-memory brain serializes it to the same directory layout — a real, durable store. +> Archiving a brain directory with **external tools** (`tar`, `rsync`, `cp`)? +> Some index files are sparse and can explode to their apparent size under a +> naive copy — see [External Backups & Sparse Storage](/docs/guides/external-backups). + Two things to know: - `persist()` requires the view to still be the store's **latest** @@ -339,8 +344,12 @@ For per-entity write coordination (rather than whole-store history), the ## Keeping history bounded Under Model-B every write is a generation, so history can grow quickly — -Brainy auto-compacts on every `flush()`/`close()` under the **`retention`** -knob (configured on the constructor): +Brainy auto-compacts at `close()` (time-bounded per pass) under the +**`retention`** knob (configured on the constructor). Since 8.9.0, `flush()` +never compacts: flushing is durability work and costs only what the current +window's writes cost, regardless of history backlog. A long-lived writer that +never closes keeps its history until its next explicit `compactHistory()` — +schedule one in your maintenance window if you run bounded retention: ```typescript // Zero-config: ADAPTIVE — keep as much history as free disk/RAM allows, @@ -354,10 +363,13 @@ new Brainy({ retention: 'all' }) new Brainy({ retention: { maxGenerations: 1000, maxAge: 7 * 86_400_000, maxBytes: 512 * 1024 ** 2 } }) ``` -Reclaim manually at any time (the same caps): +Reclaim manually at any time (the same caps, plus an optional per-pass time +budget for maintenance windows — an early stop is a consistent prefix and the +next pass resumes): ```typescript await brain.compactHistory({ maxGenerations: 100, maxAge: 7 * 24 * 60 * 60 * 1000 }) +await brain.compactHistory({ maxBytes: 512 * 1024 ** 2, timeBudgetMs: 10_000 }) ``` Compaction never breaks a pinned read — record-sets are reclaimed only when diff --git a/docs/performance-envelopes.md b/docs/performance-envelopes.md new file mode 100644 index 00000000..d29677e3 --- /dev/null +++ b/docs/performance-envelopes.md @@ -0,0 +1,83 @@ +--- +title: Performance Envelopes +slug: guides/performance-envelopes +public: true +category: guides +template: guide +order: 40 +description: Measured per-operation latency envelopes at stated scales — what to expect, on what hardware, and exactly how each number was produced. +next: + - guides/find-limits +--- + +# Performance Envelopes + +Every number on this page is **measured, never projected** — produced by the script +cited at the bottom, against the built package (the artifact you install), on the stated +hardware. Each entry says what was measured, at what scale, on which storage backend. +When a release touches a measured path, that operation is re-measured and this page +updates in the same release. + +Two scopes to keep straight: + +- **These envelopes are the pure-JS engine** (no native accelerator registered) on + filesystem storage. This is the floor every deployment gets from `npm install` alone. +- **Accelerated deployments** (the optional native provider) publish their own numbers — + this page never claims them. + +## Read operations + +Reads are where the architecture pays off: after the write path has done its indexing +work, queries answer from purpose-built indexes without scanning. + +| Operation | 1,000 entities | 10,000 entities | Notes | +|---|---|---|---| +| `get(id)` (warm) | p50 < 0.1ms | p50 < 0.1ms | served from cache/metadata index | +| `find` (metadata: indexed equality + range, limit 100) | p50 1.0ms · p95 1.8ms | p50 7.0ms · p95 8.9ms | column-store bitmap paths | +| `related(id)` (per-node adjacency) | p50 < 0.1ms · p95 0.2ms | p50 < 0.1ms | LSM adjacency index — O(degree), scale-independent | +| `find` (semantic: embed + HNSW, 1k docs) | p50 178ms · p95 393ms | — | dominated by WASM query embedding (measured on a machine under concurrent load — treat the p95 as an upper bound); the vector search itself is single-digit ms | + +## Write operations + +Under Model-B **every write is its own durable generation** — a single-op `add` pays +serialization, before-image staging, and fsync before it acks. That durability is priced +into the write path visibly, by design: + +| Operation | 1,000 entities | 10,000 entities | Notes | +|---|---|---|---| +| `add` (single-op) | p50 167ms · p95 171ms | p50 165ms · p95 172ms | full durable generation per write — flat across scale | +| `addMany` (bulk) | ~163ms/entity | ~187ms/entity | **currently per-item commits** — see the honest note below | +| `relateMany` | ~0.8ms/edge | ~0.9ms/edge | edges batch efficiently today | +| `flush` (steady-state, 1 pending write) | p50 8ms · p95 10ms | p50 45ms · p95 52ms | durability-only since 8.9.0 — cost no longer depends on history backlog or retention mode | + +**The honest note on bulk writes:** `addMany` today commits each item as its own +generation (the same durability as single-op `add`, serialized by the single-writer +lock), so bulk-load cost is N × single-op cost. Batched chunk commits (one generation +and one fsync window per chunk, as `removeMany` already does) are designed into the +unified-commit work on the current roadmap. Until that ships, size bulk imports +accordingly — 10k entities is minutes, not seconds, on filesystem storage. + +## Open / close + +| Operation | 1,000 entities | 10,000 entities | Notes | +|---|---|---|---| +| `open` (empty store) | ~560ms | ~190ms | includes embedder initialization | +| `open` (warm, populated, clean shutdown) | 763ms | 4.9s | pure-JS vector index load dominates and grows with entity count; the native accelerator exists precisely to remove this | +| `close` | bounded | bounded | auto-compaction pass is time-bounded (~5s max) since 8.9.0 | + +A store that was NOT cleanly closed pays index rebuilds on top of the warm-open +number (tens of seconds at 10k) — clean shutdown is worth engineering for. + +## How these were produced + +- **Hardware**: Intel Core i9-14900HX (32 threads), 62GB RAM, NVMe, Linux, Node v22. +- **Backend**: `storage: { type: 'filesystem' }`, pure JS (no native providers). +- **Embeddings**: deterministic stub for non-semantic ops (isolates engine cost); + the real WASM embedder for the semantic row (that's what you'll run). +- **Method**: p50/p95 over 50–200 samples per op against the built `dist/`; + the measuring script ships in the repo history and re-runs per release. + +Numbers on different hardware will differ; the *shape* (sub-2ms indexed reads, +~160ms embedding-bound semantic queries, durability-priced writes) is the envelope +you should hold your deployment against. If your measurements diverge from these +shapes by an order of magnitude, something is wrong — file it. diff --git a/package-lock.json b/package-lock.json index b2b3ce8c..37aeb81d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "8.2.8", + "version": "8.10.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "8.2.8", + "version": "8.10.0", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 095737c4..e4bc8144 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "8.2.8", + "version": "8.10.0", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", diff --git a/scripts/push-docs.js b/scripts/push-docs.js new file mode 100644 index 00000000..699d332b --- /dev/null +++ b/scripts/push-docs.js @@ -0,0 +1,116 @@ +#!/usr/bin/env node +/** + * @module scripts/push-docs + * @description Push this repo's PUBLIC docs to the soulcraft.com docs ingest + * door after an npm publish (VENUE-DOCS-RELEASE-PUSH — retires the old + * build-time docs sync). + * + * Contract (mirrors the reference implementation on the serving side): + * POST {base}/api/docs/ingest + * headers: x-service-secret: $DOCS_INGEST_SECRET, Content-Type: application/json + * body: { docs: [{ slug, title, markdown, nav: { order, section } }] } + * batches of 10, idempotent per slug. + * + * A doc is public iff its frontmatter has `public: true` AND a `slug`. The + * frontmatter is stripped; `category` → nav.section, `order` → nav.order. + * + * Deliberately NOT pushed: the combined /docs landing index. It spans BOTH + * engine corpora (this repo's and the native accelerator's), so a per-repo + * push would clobber the union — the index is authored on the serving side. + * + * Env: DOCS_INGEST_SECRET (required), DOCS_INGEST_BASE (default + * https://soulcraft.com). Exits 0 with a LOUD warning when the secret is + * absent (the npm publish has already happened; the serving side runs its + * interim sync on request) and exits 1 when a push actually fails — the docs + * site would silently trail npm otherwise, and that must be visible. + */ +import * as fs from 'node:fs' +import * as path from 'node:path' + +const BASE = (process.env.DOCS_INGEST_BASE || 'https://soulcraft.com').replace(/\/+$/, '') +const SECRET = process.env.DOCS_INGEST_SECRET +const DOCS_DIR = path.join(path.dirname(new URL(import.meta.url).pathname), '..', 'docs') +const BATCH = 10 + +if (!SECRET) { + console.warn( + '⚠️ DOCS PUSH SKIPPED: DOCS_INGEST_SECRET is not set.\n' + + ' soulcraft.com/docs now TRAILS this npm release until docs are pushed.\n' + + ' Either export DOCS_INGEST_SECRET and re-run `node scripts/push-docs.js`,\n' + + ' or ping venue on VENUE-DOCS-RELEASE-PUSH for the interim sync.' + ) + process.exit(0) +} + +/** Minimal frontmatter split — returns [meta, body] or [null, raw]. */ +function parseFrontmatter(raw) { + const m = raw.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/) + if (!m) return [null, raw] + const meta = {} + for (const line of m[1].split('\n')) { + const kv = line.match(/^(\w[\w-]*):\s*(.*)$/) + if (kv) meta[kv[1]] = kv[2].trim().replace(/^["']|["']$/g, '') + } + return [meta, m[2]] +} + +const docs = [] +;(function walk(dir) { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name) + if (entry.isDirectory()) walk(full) + else if (entry.name.endsWith('.md')) { + const [meta, body] = parseFrontmatter(fs.readFileSync(full, 'utf-8')) + if (!meta || meta.public !== 'true' || !meta.slug) continue + docs.push({ + slug: meta.slug, + title: meta.title || meta.slug, + markdown: body.trim(), + nav: { + order: Number.parseInt(meta.order || '99', 10) || 99, + section: meta.category || 'guides' + } + }) + } + } +})(DOCS_DIR) + +if (docs.length === 0) { + console.error('❌ DOCS PUSH FAILED: zero public docs collected — refusing to push an empty corpus.') + process.exit(1) +} +docs.sort((a, b) => a.slug.localeCompare(b.slug)) +console.log(`Pushing ${docs.length} public docs to ${BASE}/api/docs/ingest …`) + +let failed = false +for (let i = 0; i < docs.length; i += BATCH) { + const batch = docs.slice(i, i + BATCH) + try { + const res = await fetch(`${BASE}/api/docs/ingest`, { + method: 'POST', + headers: { + 'x-service-secret': SECRET, + 'Content-Type': 'application/json', + 'User-Agent': 'brainy-docs-push/1.0' + }, + body: JSON.stringify({ docs: batch }), + signal: AbortSignal.timeout(120_000) + }) + if (!res.ok) { + throw new Error(`HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`) + } + console.log(` batch ${i / BATCH + 1}: ${batch.map((d) => d.slug).join(', ')} → ok`) + } catch (err) { + failed = true + console.error(` batch ${i / BATCH + 1} FAILED: ${err instanceof Error ? err.message : err}`) + } +} + +if (failed) { + console.error( + '❌ DOCS PUSH INCOMPLETE — soulcraft.com/docs may trail npm. ' + + 'Re-run `node scripts/push-docs.js` or ping venue on VENUE-DOCS-RELEASE-PUSH.' + ) + process.exit(1) +} +console.log('✅ Docs pushed.') diff --git a/scripts/release.sh b/scripts/release.sh index 0e6a9c43..42f5b345 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -175,10 +175,26 @@ echo -e "${BLUE}7️⃣ Creating git tag v${NEW_VERSION}...${NC}" git tag -a "v${NEW_VERSION}" -m "Release v${NEW_VERSION}" echo -e "${GREEN}✅ Tag created${NC}\n" -# Step 9: Push to GitHub -echo -e "${BLUE}8️⃣ Pushing to GitHub...${NC}" +# Step 9: Push to origin (source of truth) and the public GitHub mirror +echo -e "${BLUE}8️⃣ Pushing to origin...${NC}" git push --follow-tags origin "$CURRENT_BRANCH" -echo -e "${GREEN}✅ Pushed to GitHub${NC}\n" +echo -e "${GREEN}✅ Pushed to origin${NC}\n" + +# The public GitHub repo is a mirror of origin with an unknown sync cadence. +# `gh release create` below targets GitHub directly: if the new tag hasn't +# reached GitHub yet, gh would CREATE it — pointed at GitHub's default-branch +# head, i.e. the wrong commit. Push branch+tag to GitHub explicitly, then +# verify the tag resolves there to the same commit before any release is cut. +GITHUB_URL="https://github.com/soulcraftlabs/brainy.git" +echo -e "${BLUE}8️⃣½ Pushing to the public GitHub mirror...${NC}" +git push --follow-tags "$GITHUB_URL" "$CURRENT_BRANCH" +LOCAL_TAG_SHA="$(git rev-parse "v${NEW_VERSION}^{}")" +GITHUB_TAG_SHA="$(git ls-remote --tags "$GITHUB_URL" "v${NEW_VERSION}^{}" | cut -f1)" +if [ "$LOCAL_TAG_SHA" != "$GITHUB_TAG_SHA" ]; then + echo -e "${RED}❌ Tag v${NEW_VERSION} on GitHub (${GITHUB_TAG_SHA:-absent}) does not match local (${LOCAL_TAG_SHA}) — aborting before npm publish. Fix the mirror, then re-run.${NC}" + exit 1 +fi +echo -e "${GREEN}✅ GitHub mirror has the tag at the right commit${NC}\n" # Step 10: Publish to npm echo -e "${BLUE}9️⃣ Publishing to npm (dist-tag: ${NPM_TAG})...${NC}" @@ -196,6 +212,18 @@ else fi echo -e "${GREEN}✅ GitHub release created${NC}\n" +# Step 12: Push public docs to the soulcraft.com docs ingest door +# (VENUE-DOCS-RELEASE-PUSH). Skips with a loud warning when +# DOCS_INGEST_SECRET is unset; fails loudly (without undoing the publish — +# that already happened) when a push errors, so the docs site never +# silently trails npm. +echo -e "${BLUE}1️⃣2️⃣ Pushing public docs to soulcraft.com/docs...${NC}" +if node scripts/push-docs.js; then + echo -e "${GREEN}✅ Docs push step done${NC}\n" +else + echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n" +fi + echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}" echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" diff --git a/src/aggregation/AggregationIndex.ts b/src/aggregation/AggregationIndex.ts index ecd16228..f9382218 100644 --- a/src/aggregation/AggregationIndex.ts +++ b/src/aggregation/AggregationIndex.ts @@ -29,6 +29,7 @@ import { matchesMetadataFilter } from '../utils/metadataFilter.js' import { compareCodePoints } from '../utils/collation.js' import { bucketTimestamp } from './timeWindows.js' import { NounType } from '../types/graphTypes.js' +import { prodLog } from '../utils/logger.js' /** Persistence key for aggregate definitions */ const DEFINITIONS_KEY = '__aggregation_definitions__' @@ -87,10 +88,18 @@ function matchesSource(entity: Record, source: AggregateDefinit if (entity.service !== source.service) return false } - // Metadata where filter — match against the entity's metadata sub-object + // Where filter — resolve each filtered field through resolveEntityField, + // the SAME single source of truth groupBy uses (top-level standard fields + // + custom metadata). Matching only the metadata sub-object made + // where:{subtype}/{visibility}/… a silent no-op: reserved fields never + // live in the custom bag, so those filters could never match anything. if (source.where && Object.keys(source.where).length > 0) { - const metadata = (entity.metadata ?? entity) as Record - if (!matchesMetadataFilter(metadata, source.where)) return false + const e = entity as unknown as HNSWNounWithMetadata + const resolved: Record = {} + for (const key of Object.keys(source.where)) { + resolved[key] = resolveEntityField(e, key) + } + if (!matchesMetadataFilter(resolved, source.where)) return false } return true @@ -327,6 +336,30 @@ export class AggregationIndex { /** Track aggregates with stale MIN/MAX (need lazy recompute) */ private staleMinMax = new Map>() + /** Resolves when init() has finished loading persisted definitions/state. */ + private initPromise: Promise | null = null + + /** True once init() has settled (success or failure). */ + private initDone = false + + /** + * Aggregates registered by the app before init() finished loading persisted + * state, awaiting reconciliation: init() adopts the persisted state when the + * definition hash matches; anything left unadopted when init settles resolves + * to a backfill. Deciding backfill eagerly at define time was the boot-order + * bug that wiped valid persisted state on every restart — the synchronous + * defineAggregate() always beats the async init(). + */ + private pendingAdopt = new Set() + + /** + * In-flight rescan targets. While a name has a staging map, ALL + * contributions (the walk's and concurrent write hooks') land there instead + * of the live map; the live map keeps serving until {@link finishBackfill} + * swaps the staging map in atomically. + */ + private backfillStaging = new Map>() + constructor(storage: StorageAdapter, nativeProvider?: AggregationProvider) { this.storage = storage this.nativeProvider = nativeProvider @@ -336,21 +369,128 @@ export class AggregationIndex { /** * Initialize: load persisted definitions and state, detect changes, rebuild stale. + * + * Idempotent — repeated calls return the same promise. Definitions registered + * *before* this completes (the normal boot order: `defineAggregate()` is + * synchronous and always beats this async load) are reconciled rather than + * clobbered: the app's definition wins, and its persisted state is adopted + * when the definition hash matches — backfill happens only on a real change. */ - async init(): Promise { + init(): Promise { + if (!this.initPromise) { + this.initPromise = this.loadPersisted().finally(() => { + this.resolvePendingAdoptToBackfill() + this.initDone = true + }) + } + return this.initPromise + } + + /** + * Await the persisted-state load (if one was started) and settle every + * pending adoption decision. After this resolves, `getPendingBackfills()` + * is authoritative: a name is listed iff it genuinely needs a rescan. + * Query paths must await this before consulting backfill state. + */ + async ready(): Promise { + if (this.initPromise) { + try { + await this.initPromise + } catch { + // The owner already surfaced the load failure loudly; backfill covers. + } + } + this.resolvePendingAdoptToBackfill() + } + + /** + * Any definition still awaiting state adoption has no persisted state to + * adopt (or init never ran / failed) — it must backfill. + */ + private resolvePendingAdoptToBackfill(): void { + if (this.pendingAdopt.size > 0) { + prodLog.info( + `[Aggregation] no adoptable persisted state for: ${Array.from(this.pendingAdopt).join(', ')} — flagged for backfill` + ) + } + for (const name of this.pendingAdopt) this.needsBackfill.add(name) + this.pendingAdopt.clear() + } + + /** + * May this persisted state be ADOPTED? When the store exposes its committed + * watermark, the state's `sourceGeneration` must EQUAL it: behind means + * later writes are missing from the state (unclean shutdown); ahead means + * it counts writes that no longer exist (e.g. a fact-log truncation on a + * copied store pulled the watermark back). Either way: one exact rescan, + * said out loud — never a silent adopt. Stores without the capability (and + * pre-stamp state on them) fall back to hash-only adoption. + */ + private stateGenerationAdoptable(name: string, stateData: unknown): boolean { + const committed = this.storage.committedGeneration?.() ?? null + if (committed === null) return true + const raw = (stateData as Record).sourceGeneration + const stamped = typeof raw === 'number' ? raw : null + if (stamped === committed) return true + prodLog.warn( + `[Aggregation] '${name}': persisted state is at generation ${stamped ?? 'unstamped'} ` + + `but the store's committed generation is ${committed} — rescanning instead of adopting` + ) + return false + } + + private async loadPersisted(): Promise { // Load persisted definitions const savedDefs = await this.storage.getMetadata(DEFINITIONS_KEY) if (savedDefs && typeof savedDefs === 'object' && savedDefs.definitions) { const defs = savedDefs.definitions as Array for (const def of defs) { - this.definitions.set(def.name, def) - const currentHash = hashDefinition(def) const savedHash = def._hash || '' - // Load persisted state + if (this.definitions.has(def.name)) { + // The app re-registered this aggregate before the load finished. + // The app's definition wins — never clobber it with the persisted + // copy. Adopt the persisted state when the definition is unchanged + // AND no write has landed for it yet (a landed write would be lost + // by adoption; the hook flips such names to backfill). + const appHash = this.definitionHashes.get(def.name) || '' + if (appHash === savedHash && this.pendingAdopt.has(def.name)) { + const stateData = await this.storage.getMetadata(`${STATE_KEY_PREFIX}${def.name}__`) + if ( + stateData && + stateData.groups && + this.stateGenerationAdoptable(def.name, stateData) + ) { + const groupMap = new Map() + for (const group of stateData.groups as AggregateGroupState[]) { + groupMap.set(serializeGroupKey(group.groupKey), group) + } + this.states.set(def.name, groupMap) + this.pendingAdopt.delete(def.name) + this.needsBackfill.delete(def.name) + prodLog.info( + `[Aggregation] '${def.name}': adopted persisted state (${groupMap.size} groups) — no rescan` + ) + } + // No/invalid persisted state: stays in pendingAdopt and resolves + // to backfill when init settles. + } + continue + } + + // Not registered this session — restore definition + state from + // persistence. + this.definitions.set(def.name, def) + const currentHash = hashDefinition(def) + const stateData = await this.storage.getMetadata(`${STATE_KEY_PREFIX}${def.name}__`) - if (stateData && stateData.groups && savedHash === currentHash) { + if ( + stateData && + stateData.groups && + savedHash === currentHash && + this.stateGenerationAdoptable(def.name, stateData) + ) { // Definition unchanged — load state const groupMap = new Map() for (const group of stateData.groups as AggregateGroupState[]) { @@ -358,6 +498,10 @@ export class AggregationIndex { groupMap.set(serialized, group) } this.states.set(def.name, groupMap) + this.needsBackfill.delete(def.name) + prodLog.info( + `[Aggregation] '${def.name}': restored definition + adopted persisted state (${groupMap.size} groups)` + ) } else { // Definition changed or no saved state — start fresh and backfill from // existing entities (the owner drains needsBackfill on first query). @@ -398,14 +542,22 @@ export class AggregationIndex { })) await this.storage.saveMetadata(DEFINITIONS_KEY, { definitions: defsToSave }) - // Persist dirty states + // Persist dirty states, stamped with the committed generation they + // reflect. The stamp is what makes reopen-adoption verifiable: state at a + // different generation than the store's committed watermark is stale (an + // unclean shutdown after later writes) or over-counts (a fact-log + // truncation on a copied store pulled the watermark BACK below the + // stamp) — either way the answer is one exact rescan, never a silent + // adopt. Read the generation after collecting groups so any racing + // commit resolves toward rescan, not wrong-adopt. for (const name of this.dirty) { const stateMap = this.states.get(name) if (stateMap) { const groups = Array.from(stateMap.values()) + const sourceGeneration = this.storage.committedGeneration?.() ?? null await this.storage.saveMetadata( `${STATE_KEY_PREFIX}${name}__`, - { groups } + sourceGeneration === null ? { groups } : { groups, sourceGeneration } ) } } @@ -452,10 +604,19 @@ export class AggregationIndex { this.definitions.set(def.name, def) this.definitionHashes.set(def.name, newHash) + // First sight this session, before init() settled: defer the backfill + // decision — init() adopts the persisted state on hash match, and anything + // left unadopted resolves to backfill. Deciding eagerly here wiped valid + // persisted state on every restart. + if (!this.states.has(def.name) && !this.initDone) { + this.states.set(def.name, new Map()) + this.pendingAdopt.add(def.name) + } // Reset state if definition changed or doesn't exist yet, and flag it for // backfill so already-stored entities are counted (write-time hooks only see // future writes). The owner drains this on the next query via getPendingBackfills(). - if (!this.states.has(def.name) || (oldHash && oldHash !== newHash)) { + else if (!this.states.has(def.name) || (oldHash && oldHash !== newHash)) { + this.pendingAdopt.delete(def.name) this.states.set(def.name, new Map()) this.needsBackfill.add(def.name) } @@ -476,6 +637,8 @@ export class AggregationIndex { this.definitionHashes.delete(name) this.states.delete(name) this.staleMinMax.delete(name) + this.pendingAdopt.delete(name) + this.needsBackfill.delete(name) // Notify native provider if (this.nativeProvider?.removeAggregate) { @@ -513,9 +676,17 @@ export class AggregationIndex { return Array.from(this.needsBackfill) } - /** Clear an aggregate's state so a full rescan cannot double-count. */ + /** + * Begin a rescan into a STAGING map. The live state is not touched — it + * keeps serving (possibly stale, but flagged pending) until the rescan + * completes and swaps in atomically. A mid-walk failure drops the staging + * map via {@link abortBackfill} and loses nothing: wiping live state before + * a scan that could throw was the destructive-before-durable defect. + * Contributions (walk + concurrent write hooks) land in staging while it + * exists, so the swapped-in result reflects writes that raced the walk. + */ beginBackfill(name: string): void { - this.states.set(name, new Map()) + this.backfillStaging.set(name, new Map()) // Reset native provider state for this aggregate too, if present. const def = this.definitions.get(name) if (def && this.nativeProvider?.removeAggregate && this.nativeProvider?.defineAggregate) { @@ -524,6 +695,15 @@ export class AggregationIndex { } } + /** + * Abandon an in-flight rescan after a failure: drop the staging map, keep + * the live state serving, leave the aggregate flagged as pending so a later + * attempt rescans. The failure itself must be surfaced loudly by the owner. + */ + abortBackfill(name: string): void { + this.backfillStaging.delete(name) + } + /** Feed one already-stored entity into a single aggregate during backfill. */ backfillEntity(name: string, entity: Record): void { if (isAggregateEntity(entity)) return @@ -537,14 +717,33 @@ export class AggregationIndex { } } - /** Mark an aggregate's backfill complete; rebuilt state persists on next flush(). */ + /** Swap the rebuilt staging state in atomically; persists on next flush(). */ finishBackfill(name: string): void { + const staged = this.backfillStaging.get(name) + if (staged) { + this.states.set(name, staged) + this.backfillStaging.delete(name) + } this.needsBackfill.delete(name) this.dirty.add(name) } // ============= Write-Time Hooks ============= + /** + * A write is landing for an aggregate whose persisted-state adoption is still + * pending — adopting after this write would lose its contribution. Settle the + * decision now: an exact rescan instead of adoption. The window is the few + * milliseconds between a boot-time defineAggregate() and init() completing, + * so this rarely fires; when it does, correctness wins over the walk. + */ + private resolveAdoptOnWrite(name: string): void { + if (this.pendingAdopt.has(name)) { + this.pendingAdopt.delete(name) + this.needsBackfill.add(name) + } + } + /** * Called when an entity is added. Updates all matching aggregates. */ @@ -553,6 +752,7 @@ export class AggregationIndex { for (const [name, def] of this.definitions) { if (!matchesSource(entity, def.source)) continue + this.resolveAdoptOnWrite(name) if (this.nativeProvider) { const results = this.nativeProvider.incrementalUpdate(name, def, entity, 'add') @@ -579,6 +779,10 @@ export class AggregationIndex { const oldMatches = matchesSource(oldEntity, def.source) const newMatches = matchesSource(newEntity, def.source) + if (oldMatches || newMatches) { + this.resolveAdoptOnWrite(name) + } + if (this.nativeProvider && (oldMatches || newMatches)) { const results = this.nativeProvider.incrementalUpdate(name, def, newEntity, 'update', oldEntity) this.applyNativeResults(name, results) @@ -605,6 +809,7 @@ export class AggregationIndex { for (const [name, def] of this.definitions) { if (!matchesSource(entity, def.source)) continue + this.resolveAdoptOnWrite(name) if (this.nativeProvider) { const results = this.nativeProvider.incrementalUpdate(name, def, entity, 'delete') @@ -757,7 +962,7 @@ export class AggregationIndex { def: AggregateDefinition, entity: Record ): void { - const stateMap = this.states.get(aggName)! + const stateMap = (this.backfillStaging.get(aggName) ?? this.states.get(aggName))! // Fan out: an unnest dimension makes one entity contribute to several groups. for (const groupKey of computeGroupKeys(entity, def.groupBy)) { @@ -815,7 +1020,7 @@ export class AggregationIndex { def: AggregateDefinition, entity: Record ): void { - const stateMap = this.states.get(aggName)! + const stateMap = (this.backfillStaging.get(aggName) ?? this.states.get(aggName))! // Fan out: reverse the entity's contribution from every group it joined. for (const groupKey of computeGroupKeys(entity, def.groupBy)) { @@ -871,7 +1076,7 @@ export class AggregationIndex { * Apply results from native provider back into the state maps. */ private applyNativeResults(aggName: string, results: AggregateGroupState[]): void { - const stateMap = this.states.get(aggName)! + const stateMap = (this.backfillStaging.get(aggName) ?? this.states.get(aggName))! for (const group of results) { const serialized = serializeGroupKey(group.groupKey) stateMap.set(serialized, group) diff --git a/src/brainy.ts b/src/brainy.ts index 72cd6fa3..fd9e1ffa 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -46,8 +46,10 @@ import { pageRank, MinHeap } from './graph/analyticsFallback.js' +import { runGraphAudit, type GraphAuditReport } from './graph/graphAudit.js' import { createPipeline } from './streaming/pipeline.js' import { configureLogger, LogLevel, prodLog } from './utils/logger.js' +import { warnOnLowOsLimits } from './utils/osLimits.js' import { setGlobalCache } from './utils/unifiedCache.js' import type { UnifiedCache } from './utils/unifiedCache.js' import { rankIndicesByScore, reorderByIndices } from './utils/resultRanking.js' @@ -61,7 +63,9 @@ import type { PathOptions, MetadataIndexProvider, OpaqueIdSet, - AtGenerationVectors + AtGenerationVectors, + VectorIndexProvider, + GraphIndexProvider } from './plugin.js' import type { BrainyPlugin, @@ -70,6 +74,7 @@ import type { } from './plugin.js' import { ConnectionsCodec } from './hnsw/connectionsCodec.js' import { TransactionManager } from './transaction/TransactionManager.js' +import { transactTimeoutBudget } from './transaction/Transaction.js' import { RevisionConflictError } from './transaction/RevisionConflictError.js' import { EntityNotFoundError, RelationNotFoundError } from './errors/notFound.js' import { @@ -85,12 +90,12 @@ import { findCallerLocation } from './utils/callerLocation.js' import { SaveNounMetadataOperation, SaveNounOperation, - AddToHNSWOperation, + AddToVectorIndexOperation, AddToMetadataIndexOperation, SaveVerbMetadataOperation, SaveVerbOperation, AddToGraphIndexOperation, - RemoveFromHNSWOperation, + RemoveFromVectorIndexOperation, RemoveFromMetadataIndexOperation, RemoveFromGraphIndexOperation, UpdateNounMetadataOperation, @@ -167,6 +172,14 @@ import { type ImportResult } from './db/portableGraph.js' import { GenerationStore, type CommitBeforeImages } from './db/generationStore.js' +import type { FactScanHandle } from './db/factLog.js' +import { + ENTITY_TREE_STAMP_PATH, + readFamilyStamp, + verifyFamilyStamp, + writeFamilyStamp, + type FamilyStamp +} from './db/familyStamp.js' import { ChangeFeed, type BrainyChangeEvent, @@ -181,6 +194,7 @@ import { MemoryStorage } from './storage/adapters/memoryStorage.js' import type { CompactHistoryOptions, CompactHistoryResult, + HistoryStats, TransactOptions, TransactReceipt, TxLogEntry, @@ -190,7 +204,7 @@ import type { HistoryVersion } from './db/types.js' import { stableDeepEqual } from './db/stableEqual.js' -import type { VersionedIndexProvider } from './plugin.js' +import type { VersionedIndexProvider, ProviderInvariantReport } from './plugin.js' import type { Operation, TransactionFunction } from './transaction/types.js' /** @@ -254,6 +268,7 @@ type ResolvedBrainyConfig = Required< | 'retention' | 'eagerEmbeddings' | 'migrationWaitTimeoutMs' + | 'transactionBudgetFloorMs' > > & Pick< @@ -266,6 +281,7 @@ type ResolvedBrainyConfig = Required< | 'retention' | 'eagerEmbeddings' | 'migrationWaitTimeoutMs' + | 'transactionBudgetFloorMs' > /** @@ -367,6 +383,64 @@ class InsertPreconditionExistsSignal extends Error { } } +/** + * @description The derived-index families a read may depend on. A read that + * consults none of them (a canonical-storage read: `get`, an entity + * enumeration, a VFS content/dir read) is index-independent and must never + * block on another family's one-time migration. Used by the family-scoped + * migration gate ({@link Brainy.awaitMigrationLock}). + */ +export type IndexFamily = 'vector' | 'metadata' | 'graph' + +/** + * @description Honest per-surface outcome for {@link Brainy.warm}. Literal + * meanings — never conflate the first two: + * - `'warmed'` — the provider's own `warm?()` hook ran (vector/graph), or the + * surface's full-hydration seam loaded EVERY shard/field/segment from + * storage (metadata; graph's fallback path). The surface is genuinely at + * steady-state cost for the next operation. + * - `'probed'` — no `warm?()` hook was available, so a best-effort read + * (e.g. one `search()` call) faulted in *some* backing storage as a side + * effect. Real work happened, but it is NOT the same guarantee as + * `'warmed'` — never reported as `'warmed'`. + * - `'unavailable'` — nothing ran: no hook, no hydration seam, and (for the + * vector probe fallback) nothing to probe (an empty index or unknown + * vector dimension). The surface is unchanged by this `warm()` call. + */ +export type WarmOutcome = 'warmed' | 'probed' | 'unavailable' + +/** + * @description Result of {@link Brainy.warm}: one {@link WarmOutcome} + + * elapsed time per index surface, plus the total wall-clock time for the + * whole call. `durationMs` is measured around exactly the work described by + * that surface's `outcome` (e.g. the vector entry's `durationMs` times the + * provider `warm()` call OR the probe `search()` call — whichever ran). + */ +export interface WarmReport { + vector: { outcome: WarmOutcome; durationMs: number } + metadata: { outcome: WarmOutcome; durationMs: number } + graph: { outcome: WarmOutcome; durationMs: number } + /** Total wall-clock time for the whole `warm()` call (all three surfaces). */ + totalDurationMs: number +} + +/** + * How long a failed aggregation-backfill walk suppresses fresh walk attempts. + * Within the window, queries rethrow the recorded failure instantly (loud, + * cheap); after it, one new attempt is allowed. Bounds the damage of a + * caller-side tight retry loop against a deterministically-failing store. + */ +const AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS = 30_000 + +/** + * Time budget for the auto-compaction pass at close() (8.9.0). Bounds how long + * a clean shutdown spends reclaiming history backlog — an early stop is a + * consistent prefix and the next close/explicit pass resumes. Explicit + * `compactHistory()` calls are unbounded unless the caller passes their own + * `timeBudgetMs` (maintenance windows choose their own budgets). + */ +const CLOSE_COMPACTION_BUDGET_MS = 5_000 + /** * The main Brainy class - Clean, Beautiful, Powerful * REAL IMPLEMENTATION - No stubs, no mocks @@ -581,6 +655,11 @@ export class Brainy implements BrainyInterface { private _hub?: IntegrationHub // Integration Hub for external tools private _pendingMigrationRunner?: MigrationRunner // Deferred migration runner for large datasets private _aggregationIndex?: AggregationIndex // Incremental aggregation engine + private _aggregationBackfillFlight: Promise | null = null // Single-flight backfill walk + // A failed walk latches its error: retries within the cooldown rethrow it + // instantly instead of re-walking, so a tight caller-side retry loop costs + // one loud error per query, never a full store walk per query. + private _aggregationBackfillFailure: { at: number; error: Error } | null = null private _materializer?: AggregateMaterializer // Debounced materialization of aggregate results /** * Fields registered via `brain.trackField()` — drives optional value validation on @@ -903,6 +982,13 @@ export class Brainy implements BrainyInterface { this.storage = await this.setupStorage() await this.storage.init() + // OS-limit detection (once per process, Linux-only, measurement-only): + // warn NOW about RLIMIT_NOFILE / vm.max_map_count values that will bite + // at pool scale, instead of letting the operator meet them as EMFILE or + // a failed mmap deep inside an index open. Fire-and-forget — the check + // never affects open. + void warnOnLowOsLimits() + // Acquire the writer lock for filesystem (and other locking-capable) backends. // Skipped in reader mode and on backends that don't support multi-process locking. // Throws if another live writer holds the directory (unless force: true). @@ -964,6 +1050,44 @@ export class Brainy implements BrainyInterface { readOnly: this.config.mode === 'reader' }) + // The generation fact log is CANONICAL state, not a derived index — no + // sweeper, GC, or blob-lifecycle path may ever delete under it. Declare + // its namespace as a protected family (rebuildable: false — a lost fact + // segment is NOT reconstructable) so the storage layer REFUSES such + // deletes; refusal beats trust. Feature-detected + idempotent per name. + if ( + this.config.mode !== 'reader' && + this.generationStore.getFactLog() && + typeof this.storage.registerDerivedFamily === 'function' + ) { + await this.storage.registerDerivedFamily({ + name: 'generation-facts', + members: ['_generations/facts/'], + namespace: true, + rebuildable: false + }) + } + + // Fact-scan capability: wire the storage seam through which index + // providers (which hold only `storage`) reach the fact log. A closure + // over the LIVE log — restore/reopen swaps the instance transparently — + // so a provider's heal can switch from the enumeration walk to one + // sequential fact scan whenever the log exists. + if (typeof (this.storage as BaseStorage).setFactScanSource === 'function') { + ;(this.storage as BaseStorage).setFactScanSource({ + factLog: () => this.generationStore?.getFactLog() ?? null, + // The committed watermark, exposed as a capability so providers + // never parse the store's private manifest format. + committedGeneration: () => this.generationStore?.committedGeneration() ?? 0 + }) + } + + // Entity-tree stamp coherence: compare the stamped sourceGeneration + + // rollup invariants against the log head + live counters. Loud on + // genuine incoherence (repairIndex heals), silent on absent/coherent, + // benign-behind refreshes at the next flush. Never blocks open. + await this.verifyEntityTreeStamp() + // 8.0 ⇄ native-provider version handshake: load the on-disk brain-format // marker (`_system/brain-format.json`) into an in-memory field NOW — // after the store-open phase, but BEFORE any derived index or native @@ -1126,6 +1250,22 @@ export class Brainy implements BrainyInterface { `(storage committed: ${committed}) — provider replays the gap per ` + `the post-commit applier contract` ) + } else if (providerGen > committed) { + // The AHEAD direction is incoherence, not a replay gap: the provider's + // persisted index claims writes the store no longer has — the signature + // of a torn copy or a log truncation that pulled the committed + // watermark back (crash recovery, byte-copy of a live store). A replay + // can never converge on it and index answers may reference vanished + // writes. Name it loudly at open so it is never diagnosed from a + // silent journal; the provider's own coherence check / heal walk (or + // brain.repairIndex()) is the cure. + prodLog.warn( + `[Brainy] Versioned index provider is AHEAD of the store: provider ` + + `generation ${providerGen} vs committed ${committed}. This store was ` + + `likely copied from a live service or truncated during crash recovery. ` + + `Derived-index answers may reference rolled-back writes until the ` + + `provider heals from canonical (brain.repairIndex() forces it).` + ) } } @@ -1278,6 +1418,17 @@ export class Brainy implements BrainyInterface { }) } + // Eager index warm (operator opt-in, `warmOnOpen: true`). Runs AFTER + // every step above — index construction, crash recovery, migrations, + // VFS bootstrap — never in place of any of it, so `warm()` always + // operates on a fully-initialized brain. Blocking BY DESIGN: the + // operator traded a longer startup for a first-request that runs at + // steady-state cost instead of paying demand-load latency on the + // critical path. See the `warmOnOpen` JSDoc in brainy.types.ts. + if (this.config.warmOnOpen) { + await this.warm() + } + // Resolve ready Promise - consumers awaiting brain.ready will now proceed if (this._readyResolve) { this._readyResolve() @@ -1287,6 +1438,13 @@ export class Brainy implements BrainyInterface { if (this._readyReject) { this._readyReject(error instanceof Error ? error : new Error(String(error))) } + // Machine-readable init failures pass through UNWRAPPED — the writer-lock + // conflict documents an err.code/err.lockInfo contract ("callers detect + // this case via err.code"), and wrapping in a fresh Error silently + // stripped both, leaving consumers only a message to regex against. + if (error instanceof Error && (error as Error & { code?: string }).code === 'BRAINY_WRITER_LOCKED') { + throw error + } throw new Error(`Failed to initialize Brainy: ${error}`) } } @@ -1439,7 +1597,10 @@ export class Brainy implements BrainyInterface { * re-initializing — using a closed Brainy is a consumer bug, not a lazy-init * opportunity. */ - private async ensureInitialized(opts?: { bypassMigrationLock?: boolean }): Promise { + private async ensureInitialized(opts?: { + bypassMigrationLock?: boolean + needs?: IndexFamily[] + }): Promise { if (this.closed) { throw new Error('Brainy instance is not initialized: it was closed via close(). Create a new instance.') } @@ -1449,12 +1610,17 @@ export class Brainy implements BrainyInterface { // Coordinated migration LOCK (#18): every data-plane read and write funnels // through here, so this is the single choke point that holds operations while // a native provider runs its one-time 7.x → 8.0 rebuild-from-canonical — no - // op touches a half-built index. Observability (`health`/`checkHealth`) and - // the lock-clearing path (`stampBrainFormat`, which does not route through - // here) opt out so an operator can always watch progress and cor can stamp. + // op touches a half-built index. The gate is FAMILY-SCOPED: `needs` names the + // derived-index families this operation actually consults, so a read served + // entirely from canonical storage (`needs: []`) or from a healthy family + // never blocks on an UNRELATED family's migration. `needs` omitted = the + // conservative whole-brain wait (writes, and any read not yet classified). + // Observability (`health`/`checkHealth`) and the lock-clearing path + // (`stampBrainFormat`, which does not route through here) opt out entirely so + // an operator can always watch progress and a native provider can stamp. // A brain that never migrates pays one boolean check (see awaitMigrationLock). if (!opts?.bypassMigrationLock) { - await this.awaitMigrationLock() + await this.awaitMigrationLock(opts?.needs) } } @@ -1659,7 +1825,13 @@ export class Brainy implements BrainyInterface { captureAndCheck({ nouns, verbs } as CommitBeforeImages) } await this.generationStore.runWithoutGeneration(() => - this.transactionManager.executeTransaction(run) + this.transactionManager.executeTransaction(run, { + timeout: transactTimeoutBudget( + (touched.nouns?.length ?? 0) + (touched.verbs?.length ?? 0), + undefined, + this.config.transactionBudgetFloorMs + ) + }) ) const timestamp = Date.now() // Bootstrap writes are not generation-stamped; emit without one. @@ -1671,7 +1843,14 @@ export class Brainy implements BrainyInterface { receipt = await this.generationStore.commitSingleOp({ touched, precommit: captureAndCheck, - execute: () => this.transactionManager.executeTransaction(run) + execute: () => + this.transactionManager.executeTransaction(run, { + timeout: transactTimeoutBudget( + (touched.nouns?.length ?? 0) + (touched.verbs?.length ?? 0), + undefined, + this.config.transactionBudgetFloorMs + ) + }) }) } catch (err) { // A failed rollback that left the store inconsistent (a remove/update @@ -1764,6 +1943,29 @@ export class Brainy implements BrainyInterface { } } + /** + * @description Build the AGGREGATION view of an entity from a stored flat + * metadata record — EVERY reserved field mapped to its top-level entity + * name (stored `noun` → `type`), custom metadata in `metadata`. This must + * mirror the add-path `entityForIndexing` shape exactly: the aggregation + * engine resolves groupBy/where fields via `resolveEntityField` + * (top-level standard fields + custom metadata), so a view that drops a + * reserved field makes every aggregate grouped by that field decrement a + * group that does not exist — counts then drift upward forever after + * deletes (SELF-AGGREGATE-DELETE-DRIFT). Do not hand-roll subsets of this. + * @param record - The stored flat metadata record (before-image or pre-delete read). + * @returns The full-fidelity entity view for aggregation hooks. + */ + private entityForAggFromRawRecord(record: Record): Record { + const { reserved, custom } = splitNounMetadataRecord(record) + const { noun, ...rest } = reserved + return { + type: noun, + ...rest, + metadata: custom + } + } + /** * @description Add an entity (noun) to the brain. Embeds `data` into a vector and * indexes the entity across all three intelligences — vector similarity, graph @@ -1958,7 +2160,7 @@ export class Brainy implements BrainyInterface { // Operation 3: Add to HNSW index (after entity saved) tx.addOperation( - new AddToHNSWOperation(this.index, id, vector) + new AddToVectorIndexOperation(this.index, id, vector) ) // Operation 4: Add to metadata index @@ -2188,7 +2390,9 @@ export class Brainy implements BrainyInterface { * */ async get(id: string, options?: GetOptions): Promise | null> { - await this.ensureInitialized() + // Canonical read: a get resolves an entity by id straight from storage and + // consults no derived index — it must not wait on any family's migration. + await this.ensureInitialized({ needs: [] }) this.warnIfReadsDegraded('get') // Id normalization (8.0): a caller may read by their natural key — resolve @@ -2247,7 +2451,8 @@ export class Brainy implements BrainyInterface { * ``` */ async batchGet(ids: string[], options?: GetOptions): Promise>> { - await this.ensureInitialized() + // Canonical read (see get): resolves by id from storage, no derived index. + await this.ensureInitialized({ needs: [] }) // Id normalization (8.0): resolve each id to its canonical UUID so callers // can batch-read by natural key. resolveEntityId is idempotent on real @@ -2944,10 +3149,10 @@ export class Brainy implements BrainyInterface { // Operation 3-4: Update HNSW index (remove and re-add if reindexing needed) if (needsReindexing) { tx.addOperation( - new RemoveFromHNSWOperation(this.index, params.id, existing.vector) + new RemoveFromVectorIndexOperation(this.index, params.id, existing.vector) ) tx.addOperation( - new AddToHNSWOperation(this.index, params.id, vector) + new AddToVectorIndexOperation(this.index, params.id, vector) ) } @@ -3003,15 +3208,16 @@ export class Brainy implements BrainyInterface { ] : undefined) - // Aggregation hook (outside transaction — derived data) + // Aggregation hook (outside transaction — derived data). `existing` is + // the full get() view — every reserved field top-level — and must be + // passed whole: a subset view makes the old-side decrement miss any + // reserved-field group (update would then double-count it). if (this._aggregationIndex) { - const oldEntityForAgg = { - type: existing.type, - service: existing.service, - data: existing.data, - metadata: existing.metadata - } - this._aggregationIndex.onEntityUpdated(params.id, entityForIndexing, oldEntityForAgg) + this._aggregationIndex.onEntityUpdated( + params.id, + entityForIndexing, + existing as unknown as Record + ) } } @@ -3062,7 +3268,7 @@ export class Brainy implements BrainyInterface { // Operation 1: Remove from vector index if (noun) { tx.addOperation( - new RemoveFromHNSWOperation(this.index, id, noun.vector) + new RemoveFromVectorIndexOperation(this.index, id, noun.vector) ) } @@ -3073,9 +3279,11 @@ export class Brainy implements BrainyInterface { ) } - // Operation 3: Delete noun metadata + // Operation 3: Delete noun (full removal). The pre-read metadata rides + // along so the count decrement never depends on re-reading the record + // being removed (a null re-read must not silently skip it). tx.addOperation( - new DeleteNounMetadataOperation(this.storage, id) + new DeleteNounMetadataOperation(this.storage, id, metadata) ) // Operations 4+: Delete all related verbs atomically @@ -3121,19 +3329,15 @@ export class Brainy implements BrainyInterface { ] : undefined) - // Aggregation hook (outside transaction — derived data) + // Aggregation hook (outside transaction — derived data). The view must + // carry EVERY reserved field top-level (not a subset): a groupBy on + // subtype/visibility/etc. otherwise decrements a nonexistent group and + // the real count never comes down. if (this._aggregationIndex && metadata) { - // Reconstruct entity-like object from stored metadata via the - // canonical reserved/custom split (the hand-rolled destructure here - // missed subtype/_rev, leaking them into the aggregation view). - const { reserved, custom } = splitNounMetadataRecord(metadata) - const entityForAgg = { - type: reserved.noun, - service: reserved.service, - data: reserved.data, - metadata: custom - } - this._aggregationIndex.onEntityDeleted(id, entityForAgg) + this._aggregationIndex.onEntityDeleted( + id, + this.entityForAggFromRawRecord(metadata as Record) + ) } } @@ -4271,7 +4475,9 @@ export class Brainy implements BrainyInterface { async related( paramsOrId?: string | RelatedParams ): Promise[]> { - await this.ensureInitialized() + // Graph read: relationship traversal consults only the graph adjacency + // family, so it waits on a graph migration but not on vector/metadata. + await this.ensureInitialized({ needs: ['graph'] }) await this.verifyGraphAdjacencyLive() // Handle string ID shorthand: related(id) -> related({ from: id }) @@ -5713,6 +5919,10 @@ export class Brainy implements BrainyInterface { ): Promise { await this.ensureInitialized() this.ensureAggregationIndex() + // Persisted definitions load asynchronously — wait for them before deciding + // the aggregate doesn't exist (an app that relies on persisted definitions + // without re-defining at boot would otherwise race a spurious throw here). + await this._aggregationIndex!.ready() if (!this._aggregationIndex!.hasAggregate(name)) { throw new Error(`Aggregate '${name}' is not defined. Call defineAggregate() first.`) } @@ -5810,7 +6020,11 @@ export class Brainy implements BrainyInterface { * }) */ async find(query: string | FindParams): Promise[]> { - await this.ensureInitialized() + // Init only here — the migration gate is applied family-scoped below, once + // the query params reveal which index families this read actually consults + // (a string query may parse to a where / connected / vector shape). The lazy + // loader and cold-read probes below already defer to a migrating provider. + await this.ensureInitialized({ needs: [] }) // Ensure indexes are loaded (lazy loading when disableAutoRebuild: true) // This is a production-safe, concurrency-controlled lazy load @@ -5849,6 +6063,13 @@ export class Brainy implements BrainyInterface { // Zero-config validation (static import for performance) validateFindParams(params) + // Family-scoped migration gate: wait only on the index families THIS query + // consults, so a filter or graph query served from a healthy family never + // blocks on an unrelated family's one-time 7.x → 8.0 migration. Placed once + // params are final (after natural-language parse + connected-id resolution) + // and before the aggregate path, which carries no gate of its own. + await this.awaitMigrationLock(this.queryIndexFamilies(params)) + // Aggregate query path — early return when params.aggregate is set if (params.aggregate) { return this.findAggregate(params) @@ -6540,10 +6761,13 @@ export class Brainy implements BrainyInterface { /** * Add multiple entities in a single batch operation * - * Uses batch embedding (embedBatch) to pre-compute all vectors in a single - * WASM forward pass instead of N individual embed() calls, providing 5-10x - * speedup on bulk inserts. Automatically adapts batch size and parallelism - * to the storage adapter (e.g., smaller batches for cloud storage). + * Uses batch embedding (embedBatch) to pre-compute all vectors before any + * storage write, keeping model inference out of the per-item write path. + * (On the default WASM engine, batch throughput is comparable to sequential + * embed() calls — measured ~160 ms/text either way; a native embedding + * provider may batch faster.) Automatically adapts batch size and + * parallelism to the storage adapter (e.g., smaller batches for cloud + * storage). * * @param params - Batch add parameters * @param params.items - Array of AddParams (same shape as brain.add()) @@ -6741,6 +6965,30 @@ export class Brainy implements BrainyInterface { this.assertWritable('removeMany') await this.ensureInitialized() + // Loud selector validation: a call with no usable selector used to + // resolve successfully having deleted NOTHING (total: 0) — the classic + // silent no-op being a bare array passed positionally + // (removeMany([id]) instead of removeMany({ ids: [id] })). The caller + // believes the delete happened; every count derived afterwards is "wrong" + // while the engine was never even asked. Refuse instead. + if (Array.isArray(params)) { + throw new Error( + `removeMany() takes a params object, not a bare array — use removeMany({ ids: [...] })` + ) + } + if (!params || (!params.ids && !params.type && !params.where)) { + throw new Error( + `removeMany() requires a selector: { ids } and/or { type, where }. ` + + `An empty selector would silently delete nothing — refusing.` + ) + } + if (params.ids && params.ids.length === 0) { + throw new Error( + `removeMany() received ids: [] — an empty id list deletes nothing. ` + + `Pass the ids to delete, or omit ids and select by { type, where }.` + ) + } + // Determine what to delete let idsToDelete: string[] = [] @@ -6828,7 +7076,7 @@ export class Brainy implements BrainyInterface { // Add delete operations to transaction if (noun) { tx.addOperation( - new RemoveFromHNSWOperation(this.index, id, noun.vector) + new RemoveFromVectorIndexOperation(this.index, id, noun.vector) ) } @@ -6838,8 +7086,10 @@ export class Brainy implements BrainyInterface { ) } + // Pre-read metadata rides along: the count decrement must not + // depend on re-reading the record being removed (see remove()). tx.addOperation( - new DeleteNounMetadataOperation(this.storage, id) + new DeleteNounMetadataOperation(this.storage, id, metadata) ) for (const verb of allVerbs) { @@ -7397,6 +7647,59 @@ export class Brainy implements BrainyInterface { await this.removeMigrationBackupSafe() } + /** + * @description Open a sequential scan over the generation FACT LOG — the + * append-only record of every committed generation as an AFTER-IMAGE fact + * (what each touched entity/relationship became, or a body-less tombstone + * for a removal). The scan is the streaming substrate for index heals and + * incremental replays: one sequential read in commit order replaces a + * per-entity directory walk. The handle carries heal-narration telemetry + * (`headGeneration` / `segmentCount` / `approxFactCount` up front; ordered + * batches each stamped with their generation range, byte size, and segment; + * a `summary()` cross-check after iteration). A detected gap or damaged + * segment ABORTS the scan loudly — never a silent skip. + * + * Returns `null` when this store hosts no fact log: the storage adapter + * lacks the binary append primitives, or the brain predates the fact log + * (its history began before dual-write — facts exist only from the first + * write after upgrade; callers fall back to the canonical enumeration walk). + * + * @param options - `fromGeneration`/`toGeneration` bound the scan (inclusive + * both ends); `kinds` filters ops to `'noun'`/`'verb'`; `batchSize` caps + * facts per yielded batch (default 256). + * @returns The scan handle, or `null` when no fact log exists. + * @example + * const scan = brain.scanFacts({ fromGeneration: 1 }) + * if (scan) { + * for await (const batch of scan.batches()) { + * for (const fact of batch.facts) { + * // fact.ops: [{ kind, id, record | null (tombstone) }, ...] + * } + * } + * } + */ + scanFacts(options?: { + fromGeneration?: number + toGeneration?: number + kinds?: Array<'noun' | 'verb'> + batchSize?: number + }): FactScanHandle | null { + const factLog = this.generationStore?.getFactLog() + return factLog ? factLog.scanFacts(options) : null + } + + /** + * @description The immutable, sealed fact-log segment files covering + * `fromGeneration` — the zero-copy handoff for consumers that map segment + * files directly instead of streaming {@link scanFacts} batches. The + * append-mutable TAIL segment is deliberately excluded (read it via + * `scanFacts`). Paths are storage-root-relative. Empty when no fact log + * exists or nothing is sealed yet. + */ + factSegmentPaths(options?: { fromGeneration?: number }): string[] { + return this.generationStore?.getFactLog()?.segmentPaths(options) ?? [] + } + /** * @description Read the reified transaction log — one entry per committed * generation, carrying the committed generation, the commit timestamp, and @@ -7627,11 +7930,23 @@ export class Brainy implements BrainyInterface { ifAtGeneration: options?.ifAtGeneration, precommit: casPrecommit, execute: async () => { - await this.transactionManager.executeTransaction(async (tx) => { - for (const operation of plan.operations) { - tx.addOperation(operation) + await this.transactionManager.executeTransaction( + async (tx) => { + for (const operation of plan.operations) { + tx.addOperation(operation) + } + }, + // Budget scales with the batch (or the caller's explicit + // timeoutMs): a flat 30s cap silently limited honest bulk work + // to ~15 ops on network disks (~2s/op measured in the field). + { + timeout: transactTimeoutBudget( + plan.operations.length, + options?.timeoutMs, + this.config.transactionBudgetFloorMs + ) } - }) + ) } })) } catch (err) { @@ -8007,6 +8322,72 @@ export class Brainy implements BrainyInterface { return this.generationStore.compact(options) } + /** + * @description Repack cold generation history into sealed segments — + * re-representation, never deletion: every record and delta stays readable + * (`asOf()` unchanged); the physical file count drops by orders of + * magnitude. Runs automatically (time-bounded) at `close()`; call this for + * explicit maintenance windows on long-lived writers. The ONLY history + * transform permitted under the archival profile (`retention: 'all'`). + * @param options - `timeBudgetMs` bounds the pass (early stop = consistent + * prefix, next pass resumes); `batchGenerations` sizes each fold. + * @returns Folded generation count and segments created. + */ + async repackHistory(options?: { + timeBudgetMs?: number + batchGenerations?: number + }): Promise<{ foldedGenerations: number; segmentsCreated: number }> { + this.assertWritable('repackHistory') + await this.ensureInitialized() + await this.generationStore.flushPendingSingleOps() + return this.generationStore.repackHistory(options) + } + + /** + * @description Read-only generational-history footprint for fleet audits: + * generation count, total on-disk bytes, generation/timestamp range, the + * compaction horizon, and the retention policy in force. Touches no data and + * changes nothing. First call pays one walk over committed deltas to seed + * the running byte total (subsequent calls — and every adaptive retention + * check — are then O(1)). + * + * A pool operator's exposure check is one call per brain: + * @example + * const stats = await brain.historyStats() + * console.log(`${stats.generations} generations, ${stats.bytes} bytes, mode=${stats.retentionMode}`) + */ + async historyStats(): Promise { + await this.ensureInitialized() + const stats = await this.generationStore.historyStats() + const policy = this.resolveRetentionPolicy() + return { + ...stats, + retentionMode: policy.mode, + effectiveBudgetBytes: + policy.mode === 'adaptive' + ? this.adaptiveHistoryBudgetBytes(policy.budgetBytes) + : null + } + } + + /** + * @description A deterministic content digest of the generation log through + * `g` (D8 — gate-to-generation provenance): identical history produces the + * identical digest on any machine; divergence produces a different one. + * Release gates and suite verdicts pin `{generation, digest}` and verify + * both at execution time instead of pinning a git commit. O(segments + + * live-tier window), never O(all generations). Throws `RangeError` out of + * range and `GenerationCompactedError` below the horizon — a gate can + * never silently pin reclaimed history. + * @example + * const gate = { generation: brain.generation(), digest: await brain.generationDigest(brain.generation()) } + */ + async generationDigest(g: number): Promise { + await this.ensureInitialized() + await this.generationStore.flushPendingSingleOps() + return this.generationStore.generationDigest(g) + } + /** * @description Drive the adaptive retention byte budget at runtime — the * settable input a machine-level coordinator (e.g. cor's `ResourceManager`, @@ -8086,19 +8467,24 @@ export class Brainy implements BrainyInterface { /** * @description Run history compaction under the resolved `retention` policy - * when `autoCompact` is on (the default). Invoked from `flush()` and - * `close()` so generational record-sets cannot accumulate unbounded across a - * long-lived writer's lifetime. + * when `autoCompact` is on (the default). Invoked from `close()` ONLY + * (8.9.0) — flush() is durability work and never pays maintenance costs; a + * production deployment measured reclaim-on-flush blocking single writes + * for 25-191s under memory pressure. Long-lived writers that never close + * accumulate history until their next explicit `compactHistory()` — the + * documented trade: predictable writes, explicit maintenance. * * - `'all'` → returns without reclaiming (index compaction for speed still * runs elsewhere; history is decoupled and kept). * - `'adaptive'` → reclaim oldest-unpinned history down to the byte budget. * - `'explicit'` → apply the supplied `maxGenerations`/`maxAge`/`maxBytes` caps. * + * Every auto pass is TIME-BOUNDED ({@link CLOSE_COMPACTION_BUDGET_MS}) so a + * large backlog can never stall a clean shutdown; the next pass resumes. * Read-only instances and an explicit `autoCompact: false` skip silently. * Pinned generations are never reclaimed ({@link GenerationStore.compact}). * Failures are logged and swallowed — compaction is housekeeping and must - * never fail a flush or a clean shutdown. + * never fail a clean shutdown. */ private async autoCompactHistory(): Promise { // Nothing to compact on a read-only instance or before init wired up the @@ -8114,12 +8500,16 @@ export class Brainy implements BrainyInterface { if (policy.mode === 'adaptive') { const budget = this.adaptiveHistoryBudgetBytes(policy.budgetBytes) if (budget === Infinity) return // no pressure signal → keep everything this pass - await this.generationStore.compact({ maxBytes: budget }) + await this.generationStore.compact({ + maxBytes: budget, + timeBudgetMs: CLOSE_COMPACTION_BUDGET_MS + }) } else { await this.generationStore.compact({ maxGenerations: policy.maxGenerations, maxAge: policy.maxAge, - maxBytes: policy.maxBytes + maxBytes: policy.maxBytes, + timeBudgetMs: CLOSE_COMPACTION_BUDGET_MS }) } } catch (error) { @@ -8144,6 +8534,14 @@ export class Brainy implements BrainyInterface { * replacement. Release them first; a warning is logged when live pins * exist. * + * NON-DESTRUCTIVE STAGING + SPARSE-AWARE: the snapshot is copied into a + * staging area BEFORE any live data is touched (all-zero blocks stay + * holes, so a sparse mmap store restores at its true allocated size — not + * its apparent size). Any copy failure, ENOSPC included, removes only the + * staging and throws; the live store is untouched. Only after the copy + * succeeds does an atomic per-entry swap move it into place; a crash + * mid-swap completes FORWARD on the next open. + * * @param path - Snapshot directory to restore from. * @param options - Must be `{ confirm: true }` — an explicit acknowledgment * that current state is destroyed. @@ -8970,7 +9368,7 @@ export class Brainy implements BrainyInterface { plan.operations.push( new SaveNounMetadataOperation(this.storage, id, storageMetadata, isNew), new SaveNounOperation(this.storage, { id, vector, connections: new Map(), level: 0 }, isNew), - new AddToHNSWOperation(this.index, id, vector), + new AddToVectorIndexOperation(this.index, id, vector), new AddToMetadataIndexOperation(this.metadataIndex, id, entityForIndexing) ) plan.touchedNouns.push(id) @@ -9138,8 +9536,8 @@ export class Brainy implements BrainyInterface { ) if (needsReindexing) { plan.operations.push( - new RemoveFromHNSWOperation(this.index, params.id, existing.vector), - new AddToHNSWOperation(this.index, params.id, vector) + new RemoveFromVectorIndexOperation(this.index, params.id, existing.vector), + new AddToVectorIndexOperation(this.index, params.id, vector) ) } plan.operations.push( @@ -9148,12 +9546,9 @@ export class Brainy implements BrainyInterface { ) plan.touchedNouns.push(params.id) - const oldEntityForAgg = { - type: existing.type, - service: existing.service, - data: existing.data, - metadata: existing.metadata - } + // The full planGetEntity view, passed whole — a subset view makes the + // old-side decrement miss reserved-field groups (double-count on update). + const oldEntityForAgg = existing as unknown as Record plan.postCommit.push(() => { if (this._aggregationIndex) { this._aggregationIndex.onEntityUpdated(params.id, entityForIndexing, oldEntityForAgg) @@ -9227,12 +9622,14 @@ export class Brainy implements BrainyInterface { } if (noun) { - plan.operations.push(new RemoveFromHNSWOperation(this.index, id, noun.vector)) + plan.operations.push(new RemoveFromVectorIndexOperation(this.index, id, noun.vector)) } if (metadata) { plan.operations.push(new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata)) } - plan.operations.push(new DeleteNounMetadataOperation(this.storage, id)) + // Pre-read metadata rides along: the count decrement must not depend on + // re-reading the record being removed (see remove()). + plan.operations.push(new DeleteNounMetadataOperation(this.storage, id, metadata)) for (const verb of cascade.values()) { plan.operations.push( // Endpoint ints resolve at EXECUTE time — a cascade verb (or its @@ -9272,14 +9669,9 @@ export class Brainy implements BrainyInterface { } if (metadata) { - // Canonical reserved/custom split — mirror of remove()'s aggregation hook. - const { reserved, custom } = splitNounMetadataRecord(metadata) - const entityForAgg = { - type: reserved.noun, - service: reserved.service, - data: reserved.data, - metadata: custom - } + // Mirror of remove()'s aggregation hook — the FULL reserved view, so + // reserved-field groupBy decrements find their group. + const entityForAgg = this.entityForAggFromRawRecord(metadata as Record) plan.postCommit.push(() => { if (this._aggregationIndex) { this._aggregationIndex.onEntityDeleted(id, entityForAgg) @@ -9914,6 +10306,30 @@ export class Brainy implements BrainyInterface { return await coordinator.import(source as Buffer | string | object, options) } + /** Brain-owned background deduplicator (lazy; see getBackgroundDeduplicator). */ + private _backgroundDedup?: import('./import/BackgroundDeduplicator.js').BackgroundDeduplicator + + /** + * The single brain-owned BackgroundDeduplicator, lazily constructed. + * + * Ownership matters here: the post-import dedup timer must outlive the + * per-call ImportCoordinator but never the brain. One instance per brain + * restores the intended cross-import debounce (per-coordinator instances + * each armed their own timer, so the "debounce" never spanned imports) and + * gives close() a handle to cancel pending work — a delete pass must never + * fire against a closed brain. + * @internal + */ + async getBackgroundDeduplicator(): Promise< + import('./import/BackgroundDeduplicator.js').BackgroundDeduplicator + > { + if (!this._backgroundDedup) { + const { BackgroundDeduplicator } = await import('./import/BackgroundDeduplicator.js') + this._backgroundDedup = new BackgroundDeduplicator(this) + } + return this._backgroundDedup + } + /** * Virtual File System API - Knowledge Operating System * @@ -10238,16 +10654,101 @@ export class Brainy implements BrainyInterface { this.generationStore.persistCounterNow() ]) - // 6. Auto-compact generational history per config.retention (default on). - // Runs after the flush so durable state is in place; respects live - // Db pins and an explicit autoCompact: false. - await this.autoCompactHistory() + // NOTE (8.9.0): flush() no longer compacts history. Flush is DURABILITY + // work — it must cost what this window's deltas cost, never what the + // history backlog costs. Auto-compaction (a MAINTENANCE concern) runs at + // close() and via explicit compactHistory(); a production deployment + // measured reclaim-on-flush stalling single writes for 25-191s under + // memory pressure, which is exactly the class this separation ends. + + // 6. Stamp the entity tree: which source generation the canonical tree + // reflects + the rollup invariants that verify it whole (the counters + // persisted in step 1). Written at flush boundaries — the tree tracks + // every commit by construction, so the stamp is a durable checkpoint, + // not a per-commit cost. Open compares stamp vs log head + rollups. + await this.stampEntityTree() const elapsed = Date.now() - startTime console.log(`All indexes flushed to disk in ${elapsed}ms`) } + /** + * @description Write the entity tree's FAMILY STAMP: `sourceGeneration` (the + * committed generation the canonical tree reflects — equal by construction, + * since the tree is written by the commit itself) plus the rollup invariants + * (entity/relationship counts) that verify the tree whole where per-file + * checks cannot scale. Verified at open by {@link verifyEntityTreeStamp}; + * healed by `repairIndex()`, whose unconditional recount rebuilds the + * rollups from a canonical walk and re-stamps. Best-effort: a stamp-write + * fault warns loudly but never fails the flush that carried real data. + */ + private async stampEntityTree(): Promise { + if (this.isReadOnly) return + try { + const [nounCount, verbCount] = await Promise.all([ + this.storage.getNounCount(), + this.storage.getVerbCount() + ]) + await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, { + family: 'entity-tree', + sourceGeneration: this.generationStore.generation(), + members: { mode: 'rollup', invariants: { nounCount, verbCount } } + }) + } catch (error) { + prodLog.warn( + `[Brainy] entity-tree stamp write failed (coherence checking degrades until the ` + + `next successful flush): ${(error as Error).message}` + ) + } + } + + /** + * @description Open-time coherence check for the entity tree's family stamp: + * compare `sourceGeneration` against the log head and the stamped rollup + * invariants against the live counters. Verdicts: + * - `coherent` / `absent` (legacy store; first flush stamps) → silent. + * - `behind` → benign for the tree (it is written BY the commit; only the + * stamp is stale — a crash landed between commit and flush). Refreshed at + * the next flush. + * - `incoherent` → LOUD: the tree or its counters diverged from what was + * stamped — `repairIndex()` recounts from canonical and re-stamps. + * Never blocks open; a fault reading the stamp is surfaced as unverifiable, + * never conflated with absence. + */ + private async verifyEntityTreeStamp(): Promise { + let stamp: FamilyStamp | null + try { + stamp = await readFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH) + } catch (error) { + prodLog.warn( + `[Brainy] entity-tree stamp is UNVERIFIABLE (read fault, not absence): ` + + `${(error as Error).message}` + ) + return + } + const [nounCount, verbCount] = await Promise.all([ + this.storage.getNounCount(), + this.storage.getVerbCount() + ]) + const verdict = verifyFamilyStamp(stamp, this.generationStore.generation(), { + nounCount, + verbCount + }) + if (verdict.state === 'incoherent') { + prodLog.warn( + `[Brainy] entity-tree stamp INCOHERENT at open: ${verdict.failures.join('; ')}. ` + + `The canonical tree or its counters diverged from the stamped state — run ` + + `brain.repairIndex() to recount from canonical and re-stamp.` + ) + } else if (verdict.state === 'behind') { + prodLog.debug( + `[Brainy] entity-tree stamp is behind the log head (${verdict.stampSource} < ` + + `${verdict.head}) — benign (stamped at last flush); refreshes at the next flush.` + ) + } + } + /** * Ask the writer process serving this data directory to flush its in-memory * indexes to disk, so a read-only inspector can observe fresh state. @@ -12530,33 +13031,96 @@ export class Brainy implements BrainyInterface { entityCount: number indexEntryCount: number recommendation: string | null + /** Cross-layer: each provider's own invariant self-report, when + * it exposes validateInvariants(). Absent providers are simply not listed. */ + providers?: ProviderInvariantReport[] }> { await this.ensureInitialized() const result = await this.metadataIndex.validateConsistency() - // Fold in a non-fatal index-rebuild failure recorded at init() so the degraded - // state is observable through the same health surface (not just the console). + + // Cross-layer integrity: validateConsistency() above only sees the + // JS metadata index — it is BLIND to a native provider whose manifest ↔ + // segments ↔ counts have diverged. Delegate to each provider's own + // validateInvariants() and aggregate, so "healthy-while-broken" is impossible. + const providers = await this.collectProviderInvariants() + const brokenProviders = providers.filter((p) => !p.healthy) + + let healthy = result.healthy + const notes: string[] = [] + if (result.recommendation) notes.push(result.recommendation) + if (this._indexRebuildFailed) { - return { - ...result, - healthy: false, - recommendation: - `Index rebuild failed at init() (degraded — queries may be incomplete): ` + - `${this._indexRebuildFailed.message}. Rebuild the indexes and re-open.` + - (result.recommendation ? ` Also: ${result.recommendation}` : '') - } + healthy = false + notes.unshift( + `Index rebuild failed at init() (degraded — queries may be incomplete): ` + + `${this._indexRebuildFailed.message}. Rebuild the indexes and re-open.` + ) } if (this._indexDegradedIds.size > 0) { - return { - ...result, - healthy: false, - recommendation: - `${this._indexDegradedIds.size} record(s) committed via adopt-forward ` + + healthy = false + notes.unshift( + `${this._indexDegradedIds.size} record(s) committed via adopt-forward ` + `failed-rollback recovery — the derived index may be incomplete for them. ` + - `Run repairIndex() to reconcile.` + - (result.recommendation ? ` Also: ${result.recommendation}` : '') + `Run repairIndex() to reconcile.` + ) + } + for (const p of brokenProviders) { + healthy = false + const failing = p.invariants.filter((i) => !i.holds) + notes.unshift( + `Provider '${p.provider}' reports ${failing.length} failing invariant(s): ` + + failing.map((i) => `${i.name} (${i.detail}; heal=${i.heal})`).join('; ') + + `. Run repairIndex() to reconcile from canonical.` + ) + } + + return { + ...result, + healthy, + recommendation: notes.length > 0 ? notes.join(' ') : null, + ...(providers.length > 0 ? { providers } : {}) + } + } + + /** + * @description Feature-detect + call each index provider's OPTIONAL + * `validateInvariants()` and collect the reports. The contract is + * that `validateInvariants()` NEVER throws and is bounded <50ms — but if a + * provider violates that, the throw is turned into an UNHEALTHY synthetic + * report (loud), never swallowed into "healthy". Providers that do not expose + * the hook are simply omitted (the JS baseline validates via + * `metadataIndex.validateConsistency()`). + * @returns One report per provider that exposes `validateInvariants()`. + */ + private async collectProviderInvariants(): Promise { + const reports: ProviderInvariantReport[] = [] + const providers: unknown[] = [this.metadataIndex, this.index, this.graphIndex] + for (const provider of providers) { + const fn = (provider as { validateInvariants?: () => Promise } | null) + ?.validateInvariants + if (typeof fn !== 'function') continue + try { + const report = await fn.call(provider) + if (report && Array.isArray(report.invariants)) reports.push(report) + } catch (err) { + reports.push({ + provider: 'unknown', + healthy: false, + serving: false, + invariants: [ + { + name: 'validate-invariants-threw', + holds: false, + detail: `validateInvariants() threw (contract violation — it must never throw): ${(err as Error).message}`, + heal: 'rebuild' + } + ], + checkedAt: Date.now(), + durationMs: 0 + }) } } - return result + return reports } /** @@ -12610,7 +13174,8 @@ export class Brainy implements BrainyInterface { limit?: number } ): Promise { - await this.ensureInitialized() + // Graph read (see related): adjacency traversal only. + await this.ensureInitialized({ needs: ['graph'] }) await this.verifyGraphAdjacencyLive() const direction = options?.direction || 'both' @@ -13772,6 +14337,118 @@ export class Brainy implements BrainyInterface { return this.embedder(textToEmbed) } + /** + * Eagerly load/fault-in the vector index, metadata index, and graph + * adjacency so the FIRST real operation after this call runs at + * steady-state cost — no page-cache-miss / demand-load latency on the + * critical path. Complements {@link warmupEmbeddings} (which warms the + * embedding engine, not the storage indexes). + * + * Sequence per surface: + * - **Vector**: calls the provider's own `warm?()` when the active + * `'vector'` provider implements it (`'warmed'`). Otherwise runs a + * best-effort probe — one `search()` with a deterministic unit vector of + * the index's known dimension, `k = min(100, size())` — which faults in + * *some* backing storage as a side effect but is reported honestly as + * `'probed'`, never `'warmed'`. An empty index or unknown dimension has + * nothing to probe (`'unavailable'`). + * - **Metadata**: full hydration — every persisted field's sparse index is + * loaded from storage (`MetadataIndexManager.hydrateAll()`), not just the + * heuristic common-fields subset `init()` warms. + * - **Graph**: calls the provider's own `warm?()` when the active graph + * provider implements it; otherwise re-runs its existing eager cold-load + * `init()` seam (idempotent — the JS adjacency index's `init()` already + * loads the full LSM manifests + SSTables, so re-invoking it here is a + * genuine full hydration, not a stat call). + * + * A single surface's `'unavailable'`/probe outcome never fails the whole + * call — `warm()` is a best-effort readiness step, not a correctness gate; + * queries still demand-load normally regardless of what `warm()` achieved. + * + * @returns A {@link WarmReport}: honest per-surface outcome + timing. + * @example + * ```typescript + * const brain = new Brainy({ storage: { path: '/data' } }) + * await brain.init() + * const report = await brain.warm() // blocking, explicit control over timing + * console.log(report.vector.outcome, report.vector.durationMs) + * + * // Or opt into the same work automatically during init(): + * const eager = new Brainy({ storage: { path: '/data' }, warmOnOpen: true }) + * await eager.init() // warm() already ran before this resolves + * ``` + */ + async warm(): Promise { + await this.ensureInitialized({ needs: ['vector', 'metadata', 'graph'] }) + + const totalStart = Date.now() + + // --- Vector --------------------------------------------------------- + const vectorStart = Date.now() + let vectorOutcome: WarmOutcome + const vectorProvider = this.index as VectorIndexProvider & { warm?: () => Promise } + if (typeof vectorProvider.warm === 'function') { + await vectorProvider.warm() + vectorOutcome = 'warmed' + } else { + const size = this.index.size() + const dimension = this.dimensions + if (size > 0 && dimension && dimension > 0) { + // A deterministic unit vector (L2 norm 1) — same probe every call, + // no reliance on stored data shape. + const probeVector = new Array(dimension).fill(1 / Math.sqrt(dimension)) + await this.index.search(probeVector, Math.min(100, size)) + vectorOutcome = 'probed' + } else { + // Nothing to probe: an empty index, or the vector dimension is not + // yet known (no entity has ever been added). + vectorOutcome = 'unavailable' + } + } + const vectorDurationMs = Date.now() - vectorStart + + // --- Metadata -------------------------------------------------------- + const metadataStart = Date.now() + let metadataOutcome: WarmOutcome + const metadataWithHydrate = this.metadataIndex as unknown as { hydrateAll?: () => Promise } + if (typeof metadataWithHydrate.hydrateAll === 'function') { + await metadataWithHydrate.hydrateAll() + metadataOutcome = 'warmed' + } else { + // No hydration seam on this metadata provider — nothing to run. + metadataOutcome = 'unavailable' + } + const metadataDurationMs = Date.now() - metadataStart + + // --- Graph ------------------------------------------------------------- + const graphStart = Date.now() + let graphOutcome: WarmOutcome + const graphProvider = this.graphIndex as GraphIndexProvider & { + warm?: () => Promise + init?: () => Promise + } + if (typeof graphProvider.warm === 'function') { + await graphProvider.warm() + graphOutcome = 'warmed' + } else if (typeof graphProvider.init === 'function') { + // Existing eager cold-load seam (readiness contract): idempotent, and + // for the JS adjacency index it's already a genuine full load of the + // LSM manifests + SSTables — a real hydration, not a stat call. + await graphProvider.init() + graphOutcome = 'warmed' + } else { + graphOutcome = 'unavailable' + } + const graphDurationMs = Date.now() - graphStart + + return { + vector: { outcome: vectorOutcome, durationMs: vectorDurationMs }, + metadata: { outcome: metadataOutcome, durationMs: metadataDurationMs }, + graph: { outcome: graphOutcome, durationMs: graphDurationMs }, + totalDurationMs: Date.now() - totalStart + } + } + /** * Explicitly warm up the embedding engine * @@ -14290,6 +14967,9 @@ export class Brainy implements BrainyInterface { // Migration LOCK wait budget — left undefined when omitted so // awaitMigrationLock() applies its 30 s default at the read site. migrationWaitTimeoutMs: config?.migrationWaitTimeoutMs ?? undefined, + // Apply-phase transaction budget floor — left undefined when omitted so + // transactTimeoutBudget() applies its 30 s default at the read site. + transactionBudgetFloorMs: config?.transactionBudgetFloorMs ?? undefined, // Pre-upgrade backup — default-on; opt out with `migrationBackup: false`. migrationBackup: config?.migrationBackup ?? true, // Vector index configuration (8.0) — algorithm-neutral surface with @@ -14305,6 +14985,9 @@ export class Brainy implements BrainyInterface { // is the active one on a non-reader writer outside unit tests). Explicit // true/false always wins. See the eagerEmbeddings JSDoc in brainy.types.ts. eagerEmbeddings: config?.eagerEmbeddings ?? undefined, + // Eager index warm at init() — default off (lazy demand-load), the + // pre-8.9 behavior. See the warmOnOpen JSDoc in brainy.types.ts. + warmOnOpen: config?.warmOnOpen ?? false, // Plugin configuration - undefined = auto-detect plugins: config?.plugins ?? undefined, // Integration Hub - undefined/false = disabled @@ -14781,6 +15464,58 @@ export class Brainy implements BrainyInterface { ) } + /** + * @description The provider instance backing a given index {@link IndexFamily}. + * The single place the family label maps to the concrete provider, so the + * scoped migration gate and any future family-aware routing agree. + */ + private providerForFamily(family: IndexFamily): unknown { + switch (family) { + case 'vector': + return this.index + case 'metadata': + return this.metadataIndex + case 'graph': + return this.graphIndex + } + } + + /** + * @description Whether the migration LOCK should hold for an operation that + * depends on the given index families: + * - `needs === undefined` → WHOLE-BRAIN: any migrating provider counts (the + * conservative default for writes and unclassified reads — a write touches + * every index). + * - `needs === []` → NEVER: a canonical-storage read consults no derived index, + * so a migration elsewhere is irrelevant; it serves immediately. + * - `needs = [families]` → SCOPED: only those families' providers count, so a + * read served from a healthy family is not blocked by an unrelated family's + * one-time migration. + */ + private neededFamiliesMigrating(needs?: IndexFamily[]): boolean { + if (needs === undefined) return this.anyProviderMigrating() + for (const family of needs) { + if (this.providerIsMigrating(this.providerForFamily(family))) return true + } + return false + } + + /** + * @description The index families a {@link find} query consults, derived from + * its params — the read-side input to the family-scoped migration gate. A + * vector / near / semantic query needs `vector`; a `where` / type / subtype / + * service filter or an aggregate needs `metadata`; a `connected` traversal + * needs `graph`. A query combining shapes needs the union. Mirrors find()'s + * own criteria split so the gate and the executor never disagree. + */ + private queryIndexFamilies(params: FindParams): IndexFamily[] { + const needs: IndexFamily[] = [] + if ((params.query && params.query.trim() !== '') || params.vector || params.near) needs.push('vector') + if (params.where || params.type || params.subtype || params.service || params.aggregate) needs.push('metadata') + if (params.connected) needs.push('graph') + return needs + } + /** * @description Rich migration progress relayed verbatim from whichever provider * exposes the OPTIONAL `migrationStatus(): { phase?, index?, percent?, … } | null`. @@ -14809,11 +15544,19 @@ export class Brainy implements BrainyInterface { /** * @description Block-and-queue on the coordinated migration LOCK (#18). Called * at the {@link ensureInitialized} choke point (and once inside `init()` before - * VFS bootstrap), so every data-plane read and write — and startup itself — - * waits here while a native provider runs its one-time 7.x → 8.0 - * rebuild-from-canonical. The caller gets the correct answer, never a partial - * read of a half-built index and never a lost write. A brain that is not - * migrating pays a single boolean check and returns immediately. + * VFS bootstrap), so a data-plane operation waits here while a native provider + * runs its one-time 7.x → 8.0 rebuild-from-canonical. The caller gets the + * correct answer, never a partial read of a half-built index and never a lost + * write. A brain that is not migrating pays a single boolean check and returns + * immediately. + * + * FAMILY-SCOPED: `needs` names the index families the caller depends on, so the + * wait holds ONLY for a migration of one of those families. A read served + * entirely from canonical storage (`needs: []`) or from a healthy family never + * blocks on an unrelated family's migration. `needs` omitted = the conservative + * whole-brain wait (writes touch every index; startup wants all of them ready). + * @param needs - Index families this operation consults, or `undefined` for the + * whole-brain wait. * * IMPORTANT: this timeout bounds THE CALLER'S WAIT, not the migration. The * migration itself is unbounded — the native provider rebuilds a billion-scale @@ -14827,8 +15570,8 @@ export class Brainy implements BrainyInterface { * for a very large brain, runs the offline migrator. The block/release lines * log once per window; the flags reset on release so a later upgrade re-logs. */ - private async awaitMigrationLock(): Promise { - if (!this.anyProviderMigrating()) return // fast path — the overwhelming common case + private async awaitMigrationLock(needs?: IndexFamily[]): Promise { + if (!this.neededFamiliesMigrating(needs)) return // fast path — the overwhelming common case const startedAt = Date.now() const timeoutMs = this.config.migrationWaitTimeoutMs ?? 30_000 @@ -14841,7 +15584,7 @@ export class Brainy implements BrainyInterface { ) } - while (this.anyProviderMigrating()) { + while (this.neededFamiliesMigrating(needs)) { const remaining = timeoutMs - (Date.now() - startedAt) if (remaining <= 0) { const status = this.providerMigrationStatus() @@ -15029,6 +15772,13 @@ export class Brainy implements BrainyInterface { */ private async ensureMetadataConsistencyProbed(): Promise { if (this._metadataConsistencyProbed) return + // Defer while the metadata provider runs its one-time in-place migration: + // probing (and self-healing via rebuild) an index the provider is mid-rebuild + // would collide with the provider that owns it. Mirrors the vector deference + // in ensureIndexesLoaded. Do NOT latch — once the migration clears, the next + // read runs the probe. (The family-scoped find() gate waits on the metadata + // family separately before any actual filter read.) + if (this.providerIsMigrating(this.metadataIndex)) return this._metadataConsistencyProbed = true const provider = this.metadataIndex as { probeConsistency?: () => Promise @@ -15096,8 +15846,143 @@ export class Brainy implements BrainyInterface { ) } + /** + * Read-only graph-truth audit — proves (or disproves) that relationship + * reads return canonical truth on THIS brain, and classifies every + * discrepancy into its failure family: + * + * - `missingFromReads` — canonical verb records the read path omits + * (PRESENT BUT INVISIBLE: adjacency/membership staleness) + * - `danglingEndpoints` — verbs whose endpoint entity is gone (SCAR class) + * - `readOnlyVerbIds` — read-path edges with no canonical record (GHOSTS) + * + * Design-hidden edges (internal/system visibility) are counted separately — + * the audit reads with every visibility tier included, so intentional + * hiding is never misclassified as index loss. Mutates nothing; safe on a + * live brain (cost: one canonical walk + one indexed read per source). + * Run it after any engine upgrade, restore, or migration; a `coherent` + * report is the verified statement that `related()`/`readdir` can be + * trusted. If it reports discrepancies, `repairIndex()` is the sanctioned + * heal — re-run the audit afterwards to prove the repair. + * + * @param options.maxExamples - Cap per example list (counts stay exact). Default 100. + * @returns The full audit report; also narrated via logs (loud on incoherence). + * @since 8.6.0 + */ + async auditGraph(options: { maxExamples?: number } = {}): Promise { + await this.ensureInitialized({ needs: ['graph'] }) + + const PAGE = 1000 + return runGraphAudit( + { + eachNounId: async (consume) => { + let cursor: string | undefined + let offset = 0 + for (;;) { + const page = await (this.storage as unknown as { + getNounIdsWithPagination(o: { + limit: number + offset?: number + cursor?: string + }): Promise<{ ids: string[]; hasMore: boolean; nextCursor?: string }> + }).getNounIdsWithPagination( + cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset } + ) + for (const id of page.ids) consume(id) + if (!page.hasMore || page.ids.length === 0) break + if (page.nextCursor) cursor = page.nextCursor + else offset += page.ids.length + } + }, + eachVerb: async (consume) => { + let cursor: string | undefined + let offset = 0 + for (;;) { + const page = await this.storage.getVerbs({ + pagination: cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset } + }) + for (const verb of page.items) { + const v = verb as unknown as Record + consume({ + id: String(v.id), + type: String(v.verb ?? 'unknown'), + sourceId: String(v.sourceId), + targetId: String(v.targetId), + visibility: typeof v.visibility === 'string' ? v.visibility : undefined + }) + } + if (!page.hasMore || page.items.length === 0) break + if (page.nextCursor) cursor = page.nextCursor + else offset += page.items.length + } + }, + readRelationsFrom: async (sourceId) => + this.related({ + from: sourceId, + includeInternal: true, + includeSystem: true, + limit: 100000 + }) + }, + options + ) + } + async repairIndex(): Promise { await this.ensureInitialized() + + // Prune orphaned canonical containers left by the pre-8.3.1 partial-delete + // defect: a delete that removed the metadata (content) leg but left the + // vector leg + the entity directory (a "ghost"), or left an empty directory + // (a "scar"). These are not live entities (getNoun needs the content leg) + // yet inflate enumerated counts and confuse locator resolution. Feature- + // detected (filesystem-only — key/prefix stores have no orphan containers); + // recompute counts afterward so the totals stop counting the ghosts. + const pruner = this.storage as { + pruneOrphanedEntities?: () => Promise<{ nouns: string[]; verbs: string[] }> + rebuildTypeCounts?: () => Promise + rebuildSubtypeCounts?: () => Promise + } + if (typeof pruner.pruneOrphanedEntities === 'function') { + const orphans = await pruner.pruneOrphanedEntities() + if (orphans.nouns.length + orphans.verbs.length > 0) { + prodLog.warn( + `[Brainy] repairIndex() pruned ${orphans.nouns.length} orphaned noun + ` + + `${orphans.verbs.length} orphaned verb container(s) left by a pre-8.3.1 ` + + `partial delete.` + ) + } + } + // SANCTIONED RECOUNT — unconditional, not gated on orphans found: the + // persisted counters can be inflated over perfectly clean shelves (deletes + // whose decrement was skipped by the removed-record re-read), and + // Math.max(totalNounCount, scanned) means an inflated scalar can never + // correct itself. rebuildTypeCounts() recomputes EVERY counter rollup + // (scalar totals + per-type maps + type-statistics arrays) from one + // canonical walk and persists them. + await pruner.rebuildTypeCounts?.() + await pruner.rebuildSubtypeCounts?.() + + // The recount changed the rollup truth — re-stamp the entity tree so the + // stamp's invariants match the healed counters (repair leaves a coherent + // stamp, not a stale one that warns on the next open). + await this.stampEntityTree() + + // VFS containment reconciliation: heal "cosmetic ghost" edges left by + // pre-fix renames (an entity Contains-linked from BOTH its old and new + // directory — readdir listed it in two places) and duplicate edges from + // concurrent writers. Canonical metadata.path is the truth; only VFS + // containment edges are touched. Loud per repair. + if (this._vfsInitialized && this._vfs) { + const containment = await this._vfs.repairContainment() + if (containment.removed + containment.restored > 0) { + prodLog.warn( + `[Brainy] repairIndex() reconciled VFS containment: removed ${containment.removed} ` + + `stale/duplicate edge(s), restored ${containment.restored} missing edge(s).` + ) + } + } + await this.metadataIndex.detectAndRepairCorruption() // Lift a failed-rollback write-quarantine: force a full rebuild so the // derived indexes are provably reconciled with canonical, then clear the @@ -15112,6 +15997,31 @@ export class Brainy implements BrainyInterface { `Writes are re-enabled.` ) } + // Cross-layer repair: repairIndex must reconcile NATIVE derived + // state from canonical, not just the JS metadata index. Consult each provider's + // own validateInvariants() and rebuild any whose failing invariant asks for it + // (heal: 'rebuild') — the native counterpart of detectAndRepairCorruption(). + for (const provider of [this.metadataIndex, this.index, this.graphIndex]) { + const p = provider as { + validateInvariants?: () => Promise + rebuild?: () => Promise + } | null + if (!p || typeof p.validateInvariants !== 'function' || typeof p.rebuild !== 'function') continue + let report: ProviderInvariantReport + try { + report = await p.validateInvariants() + } catch { + continue // a throwing validateInvariants is surfaced by validateIndexConsistency; skip repair here + } + if (report.healthy) continue + if (report.invariants.some((i) => !i.holds && i.heal === 'rebuild')) { + prodLog.warn( + `[Brainy] repairIndex(): provider '${report.provider}' has a failing invariant ` + + `requiring a rebuild — reconciling its derived state from canonical.` + ) + await p.rebuild() + } + } // detectAndRepairCorruption() above rebuilt the derived indexes from // canonical, so any adopt-forward degraded ids and a non-fatal init // rebuild failure are now reconciled — clear the queryable degraded state @@ -15372,29 +16282,107 @@ export class Brainy implements BrainyInterface { */ private async backfillAggregateIfNeeded(name: string): Promise { const index = this._aggregationIndex - if (!index || !index.getPendingBackfills().includes(name)) return + if (!index) return - index.beginBackfill(name) + // Persisted-state adoption happens inside ready(); after it resolves the + // pending-backfill set is authoritative (an unchanged definition with valid + // persisted state is NOT listed — no walk at all on a clean reopen). + await index.ready() - const PAGE = 500 - let offset = 0 - let cursor: string | undefined - for (;;) { - const page = await this.storage.getNouns({ - pagination: cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset } - }) - for (const noun of page.items) { - index.backfillEntity(name, noun as unknown as Record) + // Single-flight: concurrent queries share ONE walk instead of each wiping + // the others' partial state and starting their own (the stampede that kept + // a busy store from ever converging). The loop covers the rare case where + // the in-flight walk snapshotted its batch before `name` became pending — + // the next iteration starts a fresh walk that includes it. + while (index.getPendingBackfills().includes(name)) { + // Failure latch: a deterministically-failing walk must not be re-run at + // the caller's retry rate — that is a silent CPU loop wearing a retry + // loop's clothes. Within the cooldown, rethrow the recorded failure + // immediately; after it, one fresh attempt is allowed. + const failure = this._aggregationBackfillFailure + if (failure && Date.now() - failure.at < AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS) { + throw new Error( + `Aggregation backfill for '${name}' is in failure cooldown (retry in ` + + `${Math.ceil((AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS - (Date.now() - failure.at)) / 1000)}s). ` + + `Last failure: ${failure.error.message}` + ) } - if (!page.hasMore || page.items.length === 0) break - if (page.nextCursor) { - cursor = page.nextCursor - } else { - offset += page.items.length + if (!this._aggregationBackfillFlight) { + this._aggregationBackfillFlight = this.runAggregationBackfillWalk() + .finally(() => { + this._aggregationBackfillFlight = null + }) } + await this._aggregationBackfillFlight + } + } + + /** + * One store walk fills EVERY aggregate currently pending backfill — M pending + * aggregates cost one enumeration, not M. Only reached when an aggregate + * genuinely needs a rescan (new definition over a populated store, changed + * definition, or failed state load); a clean reopen adopts persisted state + * and never walks. + */ + private async runAggregationBackfillWalk(): Promise { + const index = this._aggregationIndex! + const names = index.getPendingBackfills() + if (names.length === 0) return + + prodLog.info(`[Aggregation] backfill walk starting for: ${names.join(', ')}`) + const startedAt = Date.now() + for (const n of names) index.beginBackfill(n) + + let scanned = 0 + try { + const PAGE = 500 + let offset = 0 + let cursor: string | undefined + for (;;) { + const page = await this.storage.getNouns({ + pagination: cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset } + }) + for (const noun of page.items) { + const record = noun as unknown as Record + for (const n of names) { + index.backfillEntity(n, record) + } + } + scanned += page.items.length + if (!page.hasMore || page.items.length === 0) break + if (page.nextCursor) { + if (page.nextCursor === cursor) { + // A non-advancing cursor with hasMore=true would loop this walk at + // CPU speed forever, silently. That is a storage pagination defect — + // fail the waiting queries loudly instead of spinning. + throw new Error( + `Aggregation backfill aborted: storage pagination returned a non-advancing cursor ` + + `after ${scanned} entities with hasMore=true — the storage adapter's getNouns cursor is broken.` + ) + } + cursor = page.nextCursor + } else { + offset += page.items.length + } + } + } catch (err) { + // Non-destructive failure: drop the staging maps (live state keeps + // serving), keep the aggregates flagged pending, latch the error so + // retries within the cooldown fail fast, and say all of it out loud. + for (const n of names) index.abortBackfill(n) + this._aggregationBackfillFailure = { at: Date.now(), error: err as Error } + prodLog.warn( + `[Aggregation] backfill walk FAILED after ${scanned} entities: ${(err as Error).message} — ` + + `prior aggregate state preserved; retries suppressed for ${AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS / 1000}s` + ) + throw err } - index.finishBackfill(name) + for (const n of names) index.finishBackfill(n) + this._aggregationBackfillFailure = null + prodLog.info( + `[Aggregation] backfill walk finished: ${scanned} entities → ${names.length} aggregate(s) in ${Date.now() - startedAt}ms` + ) } /** @@ -15404,6 +16392,10 @@ export class Brainy implements BrainyInterface { * This ensures deferred persistence mode data is saved */ async close(): Promise { + // Cancel any pending post-import background deduplication FIRST — it is a + // writer (merge-deletes), and no delete pass may start mid- or post-close. + this._backgroundDedup?.cancelPending() + // Change-feed teardown: no events are delivered for or after close(). this._changeFeed.close() @@ -15414,9 +16406,21 @@ export class Brainy implements BrainyInterface { await this.generationStore.flushPendingSingleOps() } - // Phase 0b: Auto-compact generational history per config.retention (default - // on) BEFORE the generation store closes below. Respects live Db pins and - // an explicit autoCompact: false; no-op on read-only instances. + // Phase 0b: REPACK cold history into sealed segments (D1+D3 — + // re-representation, never deletion; the only history transform under the + // archival profile), then auto-compact per config.retention. Repack runs + // FIRST so bounded-retention reclaim can drop whole segments. Both are + // time-bounded maintenance passes (8.9.0 law: flush() never pays these); + // both are housekeeping — failures warn, never fail a clean shutdown. + if (!this.isReadOnly && this.generationStore) { + try { + await this.generationStore.repackHistory({ timeBudgetMs: 5_000 }) + } catch (error) { + console.warn( + `History repacking failed (non-fatal): ${error instanceof Error ? error.message : String(error)}` + ) + } + } await this.autoCompactHistory() // Phase 1: Flush ALL components in parallel to persist buffered data @@ -15460,6 +16464,13 @@ export class Brainy implements BrainyInterface { })() ]) + // Stamp the entity tree at the close boundary (counters + counter now + // durable from Phase 1/0a), so a cleanly-closed store reopens COHERENT + // instead of benign-behind. Best-effort, never blocks the close. + if (this.generationStore && !this.isReadOnly) { + await this.stampEntityTree() + } + // Phase 2: Close components to release resources (timers, file handles) // Data is already safe on disk from Phase 1 await Promise.all([ diff --git a/src/coreTypes.ts b/src/coreTypes.ts index cce06e8a..e0248d17 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -754,6 +754,39 @@ export interface Change { data?: HNSWNounWithMetadata | HNSWVerbWithMetadata } +/** + * @description A declared derived-index blob FAMILY (the + * registered-blob contract). A family names the set of on-disk blobs that a + * derived index needs AS A SET (e.g. the vector base = `main.dkann` + + * `main.slotmap` + `main.slotrev`): losing ANY member corrupts the index. Once a + * family is declared: + * - its members are UNDELETABLE through the storage layer — `deleteBinaryBlob` / + * `removeRawPrefix` refuse with a `ProtectedArtifactError`, so an in-process + * GC/sweeper cannot remove a load-bearing file (intentional retirement = + * `unregisterDerivedFamily` first); + * - a member missing on open is a loud `DerivedArtifactMissingError` → rebuild. + * `COLD ≠ DEAD`: a write-once segment or a recovery-critical archive is + * load-bearing even when it has not been touched in a long time. + */ +export interface DerivedFamilyDeclaration { + /** Stable family id, e.g. `'vector-base'` / `'metadata-sstables'`. */ + name: string + /** + * The logical blob keys that make up the family. When {@link namespace} is + * set, each entry is a PREFIX protecting every key beneath it (for growing + * sets like `seg-*`); otherwise each entry is an exact member key. + */ + members: string[] + /** When true, {@link members} are prefixes (protect all keys beneath each). */ + namespace?: boolean + /** + * Whether a missing family can be rebuilt from the canonical records (default + * `true`). `false` marks an irreplaceable family (a missing member is data + * loss, not a rebuild). + */ + rebuildable?: boolean +} + export interface StorageAdapter { init(): Promise @@ -815,7 +848,17 @@ export interface StorageAdapter { */ getNounsByNounType(nounType: string): Promise - deleteNoun(id: string): Promise + /** + * Delete a noun — FULL canonical removal (both legs + the entity container). + * + * @param id The entity id. + * @param priorMetadata OPTIONAL already-known metadata of the entity being + * removed (the caller's pre-delete read). The count decrement must never + * REQUIRE re-reading the record being removed: when the internal read + * returns `null` (replace race, or a ghost left by an earlier version) the + * decrement falls back to this record instead of being silently skipped. + */ + deleteNoun(id: string, priorMetadata?: NounMetadata | null): Promise /** * Save verb - Pure HNSW verb with core fields only @@ -872,7 +915,15 @@ export interface StorageAdapter { */ getVerbsByType(type: string): Promise - deleteVerb(id: string): Promise + /** + * Delete a verb — FULL canonical removal (both legs + the container). + * + * @param id The relationship id. + * @param priorMetadata OPTIONAL already-known metadata of the edge being + * removed (the caller's pre-delete read); keeps the count decrement honest + * when the internal read returns `null` (see `deleteNoun`). + */ + deleteVerb(id: string, priorMetadata?: VerbMetadata | null): Promise /** * Save metadata @@ -1049,6 +1100,104 @@ export interface StorageAdapter { */ getBinaryBlobPath(key: string): string | null + /** + * @description OPTIONAL (registered-blob contract). Declare a + * derived-index blob {@link DerivedFamilyDeclaration | family} whose members + * become UNDELETABLE through this adapter — a subsequent `deleteBinaryBlob` / + * `removeRawPrefix` that would remove a declared member throws a + * `ProtectedArtifactError`. Providers declare their families on create; the + * declaration is persisted so protection survives a reopen. Idempotent per + * `name` (re-declaring replaces). + * @param family - The family to protect. + */ + registerDerivedFamily?(family: DerivedFamilyDeclaration): Promise + + /** + * @description OPTIONAL. Remove a family's protection so its members can be + * deleted again — the explicit, auditable step for intentional retirement of a + * derived index (the ONLY way a declared member becomes deletable). + * @param name - The {@link DerivedFamilyDeclaration.name} to unregister. + */ + unregisterDerivedFamily?(name: string): Promise + + /** + * @description OPTIONAL. List the currently-declared derived-index families — + * the source of truth for what `clear()` must wipe and what a + * missing-on-open check verifies. + */ + listDerivedFamilies?(): Promise + + /** + * @description OPTIONAL binary raw-byte primitives — the substrate for + * append-only log-structured files (the generation fact log's CRC-framed + * segments). Feature-detected: an adapter that omits them simply hosts no + * fact log (readers fall back to canonical enumeration). Paths are + * storage-root-relative and used VERBATIM (no `.gz`/`.bin` suffixing — + * unlike the JSON object and blob primitives). + * + * Append to a raw binary file, creating it (and parent directories) when + * absent. Append durability is the CALLER's job via `syncRawObjects` — + * matching the commit protocol, which batches fsyncs at its barrier. + */ + appendRawBytes?(path: string, bytes: Uint8Array): Promise + + /** + * Read a raw binary file whole. Absent → `null`; a real IO fault throws + * (never masked as absence). + */ + readRawBytes?(path: string): Promise + + /** + * Replace a raw binary file atomically (write-new → fsync → rename) — the + * reconcile primitive (e.g. truncating a fact-log tail back to committed + * truth after a crash). + */ + writeRawBytes?(path: string, bytes: Uint8Array): Promise + + /** + * Byte size of a raw binary file, or `null` when absent. + */ + rawByteSize?(path: string): Promise + + /** + * @description OPTIONAL fact-scan capability — how an index provider that + * holds only `storage` reaches the generation fact log (the host brain + * wires it at init; providers must never construct their own fact-log + * reader — the log's open path is writer-side). Present ⟺ this store hosts + * a fact log AND the host wired the capability. Returns a scan handle over + * committed facts (heal-grade telemetry included), or `null` when no fact + * log exists — callers fall back to the canonical enumeration walk. + */ + scanFacts?(options?: { + fromGeneration?: number + toGeneration?: number + kinds?: Array<'noun' | 'verb'> + batchSize?: number + }): import('./db/factLog.js').FactScanHandle | null + + /** + * @description OPTIONAL (rides the fact-scan capability): the fact log's + * head generation — the replay target for `stamp.sourceGeneration + 1` + * catch-ups. `null` when no fact log exists. + */ + factLogHeadGeneration?(): number | null + + /** + * @description OPTIONAL (rides the fact-scan capability): the COMMITTED + * generation watermark — the manifest truth a projection's + * `sourceGeneration` compares against. Exposed as a capability so a + * provider never parses the store's private manifest format. `null` when + * the capability is unwired. + */ + committedGeneration?(): number | null + + /** + * @description OPTIONAL (rides the fact-scan capability): the immutable, + * sealed fact-segment file paths covering `fromGeneration` — the zero-copy + * handoff. The mutable tail is excluded (read it via `scanFacts`). + */ + factSegmentPaths?(options?: { fromGeneration?: number }): string[] + /** * Save statistics data * @param statistics The statistics data to save diff --git a/src/db/db.ts b/src/db/db.ts index 62c3de0e..ac927fc5 100644 --- a/src/db/db.ts +++ b/src/db/db.ts @@ -945,6 +945,13 @@ export class Db { * {@link SpeculativeOverlayError} (commit them with `brain.transact()` * first). * + * SPARSE FILES: a native accelerator's mmap index files can be sparse — + * huge apparent size, small allocated size. `persist()` handles them + * correctly (hard links share the allocation). But if you then archive the + * snapshot with EXTERNAL tools, use the sparse-aware flags (`tar czSf`, + * `rsync --sparse`, `cp --sparse=always`) or the copy materializes every + * hole — see docs/guides/external-backups-and-sparse-storage.md. + * * @param path - Absolute directory for the snapshot (created; must be * empty or absent). * @throws GenerationConflictError when this view is no longer the latest diff --git a/src/db/factLog.ts b/src/db/factLog.ts new file mode 100644 index 00000000..94e79700 --- /dev/null +++ b/src/db/factLog.ts @@ -0,0 +1,721 @@ +/** + * @module db/factLog + * @description The generation FACT LOG — an append-only, CRC-framed record of + * every committed generation as an AFTER-IMAGE "fact": what each touched + * entity/relationship BECAME (or a body-less tombstone when it was removed). + * This is the dual-write half of the log-canonical transition: today the + * before-image history + canonical tree remain authoritative; the fact log is + * appended at the same commit points and reconciled to committed truth at + * open, so consumers (index heals, replays, scans) can read one sequential, + * self-verifying stream instead of walking the entity tree. + * + * ## Wire format (frozen; additive-only within a major) + * + * Fact (msgpack, POSITIONAL array — the segment header's formatVersion + * governs the schema): + * + * fact := [ generation:u64, timestamp:u64, ops, meta|nil, blobHashes|nil ] + * op := [ kind:u8 (0=noun, 1=verb), id:bin16 (raw uuid bytes), + * record:[metaLeg, vecLeg] | nil ] // nil = TOMBSTONE + * + * Segment file (`_generations/facts/seg-.bfl`): + * + * header := magic "BFACTS\0\0" (8B) | formatVersion:u32 LE | + * firstGeneration:u64 LE | reserved 12B (ZEROED, verified) + * frame := length:u32 LE | crc32c:u32 LE (of payload) | payload + * + * A fact is never split across segments; a torn tail (length overrun or CRC + * mismatch) terminates that segment's scan — everything before it is intact. + * Zero-padded names make lexicographic order == generation order. + * + * ## Invariant + * + * After {@link FactLog.open}, the log contains EXACTLY the committed prefix: + * facts are appended BEFORE the commit point (inside the same durability + * window), so a crash can only leave the log AHEAD of committed truth — open + * truncates any fact beyond the committed generation. Absent generation = + * never committed; present = committed. A scan can never see an uncommitted + * fact. + * + * The manifest (`_generations/facts/manifest.json`, JSON — forensics stay + * terminal-readable) is the single source of truth for the segment SET; + * rotation flips it atomically (write-new → fsync → rename) BEFORE the new + * tail's first byte exists, so no segment file is ever unaccounted for. + */ +import { encode as defaultEncode, decode as defaultDecode } from '@msgpack/msgpack' +import { crc32c } from '../utils/crc32c.js' +import { prodLog } from '../utils/logger.js' + +// Swappable msgpack implementation — defaults to the JS codec; a native +// provider (registered via the plugin registry's 'msgpack' key) may replace +// it. Byte-compatibility is the contract (positional arrays, bin16 ids). +let msgpackEncode: (value: unknown) => Uint8Array = defaultEncode +let msgpackDecode: (bytes: Uint8Array) => unknown = defaultDecode + +/** Replace the msgpack encode/decode implementation at runtime. */ +export function setFactCodec(impl: { + encode: (value: unknown) => Uint8Array + decode: (bytes: Uint8Array) => unknown +}): void { + msgpackEncode = impl.encode + msgpackDecode = impl.decode +} + +/** Storage-root-relative home of the fact log. */ +export const FACTS_PREFIX = '_generations/facts' +/** The facts manifest path (JSON). */ +export const FACTS_MANIFEST_PATH = `${FACTS_PREFIX}/manifest.json` +/** Current segment format version (header field; additive-only within a major). */ +export const FACTS_FORMAT_VERSION = 1 +/** Rotation threshold: seal the tail segment once it exceeds this many bytes. */ +const SEGMENT_ROTATE_BYTES = 8 * 1024 * 1024 +/** Segment header: magic(8) + formatVersion(4) + firstGeneration(8) + reserved(12). */ +const HEADER_BYTES = 32 +const MAGIC = new Uint8Array([0x42, 0x46, 0x41, 0x43, 0x54, 0x53, 0x00, 0x00]) // "BFACTS\0\0" +/** Frame prefix: length(4) + crc32c(4). */ +const FRAME_PREFIX_BYTES = 8 + +/** One write inside a fact: what the id became (or a tombstone). */ +export interface FactOp { + kind: 'noun' | 'verb' + id: string + /** The AFTER-IMAGE legs, or `null` for a tombstone (the id was removed). */ + record: { metadata: unknown | null; vector: unknown | null } | null +} + +/** One committed generation, as scanned back out of the log. */ +export interface CommitFact { + generation: number + timestamp: number + ops: FactOp[] + meta?: Record + blobHashes?: string[] +} + +/** The telemetry a scan batch carries (frozen shape). */ +export interface FactScanBatch { + facts: CommitFact[] + firstGeneration: number + lastGeneration: number + factCount: number + byteSize: number + segmentId: string +} + +/** + * Liveness bound on a scan's FIRST batch (Stage-2 co-freeze, D1 contract): + * `batches()` must yield its first batch — or fail loudly — within this many + * ms of the first pull. A backlogged or damaged store may be SLOW, but it may + * never be SILENT: a consumer awaiting the first batch is otherwise + * indistinguishable from a wedge (the exact failure shape a production heal + * hit against a generations-backlogged brain). + */ +export const SCANFACTS_FIRST_BATCH_MS = 10_000 + +/** The telemetry a scan OPEN returns (frozen shape). */ +export interface FactScanHandle { + headGeneration: number + segmentCount: number + approxFactCount: number + /** + * Ordered batches; a detected gap aborts LOUDLY, never a silent skip. + * Liveness contract: the FIRST batch resolves or rejects within + * {@link SCANFACTS_FIRST_BATCH_MS} of the first pull — never a silent hang. + */ + batches: () => AsyncGenerator + /** Close telemetry — the invariant cross-check, valid after iteration ends. */ + summary: () => { factsYielded: number; segmentsRead: number } +} + +/** Manifest entry for a sealed segment. */ +interface SegmentEntry { + file: string + firstGeneration: number + lastGeneration: number + facts: number + bytes: number +} + +/** The facts manifest (JSON on disk). */ +interface FactsManifest { + formatVersion: number + segments: SegmentEntry[] + /** The append target. Its true content is established by scanning (crash tolerance). */ + tailSegment: string | null + updatedAt: string +} + +/** The narrow byte-level storage surface the fact log rides. */ +export interface FactLogStorage { + appendRawBytes(path: string, bytes: Uint8Array): Promise + readRawBytes(path: string): Promise + writeRawBytes(path: string, bytes: Uint8Array): Promise + rawByteSize(path: string): Promise + readRawObject(path: string): Promise + writeRawObject(path: string, data: any): Promise + syncRawObjects(paths: string[]): Promise + deleteRawObject(path: string): Promise +} + +/** True when the storage adapter exposes every primitive the fact log needs. */ +export function storageSupportsFactLog(storage: unknown): storage is FactLogStorage { + const s = storage as Record + return ( + typeof s.appendRawBytes === 'function' && + typeof s.readRawBytes === 'function' && + typeof s.writeRawBytes === 'function' && + typeof s.rawByteSize === 'function' + ) +} + +/** uuid string → 16 raw bytes (bin16 on the wire). */ +function uuidToBytes(id: string): Uint8Array { + const hex = id.replace(/-/g, '') + if (hex.length !== 32) { + // Non-uuid ids (legacy/natural keys) ride as UTF-8 with a length prefix + // marker impossible for uuids: we refuse instead — the write API has + // guaranteed uuid ids since 8.0, so anything else is a corruption signal. + throw new Error(`fact log: id is not a uuid: ${id}`) + } + const bytes = new Uint8Array(16) + for (let i = 0; i < 16; i++) { + bytes[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16) + } + return bytes +} + +/** 16 raw bytes → canonical lowercase uuid string. */ +function bytesToUuid(bytes: Uint8Array): string { + let hex = '' + for (let i = 0; i < 16; i++) hex += bytes[i].toString(16).padStart(2, '0') + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}` +} + +/** Zero-padded segment filename: lexicographic order == generation order. */ +function segmentFileName(firstGeneration: number): string { + return `seg-${String(firstGeneration).padStart(20, '0')}.bfl` +} + +/** Build a segment header. Reserved bytes are ZEROED (and verified on open). */ +function buildHeader(firstGeneration: number): Uint8Array { + const header = new Uint8Array(HEADER_BYTES) + header.set(MAGIC, 0) + const view = new DataView(header.buffer) + view.setUint32(8, FACTS_FORMAT_VERSION, true) + view.setBigUint64(12, BigInt(firstGeneration), true) + // bytes 20..31 stay zero (reserved) + return header +} + +/** Encode one fact into a framed record (length + crc32c + msgpack payload). */ +function encodeFrame(fact: CommitFact): Uint8Array { + const payload = msgpackEncode([ + fact.generation, + fact.timestamp, + fact.ops.map((op) => [ + op.kind === 'noun' ? 0 : 1, + uuidToBytes(op.id), + op.record === null ? null : [op.record.metadata, op.record.vector] + ]), + fact.meta ?? null, + fact.blobHashes && fact.blobHashes.length > 0 ? fact.blobHashes : null + ]) + const frame = new Uint8Array(FRAME_PREFIX_BYTES + payload.length) + const view = new DataView(frame.buffer) + view.setUint32(0, payload.length, true) + view.setUint32(4, crc32c(payload), true) + frame.set(payload, FRAME_PREFIX_BYTES) + return frame +} + +/** Decode one msgpack payload back into a CommitFact. */ +function decodeFact(payload: Uint8Array): CommitFact { + const raw = msgpackDecode(payload) as unknown[] + const [generation, timestamp, ops, meta, blobHashes] = raw as [ + number, + number, + Array<[number, Uint8Array, [unknown, unknown] | null]>, + Record | null, + string[] | null + ] + return { + generation: Number(generation), + timestamp: Number(timestamp), + ops: ops.map(([kind, idBytes, record]) => ({ + kind: kind === 0 ? ('noun' as const) : ('verb' as const), + id: bytesToUuid(idBytes), + record: record === null ? null : { metadata: record[0] ?? null, vector: record[1] ?? null } + })), + ...(meta ? { meta } : {}), + ...(blobHashes && blobHashes.length > 0 ? { blobHashes } : {}) + } +} + +/** + * Parse a segment's bytes: verify the header, then walk frames until the end + * or a torn tail (length overrun / CRC mismatch), which terminates the walk — + * everything before it is intact. Returns the decoded facts plus the byte + * length of the VALID prefix (header + intact frames), which reconciliation + * uses to cut a torn tail without re-encoding. + */ +function parseSegment( + file: string, + bytes: Uint8Array +): { facts: CommitFact[]; validBytes: number } { + if (bytes.length < HEADER_BYTES) { + prodLog.warn(`[FactLog] segment ${file} shorter than its header — treating as empty`) + return { facts: [], validBytes: 0 } + } + for (let i = 0; i < MAGIC.length; i++) { + if (bytes[i] !== MAGIC[i]) { + throw new Error(`fact log: segment ${file} has a bad magic — not a fact segment`) + } + } + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) + const version = view.getUint32(8, true) + if (version !== FACTS_FORMAT_VERSION) { + throw new Error( + `fact log: segment ${file} has formatVersion ${version}; this build reads ${FACTS_FORMAT_VERSION}` + ) + } + for (let i = 20; i < HEADER_BYTES; i++) { + if (bytes[i] !== 0) { + // Non-zero reserved bytes = a future format this build cannot verify. + throw new Error(`fact log: segment ${file} has non-zero reserved header bytes — unverifiable`) + } + } + + const facts: CommitFact[] = [] + let offset = HEADER_BYTES + while (offset + FRAME_PREFIX_BYTES <= bytes.length) { + const length = view.getUint32(offset, true) + const expectedCrc = view.getUint32(offset + 4, true) + const start = offset + FRAME_PREFIX_BYTES + const end = start + length + if (end > bytes.length) break // torn tail: frame length overruns the file + const payload = bytes.subarray(start, end) + if (crc32c(payload) !== expectedCrc) break // torn tail: payload CRC mismatch + facts.push(decodeFact(payload)) + offset = end + } + return { facts, validBytes: offset } +} + +/** + * The generation fact log. One instance per open store; every method assumes + * the single-writer discipline the generation store already enforces (calls + * arrive under its commit mutex). + */ +export class FactLog { + private readonly storage: FactLogStorage + /** Rotation threshold (bytes); tests may lower it to exercise rotation. */ + private readonly rotateBytes: number + private manifest: FactsManifest = { + formatVersion: FACTS_FORMAT_VERSION, + segments: [], + tailSegment: null, + updatedAt: new Date(0).toISOString() + } + /** Decoded facts of the TAIL segment (bounded by the rotation threshold). */ + private tailFacts: CommitFact[] = [] + /** Byte size of the tail segment file (valid prefix). */ + private tailBytes = 0 + /** Highest generation in the log (0 = empty). */ + private head = 0 + /** Segment paths appended since the last sync (the fsync batch). */ + private readonly dirtySegments = new Set() + + constructor(storage: FactLogStorage, options?: { rotateBytes?: number }) { + this.storage = storage + this.rotateBytes = options?.rotateBytes ?? SEGMENT_ROTATE_BYTES + } + + /** The highest committed generation the log holds (0 = empty). */ + headGeneration(): number { + return this.head + } + + /** + * Open the log and reconcile it to committed truth: read the manifest, + * establish the tail's intact content (torn-tail scan), then TRUNCATE any + * fact with `generation > committedGeneration` — those never committed (a + * crash between fact-append and the commit point). After open, the log is + * exactly the committed prefix. + */ + async open(committedGeneration: number): Promise { + const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null + if (stored && typeof stored === 'object' && Array.isArray(stored.segments)) { + if (stored.formatVersion !== FACTS_FORMAT_VERSION) { + throw new Error( + `fact log: manifest formatVersion ${stored.formatVersion}; this build reads ${FACTS_FORMAT_VERSION}` + ) + } + this.manifest = stored + } + + // Drop sealed segments that sit ENTIRELY beyond committed truth (a crash + // right after a rotation whose facts never committed), newest first. + while (this.manifest.segments.length > 0) { + const last = this.manifest.segments[this.manifest.segments.length - 1] + if (last.firstGeneration > committedGeneration) { + prodLog.warn( + `[FactLog] dropping sealed segment ${last.file} (generations ${last.firstGeneration}..` + + `${last.lastGeneration} never committed)` + ) + await this.storage.deleteRawObject(`${FACTS_PREFIX}/${last.file}`) + this.manifest.segments.pop() + await this.persistManifest() + } else if (last.lastGeneration > committedGeneration) { + // A sealed segment STRADDLING committed truth: cut it back. + await this.truncateSegmentTo(last.file, committedGeneration) + const cut = await this.reloadSegmentEntry(last.file) + this.manifest.segments[this.manifest.segments.length - 1] = cut + await this.persistManifest() + break + } else { + break + } + } + + // Establish the tail: scan its intact prefix, then truncate beyond + // committed truth (the common crash shape: buffered single-op facts whose + // counter never went durable). + if (this.manifest.tailSegment) { + const tailPath = `${FACTS_PREFIX}/${this.manifest.tailSegment}` + const bytes = await this.storage.readRawBytes(tailPath) + if (bytes === null) { + // Manifest named a tail whose first byte never landed — an empty tail. + this.tailFacts = [] + this.tailBytes = 0 + } else { + const { facts, validBytes } = parseSegment(this.manifest.tailSegment, bytes) + const kept = facts.filter((f) => f.generation <= committedGeneration) + if (kept.length !== facts.length || validBytes !== bytes.length) { + const dropped = facts.length - kept.length + if (dropped > 0) { + prodLog.warn( + `[FactLog] truncating ${dropped} uncommitted fact(s) beyond generation ` + + `${committedGeneration} from the tail (never committed)` + ) + } + await this.rewriteTail(kept) + } else { + this.tailFacts = facts + this.tailBytes = validBytes + } + } + } + + this.head = this.computeHead() + } + + /** + * Append one committed generation's fact. NOT durable until {@link sync} — + * the caller batches durability at its commit barrier (transact syncs in + * the same call; Model-B group-commit syncs at flush). + */ + async append(fact: CommitFact): Promise { + if (fact.generation <= this.head) { + throw new Error( + `fact log: non-monotonic append (generation ${fact.generation} ≤ head ${this.head})` + ) + } + if (this.manifest.tailSegment === null) { + await this.startTail(fact.generation) + } else if (this.tailBytes >= this.rotateBytes) { + await this.rotate(fact.generation) + } + const frame = encodeFrame(fact) + const tailPath = `${FACTS_PREFIX}/${this.manifest.tailSegment}` + await this.storage.appendRawBytes(tailPath, frame) + this.tailFacts.push(fact) + this.tailBytes += frame.length + this.head = fact.generation + this.dirtySegments.add(tailPath) + } + + /** Fsync every segment appended since the last sync. */ + async sync(): Promise { + if (this.dirtySegments.size === 0) return + const paths = [...this.dirtySegments] + this.dirtySegments.clear() + await this.storage.syncRawObjects(paths) + } + + /** + * Open a scan over committed facts. The scan runs against a MANIFEST + * SNAPSHOT (sealed segments + the tail's decoded facts at open) — exactly- + * once per fact, inclusive bounds, stable under concurrent appends. Gaps + * abort LOUDLY: a missing generation inside a segment's declared range is + * corruption, never silently skipped. + */ + scanFacts(options?: { + fromGeneration?: number + toGeneration?: number + kinds?: Array<'noun' | 'verb'> + batchSize?: number + /** Test override for the first-batch liveness bound (default {@link SCANFACTS_FIRST_BATCH_MS}). */ + firstBatchTimeoutMs?: number + }): FactScanHandle { + const from = options?.fromGeneration ?? 1 + const to = options?.toGeneration ?? this.head + const kinds = options?.kinds + const batchSize = Math.max(1, options?.batchSize ?? 256) + + // Snapshot: the segment list + tail content as of NOW. + const segments = this.manifest.segments.filter( + (s) => s.lastGeneration >= from && s.firstGeneration <= to + ) + const tailSnapshot = this.tailFacts.filter((f) => f.generation >= from && f.generation <= to) + const tailId = this.manifest.tailSegment ?? 'tail' + const approxFactCount = + segments.reduce((sum, s) => sum + s.facts, 0) + tailSnapshot.length + + let factsYielded = 0 + let segmentsRead = 0 + const storage = this.storage + + async function* batches(this: void): AsyncGenerator { + let expectedNext = 0 // gap detection: generations are monotonic, not necessarily dense + const emit = (facts: CommitFact[], segmentId: string, byteSize: number): FactScanBatch => ({ + facts, + firstGeneration: facts[0].generation, + lastGeneration: facts[facts.length - 1].generation, + factCount: facts.length, + byteSize, + segmentId + }) + const filterOps = (fact: CommitFact): CommitFact => + kinds + ? { ...fact, ops: fact.ops.filter((op) => kinds.includes(op.kind)) } + : fact + + for (const entry of segments) { + const bytes = await storage.readRawBytes(`${FACTS_PREFIX}/${entry.file}`) + if (bytes === null) { + throw new Error( + `fact log: sealed segment ${entry.file} is MISSING — the log is damaged; aborting scan` + ) + } + const { facts } = parseSegment(entry.file, bytes) + segmentsRead++ + const inRange = facts.filter((f) => f.generation >= from && f.generation <= to) + for (const f of inRange) { + if (f.generation <= expectedNext - 1) { + throw new Error(`fact log: out-of-order fact ${f.generation} in ${entry.file} — aborting scan`) + } + expectedNext = f.generation + 1 + } + for (let i = 0; i < inRange.length; i += batchSize) { + const slice = inRange.slice(i, i + batchSize).map(filterOps) + if (slice.length === 0) continue + factsYielded += slice.length + yield emit(slice, entry.file, slice.reduce((n, f) => n + encodeFrame(f).length, 0)) + } + } + + if (tailSnapshot.length > 0) { + segmentsRead++ + for (const f of tailSnapshot) { + if (f.generation <= expectedNext - 1) { + throw new Error(`fact log: out-of-order fact ${f.generation} in the tail — aborting scan`) + } + expectedNext = f.generation + 1 + } + for (let i = 0; i < tailSnapshot.length; i += batchSize) { + const slice = tailSnapshot.slice(i, i + batchSize).map(filterOps) + factsYielded += slice.length + yield emit(slice, tailId, slice.reduce((n, f) => n + encodeFrame(f).length, 0)) + } + } + } + + // Liveness wrapper: the FIRST pull races the contract deadline. Only the + // first — the bound is time-to-first-batch (proof the producer is alive), + // not per-batch pacing; and it runs only while a pull is actually pending, + // so consumer think-time between pulls never counts against the producer. + const firstBatchTimeoutMs = options?.firstBatchTimeoutMs ?? SCANFACTS_FIRST_BATCH_MS + async function* batchesWithLiveness(this: void): AsyncGenerator { + const inner = batches() + let timer: NodeJS.Timeout | undefined + try { + const deadline = new Promise((_, reject) => { + timer = setTimeout( + () => + reject( + new Error( + `fact log: scanFacts produced no first batch within ${firstBatchTimeoutMs}ms ` + + `(liveness contract) — the store is wedged or unreadably slow; aborting scan LOUDLY ` + + `instead of hanging the consumer.` + ) + ), + firstBatchTimeoutMs + ) + timer.unref?.() + }) + const first = await Promise.race([inner.next(), deadline]) + if (first.done) return + yield first.value + } finally { + clearTimeout(timer) + } + yield* inner + } + + return { + headGeneration: this.head, + segmentCount: segments.length + (tailSnapshot.length > 0 ? 1 : 0), + approxFactCount, + batches: batchesWithLiveness, + summary: () => ({ factsYielded, segmentsRead }) + } + } + + /** + * The mmap fast path (capability handoff): the immutable sealed segment + * files covering `fromGeneration`, in order. The TAIL is deliberately NOT + * included — it is append-mutable; consumers read it via {@link scanFacts}. + */ + segmentPaths(options?: { fromGeneration?: number }): string[] { + const from = options?.fromGeneration ?? 1 + return this.manifest.segments + .filter((s) => s.lastGeneration >= from) + .map((s) => `${FACTS_PREFIX}/${s.file}`) + } + + /** + * Drop every fact with `generation > keepThrough` — the in-session abort + * compensation: a transact appends its fact BEFORE the commit point, so a + * real (non-crash) abort after the append must take the fact back out. The + * dropped facts can only live in the TAIL (they were just appended); the + * rewrite is atomic and bounded by the rotation threshold. + */ + async dropAbove(keepThrough: number): Promise { + if (this.head <= keepThrough) return + const kept = this.tailFacts.filter((f) => f.generation <= keepThrough) + if (kept.length === this.tailFacts.length) { + throw new Error( + `fact log: dropAbove(${keepThrough}) found no droppable facts in the tail ` + + `(head ${this.head}) — the fact to drop was already sealed; the log needs reopen` + ) + } + await this.rewriteTail(kept) + this.head = this.computeHead() + } + + // -- internals ------------------------------------------------------------- + + private computeHead(): number { + if (this.tailFacts.length > 0) return this.tailFacts[this.tailFacts.length - 1].generation + const sealed = this.manifest.segments + if (sealed.length > 0) return sealed[sealed.length - 1].lastGeneration + return 0 + } + + /** Create the very first tail segment (manifest-first, then header bytes). */ + private async startTail(firstGeneration: number): Promise { + const file = segmentFileName(firstGeneration) + this.manifest.tailSegment = file + await this.persistManifest() + await this.storage.appendRawBytes(`${FACTS_PREFIX}/${file}`, buildHeader(firstGeneration)) + this.tailFacts = [] + this.tailBytes = HEADER_BYTES + } + + /** + * Seal the tail into the manifest and start a new one. Manifest-first: the + * flip both seals the old tail AND names the new one atomically, so no + * segment file ever exists unaccounted for. + */ + private async rotate(nextGeneration: number): Promise { + const sealedFile = this.manifest.tailSegment + if (!sealedFile) return + // Seal what the tail actually holds. + await this.sync() // sealed segments are always fully durable + const entry: SegmentEntry = { + file: sealedFile, + firstGeneration: this.tailFacts[0]?.generation ?? nextGeneration, + lastGeneration: this.tailFacts[this.tailFacts.length - 1]?.generation ?? nextGeneration - 1, + facts: this.tailFacts.length, + bytes: this.tailBytes + } + const newFile = segmentFileName(nextGeneration) + this.manifest.segments.push(entry) + this.manifest.tailSegment = newFile + await this.persistManifest() + await this.storage.appendRawBytes(`${FACTS_PREFIX}/${newFile}`, buildHeader(nextGeneration)) + this.tailFacts = [] + this.tailBytes = HEADER_BYTES + } + + /** Atomically persist the manifest (write-new → fsync → rename downstream). */ + private async persistManifest(): Promise { + this.manifest.updatedAt = new Date().toISOString() + await this.storage.writeRawObject(FACTS_MANIFEST_PATH, this.manifest) + await this.storage.syncRawObjects([FACTS_MANIFEST_PATH]) + } + + /** Rewrite the tail segment to hold exactly `facts` (atomic replace). */ + private async rewriteTail(facts: CommitFact[]): Promise { + const file = this.manifest.tailSegment + if (!file) return + const first = facts[0]?.generation ?? this.segmentFirstGenerationFromName(file) + const parts: Uint8Array[] = [buildHeader(first)] + for (const f of facts) parts.push(encodeFrame(f)) + const total = parts.reduce((n, p) => n + p.length, 0) + const merged = new Uint8Array(total) + let offset = 0 + for (const p of parts) { + merged.set(p, offset) + offset += p.length + } + await this.storage.writeRawBytes(`${FACTS_PREFIX}/${file}`, merged) + this.tailFacts = facts + this.tailBytes = total + } + + /** Cut a SEALED segment back to `committedGeneration` (atomic replace). */ + private async truncateSegmentTo(file: string, committedGeneration: number): Promise { + const path = `${FACTS_PREFIX}/${file}` + const bytes = await this.storage.readRawBytes(path) + if (bytes === null) return + const { facts } = parseSegment(file, bytes) + const kept = facts.filter((f) => f.generation <= committedGeneration) + prodLog.warn( + `[FactLog] truncating sealed segment ${file} to generation ${committedGeneration} ` + + `(${facts.length - kept.length} uncommitted fact(s) dropped)` + ) + const first = kept[0]?.generation ?? this.segmentFirstGenerationFromName(file) + const parts: Uint8Array[] = [buildHeader(first)] + for (const f of kept) parts.push(encodeFrame(f)) + const total = parts.reduce((n, p) => n + p.length, 0) + const merged = new Uint8Array(total) + let offset = 0 + for (const p of parts) { + merged.set(p, offset) + offset += p.length + } + await this.storage.writeRawBytes(path, merged) + } + + /** Re-derive a sealed segment's manifest entry from its actual bytes. */ + private async reloadSegmentEntry(file: string): Promise { + const bytes = await this.storage.readRawBytes(`${FACTS_PREFIX}/${file}`) + const { facts, validBytes } = bytes + ? parseSegment(file, bytes) + : { facts: [] as CommitFact[], validBytes: 0 } + return { + file, + firstGeneration: facts[0]?.generation ?? this.segmentFirstGenerationFromName(file), + lastGeneration: facts[facts.length - 1]?.generation ?? 0, + facts: facts.length, + bytes: validBytes + } + } + + /** Parse the zero-padded firstGeneration back out of a segment filename. */ + private segmentFirstGenerationFromName(file: string): number { + const match = /^seg-(\d{20})\.bfl$/.exec(file) + return match ? Number(match[1]) : 0 + } +} diff --git a/src/db/familyStamp.ts b/src/db/familyStamp.ts new file mode 100644 index 00000000..98342884 --- /dev/null +++ b/src/db/familyStamp.ts @@ -0,0 +1,152 @@ +/** + * @module db/familyStamp + * @description The generalized FAMILY STAMP — one JSON shape that declares, + * for any derived projection, WHICH source state it reflects and HOW to verify + * it is whole. The entity tree (canonical current-state files) carries the + * first brainy-side stamp; native index families carry the same shape. One + * verifier reads both member modes: + * + * - `enumerated` — bounded families: exact byte size per member file, + * verified at open. + * - `rollup` — unbounded families (the entity tree: millions of files): + * the verified surface is a small set of rollup invariants (entity/ + * relationship counts) plus `sourceGeneration`. + * + * `sourceGeneration` is the generation of the source-of-truth log this + * projection reflects — open-time coherence becomes a COMPARISON (stamp vs + * log head), not a walk: + * + * - equal + invariants hold → coherent, serve. + * - behind → the projection missed the tail (crash between commit and stamp); + * for the Stage-1 tree this is benign by construction (the tree is written + * BY the commit), so the stamp refreshes; a DERIVED projection would replay + * the gap instead. + * - invariants FAIL at equal generation → genuine incoherence: loud, and the + * repair ritual (`repairIndex()`, whose recount rebuilds the rollups from a + * canonical walk) heals it. + * + * Stamps are JSON on purpose — every incident gets debugged by reading a + * stamp in a terminal. + */ + +/** Storage-root-relative directory holding family stamps. */ +export const FAMILY_STAMPS_PREFIX = '_system/family-stamps' + +/** The entity tree's stamp path. */ +export const ENTITY_TREE_STAMP_PATH = `${FAMILY_STAMPS_PREFIX}/entity-tree.json` + +/** One enumerated member: a file and its exact expected byte size. */ +export interface EnumeratedMember { + path: string + bytes: number +} + +/** + * The stamp's verified surface, in one of the two member modes. Rollup + * invariant values may be numbers (counts, byte sizes) or strings (content + * fingerprints, e.g. a per-tree SHA-256) — the verifier compares by strict + * equality either way, so a type mismatch reads as incoherence, never a pass. + */ +export type StampMembers = + | { mode: 'enumerated'; files: EnumeratedMember[] } + | { mode: 'rollup'; invariants: Record } + +/** The generalized family stamp (one shape, one verifier, both engines). */ +export interface FamilyStamp { + /** Which projection this stamps (e.g. `'entity-tree'`). */ + family: string + /** Monotonic per-family stamp generation — bumps on every committed stamp. */ + generation: number + /** ISO timestamp of the stamp write. */ + committedAt: string + /** The source-of-truth generation this projection reflects. */ + sourceGeneration: number + /** The verified surface. */ + members: StampMembers +} + +/** The verdict of an open-time stamp verification. */ +export type StampVerdict = + | { state: 'coherent' } + | { state: 'absent' } // legacy store — first stamp writes at the next flush + | { state: 'behind'; stampSource: number; head: number } + | { state: 'incoherent'; failures: string[] } + | { state: 'unverifiable'; reason: string } // a FAULT reading the stamp — never conflated with absence + +/** The narrow storage surface stamps ride (JSON objects + fsync). */ +export interface StampStorage { + readRawObject(path: string): Promise + writeRawObject(path: string, data: any): Promise + syncRawObjects(paths: string[]): Promise +} + +/** Read a family's stamp; `null` when none was ever written. */ +export async function readFamilyStamp( + storage: StampStorage, + path: string +): Promise { + const stored = (await storage.readRawObject(path)) as FamilyStamp | null + if (!stored || typeof stored !== 'object' || typeof stored.family !== 'string') return null + return stored +} + +/** Write a family's stamp durably (atomic object write + fsync). */ +export async function writeFamilyStamp( + storage: StampStorage, + path: string, + stamp: Omit & { generation?: number } +): Promise { + const prior = await readFamilyStamp(storage, path) + const full: FamilyStamp = { + ...stamp, + generation: (prior?.generation ?? 0) + 1, + committedAt: new Date().toISOString() + } + await storage.writeRawObject(path, full) + await storage.syncRawObjects([path]) +} + +/** + * The ONE verifier, both member modes. `actual` supplies the observed rollup + * values (rollup mode) or file sizes (enumerated mode, keyed by path); + * `head` is the source-of-truth generation now. + */ +export function verifyFamilyStamp( + stamp: FamilyStamp | null, + head: number, + actual: Record +): StampVerdict { + if (stamp === null) return { state: 'absent' } + if (stamp.sourceGeneration > head) { + // A stamp AHEAD of the log claims state that never committed — the + // projection was stamped against truth that a crash rolled back. + return { + state: 'incoherent', + failures: [`sourceGeneration ${stamp.sourceGeneration} is ahead of the log head ${head}`] + } + } + if (stamp.sourceGeneration < head) { + return { state: 'behind', stampSource: stamp.sourceGeneration, head } + } + const failures: string[] = [] + if (stamp.members.mode === 'rollup') { + for (const [name, expected] of Object.entries(stamp.members.invariants)) { + const observed = actual[name] + if (observed === undefined) { + failures.push(`rollup invariant '${name}' has no observed value`) + } else if (observed !== expected) { + failures.push(`rollup invariant '${name}': stamped ${expected}, observed ${observed}`) + } + } + } else { + for (const member of stamp.members.files) { + const observed = actual[member.path] + if (observed === undefined) { + failures.push(`member '${member.path}' is missing`) + } else if (observed !== member.bytes) { + failures.push(`member '${member.path}': stamped ${member.bytes} bytes, observed ${observed}`) + } + } + } + return failures.length > 0 ? { state: 'incoherent', failures } : { state: 'coherent' } +} diff --git a/src/db/generationSegments.ts b/src/db/generationSegments.ts new file mode 100644 index 00000000..0c14b60c --- /dev/null +++ b/src/db/generationSegments.ts @@ -0,0 +1,459 @@ +/** + * @module db/generationSegments + * @description The generation-segment store — Stage-2 D1+D3+repacking's file + * format (co-frozen 2026-07-19; design: the d1-d3-repacking spec). + * + * Packs CONSECUTIVE cold generations' record-sets (before-images + delta) + * into append-once segment files with derived sidecar indexes, so history + * scales in SEGMENTS (tens) instead of FILES-PER-GENERATION (hundreds of + * thousands), and cold-open reads ONE manifest instead of listing the + * backlog. Layout under `_generations/segments/`: + * + * - `seg-.bgs` — magic "BGS1", then one frame per + * generation: `u32 payloadLen | u32 crc32c | msgpack payload`. Payload is + * POSITIONAL: `[generation, timestamp, delta, records[], flags]` with + * records `[kindByte, id, record]`. `flags` reserves encoding evolution + * (bit 0 = compressed payload — v1 always 0; a future writer upgrade, + * never a format break). Sealed segments are IMMUTABLE — the fact log's + * own law, generalized. + * - `seg-.idx` — DERIVED sidecar (msgpack): per-generation frame + * offsets (point reads = one ranged read, never a listing) + per-id + * generation postings (per-id chain rebuilds read only what they need). + * Corrupt/missing → rebuilt from its segment in one sequential read, + * loudly. + * - `manifest.json` — the segment catalogue + `compactedBelow` (D3's + * horizon marker). Cold-open reads THIS; the packed backlog is never + * listed. + * + * D3 semantics carried here: bounded-retention reclaim drops WHOLE segments + * at boundaries (O(1) per segment, no rewrite); under the archival profile + * (`retention: 'all'`) nothing here is ever dropped — folding is the only + * transform (re-representation, never deletion). + */ + +import { encode as msgpackEncode, decode as msgpackDecode } from '@msgpack/msgpack' +import { crc32c } from '../utils/crc32c.js' +import type { FactLogStorage } from './factLog.js' +import { prodLog } from '../utils/logger.js' + +/** Directory for segment files + manifest, under the generations prefix. */ +export const SEGMENTS_PREFIX = '_generations/segments' + +/** Target sealed-segment size (co-freeze proposal; tunable on evidence). */ +export const SEGMENT_TARGET_BYTES = 64 * 1024 * 1024 + +const MAGIC = new TextEncoder().encode('BGS1') +const FRAME_PREFIX_BYTES = 8 // u32 payloadLen + u32 crc32c +const MANIFEST_PATH = `${SEGMENTS_PREFIX}/manifest.json` + +/** One generation's fold input — exactly what the live tier holds for it. */ +export interface FoldGeneration { + generation: number + timestamp: number + /** The tx.json delta object, carried verbatim. */ + delta: unknown + /** The before-image record-set (empty for record-less generations). */ + records: Array<{ kind: 'noun' | 'verb'; id: string; record: unknown }> +} + +/** Manifest entry for one sealed segment. */ +export interface SegmentMeta { + file: string + firstGeneration: number + lastGeneration: number + frames: number + bytes: number + /** crc32c of the full segment byte stream — the digest chain's link. */ + checksum: number +} + +interface SegmentManifest { + version: 1 + compactedBelow: number + segments: SegmentMeta[] +} + +interface SidecarIndex { + version: 1 + /** [generation, frameOffset, frameLen] ascending by generation. */ + generations: Array<[number, number, number]> + /** `${kindByte}:${id}` → ascending generations holding a record for it. */ + ids: Record +} + +const segmentFileName = (firstGeneration: number): string => + `seg-${String(firstGeneration).padStart(20, '0')}.bgs` +const sidecarFileName = (firstGeneration: number): string => + `seg-${String(firstGeneration).padStart(20, '0')}.idx` + +/** + * The generation-segment store. Owns the packed tier ONLY — the live + * per-generation tier and the routing between tiers belong to + * `GenerationStore`. All mutating entry points here are called under the + * generation store's commit mutex. + */ +export class GenerationSegmentStore { + private readonly storage: FactLogStorage + private manifest: SegmentManifest = { version: 1, compactedBelow: 0, segments: [] } + /** Sidecar cache — segments are immutable, so entries never invalidate. */ + private readonly sidecars = new Map() + + constructor(storage: FactLogStorage) { + this.storage = storage + } + + /** Load the manifest (ONE read — never a directory listing). */ + async open(): Promise { + const raw = (await this.storage.readRawObject(MANIFEST_PATH)) as SegmentManifest | null + if (raw) { + if (raw.version !== 1) { + throw new Error( + `[GenerationSegments] manifest version ${String(raw.version)} is newer than this ` + + `engine understands — refusing to serve partial history. Upgrade the engine.` + ) + } + this.manifest = raw + } + } + + /** The packed tier's catalogue (ascending, immutable snapshot). */ + segments(): readonly SegmentMeta[] { + return this.manifest.segments + } + + /** D3's horizon marker: generations below this were reclaimed (bounded profiles only). */ + compactedBelow(): number { + return this.manifest.compactedBelow + } + + /** The covering sealed segment for `gen`, or null if it lives outside the packed tier. */ + private coveringSegment(gen: number): SegmentMeta | null { + // Manifest is ascending and ranges never overlap — binary search. + const segs = this.manifest.segments + let lo = 0 + let hi = segs.length - 1 + while (lo <= hi) { + const mid = (lo + hi) >> 1 + const s = segs[mid] + if (gen < s.firstGeneration) hi = mid - 1 + else if (gen > s.lastGeneration) lo = mid + 1 + else return s + } + return null + } + + /** True when `gen` is packed (readable from this tier). */ + hasGeneration(gen: number): boolean { + return this.coveringSegment(gen) !== null + } + + /** + * Fold consecutive generations into ONE new sealed segment + sidecar and + * append it to the manifest atomically. Caller guarantees: `gens` is + * ascending, contiguous with the packed tier (first = last packed + 1 when + * segments exist), and already durable in the live tier. Crash between the + * segment write and the caller's live-tier delete leaves a DUPLICATE + * representation — resolved live-tier-wins by the reader; never a gap. + */ + async fold(gens: FoldGeneration[]): Promise { + if (gens.length === 0) { + throw new Error('[GenerationSegments] fold() requires at least one generation') + } + for (let i = 1; i < gens.length; i++) { + if (gens[i].generation <= gens[i - 1].generation) { + throw new Error('[GenerationSegments] fold() input must be strictly ascending') + } + } + const last = this.manifest.segments[this.manifest.segments.length - 1] + if (last && gens[0].generation <= last.lastGeneration) { + throw new Error( + `[GenerationSegments] fold() overlaps the packed tier: ${gens[0].generation} ≤ ` + + `sealed ${last.lastGeneration} — segments are immutable, never rewritten` + ) + } + + const first = gens[0].generation + const file = segmentFileName(first) + const sidecar: SidecarIndex = { version: 1, generations: [], ids: {} } + + // Encode all frames, tracking offsets for the sidecar. + const parts: Uint8Array[] = [MAGIC] + let offset = MAGIC.length + for (const g of gens) { + const payload = msgpackEncode([ + g.generation, + g.timestamp, + g.delta, + g.records.map((r) => [r.kind === 'noun' ? 0 : 1, r.id, r.record]), + 0 // flags: v1 = uncompressed + ]) + const frame = new Uint8Array(FRAME_PREFIX_BYTES + payload.length) + const view = new DataView(frame.buffer) + view.setUint32(0, payload.length, true) + view.setUint32(4, crc32c(payload), true) + frame.set(payload, FRAME_PREFIX_BYTES) + sidecar.generations.push([g.generation, offset, frame.length]) + for (const r of g.records) { + const key = `${r.kind === 'noun' ? 0 : 1}:${r.id}` + ;(sidecar.ids[key] ??= []).push(g.generation) + } + parts.push(frame) + offset += frame.length + } + const total = parts.reduce((n, p) => n + p.length, 0) + const bytes = new Uint8Array(total) + let at = 0 + for (const p of parts) { + bytes.set(p, at) + at += p.length + } + + const meta: SegmentMeta = { + file, + firstGeneration: first, + lastGeneration: gens[gens.length - 1].generation, + frames: gens.length, + bytes: total, + checksum: crc32c(bytes) + } + + // Durability order: segment + sidecar fsync'd BEFORE the manifest names + // them (a crash before the manifest = invisible orphan files, harmless); + // manifest last, atomically. + const segPath = `${SEGMENTS_PREFIX}/${file}` + const idxPath = `${SEGMENTS_PREFIX}/${sidecarFileName(first)}` + await this.storage.writeRawBytes(segPath, bytes) + await this.storage.writeRawBytes(idxPath, msgpackEncode(sidecar)) + await this.storage.syncRawObjects([segPath, idxPath]) + const next: SegmentManifest = { + ...this.manifest, + segments: [...this.manifest.segments, meta] + } + await this.storage.writeRawObject(MANIFEST_PATH, next) + await this.storage.syncRawObjects([MANIFEST_PATH]) + this.manifest = next + this.sidecars.set(file, sidecar) + return meta + } + + /** Load (or rebuild, loudly) a segment's sidecar. */ + private async sidecarFor(meta: SegmentMeta): Promise { + const cached = this.sidecars.get(meta.file) + if (cached) return cached + const idxPath = `${SEGMENTS_PREFIX}/${sidecarFileName(meta.firstGeneration)}` + const raw = await this.storage.readRawBytes(idxPath) + if (raw) { + try { + const idx = msgpackDecode(raw) as SidecarIndex + if (idx.version === 1) { + this.sidecars.set(meta.file, idx) + return idx + } + } catch { + // fall through to rebuild + } + } + // Sidecars are DERIVED: rebuild from the segment, loudly — never serve + // wrong offsets silently. + prodLog.warn( + `[GenerationSegments] sidecar for ${meta.file} missing or unreadable — rebuilding from the segment` + ) + const rebuilt = await this.rebuildSidecar(meta) + await this.storage.writeRawBytes(idxPath, msgpackEncode(rebuilt)) + this.sidecars.set(meta.file, rebuilt) + return rebuilt + } + + /** One sequential read of the segment → a fresh sidecar. Verifies every frame CRC. */ + private async rebuildSidecar(meta: SegmentMeta): Promise { + const frames = await this.readAllFrames(meta) + const idx: SidecarIndex = { version: 1, generations: [], ids: {} } + for (const f of frames) { + idx.generations.push([f.generation, f.offset, f.frameLen]) + for (const r of f.records) { + const key = `${r.kind === 'noun' ? 0 : 1}:${r.id}` + ;(idx.ids[key] ??= []).push(f.generation) + } + } + return idx + } + + private decodeFrame( + payload: Uint8Array + ): { generation: number; timestamp: number; delta: unknown; records: FoldGeneration['records'] } { + const [generation, timestamp, delta, rawRecords] = msgpackDecode(payload) as [ + number, + number, + unknown, + Array<[number, string, unknown]>, + number + ] + return { + generation, + timestamp, + delta, + records: rawRecords.map(([kindByte, id, record]) => ({ + kind: kindByte === 0 ? ('noun' as const) : ('verb' as const), + id, + record + })) + } + } + + private async readAllFrames(meta: SegmentMeta): Promise< + Array & { offset: number; frameLen: number }> + > { + const bytes = await this.storage.readRawBytes(`${SEGMENTS_PREFIX}/${meta.file}`) + if (!bytes) { + throw new Error( + `[GenerationSegments] sealed segment ${meta.file} is MISSING — packed history is damaged; ` + + `refusing to continue silently` + ) + } + const out: Array & { offset: number; frameLen: number }> = [] + let at = MAGIC.length + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) + while (at + FRAME_PREFIX_BYTES <= bytes.length) { + const payloadLen = view.getUint32(at, true) + const crc = view.getUint32(at + 4, true) + const payload = bytes.subarray(at + FRAME_PREFIX_BYTES, at + FRAME_PREFIX_BYTES + payloadLen) + if (payload.length !== payloadLen || crc32c(payload) !== crc) { + throw new Error( + `[GenerationSegments] frame CRC mismatch in ${meta.file} at offset ${at} — ` + + `packed history is damaged; refusing to serve it` + ) + } + out.push({ ...this.decodeFrame(payload), offset: at, frameLen: FRAME_PREFIX_BYTES + payloadLen }) + at += FRAME_PREFIX_BYTES + payloadLen + } + return out + } + + /** Read one packed generation's frame via its sidecar offset (one ranged read). */ + private async readFrame( + gen: number + ): Promise | null> { + const meta = this.coveringSegment(gen) + if (!meta) return null + const idx = await this.sidecarFor(meta) + // generations ascending → binary search. + const gens = idx.generations + let lo = 0 + let hi = gens.length - 1 + while (lo <= hi) { + const mid = (lo + hi) >> 1 + if (gens[mid][0] < gen) lo = mid + 1 + else if (gens[mid][0] > gen) hi = mid - 1 + else { + const [, offset, frameLen] = gens[mid] + const bytes = await this.storage.readRawBytes(`${SEGMENTS_PREFIX}/${meta.file}`) + if (!bytes) { + throw new Error(`[GenerationSegments] sealed segment ${meta.file} is MISSING`) + } + const frame = bytes.subarray(offset, offset + frameLen) + const view = new DataView(frame.buffer, frame.byteOffset, frame.byteLength) + const payloadLen = view.getUint32(0, true) + const crc = view.getUint32(4, true) + const payload = frame.subarray(FRAME_PREFIX_BYTES, FRAME_PREFIX_BYTES + payloadLen) + if (payload.length !== payloadLen || crc32c(payload) !== crc) { + throw new Error( + `[GenerationSegments] frame CRC mismatch for generation ${gen} in ${meta.file} — ` + + `packed history is damaged; refusing to serve it` + ) + } + return this.decodeFrame(payload) + } + } + // In the covering range but not present: the packed tier is dense by + // construction (fold packs every generation it is handed, including + // record-less ones) — absence inside a sealed range is damage. + throw new Error( + `[GenerationSegments] generation ${gen} is inside sealed segment ${meta.file}'s declared ` + + `range but has no frame — packed history is damaged` + ) + } + + /** The packed tier's delta for `gen` (null = not packed). */ + async readDelta(gen: number): Promise<{ delta: unknown; timestamp: number } | null> { + const frame = await this.readFrame(gen) + return frame ? { delta: frame.delta, timestamp: frame.timestamp } : null + } + + /** The packed tier's full record-set for `gen` (null = not packed). */ + async readRecords(gen: number): Promise { + const frame = await this.readFrame(gen) + return frame ? frame.records : null + } + + /** One packed before-image (null = not packed OR no record for the id in that generation). */ + async readRecord(gen: number, kind: 'noun' | 'verb', id: string): Promise { + const frame = await this.readFrame(gen) + if (!frame) return null + const hit = frame.records.find((r) => r.kind === kind && r.id === id) + return hit ? hit.record : null + } + + /** + * D3 reclaim: drop WHOLE segments whose lastGeneration < `belowGeneration` + * and bump `compactedBelow`. Partial segments are never dropped — the + * boundary waits. NEVER called under the archival profile (the caller + * enforces retention semantics; this method only executes boundary drops). + */ + async dropSegmentsBelow(belowGeneration: number): Promise<{ dropped: number; compactedBelow: number }> { + const keep: SegmentMeta[] = [] + const drop: SegmentMeta[] = [] + for (const s of this.manifest.segments) { + ;(s.lastGeneration < belowGeneration ? drop : keep).push(s) + } + if (drop.length === 0) { + return { dropped: 0, compactedBelow: this.manifest.compactedBelow } + } + const compactedBelow = Math.max( + this.manifest.compactedBelow, + drop[drop.length - 1].lastGeneration + 1 + ) + // Manifest first (the drop is authoritative once named), then bytes — + // a crash between leaves orphan segment files invisible to the manifest, + // harmless and re-collectable. + const next: SegmentManifest = { ...this.manifest, compactedBelow, segments: keep } + await this.storage.writeRawObject(MANIFEST_PATH, next) + await this.storage.syncRawObjects([MANIFEST_PATH]) + this.manifest = next + for (const s of drop) { + await this.storage.deleteRawObject(`${SEGMENTS_PREFIX}/${s.file}`) + await this.storage.deleteRawObject(`${SEGMENTS_PREFIX}/${sidecarFileName(s.firstGeneration)}`) + this.sidecars.delete(s.file) + } + return { dropped: drop.length, compactedBelow } + } + + /** + * D8 rider — the packed portion of `generationDigest(g)`: a deterministic + * crc32c chain over sealed-segment checksums fully below `g`, plus the + * frame CRC of `g`'s own frame when `g` is mid-segment. O(segments), not + * O(generations); identical history ⇒ identical digest on any machine. + * The live-tier portion is composed by the caller. + */ + async digestThroughPacked(g: number): Promise { + let digest = 0 + let covered = false + for (const s of this.manifest.segments) { + if (s.lastGeneration <= g) { + digest = crc32c(new TextEncoder().encode(`${digest}:${s.checksum}`)) + if (s.lastGeneration === g) covered = true + } else if (s.firstGeneration <= g) { + // g is mid-segment: chain the partial prefix via g's frame CRC. + const frame = await this.readFrame(g) + if (frame === null) return null + const idx = await this.sidecarFor(s) + const upTo = idx.generations.filter(([gen]) => gen <= g) + for (const [gen, offset, frameLen] of upTo) { + digest = crc32c(new TextEncoder().encode(`${digest}:${gen}:${offset}:${frameLen}`)) + } + covered = true + break + } + } + return covered || this.manifest.segments.length > 0 ? digest : null + } +} diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index d7a302c9..aede17a4 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -45,6 +45,9 @@ import type { GenerationStorage, TxLogEntry } from './types.js' +import { FactLog, storageSupportsFactLog, type CommitFact, type FactOp } from './factLog.js' +import { GenerationSegmentStore, type FoldGeneration } from './generationSegments.js' +import { crc32c } from '../utils/crc32c.js' /** * The byte-identical before-images of every id a commit touches, read UNDER @@ -121,6 +124,16 @@ export interface GenerationStoreOpenResult { export class GenerationStore { private readonly storage: GenerationStorage + /** + * The generation FACT LOG (dual-write transition) — an append-only, + * CRC-framed record of every committed generation as an AFTER-IMAGE fact. + * `null` when the storage layer lacks the binary raw-byte primitives. + * Appends ride the same commit protocol: a fact-append failure FAILS the + * write (loud — a silent fact gap would make the log a lie that a later + * replay discovers), and open() reconciles the log to committed truth. + */ + private factLog: FactLog | null = null + /** Latest reserved/observed generation (≥ {@link committed}). */ private counter = 0 /** Committed-transaction watermark (manifest generation). */ @@ -246,6 +259,30 @@ export class GenerationStore { */ private deltaCacheMax = 4096 + /** + * Running total of on-disk history bytes across committed generations — + * `null` until {@link historyBytes} pays its one seeding walk. Maintained + * incrementally at commit/reclaim so the adaptive retention check on every + * flush() is O(1), never a tail re-walk. Never updated by cache re-reads + * ({@link setDelta} inserts are cache population, not new history). + */ + private historyBytesTotal: number | null = null + + /** + * The packed tier (D1+D3): sealed segments holding folded cold + * generations. Null until {@link open} wires it (and on storage adapters + * without raw-byte primitives — the live tier then carries everything, + * exactly as before the packed tier existed). + */ + private segments: GenerationSegmentStore | null = null + + /** + * Live-tier window: generations newer than `committed - REPACK_LIVE_WINDOW` + * are never folded — the hot tail stays in the per-generation layout the + * write path owns. Matches the resident chain window's scale. + */ + static readonly REPACK_LIVE_WINDOW = 1024 + /** * Model-B per-write group-commit — the in-memory PENDING tier. * @@ -400,6 +437,46 @@ export class GenerationStore { this.opened = true + // Generation FACT LOG (dual-write transition): when the storage layer + // exposes the binary raw-byte primitives, open the after-image fact log + // and reconcile it to committed truth — facts are appended BEFORE the + // commit point, so a crash can only leave the log AHEAD; open truncates + // any fact beyond `committed`. Storage without the primitives simply + // hosts no fact log (readers fall back to canonical enumeration). + if (storageSupportsFactLog(this.storage)) { + this.factLog = new FactLog(this.storage) + await this.factLog.open(this.committed) + } else { + this.factLog = null + } + + // PACKED TIER (D1+D3): same capability gate as the fact log. Opening + // reads ONE manifest — never a listing of the packed backlog — and seeds + // committedRanges with the sealed ranges so packed generations resolve + // exactly like live ones. + if (storageSupportsFactLog(this.storage)) { + this.segments = new GenerationSegmentStore(this.storage) + await this.segments.open() + const packedRanges = this.segments + .segments() + .map((s): [number, number] => [s.firstGeneration, Math.min(s.lastGeneration, this.committed)]) + .filter(([lo, hi]) => lo <= hi) + if (packedRanges.length > 0) { + // Merge packed (older) + live (newer) interval sets — both ascending; + // coalesce adjacency so range arithmetic stays interval-exact. + const merged: Array<[number, number]> = [] + for (const r of [...packedRanges, ...this.committedRanges].sort((a, b) => a[0] - b[0])) { + const last = merged[merged.length - 1] + if (last && r[0] <= last[1] + 1) last[1] = Math.max(last[1], r[1]) + else merged.push([r[0], r[1]]) + } + this.committedRanges = merged + } + this.horizonGen = Math.max(this.horizonGen, this.segments.compactedBelow() - 1) + } else { + this.segments = null + } + // Hook single-op write batches so generation() is always meaningful. // Suppressed while a transact batch executes (the batch is ONE generation). if (!options?.readOnly) { @@ -459,6 +536,85 @@ export class GenerationStore { return this.horizonGen } + /** + * @description Read-only history footprint for fleet audits: how much + * generational history this store holds on disk. `bytes` pays (and seeds) + * the one-time {@link historyBytes} walk on first call — subsequent calls + * are O(1). The oldest/newest timestamps come from those generations' + * deltas (cache-bounded reads). + * @returns Counts, bytes, generation range, and the compaction horizon. + */ + /** + * @description D8 (gate-to-generation provenance): a deterministic content + * digest of the generation log THROUGH `g` — identical history ⇒ identical + * digest on any machine; any divergence (different records, different + * order, reclaimed range) ⇒ different digest. Composed from the packed + * tier's sealed-segment checksum chain (O(segments)) plus the live tier's + * per-generation delta digests (O(live window at most)). Release gates pin + * {generation, digest} and verify both at execution time. + * @param g - The generation to digest through (≤ committed). + * @returns A hex digest string, stable across reopen and repacking states + * ONLY for fully-packed prefixes — repacking changes representation, so + * the composed digest is defined over CONTENT: live-tier gens hash their + * delta + record ids, packed gens hash via frame CRCs. A gate should pin + * after a repack pass for long-term stability, or re-pin on repack. + */ + async generationDigest(g: number): Promise { + if (!Number.isInteger(g) || g < 1 || g > this.committed) { + throw new RangeError( + `generationDigest(): generation ${g} is out of range [1, ${this.committed}]` + ) + } + if (g <= this.horizonGen) { + throw new GenerationCompactedError(g, this.horizonGen) + } + let digest = 0 + const enc = new TextEncoder() + if (this.segments) { + const packed = await this.segments.digestThroughPacked(g) + if (packed !== null) digest = packed + } + // Live-tier composition: every committed gen ≤ g not covered by a sealed + // segment hashes its delta content in ascending order. + for (const gen of this.committedGensAsc()) { + if (gen > g) break + if (this.segments?.hasGeneration(gen)) continue + const delta = await this.getDelta(gen) + digest = crc32c( + enc.encode( + `${digest}:${gen}:${delta.timestamp}:${[...delta.nouns].sort().join(',')}:${[...delta.verbs].sort().join(',')}` + ) + ) + } + return digest.toString(16).padStart(8, '0') + } + + async historyStats(): Promise<{ + generations: number + bytes: number + oldestGeneration: number | null + newestGeneration: number | null + oldestTimestamp: number | null + newestTimestamp: number | null + horizon: number + }> { + let oldest: number | null = null + let newest: number | null = null + for (const gen of this.committedGensAsc()) { + if (oldest === null) oldest = gen + newest = gen + } + return { + generations: this.committedCount(), + bytes: await this.historyBytes(), + oldestGeneration: oldest, + newestGeneration: newest, + oldestTimestamp: oldest !== null ? (await this.getDelta(oldest)).timestamp : null, + newestTimestamp: newest !== null ? (await this.getDelta(newest)).timestamp : null, + horizon: this.horizonGen + } + } + /** * @description Read one generation's persisted before-image records — the * compaction fallback for generations written before deltas carried @@ -471,14 +627,17 @@ export class GenerationStore { try { paths = await this.storage.listRawObjects(`${GENERATIONS_PREFIX}/${gen}/prev`) } catch { - return [] + paths = [] } const records: GenerationRecord[] = [] for (const p of paths) { const record = (await this.storage.readRawObject(p)) as GenerationRecord | null if (record) records.push(record) } - return records + if (records.length > 0) return records + // Two-tier: folded generations serve their record-set from the segment. + const packed = await this.segments?.readRecords(gen) + return packed ? (packed.map((r) => r.record) as GenerationRecord[]) : [] } /** @@ -600,6 +759,58 @@ export class GenerationStore { * @returns The committed generation and its commit timestamp. * @throws GenerationConflictError when the CAS expectation fails. */ + /** + * The generation fact log, or `null` when the storage layer cannot host one. + * Consumers scan committed facts through it (`scanFacts` / `segmentPaths`). + */ + getFactLog(): FactLog | null { + return this.factLog + } + + /** + * @description Build one commit's AFTER-IMAGE fact by reading canonical + * state back for every touched id — under the commit mutex, immediately + * after the operations applied, so canonical IS the after-image (and the + * reads are page-cache-warm: the operations just wrote these files). An + * absent id (both legs null) becomes a body-less TOMBSTONE — the delete + * fact needs no body, so removal never requires reading the removed thing. + * The fact's blobHashes are extracted from the AFTER records (the content + * this generation's state references), unlike the history path's + * before-image hashes. + */ + private async buildCommitFact(args: { + generation: number + timestamp: number + nouns: string[] + verbs: string[] + meta?: Record + }): Promise { + const ops: FactOp[] = [] + const afterRecords: GenerationRecord[] = [] + for (const id of args.nouns) { + const after = await this.storage.readNounRaw(id) + const absent = after.metadata === null && after.vector === null + ops.push({ kind: 'noun', id, record: absent ? null : after }) + if (!absent) afterRecords.push({ kind: 'noun', metadata: after.metadata, vector: after.vector }) + } + for (const id of args.verbs) { + const after = await this.storage.readVerbRaw(id) + const absent = after.metadata === null && after.vector === null + ops.push({ kind: 'verb', id, record: absent ? null : after }) + if (!absent) afterRecords.push({ kind: 'verb', metadata: after.metadata, vector: after.vector }) + } + const blobHashes = this.storage.extractBlobHashesFromRecords + ? this.storage.extractBlobHashesFromRecords(afterRecords) + : [] + return { + generation: args.generation, + timestamp: args.timestamp, + ops, + ...(args.meta ? { meta: args.meta } : {}), + ...(blobHashes.length > 0 ? { blobHashes } : {}) + } + } + async commitTransaction(args: { touched: TouchedIds meta?: Record @@ -733,6 +944,24 @@ export class GenerationStore { await this.storage.flushWriteBarrier?.() faultPoint('after-execute') + // Fact log (dual-write): append + fsync this generation's AFTER-IMAGE + // fact BEFORE the commit point, inside the same durability window — + // so a crash can only leave the log AHEAD (open truncates), never a + // committed generation without its fact. A real abort below this point + // compensates via dropAbove in the catch. An append failure fails the + // write, loudly — a silent fact gap would be a lie a replay discovers. + if (this.factLog) { + const fact = await this.buildCommitFact({ + generation: gen, + timestamp, + nouns, + verbs, + ...(args.meta ? { meta: args.meta } : {}) + }) + await this.factLog.append(fact) + await this.factLog.sync() + } + // -- 5. Counter + manifest rename (COMMIT POINT) ---------------------- await this.persistCounterUnlocked() faultPoint('before-manifest-rename') @@ -755,6 +984,9 @@ export class GenerationStore { timestamp, bytes: delta.bytes ?? 0 }) + if (this.historyBytesTotal !== null) { + this.historyBytesTotal += delta.bytes ?? 0 + } this.extendChains(gen, nouns, verbs) const logEntry: TxLogEntry = { generation: gen, timestamp, ...(args.meta && { meta: args.meta }) } await this.storage.appendTxLogLine(JSON.stringify(logEntry)) @@ -800,6 +1032,13 @@ export class GenerationStore { // over-count-safe; the scrub restores exactness } } + // Fact-log compensation: a real (non-crash) abort after the fact was + // appended must take the fact back out — the generation never + // committed. A crash instead reaches open(), whose truncation does the + // same reconcile from disk. + if (this.factLog && this.factLog.headGeneration() >= gen) { + await this.factLog.dropAbove(gen - 1) + } // Return the reservation when no concurrent bump consumed a later // number, so a failed transaction leaves generation() unchanged. if (this.counter === gen) this.counter = gen - 1 @@ -991,6 +1230,14 @@ export class GenerationStore { this.pendingBuffer.set(gen, { nouns: nounBefore, verbs: verbBefore, timestamp }) this.pendingGens.push(gen) this.extendChains(gen, nouns, verbs) + // The adopted generation is committed — it gets its fact like any + // other (durability rides the group-commit flush, same as the + // buffered history). + if (this.factLog) { + await this.factLog.append( + await this.buildCommitFact({ generation: gen, timestamp, nouns, verbs }) + ) + } prodLog.warn( `[GenerationStore] Recovered a failed rollback FORWARD: single-op write ` + `committed as generation ${gen} because its canonical undo could not be ` + @@ -1020,6 +1267,17 @@ export class GenerationStore { this.pendingBuffer.set(gen, { nouns: nounBefore, verbs: verbBefore, timestamp }) this.pendingGens.push(gen) this.extendChains(gen, nouns, verbs) + // Fact log (dual-write): the acked write's AFTER-IMAGE fact, appended + // now (read back warm, under the mutex — group-commit means flush-time + // canonical only holds the LATEST state, so each generation's after-image + // exists only here). Durability rides the group-commit flush, exactly + // like the buffered before-image history: a crash before the flush loses + // the fact AND the generation together — never a torn state. + if (this.factLog) { + await this.factLog.append( + await this.buildCommitFact({ generation: gen, timestamp, nouns, verbs }) + ) + } this.schedulePendingFlush() return { generation: gen, timestamp } }) @@ -1141,6 +1399,12 @@ export class GenerationStore { // ONE fsync for the whole window — the durability-batching win. await this.storage.syncRawObjects(stagedPaths) + // Fact log (dual-write): make the window's buffered facts durable in the + // same batch, BEFORE the commit point below — so a crash can only leave + // the log AHEAD of the counter (open truncates), never a committed + // generation without its durable fact. + await this.factLog?.sync() + // Test-only crash simulation: a throwing injector here leaves the staged // group-commit generation dirs on disk with NO manifest advance — the // exact "crashed mid-flush" state recovery must DROP-WITHOUT-RESTORE @@ -1176,6 +1440,9 @@ export class GenerationStore { timestamp: buf.timestamp, bytes: genBytes.get(gen) ?? 0 }) + if (this.historyBytesTotal !== null) { + this.historyBytesTotal += genBytes.get(gen) ?? 0 + } this.pendingBuffer.delete(gen) } this.pendingGens = [] @@ -1608,9 +1875,15 @@ export class GenerationStore { if (pending) { return (kind === 'noun' ? pending.nouns : pending.verbs).get(id) ?? null } - return (await this.storage.readRawObject( + const live = (await this.storage.readRawObject( `${GENERATIONS_PREFIX}/${gen}/prev/${id}.json` )) as GenerationRecord | null + if (live) return live + // Two-tier: the packed tier serves folded generations (live-tier-wins). + if (this.segments?.hasGeneration(gen)) { + return (await this.segments.readRecord(gen, kind, id)) as GenerationRecord | null + } + return null } /** @@ -1957,6 +2230,21 @@ export class GenerationStore { `${GENERATIONS_PREFIX}/${gen}/tx.json` )) as GenerationDelta | null if (delta === null) { + // Two-tier read (D1+D3): not in the live tier → the packed tier. + // Live-tier-wins ordering (a crash mid-fold leaves a duplicate, never + // a gap), so the segment lookup runs only after the live miss. + const packed = await this.segments?.readDelta(gen) + if (packed) { + const d = packed.delta as GenerationDelta + const entry = { + nouns: new Set(d.nouns), + verbs: new Set(d.verbs), + timestamp: packed.timestamp, + bytes: d.bytes ?? 0 + } + this.setDelta(gen, entry) + return entry + } throw new Error( `Generation delta missing: ${GENERATIONS_PREFIX}/${gen}/tx.json ` + `(store corrupted or records removed outside compactHistory())` @@ -1996,17 +2284,26 @@ export class GenerationStore { /** * @description Total serialized bytes of the ON-DISK generational history — * the sum of every committed generation's recorded `bytes`. Backs the - * `maxBytes` and adaptive retention caps. Reads each committed generation's - * delta (cached; a re-read only for cache-evicted ones) — O(committed - * generations), bounded by retention itself and invoked only at compaction - * time. Pending (un-flushed) generations are excluded (they are not on disk). + * `maxBytes` and adaptive retention caps. O(1) after the first call: the + * total is computed by ONE walk over committed deltas, then maintained + * incrementally at every commit (+bytes) and reclaim (−bytes) and dropped on + * a wholesale state replacement (restore). Without the running total, the + * adaptive auto-compaction on every flush() re-walked the ENTIRE history — + * O(committed generations) file reads per flush past the delta-cache bound — + * which is how a 70k-generation production brain turned every write into a + * full-tail scan (SELF-GENERATIONS-GROWTH). Pending (un-flushed) generations + * are excluded (they are not on disk). * @returns The total on-disk history byte count. */ async historyBytes(): Promise { + if (this.historyBytesTotal !== null) { + return this.historyBytesTotal + } let total = 0 for (const gen of this.committedGensAsc()) { total += (await this.getDelta(gen)).bytes } + this.historyBytesTotal = total return total } @@ -2029,6 +2326,94 @@ export class GenerationStore { * @param options - Retention caps (see {@link CompactHistoryOptions}). * @returns Count of removed record-sets and the new horizon. */ + /** + * @description The REPACKER (D1+D3+repacking): fold cold live-tier + * generations into sealed segments — re-representation, never deletion. + * Every record and delta stays readable (asOf/chains unchanged); the + * per-generation directories are deleted only AFTER their segment is + * durable (crash between = duplicate representation, resolved + * live-tier-wins by every reader; never a gap). This is the transform that + * takes a 70k-file history to tens of segment files, and the ONLY history + * transform permitted under the archival profile. + * + * Folds oldest-first, contiguous from the packed boundary, in batches, and + * stops at the live window ({@link GenerationStore.REPACK_LIVE_WINDOW}) + * or when `timeBudgetMs` is spent — an early stop is a consistent prefix; + * the next pass resumes. + */ + async repackHistory(options?: { timeBudgetMs?: number; batchGenerations?: number }): Promise<{ + foldedGenerations: number + segmentsCreated: number + }> { + if (!this.segments) return { foldedGenerations: 0, segmentsCreated: 0 } + const segments = this.segments + return this.withMutex(async () => { + const deadline = + options?.timeBudgetMs !== undefined ? Date.now() + options.timeBudgetMs : undefined + const batchSize = options?.batchGenerations ?? 512 + const coldCeiling = this.committed - GenerationStore.REPACK_LIVE_WINDOW + const packedThrough = + segments.segments().length > 0 + ? segments.segments()[segments.segments().length - 1].lastGeneration + : 0 + + // Cold, unpacked, committed generations — ascending, contiguous scan. + const eligible: number[] = [] + for (const gen of this.committedGensAsc()) { + if (gen > coldCeiling) break + if (gen <= packedThrough) continue // already packed (dup fold barred) + if (this.pendingBuffer.has(gen)) continue // un-flushed = live by definition + eligible.push(gen) + } + + let folded = 0 + let segmentsCreated = 0 + for (let i = 0; i < eligible.length; i += batchSize) { + if (deadline !== undefined && Date.now() >= deadline) break + const batch = eligible.slice(i, i + batchSize) + const foldInput: FoldGeneration[] = [] + for (const gen of batch) { + const delta = (await this.storage.readRawObject( + `${GENERATIONS_PREFIX}/${gen}/tx.json` + )) as GenerationDelta | null + if (delta === null) { + // Already folded by a prior crashed pass whose dirs were removed, + // or damage — getDelta's two-tier read decides which, loudly, + // when someone asks. Skip; never fold a generation we cannot read. + continue + } + const records: FoldGeneration['records'] = [] + for (const [kind, ids] of [ + ['noun', delta.nouns] as const, + ['verb', delta.verbs] as const + ]) { + for (const id of ids) { + const record = await this.storage.readRawObject( + `${GENERATIONS_PREFIX}/${gen}/prev/${id}.json` + ) + if (record) records.push({ kind, id, record }) + } + } + foldInput.push({ generation: gen, timestamp: delta.timestamp, delta, records }) + } + if (foldInput.length === 0) continue + await segments.fold(foldInput) + segmentsCreated++ + // Segment + manifest durable → the live copies retire. + for (const g of foldInput) { + await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`) + } + folded += foldInput.length + } + if (folded > 0) { + prodLog.info( + `[GenerationStore] repacked ${folded} cold generation(s) into ${segmentsCreated} segment(s) — history preserved, file count reduced` + ) + } + return { foldedGenerations: folded, segmentsCreated } + }) + } + async compact(options?: CompactHistoryOptions): Promise { return this.withMutex(async () => { const minPinned = this.minPinnedGeneration() @@ -2036,6 +2421,11 @@ export class GenerationStore { const maxAge = options?.maxAge const maxBytes = options?.maxBytes const ageCutoff = maxAge !== undefined ? Date.now() - maxAge : undefined + // Bounded maintenance pass (8.9.0): stop reclaiming once the budget is + // spent. Safe mid-loop — reclamation is oldest-first, so an early stop + // leaves a consistent contiguous prefix and the next pass resumes. + const deadline = + options?.timeBudgetMs !== undefined ? Date.now() + options.timeBudgetMs : undefined const noCaps = maxGenerations === undefined && maxAge === undefined && maxBytes === undefined @@ -2049,6 +2439,7 @@ export class GenerationStore { for (const gen of [...this.committedGensAsc()]) { // Pins are always exempt: never reclaim a generation a live pin needs. if (gen > minPinned) break // committedGensAsc ascending → nothing newer is eligible either + if (deadline !== undefined && Date.now() >= deadline) break // budget spent — resume next pass const delta = await this.getDelta(gen) if (!noCaps) { const violatesCount = maxGenerations !== undefined && remainingCount > maxGenerations @@ -2080,6 +2471,9 @@ export class GenerationStore { await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${gen}`) this.deltaCache.delete(gen) + if (this.historyBytesTotal !== null) { + this.historyBytesTotal -= delta.bytes + } // AFTER the record-set is gone (over-count-only crash ordering): // release its history references and reclaim any blob left with zero @@ -2111,6 +2505,16 @@ export class GenerationStore { // Reclaimed generations leave the per-id chains stale → rebuild on next read. this.invalidateChains() this.horizonGen = Math.max(this.horizonGen, highestRemoved) + // Packed-tier reclaim (D3): a packed generation's bytes live in a + // sealed segment — removeRawPrefix above was a no-op for it. Drop + // WHOLE segments now fully below the horizon; a partially-reclaimed + // segment keeps its bytes until the boundary passes it (the frozen + // partial-segments-wait rule; logical reclamation above still holds — + // the generations left committedRanges and asOf below the horizon + // throws regardless). + if (this.segments) { + await this.segments.dropSegmentsBelow(this.horizonGen + 1) + } const manifest: GenerationManifest = { version: 1, generation: this.committed, @@ -2140,6 +2544,9 @@ export class GenerationStore { async reopenAfterRestore(floorGeneration: number): Promise { await this.withMutex(async () => { this.deltaCache.clear() + // The running history-byte total describes the REPLACED store — drop it; + // the next historyBytes() re-seeds with one walk over the new state. + this.historyBytesTotal = null // A wholesale state replacement invalidates any buffered single-op // history — discard the pending tier (its live writes are gone with the // replaced store). diff --git a/src/db/types.ts b/src/db/types.ts index 4f932e76..4c8a4957 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -116,6 +116,16 @@ export interface TransactOptions { * record is staged. */ ifAtGeneration?: number + /** + * Budget (ms) for the atomic apply phase. When omitted, the budget SCALES + * with the batch: `max(30 000, opCount × 2 000)` — production imports on + * network-attached disks measure ~2 s per operation, so a flat 30 s budget + * silently capped honest bulk work at ~15 operations. A tripped budget + * rolls the whole batch back and throws a retryable + * `TransactionTimeoutError` naming the operation it stopped at, the batch + * size, and the elapsed/budget times. + */ + timeoutMs?: number } /** @@ -166,6 +176,15 @@ export interface CompactHistoryOptions { * of each surviving generation's serialized record set (`GenerationDelta.bytes`). */ maxBytes?: number + /** + * Stop reclaiming after this many milliseconds even if caps are still + * exceeded (8.9.0). Compaction is maintenance — a bounded pass keeps + * `close()` (and any explicit maintenance window) from stalling on a large + * backlog; the next pass resumes where this one stopped (reclamation is + * oldest-first, so an early stop is always a consistent prefix). Unset = + * run to completion. + */ + timeBudgetMs?: number } /** @@ -183,6 +202,36 @@ export interface CompactHistoryResult { horizon: number } +/** + * @description Result of `brain.historyStats()` — the read-only generational + * history footprint, for fleet audits and ops doors. A pool operator runs this + * per brain to size retention exposure (how much MVCC history each brain + * carries and under which policy) without touching any data. + */ +export interface HistoryStats { + /** Committed generation record-sets currently on disk. */ + generations: number + /** Total on-disk history bytes across those record-sets. */ + bytes: number + /** Oldest committed generation still on disk (null when history is empty). */ + oldestGeneration: number | null + /** Newest committed generation (null when history is empty). */ + newestGeneration: number | null + /** Commit timestamp (ms) of the oldest on-disk generation. */ + oldestTimestamp: number | null + /** Commit timestamp (ms) of the newest on-disk generation. */ + newestTimestamp: number | null + /** Compaction horizon — generations below it were reclaimed. */ + horizon: number + /** The effective retention mode this brain runs under. */ + retentionMode: 'all' | 'adaptive' | 'explicit' + /** + * The adaptive byte budget in force (coordinator-driven or the local + * free-memory probe); null under 'all' or explicit caps. + */ + effectiveBudgetBytes: number | null +} + // ============================================================================ // Db surfaces // ============================================================================ @@ -423,6 +472,22 @@ export interface GenerationStorage { /** Read all lines of `_system/tx-log.jsonl` (empty array if absent). */ readTxLogLines(): Promise + /** + * OPTIONAL binary raw-byte primitives — the substrate for the generation + * fact log's append-only CRC-framed segments. Feature-detected: a storage + * layer that omits them hosts no fact log (dual-write is skipped; readers + * fall back to canonical enumeration). Paths are used VERBATIM (no + * suffixing). Append durability rides `syncRawObjects` at the commit + * barrier, exactly like the staged history files. + */ + appendRawBytes?(path: string, bytes: Uint8Array): Promise + /** Read a raw binary file whole; absent → null; a real fault throws. */ + readRawBytes?(path: string): Promise + /** Replace a raw binary file atomically (tmp → fsync → rename). */ + writeRawBytes?(path: string, bytes: Uint8Array): Promise + /** Byte size of a raw binary file, or null when absent. */ + rawByteSize?(path: string): Promise + /** * OPTIONAL temporal-blob contract (implemented by blob-aware storage; the * generation store treats the hashes as opaque strings). Extract the diff --git a/src/errors/brainyError.ts b/src/errors/brainyError.ts index c2667009..a58236e3 100644 --- a/src/errors/brainyError.ts +++ b/src/errors/brainyError.ts @@ -15,6 +15,8 @@ export type BrainyErrorType = | 'GRAPH_INDEX_NOT_READY' | 'METADATA_INDEX_NOT_READY' | 'VECTOR_INDEX_NOT_READY' + | 'PROTECTED_ARTIFACT' + | 'DERIVED_ARTIFACT_MISSING' | 'MIGRATION_IN_PROGRESS' /** @@ -307,6 +309,60 @@ export class VectorIndexNotReadyError extends BrainyError { } } +/** + * Thrown when a delete (`deleteBinaryBlob` / `removeRawPrefix`) would remove a + * blob that is a declared member of a protected derived-index FAMILY (the + * registered-blob contract). Declared derived artifacts are undeletable through + * the storage layer — this makes an in-process GC / sweeper INCAPABLE of removing + * a load-bearing index file (the lost-`main.dkann` class). Intentional retirement + * is the explicit `unregisterDerivedFamily(name)` step, then the delete. + */ +export class ProtectedArtifactError extends BrainyError { + /** The blob key the delete targeted. */ + public readonly key: string + /** The protected family the key belongs to. */ + public readonly family: string + constructor(key: string, family: string) { + super( + `Refused to delete '${key}': it is a declared member of the protected ` + + `derived-index family '${family}'. Declared derived artifacts are undeletable ` + + `through the storage layer (COLD ≠ DEAD) — unregisterDerivedFamily('${family}') ` + + `first if retirement is intentional.`, + 'PROTECTED_ARTIFACT', + false + ) + this.name = 'ProtectedArtifactError' + this.key = key + this.family = family + } +} + +/** + * Raised (loudly) when a declared derived-index family is missing one or more of + * its members on open — i.e. a load-bearing blob was deleted OUTSIDE the write + * path (an external sweeper the in-process refusal cannot stop). The index must + * be rebuilt from canonical; "healthy-while-broken" is impossible because the + * missing member is named, not silently tolerated. + */ +export class DerivedArtifactMissingError extends BrainyError { + /** The family with missing members. */ + public readonly family: string + /** The member keys that are absent. */ + public readonly missing: string[] + constructor(family: string, missing: string[]) { + super( + `Derived-index family '${family}' is missing ${missing.length} declared ` + + `member(s) on open (${missing.join(', ')}) — deleted outside the write path. ` + + `The index must be rebuilt from canonical.`, + 'DERIVED_ARTIFACT_MISSING', + false + ) + this.name = 'DerivedArtifactMissingError' + this.family = family + this.missing = missing + } +} + /** * Thrown when a data-plane read or write is issued against a brain that is * running its one-time, automatic 7.x → 8.0 on-disk upgrade — the coordinated diff --git a/src/graph/graphAdjacencyIndex.ts b/src/graph/graphAdjacencyIndex.ts index 816856b1..b37391aa 100644 --- a/src/graph/graphAdjacencyIndex.ts +++ b/src/graph/graphAdjacencyIndex.ts @@ -282,6 +282,16 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { } hasMore = result.hasMore + if (hasMore && (!result.nextCursor || result.nextCursor === cursor)) { + // A stalled cursor with hasMore=true would re-read the same page + // forever — a silent full-CPU loop at cold open. Abort loudly; a + // graph read failing beats a process that spins without a log line. + throw new Error( + `GraphAdjacencyIndex: verb walk stalled after ${count} verbs — storage returned ` + + `hasMore=true with ${result.nextCursor ? 'a non-advancing' : 'no'} cursor. ` + + `Aborting the cold-load; run brain.repairIndex() if this persists.` + ) + } cursor = result.nextCursor } diff --git a/src/graph/graphAudit.ts b/src/graph/graphAudit.ts new file mode 100644 index 00000000..d0e44cd9 --- /dev/null +++ b/src/graph/graphAudit.ts @@ -0,0 +1,214 @@ +/** + * @module graph/graphAudit + * @description Read-only graph-truth audit — the graph sibling of `repairIndex()`'s + * diagnosis half. Verifies three layers against each other without mutating anything: + * + * 1. CANONICAL verb records (the storage walk — the source of truth) + * 2. the RELATIONSHIP READ PATH (`related()` with all visibility tiers — exactly + * what application reads like a VFS `readdir` consult) + * 3. ENTITY ENDPOINTS (does each verb's source/target still exist?) + * + * and classifies every discrepancy into the three failure families production + * incidents have shown: + * + * - `missingFromReads` — a canonical verb record the read path does NOT return + * for its source: PRESENT BUT INVISIBLE (adjacency/membership staleness). + * - `danglingEndpoints` — a canonical verb whose endpoint entity is gone: + * the SCAR class (write-path loss / partial delete). + * - `readOnlyVerbIds` — the read path returns an edge with NO canonical + * record: GHOST edges (stale index entries). + * + * `visibilityHiddenCount` is reported separately: an internal/system edge that is + * indexed and present but hidden from DEFAULT reads is working as designed — the + * audit reads with all tiers included so design-hiding is never misclassified as + * index loss. + * + * Full counts are always exact; only the example LISTS are capped (`maxExamples`) + * — a capped report says so via `truncatedExamples`, never silently. + */ + +import { prodLog } from '../utils/logger.js' + +/** One discrepant relationship, identified fully enough to inspect by hand. */ +export interface GraphAuditDiscrepancy { + verbId: string + from: string + to: string + type: string +} + +export interface GraphAuditReport { + /** True iff every discrepancy count is zero — `related()` returns canonical truth. */ + coherent: boolean + verbsInCanonical: number + entitiesInCanonical: number + /** Distinct source entities whose read path was actually consulted (coverage honesty). */ + sourcesChecked: number + + /** PRESENT BUT INVISIBLE: canonical records the read path omits. */ + missingFromReadsCount: number + missingFromReads: GraphAuditDiscrepancy[] + + /** SCAR CLASS: canonical verbs with a missing endpoint entity. */ + danglingEndpointsCount: number + danglingEndpoints: Array + + /** GHOST EDGES: read-path verb ids with no canonical record. */ + readOnlyCount: number + readOnlyVerbIds: string[] + + /** Canonical verbs hidden from DEFAULT reads by design (internal/system visibility). */ + visibilityHiddenCount: number + + /** Example lists above were capped at maxExamples; counts remain exact. */ + truncatedExamples: boolean + durationMs: number +} + +/** A canonical verb record, as the audit needs it. */ +export interface AuditVerbRecord { + id: string + type: string + sourceId: string + targetId: string + visibility?: string +} + +/** The seams the audit runs over — injected so the walk is testable in isolation. */ +export interface GraphAuditDeps { + /** Stream every canonical entity id (id-only; no per-entity reads needed). */ + eachNounId(consume: (id: string) => void): Promise + /** Stream every canonical verb record. */ + eachVerb(consume: (verb: AuditVerbRecord) => void): Promise + /** + * The END-TO-END relationship read for one source, ALL visibility tiers + * included — must be the same path application reads consult. + */ + readRelationsFrom(sourceId: string): Promise> +} + +export interface GraphAuditOptions { + /** Cap on entries per example list (counts stay exact). Default 100. */ + maxExamples?: number +} + +export async function runGraphAudit( + deps: GraphAuditDeps, + options: GraphAuditOptions = {} +): Promise { + const maxExamples = options.maxExamples ?? 100 + const started = Date.now() + + // 1. Canonical entity ids — endpoint existence oracle. + const entityIds = new Set() + await deps.eachNounId((id) => entityIds.add(id)) + + // 2. Canonical verb walk: group by source, check endpoints, note visibility. + const canonicalVerbIds = new Set() + const bySource = new Map() + let verbsInCanonical = 0 + let visibilityHiddenCount = 0 + let danglingEndpointsCount = 0 + const danglingEndpoints: GraphAuditReport['danglingEndpoints'] = [] + + await deps.eachVerb((verb) => { + verbsInCanonical++ + canonicalVerbIds.add(verb.id) + const list = bySource.get(verb.sourceId) + if (list) list.push(verb) + else bySource.set(verb.sourceId, [verb]) + + if (verb.visibility === 'internal' || verb.visibility === 'system') { + visibilityHiddenCount++ + } + + const fromMissing = !entityIds.has(verb.sourceId) + const toMissing = !entityIds.has(verb.targetId) + if (fromMissing || toMissing) { + danglingEndpointsCount++ + if (danglingEndpoints.length < maxExamples) { + danglingEndpoints.push({ + verbId: verb.id, + from: verb.sourceId, + to: verb.targetId, + type: verb.type, + missingEnd: fromMissing && toMissing ? 'both' : fromMissing ? 'from' : 'to' + }) + } + } + }) + + // 3. Per-source read-path comparison. A verb must be returned by the read + // path of ITS OWN source — the exact consult a readdir/traversal makes. + let missingFromReadsCount = 0 + const missingFromReads: GraphAuditDiscrepancy[] = [] + let readOnlyCount = 0 + const readOnlyVerbIds: string[] = [] + const readOnlySeen = new Set() + + for (const [sourceId, verbs] of bySource) { + const readIds = new Set((await deps.readRelationsFrom(sourceId)).map((r) => r.id)) + + for (const verb of verbs) { + if (!readIds.has(verb.id)) { + missingFromReadsCount++ + if (missingFromReads.length < maxExamples) { + missingFromReads.push({ + verbId: verb.id, + from: verb.sourceId, + to: verb.targetId, + type: verb.type + }) + } + } + } + + for (const readId of readIds) { + if (!canonicalVerbIds.has(readId) && !readOnlySeen.has(readId)) { + readOnlySeen.add(readId) + readOnlyCount++ + if (readOnlyVerbIds.length < maxExamples) { + readOnlyVerbIds.push(readId) + } + } + } + } + + const coherent = + missingFromReadsCount === 0 && danglingEndpointsCount === 0 && readOnlyCount === 0 + + const report: GraphAuditReport = { + coherent, + verbsInCanonical, + entitiesInCanonical: entityIds.size, + sourcesChecked: bySource.size, + missingFromReadsCount, + missingFromReads, + danglingEndpointsCount, + danglingEndpoints, + readOnlyCount, + readOnlyVerbIds, + visibilityHiddenCount, + truncatedExamples: + missingFromReadsCount > missingFromReads.length || + danglingEndpointsCount > danglingEndpoints.length || + readOnlyCount > readOnlyVerbIds.length, + durationMs: Date.now() - started + } + + if (coherent) { + prodLog.info( + `[GraphAudit] coherent: ${verbsInCanonical} verbs across ${bySource.size} sources — ` + + `the read path returns canonical truth (${report.durationMs}ms)` + ) + } else { + prodLog.warn( + `[GraphAudit] INCOHERENT: ${missingFromReadsCount} present-but-invisible, ` + + `${danglingEndpointsCount} dangling-endpoint, ${readOnlyCount} ghost ` + + `(of ${verbsInCanonical} canonical verbs, ${bySource.size} sources, ` + + `${visibilityHiddenCount} visibility-hidden by design) — ${report.durationMs}ms` + ) + } + + return report +} diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index 605d2a0b..eb2acd71 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -54,6 +54,9 @@ export class HnswFlushError extends Error { * acceleration provider). */ export class JsHnswVectorIndex implements VectorIndexProvider { + /** Self-identifies as the built-in JS fallback engine — see {@link VectorIndexProvider.name}. */ + readonly name = 'hnsw-js' + private nouns: Map = new Map() /** * Reverse adjacency: `target id → (level → set of node ids that link TO target)`. diff --git a/src/import/BackgroundDeduplicator.ts b/src/import/BackgroundDeduplicator.ts index 53b28262..073fbd9e 100644 --- a/src/import/BackgroundDeduplicator.ts +++ b/src/import/BackgroundDeduplicator.ts @@ -41,6 +41,14 @@ export interface DeduplicationStats { * - Import-scoped deduplication (no cross-contamination) * - 3-tier strategy (ID → Name → Similarity) * - Uses existing indexes (EntityIdMapper, MetadataIndexManager, TypeAware HNSW) + * + * Lifecycle: ONE instance per brain, owned by Brainy (getBackgroundDeduplicator) + * so the debounce genuinely spans imports and brain.close() cancels pending + * work via cancelPending() — this pass merge-DELETES duplicate entities, so it + * must never fire against a closed brain. The enableDeduplication gate lives + * at the scheduling call site (ImportCoordinator); scheduleDedup itself is + * unconditional. The timer is unref'd — a pending pass never holds the + * process open. */ export class BackgroundDeduplicator { private brain: Brainy @@ -67,12 +75,15 @@ export class BackgroundDeduplicator { clearTimeout(this.debounceTimer) } - // Schedule for 5 minutes from now + // Schedule for 5 minutes from now. unref'd: a pending dedup pass must + // never hold the process open (exit-hang class) — if the process exits + // first, the pass simply never runs; imports are already durable. this.debounceTimer = setTimeout(() => { this.runBatchDedup().catch(error => { prodLog.error('[BackgroundDedup] Batch dedup failed:', error) }) }, 5 * 60 * 1000) + this.debounceTimer.unref?.() } /** diff --git a/src/import/ImportCoordinator.ts b/src/import/ImportCoordinator.ts index 21d09410..1e1316b7 100644 --- a/src/import/ImportCoordinator.ts +++ b/src/import/ImportCoordinator.ts @@ -13,7 +13,6 @@ import { Brainy } from '../brainy.js' import { FormatDetector, SupportedFormat } from './FormatDetector.js' import { ImportHistory, type ImportHistoryEntry } from './ImportHistory.js' -import { BackgroundDeduplicator } from './BackgroundDeduplicator.js' import { SmartExcelImporter } from '../importers/SmartExcelImporter.js' import { SmartPDFImporter } from '../importers/SmartPDFImporter.js' import { SmartCSVImporter } from '../importers/SmartCSVImporter.js' @@ -112,7 +111,12 @@ export interface ValidImportOptions { /** Confidence threshold for entities */ confidenceThreshold?: number - /** Enable entity deduplication across imports */ + /** + * Enable entity deduplication (default: true). Gates BOTH passes: the + * inline merge during import AND the debounced background pass that runs + * ~5 minutes after the last import (which merge-DELETES duplicate entities). + * Set false for deployments that must never auto-remove records. + */ enableDeduplication?: boolean /** Similarity threshold for deduplication (0-1) */ @@ -286,7 +290,6 @@ export class ImportCoordinator { private brain: Brainy private detector: FormatDetector private history: ImportHistory - private backgroundDedup: BackgroundDeduplicator private excelImporter: SmartExcelImporter private pdfImporter: SmartPDFImporter private csvImporter: SmartCSVImporter @@ -300,7 +303,6 @@ export class ImportCoordinator { this.brain = brain this.detector = new FormatDetector() this.history = new ImportHistory(brain) - this.backgroundDedup = new BackgroundDeduplicator(brain) this.excelImporter = new SmartExcelImporter(brain) this.pdfImporter = new SmartPDFImporter(brain) this.csvImporter = new SmartCSVImporter(brain) @@ -1459,9 +1461,16 @@ export class ImportCoordinator { } } - // Schedule background deduplication (debounced 5 minutes) - if (trackingContext && trackingContext.importId) { - this.backgroundDedup.scheduleDedup(trackingContext.importId) + // Schedule background deduplication (debounced 5 minutes, brain-owned so + // close() can cancel it). Honors the same enableDeduplication gate as the + // inline pass — false means NO dedup, inline or background. + if ( + trackingContext && + trackingContext.importId && + options.enableDeduplication !== false + ) { + const backgroundDedup = await this.brain.getBackgroundDeduplicator() + backgroundDedup.scheduleDedup(trackingContext.importId) } return { diff --git a/src/index.ts b/src/index.ts index 619a3470..bfab39da 100644 --- a/src/index.ts +++ b/src/index.ts @@ -28,6 +28,19 @@ export type { FileVersion } from './vfs/types.js' // Export diagnostics result type export type { DiagnosticsResult } from './brainy.js' +// brain.warm() — eager index/storage readiness report (per-surface honest +// outcome + timing). See the WarmReport JSDoc in brainy.ts. +export type { WarmReport, WarmOutcome } from './brainy.js' +export type { + GraphAuditReport, + GraphAuditDiscrepancy +} from './graph/graphAudit.js' +export { + checkOsLimits, + NOFILE_POOL_FLOOR, + MAX_MAP_COUNT_POOL_FLOOR +} from './utils/osLimits.js' +export type { OsLimitsReport } from './utils/osLimits.js' // Export Brainy configuration and types export type { @@ -150,7 +163,7 @@ export { EntityNotFoundError, RelationNotFoundError } from './errors/notFound.js // Base error + typed migration-lock error — thrown by any data-plane call while a // brain runs its one-time 7.x→8.0 upgrade; catch to answer HTTP 503 + Retry-After. -export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError } from './errors/brainyError.js' +export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError, ProtectedArtifactError, DerivedArtifactMissingError } from './errors/brainyError.js' export type { BrainyErrorType } from './errors/brainyError.js' // ============= 8.0 Db API — generational MVCC ============= @@ -191,14 +204,30 @@ export type { TxLogEntry, CompactHistoryOptions, CompactHistoryResult, + HistoryStats, ChangedIds, DiffResult, HistoryVersion, EntityHistory } from './db/types.js' +// The generation fact log — sequential after-image scan surface +// (brain.scanFacts / brain.factSegmentPaths) for index heals and replays. +export type { + CommitFact, + FactOp, + FactScanBatch, + SCANFACTS_FIRST_BATCH_MS, + FactScanHandle +} from './db/factLog.js' +// The generalized family stamp — which source generation a projection +// reflects + the surface that verifies it whole; one verifier, both member +// modes (enumerated byte-exact / rollup invariants). +export { readFamilyStamp, verifyFamilyStamp, ENTITY_TREE_STAMP_PATH } from './db/familyStamp.js' +export type { FamilyStamp, StampMembers, StampVerdict } from './db/familyStamp.js' // Optional provider capability for generation-aware native indexes export { isVersionedIndexProvider } from './plugin.js' export type { VersionedIndexProvider } from './plugin.js' +export type { ProviderInvariantReport, InvariantResult, InvariantHeal } from './plugin.js' // Optional native graph-acceleration engine (cor 3.0) — the published provider // contract + its columnar wire types. Brainy feature-detects an implementation // and falls back to its pure-TS adjacency when absent. @@ -300,7 +329,8 @@ import type { HNSWNoun, HNSWVerb, HNSWConfig, - StorageAdapter + StorageAdapter, + DerivedFamilyDeclaration } from './coreTypes.js' // Export vector index implementation (the JS HNSW path) @@ -318,7 +348,8 @@ export type { HNSWNoun, HNSWVerb, HNSWConfig, - StorageAdapter + StorageAdapter, + DerivedFamilyDeclaration } // Export graph types diff --git a/src/internals.ts b/src/internals.ts index 00a24638..009124fe 100644 --- a/src/internals.ts +++ b/src/internals.ts @@ -19,3 +19,29 @@ export type { MemoryInfo, CacheAllocationStrategy } from './utils/memoryDetectio // HNSWNounWithMetadata. First-party plugins (Cor) use this to stay in // lockstep with the entity shape contract. export { resolveEntityField, STANDARD_ENTITY_FIELDS } from './coreTypes.js' + +// The generalized family stamp — ONE verifier, both member modes, shared +// verbatim with native providers so stamp verification is literally one +// function, never two synchronized copies. Providers write the same shape +// (their set-swap rewrites stamps, so adoption is migration-free). +export { + readFamilyStamp, + writeFamilyStamp, + verifyFamilyStamp, + FAMILY_STAMPS_PREFIX, + ENTITY_TREE_STAMP_PATH +} from './db/familyStamp.js' +export type { + FamilyStamp, + StampMembers, + StampVerdict, + EnumeratedMember, + StampStorage +} from './db/familyStamp.js' + +// Generation fact-log types — the scan surface a provider reaches through the +// storage capability (`storage.scanFacts` / `storage.factLogHeadGeneration` / +// `storage.factSegmentPaths`, wired by the host brain at init). Providers +// never construct a FactLog themselves: open() is writer-side (it reconciles +// by truncating/rewriting) and there is exactly one writer. +export type { CommitFact, FactOp, FactScanBatch, FactScanHandle } from './db/factLog.js' diff --git a/src/plugin.ts b/src/plugin.ts index 12455b3a..02639955 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -115,6 +115,62 @@ export interface BrainyPluginContext { // implementation then fails to compile until it provides the member. // =========================================================================== +/** + * @description How a failed provider invariant should be remediated: + * - `'none'` — informational; the invariant held or nothing to do. + * - `'repair'` — a targeted, cheap fix exists (e.g. re-derive a count/manifest field). + * - `'rebuild'` — the derived state must be rebuilt from canonical (`provider.rebuild()`). + */ +export type InvariantHeal = 'none' | 'repair' | 'rebuild' + +/** + * @description The result of ONE provider invariant check. A failure + * (`holds === false`) NAMES what diverged, with numbers, so it is diagnosable + * from the report alone — never a bare boolean. `name` is a stable kebab-case id + * for telemetry / remediation routing. + */ +export interface InvariantResult { + /** Stable kebab-case id, e.g. `'manifest-residency'` / `'posted-count-floor'`. */ + name: string + /** `true` when the invariant holds. */ + holds: boolean + /** Human-readable detail; on failure, names the divergence WITH numbers. */ + detail: string + /** The value the invariant expected (optional, for diagnosis). */ + expected?: unknown + /** The value actually observed (optional, for diagnosis). */ + actual?: unknown + /** How a failure should be remediated. Ignored when `holds === true`. */ + heal: InvariantHeal +} + +/** + * @description A provider's self-report of its own cross-layer invariants + * (the `validateInvariants()` hook). Contract: + * - It NEVER throws — a failure is DATA (`healthy: false` + a failing invariant), + * not an exception. + * - It is BOUNDED (<50ms): residency checks + O(1) counts only, NO canonical + * walks — safe to call on a live brain, repeatedly. + * - `serving` = can the provider answer queries right now (the `isReady()` truth); + * `healthy` = do ALL invariants hold. A provider can be `serving` while an + * invariant flags a latent divergence, or `healthy` but not-yet-`serving` on a + * cold open. + */ +export interface ProviderInvariantReport { + /** Which provider produced this report, e.g. `'vector'` / `'graph'` / `'metadata'` / `'column'`. */ + provider: string + /** `true` iff every invariant in {@link invariants} holds. */ + healthy: boolean + /** `true` iff the provider can serve queries now (the `isReady()` truth). */ + serving: boolean + /** Each checked invariant and its verdict. */ + invariants: InvariantResult[] + /** Epoch millis when the check ran. */ + checkedAt: number + /** How long the check took (must stay well under 50ms). */ + durationMs: number +} + /** * The `'metadataIndex'` provider — a drop-in for `MetadataIndexManager`. * Brainy calls this surface via `this.metadataIndex.*` (see `brainy.ts`) and @@ -139,6 +195,18 @@ export interface MetadataIndexProvider { */ isReady?(): boolean + /** + * @description OPTIONAL. The provider's self-report of its own + * cross-layer invariants (manifest ↔ segments ↔ counts residency/coherence). + * MUST NOT throw — a failure is DATA (`healthy: false` + a failing invariant). + * MUST be bounded (<50ms): residency + O(1) counts only, NO canonical walks, so + * brainy's {@link } `validateIndexConsistency()` can call it on a live brain. + * Absent → brainy skips this provider in the cross-layer check (feature-detected). + * `repairIndex()` maps any failing invariant with `heal: 'rebuild'` to this + * provider's `rebuild()`. + */ + validateInvariants?(): Promise + /** * @description OPTIONAL. A native provider returns true from the moment its * `init()` detects a large epoch-drift until its background @@ -289,6 +357,18 @@ export interface GraphIndexProvider { */ isReady?(): boolean + /** + * @description OPTIONAL. The provider's self-report of its own + * cross-layer invariants (manifest ↔ segments ↔ counts residency/coherence). + * MUST NOT throw — a failure is DATA (`healthy: false` + a failing invariant). + * MUST be bounded (<50ms): residency + O(1) counts only, NO canonical walks, so + * brainy's {@link } `validateIndexConsistency()` can call it on a live brain. + * Absent → brainy skips this provider in the cross-layer check (feature-detected). + * `repairIndex()` maps any failing invariant with `heal: 'rebuild'` to this + * provider's `rebuild()`. + */ + validateInvariants?(): Promise + /** * @description OPTIONAL eager cold-load. Called once during brain init — AFTER * the metadata provider's `init()` (so the id-mapper is hydrated; a native int @@ -301,6 +381,20 @@ export interface GraphIndexProvider { */ init?(): Promise + /** + * @description OPTIONAL. Eagerly load/fault-in backing storage (e.g. mmap + * pretouch) so first operations run at steady-state cost. Optional; + * absence means the provider demand-loads. Distinct from `init?()`: `init` + * is called once automatically during brain startup as part of the + * readiness contract (cold-load + rebuild gating); `warm` is an explicit, + * separate readiness step a caller opts into via `brain.warm()` (or + * `warmOnOpen`) specifically to pre-pay demand-load cost that `init` left + * lazy. A provider that already loads everything eagerly in `init?()` may + * implement `warm` as a no-op or omit it — `brain.warm()` falls back to a + * best-effort read-through probe when absent. + */ + warm?(): Promise + /** * @description OPTIONAL. A native provider returns true from the moment its * `init()` detects a large epoch-drift until its background @@ -867,6 +961,26 @@ export interface AtGenerationVectors { * are retired and never looked up. */ export interface VectorIndexProvider { + /** + * @description REQUIRED self-reported implementation identity, rendered as + * `[vector-index:]` in prose log lines and stamped into the + * transaction op-name strings that surface in journals/timings (e.g. + * `AddToVectorIndex(hnsw-js)`) — see + * `src/transaction/operations/IndexOperations.ts`. A provider must name + * itself TRUTHFULLY (its own algorithm/engine, e.g. its plugin/package + * name) and must never inherit a default — guessing a backend name would + * resurrect the exact bug this field exists to prevent: an operator + * hunting an index that isn't the one actually running. The built-in JS + * index always sets this to `'hnsw-js'`; a native provider picks its own + * string. At the TypeScript level this field is required; a runtime + * instance from an older provider compiled against the previous optional + * `providerId` contract is tolerated (never crashes, never silently + * mislabeled) — see `resolveVectorProviderId` in + * `src/transaction/operations/IndexOperations.ts`, which stamps + * `'unknown-provider'` and emits one loud warning for that case. + */ + readonly name: string + addItem(item: VectorDocument): Promise removeItem(id: string): Promise search( @@ -929,6 +1043,20 @@ export interface VectorIndexProvider { */ init?(): Promise + /** + * @description OPTIONAL. Eagerly load/fault-in backing storage (e.g. mmap + * pretouch) so first operations run at steady-state cost. Optional; + * absence means the provider demand-loads. Distinct from `init?()`: `init` + * runs automatically once during brain startup (the cold-load + rebuild + * readiness contract above); `warm` is a separate, explicit step a caller + * opts into via `brain.warm()` (or `warmOnOpen`) to pre-pay demand-load + * cost `init` left lazy — e.g. touching every mmap page rather than just + * opening the file. A provider that already loads everything eagerly in + * `init?()` may implement `warm` as a no-op or omit it — `brain.warm()` + * falls back to a best-effort probe `search()` when absent. + */ + warm?(): Promise + /** * @description OPTIONAL honest durability signal (readiness contract, * mirrors {@link GraphIndexProvider.isReady}). `true` ⇔ the persisted @@ -943,6 +1071,18 @@ export interface VectorIndexProvider { */ isReady?(): boolean + /** + * @description OPTIONAL. The provider's self-report of its own + * cross-layer invariants (manifest ↔ segments ↔ counts residency/coherence). + * MUST NOT throw — a failure is DATA (`healthy: false` + a failing invariant). + * MUST be bounded (<50ms): residency + O(1) counts only, NO canonical walks, so + * brainy's {@link } `validateIndexConsistency()` can call it on a live brain. + * Absent → brainy skips this provider in the cross-layer check (feature-detected). + * `repairIndex()` maps any failing invariant with `heal: 'rebuild'` to this + * provider's `rebuild()`. + */ + validateInvariants?(): Promise + /** * @description OPTIONAL. A native provider returns true from the moment its * `init()` detects a large epoch-drift until its background diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 9ff0539e..5eb4785a 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -96,6 +96,14 @@ export class FileSystemStorage extends BaseStorage { private static readonly WRITER_STALE_THRESHOLD_MS = 60_000 private writerLockHeartbeat?: NodeJS.Timeout private writerLockInfo?: WriterLockInfo + /** + * The currently-executing heartbeat refresh, if any. `releaseWriterLock()` + * awaits it before unlinking: clearInterval() stops FUTURE ticks but not a + * tick already in flight, and a straggler landing after the unlink would + * RE-CREATE the lock file — a phantom lock blocking the next writer until + * the stale TTL expires (the pool-eviction reopen case). + */ + private writerHeartbeatInFlight?: Promise // Flush-request RPC state. The writer polls `locks/_flush_requests/` for // new `.req` files and emits `.ack` files in `locks/_flush_responses/` after @@ -502,6 +510,104 @@ export class FileSystemStorage extends BaseStorage { this.writeBarrierDeleteDirs?.add(path.dirname(pathStr)) } + /** + * @description Remove the entity container directory after both canonical legs + * were deleted (full-removal delete). `objectLegPath` is one leg's + * storage-root-relative path (e.g. `entities/nouns///vectors.json`); + * its parent is the `/` entity directory. `rmdir` removes it ONLY if empty + * — both legs are gone by this point, so it should be. A non-empty dir means an + * unexpected leftover (a leg whose delete faulted — already surfaced upstream — + * or foreign data): we do NOT recursively nuke it (loud errors, never quiet + * losses); we log and leave it for the orphan repair sweep (`repairIndex`). + */ + protected override async removeCanonicalContainer(objectLegPath: string): Promise { + const relDir = path.dirname(objectLegPath) + const absDir = path.join(this.rootDir, relDir) + try { + await fs.promises.rmdir(absDir) + // The dir removal is durable once its PARENT (the shard dir) is fsync'd. + this.writeBarrierDeleteDirs?.add(path.dirname(relDir)) + } catch (error: any) { + if (error?.code === 'ENOENT') return // container already gone — fine + if (error?.code === 'ENOTEMPTY' || error?.code === 'EEXIST') { + console.warn( + `[FileSystemStorage] entity container ${relDir} not empty after delete — ` + + `leaving it for the orphan repair sweep (brain.repairIndex()).` + ) + return + } + throw error + } + } + + /** + * @description Prune orphaned entity directories left by the pre-8.3.1 + * partial-delete defect. A delete used to remove the metadata (content) leg + * but leave the `vectors.json` leg + the `/` directory (a "ghost"), or — + * on the direct storage path — remove both legs but leave an empty directory + * (a "scar"). Neither is a live entity (`getNoun` needs the metadata content + * leg) yet each lingers on disk, inflating enumerated counts and confusing + * locator resolution. + * + * CONSERVATIVE by design: removes ONLY dirs with NO metadata content leg — a + * directory that still holds its content is left untouched. LOUD: logs every + * removed orphan. Operator-invoked via {@link Brainy.repairIndex}; never runs + * automatically. Returns the pruned container ids so the caller can recompute + * counts. + */ + public async pruneOrphanedEntities(): Promise<{ nouns: string[]; verbs: string[] }> { + await this.ensureInitialized() + const pruned: { nouns: string[]; verbs: string[] } = { nouns: [], verbs: [] } + + for (const [kind, root] of [ + ['nouns', 'entities/nouns'], + ['verbs', 'entities/verbs'] + ] as const) { + const rootAbs = path.join(this.rootDir, root) + let shards: string[] + try { + shards = await fs.promises.readdir(rootAbs) + } catch (error: any) { + if (error?.code === 'ENOENT') continue // no entities of this kind yet + throw error + } + + for (const shard of shards) { + const shardAbs = path.join(rootAbs, shard) + let entries: import('fs').Dirent[] + try { + entries = await fs.promises.readdir(shardAbs, { withFileTypes: true }) + } catch (error: any) { + if (error?.code === 'ENOENT') continue + throw error + } + + for (const entry of entries) { + if (!entry.isDirectory()) continue + const idAbs = path.join(shardAbs, entry.name) + let legs: string[] + try { + legs = await fs.promises.readdir(idAbs) + } catch (error: any) { + if (error?.code === 'ENOENT') continue + throw error + } + // A live entity has its metadata content leg. No content leg → a + // vector-only ghost or an empty scar → prune the whole container. + if (legs.some((f) => f.startsWith('metadata.json'))) continue + await fs.promises.rm(idAbs, { recursive: true, force: true }) + pruned[kind].push(entry.name) + console.warn( + `[FileSystemStorage] pruned orphaned ${kind === 'nouns' ? 'noun' : 'verb'} ` + + `container ${root}/${shard}/${entry.name} (no metadata content leg)` + ) + } + } + } + + return pruned + } + /** * Primitive operation: List objects under path prefix * All metadata operations use this internally via base class routing @@ -595,6 +701,17 @@ export class FileSystemStorage extends BaseStorage { */ private static readonly SNAPSHOT_BYTE_COPY_DIRS = new Set(['_id_mapper']) + /** + * Nested path PREFIXES whose files are byte-copied into snapshots, not + * hard-linked — for append-in-place files below the top level. The + * generation fact log's tail segment is appended in place between rotations; + * a hard-linked tail would let post-snapshot appends reach through into the + * snapshot. (Sealed segments are immutable and would be link-safe, but the + * prefix rule keeps the discipline simple; segments are bounded by the + * rotation threshold, so the copy cost is small.) + */ + private static readonly SNAPSHOT_BYTE_COPY_PREFIXES: string[] = ['_generations/facts/'] + /** * Top-level directories excluded from snapshots: process-local lock state * (writer lock, flush-request RPC files) must never travel with the data, and @@ -630,6 +747,9 @@ export class FileSystemStorage extends BaseStorage { */ public override async removeRawPrefix(prefix: string): Promise { await this.ensureInitialized() + // Registered-blob contract: a prefix-nuke must not take out a protected + // family member. Throws if the prefix intersects one. + await this.assertPrefixNotProtected(prefix) await fs.promises.rm(path.join(this.rootDir, prefix), { recursive: true, force: true }) } @@ -769,6 +889,75 @@ export class FileSystemStorage extends BaseStorage { } } + // ========================================================================== + // Binary raw-byte primitives — the substrate for append-only log-structured + // files (the generation fact log's CRC-framed segments). Paths are used + // VERBATIM (no .gz/.bin suffixing). Append durability rides syncRawObjects + // at the commit barrier, like every other staged write. + // ========================================================================== + + /** + * Append bytes to a raw binary file, creating it (and parent directories) + * when absent. NOT fsync'd here — the caller batches durability via + * `syncRawObjects` at its commit barrier. + */ + public async appendRawBytes(rawPath: string, bytes: Uint8Array): Promise { + await this.ensureInitialized() + const fullPath = path.join(this.rootDir, rawPath) + await fs.promises.mkdir(path.dirname(fullPath), { recursive: true }) + await fs.promises.appendFile(fullPath, bytes) + } + + /** + * Read a raw binary file whole. Absent → `null`; a real IO fault throws — + * a present-but-unreadable log segment must never read as "no facts". + */ + public async readRawBytes(rawPath: string): Promise { + await this.ensureInitialized() + try { + const buf: Buffer = await fs.promises.readFile(path.join(this.rootDir, rawPath)) + return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength) + } catch (error: any) { + if (isAbsentError(error)) return null + throw error + } + } + + /** + * Replace a raw binary file atomically: write-new → fsync → rename. The + * reconcile primitive (e.g. truncating a fact-log tail back to committed + * truth after a crash) — a crash mid-replace leaves either the old file or + * the new one, never a mix. + */ + public async writeRawBytes(rawPath: string, bytes: Uint8Array): Promise { + await this.ensureInitialized() + const fullPath = path.join(this.rootDir, rawPath) + await fs.promises.mkdir(path.dirname(fullPath), { recursive: true }) + const tmpPath = `${fullPath}.tmp.${Date.now()}.${Math.random().toString(36).slice(2)}` + const handle = await fs.promises.open(tmpPath, 'w') + try { + await handle.writeFile(bytes) + await handle.sync() + } finally { + await handle.close() + } + await fs.promises.rename(tmpPath, fullPath) + } + + /** + * Byte size of a raw binary file, or `null` when absent. + */ + public async rawByteSize(rawPath: string): Promise { + await this.ensureInitialized() + try { + const stat = await fs.promises.stat(path.join(this.rootDir, rawPath)) + return stat.size + } catch (error: any) { + if (isAbsentError(error)) return null + throw error + } + } + /** * Snapshot the entire store into `targetPath` as a hard-link farm * (Cassandra-style: instant, space-shared). Safe because every data file @@ -818,7 +1007,8 @@ export class FileSystemStorage extends BaseStorage { // msync/truncate reach through into the snapshot. if ( FileSystemStorage.SNAPSHOT_BYTE_COPY_PATHS.has(normalized) || - FileSystemStorage.SNAPSHOT_BYTE_COPY_DIRS.has(normalized.split('/')[0]) + FileSystemStorage.SNAPSHOT_BYTE_COPY_DIRS.has(normalized.split('/')[0]) || + FileSystemStorage.SNAPSHOT_BYTE_COPY_PREFIXES.some((p) => normalized.startsWith(p)) ) { await fs.promises.copyFile(sourceFile, targetFile) continue @@ -1237,7 +1427,7 @@ export class FileSystemStorage extends BaseStorage { // errors, never quiet losses. Fault-propagation restored lockstep with // cortex 3.0.13, whose two column-store call sites now handle the throw // (they mark the field unavailable + throw a named error) instead of - // relying on null-on-error (CORTEX-BILLION-SCALE-SPINE). + // relying on null-on-error. if (isAbsentError(err)) return null throw err } @@ -1250,6 +1440,11 @@ export class FileSystemStorage extends BaseStorage { */ public async deleteBinaryBlob(key: string): Promise { await this.ensureInitialized() + // Registered-blob contract: refuse to delete a declared + // derived-index family member — an in-process GC/sweeper cannot remove a + // load-bearing index file. Throws ProtectedArtifactError; no-op when no + // families are registered. + await this.assertBlobKeyDeletable(key) try { await fs.promises.unlink(this.blobPath(key)) } catch { @@ -1577,79 +1772,158 @@ export class FileSystemStorage extends BaseStorage { const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE) const os = await import('node:os') const hostname = os.hostname() - const now = new Date().toISOString() + const myPid = typeof process !== 'undefined' && process.pid ? process.pid : 0 - const existing = await this.readWriterLock() - if (existing && !options?.force) { - // Same-process re-open: a second Brainy instance in this Node process - // (e.g. test "simulate server restart" patterns, or a consumer that - // explicitly re-instantiates without closing first). This isn't the - // dangerous cross-process case the lock exists to prevent — the two - // instances share a memory space and can't silently diverge from each - // other beyond what their callers already see. Warn and take over the lock. - if (existing.pid === (typeof process !== 'undefined' ? process.pid : 0) && - existing.hostname === hostname) { - console.warn( - `[brainy] Re-acquiring writer lock for ${this.rootDir} held by the same process (PID ${existing.pid}). ` + - `If you intended to keep the previous Brainy instance alive, this is a bug — close it first.` - ) - } else { - const stale = await this.isWriterLockStale(existing) - if (!stale) { + // Bounded acquire loop. The CLAIM itself is an atomic create-exclusive + // write (O_EXCL) — two processes racing an ABSENT lock can never both + // succeed, which closes the read-then-write window where the loser used + // to keep running unlocked, silently. An EEXIST loser loops, re-reads, + // and handles whatever it finds honestly (fresh foreign lock → loud + // throw; stale/forced → verified takeover). + const MAX_ATTEMPTS = 3 + for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) { + const now = new Date().toISOString() + const existing = await this.readWriterLock() + + if (existing) { + // Same-process re-open: a second Brainy instance in this Node process + // (e.g. test "simulate server restart" patterns, or a consumer that + // explicitly re-instantiates without closing first). This isn't the + // dangerous cross-process case the lock exists to prevent — the two + // instances share a memory space and can't silently diverge from each + // other beyond what their callers already see. Warn and take over. + if (existing.pid === myPid && existing.hostname === hostname && !options?.force) { + console.warn( + `[brainy] Re-acquiring writer lock for ${this.rootDir} held by the same process (PID ${existing.pid}). ` + + `If you intended to keep the previous Brainy instance alive, this is a bug — close it first.` + ) + const info: WriterLockInfo = { + pid: myPid, + hostname, + startedAt: now, + lastHeartbeat: now, + version: getBrainyVersion(), + rootDir: this.rootDir + } + await this.writeFileAtomic(lockFile, JSON.stringify(info, null, 2)) + this.installWriterLock(info) + return info + } + + const stale = !options?.force && (await this.isWriterLockStale(existing)) + if (!options?.force && !stale) { // Consumer-facing error contract: callers detect this case via // err.code and read the holder's details from err.lockInfo. - const err = new Error( - `Another writer holds this Brainy directory.\n` + - ` PID: ${existing.pid} on host ${existing.hostname}\n` + - ` Started: ${existing.startedAt}\n` + - ` Heartbeat: ${existing.lastHeartbeat}\n` + - ` Version: ${existing.version}\n` + - ` Directory: ${this.rootDir}\n\n` + - `For diagnostic queries against this live store, use:\n` + - ` const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: '${this.rootDir}' } })\n\n` + - `If you have verified the existing lock is stale (e.g. a crashed writer on a different host that PID liveness cannot reach), pass { force: true }.` - ) as Error & { code: string; lockInfo: WriterLockInfo } - err.code = 'BRAINY_WRITER_LOCKED' - err.lockInfo = existing - throw err + throw this.writerLockedError(existing) } + console.warn( - `[brainy] Overwriting stale writer lock for ${this.rootDir} ` + - `(PID ${existing.pid} on ${existing.hostname} appears dead).` + options?.force + ? `[brainy] Force-overwriting writer lock for ${this.rootDir} ` + + `(was held by PID ${existing.pid} on ${existing.hostname}).` + : `[brainy] Overwriting stale writer lock for ${this.rootDir} ` + + `(PID ${existing.pid} on ${existing.hostname} appears dead).` ) + // Takeover: verify the file still holds the lock we judged (a live + // successor may have claimed meanwhile), then remove it and fall + // through to the atomic claim below. A racing claimer who beats us to + // the create simply wins — our next loop iteration reads their fresh + // lock and throws honestly. (Advisory file locking has no + // compare-and-delete; staleness requiring a 60s-old heartbeat keeps + // the residual verify-to-unlink window practically unreachable.) + const recheck = await this.readWriterLock() + if ( + recheck && + (recheck.pid !== existing.pid || + recheck.startedAt !== existing.startedAt || + recheck.lastHeartbeat !== existing.lastHeartbeat) + ) { + continue // the lock changed hands while we deliberated — re-evaluate + } + try { + await fs.promises.unlink(lockFile) + } catch (err: any) { + if (err.code !== 'ENOENT') throw err + } } - } else if (existing && options?.force) { - console.warn( - `[brainy] Force-overwriting writer lock for ${this.rootDir} ` + - `(was held by PID ${existing.pid} on ${existing.hostname}).` - ) + + const info: WriterLockInfo = { + pid: myPid, + hostname, + startedAt: existing && options?.force ? existing.startedAt : now, + lastHeartbeat: now, + version: getBrainyVersion(), + rootDir: this.rootDir + } + + // The atomic claim: create-exclusive, so exactly ONE racer wins. + try { + await fs.promises.writeFile(lockFile, JSON.stringify(info, null, 2), { flag: 'wx' }) + } catch (err: any) { + if (err.code === 'EEXIST') { + continue // someone else claimed between our read and create — re-evaluate + } + throw err + } + + this.installWriterLock(info) + return info } - const info: WriterLockInfo = { - pid: typeof process !== 'undefined' && process.pid ? process.pid : 0, - hostname, - startedAt: existing && options?.force ? existing.startedAt : now, - lastHeartbeat: now, - version: getBrainyVersion(), - rootDir: this.rootDir - } + // Attempts exhausted: something is claiming this directory faster than we + // can evaluate it. Read whoever holds it now and fail loudly with their + // details rather than degrading into a lockless open. + const holder = await this.readWriterLock() + if (holder) throw this.writerLockedError(holder) + throw new Error( + `Failed to acquire the writer lock for ${this.rootDir} after ${MAX_ATTEMPTS} attempts — ` + + `the lock file is being contended. Retry, or inspect ${lockFile}.` + ) + } - await this.writeFileAtomic(lockFile, JSON.stringify(info, null, 2)) + /** Record lock ownership + start the unref'd heartbeat. */ + private installWriterLock(info: WriterLockInfo): void { this.writerLockInfo = info // Heartbeat — rewrite lastHeartbeat every WRITER_HEARTBEAT_MS so other // processes can tell a live writer from one that crashed without releasing. this.writerLockHeartbeat = setInterval(() => { - this.refreshWriterLockHeartbeat().catch((err) => { - console.warn('[brainy] Failed to refresh writer lock heartbeat:', err) + const tick = this.refreshWriterLockHeartbeat().catch((err) => { + // ENOENT = the lock (or its directory) vanished mid-refresh — the + // store was released or removed under us; the next acquire recreates + // it. Benign by construction; anything else stays loud. + if ((err as NodeJS.ErrnoException)?.code !== 'ENOENT') { + console.warn('[brainy] Failed to refresh writer lock heartbeat:', err) + } + }) + this.writerHeartbeatInFlight = tick.finally(() => { + if (this.writerHeartbeatInFlight === tick) { + this.writerHeartbeatInFlight = undefined + } }) }, FileSystemStorage.WRITER_HEARTBEAT_MS) if (typeof this.writerLockHeartbeat.unref === 'function') { // Don't keep the event loop alive just for the heartbeat. this.writerLockHeartbeat.unref() } + } - return info + /** The consumer-facing BRAINY_WRITER_LOCKED error, holder details attached. */ + private writerLockedError(existing: WriterLockInfo): Error { + const err = new Error( + `Another writer holds this Brainy directory.\n` + + ` PID: ${existing.pid} on host ${existing.hostname}\n` + + ` Started: ${existing.startedAt}\n` + + ` Heartbeat: ${existing.lastHeartbeat}\n` + + ` Version: ${existing.version}\n` + + ` Directory: ${this.rootDir}\n\n` + + `For diagnostic queries against this live store, use:\n` + + ` const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: '${this.rootDir}' } })\n\n` + + `If you have verified the existing lock is stale (e.g. a crashed writer on a different host that PID liveness cannot reach), pass { force: true }.` + ) as Error & { code: string; lockInfo: WriterLockInfo } + err.code = 'BRAINY_WRITER_LOCKED' + err.lockInfo = existing + return err } public override async releaseWriterLock(): Promise { @@ -1657,6 +1931,15 @@ export class FileSystemStorage extends BaseStorage { clearInterval(this.writerLockHeartbeat) this.writerLockHeartbeat = undefined } + // Drain an in-flight heartbeat tick BEFORE unlinking: clearInterval stops + // future ticks only, and a straggler write landing after the unlink would + // re-create the lock as a phantom (blocking the next writer until the + // stale TTL). After the drain, any refresh is either fully landed (we + // unlink its output below) or not started (it sees writerLockInfo + // undefined and returns). + if (this.writerHeartbeatInFlight) { + await this.writerHeartbeatInFlight + } if (!this.writerLockInfo) { return } diff --git a/src/storage/adapters/memoryStorage.ts b/src/storage/adapters/memoryStorage.ts index 05dfb0b4..1b1f412e 100644 --- a/src/storage/adapters/memoryStorage.ts +++ b/src/storage/adapters/memoryStorage.ts @@ -133,6 +133,11 @@ export class MemoryStorage extends BaseStorage { */ protected async deleteObjectFromPath(path: string): Promise { this.objectStore.delete(path) + // Filesystem parity: on disk, objects and raw BYTE files are both just + // files — unlink removes whichever exists. Without this, deleteRawObject + // on a raw-bytes path (fact-log/generation segments) silently no-ops on + // memory storage: the delete "succeeds" and the bytes remain. + this.rawBytesStore.delete(path) } /** @@ -183,6 +188,9 @@ export class MemoryStorage extends BaseStorage { * @param key - The blob key. */ public async deleteBinaryBlob(key: string): Promise { + // Registered-blob contract — parity with the filesystem adapter: + // a declared family member is undeletable (throws ProtectedArtifactError). + await this.assertBlobKeyDeletable(key) this.blobStore.delete(key) } @@ -219,6 +227,45 @@ export class MemoryStorage extends BaseStorage { return [...this.txLogLines] } + // =========================================================================== + // Binary raw-byte primitives — in-memory mirror of the filesystem adapter's + // append-only substrate (the generation fact log's segments), so memory + // brains dual-write facts too and the compat suite runs on both adapters. + // =========================================================================== + + /** Raw binary files, keyed by verbatim path. */ + private rawBytesStore: Map = new Map() + + /** Append bytes to a raw binary file, creating it when absent. */ + public async appendRawBytes(rawPath: string, bytes: Uint8Array): Promise { + const existing = this.rawBytesStore.get(rawPath) + if (!existing) { + this.rawBytesStore.set(rawPath, bytes.slice()) + return + } + const merged = new Uint8Array(existing.length + bytes.length) + merged.set(existing, 0) + merged.set(bytes, existing.length) + this.rawBytesStore.set(rawPath, merged) + } + + /** Read a raw binary file whole (a copy); absent → null. */ + public async readRawBytes(rawPath: string): Promise { + const bytes = this.rawBytesStore.get(rawPath) + return bytes ? bytes.slice() : null + } + + /** Replace a raw binary file (atomic by construction in memory). */ + public async writeRawBytes(rawPath: string, bytes: Uint8Array): Promise { + this.rawBytesStore.set(rawPath, bytes.slice()) + } + + /** Byte size of a raw binary file, or null when absent. */ + public async rawByteSize(rawPath: string): Promise { + const bytes = this.rawBytesStore.get(rawPath) + return bytes ? bytes.length : null + } + /** * Serialize the entire in-memory store to a directory in the exact layout * the filesystem adapter uses (uncompressed JSON objects, `_blobs/*.bin` @@ -351,6 +398,7 @@ export class MemoryStorage extends BaseStorage { public async clear(): Promise { this.objectStore.clear() this.blobStore.clear() + this.rawBytesStore.clear() this.txLogLines = [] this.statistics = null diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index 159a0593..6daf09c0 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -15,7 +15,8 @@ import { VerbMetadata, HNSWNounWithMetadata, HNSWVerbWithMetadata, - StatisticsData + StatisticsData, + DerivedFamilyDeclaration } from '../coreTypes.js' import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js' import { validateNounType, validateVerbType } from '../utils/typeValidation.js' @@ -31,7 +32,7 @@ import { BlobStorage, type BlobStoreAdapter } from './blobStorage.js' import { unwrapBinaryData } from './binaryDataCodec.js' import { prodLog } from '../utils/logger.js' import { isAbsentError } from '../utils/errorClassification.js' -import { BrainyError } from '../errors/brainyError.js' +import { BrainyError, ProtectedArtifactError, DerivedArtifactMissingError } from '../errors/brainyError.js' import { MetadataWriteBuffer } from '../utils/metadataWriteBuffer.js' import { splitNounMetadataRecord, @@ -251,6 +252,25 @@ export abstract class BaseStorage extends BaseStorageAdapter { protected isInitialized = false /** One-shot guard so the graph fast-path cold-load probe runs once per adapter. */ private _graphFastPathProbed = false + /** + * Registered-blob contract: declared derived-index families, keyed + * by name. Members are undeletable through the blob delete seams. Loaded lazily + * from `_system/derived-artifacts.json` and re-persisted on every change. + */ + private _derivedFamilies = new Map() + /** One-shot guard for loading the persisted family registry. */ + private _derivedFamiliesLoaded = false + /** Storage-root-relative path of the persisted family registry. */ + private static readonly DERIVED_FAMILIES_KEY = '_system/derived-artifacts.json' + /** + * Bounded concurrency for hydrating enumerated nouns during a pagination walk + * (readCanonicalObject + getNounMetadata per item). A canonical enumeration — + * which every index heal performs — otherwise pays N×per-op-latency serially; + * 16-way matches the wave the native rebuild uses so heal wall-clock is + * ~2×N/16×per-op instead of N×per-op. Bounded so a huge dataset can't spawn a + * read per entity at once. + */ + private static readonly HYDRATE_CONCURRENCY = 16 protected graphIndex?: GraphAdjacencyIndex protected graphIndexPromise?: Promise /** @@ -357,7 +377,14 @@ export abstract class BaseStorage extends BaseStorageAdapter { id.startsWith('statistics_') || id === 'statistics' || id.startsWith('__chunk__') || // Metadata index chunks (roaring bitmap data) - id.startsWith('__sparse_index__') // Metadata sparse indices (zone maps + bloom filters) + id.startsWith('__sparse_index__') || // Metadata sparse indices (zone maps + bloom filters) + id.startsWith('__aggregation_') || // Aggregation engine definitions + state (routing is + // identical to the unknown-key fallback these keys hit + // before being listed here — this only kills the + // per-boot "Unknown key format" warning) + isSingletonSystemKey(id) // Known singletons (e.g. brainy:entityIdMapper) hit the + // same warn-then-route fallback without this — the + // routing below already handles them identically if (isSystemKey) { if (isSingletonSystemKey(id)) { @@ -822,6 +849,82 @@ export abstract class BaseStorage extends BaseStorageAdapter { return this.deleteObjectFromPath(path) } + // ========================================================================== + // Fact-scan capability — the seam through which an index provider holding + // only `storage` reaches the generation fact log. The HOST brain wires the + // source at init (a closure over its live fact log, so restore/reopen stays + // transparent); providers call storage.scanFacts?.(...) and fall back to the + // enumeration walk when it returns null. Providers never construct a + // fact-log reader themselves — the log's open path is writer-side. + // ========================================================================== + + /** The host-wired fact-scan source (closures over the live generation state). */ + private _factScanSource: { + factLog: () => import('../db/factLog.js').FactLog | null + committedGeneration: () => number + } | null = null + + /** HOST-ONLY: wire (or clear) the fact-scan capability's source. */ + public setFactScanSource( + source: { + factLog: () => import('../db/factLog.js').FactLog | null + committedGeneration: () => number + } | null + ): void { + this._factScanSource = source + } + + /** Open a scan over committed facts, or `null` when no fact log exists. */ + public scanFacts(options?: { + fromGeneration?: number + toGeneration?: number + kinds?: Array<'noun' | 'verb'> + batchSize?: number + }): import('../db/factLog.js').FactScanHandle | null { + const log = this._factScanSource?.factLog() + return log ? log.scanFacts(options) : null + } + + /** The fact log's head generation, or `null` when no fact log exists. */ + public factLogHeadGeneration(): number | null { + const log = this._factScanSource?.factLog() + return log ? log.headGeneration() : null + } + + /** + * The COMMITTED generation watermark (the manifest truth) — the value a + * projection's `sourceGeneration` compares against. Exposed here so a + * provider never parses the store's private manifest format. `null` when + * the capability is unwired (no host, or a bare adapter). + */ + public committedGeneration(): number | null { + return this._factScanSource ? this._factScanSource.committedGeneration() : null + } + + /** Sealed fact-segment paths (zero-copy handoff); empty when none. */ + public factSegmentPaths(options?: { fromGeneration?: number }): string[] { + const log = this._factScanSource?.factLog() + return log ? log.segmentPaths(options) : [] + } + + /** + * @description Remove the container that held a canonical entity's leg files, + * called after both legs are deleted so a delete leaves NOTHING behind — no + * orphan "scar" directory. `objectLegPath` is any one of the entity's leg + * paths (e.g. its `vectors.json`); the container is that path's parent. + * + * Default: a no-op. Key/prefix-addressed stores (in-memory, cloud object + * stores) have no empty-container concept — deleting the leg keys already + * removes the entity entirely. The filesystem adapter overrides this to + * `rmdir` the emptied entity directory. + * + * @param objectLegPath - Storage-root-relative path of one of the entity's legs. + * @protected + */ + protected async removeCanonicalContainer(objectLegPath: string): Promise { + void objectLegPath + } + /** * @description List canonical objects under a storage-root-relative prefix. * @@ -1041,6 +1144,154 @@ export abstract class BaseStorage extends BaseStorageAdapter { await this.writeObjectToPath(path, data) } + // ========================================================================== + // Registered-blob contract — declared derived-index families are + // undeletable through the blob delete seams. Shared here so every adapter that + // extends BaseStorage inherits the same enforcement; the concrete + // deleteBinaryBlob / removeRawPrefix call assertBlobKeyDeletable / + // assertPrefixNotProtected before removing anything. + // ========================================================================== + + /** Load the persisted family registry once (lazy). */ + private async ensureDerivedFamiliesLoaded(): Promise { + if (this._derivedFamiliesLoaded) return + const stored = await this.readRawObject(BaseStorage.DERIVED_FAMILIES_KEY).catch(() => null) + const families = (stored as { families?: DerivedFamilyDeclaration[] } | null)?.families + if (Array.isArray(families)) { + for (const f of families) { + if (f && typeof f.name === 'string' && Array.isArray(f.members)) this._derivedFamilies.set(f.name, f) + } + } + this._derivedFamiliesLoaded = true + } + + /** Persist the current family registry (fsync'd via the raw-object write). */ + private async persistDerivedFamilies(): Promise { + await this.writeRawObject(BaseStorage.DERIVED_FAMILIES_KEY, { + families: [...this._derivedFamilies.values()] + }) + } + + public async registerDerivedFamily(family: DerivedFamilyDeclaration): Promise { + await this.ensureInitialized() + await this.ensureDerivedFamiliesLoaded() + this._derivedFamilies.set(family.name, { + name: family.name, + members: [...family.members], + ...(family.namespace !== undefined ? { namespace: family.namespace } : {}), + ...(family.rebuildable !== undefined ? { rebuildable: family.rebuildable } : {}) + }) + await this.persistDerivedFamilies() + } + + public async unregisterDerivedFamily(name: string): Promise { + await this.ensureInitialized() + await this.ensureDerivedFamiliesLoaded() + if (this._derivedFamilies.delete(name)) await this.persistDerivedFamilies() + } + + public async listDerivedFamilies(): Promise { + await this.ensureInitialized() + await this.ensureDerivedFamiliesLoaded() + return [...this._derivedFamilies.values()] + } + + /** + * @description A blob key that is a transient write-scratch file (a `*.tmp.*` + * temp, a `*.rebuild-tmp`, a `*.rotate-tmp`). Its OWNER renames/removes it as + * part of an atomic write; it is never a protected family member. + */ + private isTransientBlobKey(key: string): boolean { + return /\.rebuild-tmp$|\.rotate-tmp$|\.tmp(\.|$)/.test(key) + } + + /** + * @description The name of the protected family a blob key belongs to, or + * `null`. A `namespace` member protects every key beneath it; a plain member + * protects that exact key. + */ + private protectingFamilyOf(key: string): string | null { + for (const family of this._derivedFamilies.values()) { + for (const member of family.members) { + if (family.namespace ? key === member || key.startsWith(member) : key === member) { + return family.name + } + } + } + return null + } + + /** + * @description Enforcement point for `deleteBinaryBlob`: refuse (throw + * {@link ProtectedArtifactError}) when the key is a declared family member. + * Transients pass through. An undeclared blob delete under an ACTIVE contract + * (families are registered) is logged loudly — nothing under `_blobs/` should + * vanish unremarked once the contract is in force. + */ + protected async assertBlobKeyDeletable(key: string): Promise { + await this.ensureDerivedFamiliesLoaded() + if (this._derivedFamilies.size === 0) return // contract inactive — no enforcement + if (this.isTransientBlobKey(key)) return + const family = this.protectingFamilyOf(key) + if (family) throw new ProtectedArtifactError(key, family) + prodLog.warn( + `[BaseStorage] deleteBinaryBlob('${key}') removes an UNDECLARED blob while the ` + + `registered-blob contract is active — permitted, but surfaced so no _blobs/ file ` + + `disappears silently.` + ) + } + + /** + * @description Enforcement point for `removeRawPrefix`: refuse when the prefix + * would take out a protected family member (a prefix-nuke must not remove a + * declared blob). Transients are ignored. + */ + protected async assertPrefixNotProtected(prefix: string): Promise { + await this.ensureDerivedFamiliesLoaded() + if (this._derivedFamilies.size === 0) return + for (const family of this._derivedFamilies.values()) { + for (const member of family.members) { + // Under the shared `_blobs/` root, member keys resolve beneath it; a + // prefix intersects a member when either contains the other. + const memberBlobPath = `_blobs/${member}` + if ( + memberBlobPath.startsWith(prefix) || + prefix.startsWith(memberBlobPath) || + member.startsWith(prefix) || + prefix.startsWith(member) + ) { + if (!this.isTransientBlobKey(member)) throw new ProtectedArtifactError(member, family.name) + } + } + } + } + + /** + * @description Verify every declared family has all its members present on + * disk (missing-on-open catch for an EXTERNAL deleter that bypasses + * the in-process refusal). Returns the incomplete families (name + missing + * members) — the caller decides how to heal (rebuild from canonical). Loud by + * construction: a missing load-bearing blob is named, never silently tolerated. + */ + public async checkDerivedFamiliesPresent(): Promise> { + await this.ensureInitialized() + await this.ensureDerivedFamiliesLoaded() + const incomplete: Array<{ name: string; missing: string[]; rebuildable: boolean }> = [] + for (const family of this._derivedFamilies.values()) { + if (family.namespace) continue // a growing prefix has no fixed member set to verify + const missing: string[] = [] + for (const member of family.members) { + const blob = await this.loadBinaryBlob(member).catch(() => null) + if (!blob) missing.push(member) + } + if (missing.length > 0) { + prodLog.warn(new DerivedArtifactMissingError(family.name, missing).message) + incomplete.push({ name: family.name, missing, rebuildable: family.rebuildable !== false }) + } + } + return incomplete + } + /** * Delete a raw object at a storage-root-relative path (no-op if absent). * @@ -1224,6 +1475,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { */ protected async reloadDerivedState(): Promise { this.clearWriteCache() + // Registered-blob registry: clear() wipes the persisted `_system/` copy and + // the family members under `_blobs/`, so drop the in-memory cache too — a + // reset brain must not keep stale family protection or report their members + // "missing" on the next check. + this._derivedFamilies.clear() + this._derivedFamiliesLoaded = false this.nounCountsByType.fill(0) this.verbCountsByType.fill(0) this.subtypeCountsByType.clear() @@ -1424,19 +1681,27 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Delete a noun from storage */ - public async deleteNoun(id: string): Promise { + public async deleteNoun(id: string, priorMetadata?: NounMetadata | null): Promise { await this.ensureInitialized() - // Delete both the vector file and metadata file (2-file system) - await this.deleteNoun_internal(id) + // FULL removal (live-HEAD hygiene): remove BOTH canonical legs AND the + // entity's container, so a delete leaves nothing behind — no orphan/scar + // directory to inflate the enumerated count or ghost resolveLocator. The + // generation store holds the immutable before-image, so asOf() still + // reconstructs the deleted entity until retention expires. + await this.deleteNoun_internal(id) // vectors.json leg - // Delete metadata file (if it exists) - try { - await this.deleteNounMetadata(id) - } catch (error) { - // Ignore if metadata file doesn't exist - prodLog.debug(`No metadata file to delete for noun ${id}`) - } + // Metadata leg + count decrement. A genuine "already absent" is a no-op + // downstream (readCanonicalObject / deleteObjectFromPath both treat ENOENT as + // success); a REAL fault must surface loudly (loud errors, never quiet + // losses) and must never silently skip the count decrement — so this is NO + // LONGER wrapped in a blind catch that masked faults as "file didn't exist". + // `priorMetadata` (the caller's pre-delete read) keeps the decrement honest + // even when the canonical read inside returns null (replace race / ghost). + await this.deleteNounMetadata(id, priorMetadata) + + // Remove the now-empty entity container (a no-op for key/prefix stores). + await this.removeCanonicalContainer(getNounVectorPath(id)) } /** @@ -1803,11 +2068,20 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Cursor (8.0): resume token carrying the (shard, nounId) of the last returned // noun — the noun mirror of getVerbsWithPagination. When present it supersedes // `offset` and resumes the shard walk immediately AFTER that position, so a full - // walk is O(N) instead of the O(N²) of offset paging. Malformed/foreign tokens - // decode to null → offset fallback. (Previously the cursor was ignored, which - // was latent — the only multi-page consumer used a single big page — until small - // chunk sizes needed page 2 and an offset-0-on-every-call walk never terminated.) + // walk is O(N) instead of the O(N²) of offset paging. (Previously the cursor was + // ignored, which was latent — the only multi-page consumer used a single big + // page — until small chunk sizes needed page 2 and an offset-0-on-every-call + // walk never terminated.) const cursor = this.decodeNounWalkCursor(options.cursor) + if (options.cursor && cursor === null) { + // A supplied-but-undecodable resume token must FAIL, not silently restart + // at offset 0 — to a while(hasMore) caller the silent fallback re-serves + // page 1 forever: an unbounded CPU loop wearing pagination's clothes. + throw BrainyError.storage( + `getNouns: invalid pagination cursor '${options.cursor}' — cannot resume this walk. ` + + `Restart it without a cursor.` + ) + } const collected: Array<{ noun: HNSWNounWithMetadata; shard: number }> = [] // Peek one past the window so `hasMore` is decidable. Cursor mode collects one @@ -1830,39 +2104,63 @@ export abstract class BaseStorage extends BaseStorageAdapter { .map((p) => ({ path: p, id: idFromVectorPath(p) })) .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)) - for (const { path: nounPath, id: nounId } of entries) { - if (collected.length >= peekCount) break - // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor id. - if (cursor && shard === cursor.shard && nounId <= cursor.id) continue + // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor + // id BEFORE hydrating — a cheap id compare (from the path), so skipped + // nouns are never read. + const toHydrate = + cursor && shard === cursor.shard + ? entries.filter((e) => e.id > cursor.id) + : entries - try { - const noun = await this.readCanonicalObject(nounPath) - if (!noun) continue - const deserialized = this.deserializeNoun(noun) - const metadata = await this.getNounMetadata(deserialized.id) - if (!metadata) continue + // Hydrate in bounded-concurrency batches (16-way) instead of one-at-a-time. + // Every canonical enumeration — and every index heal enumerates canonical — + // otherwise pays N×per-op-latency SERIALLY (the dominant heal-time term). + // Order is preserved (the batch is a slice of the sorted entries and its + // results are consumed in order), so offset windows / cursor resume stay + // deterministic. peekCount stops the walk; the final batch over-hydrates by + // at most BASE_STORAGE_HYDRATE_CONCURRENCY entries (bounded, acceptable). + for ( + let i = 0; + i < toHydrate.length && collected.length < peekCount; + i += BaseStorage.HYDRATE_CONCURRENCY + ) { + const batch = toHydrate.slice(i, i + BaseStorage.HYDRATE_CONCURRENCY) + const hydrated = await Promise.all( + batch.map(async ({ path: nounPath }) => { + try { + const noun = await this.readCanonicalObject(nounPath) + if (!noun) return null + const deserialized = this.deserializeNoun(noun) + const metadata = await this.getNounMetadata(deserialized.id) + if (!metadata) return null + return { deserialized, metadata } + } catch (error) { + // Skip nouns that fail to load + return null + } + }) + ) + + for (const h of hydrated) { + if (collected.length >= peekCount) break + if (!h) continue + const { deserialized, metadata } = h // Apply type filter if (filter?.nounType && metadata.noun) { const types = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType] - if (!types.includes(metadata.noun)) { - continue - } + if (!types.includes(metadata.noun)) continue } // Apply service filter if (filter?.service) { const services = Array.isArray(filter.service) ? filter.service : [filter.service] - if (metadata.service && !services.includes(metadata.service)) { - continue - } + if (metadata.service && !services.includes(metadata.service)) continue } // Combine noun + metadata via the canonical hydration helper — // reserved fields top-level, ONLY custom fields in `metadata`. collected.push({ noun: this.hydrateNounWithMetadata(deserialized, metadata), shard }) - } catch (error) { - // Skip nouns that fail to load } } } catch (error) { @@ -1902,6 +2200,119 @@ export abstract class BaseStorage extends BaseStorageAdapter { } } + /** + * @description Enumerate noun IDS ONLY, without hydrating each entity's vector + * and metadata — the opt-out for callers (e.g. an index heal) that own their + * own IO schedule and index straight from a stream. Shares the exact + * shard-walk, cursor, offset and `nextCursor` contract of + * {@link getNounsWithPagination}, so the two are page-compatible. + * + * - UNFILTERED (the heal case): ids come straight from the shard paths — ZERO + * per-entity reads. A full enumeration is O(entries listed), not O(N reads). + * - FILTERED: the type/service filter needs metadata, so only the metadata is + * hydrated (16-way bounded concurrency), never the full noun. + * + * @param options - `limit`/`offset`/`cursor`/`filter` — same semantics as + * {@link getNounsWithPagination}. + * @returns The page of ids plus `totalCount` / `hasMore` / `nextCursor`. + */ + public async getNounIdsWithPagination(options: { + limit: number + offset?: number + cursor?: string + filter?: { + nounType?: string | string[] + service?: string | string[] + metadata?: Record + } + }): Promise<{ ids: string[]; totalCount: number; hasMore: boolean; nextCursor?: string }> { + await this.ensureInitialized() + + const { limit, offset = 0, filter } = options + const cursor = this.decodeNounWalkCursor(options.cursor) + if (options.cursor && cursor === null) { + // Same law as getNouns/getVerbs: an undecodable resume token FAILS + // instead of silently restarting the walk at offset 0. + throw BrainyError.storage( + `getNounIds: invalid pagination cursor '${options.cursor}' — cannot resume this walk. ` + + `Restart it without a cursor.` + ) + } + const collected: Array<{ id: string; shard: number }> = [] + const peekCount = cursor ? limit + 1 : offset + limit + 1 + const startShard = cursor ? cursor.shard : 0 + + for (let shard = startShard; shard < 256 && collected.length < peekCount; shard++) { + const shardHex = shard.toString(16).padStart(2, '0') + const shardDir = `entities/nouns/${shardHex}` + try { + const nounFiles = await this.listCanonicalObjects(shardDir) + const entries = nounFiles + .filter((p) => p.includes('/vectors.json')) + .map((p) => idFromVectorPath(p)) + .sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)) + const toWalk = + cursor && shard === cursor.shard ? entries.filter((id) => id > cursor.id) : entries + + if (!filter) { + // Unfiltered — ids straight from the paths, no reads at all. + for (const id of toWalk) { + if (collected.length >= peekCount) break + collected.push({ id, shard }) + } + } else { + // Filtered — hydrate metadata ONLY (16-way) to apply the filter. + for ( + let i = 0; + i < toWalk.length && collected.length < peekCount; + i += BaseStorage.HYDRATE_CONCURRENCY + ) { + const batch = toWalk.slice(i, i + BaseStorage.HYDRATE_CONCURRENCY) + const metas = await Promise.all( + batch.map(async (id) => { + try { + return { id, metadata: await this.getNounMetadata(id) } + } catch { + return null + } + }) + ) + for (const m of metas) { + if (collected.length >= peekCount) break + if (!m || !m.metadata) continue + const metadata = m.metadata + if (filter.nounType && metadata.noun) { + const types = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType] + if (!types.includes(metadata.noun)) continue + } + if (filter.service) { + const services = Array.isArray(filter.service) ? filter.service : [filter.service] + if (metadata.service && !services.includes(metadata.service)) continue + } + collected.push({ id: m.id, shard }) + } + } + } + } catch (error) { + // Skip shards with no data + } + } + + const windowStart = cursor ? 0 : offset + const pagePairs = collected.slice(windowStart, windowStart + limit) + const ids = pagePairs.map((p) => p.id) + const hasMore = collected.length > windowStart + limit + const totalCount = filter ? collected.length : Math.max(this.totalNounCount, collected.length) + + let nextCursor: string | undefined = undefined + if (hasMore && pagePairs.length > 0) { + const lastPair = pagePairs[pagePairs.length - 1] + nextCursor = this.encodeNounWalkCursor(lastPair.shard, lastPair.id) + } + + return { ids, totalCount, hasMore, nextCursor } + } + /** * @description Encode a noun-walk resume cursor — the `(shard, nounId)` of the * last returned noun — as an opaque, version-tagged token (`cn1:` prefix lets @@ -1972,8 +2383,17 @@ export abstract class BaseStorage extends BaseStorageAdapter { // (shard, verbId) of the last returned verb. When present it SUPERSEDES `offset` // and resumes the shard walk immediately AFTER that position, so a full walk is // O(N) total instead of the O(N²) of offset paging (which re-scans from shard 0 - // every page). Malformed / foreign tokens decode to null → offset fallback. + // every page). const cursor = this.decodeVerbWalkCursor(options.cursor) + if (options.cursor && cursor === null) { + // A supplied-but-undecodable resume token must FAIL, not silently restart + // at offset 0 — to a while(hasMore) caller the silent fallback re-serves + // page 1 forever: an unbounded CPU loop wearing pagination's clothes. + throw BrainyError.storage( + `getVerbs: invalid pagination cursor '${options.cursor}' — cannot resume this walk. ` + + `Restart it without a cursor.` + ) + } // Each collected entry remembers its shard so nextCursor can point at the exact // (shard, id) resume position. @@ -2609,19 +3029,16 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Delete a verb from storage */ - public async deleteVerb(id: string): Promise { + public async deleteVerb(id: string, priorMetadata?: VerbMetadata | null): Promise { await this.ensureInitialized() - // Delete both the vector file and metadata file (2-file system) - await this.deleteVerb_internal(id) - - // Delete metadata file (if it exists) - try { - await this.deleteVerbMetadata(id) - } catch (error) { - // Ignore if metadata file doesn't exist - prodLog.debug(`No metadata file to delete for verb ${id}`) - } + // FULL removal (see deleteNoun): both canonical legs + the verb container. + // The blind catch that masked real faults as "no metadata file" is gone — a + // genuine absence is already a no-op downstream; a real fault surfaces loudly. + // `priorMetadata` keeps the count decrement honest on a null internal read. + await this.deleteVerb_internal(id) // vectors.json leg + await this.deleteVerbMetadata(id, priorMetadata) // metadata leg + count decrement + await this.removeCanonicalContainer(getVerbVectorPath(id)) } /** * Get graph index (lazy initialization with concurrent access protection) @@ -3273,19 +3690,32 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Delete noun metadata from storage (ID-first, O(1) delete) + * Delete noun metadata from storage (ID-first, O(1) delete). + * + * @param id - The entity id. + * @param priorRecord - OPTIONAL already-known metadata of the entity being + * removed (e.g. the pre-delete read `remove()` performs, or a captured + * before-image). The count decrement must never REQUIRE re-reading the + * thing being removed: when the canonical read here returns `null` (a + * replace race, or a partial-delete ghost from an earlier version) the + * decrement falls back to this record instead of being silently skipped — + * the skip permanently inflated the persisted totals (adds counted, paired + * removals not decremented), and `Math.max(totalNounCount, scanned)` made + * the inflation unfixable by any disk cleanup. */ - public async deleteNounMetadata(id: string): Promise { + public async deleteNounMetadata(id: string, priorRecord?: NounMetadata | null): Promise { await this.ensureInitialized() // Direct O(1) delete with ID-first path. Read the canonical record BEFORE // removing it: the per-type and subtype decrements are sourced from the // entity's own metadata (`noun` type, `subtype`, `visibility`) rather than an // id-keyed cache, keeping type-statistics honest across deletes — symmetric - // with the increments in `saveNounMetadata_internal()`. + // with the increments in `saveNounMetadata_internal()`. A null read falls + // back to the caller-provided prior record (see @param priorRecord). const path = getNounMetadataPath(id) - const record = await this.readCanonicalObject(path) + const read = await this.readCanonicalObject(path) await this.deleteCanonicalObject(path) + const record = read ?? priorRecord const priorType = record?.noun as NounType | undefined // 8.0 visibility: an internal/system entity was never added to `nounCountsByType` @@ -3459,16 +3889,20 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Delete verb metadata from storage (ID-first, O(1) delete) */ - public async deleteVerbMetadata(id: string): Promise { + public async deleteVerbMetadata(id: string, priorRecord?: VerbMetadata | null): Promise { await this.ensureInitialized() // Direct O(1) delete with ID-first path. Read the canonical record BEFORE // removing it so every decrement is sourced from the edge's own metadata // (`verb` type, `subtype`, `visibility`) rather than an id-keyed cache — - // symmetric with the increments in `saveVerbMetadata_internal()`. + // symmetric with the increments in `saveVerbMetadata_internal()`. A null + // read falls back to the caller-provided prior record: the decrement must + // never REQUIRE re-reading the thing being removed (a silent skip minted + // permanent counter inflation — see deleteNounMetadata). const path = getVerbMetadataPath(id) - const record = await this.readCanonicalObject(path) + const read = await this.readCanonicalObject(path) await this.deleteCanonicalObject(path) + const record = read ?? priorRecord const priorVerb = record?.verb as VerbType | undefined // Symmetric count decrement (previously OMITTED — verb deletes touched neither the @@ -3905,6 +4339,17 @@ export abstract class BaseStorage extends BaseStorageAdapter { this.nounCountsByType = new Uint32Array(NOUN_TYPE_COUNT) this.verbCountsByType = new Uint32Array(VERB_TYPE_COUNT) + // The SAME walk also rebuilds the user-facing scalar totals + per-type maps + // persisted in counts.json (totalNounCount / totalVerbCount / entityCounts / + // verbCounts). Previously only the type-statistics arrays were rebuilt and + // the total was computed just to LOG it — so a drifted persisted scalar + // (deletes whose decrement was skipped) survived every "rebuild" forever, + // and Math.max(totalNounCount, scanned) made the inflation unfixable by any + // disk cleanup. This method is now the SANCTIONED RECOUNT: one canonical + // walk, every counter rollup rebuilt and persisted from it. + const countedNouns = new Map() + const countedVerbs = new Map() + // Scan noun shards for (let shard = 0; shard < 256; shard++) { const shardHex = shard.toString(16).padStart(2, '0') @@ -3925,6 +4370,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { if (typeIndex >= 0 && typeIndex < NOUN_TYPE_COUNT) { this.nounCountsByType[typeIndex]++ } + countedNouns.set(metadata.noun, (countedNouns.get(metadata.noun) || 0) + 1) } } } catch (error) { @@ -3956,6 +4402,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { if (typeIndex >= 0 && typeIndex < VERB_TYPE_COUNT) { this.verbCountsByType[typeIndex]++ } + countedVerbs.set(metadata.verb, (countedVerbs.get(metadata.verb) || 0) + 1) } } } catch (error) { @@ -3972,7 +4419,19 @@ export abstract class BaseStorage extends BaseStorageAdapter { const totalVerbs = this.verbCountsByType.reduce((sum, count) => sum + count, 0) const totalNouns = this.nounCountsByType.reduce((sum, count) => sum + count, 0) - prodLog.info(`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs`) + + // The sanctioned recount half: replace the user-facing scalar totals + the + // per-type maps with the walk's truth and PERSIST them (counts.json), so an + // inflated persisted counter is actually corrected — not merely out-voted + // in memory until the next reopen rehydrates the stale file. + this.entityCounts = countedNouns + this.verbCounts = countedVerbs + this.totalNounCount = totalNouns + this.totalVerbCount = totalVerbs + this.countCache.clear() + await this.persistCounts() + + prodLog.info(`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs (scalar + per-type persisted)`) } /** diff --git a/src/transaction/Transaction.ts b/src/transaction/Transaction.ts index 75dccc05..79b53006 100644 --- a/src/transaction/Transaction.ts +++ b/src/transaction/Transaction.ts @@ -37,6 +37,49 @@ const DEFAULT_OPTIONS: Required = { maxRollbackRetries: 3 } +/** + * The floor term of {@link transactTimeoutBudget}'s scaling formula when the + * caller supplies neither an explicit override nor `config.transactionBudgetFloorMs`. + */ +const DEFAULT_BUDGET_FLOOR_MS = 30_000 + +/** + * The apply-phase budget for a batch of `opCount` operations. + * + * An explicit `override` wins untouched (a caller-specified deadline for this + * one batch). Otherwise the budget SCALES with the batch: + * `max(floorMs, opCount × 2 000)`, where `floorMs` defaults to `30 000` and + * is overridable via `BrainyConfig.transactionBudgetFloorMs` for the whole + * brain. The per-op term is calibrated from field data — bulk imports on + * network-attached disks measure ~2 s per operation (each op pays canonical + * writes + fsync + index maintenance) — so a flat 30 s budget silently capped + * honest work at ~15 operations while looking generous for small batches. A + * 16-op batch, for example, scales to `max(30 000, 16 × 2 000) = 32 000`. + * Scaling keeps small transacts fast-failing and gives bulk ones a budget + * proportional to the work they actually asked for. + * + * This is a **mid-batch** budget only: it governs whether the transaction's + * NEXT operation may start (see {@link Transaction.execute}), never whether + * already-completed work is rolled back after the fact. A trip mid-batch + * still rolls back every operation applied so far, atomically, and throws a + * retryable, fully-labeled TransactionTimeoutError — that zero-loss guarantee + * doesn't change; only the point at which the clock stops mattering does (at + * the last operation, not one check later). + * + * @param opCount - Number of operations in the batch. + * @param override - A full override for this call; wins over everything else. + * @param floorMs - The scaling floor for this call (e.g. `config.transactionBudgetFloorMs`). + * Ignored when `override` is set. Defaults to `30 000` when omitted. + */ +export function transactTimeoutBudget( + opCount: number, + override?: number, + floorMs?: number +): number { + if (override !== undefined) return override + return Math.max(floorMs ?? DEFAULT_BUDGET_FLOOR_MS, opCount * 2_000) +} + /** * Transaction class */ @@ -80,7 +123,20 @@ export class Transaction implements TransactionContext { } /** - * Execute all operations atomically + * Execute all operations atomically. + * + * Budget semantics: the configured budget gates starting the next + * operation; completed work is never rolled back for elapsed time. Elapsed + * time is checked ONLY before starting operation `i` for `i ≥ 1` — never + * before operation 0 (an operation always gets to start) and never again + * after the final operation completes. Concretely: a single-op transaction + * can never time out post-hoc — it either runs (and its result stands, no + * matter how long it took) or a `TransactionTimeoutError` is thrown before + * it starts, which cannot happen since there is no operation before it. A + * multi-op transaction whose op `i` overruns the budget stops op `i+1` from + * starting, rolls back operations `0..i` (reverse order), and throws + * `TransactionTimeoutError` — that zero-loss rollback guarantee for + * mid-batch trips is unchanged; only the post-completion check is gone. */ async execute(): Promise { if (this.state !== 'pending') { @@ -110,11 +166,19 @@ export class Transaction implements TransactionContext { // already-applied writes as torn, generation-less state in canonical // storage. for (let i = 0; i < this.operations.length; i++) { - // Budget check BEFORE starting the next operation. A trip here throws - // into the catch below and rolls back like any other failure — it must - // never bypass rollback. - if (Date.now() - this.startTime > this.options.timeout) { - throw new TransactionTimeoutError(this.options.timeout, i) + // Budget gates STARTING the next operation; completed work is never + // rolled back for elapsed time. Skipped for i === 0 (operation 0 + // always gets to start — nothing has run yet to have overrun) and + // never re-checked after the final operation completes (there is no + // "next operation" left to gate). A trip here throws into the catch + // below and rolls back like any other failure — it must never bypass + // rollback. + if (i > 0 && Date.now() - this.startTime > this.options.timeout) { + throw new TransactionTimeoutError(this.options.timeout, i, { + elapsedMs: Date.now() - this.startTime, + totalOperations: this.operations.length, + operationName: this.operations[i]?.name + }) } const operation = this.operations[i] diff --git a/src/transaction/errors.ts b/src/transaction/errors.ts index c9f6eee1..c270d0ed 100644 --- a/src/transaction/errors.ts +++ b/src/transaction/errors.ts @@ -77,11 +77,24 @@ export class InvalidTransactionStateError extends TransactionError { export class TransactionTimeoutError extends TransactionError { constructor( timeoutMs: number, - operationIndex: number + operationIndex: number, + telemetry?: { + elapsedMs?: number + totalOperations?: number + operationName?: string + } ) { + const progress = + telemetry?.totalOperations !== undefined + ? `${operationIndex}/${telemetry.totalOperations}` + : String(operationIndex) + const name = telemetry?.operationName ? ` ('${telemetry.operationName}')` : '' + const elapsed = + telemetry?.elapsedMs !== undefined ? `${telemetry.elapsedMs}ms elapsed, ` : '' super( - `Transaction timed out after ${timeoutMs}ms at operation ${operationIndex}`, - { timeoutMs, operationIndex } + `Transaction timed out at operation ${progress}${name} — ${elapsed}budget ${timeoutMs}ms. ` + + `The batch rolled back atomically; retry with a higher timeoutMs or a smaller batch.`, + { timeoutMs, operationIndex, ...telemetry } ) this.name = 'TransactionTimeoutError' } diff --git a/src/transaction/operations/IndexOperations.ts b/src/transaction/operations/IndexOperations.ts index 97c39270..d130bb3f 100644 --- a/src/transaction/operations/IndexOperations.ts +++ b/src/transaction/operations/IndexOperations.ts @@ -2,34 +2,73 @@ * Index Operations with Rollback Support * * Provides transactional operations for all indexes: - * - JsHnswVectorIndex (unified vector index) + * - VectorIndexProvider (the JS HNSW fallback, or a native acceleration provider) * - MetadataIndexManager (roaring bitmap filtering) * - GraphAdjacencyIndex (LSM-tree graph storage) * * Each operation can be executed and rolled back atomically. */ -import type { JsHnswVectorIndex } from '../../hnsw/hnswIndex.js' +import type { VectorIndexProvider, GraphIndexProvider } from '../../plugin.js' import type { MetadataIndexManager } from '../../utils/metadataIndex.js' -import type { GraphIndexProvider } from '../../plugin.js' import type { GraphVerb } from '../../coreTypes.js' import type { Operation, RollbackAction } from '../types.js' /** - * Add to HNSW index with rollback support + * Backend identity stamped into an operation's emitted `name` string (e.g. + * `AddToVectorIndex(hnsw-js)`), resolved from the provider's own + * {@link VectorIndexProvider.name} self-report. + * + * These operation classes are backend-neutral (the vector index they wrap may + * be Brainy's own JS HNSW fallback OR a native acceleration provider), but + * their names surface directly in consumer-visible transaction journals and + * timings. `name` is REQUIRED at the TypeScript level — but a native provider + * instance compiled against the previous (pre-required) contract can still + * reach this function at runtime without it. That legacy case is tolerated, + * never crashed on and never silently mislabeled: it resolves to + * `'unknown-provider'` and emits ONE loud warning naming the missing contract + * field, so the fix (implement `name`) is discoverable rather than a silent + * fossil label in every journal line thereafter. + */ +const warnedMissingName = new WeakSet() + +function resolveVectorProviderId(index: VectorIndexProvider): string { + const name = (index as { name?: unknown }).name + if (typeof name === 'string') return name + if (!warnedMissingName.has(index)) { + warnedMissingName.add(index) + console.warn( + '[vector-index] provider is missing the required `name` field (VectorIndexProvider.name, ' + + 'required since 8.10.0) — stamping "unknown-provider" in transaction op names until the ' + + 'provider declares its own identity.' + ) + } + return 'unknown-provider' +} + +/** + * Add to the vector index with rollback support. + * + * Backend-neutral: `index` is whatever the `'vector'` provider factory + * returns — Brainy's own JS HNSW fallback, or a native acceleration provider + * (e.g. DiskANN). The emitted `name` stamps the active backend (see + * {@link resolveVectorProviderId}) so operators reading a transaction journal + * or timing trace see which engine actually ran, never a fossil name from + * whichever engine happened to be active when this op class was written. * * Rollback strategy: * - Remove item from index - */ -export class AddToHNSWOperation implements Operation { - readonly name = 'AddToHNSW' +export class AddToVectorIndexOperation implements Operation { + readonly name: string constructor( - private readonly index: JsHnswVectorIndex, + private readonly index: VectorIndexProvider, private readonly id: string, private readonly vector: number[] - ) {} + ) { + this.name = `AddToVectorIndex(${resolveVectorProviderId(index)})` + } async execute(): Promise { // Check if item already exists (for rollback decision) @@ -59,11 +98,12 @@ export class AddToHNSWOperation implements Operation { * pre-existence as "existed" made every rollback skip removeItem, leaving * phantom entries in the index after a failed transaction. The safe default * is to remove what this operation added — update flows pair this op with a - * RemoveFromHNSWOperation whose own rollback restores the prior vector, so - * reverse-order rollback reconstructs the original state either way. + * RemoveFromVectorIndexOperation whose own rollback restores the prior + * vector, so reverse-order rollback reconstructs the original state either + * way. */ private async itemExists(id: string): Promise { - const index = this.index as JsHnswVectorIndex & { + const index = this.index as VectorIndexProvider & { getItem?: (id: string) => Promise } if (typeof index.getItem !== 'function') return false @@ -77,21 +117,27 @@ export class AddToHNSWOperation implements Operation { } /** - * Remove from HNSW index with rollback support + * Remove from the vector index with rollback support. + * + * Backend-neutral: see {@link AddToVectorIndexOperation} — `index` may be the + * JS HNSW fallback or a native acceleration provider; the emitted `name` + * stamps the active backend. * * Rollback strategy: * - Re-add item to index with original vector * * Note: Requires storing the vector for rollback */ -export class RemoveFromHNSWOperation implements Operation { - readonly name = 'RemoveFromHNSW' +export class RemoveFromVectorIndexOperation implements Operation { + readonly name: string constructor( - private readonly index: JsHnswVectorIndex, + private readonly index: VectorIndexProvider, private readonly id: string, private readonly vector: number[] // Required for rollback - ) {} + ) { + this.name = `RemoveFromVectorIndex(${resolveVectorProviderId(index)})` + } async execute(): Promise { // Remove from index @@ -285,22 +331,24 @@ export class RemoveFromGraphIndexOperation implements Operation { } /** - * Batch operation: Add multiple items to HNSW index + * Batch operation: Add multiple items to the vector index (backend-neutral — + * see {@link AddToVectorIndexOperation}). * * Useful for bulk imports with transaction support. * Rolls back all items if any fail. */ -export class BatchAddToHNSWOperation implements Operation { - readonly name = 'BatchAddToHNSW' +export class BatchAddToVectorIndexOperation implements Operation { + readonly name: string - private operations: AddToHNSWOperation[] + private operations: AddToVectorIndexOperation[] constructor( - index: JsHnswVectorIndex, + index: VectorIndexProvider, items: Array<{ id: string; vector: number[] }> ) { + this.name = `BatchAddToVectorIndex(${resolveVectorProviderId(index)})` this.operations = items.map( - item => new AddToHNSWOperation(index, item.id, item.vector) + item => new AddToVectorIndexOperation(index, item.id, item.vector) ) } diff --git a/src/transaction/operations/StorageOperations.ts b/src/transaction/operations/StorageOperations.ts index ca513cd4..316f1ac0 100644 --- a/src/transaction/operations/StorageOperations.ts +++ b/src/transaction/operations/StorageOperations.ts @@ -104,35 +104,72 @@ export class SaveNounOperation implements Operation { } /** - * Delete noun metadata with rollback support + * Delete a noun — FULL canonical removal, with rollback support. + * + * Despite the historical name, this removes the WHOLE entity: both canonical + * legs (metadata + vector) AND the entity's container. Previously it deleted + * only the metadata leg via `deleteNounMetadata`, leaving the canonical + * `vectors.json` leg and the `/` directory orphaned on disk — a "ghost" + * that reads as absent (getNoun needs both legs) yet inflates the enumerated + * count and can never be told apart from a damage scar. Routing through + * `storage.deleteNoun` removes both legs + the container in one place. + * + * Immutability is preserved: this cleans only the live-HEAD projection; the + * generation store retains the before-image so `asOf()` still reconstructs the + * deleted entity until retention expires. * * Rollback strategy: - * - Restore deleted metadata + * - Restore BOTH legs from the before-image (vector leg raw, metadata leg via + * the count-aware save so deleteNoun()'s decrement is reversed). */ export class DeleteNounMetadataOperation implements Operation { - readonly name = 'DeleteNounMetadata' + readonly name = 'DeleteNoun' constructor( private readonly storage: StorageAdapter, - private readonly id: string + private readonly id: string, + /** + * OPTIONAL already-known metadata of the entity being removed (the caller's + * pre-delete read). Removal must never REQUIRE re-reading the thing being + * removed: if the reads here return null (replace race, or a ghost left by + * an earlier version), the count decrement downstream falls back to this + * record instead of being silently skipped — the skip minted permanent + * counter inflation (adds counted, paired removals not decremented). + */ + private readonly priorMetadata?: NounMetadata | null ) {} async execute(): Promise { - // Get metadata before deletion (for rollback) - const previousMetadata = await this.storage.getNounMetadata(this.id) + // Capture the FULL before-image (both legs) so the undo restores the whole + // entity — a metadata-only rollback would leave the vector leg unrestored. + // A null metadata read falls back to the caller's pre-delete read. + const previousNoun = await this.storage.getNoun(this.id) + const previousMetadata = (await this.storage.getNounMetadata(this.id)) ?? this.priorMetadata ?? null - if (!previousMetadata) { + if (!previousNoun && !previousMetadata) { // Nothing to delete - no rollback needed return async () => {} } - // Delete metadata - await this.storage.deleteNounMetadata(this.id) + // Full removal: both canonical legs + the entity container + count decrement + // (the prior record keeps the decrement honest on a null canonical read). + await this.storage.deleteNoun(this.id, previousMetadata) // Return rollback action return async () => { - // Restore deleted metadata - await this.storage.saveNounMetadata(this.id, previousMetadata) + // Restore the vector leg, then the metadata leg through the count-aware + // save so deleteNoun()'s decrement is reversed. + if (previousNoun) { + await this.storage.saveNoun({ + id: previousNoun.id, + vector: previousNoun.vector, + connections: previousNoun.connections || new Map(), + level: previousNoun.level || 0 + }) + } + if (previousMetadata) { + await this.storage.saveNounMetadata(this.id, previousMetadata) + } } } } @@ -242,9 +279,9 @@ export class DeleteVerbMetadataOperation implements Operation { return async () => {} } - // Delete verb (metadata + vector) - // Note: StorageAdapter has deleteVerb but not deleteVerbMetadata - await this.storage.deleteVerb(this.id) + // Delete verb (metadata + vector). The pre-read rides along so the count + // decrement never depends on re-reading the record being removed. + await this.storage.deleteVerb(this.id, previousMetadata) // Return rollback action return async () => { diff --git a/src/transaction/operations/index.ts b/src/transaction/operations/index.ts index 422f6dad..c5548e70 100644 --- a/src/transaction/operations/index.ts +++ b/src/transaction/operations/index.ts @@ -21,12 +21,12 @@ export { // Index Operations export { - AddToHNSWOperation, - RemoveFromHNSWOperation, + AddToVectorIndexOperation, + RemoveFromVectorIndexOperation, AddToMetadataIndexOperation, RemoveFromMetadataIndexOperation, AddToGraphIndexOperation, RemoveFromGraphIndexOperation, - BatchAddToHNSWOperation, + BatchAddToVectorIndexOperation, BatchAddToMetadataIndexOperation } from './IndexOperations.js' diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index c2344375..8bede8d3 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -1651,6 +1651,25 @@ export interface BrainyConfig { */ disableAutoRebuild?: boolean + /** + * The floor (ms) of the apply-phase transaction budget's scaling formula: + * `max(transactionBudgetFloorMs, opCount × 2 000)`. The budget governs + * whether a `transact()` / single-op write's NEXT internal operation may + * **start** — never whether already-completed work gets rolled back after + * the fact (a single-op write can never time out post-hoc: it either runs + * or it commits). A trip mid-batch still rolls back every applied operation + * atomically and throws a retryable `TransactionTimeoutError`; only the + * floor of the formula is configurable here. + * + * Raise this when a cold store's first writes after a restart legitimately + * take longer than 30s per operation (e.g. page-cache-cold canonical writes + * on a large brain) so a bulk `transact()` batch gets a proportionally + * larger runway instead of tripping mid-batch. Lower it to fail faster on a + * latency-sensitive write path. Default: `30000` (30 s) — unchanged from + * pre-8.9 behavior. + */ + transactionBudgetFloorMs?: number + /** * How long (ms) an operation **waits** on the coordinated 7.x → 8.0 migration * before throwing a retryable `MigrationInProgressError`. @@ -1737,7 +1756,10 @@ export interface BrainyConfig { * Under Model-B EVERY write (`transact()` AND single-op `add`/`update`/ * `remove`/`relate`) produces an immutable generation record-set serving * historical reads (`asOf()`, pinned `Db` values). Without compaction those - * accumulate, so Brainy **auto-compacts on every `flush()` and `close()`**. + * accumulate, so Brainy **auto-compacts at `close()`** (time-bounded per + * pass; 8.9.0 removed compaction from `flush()` — flush is durability work + * and never pays maintenance costs). A long-lived writer that never closes + * accumulates history until its next explicit `compactHistory()` call. * Live `Db` pins are ALWAYS exempt from reclamation, in every mode. * * Modes: @@ -1754,7 +1776,7 @@ export interface BrainyConfig { * the oldest unpinned generations while ANY supplied cap is exceeded * (predictable ops). `maxAge` in ms; `maxBytes` total history bytes. * - * `autoCompact: false` disables the automatic flush/close compaction (manage + * `autoCompact: false` disables the automatic close() compaction (manage * manually via `brain.compactHistory()`). `budgetBytes` is the settable * adaptive byte budget a coordinator drives (also via `brain.setRetentionBudget()`). * Long-term archives belong in `db.persist(path)` snapshots, which compaction @@ -1772,7 +1794,7 @@ export interface BrainyConfig { maxBytes?: number /** Adaptive byte budget for this brain, driven by a coordinator (e.g. cor). */ budgetBytes?: number - /** Run compaction automatically on flush()/close() (default: true). */ + /** Run compaction automatically at close() (default: true; 8.9.0 — flush() never compacts). */ autoCompact?: boolean } @@ -1803,6 +1825,23 @@ export interface BrainyConfig { */ eagerEmbeddings?: boolean + /** + * When `true`, `init()` **awaits `warm()`** (see {@link Brainy.warm}) before + * it resolves — blocking startup until the vector index, metadata index, + * and graph adjacency have all faulted their backing storage in. This is a + * deliberate operator opt-in trade: startup takes longer, but the FIRST + * request after a cold restart runs at steady-state cost instead of paying + * page-cache-miss / demand-load latency on the critical path. + * + * Runs AFTER `init()`'s own sequence completes (index construction, crash + * recovery, migrations, VFS bootstrap) — never in place of any of it — so + * `warm()` always operates on a fully-initialized brain. + * + * Default: `false` (lazy — the pre-8.9 behavior: indexes demand-load on + * first access). + */ + warmOnOpen?: boolean + // Plugin configuration // Controls which plugins are loaded during init(). // - undefined (default): guarded auto-detection of the first-party diff --git a/src/utils/crc32c.ts b/src/utils/crc32c.ts new file mode 100644 index 00000000..0c2a1c14 --- /dev/null +++ b/src/utils/crc32c.ts @@ -0,0 +1,43 @@ +/** + * @module utils/crc32c + * @description CRC-32C (Castagnoli, polynomial 0x1EDC6F41, reflected 0x82F63B78) + * — the storage-industry frame checksum (ext4, iSCSI, SCTP, LSM segment files). + * Used to frame generation-fact segments: every appended record carries the + * CRC-32C of its payload, so a torn tail (crash mid-append) or bit rot is + * DETECTED at scan time and never silently read as data. + * + * Table-driven, dependency-free reference implementation. Native providers may + * substitute a hardware-accelerated (SSE4.2 / ARMv8 CRC) implementation — the + * polynomial is the contract, byte-identical results required. + */ + +/** The 256-entry lookup table for the reflected CRC-32C polynomial. */ +const TABLE: Uint32Array = (() => { + const table = new Uint32Array(256) + for (let n = 0; n < 256; n++) { + let c = n + for (let k = 0; k < 8; k++) { + c = c & 1 ? 0x82f63b78 ^ (c >>> 1) : c >>> 1 + } + table[n] = c >>> 0 + } + return table +})() + +/** + * Compute the CRC-32C checksum of a byte buffer. + * + * Known-answer vectors (RFC 3720 appendix / the standard test suite): + * - ASCII "123456789" → 0xE3069283 + * - 32 zero bytes → 0x8A9136AA + * + * @param bytes - The payload to checksum. + * @returns The CRC-32C as an unsigned 32-bit integer. + */ +export function crc32c(bytes: Uint8Array): number { + let crc = 0xffffffff + for (let i = 0; i < bytes.length; i++) { + crc = TABLE[(crc ^ bytes[i]) & 0xff] ^ (crc >>> 8) + } + return (crc ^ 0xffffffff) >>> 0 +} diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index fd325319..fdb17c22 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -423,6 +423,46 @@ export class MetadataIndexManager implements MetadataIndexProvider { prodLog.debug('✅ Type-aware cache warming completed') } + /** + * Full hydration — the {@link Brainy.warm} readiness seam for the metadata + * index. Unlike {@link warmCache} / {@link warmCacheForTopTypes} (which + * warm only a heuristic subset: common fields plus the top-N types' top + * fields), this loads EVERY field's sparse index the field registry knows + * about — a real read through {@link loadSparseIndex} into the unified + * cache for each field, not a stat/existence check. Idempotent: an + * already-cached field's `loadSparseIndex` call is a cheap cache hit. + * + * Re-reads the field registry first when `fieldIndexes` is empty (a warm() + * call issued before `init()` populated it would otherwise hydrate + * nothing), then loads every discovered field in parallel. + */ + async hydrateAll(): Promise { + if (this.fieldIndexes.size === 0) { + await this.loadFieldRegistry() + } + + const fields = Array.from(this.fieldIndexes.keys()) + if (fields.length === 0) { + prodLog.debug('[MetadataIndex] hydrateAll: no persisted fields to hydrate') + return + } + + prodLog.debug(`[MetadataIndex] hydrateAll: loading ${fields.length} field(s) — ${fields.join(', ')}`) + + await Promise.all( + fields.map(async field => { + try { + await this.loadSparseIndex(field) + } catch (error) { + // A single field's load failure doesn't abort the rest of the + // hydration — warm() is a best-effort readiness step, never a + // correctness gate (queries still demand-load on miss). + prodLog.debug(`[MetadataIndex] hydrateAll: field '${field}' failed to load:`, error) + } + }) + ) + } + /** * Acquire an in-memory lock for coordinating concurrent metadata index writes * Uses in-memory locks since MetadataIndexManager doesn't have direct file system access @@ -1867,9 +1907,26 @@ export class MetadataIndexManager implements MetadataIndexProvider { // not once per AND-clause inside it. const unindexedFields: string[] = [] - for (const [field, condition] of Object.entries(filter)) { + for (const [rawField, condition] of Object.entries(filter)) { // Skip logical operators - if (field === 'allOf' || field === 'anyOf' || field === 'not') continue + if (rawField === 'allOf' || rawField === 'anyOf' || rawField === 'not') continue + + // Metadata is FLATTENED at index time (metadata.entry.title indexes as + // entry.title), so a `metadata.`-prefixed where key is almost always + // the caller spelling the STORAGE shape rather than the index shape. + // Accept both spellings: when the key as spelled is unindexed but its + // stripped spelling is, query the stripped one. A literal nested + // custom key named `metadata` still wins when indexed as spelled + // (checked first), so that rare shape keeps working. + let field = rawField + if ( + rawField.startsWith('metadata.') && + this.columnStore && + !this.columnStore.hasField(rawField) && + this.columnStore.hasField(rawField.slice('metadata.'.length)) + ) { + field = rawField.slice('metadata.'.length) + } let fieldResults: string[] = [] diff --git a/src/utils/osLimits.ts b/src/utils/osLimits.ts new file mode 100644 index 00000000..57e1ccf1 --- /dev/null +++ b/src/utils/osLimits.ts @@ -0,0 +1,141 @@ +/** + * @module utils/osLimits + * @description Detect-and-warn for OS resource limits that bite at POOL scale. + * + * A single brain rarely notices them, but a pool of brains — especially with a + * native accelerator memory-mapping many index files per brain — consumes file + * descriptors and memory mappings multiplicatively. On stock Linux defaults + * (RLIMIT_NOFILE soft 1024, vm.max_map_count 65530) the failure arrives as + * EMFILE or a failed mmap deep inside an index open, long after the real cause + * (the limit) stopped being visible. This module reads the limits at open and + * WARNS ONCE per process with the exact raise commands, so the operator learns + * the fix before the incident instead of from it. + * + * Read-only and Linux-only by construction: both sources are `/proc` files. + * On platforms where they are absent the check reports nulls and stays silent — + * no limit read means no claim made, never a guessed warning. + */ + +import * as fs from 'node:fs' +import { prodLog } from './logger.js' + +/** Soft-NOFILE floor below which pool-scale use is at EMFILE risk. */ +export const NOFILE_POOL_FLOOR = 65536 + +/** vm.max_map_count floor below which mmap-heavy native indexes are at risk. */ +export const MAX_MAP_COUNT_POOL_FLOOR = 262144 + +export interface OsLimitsReport { + /** RLIMIT_NOFILE soft limit (null when unreadable; Infinity for 'unlimited'). */ + nofileSoft: number | null + /** RLIMIT_NOFILE hard limit (null when unreadable; Infinity for 'unlimited'). */ + nofileHard: number | null + /** vm.max_map_count (null when unreadable). */ + maxMapCount: number | null + /** Human-actionable warnings for limits below the pool floors. Empty = fine. */ + warnings: string[] +} + +/** + * Parse the `Max open files` row of a `/proc//limits` document into + * soft/hard values. Returns nulls when the row is absent or malformed. + */ +export function parseProcLimits(content: string): { soft: number | null; hard: number | null } { + const line = content.split('\n').find((l) => l.startsWith('Max open files')) + if (!line) return { soft: null, hard: null } + const m = line.match(/^Max open files\s+(\S+)\s+(\S+)/) + if (!m) return { soft: null, hard: null } + const parse = (v: string): number | null => { + if (v === 'unlimited') return Infinity + const n = Number.parseInt(v, 10) + return Number.isNaN(n) ? null : n + } + return { soft: parse(m[1]), hard: parse(m[2]) } +} + +/** + * Assess readable limits against the pool floors. Pure — feed it any values. + * A null (unreadable) limit produces NO warning: no measurement, no claim. + */ +export function assessOsLimits(limits: { + nofileSoft: number | null + nofileHard: number | null + maxMapCount: number | null +}): string[] { + const warnings: string[] = [] + + if (limits.nofileSoft !== null && limits.nofileSoft < NOFILE_POOL_FLOOR) { + const hardNote = + limits.nofileHard !== null && limits.nofileHard >= NOFILE_POOL_FLOOR + ? ` (the hard limit ${limits.nofileHard === Infinity ? 'unlimited' : limits.nofileHard} already allows it — raise the soft limit only)` + : '' + warnings.push( + `RLIMIT_NOFILE soft limit is ${limits.nofileSoft} — below the ${NOFILE_POOL_FLOOR} recommended ` + + `for pool-scale use (a pool of brains with a native accelerator opens many index files per brain; ` + + `the failure mode is EMFILE deep inside an index open). Raise with \`ulimit -n ${NOFILE_POOL_FLOOR}\` ` + + `or LimitNOFILE=${NOFILE_POOL_FLOOR} in the service unit${hardNote}.` + ) + } + + if (limits.maxMapCount !== null && limits.maxMapCount < MAX_MAP_COUNT_POOL_FLOOR) { + warnings.push( + `vm.max_map_count is ${limits.maxMapCount} — below the ${MAX_MAP_COUNT_POOL_FLOOR} recommended ` + + `for mmap-heavy native indexes at pool scale (each mapped index segment consumes map entries; ` + + `the failure mode is a failed mmap mid-heal). Raise with ` + + `\`sysctl -w vm.max_map_count=${MAX_MAP_COUNT_POOL_FLOOR}\` (persist in /etc/sysctl.d/).` + ) + } + + return warnings +} + +/** + * Read the limits from /proc and assess them. `readFile` is injectable for + * tests; absent/unreadable sources yield nulls (and therefore no warnings). + */ +export async function checkOsLimits( + readFile: (path: string) => Promise = async (p) => fs.promises.readFile(p, 'utf-8') +): Promise { + let nofileSoft: number | null = null + let nofileHard: number | null = null + let maxMapCount: number | null = null + + try { + const parsed = parseProcLimits(await readFile('/proc/self/limits')) + nofileSoft = parsed.soft + nofileHard = parsed.hard + } catch { + // Not Linux (or /proc unavailable) — no measurement, no claim. + } + + try { + const raw = (await readFile('/proc/sys/vm/max_map_count')).trim() + const n = Number.parseInt(raw, 10) + maxMapCount = Number.isNaN(n) ? null : n + } catch { + // Not Linux — same rule. + } + + const warnings = assessOsLimits({ nofileSoft, nofileHard, maxMapCount }) + return { nofileSoft, nofileHard, maxMapCount, warnings } +} + +/** Once-per-process latch so a brain pool warns once, not once per brain. */ +let osLimitsWarned = false + +/** + * Run the check and warn (once per process) about limits below the pool + * floors. Called from brain open; safe everywhere (silent off-Linux). + */ +export async function warnOnLowOsLimits(): Promise { + if (osLimitsWarned) return + osLimitsWarned = true + try { + const report = await checkOsLimits() + for (const warning of report.warnings) { + prodLog.warn(`[Brainy] OS limit check: ${warning}`) + } + } catch { + // The check must never affect open — measurement-only. + } +} diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index 57a9a1c0..ca439524 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -209,8 +209,10 @@ export interface ValidationConfigOptions { } /** - * Auto-configured limits based on system resources - * These adapt to available memory and observed performance + * Auto-configured limits based on system resources. + * Derived from memory (explicit overrides > reserved memory > container limit > + * free memory). Query timing is recorded for diagnostics only — it never + * changes the cap (see `recordQuery`). */ export class ValidationConfig { private static instance: ValidationConfig | null = null @@ -323,24 +325,23 @@ export class ValidationConfig { } /** - * Learn from actual usage to adjust limits + * Record query timing for diagnostics. Telemetry ONLY — never mutates the cap. + * + * `maxLimit` is a MEMORY-protection bound; query duration says nothing about + * memory-per-result, so duration must never drive it. An earlier version + * "learned" here: while the lifetime-average query time exceeded 1s it shrank + * `maxLimit` by 20% per recorded query down to a floor of 1000 — below the + * documented `MIN_AUTO_QUERY_LIMIT` (10 000) and with no recovery once slow + * samples poisoned the cumulative average. On a production host a burst of + * slow aggregate queries silently strangled every consumer's `find()` to + * 1000 while the error message blamed "available free memory" — exactly the + * silent throttling this module's own contract forbids. The cap now comes + * from its construction-time basis (or explicit overrides) alone. */ recordQuery(duration: number, resultCount: number) { + void resultCount this.queryCount++ this.avgQueryTime = (this.avgQueryTime * (this.queryCount - 1) + duration) / this.queryCount - - // Only auto-adjust if not using explicit overrides - if (this.limitBasis !== 'override') { - // If queries are consistently fast with large results, increase limits - if (this.avgQueryTime < 100 && resultCount > this.maxLimit * 0.8) { - this.maxLimit = Math.min(this.maxLimit * 1.5, 100000) - } - - // If queries are slow, reduce limits - if (this.avgQueryTime > 1000) { - this.maxLimit = Math.max(this.maxLimit * 0.8, 1000) - } - } } } diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts index 013f04f3..00bddefb 100644 --- a/src/vfs/VirtualFileSystem.ts +++ b/src/vfs/VirtualFileSystem.ts @@ -1954,15 +1954,27 @@ export class VirtualFileSystem implements IVirtualFileSystem { const newParentPath = this.getParentPath(newPath) if (oldParentPath !== newParentPath) { - // Remove from old parent + // Remove the OLD parent's containment edge(s) — by edge id, resolved from + // the graph's own adjacency (the removal law: a removal never requires + // reading the thing being removed). This step used to be skipped as "not + // critical", which left the moved entity a child of BOTH directories: + // readdir(oldDir) kept listing it, re-creating the old path showed the + // name twice, and tree-walking consumers saw the file in two places. if (oldParentPath) { const oldParentId = await this.pathResolver.resolve(oldParentPath) - // unrelate takes the relation ID, not params - need to find and remove relation - // For now, skip unrelate as it's not critical for rename + const staleEdges = await this.brain.related({ + from: oldParentId, + to: entityId, + type: VerbType.Contains + }) + for (const edge of staleEdges) { + await this.brain.unrelate(edge.id) + } } - // Add to new parent - if (newParentPath && newParentPath !== '/') { + // Add to the new parent. The root ('/') is a REAL parent — skipping it + // orphaned a move-to-root out of readdir('/') entirely. + if (newParentPath) { const newParentId = await this.pathResolver.resolve(newParentPath) await this.brain.relate({ from: newParentId, @@ -2003,6 +2015,95 @@ export class VirtualFileSystem implements IVirtualFileSystem { this.triggerWatchers(newPath, 'rename') } + /** + * Reconcile every VFS entity's containment edges against its canonical + * `metadata.path` — the path is the truth (maintained by write/rename); the + * `Contains` edges are a projection of it. Heals the "cosmetic ghost" class + * left by pre-fix renames that added the new parent's edge without removing + * the old one (an entity listed in TWO directories; a re-created old path + * showing its name twice), plus duplicate edges from the same parent left by + * concurrent writers. + * + * CONSERVATIVE by design: only VFS containment edges (subtype + * `'vfs-contains'` or `metadata.isVFS`) are ever touched — a user's own + * knowledge-graph `Contains` edge between the same entities is never + * removed. An entity whose expected parent path has no entity is logged + * loudly and left alone (never orphaned further). Operator-invoked via + * `brain.repairIndex()`. + * + * The canonical pagination walk (not an index query) is deliberate: this is + * a repair op — the projections are the thing under suspicion, so the walk + * reads the source of truth. + * + * @returns Counts of stale edges removed and missing expected edges restored. + */ + async repairContainment(): Promise<{ removed: number; restored: number }> { + await this.ensureInitialized() + + // Pass 1: canonical walk → every VFS entity's id + path. + const idByPath = new Map() + const vfsEntities: Array<{ id: string; path: string }> = [] + let cursor: string | undefined + for (;;) { + const page = await (this.brain as any).storage.getNounsWithPagination({ limit: 500, cursor }) + for (const noun of page.items) { + const meta = (noun as any).metadata ?? noun + const p = meta?.path + if (meta?.vfsType && typeof p === 'string') { + idByPath.set(p, noun.id) + vfsEntities.push({ id: noun.id, path: p }) + } + } + if (!page.hasMore) break + cursor = page.nextCursor + } + + let removed = 0 + let restored = 0 + for (const { id, path } of vfsEntities) { + if (path === '/') continue // the root has no parent + const expectedParentId = idByPath.get(this.getParentPath(path)) + if (!expectedParentId) { + console.warn( + `[VFS] repairContainment: no entity found for parent of ${path} — leaving its edges untouched.` + ) + continue + } + + const incoming = await this.brain.related({ to: id, type: VerbType.Contains }) + let expectedSeen = false + for (const edge of incoming) { + const isVfsEdge = edge.subtype === 'vfs-contains' || (edge.metadata as any)?.isVFS === true + if (!isVfsEdge) continue // never touch user knowledge edges + if (edge.from === expectedParentId && !expectedSeen) { + expectedSeen = true // keep exactly one correct edge + continue + } + // Stale parent (a pre-fix rename ghost) or a duplicate of the correct + // edge (concurrent-writer artifact) — remove it, loudly. + await this.brain.unrelate(edge.id) + removed++ + console.warn( + `[VFS] repairContainment: removed ${edge.from === expectedParentId ? 'duplicate' : 'stale'} ` + + `containment edge ${edge.from} -> ${id} (${path})` + ) + } + if (!expectedSeen) { + await this.brain.relate({ + from: expectedParentId, + to: id, + type: VerbType.Contains, + subtype: 'vfs-contains', + metadata: { isVFS: true } + }) + restored++ + console.warn(`[VFS] repairContainment: restored missing containment edge for ${path}`) + } + } + + return { removed, restored } + } + /** * Copy a file or directory to a new path. * diff --git a/tests/integration/aggregate-reserved-fields.test.ts b/tests/integration/aggregate-reserved-fields.test.ts new file mode 100644 index 00000000..e81692d6 --- /dev/null +++ b/tests/integration/aggregate-reserved-fields.test.ts @@ -0,0 +1,169 @@ +/** + * @module tests/integration/aggregate-reserved-fields + * @description One field-resolution law across the whole aggregation + query + * surface (SELF-AGGREGATE-DELETE-DRIFT). Laws: + * (1) aggregates grouped by a RESERVED field (subtype) decrement on delete — + * the delete-side entity view carries every reserved field, so the + * decrement finds its group (counts must never drift from ground truth); + * (2) same for update: moving an entity between reserved-field groups + * decrements the old group and increments the new one (no double-count); + * (3) aggregation source.where on a reserved field (subtype) FILTERS instead + * of silently matching nothing; + * (4) removeMany refuses empty/invalid selectors loudly (bare array, empty + * object, ids: []) instead of resolving as a silent no-op; + * (5) find() accepts both where spellings: flattened (entry.title) and + * storage-shaped (metadata.entry.title) resolve to the same rows. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +const stubEmbedding = async (text: string): Promise => { + const hash = text.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0) + return new Array(384).fill(0).map((_, i) => Math.sin(hash + i)) +} + +describe('aggregation + query field-resolution law', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' as const }, + embeddingFunction: stubEmbedding + }) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + it('reserved-field groupBy decrements on delete (the drift bug)', async () => { + brain.defineAggregate({ + name: 'by_subtype', + source: { type: NounType.Document }, + groupBy: ['subtype'], + metrics: { count: { op: 'count' } } + }) + + const ids: string[] = [] + for (let i = 0; i < 5; i++) { + ids.push( + await brain.add({ + data: `doc-${i}`, + type: NounType.Document, + subtype: 'note', + metadata: { team: 'alpha' } + }) + ) + } + let groups = await brain.queryAggregate('by_subtype') + expect(groups).toHaveLength(1) + expect(groups[0].groupKey).toEqual({ subtype: 'note' }) + expect(groups[0].metrics.count).toBe(5) + + await brain.remove(ids[0]) + await brain.flush() + + groups = await brain.queryAggregate('by_subtype') + expect(groups[0].metrics.count).toBe(4) + const live = await brain.find({ type: NounType.Document, limit: 100 }) + expect(groups[0].metrics.count).toBe(live.length) + }) + + it('reserved-field groupBy moves between groups on update (no double-count)', async () => { + brain.defineAggregate({ + name: 'by_subtype', + source: { type: NounType.Document }, + groupBy: ['subtype'], + metrics: { count: { op: 'count' } } + }) + const id = await brain.add({ + data: 'doc-move', + type: NounType.Document, + subtype: 'draft' + }) + await brain.update({ id, subtype: 'published' }) + + const groups = await brain.queryAggregate('by_subtype') + const byKey = Object.fromEntries( + groups.map((g) => [String(g.groupKey.subtype), g.metrics.count]) + ) + expect(byKey['published']).toBe(1) + // The old group must be gone or zero — never still counting the entity. + expect(byKey['draft'] ?? 0).toBe(0) + }) + + it('source.where on a reserved field filters instead of matching nothing', async () => { + brain.defineAggregate({ + name: 'notes_only', + source: { type: NounType.Document, where: { subtype: 'note' } }, + groupBy: ['team'], + metrics: { count: { op: 'count' } } + }) + await brain.add({ + data: 'n1', + type: NounType.Document, + subtype: 'note', + metadata: { team: 'alpha' } + }) + await brain.add({ + data: 'd1', + type: NounType.Document, + subtype: 'draft', + metadata: { team: 'alpha' } + }) + + const groups = await brain.queryAggregate('notes_only') + expect(groups).toHaveLength(1) + expect(groups[0].metrics.count).toBe(1) // the note, never the draft + }) + + it('removeMany refuses empty/invalid selectors loudly', async () => { + const id = await brain.add({ data: 'keep-me', type: NounType.Document }) + + // Bare array passed positionally — the classic silent no-op. + await expect( + brain.removeMany([id] as unknown as Parameters[0]) + ).rejects.toThrow(/bare array/) + // Empty selector object. + await expect( + brain.removeMany({} as Parameters[0]) + ).rejects.toThrow(/requires a selector/) + // Explicit empty id list. + await expect(brain.removeMany({ ids: [] })).rejects.toThrow(/ids: \[\]/) + + // Nothing was deleted by any of the refused calls. + expect(await brain.get(id)).toBeTruthy() + }) + + it('find() accepts both flattened and metadata.-prefixed where spellings', async () => { + await brain.add({ + data: 'nested-doc', + type: NounType.Document, + metadata: { entry: { title: 'T1' }, classifier: { contextHints: { vfsPath: '/n/a.md' } } } + }) + await brain.flush() + + const flat = await brain.find({ + type: NounType.Document, + where: { 'entry.title': 'T1' }, + limit: 10 + }) + const prefixed = await brain.find({ + type: NounType.Document, + where: { 'metadata.entry.title': 'T1' }, + limit: 10 + }) + const deepPrefixed = await brain.find({ + type: NounType.Document, + where: { 'metadata.classifier.contextHints.vfsPath': '/n/a.md' }, + limit: 10 + }) + expect(flat).toHaveLength(1) + expect(prefixed).toHaveLength(1) + expect(prefixed[0].id).toBe(flat[0].id) + expect(deepPrefixed).toHaveLength(1) + }) +}) diff --git a/tests/integration/aggregation-state-persistence.test.ts b/tests/integration/aggregation-state-persistence.test.ts new file mode 100644 index 00000000..db8fc088 --- /dev/null +++ b/tests/integration/aggregation-state-persistence.test.ts @@ -0,0 +1,311 @@ +/** + * @module tests/integration/aggregation-state-persistence + * @description The boot-order contract for the aggregation engine. Five laws: + * (1) STATE ADOPTION — a reopen with an unchanged defineAggregate() adopts the + * persisted state and performs NO store walk. (The pre-fix behavior: the + * synchronous define always beat the async init, flagged a backfill, and + * the first query wiped the just-loaded state and re-walked the whole + * store — every restart, forever.) + * (2) SINGLE-FLIGHT + BATCH — concurrent cold queries across multiple pending + * aggregates share exactly ONE store walk; a query never wipes another's + * partial progress and M pending aggregates cost one enumeration, not M. + * (3) CHANGED DEFINITION — a real definition change still backfills, exactly. + * (4) PERSISTED-ONLY DEFINITIONS — an app that does not re-define at boot can + * query a persisted aggregate without racing a spurious "not defined". + * (5) QUIET KEYS — the engine's persistence keys (__aggregation_*) are + * recognized system keys: no "Unknown key format" warning at boot. + */ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' +import type { AggregateDefinition } from '../../src/types/brainy.types.js' +import { prodLog } from '../../src/utils/logger.js' + +const SPENDING: AggregateDefinition = { + name: 'spending', + source: { type: NounType.Event, where: { domain: 'financial' } }, + groupBy: ['category'], + metrics: { + total: { op: 'sum', field: 'amount' }, + count: { op: 'count' } + } +} + +/** Same name, different metrics — a REAL definition change (hash differs). */ +const SPENDING_CHANGED: AggregateDefinition = { + ...SPENDING, + metrics: { + total: { op: 'sum', field: 'amount' }, + count: { op: 'count' }, + average: { op: 'avg', field: 'amount' } + } +} + +describe('aggregation state persistence — boot-order contract', () => { + let dir: string + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-agg-persist-')) + }) + + afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }) + }) + + const open = async (): Promise => { + const b: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true + }) + await b.init() + return b + } + + /** Count store walks by intercepting the storage adapter's getNouns. */ + const countWalks = (brain: any): { count: () => number } => { + const storage = brain.storage + const orig = storage.getNouns.bind(storage) + let calls = 0 + storage.getNouns = async (opts: unknown) => { + calls++ + return orig(opts) + } + return { count: () => calls } + } + + const seed = async (brain: any): Promise => { + for (let i = 0; i < 12; i++) { + await brain.add({ + data: `tx ${i}`, + type: NounType.Event, + metadata: { + domain: 'financial', + category: i % 2 === 0 ? 'food' : 'transport', + amount: 10 + i + } + }) + } + } + + it('adopts persisted state on reopen with an unchanged definition — zero walks', async () => { + const brain1 = await open() + brain1.defineAggregate(SPENDING) + await seed(brain1) + const before = await brain1.queryAggregate('spending') + expect(before.length).toBe(2) + await brain1.close() + + const brain2 = await open() + brain2.defineAggregate(SPENDING) // the standard declarative boot pattern + await brain2.getNounCount() // settle init paths before counting walks + const walks = countWalks(brain2) + + const after = await brain2.queryAggregate('spending') + + expect(walks.count()).toBe(0) + const key = (r: any) => r.groupKey.category + expect( + after.map((r: any) => [key(r), r.metrics.total, r.metrics.count]).sort() + ).toEqual( + before.map((r: any) => [key(r), r.metrics.total, r.metrics.count]).sort() + ) + await brain2.close() + }) + + it('adopted state keeps accumulating: post-reopen writes land on top of it', async () => { + const brain1 = await open() + brain1.defineAggregate(SPENDING) + await seed(brain1) + await brain1.queryAggregate('spending') + await brain1.close() + + const brain2 = await open() + brain2.defineAggregate(SPENDING) + // A write BEFORE the first query: if it lands before state adoption the + // engine must choose an exact rescan over adoption — either way the + // result must include all 13 entities. + await brain2.add({ + data: 'late tx', + type: NounType.Event, + metadata: { domain: 'financial', category: 'food', amount: 100 } + }) + const rows = await brain2.queryAggregate('spending') + const food = rows.find((r: any) => r.groupKey.category === 'food') + expect(food.metrics.count).toBe(7) // 6 seeded + 1 late + await brain2.close() + }) + + it('a changed definition still backfills — exactly once, with correct results', async () => { + const brain1 = await open() + brain1.defineAggregate(SPENDING) + await seed(brain1) + await brain1.queryAggregate('spending') + await brain1.close() + + const brain2 = await open() + brain2.defineAggregate(SPENDING_CHANGED) + await brain2.getNounCount() + const walks = countWalks(brain2) + + const rows = await brain2.queryAggregate('spending') + + expect(walks.count()).toBe(1) // 12 entities = one page = one getNouns call + const food = rows.find((r: any) => r.groupKey.category === 'food') + expect(food.metrics.count).toBe(6) + expect(food.metrics.average).toBeCloseTo(food.metrics.total / 6) + await brain2.close() + }) + + it('concurrent cold queries across two aggregates share exactly ONE walk', async () => { + const brain = await open() + brain.defineAggregate(SPENDING) + brain.defineAggregate({ + ...SPENDING, + name: 'by_category_count', + metrics: { count: { op: 'count' } } + }) + await seed(brain) + await brain.getNounCount() + const walks = countWalks(brain) + + const results = await Promise.all([ + brain.queryAggregate('spending'), + brain.queryAggregate('by_category_count'), + brain.queryAggregate('spending'), + brain.queryAggregate('by_category_count'), + brain.queryAggregate('spending'), + brain.queryAggregate('by_category_count') + ]) + + expect(walks.count()).toBe(1) // 12 entities = one page; one walk fills both + for (const rows of results) { + const total = rows.reduce((s: number, r: any) => s + r.metrics.count, 0) + expect(total).toBe(12) + } + + // Warm re-query: converged, no further walks. + await brain.queryAggregate('spending') + expect(walks.count()).toBe(1) + await brain.close() + }) + + it('persisted-only definitions are queryable without re-defining at boot', async () => { + const brain1 = await open() + brain1.defineAggregate(SPENDING) + await seed(brain1) + await brain1.queryAggregate('spending') + await brain1.close() + + const brain2 = await open() + // NO defineAggregate — the app relies on the persisted definition. + const walks = countWalks(brain2) + const rows = await brain2.queryAggregate('spending') + expect(walks.count()).toBe(0) // persisted state adopted here too + expect(rows.length).toBe(2) + await brain2.close() + }) + + it('generation-mismatched persisted state is rescanned once, loudly — never adopted', async () => { + const brain1 = await open() + brain1.defineAggregate(SPENDING) + await seed(brain1) + await brain1.queryAggregate('spending') + await brain1.close() + + // Simulate the copied-store incident class: a fact-log truncation (or an + // unclean shutdown) leaves the committed watermark different from the + // generation the flushed state was stamped with. + const tamper: any = await open() + const key = '__aggregation_state_spending__' + const stored = await tamper.storage.getMetadata(key) + expect(typeof stored.sourceGeneration).toBe('number') // the stamp is really persisted + await tamper.storage.saveMetadata(key, { + ...stored, + sourceGeneration: stored.sourceGeneration + 5 + }) + await tamper.close() + + const warnSpy = vi.spyOn(prodLog, 'warn') + const brain2 = await open() + brain2.defineAggregate(SPENDING) + await brain2.getNounCount() + const walks = countWalks(brain2) + + const rows = await brain2.queryAggregate('spending') + + expect(walks.count()).toBe(1) // exactly ONE rescan — no silent adopt, no spin + const food = rows.find((r: any) => r.groupKey.category === 'food') + expect(food.metrics.count).toBe(6) // rescan produced exact results + expect( + warnSpy.mock.calls.some(args => String(args[0]).includes('rescanning instead of adopting')) + ).toBe(true) // and it said so out loud + warnSpy.mockRestore() + await brain2.close() + }) + + it('a failing walk is loud, non-destructive, and latched — never a silent retry loop', async () => { + // Fresh define + seeded writes: the write hooks have populated LIVE state, + // and the first-query rescan is still pending. The incident shape + // (wipe-before-scan + no try/catch + per-query re-walk) would have wiped + // that live state and silently re-walked on every query. + const brain: any = await open() + brain.defineAggregate(SPENDING) + await seed(brain) + expect(brain._aggregationIndex.queryAggregate({ name: 'spending' }).length).toBe(2) + + const storage = brain.storage + const origGetNouns = storage.getNouns.bind(storage) + let walkAttempts = 0 + storage.getNouns = async () => { + walkAttempts++ + throw new Error('injected storage failure') + } + + // First query: the walk fails LOUDLY with the storage error. + await expect(brain.queryAggregate('spending')).rejects.toThrow('injected storage failure') + expect(walkAttempts).toBe(1) + + // Live state was NOT destroyed by the failed walk (staging was dropped). + expect(brain._aggregationIndex.queryAggregate({ name: 'spending' }).length).toBe(2) + + // Second query inside the cooldown: instant loud failure, NO new walk. + await expect(brain.queryAggregate('spending')).rejects.toThrow('failure cooldown') + expect(walkAttempts).toBe(1) + + // Heal the storage + expire the cooldown: one fresh walk succeeds exactly. + storage.getNouns = origGetNouns + brain._aggregationBackfillFailure.at = Date.now() - 60_000 + const rows = await brain.queryAggregate('spending') + const food = rows.find((r: any) => r.groupKey.category === 'food') + expect(food.metrics.count).toBe(6) + await brain.close() + }) + + it('aggregation persistence keys never log "Unknown key format"', async () => { + const warnSpy = vi.spyOn(prodLog, 'warn') + const brain1 = await open() + brain1.defineAggregate(SPENDING) + await seed(brain1) + await brain1.queryAggregate('spending') + await brain1.close() + + const brain2 = await open() + brain2.defineAggregate(SPENDING) + await brain2.queryAggregate('spending') + await brain2.close() + + const offenders = warnSpy.mock.calls + .map(args => String(args[0])) + .filter( + msg => + msg.includes('Unknown key format') && + (msg.includes('__aggregation_') || msg.includes('brainy:entityIdMapper')) + ) + expect(offenders).toEqual([]) + warnSpy.mockRestore() + }) +}) diff --git a/tests/integration/background-dedup-lifecycle.test.ts b/tests/integration/background-dedup-lifecycle.test.ts new file mode 100644 index 00000000..48c7a4a7 --- /dev/null +++ b/tests/integration/background-dedup-lifecycle.test.ts @@ -0,0 +1,85 @@ +/** + * @module tests/integration/background-dedup-lifecycle + * @description The post-import background deduplication pass (a merge-DELETE + * writer) obeys the same contract as the inline pass. Laws: + * (1) enableDeduplication:false schedules NO background pass — the brain-owned + * deduplicator is never even constructed; + * (2) by default the pass IS scheduled, brain-owned, with an unref'd timer + * (a pending pass never holds the process open); + * (3) repeated imports debounce into ONE pending batch on ONE instance + * (per-coordinator instances used to arm one timer per import); + * (4) close() cancels pending work — no delete pass can fire after close. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../src/brainy.js' + +const ROWS = [ + { name: 'Alice Zephyr', role: 'engineer' }, + { name: 'Bob Quill', role: 'writer' } +] + +// Keep imports fast and deterministic — dedup scheduling is what's under test. +const FAST = { + enableNeuralExtraction: false, + enableRelationshipInference: false, + enableConceptExtraction: false +} as const + +// Deterministic stub embedder (hnsw-rebuild.test.ts pattern) — dedup +// scheduling never inspects vector CONTENT, so skip the WASM model load. +const stubEmbedding = async (text: string): Promise => { + const hash = text.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0) + const vector = new Array(384).fill(0).map((_, i) => Math.sin(hash + i)) + return vector +} + +describe('background dedup lifecycle', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' as const }, + embeddingFunction: stubEmbedding + }) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + it('enableDeduplication:false schedules no background pass at all', async () => { + await brain.import(ROWS, { ...FAST, enableDeduplication: false }) + expect((brain as any)._backgroundDedup).toBeUndefined() + }) + + it('default schedules a brain-owned pass with an unref-ed timer', async () => { + await brain.import(ROWS, { ...FAST }) + const dedup = (brain as any)._backgroundDedup + expect(dedup).toBeDefined() + expect(dedup.pendingImports.size).toBe(1) + const timer = dedup.debounceTimer + expect(timer).toBeDefined() + // Node timers expose hasRef(); an unref'd timer must not hold the process. + expect(typeof timer.hasRef).toBe('function') + expect(timer.hasRef()).toBe(false) + }) + + it('imports debounce into one pending batch on one brain-owned instance', async () => { + await brain.import(ROWS, { ...FAST }) + const first = (brain as any)._backgroundDedup + await brain.import([{ name: 'Cara Vex', role: 'analyst' }], { ...FAST }) + expect((brain as any)._backgroundDedup).toBe(first) + expect(first.pendingImports.size).toBe(2) + }) + + it('close() cancels pending background dedup', async () => { + await brain.import(ROWS, { ...FAST }) + const dedup = (brain as any)._backgroundDedup + expect(dedup.debounceTimer).toBeDefined() + await brain.close() + expect(dedup.debounceTimer).toBeUndefined() + expect(dedup.pendingImports.size).toBe(0) + }) +}) diff --git a/tests/integration/counter-recount.test.ts b/tests/integration/counter-recount.test.ts new file mode 100644 index 00000000..310ea4d5 --- /dev/null +++ b/tests/integration/counter-recount.test.ts @@ -0,0 +1,122 @@ +/** + * @module tests/integration/counter-recount + * @description Counter honesty. Two laws under test: + * (1) REMOVAL NEVER REQUIRES RE-READING THE REMOVED RECORD — the count + * decrement falls back to the caller's pre-delete read when the canonical + * metadata re-read returns null (replace race / ghost), instead of being + * silently skipped. The skip minted permanent inflation: adds counted, + * paired removals not decremented, and Math.max(totalNounCount, scanned) + * pinned the inflated scalar forever. + * (2) THE SANCTIONED RECOUNT — repairIndex() unconditionally recomputes and + * PERSISTS every counter rollup (scalar totals + per-type maps + + * type-statistics) from one canonical walk, so an already-inflated brain + * is permanently corrected (survives reopen). + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/index.js' + +/** Absolute path of one entity's `` directory (or null if not present). */ +function entityDir(root: string, id: string): string | null { + const base = path.join(root, 'entities', 'nouns') + if (!fs.existsSync(base)) return null + for (const shard of fs.readdirSync(base)) { + const candidate = path.join(base, shard, id) + if (fs.existsSync(candidate)) return candidate + } + return null +} + +describe('counter honesty — removal without re-reading + the sanctioned recount', () => { + let dir: string + let brain: any + + const open = async () => { + const b: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) + await b.init() + return b + } + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-recount-')) + brain = await open() + }) + afterEach(async () => { + await brain.close?.().catch(() => {}) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('the counter returns to baseline across add→remove→re-create cycles (no drift)', async () => { + const baseline = await brain.storage.getNounCount() + for (let i = 0; i < 4; i++) { + const id = await brain.add({ data: `cycle ${i}`, type: 'document', metadata: { i } }) + expect(await brain.storage.getNounCount()).toBe(baseline + 1) + await brain.remove(id) + expect(await brain.storage.getNounCount()).toBe(baseline) + } + }) + + it('deleteNoun decrements from the provided prior record when the canonical re-read is null', async () => { + const id = await brain.add({ data: 'to be ghosted', type: 'document', metadata: { g: 1 } }) + await brain.flush() + const baseline = await brain.storage.getNounCount() + + // Capture the pre-delete read (what remove() holds), then simulate the + // replace-race / ghost shape: the metadata leg vanishes before the delete's + // internal re-read. + const prior = await brain.storage.getNounMetadata(id) + expect(prior).not.toBeNull() + const eDir = entityDir(dir, id)! + for (const f of fs.readdirSync(eDir)) { + if (f.startsWith('metadata.json')) fs.rmSync(path.join(eDir, f)) + } + + // Without the prior record this decrement used to be silently skipped. + await brain.storage.deleteNoun(id, prior) + expect(await brain.storage.getNounCount()).toBe(baseline - 1) + // Full removal still holds: nothing left on disk. + expect(entityDir(dir, id)).toBeNull() + }) + + it('repairIndex() recounts an inflated persisted scalar over CLEAN shelves — and it survives reopen', async () => { + for (let i = 0; i < 3; i++) { + await brain.add({ data: `real ${i}`, type: 'document', metadata: { i } }) + } + await brain.flush() + const honest = await brain.storage.getNounCount() + // Pagination's totalCount has its own baseline: it enumerates EVERYTHING + // (including the internal VFS root), while the user-facing scalar counts + // only public entities — so the two legitimately differ by the internals. + const pageHonest = (await brain.storage.getNounsWithPagination({ limit: 1000, offset: 0 })).totalCount + + // Simulate the historical drift: an inflated persisted scalar (deletes whose + // decrement was skipped). Persist it so a reopen rehydrates the lie. + ;(brain.storage as any).totalNounCount = honest + 52 + await (brain.storage as any).persistCounts() + await brain.close() + brain = await open() + expect(await brain.storage.getNounCount()).toBe(honest + 52) // the lie survived reopen + + // The sanctioned recount — unconditional in repairIndex (no orphans needed). + await brain.repairIndex() + expect(await brain.storage.getNounCount()).toBe(honest) + + // Permanently: the corrected counter survives another reopen. + await brain.close() + brain = await open() + expect(await brain.storage.getNounCount()).toBe(honest) + + // And the paginated totalCount (the Math.max consumer) is honest too — + // back to ITS baseline, no longer pinned high by the inflated scalar. + const page = await brain.storage.getNounsWithPagination({ limit: 1000, offset: 0 }) + expect(page.totalCount).toBe(pageHonest) + }) +}) diff --git a/tests/integration/db-mvcc.test.ts b/tests/integration/db-mvcc.test.ts index 764a755a..959d0053 100644 --- a/tests/integration/db-mvcc.test.ts +++ b/tests/integration/db-mvcc.test.ts @@ -515,7 +515,15 @@ describe('8.0 Db API — generational MVCC', () => { await brain.transact([{ op: 'update', id: uid('compact-e'), metadata: { v: 4 } }]) ).release() - const recordsBefore = (await storage.listRawObjects('_generations')).length + // History record-sets only — the generation FACT LOG also lives under + // `_generations/` (at `facts/`) and is deliberately NOT reclaimed by + // history compaction (facts are the future canonical, not undo history). + const historyRecords = async (): Promise => + (await storage.listRawObjects('_generations')).filter( + (p: string) => !p.startsWith('_generations/facts/') + ).length + + const recordsBefore = await historyRecords() expect(recordsBefore).toBeGreaterThan(0) // Compact while pinned: record-sets above the pin survive, pinned reads stay correct. @@ -528,7 +536,7 @@ describe('8.0 Db API — generational MVCC', () => { const second = await brain.compactHistory() expect(first.removedGenerations + second.removedGenerations).toBeGreaterThan(0) - const recordsAfter = (await storage.listRawObjects('_generations')).length + const recordsAfter = await historyRecords() expect(recordsAfter).toBeLessThan(recordsBefore) expect(recordsAfter).toBe(0) @@ -1272,24 +1280,32 @@ describe('8.0 Db API — generational MVCC', () => { await expect(reopened.asOf(1)).rejects.toBeInstanceOf(GenerationCompactedError) }) - it('Model-B retention — setRetentionBudget drives adaptive reclaim on flush; live data intact', async () => { - // Default brain → ADAPTIVE retention. A coordinator (e.g. cor's ResourceManager) - // pushes a byte budget via setRetentionBudget(); auto-compaction on flush() reclaims - // oldest history down toward it. Each update's before-image carries the full prior - // 384-dim vector (~KBs), so ~13 generations far exceed a few-KB budget. - const { brain } = await openFsBrain() + it('Model-B retention — flush() NEVER compacts (8.9.0); adaptive reclaim runs at close()', async () => { + // Default brain → ADAPTIVE retention with a driven byte budget far below + // the accumulated history (~13 generations of full-vector before-images). + // The 8.9.0 law: flush() is durability-only — it must not reclaim even + // when the budget is exceeded (reclaim-on-flush blocked production writes + // for 25-191s). Maintenance runs at close(), time-bounded. + const { brain, dir } = await openFsBrain() const a = uid('ret-budget') await brain.add({ id: a, type: NounType.Document, data: 'v0', vector: vec(1), metadata: { v: 0 } }) for (let v = 1; v <= 12; v++) await brain.update({ id: a, metadata: { v } }) brain.setRetentionBudget(6000) // ~6 KB — well below the accumulated history - await brain.flush() // group-commit + adaptive auto-compaction under the budget + await brain.flush() - // History was reclaimed (the horizon advanced past the oldest generations)… - expect(generationStoreOf(brain).horizon()).toBeGreaterThan(0) - await expect(brain.asOf(1)).rejects.toBeInstanceOf(GenerationCompactedError) - // …but the budget reclaims HISTORY only — the live record is untouched. - expect((await brain.get(a))?.metadata?.v).toBe(12) + // flush() paid durability only: nothing reclaimed, all history readable. + expect(generationStoreOf(brain).horizon()).toBe(0) + const probe = await brain.asOf(1) // readable proves nothing was reclaimed… + await probe.release() // …and MUST be released: a held pin would (correctly) + // protect every newer generation through the close() compaction below. + await brain.close() // ← THE auto-compaction site now + + // close() reclaimed under the budget; live record intact; horizon durable. + const { brain: reopened } = await openFsBrain(dir) + expect(generationStoreOf(reopened).horizon()).toBeGreaterThan(0) + await expect(reopened.asOf(1)).rejects.toBeInstanceOf(GenerationCompactedError) + expect((await reopened.get(a))?.metadata?.v).toBe(12) }) // ========================================================================== diff --git a/tests/integration/delete-full-removal.test.ts b/tests/integration/delete-full-removal.test.ts new file mode 100644 index 00000000..83675a13 --- /dev/null +++ b/tests/integration/delete-full-removal.test.ts @@ -0,0 +1,161 @@ +/** + * @module tests/integration/delete-full-removal + * @description Canonical noun/verb deletes are FULL removals: both legs + * (metadata + vectors) AND the entity's `/` container are removed, so a + * delete leaves NOTHING behind. Regression for the ghost/scar defect where + * remove() deleted the vector INDEX entry + the canonical metadata leg but never + * the canonical vectors.json leg or the directory — leaving an orphan that reads + * as absent (getNoun needs both legs) yet inflated the enumerated count and + * confused locator resolution. Also covers the operator repair sweep + * (brain.repairIndex()) that prunes orphans left by the pre-fix behavior. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/index.js' + +/** All entity-directory names (the `` dirs) under entities/. */ +function entityDirs(root: string, kind: 'nouns' | 'verbs'): string[] { + const base = path.join(root, 'entities', kind) + if (!fs.existsSync(base)) return [] + const ids: string[] = [] + for (const shard of fs.readdirSync(base)) { + const shardDir = path.join(base, shard) + if (!fs.statSync(shardDir).isDirectory()) continue + for (const id of fs.readdirSync(shardDir)) { + if (fs.statSync(path.join(shardDir, id)).isDirectory()) ids.push(id) + } + } + return ids +} + +/** Absolute path of one entity's `` directory (or null if not present). */ +function entityDir(root: string, kind: 'nouns' | 'verbs', id: string): string | null { + const base = path.join(root, 'entities', kind) + if (!fs.existsSync(base)) return null + for (const shard of fs.readdirSync(base)) { + const candidate = path.join(base, shard, id) + if (fs.existsSync(candidate)) return candidate + } + return null +} + +describe('canonical delete is a full removal (no ghost/scar directory)', () => { + let dir: string + let brain: any + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-del-')) + brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 }) + await brain.init() + }) + afterEach(async () => { + await brain.close?.().catch(() => {}) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('remove() deletes BOTH legs and the container — nothing left on disk', async () => { + const ids: string[] = [] + for (let i = 0; i < 3; i++) ids.push(await brain.add({ data: `doc ${i}`, type: 'document', metadata: { i } })) + await brain.flush() + + // (A filesystem brain also has its VFS-root noun, so don't assume an exact set.) + const before = entityDirs(dir, 'nouns') + expect(before).toEqual(expect.arrayContaining(ids)) + + await brain.remove(ids[1]) + await brain.flush() + + // getNoun is null AND the on-disk container is fully gone (no orphan), and + // EXACTLY one directory disappeared (the removed entity's). + expect(await brain.get(ids[1])).toBeNull() + expect(entityDir(dir, 'nouns', ids[1])).toBeNull() + const after = entityDirs(dir, 'nouns') + expect(after).toHaveLength(before.length - 1) + expect(after).toEqual(expect.arrayContaining([ids[0], ids[2]])) + expect(after).not.toContain(ids[1]) + }) + + it('the enumerated count is honest after a delete (no monotonic inflation)', async () => { + const ids: string[] = [] + for (let i = 0; i < 4; i++) ids.push(await brain.add({ data: `n${i}`, type: 'document', metadata: { i } })) + await brain.flush() + + const before = await brain.storage.getNounsWithPagination({ limit: 100, offset: 0 }) + + await brain.remove(ids[0]) + await brain.remove(ids[1]) + await brain.flush() + + // The count drops by EXACTLY the two removed entities — no ghost lingers to + // hold the total up (the pre-fix defect left it monotonic). + const after = await brain.storage.getNounsWithPagination({ limit: 100, offset: 0 }) + expect(after.totalCount).toBe(before.totalCount - 2) + const remaining = after.items.map((n: any) => n.id) + expect(remaining).toEqual(expect.arrayContaining([ids[2], ids[3]])) + expect(remaining).not.toContain(ids[0]) + expect(remaining).not.toContain(ids[1]) + }) +}) + +describe('repairIndex() prunes orphan containers left by the pre-fix partial delete', () => { + let dir: string + let brain: any + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-orphan-')) + brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 }) + await brain.init() + }) + afterEach(async () => { + await brain.close?.().catch(() => {}) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('a vector-only "ghost" (metadata leg deleted out-of-band) is pruned', async () => { + const ids: string[] = [] + for (let i = 0; i < 3; i++) ids.push(await brain.add({ data: `g${i}`, type: 'document', metadata: { i } })) + await brain.flush() + + // Simulate the pre-fix defect on ids[0]: remove ONLY its metadata leg, + // leaving vectors.json + the directory (the exact ghost shape). + const ghostDir = entityDir(dir, 'nouns', ids[0])! + for (const f of fs.readdirSync(ghostDir)) { + if (f.startsWith('metadata.json')) fs.rmSync(path.join(ghostDir, f)) + } + // The ghost dir (vectors.json, no metadata content leg) still sits on disk. + expect(entityDir(dir, 'nouns', ids[0])).not.toBeNull() + + await brain.repairIndex() + + // The sweep removes the ghost container; the two healthy entities survive. + expect(entityDir(dir, 'nouns', ids[0])).toBeNull() + expect(entityDir(dir, 'nouns', ids[1])).not.toBeNull() + expect(entityDir(dir, 'nouns', ids[2])).not.toBeNull() + }) + + it('an empty "scar" directory is pruned', async () => { + const id = await brain.add({ data: 'lonely', type: 'document', metadata: {} }) + await brain.flush() + const scarDir = entityDir(dir, 'nouns', id)! + for (const f of fs.readdirSync(scarDir)) fs.rmSync(path.join(scarDir, f)) // empty the dir, keep it + expect(fs.existsSync(scarDir)).toBe(true) + + await brain.repairIndex() + + expect(fs.existsSync(scarDir)).toBe(false) + }) + + it('a healthy entity (both legs present) is NEVER pruned', async () => { + const id = await brain.add({ data: 'keep me', type: 'document', metadata: { keep: true } }) + await brain.flush() + + await brain.repairIndex() + + expect(entityDir(dir, 'nouns', id)).not.toBeNull() + expect(await brain.get(id)).not.toBeNull() + }) +}) diff --git a/tests/integration/entity-tree-stamp.test.ts b/tests/integration/entity-tree-stamp.test.ts new file mode 100644 index 00000000..deefc5e6 --- /dev/null +++ b/tests/integration/entity-tree-stamp.test.ts @@ -0,0 +1,163 @@ +/** + * @module tests/integration/entity-tree-stamp + * @description The entity tree's FAMILY STAMP: written at flush/close with + * `sourceGeneration` (the committed generation the canonical tree reflects) + * plus rollup invariants (entity/relationship counts); verified at open by + * comparison — coherent on a clean close, LOUD on genuine incoherence + * (tampered counters), healed by repairIndex()'s recount + re-stamp. One + * verifier reads both member modes. + */ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { + Brainy, + readFamilyStamp, + verifyFamilyStamp, + ENTITY_TREE_STAMP_PATH, + type FamilyStamp +} from '../../src/index.js' +import { prodLog } from '../../src/utils/logger.js' + +describe('entity-tree family stamp', () => { + let dir: string + let brain: any + + const open = async () => { + const b: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) + await b.init() + return b + } + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-stamp-')) + brain = await open() + }) + afterEach(async () => { + vi.restoreAllMocks() + await brain.close?.().catch(() => {}) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('flush() writes the stamp: rollup mode, counts + sourceGeneration match live state', async () => { + for (let i = 0; i < 3; i++) await brain.add({ data: `s${i}`, type: 'document', metadata: { i } }) + await brain.flush() + + const stamp = (await readFamilyStamp(brain.storage, ENTITY_TREE_STAMP_PATH)) as FamilyStamp + expect(stamp).not.toBeNull() + expect(stamp.family).toBe('entity-tree') + expect(stamp.members.mode).toBe('rollup') + const invariants = (stamp.members as any).invariants + expect(invariants.nounCount).toBe(await brain.storage.getNounCount()) + expect(invariants.verbCount).toBe(await brain.storage.getVerbCount()) + expect(stamp.sourceGeneration).toBe(brain.generation()) + expect(stamp.generation).toBeGreaterThanOrEqual(1) + }) + + it('a cleanly-closed store reopens COHERENT (no incoherence warning)', async () => { + await brain.add({ data: 'clean', type: 'document', metadata: {} }) + await brain.close() + + const warn = vi.spyOn(prodLog, 'warn') + brain = await open() + const stampWarnings = warn.mock.calls.filter((c) => String(c[0]).includes('entity-tree stamp')) + expect(stampWarnings).toEqual([]) + }) + + it('tampered counters surface as INCOHERENT at open; repairIndex() heals + re-stamps', async () => { + for (let i = 0; i < 3; i++) await brain.add({ data: `t${i}`, type: 'document', metadata: { i } }) + await brain.close() + + // Simulate counter drift AFTER the stamp was written: inflate the + // persisted scalar the way the historical decrement-skip did. + brain = await open() + ;(brain.storage as any).totalNounCount += 52 + await (brain.storage as any).persistCounts() + await brain.close() + // The close boundary re-stamps with the inflated counter — so tamper the + // STAMP instead for a deterministic mismatch: stamped counts differ from + // the (inflated) live ones at the NEXT open only if the stamp is older. + // Rewrite the stamp with the honest counts + current sourceGeneration. + const raw = JSON.parse( + require('node:zlib') + .gunzipSync(fs.readFileSync(path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`))) + .toString('utf-8') + ) as FamilyStamp + const honest = { ...raw } + ;(honest.members as any).invariants.nounCount -= 52 + fs.writeFileSync( + path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`), + require('node:zlib').gzipSync(JSON.stringify(honest)) + ) + + const warn = vi.spyOn(prodLog, 'warn') + brain = await open() + const incoherent = warn.mock.calls.filter((c) => String(c[0]).includes('INCOHERENT')) + expect(incoherent.length).toBeGreaterThanOrEqual(1) + expect(String(incoherent[0][0])).toMatch(/nounCount/) + + // The heal: recount from canonical + re-stamp → next open is quiet. + await brain.repairIndex() + await brain.close() + const warn2 = vi.spyOn(prodLog, 'warn') + brain = await open() + const stillIncoherent = warn2.mock.calls.filter((c) => String(c[0]).includes('INCOHERENT')) + expect(stillIncoherent).toEqual([]) + }) + + it('the one verifier handles both member modes', () => { + const rollup: FamilyStamp = { + family: 'x', + generation: 1, + committedAt: new Date().toISOString(), + sourceGeneration: 5, + members: { mode: 'rollup', invariants: { nounCount: 10 } } + } + expect(verifyFamilyStamp(rollup, 5, { nounCount: 10 })).toEqual({ state: 'coherent' }) + expect(verifyFamilyStamp(rollup, 5, { nounCount: 11 }).state).toBe('incoherent') + expect(verifyFamilyStamp(rollup, 9, { nounCount: 10 })).toEqual({ + state: 'behind', + stampSource: 5, + head: 9 + }) + expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 }).state).toBe('incoherent') // ahead of head + expect(verifyFamilyStamp(null, 5, {})).toEqual({ state: 'absent' }) + + const enumerated: FamilyStamp = { + family: 'y', + generation: 1, + committedAt: new Date().toISOString(), + sourceGeneration: 2, + members: { mode: 'enumerated', files: [{ path: 'a.bin', bytes: 128 }] } + } + expect(verifyFamilyStamp(enumerated, 2, { 'a.bin': 128 })).toEqual({ state: 'coherent' }) + expect(verifyFamilyStamp(enumerated, 2, { 'a.bin': 64 }).state).toBe('incoherent') + expect(verifyFamilyStamp(enumerated, 2, {}).state).toBe('incoherent') // missing member + + // Rollup invariants may be STRING fingerprints (e.g. a per-tree SHA-256): + // strict equality either way; a type mismatch reads as incoherence. + const fingerprinted: FamilyStamp = { + family: 'z', + generation: 1, + committedAt: new Date().toISOString(), + sourceGeneration: 3, + members: { mode: 'rollup', invariants: { treeDigest: 'abc123', rows: 42 } } + } + expect(verifyFamilyStamp(fingerprinted, 3, { treeDigest: 'abc123', rows: 42 })).toEqual({ + state: 'coherent' + }) + expect(verifyFamilyStamp(fingerprinted, 3, { treeDigest: 'deadbeef', rows: 42 }).state).toBe( + 'incoherent' + ) + expect(verifyFamilyStamp(fingerprinted, 3, { treeDigest: 'abc123', rows: '42' }).state).toBe( + 'incoherent' // type mismatch never passes + ) + }) +}) diff --git a/tests/integration/fact-log-contracts.test.ts b/tests/integration/fact-log-contracts.test.ts new file mode 100644 index 00000000..eb579da9 --- /dev/null +++ b/tests/integration/fact-log-contracts.test.ts @@ -0,0 +1,125 @@ +/** + * @module tests/integration/fact-log-contracts + * @description Pinned durability + stability contracts for the fact log. + * + * (1) FSYNC-BEFORE-ACK: an acknowledged write's fact survives an abrupt + * process end (no flush, no close — reopen from disk). + * - transact(): HOLDS TODAY — the fact is fsync'd before transact returns. + * - single-op: PINNED AS `it.fails` — today's group-commit batches + * DURABILITY (ack precedes the group fsync; a hard kill loses the fact + * AND the generation together, coherently — the documented Model-B + * contract, fine while the tree is authoritative). The destination + * (ack-at-log) requires group commit to become LATENCY batching: the + * ack waits for the shared fsync. When that lands, this pin flips red — + * remove `.fails` and the contract is permanent. No cliff to discover. + * + * (2) SCAN STABILITY UNDER ROTATION: a scan handle opened before segment + * rotation yields exactly its snapshot — byte-identical facts, no gaps, + * no duplicates, and no bleed-in of facts appended after the snapshot. + * (The reclaim-during-scan variant lands with fact-log compaction, which + * does not exist yet — segments only rotate today, never reclaim.) + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy, type CommitFact } from '../../src/index.js' +import { MemoryStorage } from '../../src/storage/adapters/memoryStorage.js' +import { FactLog, type FactLogStorage } from '../../src/db/factLog.js' + +describe('fsync-before-ack contract (fact durability at the ack boundary)', () => { + let dir: string + let brain: any + + const open = async () => { + const b: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) + await b.init() + return b + } + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-factack-')) + brain = await open() + }) + afterEach(async () => { + await brain.close?.().catch(() => {}) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('transact(): the fact is durable the moment the ack returns (kill-after-ack safe)', async () => { + const receipt = await brain.transact([ + { op: 'add', type: 'document', metadata: { durable: 1 }, data: 'ack-at-commit' } + ]) + // Abrupt end: no flush(), no close() — a new instance reads only disk. + brain = await open() + const facts: CommitFact[] = [] + for await (const b of brain.scanFacts()!.batches()) facts.push(...b.facts) + expect(facts.some((f) => f.generation === receipt.generation)).toBe(true) + }) + + // PINNED (flips red when group commit becomes latency batching — then + // remove `.fails` and the ack-at-log contract is permanent on every path). + it.fails('single-op: the fact is durable the moment the ack returns (the ack-at-log target)', async () => { + await brain.add({ data: 'acked single-op', type: 'document', metadata: { n: 1 } }) + const ackedHead = brain.scanFacts()!.headGeneration + // Abrupt end immediately after the ack — before any flush window. + brain = await open() + const facts: CommitFact[] = [] + for await (const b of brain.scanFacts()!.batches()) facts.push(...b.facts) + expect(facts.some((f) => f.generation === ackedHead)).toBe(true) + }) +}) + +describe('scan stability under rotation (the snapshot contract)', () => { + const UUID = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, '0')}` + const fact = (generation: number): CommitFact => ({ + generation, + timestamp: 1_700_000_000_000 + generation, + ops: [ + { + kind: 'noun', + id: UUID(generation), + // Padding makes each frame ~1KB so a small rotateBytes forces rotations. + record: { metadata: { noun: 'document', pad: 'x'.repeat(900), g: generation }, vector: null } + } + ] + }) + + it('a scan opened before rotations yields its exact snapshot — no gaps, dups, or bleed-in', async () => { + const mem: any = new MemoryStorage() + await mem.init() + const log = new FactLog(mem as FactLogStorage, { rotateBytes: 4096 }) // ~4 facts per segment + await log.open(0) + for (let g = 1; g <= 10; g++) await log.append(fact(g)) + await log.sync() + + // Open the snapshot, THEN keep appending — forcing further rotations. + const scan = log.scanFacts() + expect(scan.headGeneration).toBe(10) + for (let g = 11; g <= 25; g++) await log.append(fact(g)) + await log.sync() + expect(log.headGeneration()).toBe(25) + + const seen: number[] = [] + for await (const batch of scan.batches()) { + for (const f of batch.facts) seen.push(f.generation) + } + // Exactly the snapshot: 1..10 in order, nothing appended-after bleeds in. + expect(seen).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + expect(scan.summary().factsYielded).toBe(10) + + // And a fresh scan sees everything, across all rotated segments. + const all: number[] = [] + for await (const batch of log.scanFacts().batches()) { + for (const f of batch.facts) all.push(f.generation) + } + expect(all).toEqual(Array.from({ length: 25 }, (_, i) => i + 1)) + expect(log.segmentPaths().length).toBeGreaterThanOrEqual(2) // rotations actually happened + }) +}) diff --git a/tests/integration/fact-log-dual-write.test.ts b/tests/integration/fact-log-dual-write.test.ts new file mode 100644 index 00000000..5ec66273 --- /dev/null +++ b/tests/integration/fact-log-dual-write.test.ts @@ -0,0 +1,219 @@ +/** + * @module tests/integration/fact-log-dual-write + * @description The generation fact log end-to-end through real commits: every + * committed generation (single-op AND transact) appends its AFTER-IMAGE fact + * at the commit point; removals append body-less tombstones; an aborted + * transaction leaves no fact; facts survive reopen and continue monotonically; + * the scan surface (brain.scanFacts) carries the frozen telemetry shape; and + * the fact-log namespace is protected against prefix-nuking. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy, ProtectedArtifactError, type CommitFact } from '../../src/index.js' + +async function allFacts(brain: any): Promise { + const scan = brain.scanFacts() + expect(scan).not.toBeNull() + const facts: CommitFact[] = [] + for await (const batch of scan!.batches()) facts.push(...batch.facts) + return facts +} + +describe('fact log dual-write (memory adapter)', () => { + let brain: any + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true, dimensions: 384 }) + await brain.init() + }) + afterEach(async () => { + await brain.close?.().catch(() => {}) + }) + + it('every single-op write appends its after-image fact; a remove appends a tombstone', async () => { + const id = await brain.add({ data: 'first', type: 'document', metadata: { rev: 1 } }) + await brain.update({ id, metadata: { rev: 2 } }) + await brain.remove(id) + + const facts = await allFacts(brain) + // add + update + remove each committed a generation (the remove may span + // cascade ops but is ONE generation). Facts are monotonic. + const gens = facts.map((f) => f.generation) + expect([...gens].sort((a, b) => a - b)).toEqual(gens) + expect(facts.length).toBeGreaterThanOrEqual(3) + + // The add fact carries the after-image of the new entity. + const addFact = facts.find((f) => f.ops.some((op) => op.id === id && op.record !== null)) + expect(addFact).toBeDefined() + + // The remove fact carries a body-less tombstone for the id. + const removeFact = facts[facts.length - 1] + const tombstone = removeFact.ops.find((op) => op.id === id) + expect(tombstone).toBeDefined() + expect(tombstone!.record).toBeNull() + expect(tombstone!.kind).toBe('noun') + }) + + it('the update fact holds the NEW state (after-image, not before)', async () => { + const id = await brain.add({ data: 'versioned', type: 'document', metadata: { v: 'old' } }) + await brain.update({ id, metadata: { v: 'new' } }) + + const facts = await allFacts(brain) + const updateFact = facts[facts.length - 1] + const op = updateFact.ops.find((o) => o.id === id)! + expect(op.record).not.toBeNull() + expect((op.record!.metadata as any).v).toBe('new') + }) + + it('a transact commits ONE fact carrying all its ops, with meta', async () => { + const receipt = await brain.transact( + [ + { op: 'add', type: 'document', metadata: { part: 1 }, data: 'a' }, + { op: 'add', type: 'document', metadata: { part: 2 }, data: 'b' } + ], + { meta: { source: 'batch-import' } } + ) + + const facts = await allFacts(brain) + const txFact = facts.find((f) => f.generation === receipt.generation) + expect(txFact).toBeDefined() + expect(txFact!.ops.filter((op) => op.kind === 'noun').length).toBeGreaterThanOrEqual(2) + expect(txFact!.meta).toEqual({ source: 'batch-import' }) + }) + + it('an aborted transact leaves NO fact (absent = never committed)', async () => { + const id = await brain.add({ data: 'cas target', type: 'document', metadata: { n: 1 } }) + const before = (await allFacts(brain)).length + + await expect( + brain.transact([{ op: 'update', id, ifRev: 999, metadata: { n: 2 } }]) + ).rejects.toThrow() + + const after = await allFacts(brain) + expect(after.length).toBe(before) + }) + + it('fact generations line up with the transaction log', async () => { + await brain.add({ data: 'x', type: 'document', metadata: {} }) + await brain.add({ data: 'y', type: 'document', metadata: {} }) + await brain.flush() + + const facts = await allFacts(brain) + const logGens = new Set((await brain.transactionLog()).map((e: any) => e.generation)) + for (const f of facts) { + expect(logGens.has(f.generation)).toBe(true) + } + }) + + it('the storage fact-scan capability serves a provider holding only `storage`', async () => { + // An index provider receives `storage` — never the brain — and reaches the + // fact log through the host-wired capability (it must never construct its + // own fact-log reader: the log's open path is writer-side). + const id = await brain.add({ data: 'via storage', type: 'document', metadata: { s: 1 } }) + await brain.remove(id) + + const storage = brain.storage + expect(typeof storage.scanFacts).toBe('function') + expect(storage.factLogHeadGeneration()).toBe(brain.scanFacts()!.headGeneration) + + const viaStorage: CommitFact[] = [] + for await (const b of storage.scanFacts()!.batches()) viaStorage.push(...b.facts) + const viaBrain: CommitFact[] = [] + for await (const b of brain.scanFacts()!.batches()) viaBrain.push(...b.facts) + expect(viaStorage.map((f) => f.generation)).toEqual(viaBrain.map((f) => f.generation)) + expect(storage.factSegmentPaths()).toEqual(brain.factSegmentPaths()) + }) + + it('scan telemetry carries the frozen shape end-to-end', async () => { + for (let i = 0; i < 5; i++) await brain.add({ data: `t${i}`, type: 'document', metadata: { i } }) + + const scan = brain.scanFacts({ batchSize: 2 })! + expect(scan.headGeneration).toBeGreaterThanOrEqual(5) + expect(scan.approxFactCount).toBeGreaterThanOrEqual(5) + let batches = 0 + for await (const b of scan.batches()) { + batches++ + expect(b.factCount).toBe(b.facts.length) + expect(b.firstGeneration).toBe(b.facts[0].generation) + expect(b.lastGeneration).toBe(b.facts[b.facts.length - 1].generation) + expect(b.byteSize).toBeGreaterThan(0) + expect(typeof b.segmentId).toBe('string') + } + expect(batches).toBeGreaterThan(1) + expect(scan.summary().factsYielded).toBe(scan.approxFactCount) + }) +}) + +describe('fact log dual-write (filesystem adapter — durability + protection)', () => { + let dir: string + let brain: any + + const open = async () => { + const b: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) + await b.init() + return b + } + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-factlog-')) + brain = await open() + }) + afterEach(async () => { + await brain.close?.().catch(() => {}) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('facts survive close + reopen and appends continue monotonically', async () => { + const id = await brain.add({ data: 'persist me', type: 'document', metadata: { k: 1 } }) + await brain.remove(id) + await brain.close() + + brain = await open() + const facts = await allFacts(brain) + expect(facts.length).toBeGreaterThanOrEqual(2) + const headBefore = facts[facts.length - 1].generation + + await brain.add({ data: 'after reopen', type: 'document', metadata: { k: 2 } }) + const facts2 = await allFacts(brain) + expect(facts2[facts2.length - 1].generation).toBeGreaterThan(headBefore) + }) + + it('the fact segments exist on disk under _generations/facts/ with zero-padded names', async () => { + await brain.add({ data: 'on disk', type: 'document', metadata: {} }) + await brain.flush() + const factsDir = path.join(dir, '_generations', 'facts') + const files = fs.readdirSync(factsDir) + // The manifest rides the store's JSON object discipline (gzip on disk). + expect(files.some((f) => f.startsWith('manifest.json'))).toBe(true) + const segs = files.filter((f) => /^seg-\d{20}\.bfl$/.test(f)) + expect(segs.length).toBeGreaterThanOrEqual(1) + }) + + it('the fact-log namespace is PROTECTED: a prefix-nuke is refused', async () => { + await brain.add({ data: 'protected', type: 'document', metadata: {} }) + await expect(brain.storage.removeRawPrefix('_generations/facts')).rejects.toBeInstanceOf( + ProtectedArtifactError + ) + // Per-generation history cleanup remains unaffected (no false intersect). + await expect(brain.storage.removeRawPrefix('_generations/999999')).resolves.toBeUndefined() + }) + + it('transact facts are durable-on-return (no flush needed before reopen)', async () => { + const receipt = await brain.transact([ + { op: 'add', type: 'document', metadata: { durable: true }, data: 'tx' } + ]) + // Simulate an abrupt end: no flush(), no close() — reopen from disk. + brain = await open() + const facts = await allFacts(brain) + expect(facts.some((f) => f.generation === receipt.generation)).toBe(true) + }) +}) diff --git a/tests/integration/graph-audit.test.ts b/tests/integration/graph-audit.test.ts new file mode 100644 index 00000000..74fcc96a --- /dev/null +++ b/tests/integration/graph-audit.test.ts @@ -0,0 +1,164 @@ +/** + * @module tests/integration/graph-audit + * @description brain.auditGraph() — the read-only graph-truth instrument. + * Laws under test: + * (1) a healthy brain audits COHERENT: every canonical verb is returned by the + * read path of its source, endpoints exist, no ghosts; + * (2) design-hidden edges (internal/system visibility) are counted separately + * and never misclassified as index loss; + * (3) a verb whose endpoint entity was destroyed at the storage layer (the + * scar class) is flagged as a dangling endpoint, loudly; + * (4) the classification core flags present-but-invisible and ghost edges + * exactly (exercised via injected seams — manufacturing a genuinely stale + * adjacency index end-to-end would require corrupting internals the + * public API rightly refuses to corrupt). + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/index.js' +import { NounType, VerbType } from '../../src/types/graphTypes.js' +import { runGraphAudit, type AuditVerbRecord } from '../../src/graph/graphAudit.js' + +describe('brain.auditGraph() — graph-truth audit', () => { + let dir: string + let brain: any + + beforeEach(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-graph-audit-')) + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true + }) + await brain.init() + }) + + afterEach(async () => { + await brain.close().catch(() => {}) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + async function seedGraph(): Promise<{ ids: string[]; verbIds: string[] }> { + const ids: string[] = [] + for (let i = 0; i < 5; i++) { + ids.push( + await brain.add({ + data: `entity ${i}`, + type: NounType.Concept, + metadata: { n: i } + }) + ) + } + const verbIds: string[] = [] + verbIds.push(await brain.relate({ from: ids[0], to: ids[1], type: VerbType.RelatedTo })) + verbIds.push(await brain.relate({ from: ids[0], to: ids[2], type: VerbType.Contains })) + verbIds.push(await brain.relate({ from: ids[1], to: ids[3], type: VerbType.DependsOn })) + verbIds.push(await brain.relate({ from: ids[3], to: ids[4], type: VerbType.RelatedTo })) + return { ids, verbIds } + } + + it('audits a healthy brain as coherent, with exact counts', async () => { + const { ids } = await seedGraph() + const report = await brain.auditGraph() + + expect(report.coherent).toBe(true) + expect(report.verbsInCanonical).toBe(4) + expect(report.entitiesInCanonical).toBeGreaterThanOrEqual(ids.length) // VFS root etc. may add system nouns + expect(report.sourcesChecked).toBe(3) // ids[0], ids[1], ids[3] + expect(report.missingFromReadsCount).toBe(0) + expect(report.danglingEndpointsCount).toBe(0) + expect(report.readOnlyCount).toBe(0) + expect(report.truncatedExamples).toBe(false) + }) + + it('counts design-hidden edges separately and stays coherent', async () => { + const { ids } = await seedGraph() + await brain.relate({ + from: ids[2], + to: ids[4], + type: VerbType.RelatedTo, + visibility: 'internal' + }) + + const report = await brain.auditGraph() + expect(report.coherent).toBe(true) // hidden-by-design is NOT a discrepancy + expect(report.verbsInCanonical).toBe(5) + expect(report.visibilityHiddenCount).toBeGreaterThanOrEqual(1) + }) + + it('flags a destroyed endpoint as a dangling verb (the scar class)', async () => { + const { ids } = await seedGraph() + // Destroy ids[4] at the STORAGE layer (bypassing remove(), which would + // also delete its verbs) — the historical partial-delete scar shape. + await brain.storage.deleteNoun(ids[4]) + + const report = await brain.auditGraph() + expect(report.coherent).toBe(false) + expect(report.danglingEndpointsCount).toBe(1) + expect(report.danglingEndpoints[0].to).toBe(ids[4]) + expect(report.danglingEndpoints[0].missingEnd).toBe('to') + }) +}) + +describe('runGraphAudit classification core (injected seams)', () => { + const verb = (id: string, from: string, to: string): AuditVerbRecord => ({ + id, + type: 'relatedTo', + sourceId: from, + targetId: to + }) + + const deps = (opts: { + nouns: string[] + verbs: AuditVerbRecord[] + reads: Record // sourceId -> verb ids the read path returns + }) => ({ + eachNounId: async (consume: (id: string) => void) => { + for (const id of opts.nouns) consume(id) + }, + eachVerb: async (consume: (v: AuditVerbRecord) => void) => { + for (const v of opts.verbs) consume(v) + }, + readRelationsFrom: async (sourceId: string) => + (opts.reads[sourceId] ?? []).map((id) => ({ id })) + }) + + it('flags a canonical verb the read path omits — present but invisible', async () => { + const report = await runGraphAudit( + deps({ + nouns: ['A', 'B', 'C'], + verbs: [verb('v1', 'A', 'B'), verb('v2', 'A', 'C')], + reads: { A: ['v1'] } // v2 exists canonically but reads miss it + }) + ) + expect(report.coherent).toBe(false) + expect(report.missingFromReadsCount).toBe(1) + expect(report.missingFromReads[0].verbId).toBe('v2') + }) + + it('flags a read-path edge with no canonical record — a ghost', async () => { + const report = await runGraphAudit( + deps({ + nouns: ['A', 'B'], + verbs: [verb('v1', 'A', 'B')], + reads: { A: ['v1', 'ghost-9'] } + }) + ) + expect(report.coherent).toBe(false) + expect(report.readOnlyCount).toBe(1) + expect(report.readOnlyVerbIds).toEqual(['ghost-9']) + }) + + it('caps example lists but keeps counts exact, and says so', async () => { + const verbs = Array.from({ length: 10 }, (_, i) => verb(`v${i}`, 'A', 'B')) + const report = await runGraphAudit( + deps({ nouns: ['A', 'B'], verbs, reads: { A: [] } }), + { maxExamples: 3 } + ) + expect(report.missingFromReadsCount).toBe(10) + expect(report.missingFromReads.length).toBe(3) + expect(report.truncatedExamples).toBe(true) + }) +}) diff --git a/tests/integration/history-repacking.test.ts b/tests/integration/history-repacking.test.ts new file mode 100644 index 00000000..2bcee038 --- /dev/null +++ b/tests/integration/history-repacking.test.ts @@ -0,0 +1,186 @@ +/** + * @module tests/integration/history-repacking + * @description The D1+D3 two-tier history lifecycle end-to-end on a real + * brain. Laws: (1) repacking is RE-REPRESENTATION — after folding, every + * asOf() read below the fold boundary answers exactly as before, across a + * cold reopen; (2) folded per-generation directories are physically gone + * (the file-count cure is real, not cosmetic); (3) repack + reclaim compose: + * bounded retention after repacking drops whole segments and asOf below the + * horizon throws GenerationCompactedError; (4) repackHistory is explicit + * API and time-bounded (spent budget = consistent no-op). + * + * Uses a tiny REPACK_LIVE_WINDOW override so a small history has a cold + * tier at all (the production window is 1024). + */ +import { describe, it, expect, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as path from 'node:path' +import * as os from 'node:os' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import { GenerationStore } from '../../src/db/generationStore.js' +import { GenerationCompactedError } from '../../src/db/errors.js' +import { SEGMENTS_PREFIX } from '../../src/db/generationSegments.js' + +const stub = async (text: string): Promise => { + const h = text.split('').reduce((a, c) => a + c.charCodeAt(0), 0) + return new Array(384).fill(0).map((_, i) => Math.sin(h + i)) +} + +const openBrain = async (dir: string): Promise => { + const brain = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + embeddingFunction: stub + }) + await brain.init() + return brain +} + +describe('history repacking — the two-tier lifecycle', () => { + const dirs: string[] = [] + const tempDir = (): string => { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-repack-')) + dirs.push(d) + return d + } + const originalWindow = GenerationStore.REPACK_LIVE_WINDOW + + afterEach(() => { + ;(GenerationStore as any).REPACK_LIVE_WINDOW = originalWindow + for (const d of dirs.splice(0)) { + try { + fs.rmSync(d, { recursive: true, force: true }) + } catch { + /* best effort */ + } + } + }) + + it('repack preserves every historical read across cold reopen; folded dirs are gone', async () => { + ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3 + const dir = tempDir() + const brain = await openBrain(dir) + + const id = await brain.add({ + data: 'versioned-entity', + type: NounType.Document, + metadata: { v: 0 } + }) + for (let v = 1; v <= 10; v++) await brain.update({ id, metadata: { v } }) + await brain.flush() + + // Ground truth BEFORE repacking: capture asOf views for early generations. + const before: Record = {} + for (const g of [2, 4, 6]) { + const db = await brain.asOf(g) + before[g] = (await db.get(id))?.metadata?.v as number + await db.release() + } + + const result = await brain.repackHistory() + expect(result.foldedGenerations).toBeGreaterThan(0) + expect(result.segmentsCreated).toBeGreaterThan(0) + + // The folded per-generation directories are PHYSICALLY gone… + const genDirs = fs + .readdirSync(path.join(dir, '_generations'), { withFileTypes: true }) + .filter((e) => e.isDirectory() && /^\d+$/.test(e.name)).length + expect(genDirs).toBeLessThanOrEqual(4) // live window (3) + at most the newest + // …and the segment tier exists (the filesystem adapter stores objects + // gzipped, so the manifest may live at either spelling). + const segDir = path.join(dir, SEGMENTS_PREFIX) + expect( + fs.existsSync(path.join(segDir, 'manifest.json')) || + fs.existsSync(path.join(segDir, 'manifest.json.gz')) + ).toBe(true) + expect(fs.readdirSync(segDir).some((f) => f.endsWith('.bgs'))).toBe(true) + + // Same asOf answers from the packed tier, same process… + for (const g of [2, 4, 6]) { + const db = await brain.asOf(g) + expect((await db.get(id))?.metadata?.v).toBe(before[g]) + await db.release() + } + await brain.close() + + // …and across a COLD REOPEN (manifest discovery, no live dirs to list). + const reopened = await openBrain(dir) + for (const g of [2, 4, 6]) { + const db = await reopened.asOf(g) + expect((await db.get(id))?.metadata?.v).toBe(before[g]) + await db.release() + } + expect((await reopened.get(id))?.metadata?.v).toBe(10) // live state untouched + await reopened.close() + }) + + it('repack + bounded reclaim compose: whole segments drop, horizon is loud', async () => { + ;(GenerationStore as any).REPACK_LIVE_WINDOW = 2 + const dir = tempDir() + const brain = await openBrain(dir) + const id = await brain.add({ data: 'reclaim-probe', type: NounType.Document, metadata: { v: 0 } }) + for (let v = 1; v <= 8; v++) await brain.update({ id, metadata: { v } }) + await brain.flush() + await brain.repackHistory() + + // Reclaim down to the 3 newest generations — packed segments below the + // horizon drop whole; asOf below throws loudly. + const res = await brain.compactHistory({ maxGenerations: 3 }) + expect(res.removedGenerations).toBeGreaterThan(0) + await expect(brain.asOf(1)).rejects.toBeInstanceOf(GenerationCompactedError) + expect((await brain.get(id))?.metadata?.v).toBe(8) + await brain.close() + }) + + it('generationDigest: reopen-stable, divergence-sensitive, loud below the horizon', async () => { + ;(GenerationStore as any).REPACK_LIVE_WINDOW = 2 + const dir = tempDir() + const brain = await openBrain(dir) + const id = await brain.add({ data: 'digest-probe', type: NounType.Document, metadata: { v: 0 } }) + for (let v = 1; v <= 6; v++) await brain.update({ id, metadata: { v } }) + await brain.flush() + await brain.repackHistory() + + const gen = brain.generation() + const atHead = await brain.generationDigest(gen) + const atMid = await brain.generationDigest(3) + expect(atHead).toMatch(/^[0-9a-f]{8}$/) + expect(atMid).not.toBe(atHead) // more history ⇒ different digest + await brain.close() + + // Reopen-stable: same history, same digests (packed prefix stability). + const reopened = await openBrain(dir) + expect(await reopened.generationDigest(gen)).toBe(atHead) + expect(await reopened.generationDigest(3)).toBe(atMid) + + // New history diverges the head digest. + await reopened.update({ id, metadata: { v: 7 } }) + await reopened.flush() + expect(await reopened.generationDigest(reopened.generation())).not.toBe(atHead) + + // Below the horizon: LOUD, never a silent pin of reclaimed history. + await reopened.compactHistory({ maxGenerations: 2 }) + await expect(reopened.generationDigest(1)).rejects.toBeInstanceOf(GenerationCompactedError) + await reopened.close() + }) + + it('a spent time budget is a consistent no-op; the next pass resumes', async () => { + ;(GenerationStore as any).REPACK_LIVE_WINDOW = 2 + const dir = tempDir() + const brain = await openBrain(dir) + const id = await brain.add({ data: 'budget-probe', type: NounType.Document, metadata: { v: 0 } }) + for (let v = 1; v <= 6; v++) await brain.update({ id, metadata: { v } }) + await brain.flush() + + const bounded = await brain.repackHistory({ timeBudgetMs: 0 }) + expect(bounded).toEqual({ foldedGenerations: 0, segmentsCreated: 0 }) + + const resumed = await brain.repackHistory() + expect(resumed.foldedGenerations).toBeGreaterThan(0) + const db = await brain.asOf(3) + expect((await db.get(id))?.metadata?.v).toBeDefined() + await db.release() + await brain.close() + }) +}) diff --git a/tests/integration/lens-consistency.test.ts b/tests/integration/lens-consistency.test.ts new file mode 100644 index 00000000..64484a38 --- /dev/null +++ b/tests/integration/lens-consistency.test.ts @@ -0,0 +1,138 @@ +/** + * @module tests/integration/lens-consistency + * @description The three metadata "lenses" over one corpus must agree with + * canonical ground truth id-for-id, warm AND after a cold reopen: + * - combined: find({ type: T, where: { subtype: S } }) + * - subtype-only: find({ where: { subtype: S } }) + * - type-only: find({ type: T }) + * Ported from the fresh-brain probe that closed the type+subtype lens-drop + * investigation (a restored pre-8.2.2 torn capture had entities visible to the + * subtype-only lens but dropped by the combined lens — "0 of 2 migrated, all + * gates green"). The corpus is seeded through the REAL write API — never + * restored bytes — which is what made the original datapoint decisive. The + * invariants: every lens matches an unfiltered canonical scan exactly (no + * missing ids, no extras) and combined ⊆ subtype-only always holds. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/index.js' + +/** The corpus: 7 (type, subtype) pairs, uneven counts, incl. the incident's 2-of-a-pair shape. */ +const CORPUS: Array<{ type: string; subtype: string; count: number }> = [ + { type: 'proposition', subtype: 'decision', count: 2 }, // the incident shape: "0 of 2" + { type: 'concept', subtype: 'decision', count: 3 }, + { type: 'task', subtype: 'decision', count: 2 }, + { type: 'concept', subtype: 'action', count: 4 }, + { type: 'message', subtype: 'note', count: 5 }, + { type: 'message', subtype: 'ship', count: 3 }, + { type: 'document', subtype: 'guide', count: 4 } +] + +/** Canonical ground truth: unfiltered enumeration, post-filtered IN THE TEST. */ +async function groundTruth( + brain: any, + match: { type?: string; subtype?: string } +): Promise> { + const ids = new Set() + let cursor: string | undefined + for (;;) { + const page = await brain.storage.getNounsWithPagination({ limit: 500, cursor }) + for (const noun of page.items) { + // Hydrated shape: `type`/`subtype` are TOP-LEVEL; `metadata` holds only + // custom user fields (vfsType is one — the VFS plumbing marker). + const n = noun as any + if (n.metadata?.vfsType) continue // VFS plumbing is not corpus + if (!n.type || !n.subtype) continue + if (match.type && n.type !== match.type) continue + if (match.subtype && n.subtype !== match.subtype) continue + ids.add(n.id) + } + if (!page.hasMore) break + cursor = page.nextCursor + } + return ids +} + +const idSet = (results: Array<{ id: string }>): Set => new Set(results.map((r) => r.id)) + +/** Every lens vs ground truth, id-for-id, for every pair in the corpus. */ +async function assertAllLenses(brain: any): Promise { + const types = [...new Set(CORPUS.map((c) => c.type))] + const subtypes = [...new Set(CORPUS.map((c) => c.subtype))] + + for (const { type, subtype } of CORPUS) { + const combined = idSet(await brain.find({ type, where: { subtype }, limit: 1000 })) + const subtypeOnly = idSet(await brain.find({ where: { subtype }, limit: 1000 })) + const truthPair = await groundTruth(brain, { type, subtype }) + const truthSubtype = await groundTruth(brain, { subtype }) + + expect([...combined].sort()).toEqual([...truthPair].sort()) // no drops, no extras + expect([...subtypeOnly].sort()).toEqual([...truthSubtype].sort()) + for (const id of combined) expect(subtypeOnly.has(id)).toBe(true) // combined ⊆ subtype-only + } + + for (const type of types) { + const typeOnly = idSet(await brain.find({ type, limit: 1000 })) + const truthType = await groundTruth(brain, { type }) + expect([...typeOnly].sort()).toEqual([...truthType].sort()) + } + + // Count cross-check against the corpus definition itself. + for (const subtype of subtypes) { + const expected = CORPUS.filter((c) => c.subtype === subtype).reduce((s, c) => s + c.count, 0) + const got = (await brain.find({ where: { subtype }, limit: 1000 })).length + expect(got).toBe(expected) + } +} + +describe('lens consistency — combined vs subtype-only vs canonical ground truth', () => { + let dir: string + let brain: any + + beforeAll(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-lens-')) + brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 }) + await brain.init() + // Seed through the REAL write API — never restored bytes. + let i = 0 + for (const { type, subtype, count } of CORPUS) { + for (let k = 0; k < count; k++) { + await brain.add({ data: `${type} ${subtype} ${i++}`, type, subtype, metadata: { k } }) + } + } + await brain.flush() + }) + afterAll(async () => { + await brain.close?.().catch(() => {}) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('WARM: all lenses agree with ground truth id-for-id', async () => { + await assertAllLenses(brain) + }) + + it('COLD REOPEN: all lenses still agree after close + reopen from disk', async () => { + await brain.close() + brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 }) + await brain.init() + await assertAllLenses(brain) + }) + + it('after an update() flips type AND subtype, every lens tracks the move exactly', async () => { + // The historical cross-bucket-staleness path: change (concept, action) -> (task, review). + const victims = await brain.find({ type: 'concept', where: { subtype: 'action' }, limit: 1 }) + expect(victims.length).toBe(1) + const id = victims[0].id + await brain.update({ id, type: 'task', subtype: 'review' }) + + const oldCombined = idSet(await brain.find({ type: 'concept', where: { subtype: 'action' }, limit: 1000 })) + expect(oldCombined.has(id)).toBe(false) // unposted from the old buckets + const newCombined = idSet(await brain.find({ type: 'task', where: { subtype: 'review' }, limit: 1000 })) + expect(newCombined.has(id)).toBe(true) // posted to the new buckets + const subtypeOnly = idSet(await brain.find({ where: { subtype: 'review' }, limit: 1000 })) + expect(subtypeOnly.has(id)).toBe(true) + }) +}) diff --git a/tests/integration/multi-process-safety.test.ts b/tests/integration/multi-process-safety.test.ts index 6ac5265d..592d7969 100644 --- a/tests/integration/multi-process-safety.test.ts +++ b/tests/integration/multi-process-safety.test.ts @@ -110,6 +110,89 @@ describe('Multi-process safety + read-only mode', () => { // Don't track `blocked` for afterEach cleanup since init failed. }) + it('takes over a STALE foreign lock (dead PID + old heartbeat) and claims atomically', async () => { + const { mkdirSync, writeFileSync, readFileSync } = await import('node:fs') + const { join } = await import('node:path') + const os = await import('node:os') + mkdirSync(join(dir, 'locks'), { recursive: true }) + const tenMinutesAgo = new Date(Date.now() - 10 * 60 * 1000).toISOString() + writeFileSync(join(dir, 'locks', '_writer.lock'), JSON.stringify({ + pid: 999999999, // no such process — provably dead + hostname: os.hostname(), + startedAt: tenMinutesAgo, + lastHeartbeat: tenMinutesAgo, + version: '8.0.0', + rootDir: dir + })) + + writer = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await writer.init() // stale takeover must succeed + + const lock = JSON.parse(readFileSync(join(dir, 'locks', '_writer.lock'), 'utf-8')) + expect(lock.pid).toBe(process.pid) // the atomic wx claim installed OUR lock + }) + + it('the writer-locked error carries the machine-readable contract (code + lockInfo)', async () => { + const { mkdirSync, writeFileSync } = await import('node:fs') + const { join } = await import('node:path') + const os = await import('node:os') + mkdirSync(join(dir, 'locks'), { recursive: true }) + const otherPid = (process as any).ppid || 1 + writeFileSync(join(dir, 'locks', '_writer.lock'), JSON.stringify({ + pid: otherPid, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + lastHeartbeat: new Date().toISOString(), + version: '8.7.0', + rootDir: dir + })) + + const blocked = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + const err: any = await blocked.init().catch((e) => e) + expect(err.code).toBe('BRAINY_WRITER_LOCKED') + expect(err.lockInfo?.pid).toBe(otherPid) + }) + + it('release drains an in-flight heartbeat — no phantom lock re-created after unlink', async () => { + // The race (8.9.0): clearInterval stops FUTURE heartbeat ticks, but a + // tick already in flight could land its lock rewrite AFTER release's + // unlink — re-creating the lock as a phantom that blocks the next + // writer until the stale TTL. Simulate the in-flight tick explicitly + // and prove release waits for it. + writer = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await writer.init() + const storage: any = (writer as any).storage + + // An in-flight refresh that is ALREADY PAST its ownership guards + // (captured the lock info before release ran) and lands its atomic + // rewrite slowly — the exact straggler shape; absent the drain it + // writes after the unlink. + const { join: joinPath } = await import('node:path') + const capturedInfo = { ...storage.writerLockInfo } + const lockPath = joinPath(dir, 'locks', '_writer.lock') + const slowTick = (async () => { + await new Promise((r) => setTimeout(r, 100)) + await storage.writeFileAtomic( + lockPath, + JSON.stringify({ ...capturedInfo, lastHeartbeat: new Date().toISOString() }) + ) + })() + storage.writerHeartbeatInFlight = slowTick.catch(() => {}) + + await writer.close() // → releaseWriterLock must drain slowTick first + await slowTick.catch(() => {}) // both paths fully settled either way + writer = null + + const { existsSync } = await import('node:fs') + const { join } = await import('node:path') + expect(existsSync(join(dir, 'locks', '_writer.lock'))).toBe(false) + + // And the directory is immediately claimable — no stale-TTL wait. + const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await expect(next.init()).resolves.toBeUndefined() + await next.close() + }) + it('allows a second in-process writer with a warning (same PID)', async () => { // Two Brainy instances in the same Node process: not the dangerous // cross-process case. Should succeed (with a console warning). diff --git a/tests/integration/readdir-no-duplicate-replace.test.ts b/tests/integration/readdir-no-duplicate-replace.test.ts new file mode 100644 index 00000000..8043bd40 --- /dev/null +++ b/tests/integration/readdir-no-duplicate-replace.test.ts @@ -0,0 +1,56 @@ +/** + * @module tests/integration/readdir-no-duplicate-replace + * @description A re-created path must NOT show up twice in readdir. The reported + * defect (memory-vm) was readdir serving DUPLICATE entries for re-created paths, + * rooted in a delete that left a ghost (partial removal) whose next delete read + * null metadata and skipped unposting the stale Contains edge. With full-removal + * deletes there is no ghost, so the edge is always unposted and the path appears + * exactly once after any number of delete→recreate cycles. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/index.js' + +describe('readdir shows a re-created path exactly once (no duplicate on replace)', () => { + let dir: string + let brain: any + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-readdir-')) + brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 }) + await brain.init() + }) + afterEach(async () => { + await brain.close?.().catch(() => {}) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('delete then re-create the same file path — readdir lists it once', async () => { + await brain.vfs.mkdir('/d', { recursive: true }) + await brain.vfs.writeFile('/d/x.txt', 'v1') + expect(await brain.vfs.readdir('/d')).toEqual(['x.txt']) + + await brain.vfs.unlink('/d/x.txt') + expect(await brain.vfs.readdir('/d')).toEqual([]) + + await brain.vfs.writeFile('/d/x.txt', 'v2') + const listing = (await brain.vfs.readdir('/d')) as string[] + expect(listing).toEqual(['x.txt']) // exactly once — no ghost duplicate + expect(listing.filter((n) => n === 'x.txt')).toHaveLength(1) + }) + + it('survives several delete→recreate cycles without accumulating duplicates', async () => { + await brain.vfs.mkdir('/c', { recursive: true }) + for (let i = 0; i < 4; i++) { + await brain.vfs.writeFile('/c/f.txt', `gen ${i}`) + await brain.vfs.unlink('/c/f.txt') + } + await brain.vfs.writeFile('/c/f.txt', 'final') + const listing = (await brain.vfs.readdir('/c')) as string[] + expect(listing).toEqual(['f.txt']) + expect(await brain.vfs.readFile('/c/f.txt')).toBeDefined() + }) +}) diff --git a/tests/integration/vfs-rename-containment.test.ts b/tests/integration/vfs-rename-containment.test.ts new file mode 100644 index 00000000..0169bf01 --- /dev/null +++ b/tests/integration/vfs-rename-containment.test.ts @@ -0,0 +1,157 @@ +/** + * @module tests/integration/vfs-rename-containment + * @description A cross-directory rename must MOVE the containment edge, not + * accumulate one per parent. The pre-fix rename added the new parent's + * Contains edge but skipped removing the old one ("not critical") — leaving + * the entity a child of BOTH directories: readdir(oldDir) kept listing it, + * re-creating the old path showed the name twice (the duplicate-readdir / + * "cosmetic ghost" field report), and tree-walking consumers saw the file in + * two places. Also covers the repair sweep (repairIndex → vfs.repairContainment) + * that heals ghosts left by earlier versions, and move-to-root (whose edge + * used to be skipped entirely). + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy, VerbType } from '../../src/index.js' + +describe('VFS rename moves the containment edge (no ghost in the old directory)', () => { + let brain: any + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true, dimensions: 384 }) + await brain.init() + await brain.vfs.mkdir('/a', { recursive: true }) + await brain.vfs.mkdir('/b', { recursive: true }) + }) + afterEach(async () => { + await brain.close?.().catch(() => {}) + }) + + it('cross-directory move: old dir stops listing it, new dir lists it exactly once', async () => { + await brain.vfs.writeFile('/a/x.txt', 'v1') + await brain.vfs.rename('/a/x.txt', '/b/x.txt') + + expect(await brain.vfs.readdir('/a')).toEqual([]) + expect(await brain.vfs.readdir('/b')).toEqual(['x.txt']) + expect(String(await brain.vfs.readFile('/b/x.txt'))).toBe('v1') + }) + + it('re-creating the old path lists each name exactly once in each directory', async () => { + await brain.vfs.writeFile('/a/x.txt', 'moved away') + await brain.vfs.rename('/a/x.txt', '/b/x.txt') + await brain.vfs.writeFile('/a/x.txt', 'new file at old path') + + const a = (await brain.vfs.readdir('/a')) as string[] + const b = (await brain.vfs.readdir('/b')) as string[] + expect(a).toEqual(['x.txt']) // exactly once — the pre-fix ghost made this list the moved entity too + expect(b).toEqual(['x.txt']) + expect(String(await brain.vfs.readFile('/a/x.txt'))).toBe('new file at old path') + expect(String(await brain.vfs.readFile('/b/x.txt'))).toBe('moved away') + }) + + it('repeated moves never accumulate containment edges', async () => { + await brain.vfs.writeFile('/a/f.txt', 'wanderer') + for (let i = 0; i < 3; i++) { + await brain.vfs.rename('/a/f.txt', '/b/f.txt') + await brain.vfs.rename('/b/f.txt', '/a/f.txt') + } + expect(await brain.vfs.readdir('/a')).toEqual(['f.txt']) + expect(await brain.vfs.readdir('/b')).toEqual([]) + + // Exactly ONE containment edge exists on the entity. + const stat = await brain.vfs.stat('/a/f.txt') + const edges = await brain.related({ to: stat.entityId, type: VerbType.Contains }) + expect(edges).toHaveLength(1) + }) + + it('move to the root gets a containment edge (used to be skipped → orphan)', async () => { + await brain.vfs.writeFile('/a/up.txt', 'to the top') + await brain.vfs.rename('/a/up.txt', '/up.txt') + + const rootListing = (await brain.vfs.readdir('/')) as string[] + expect(rootListing).toContain('up.txt') + expect(await brain.vfs.readdir('/a')).toEqual([]) + expect(String(await brain.vfs.readFile('/up.txt'))).toBe('to the top') + }) +}) + +describe('repairIndex() heals pre-fix containment ghosts (vfs.repairContainment)', () => { + let brain: any + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true, dimensions: 384 }) + await brain.init() + await brain.vfs.mkdir('/a', { recursive: true }) + await brain.vfs.mkdir('/b', { recursive: true }) + }) + afterEach(async () => { + await brain.close?.().catch(() => {}) + }) + + /** Reproduce the PRE-FIX defect state: file lives at /b/g.txt but a stale + * vfs-contains edge from /a lingers (what old renames left behind). */ + const synthesizeGhost = async () => { + await brain.vfs.writeFile('/b/g.txt', 'ghost target') + const fileId = (await brain.vfs.stat('/b/g.txt')).entityId + const aId = (await brain.vfs.stat('/a')).entityId + await brain.relate({ + from: aId, + to: fileId, + type: VerbType.Contains, + subtype: 'vfs-contains', + metadata: { isVFS: true } + }) + return { fileId, aId } + } + + it('a stale old-parent edge is removed; listings become honest', async () => { + const { fileId } = await synthesizeGhost() + // The defect state is visible: /a lists a file whose path says /b. + expect((await brain.vfs.readdir('/a')) as string[]).toContain('g.txt') + + await brain.repairIndex() + + expect(await brain.vfs.readdir('/a')).toEqual([]) + expect(await brain.vfs.readdir('/b')).toEqual(['g.txt']) + const edges = await brain.related({ to: fileId, type: VerbType.Contains }) + expect(edges).toHaveLength(1) + }) + + it("a user's own knowledge Contains edge onto the file survives the repair", async () => { + const { fileId } = await synthesizeGhost() + // A knowledge-graph containment from a NON-directory entity (a collection + // curating the file) — same verb TYPE, not a vfs-contains edge. The repair + // must remove only VFS containment ghosts, never user knowledge edges. + // (Edges dedupe by (from,to,type), so the user edge needs its own source.) + const collectionId = await brain.add({ + data: 'reading list', + type: 'collection', + metadata: { kind: 'curation' } + }) + await brain.relate({ from: collectionId, to: fileId, type: VerbType.Contains, subtype: 'curates' }) + + await brain.repairIndex() + + const edges = await brain.related({ to: fileId, type: VerbType.Contains }) + const subtypes = edges.map((e: any) => e.subtype).sort() + // The stale vfs ghost edge is gone; the correct vfs edge + the user's + // curation edge remain. + expect(subtypes).toEqual(['curates', 'vfs-contains']) + expect(edges.find((e: any) => e.subtype === 'curates')?.from).toBe(collectionId) + }) + + it('a missing expected edge is restored (entity unreachable from its own directory)', async () => { + await brain.vfs.writeFile('/b/lost.txt', 'find me') + const fileId = (await brain.vfs.stat('/b/lost.txt')).entityId + // Simulate total edge loss (an older damage shape). + for (const e of await brain.related({ to: fileId, type: VerbType.Contains })) { + await brain.unrelate(e.id) + } + expect((await brain.vfs.readdir('/b')) as string[]).not.toContain('lost.txt') + + await brain.repairIndex() + + expect((await brain.vfs.readdir('/b')) as string[]).toContain('lost.txt') + }) +}) diff --git a/tests/regression/metadata-index-cleanup.unit.test.ts b/tests/regression/metadata-index-cleanup.unit.test.ts index 266b9a4d..0984d727 100644 --- a/tests/regression/metadata-index-cleanup.unit.test.ts +++ b/tests/regression/metadata-index-cleanup.unit.test.ts @@ -205,10 +205,10 @@ describe('Metadata index cleanup after remove / removeMany', () => { } }) - it('handles empty ids array gracefully', async () => { - const result = await brain.removeMany({ ids: [] }) - expect(result.successful).toHaveLength(0) - expect(result.failed).toHaveLength(0) + it('refuses an empty ids array loudly (a silent no-op is not "graceful")', async () => { + // 8.8.2: an empty selector used to resolve successfully having deleted + // NOTHING — the caller believed the delete happened. Now it throws. + await expect(brain.removeMany({ ids: [] })).rejects.toThrow(/ids: \[\]/) }) it('handles large batch (> 1 chunk) without leaving stale index entries', async () => { diff --git a/tests/transaction/TransactionManager.unit.test.ts b/tests/transaction/TransactionManager.unit.test.ts index 5ffee866..86b1692c 100644 --- a/tests/transaction/TransactionManager.unit.test.ts +++ b/tests/transaction/TransactionManager.unit.test.ts @@ -10,6 +10,7 @@ import { describe, it, expect, beforeEach } from 'vitest' import { TransactionManager } from '../../src/transaction/TransactionManager.js' +import { transactTimeoutBudget } from '../../src/transaction/Transaction.js' import { TransactionError } from '../../src/transaction/errors.js' describe('TransactionManager', () => { @@ -105,14 +106,17 @@ describe('TransactionManager', () => { const result = await manager.executeTransactionWithResult(async (tx) => { tx.addOperation({ execute: async () => { - await new Promise(resolve => setTimeout(resolve, 10)) + await new Promise(resolve => setTimeout(resolve, 25)) return async () => {} } }) return 'done' }) - expect(result.executionTimeMs).toBeGreaterThanOrEqual(10) + // Timer coalescing can fire a setTimeout up to a few ms EARLY under + // load, so assert well below the sleep — this tests that time is + // MEASURED, not the OS timer's precision. + expect(result.executionTimeMs).toBeGreaterThanOrEqual(20) }) }) @@ -325,4 +329,45 @@ describe('TransactionManager', () => { expect(stats1).toEqual(stats2) // Same values }) }) + + describe('Timeout budget + telemetry', () => { + it('transactTimeoutBudget: explicit override wins; default scales with batch size', () => { + expect(transactTimeoutBudget(1)).toBe(30_000) // small batches keep the 30s floor + expect(transactTimeoutBudget(15)).toBe(30_000) // the old flat cap's break-even point + expect(transactTimeoutBudget(100)).toBe(200_000) // 100 ops × 2s — bulk gets an honest budget + expect(transactTimeoutBudget(1000, 5_000)).toBe(5_000) // caller override is untouched + }) + + it('a tripped budget rolls back and names the operation, progress, and budget', async () => { + const rolledBack: string[] = [] + + const failing = manager.executeTransaction( + async (tx) => { + tx.addOperation({ + name: 'slow-first-op', + execute: async () => { + await new Promise((r) => setTimeout(r, 30)) + return async () => { + rolledBack.push('slow-first-op') + } + } + }) + tx.addOperation({ + name: 'never-reached', + execute: async () => undefined + }) + }, + { timeout: 5 } // the first op's 30ms sleep guarantees the pre-op-2 check trips + ) + + await expect(failing).rejects.toThrow(TransactionError) + const err = await failing.catch((e) => e) + expect(err.name).toBe('TransactionTimeoutError') + expect(err.message).toContain('operation 1/2') // which op, of how many + expect(err.message).toContain("('never-reached')") // its name + expect(err.message).toContain('budget 5ms') // the budget that tripped + expect(err.message).toContain('rolled back') // the retryability statement + expect(rolledBack).toEqual(['slow-first-op']) // the applied op was undone + }) + }) }) diff --git a/tests/unit/aggregation/AggregationIndex.test.ts b/tests/unit/aggregation/AggregationIndex.test.ts index 05f4a62a..a0c10e62 100644 --- a/tests/unit/aggregation/AggregationIndex.test.ts +++ b/tests/unit/aggregation/AggregationIndex.test.ts @@ -675,4 +675,104 @@ describe('AggregationIndex', () => { await reloaded.close() }) }) + + // ============= Boot-order reconciliation ============= + // + // The production boot pattern: defineAggregate() is synchronous and always + // beats the async init() that loads persisted state. The old code flagged a + // backfill at define time and init never cleared it — so the loaded state + // was wiped and the whole store re-walked on EVERY restart. + + describe('boot-order reconciliation (define-before-init)', () => { + const DEF: AggregateDefinition = { + name: 'boot_agg', + source: { type: NounType.Event }, + groupBy: ['category'], + metrics: { count: { op: 'count' } } + } + + const entity = (id: string, category: string): Record => ({ + id, + noun: NounType.Event, + metadata: { category }, + createdAt: Date.now(), + updatedAt: Date.now() + }) + + /** Simulate the previous session: define, contribute, flush. */ + async function seedAndFlush(store: MemoryStorage): Promise { + const first = new AggregationIndex(store) + await first.init() + first.defineAggregate(DEF) + first.onEntityAdded('e1', entity('e1', 'food')) + first.onEntityAdded('e2', entity('e2', 'food')) + first.onEntityAdded('e3', entity('e3', 'transport')) + await first.flush() + } + + it('adopts persisted state when define beats init with an unchanged definition', async () => { + const store = new MemoryStorage() + await store.init() + await seedAndFlush(store) + + const second = new AggregationIndex(store) + second.defineAggregate(DEF) // synchronous define FIRST — the real boot order + await second.init() + await second.ready() + + expect(second.getPendingBackfills()).toEqual([]) + const rows = second.queryAggregate({ name: 'boot_agg' }) + const food = rows.find(r => r.groupKey.category === 'food')! + expect(food.metrics.count).toBe(2) + }) + + it('a write landing before adoption forces an exact rescan instead', async () => { + const store = new MemoryStorage() + await store.init() + await seedAndFlush(store) + + const second = new AggregationIndex(store) + second.defineAggregate(DEF) + second.onEntityAdded('e4', entity('e4', 'food')) // lands before init settles + await second.init() + await second.ready() + + // Adoption would lose e4's contribution — the engine must rescan. + expect(second.getPendingBackfills()).toEqual(['boot_agg']) + }) + + it('init never clobbers a changed app definition registered before it', async () => { + const store = new MemoryStorage() + await store.init() + await seedAndFlush(store) + + const CHANGED: AggregateDefinition = { + ...DEF, + metrics: { count: { op: 'count' }, total: { op: 'sum', field: 'amount' } } + } + const second = new AggregationIndex(store) + second.defineAggregate(CHANGED) + await second.init() + await second.ready() + + const def = second.getDefinitions().find(d => d.name === 'boot_agg')! + expect(Object.keys(def.metrics).sort()).toEqual(['count', 'total']) + expect(second.getPendingBackfills()).toEqual(['boot_agg']) + }) + + it('init alone restores persisted definitions with adopted state, no backfill', async () => { + const store = new MemoryStorage() + await store.init() + await seedAndFlush(store) + + const second = new AggregationIndex(store) + await second.init() + await second.ready() + + expect(second.hasAggregate('boot_agg')).toBe(true) + expect(second.getPendingBackfills()).toEqual([]) + const rows = second.queryAggregate({ name: 'boot_agg' }) + expect(rows.reduce((s, r) => s + (r.metrics.count as number), 0)).toBe(3) + }) + }) }) diff --git a/tests/unit/brainy/batch-operations.test.ts b/tests/unit/brainy/batch-operations.test.ts index b2cc7f09..58b25744 100644 --- a/tests/unit/brainy/batch-operations.test.ts +++ b/tests/unit/brainy/batch-operations.test.ts @@ -533,8 +533,10 @@ describe('Brainy Batch Operations', () => { expect(result.successful).toHaveLength(0) await brain.updateMany({ items: [] }) - await brain.removeMany({ ids: [] }) - // Should not throw + // removeMany is the exception (8.8.2): an empty id list is a refused + // selector, not an empty batch — deleting "nothing" silently was the + // bug class (a positional/bare-array call looked identical). + await expect(brain.removeMany({ ids: [] })).rejects.toThrow(/ids: \[\]/) }) it('should validate batch size limits', async () => { diff --git a/tests/unit/brainy/migration-gate-family-scoped.test.ts b/tests/unit/brainy/migration-gate-family-scoped.test.ts new file mode 100644 index 00000000..b71c3899 --- /dev/null +++ b/tests/unit/brainy/migration-gate-family-scoped.test.ts @@ -0,0 +1,94 @@ +/** + * @module tests/unit/brainy/migration-gate-family-scoped + * @description The coordinated migration LOCK is FAMILY-SCOPED: a read waits only + * on the index families it actually consults. A one-time native migration of ONE + * family (its `isMigrating()` held) must NOT block a read served entirely from + * canonical storage (get / VFS content + dir) or from a HEALTHY family — only a + * read that needs the migrating family blocks. Regression for the whole-brain + * gate that hung getStats / readdir / readFile behind an unrelated family's + * migration until the wait timed out. + */ +import { describe, it, expect, beforeEach } from 'vitest' +import { Brainy } from '../../../src/brainy.js' +import { MigrationInProgressError } from '../../../src/errors/brainyError.js' + +const seed = async () => { + const brain = new Brainy({ + storage: { type: 'memory' }, + dimensions: 384, + requireSubtype: false, + silent: true, + // Short so a genuine block resolves fast; a real block would otherwise wait + // the 30 s default — this test asserts the wait does NOT happen for scoped + // reads, and DOES (bounded) for the one that needs the migrating family. + migrationWaitTimeoutMs: 300 + }) + await brain.init() + for (let i = 0; i < 5; i++) { + await brain.add({ data: `doc ${i}`, type: 'document', metadata: { i } }) + } + await brain.vfs.writeFile('/notes/hello.txt', 'canonical bytes — no index needed') + await brain.flush() + return brain +} + +/** Force a provider to report an in-flight (stuck) migration. */ +const jam = (provider: unknown) => { + ;(provider as { isMigrating: () => boolean }).isMigrating = () => true +} + +describe('migration LOCK is family-scoped', () => { + beforeEach(() => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + }) + + it('a stuck VECTOR migration does not block canonical or graph/metadata reads', async () => { + const brain = await seed() + const childId = ( + (await brain.vfs.readdir('/notes', { withFileTypes: true })) as Array<{ entityId: string }> + )[0].entityId + + jam((brain as any).index) // vector provider migrating; graph + metadata healthy + + // None of these consult the vector family → they serve immediately. + await expect(brain.getStats()).resolves.toBeDefined() + await expect(brain.vfs.readdir('/notes')).resolves.toHaveLength(1) // graph traversal + await expect(brain.vfs.readFile('/notes/hello.txt')).resolves.toBeDefined() // canonical blob + await expect(brain.get(childId)).resolves.not.toBeNull() // canonical by id + await expect(brain.find({ where: { i: 1 } })).resolves.toBeDefined() // metadata filter + }) + + it('a stuck VECTOR migration STILL blocks a read that needs the vector family', async () => { + const brain = await seed() + jam((brain as any).index) + + // A semantic query consults the vector index — it must wait, and (bounded by + // migrationWaitTimeoutMs) surface the retryable MigrationInProgressError + // rather than serve a half-built result. + await expect(brain.find({ query: 'doc' })).rejects.toBeInstanceOf(MigrationInProgressError) + }) + + it('a stuck GRAPH migration blocks traversal but not vector/canonical reads', async () => { + const brain = await seed() + const childId = ( + (await brain.vfs.readdir('/notes', { withFileTypes: true })) as Array<{ entityId: string }> + )[0].entityId + + jam((brain as any).graphIndex) // graph provider migrating; vector + metadata healthy + + // Canonical + vector + metadata are unaffected. + await expect(brain.get(childId)).resolves.not.toBeNull() + await expect(brain.find({ query: 'doc' })).resolves.toBeDefined() + await expect(brain.find({ where: { i: 1 } })).resolves.toBeDefined() + + // A graph traversal needs the migrating family → it blocks (bounded). + await expect(brain.related({ from: childId })).rejects.toBeInstanceOf(MigrationInProgressError) + }) + + it('with no migration in flight, every read serves (the fast path is a no-op)', async () => { + const brain = await seed() + await expect(brain.getStats()).resolves.toBeDefined() + await expect(brain.find({ query: 'doc' })).resolves.toBeDefined() + await expect(brain.vfs.readdir('/notes')).resolves.toHaveLength(1) + }) +}) diff --git a/tests/unit/brainy/warm.test.ts b/tests/unit/brainy/warm.test.ts new file mode 100644 index 00000000..ce213696 --- /dev/null +++ b/tests/unit/brainy/warm.test.ts @@ -0,0 +1,310 @@ +/** + * @module tests/unit/brainy/warm + * @description Coverage for `brain.warm()` / `warmOnOpen` / the provider + * `warm?()` contract (the cold-restart readiness fix): first operations after + * a cold restart should run at steady-state cost instead of paying + * demand-load latency on the critical path. + * + * Uses a real, in-process fake plugin provider implementing the actual + * `VectorIndexProvider` contract from `src/plugin.ts` — the real seam a + * native provider (e.g. a disk-native accelerator) plugs into. The real + * built-in JS metadata index and graph adjacency index run against real + * (filesystem or in-memory) storage with pre-existing data, so their + * hydration paths are exercised for real, not mocked. + */ +import { describe, it, expect, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../../src/brainy.js' +import { NounType, VerbType } from '../../../src/types/graphTypes.js' +import type { VectorIndexProvider } from '../../../src/plugin.js' +import type { VectorDocument, Vector } from '../../../src/coreTypes.js' +import { MetadataIndexManager } from '../../../src/utils/metadataIndex.js' +import { GraphAdjacencyIndex } from '../../../src/graph/graphAdjacencyIndex.js' + +const tmpDirs: string[] = [] +function mkTmp(): string { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-warm-')) + tmpDirs.push(d) + return d +} +afterEach(() => { + for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }) +}) + +// Brainy's ValidationConfig fixes vectors at exactly 384 dimensions +// (src/utils/paramValidation.ts) — match it so `add()` doesn't reject test data. +const DIM = 384 +const V = (seed = 1): number[] => Array.from({ length: DIM }, (_, i) => Math.sin(seed + i)) + +/** + * A real (not mocked) VectorIndexProvider implementation, backed by a plain + * Map, that optionally implements `warm()` — the exact seam + * `AddToVectorIndexOperation` / `brain.warm()` call through. + */ +class FakeVectorProvider implements VectorIndexProvider { + readonly name = 'fake-vector-provider' + readonly items = new Map() + warmCalls = 0 + searchCalls: Array<{ k?: number }> = [] + // Only present on the instance when the constructor is told to — mirrors a + // real provider that may or may not implement the optional hook. Assigned + // in the constructor BODY (not as a field initializer): under native ES + // class fields, field initializers run before constructor-body statements + // — including the parameter-property assignment — so referencing a + // parameter property from a field initializer would see it as still + // `undefined`. + warm?: () => Promise + + constructor(hasWarm: boolean) { + if (hasWarm) { + this.warm = async (): Promise => { + this.warmCalls++ + } + } + } + + async addItem(item: VectorDocument): Promise { + this.items.set(item.id, item.vector) + return item.id + } + async removeItem(id: string): Promise { + return this.items.delete(id) + } + async search(_queryVector: Vector, k?: number): Promise> { + this.searchCalls.push({ k }) + return [...this.items.keys()].slice(0, k ?? 10).map((id) => [id, 0]) + } + size(): number { + return this.items.size + } + clear(): void { + this.items.clear() + } + async rebuild(): Promise {} + async flush(): Promise { + return 0 + } + getPersistMode(): 'immediate' | 'deferred' { + return 'deferred' + } +} + +/** Registers `provider` under the `'vector'` plugin key, before `init()`. */ +function useFakeVectorProvider(brain: Brainy, provider: FakeVectorProvider): void { + brain.use({ + name: 'fake-vector-provider', + activate: async (ctx: any) => { + ctx.registerProvider('vector', () => provider) + return true + } + }) +} + +describe('brain.warm()', () => { + it('(a) calls the vector provider\'s warm() when present and reports "warmed"', async () => { + const brain = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' }, + silent: true + }) + const provider = new FakeVectorProvider(true) + useFakeVectorProvider(brain, provider) + await brain.init() + await brain.add({ data: 'a', type: NounType.Thing, vector: V(1) }) + + const report = await brain.warm() + + expect(provider.warmCalls).toBe(1) + expect(provider.searchCalls.length).toBe(0) // warm() ran — no probe fallback + expect(report.vector.outcome).toBe('warmed') + await brain.close() + }) + + it('(b) falls back to a probe search() when warm() is absent and reports "probed", never "warmed"', async () => { + const brain = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' }, + silent: true + }) + const provider = new FakeVectorProvider(false) + useFakeVectorProvider(brain, provider) + await brain.init() + await brain.add({ data: 'a', type: NounType.Thing, vector: V(1) }) + await brain.add({ data: 'b', type: NounType.Thing, vector: V(2) }) + + const report = await brain.warm() + + expect(provider.warmCalls).toBe(0) // no warm() on this provider + expect(provider.searchCalls.length).toBe(1) // the probe ran + expect(provider.searchCalls[0].k).toBe(Math.min(100, provider.size())) // k = min(100, size) + expect(report.vector.outcome).toBe('probed') + expect(report.vector.outcome).not.toBe('warmed') // never conflated + await brain.close() + }) + + it('vector: reports "unavailable" when there is nothing to probe (empty index, unknown dimension)', async () => { + const brain = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' }, + silent: true + }) + // A disk-native provider MAY honestly report 0 resident entries while + // durable data exists on disk (the same posture documented on + // `VectorIndexProvider.isReady` — "an mmap/disk-native index may + // legitimately report 0 resident entries"). warm()'s probe fallback + // reads `size()`, so this is the real trigger for "nothing to probe": + // never a k=0 search, an honest skip. + const provider = new FakeVectorProvider(false) + provider.size = () => 0 + useFakeVectorProvider(brain, provider) + await brain.init() // the VFS root bootstrap write sets `dimensions`, but size() still reports 0 + + const report = await brain.warm() + + expect(provider.searchCalls.length).toBe(0) // nothing probed + expect(report.vector.outcome).toBe('unavailable') + await brain.close() + }) + + it('(c) metadata + graph hydration paths actually execute against filesystem storage with pre-existing data', async () => { + const dir = mkTmp() + + // Build a brain with real data (including a field NOT in the metadata + // index's own common-fields warm subset — 'wave' — so hydrateAll()'s + // FULL hydration is distinguishable from init()'s partial warmCache()), + // and real graph edges, then close it (persisting everything). + const seed = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true + }) + await seed.init() + const ids: string[] = [] + for (let i = 0; i < 6; i++) { + ids.push( + await seed.add({ + data: `entity ${i}`, + type: NounType.Thing, + metadata: { wave: i % 3 }, + vector: V(i + 1) + }) + ) + } + for (let i = 0; i + 1 < ids.length; i++) { + await seed.relate({ from: ids[i], to: ids[i + 1], type: VerbType.RelatedTo }) + } + await seed.close() + + // Cold-reopen a FRESH instance and spy on the real hydration seams before + // calling warm(), so we assert they actually ran (not just that the + // report claims they did). + const metaHydrateCalls: number[] = [] + const loadedFields: string[] = [] + const graphInitCalls: number[] = [] + + const origHydrateAll = MetadataIndexManager.prototype.hydrateAll + const origLoadSparseIndex = (MetadataIndexManager.prototype as any).loadSparseIndex + const origGraphInit = GraphAdjacencyIndex.prototype.init + + MetadataIndexManager.prototype.hydrateAll = async function (...args: any[]) { + metaHydrateCalls.push(1) + return origHydrateAll.apply(this, args as any) + } + ;(MetadataIndexManager.prototype as any).loadSparseIndex = async function ( + field: string, + ...args: any[] + ) { + loadedFields.push(field) + return origLoadSparseIndex.apply(this, [field, ...args] as any) + } + GraphAdjacencyIndex.prototype.init = async function (...args: any[]) { + graphInitCalls.push(1) + return origGraphInit.apply(this, args as any) + } + + try { + const brain = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true + }) + await brain.init() + + const report = await brain.warm() + + expect(metaHydrateCalls.length).toBe(1) // hydrateAll() actually ran + expect(loadedFields).toContain('wave') // a NON-common field was loaded — full hydration, not the heuristic subset + expect(report.metadata.outcome).toBe('warmed') + + expect(graphInitCalls.length).toBeGreaterThanOrEqual(1) // graph's full-load seam ran (once at brain init, once via warm()) + expect(report.graph.outcome).toBe('warmed') + + // Correctness survives: the hydrated data still answers queries. + const byWhere = await brain.find({ type: NounType.Thing, where: { wave: 1 } }) + expect(byWhere.length).toBe(2) // waves 1,4 of 0..5 + + await brain.close() + } finally { + MetadataIndexManager.prototype.hydrateAll = origHydrateAll + ;(MetadataIndexManager.prototype as any).loadSparseIndex = origLoadSparseIndex + GraphAdjacencyIndex.prototype.init = origGraphInit + } + }) + + it('(d) warmOnOpen: true runs warm() during init() — observable via the fake provider', async () => { + const brain = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' }, + warmOnOpen: true, + silent: true + }) + const provider = new FakeVectorProvider(true) + useFakeVectorProvider(brain, provider) + + // No explicit brain.warm() call — warmOnOpen must have run it as part of init(). + await brain.init() + + expect(provider.warmCalls).toBe(1) + await brain.close() + }) + + it('warmOnOpen defaults to false — init() does NOT run warm() unless opted in', async () => { + const brain = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' }, + silent: true + }) + const provider = new FakeVectorProvider(true) + useFakeVectorProvider(brain, provider) + + await brain.init() + + expect(provider.warmCalls).toBe(0) + await brain.close() + }) + + it('(e) WarmReport shape: outcome literal + durationMs per surface, plus totalDurationMs', async () => { + const brain = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' }, + silent: true + }) + const provider = new FakeVectorProvider(true) + useFakeVectorProvider(brain, provider) + await brain.init() + await brain.add({ data: 'a', type: NounType.Thing, vector: V(1) }) + + const report = await brain.warm() + + for (const surface of ['vector', 'metadata', 'graph'] as const) { + expect(['warmed', 'probed', 'unavailable']).toContain(report[surface].outcome) + expect(typeof report[surface].durationMs).toBe('number') + expect(report[surface].durationMs).toBeGreaterThanOrEqual(0) + } + expect(typeof report.totalDurationMs).toBe('number') + expect(report.totalDurationMs).toBeGreaterThanOrEqual(0) + await brain.close() + }) +}) diff --git a/tests/unit/db/fact-log.test.ts b/tests/unit/db/fact-log.test.ts new file mode 100644 index 00000000..f1c226cc --- /dev/null +++ b/tests/unit/db/fact-log.test.ts @@ -0,0 +1,237 @@ +/** + * @module tests/unit/db/fact-log + * @description The generation fact log in isolation: wire-format round-trip + * (positional msgpack facts, bin16 uuids, body-less tombstones), crc32c + * framing with torn-tail detection, open-time truncation to committed truth + * (the log can only ever be AHEAD after a crash; open cuts it back), rotation + * with a manifest-first flip, exactly-once scans with the frozen telemetry + * shape, and the mmap segment handoff excluding the mutable tail. + */ +import { describe, it, expect, beforeEach } from 'vitest' +import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' +import { + FactLog, + FACTS_PREFIX, + type CommitFact, + type FactLogStorage, + storageSupportsFactLog +} from '../../../src/db/factLog.js' +import { crc32c } from '../../../src/utils/crc32c.js' + +const UUID = (n: number): string => + `00000000-0000-4000-8000-${String(n).padStart(12, '0')}` + +const fact = (generation: number, overrides?: Partial): CommitFact => ({ + generation, + timestamp: 1_700_000_000_000 + generation, + ops: [ + { + kind: 'noun', + id: UUID(generation), + record: { metadata: { noun: 'document', title: `doc ${generation}` }, vector: { v: [1, 2] } } + } + ], + ...overrides +}) + +describe('crc32c known-answer vectors', () => { + it('matches the RFC 3720 test vectors', () => { + expect(crc32c(new TextEncoder().encode('123456789'))).toBe(0xe3069283) + expect(crc32c(new Uint8Array(32))).toBe(0x8a9136aa) + }) +}) + +describe('fact log — round-trip, framing, reconcile, rotation, scan', () => { + let storage: FactLogStorage + let log: FactLog + + beforeEach(async () => { + const mem: any = new MemoryStorage() + await mem.init() + expect(storageSupportsFactLog(mem)).toBe(true) + storage = mem + log = new FactLog(storage) + await log.open(0) + }) + + it('facts round-trip byte-exactly: ops, tombstones, meta, blobHashes', async () => { + await log.append(fact(1)) + await log.append( + fact(2, { + ops: [ + { kind: 'verb', id: UUID(21), record: { metadata: { verb: 'contains' }, vector: null } }, + { kind: 'noun', id: UUID(22), record: null } // TOMBSTONE + ], + meta: { source: 'test' }, + blobHashes: ['abc123', 'abc123'] // multiset — duplicates preserved + }) + ) + await log.sync() + + const scan = log.scanFacts() + expect(scan.headGeneration).toBe(2) + const all: CommitFact[] = [] + for await (const batch of scan.batches()) all.push(...batch.facts) + + expect(all).toHaveLength(2) + expect(all[0].generation).toBe(1) + expect(all[0].ops[0].id).toBe(UUID(1)) + expect(all[0].ops[0].record?.metadata).toEqual({ noun: 'document', title: 'doc 1' }) + expect(all[1].ops[0].kind).toBe('verb') + expect(all[1].ops[1].record).toBeNull() // the tombstone is body-less + expect(all[1].meta).toEqual({ source: 'test' }) + expect(all[1].blobHashes).toEqual(['abc123', 'abc123']) + expect(scan.summary().factsYielded).toBe(2) + }) + + it('appends are monotonic — a replayed/duplicate generation throws', async () => { + await log.append(fact(5)) + await expect(log.append(fact(5))).rejects.toThrow(/non-monotonic/) + await expect(log.append(fact(3))).rejects.toThrow(/non-monotonic/) + await expect(log.append(fact(6))).resolves.toBeUndefined() // gaps are fine (aborted reservations) + }) + + it('a torn tail (partial frame) is detected and ignored — intact prefix survives', async () => { + await log.append(fact(1)) + await log.append(fact(2)) + await log.sync() + + // Simulate a crash mid-append: chop bytes off the tail file. + const tailPath = `${FACTS_PREFIX}/seg-${'1'.padStart(20, '0')}.bfl` + const bytes = (await storage.readRawBytes(tailPath))! + await storage.writeRawBytes(tailPath, bytes.subarray(0, bytes.length - 7)) + + const reopened = new FactLog(storage) + await reopened.open(2) + expect(reopened.headGeneration()).toBe(1) // fact 2's frame was torn → gone + + const all: CommitFact[] = [] + for await (const b of reopened.scanFacts().batches()) all.push(...b.facts) + expect(all.map((f) => f.generation)).toEqual([1]) + }) + + it('open() truncates facts beyond committed truth (the crash-ahead shape)', async () => { + await log.append(fact(1)) + await log.append(fact(2)) + await log.append(fact(3)) + await log.sync() + + // The store's committed generation is 1 — facts 2..3 never committed. + const reopened = new FactLog(storage) + await reopened.open(1) + expect(reopened.headGeneration()).toBe(1) + + const all: CommitFact[] = [] + for await (const b of reopened.scanFacts().batches()) all.push(...b.facts) + expect(all.map((f) => f.generation)).toEqual([1]) + + // And appends continue cleanly from the truncated head. + await reopened.append(fact(2)) + expect(reopened.headGeneration()).toBe(2) + }) + + it('a cleared store (committed=0) truncates everything', async () => { + await log.append(fact(1)) + await log.append(fact(2)) + await log.sync() + const reopened = new FactLog(storage) + await reopened.open(0) + expect(reopened.headGeneration()).toBe(0) + }) + + it('scan honors fromGeneration/toGeneration inclusively and filters kinds', async () => { + for (let g = 1; g <= 6; g++) await log.append(fact(g)) + await log.sync() + + const scan = log.scanFacts({ fromGeneration: 2, toGeneration: 4 }) + const all: CommitFact[] = [] + for await (const b of scan.batches()) all.push(...b.facts) + expect(all.map((f) => f.generation)).toEqual([2, 3, 4]) + + const verbsOnly = log.scanFacts({ kinds: ['verb'] }) + for await (const b of verbsOnly.batches()) { + for (const f of b.facts) expect(f.ops.every((op) => op.kind === 'verb')).toBe(true) + } + }) + + it('batch telemetry carries the frozen shape', async () => { + for (let g = 1; g <= 5; g++) await log.append(fact(g)) + await log.sync() + + const scan = log.scanFacts({ batchSize: 2 }) + expect(scan.approxFactCount).toBe(5) + const batches = [] + for await (const b of scan.batches()) batches.push(b) + expect(batches.length).toBe(3) + expect(batches[0]).toMatchObject({ firstGeneration: 1, lastGeneration: 2, factCount: 2 }) + expect(batches[0].byteSize).toBeGreaterThan(0) + expect(typeof batches[0].segmentId).toBe('string') + expect(scan.summary()).toEqual({ factsYielded: 5, segmentsRead: 1 }) + }) + + it('survives reopen: head and content come back from disk', async () => { + for (let g = 1; g <= 3; g++) await log.append(fact(g)) + await log.sync() + + const reopened = new FactLog(storage) + await reopened.open(3) + expect(reopened.headGeneration()).toBe(3) + const all: CommitFact[] = [] + for await (const b of reopened.scanFacts().batches()) all.push(...b.facts) + expect(all.map((f) => f.generation)).toEqual([1, 2, 3]) + }) + + it('segmentPaths excludes the mutable tail (mmap handoff = sealed only)', async () => { + await log.append(fact(1)) + await log.sync() + expect(log.segmentPaths()).toEqual([]) // only a tail exists — nothing sealed + }) + + describe('scanFacts liveness contract (Stage-2 D1)', () => { + it('a wedged store fails LOUDLY within the first-batch bound — never a silent hang', async () => { + // Force a sealed segment (tiny rotateBytes) so the scan must READ from + // storage, then wedge that read: the exact production shape (a + // backlogged brain whose segment read never returned). + const mem: any = new MemoryStorage() + await mem.init() + const wedgeable = new FactLog(mem, { rotateBytes: 1 }) + await wedgeable.open(0) + await wedgeable.append(fact(1)) + await wedgeable.append(fact(2)) // second append rotates → seg 1 sealed + await wedgeable.sync() + + const realRead = mem.readRawBytes.bind(mem) + mem.readRawBytes = (p: string) => + p.includes('facts/seg-') ? new Promise(() => {}) : realRead(p) // hangs forever + + const scan = wedgeable.scanFacts({ firstBatchTimeoutMs: 200 }) + const started = Date.now() + await expect(scan.batches().next()).rejects.toThrow(/no first batch within 200ms/) + expect(Date.now() - started).toBeLessThan(5_000) // bound held, not a hang + }) + + it('a healthy scan is unaffected — first batch well inside the bound, all facts delivered', async () => { + for (let g = 1; g <= 5; g++) await log.append(fact(g)) + await log.sync() + const scan = log.scanFacts({ batchSize: 2 }) + const all: CommitFact[] = [] + for await (const b of scan.batches()) all.push(...b.facts) + expect(all.map((f) => f.generation)).toEqual([1, 2, 3, 4, 5]) + expect(scan.summary().factsYielded).toBe(5) + }) + + it('consumer think-time between pulls never counts against the producer', async () => { + for (let g = 1; g <= 4; g++) await log.append(fact(g)) + await log.sync() + // Bound tighter than the consumer's pause: only the FIRST pull is + // raced, so a slow consumer after batch 1 must not trip the deadline. + const gen = log.scanFacts({ batchSize: 2, firstBatchTimeoutMs: 150 }).batches() + const first = await gen.next() + expect(first.done).toBe(false) + await new Promise((r) => setTimeout(r, 400)) // dawdle past the bound + const second = await gen.next() + expect(second.done).toBe(false) + expect((await gen.next()).done).toBe(true) + }) + }) +}) diff --git a/tests/unit/db/generation-segments.test.ts b/tests/unit/db/generation-segments.test.ts new file mode 100644 index 00000000..27ab85cb --- /dev/null +++ b/tests/unit/db/generation-segments.test.ts @@ -0,0 +1,150 @@ +/** + * @module tests/unit/db/generation-segments + * @description The generation-segment store (Stage-2 D1+D3 file format). + * Laws: (1) fold → read round-trips deltas and records byte-faithfully via + * sidecar point-reads; (2) the manifest is the ONLY discovery path — reopen + * reads one file, never a listing; (3) a lost/corrupt sidecar rebuilds from + * its segment loudly, a damaged SEGMENT fails loudly (never silent wrong + * data); (4) D3 reclaim drops whole segments only and bumps compactedBelow; + * (5) the packed digest is deterministic across reopen; (6) immutability — + * fold refuses overlap with sealed ranges. + */ +import { describe, it, expect, beforeEach } from 'vitest' +import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' +import { + GenerationSegmentStore, + SEGMENTS_PREFIX, + type FoldGeneration +} from '../../../src/db/generationSegments.js' + +const UUID = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, '0')}` + +const gen = (g: number, recordCount = 2): FoldGeneration => ({ + generation: g, + timestamp: 1_700_000_000_000 + g, + delta: { generation: g, nouns: [UUID(g)], verbs: [], bytes: 123 + g }, + records: Array.from({ length: recordCount }, (_, i) => ({ + kind: (i % 2 === 0 ? 'noun' : 'verb') as 'noun' | 'verb', + id: UUID(g * 100 + i), + record: { metadata: { noun: 'document', v: g }, vector: { v: [g, i] } } + })) +}) + +describe('db/GenerationSegmentStore — the D1+D3 packed tier', () => { + let storage: MemoryStorage + let store: GenerationSegmentStore + + beforeEach(async () => { + storage = new MemoryStorage() + await storage.init() + store = new GenerationSegmentStore(storage as any) + await store.open() + }) + + it('fold → read round-trips deltas and records via sidecar point-reads', async () => { + const meta = await store.fold([gen(1), gen(2), gen(3)]) + expect(meta).toMatchObject({ firstGeneration: 1, lastGeneration: 3, frames: 3 }) + expect(meta.checksum).toBeGreaterThan(0) + + expect(store.hasGeneration(2)).toBe(true) + expect(store.hasGeneration(4)).toBe(false) + + const d2 = await store.readDelta(2) + expect(d2?.delta).toEqual({ generation: 2, nouns: [UUID(2)], verbs: [], bytes: 125 }) + expect(d2?.timestamp).toBe(1_700_000_000_002) + + const records = await store.readRecords(3) + expect(records).toHaveLength(2) + expect(records![0]).toEqual({ + kind: 'noun', + id: UUID(300), + record: { metadata: { noun: 'document', v: 3 }, vector: { v: [3, 0] } } + }) + // Point read by id, both kinds. + expect(await store.readRecord(3, 'verb', UUID(301))).toEqual({ + metadata: { noun: 'document', v: 3 }, + vector: { v: [3, 1] } + }) + expect(await store.readRecord(3, 'noun', UUID(999))).toBeNull() + }) + + it('reopen discovers everything from the manifest alone — no listing', async () => { + await store.fold([gen(1), gen(2)]) + await store.fold([gen(3), gen(4)]) + + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + expect(reopened.segments()).toHaveLength(2) + expect(reopened.hasGeneration(4)).toBe(true) + expect((await reopened.readDelta(1))?.timestamp).toBe(1_700_000_000_001) + }) + + it('a lost sidecar rebuilds from its segment; a damaged segment fails LOUDLY', async () => { + const meta = await store.fold([gen(1), gen(2)]) + const idxPath = `${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.idx` + await storage.deleteRawObject(idxPath) + + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + // Rebuild path: still serves correct data. + expect((await reopened.readRecords(2))!).toHaveLength(2) + + // Now damage the SEGMENT itself: flip a payload byte → CRC mismatch, loud. + const segPath = `${SEGMENTS_PREFIX}/${meta.file}` + const bytes = (await storage.readRawBytes(segPath))! + bytes[bytes.length - 3] ^= 0xff + await storage.writeRawBytes(segPath, bytes) + const damaged = new GenerationSegmentStore(storage as any) + await damaged.open() + ;(damaged as any).sidecars.clear() + await storage.deleteRawObject(idxPath) // force the sequential rebuild over damaged bytes + await expect(damaged.readRecords(2)).rejects.toThrow(/CRC mismatch|damaged/) + }) + + it('D3 reclaim drops whole segments only and bumps compactedBelow', async () => { + await store.fold([gen(1), gen(2)]) + await store.fold([gen(3), gen(4)]) + await store.fold([gen(5), gen(6)]) + + // Horizon mid-segment-2 (below 4): only segment 1 is FULLY below → drops. + const r1 = await store.dropSegmentsBelow(4) + expect(r1).toEqual({ dropped: 1, compactedBelow: 3 }) + expect(store.hasGeneration(1)).toBe(false) + expect(store.hasGeneration(3)).toBe(true) // partial segment survives whole + + // Bytes actually gone. + expect(await storage.readRawBytes(`${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.bgs`)).toBeNull() + + // Horizon past everything: the rest drop; compactedBelow is durable. + const r2 = await store.dropSegmentsBelow(7) + expect(r2.dropped).toBe(2) + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + expect(reopened.compactedBelow()).toBe(7) + expect(reopened.segments()).toHaveLength(0) + }) + + it('the packed digest is deterministic across reopen and changes with history', async () => { + await store.fold([gen(1), gen(2), gen(3)]) + const atSeal = await store.digestThroughPacked(3) + const midSegment = await store.digestThroughPacked(2) + expect(atSeal).not.toBeNull() + expect(midSegment).not.toBeNull() + expect(midSegment).not.toBe(atSeal) + + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + expect(await reopened.digestThroughPacked(3)).toBe(atSeal) + expect(await reopened.digestThroughPacked(2)).toBe(midSegment) + + await reopened.fold([gen(4)]) + expect(await reopened.digestThroughPacked(4)).not.toBe(atSeal) + }) + + it('sealed segments are immutable — fold refuses overlap, requires ascending input', async () => { + await store.fold([gen(1), gen(2)]) + await expect(store.fold([gen(2), gen(3)])).rejects.toThrow(/overlaps the packed tier/) + await expect(store.fold([gen(4), gen(4)])).rejects.toThrow(/strictly ascending/) + await expect(store.fold([])).rejects.toThrow(/at least one generation/) + }) +}) diff --git a/tests/unit/db/generationStore.test.ts b/tests/unit/db/generationStore.test.ts index 8073d7ee..5b667415 100644 --- a/tests/unit/db/generationStore.test.ts +++ b/tests/unit/db/generationStore.test.ts @@ -249,7 +249,11 @@ describe('db/GenerationStore', () => { store.release(pinned) const result = await store.compact() expect(result.removedGenerations).toBeGreaterThan(0) - const remaining = await storage.listRawObjects(GENERATIONS_PREFIX) + // History record-sets only — the fact log (at `_generations/facts/`) is + // deliberately NOT reclaimed by history compaction. + const remaining = (await storage.listRawObjects(GENERATIONS_PREFIX)).filter( + (p: string) => !p.startsWith(`${GENERATIONS_PREFIX}/facts/`) + ) expect(remaining).toEqual([]) }) @@ -484,5 +488,95 @@ describe('db/GenerationStore', () => { expect(result.removedGenerations).toBe(2) store.release(2) }) + + it('timeBudgetMs bounds a pass; the next pass resumes the same prefix', async () => { + await manyGens(4) + // A spent budget (0ms) stops before reclaiming anything — an early stop + // is a consistent prefix, never a partial generation. + const bounded = await store.compact({ timeBudgetMs: 0 }) + expect(bounded.removedGenerations).toBe(0) + expect(bounded.horizon).toBe(0) + // The next (unbounded) pass picks up exactly where the bounded one + // stopped and completes the same work. + const resumed = await store.compact() + expect(resumed.removedGenerations).toBe(4) + expect(resumed.horizon).toBe(4) + }) + }) + + // ========================================================================== + describe('history-bytes running total (the O(1) retention check)', () => { + /** A fresh walk with the cache dropped — ground truth for the invariant. */ + async function groundTruthBytes(): Promise { + ;(store as any).historyBytesTotal = null + return store.historyBytes() + } + + it('is seeded once, then maintained through commits WITHOUT re-walks', async () => { + await commitWrite(ID_A, 1) + await commitWrite(ID_A, 2) + const seeded = await store.historyBytes() + expect(seeded).toBe(await groundTruthBytes()) + + // From here every read must come from the running total, not a walk: + // getDelta re-reads are the walk's cost — commits must not trigger any. + const getDeltaSpy = vi.spyOn(store as any, 'getDelta') + await commitWrite(ID_B, 1) + const afterCommit = await store.historyBytes() + expect(getDeltaSpy).not.toHaveBeenCalled() + getDeltaSpy.mockRestore() + expect(afterCommit).toBe(await groundTruthBytes()) + }) + + it('stays exact through single-op group commits and compaction', async () => { + await commitWrite(ID_A, 1) + await store.historyBytes() // seed + // Single-op path: buffered generations flushed as one group commit. + await store.commitSingleOp({ + touched: { nouns: [ID_B] }, + execute: async () => { + await storage.saveNounMetadata(ID_B, metadataFixture(1)) + } + }) + await store.flushPendingSingleOps() + expect(await store.historyBytes()).toBe(await groundTruthBytes()) + + await store.historyBytes() // re-seed after ground-truth reset + await store.compact({ maxGenerations: 1 }) + expect(await store.historyBytes()).toBe(await groundTruthBytes()) + }) + + it('historyStats reports counts, bytes, range, and horizon read-only', async () => { + await commitWrite(ID_A, 1) + await commitWrite(ID_B, 1) + const stats = await store.historyStats() + expect(stats.generations).toBe(2) + expect(stats.bytes).toBe(await store.historyBytes()) + expect(stats.oldestGeneration).toBe(1) + expect(stats.newestGeneration).toBe(2) + expect(stats.oldestTimestamp).toBeLessThanOrEqual(stats.newestTimestamp!) + expect(stats.horizon).toBe(0) + // Read-only: nothing was reclaimed by asking. + expect(store.committedGeneration()).toBe(2) + + await store.compact({ maxGenerations: 1 }) + const after = await store.historyStats() + expect(after.generations).toBe(1) + expect(after.oldestGeneration).toBe(2) + expect(after.horizon).toBe(1) + }) + + it('empty history reports null range and zero bytes', async () => { + const stats = await store.historyStats() + expect(stats).toMatchObject({ + generations: 0, + bytes: 0, + oldestGeneration: null, + newestGeneration: null, + oldestTimestamp: null, + newestTimestamp: null, + horizon: 0 + }) + }) }) }) diff --git a/tests/unit/migration-lock.test.ts b/tests/unit/migration-lock.test.ts index 81d44a9f..f0fbbe4c 100644 --- a/tests/unit/migration-lock.test.ts +++ b/tests/unit/migration-lock.test.ts @@ -2,11 +2,16 @@ * Migration LOCK (#18) — coordinated, automatic 7.x → 8.0 auto-upgrade. * * Exercises the real block-and-queue behavior of `awaitMigrationLock` at the - * `ensureInitialized` choke point: while ANY index provider reports - * `isMigrating() === true`, data-plane reads and writes WAIT (no operation - * touches a half-built index); observability (`getIndexStatus`, `health`) and - * the lock-clearing `stampBrainFormat` stay exempt; and a lock that outlives the - * configured window surfaces a retryable `MigrationInProgressError`. + * `ensureInitialized` choke point. The gate is FAMILY-SCOPED: a write (which + * touches every index) WAITS while ANY provider reports `isMigrating() === true`, + * and a read WAITS only when the migrating provider is one of the index families + * that read consults (so a read served from canonical storage or a healthy family + * is never blocked by an unrelated family's migration — see + * `tests/unit/brainy/migration-gate-family-scoped.test.ts`). Observability + * (`getIndexStatus`, `health`) and the lock-clearing `stampBrainFormat` stay + * exempt; a lock that outlives the configured window surfaces a retryable + * `MigrationInProgressError`. Here the graph provider holds the lock, so the + * read cases use a graph traversal (`related`) — the family that actually waits. * * A native (cortex) provider owns the real migration; here we simulate the lock * by feature-detect-injecting `isMigrating()` onto a live provider, exactly as @@ -84,11 +89,12 @@ describe('Migration LOCK (#18) — coordinated 7.x→8.0 auto-upgrade', () => { expect(id).toBeTruthy() }) - it('blocks a read while migrating, then releases when the flag clears (poll path)', async () => { + it('blocks a graph read while the graph provider migrates, then releases when the flag clears (poll path)', async () => { + const anchor = await brain.add({ data: 'anchor', type: NounType.Concept }) let migrating = true brain.graphIndex.isMigrating = () => migrating - const p = brain.find({ type: NounType.Concept }) // a read → gated + const p = brain.related({ from: anchor }) // a GRAPH read → gated on the graph migration let resolved = false p.then(() => { resolved = true @@ -109,9 +115,12 @@ describe('Migration LOCK (#18) — coordinated 7.x→8.0 auto-upgrade', () => { migrationWaitTimeoutMs: 120 }) await shortBrain.init() - setMigrating(shortBrain, true) // never clears + const anchor = await shortBrain.add({ data: 'anchor', type: NounType.Concept }) + setMigrating(shortBrain, true) // graph lock never clears - await expect(shortBrain.find({ type: NounType.Concept })).rejects.toBeInstanceOf( + // A graph read needs the migrating family → it waits out the window and + // surfaces the retryable error rather than serving a half-built traversal. + await expect(shortBrain.related({ from: anchor })).rejects.toBeInstanceOf( MigrationInProgressError ) try { diff --git a/tests/unit/storage/pagination-parallel-hydration.test.ts b/tests/unit/storage/pagination-parallel-hydration.test.ts new file mode 100644 index 00000000..ada324bb --- /dev/null +++ b/tests/unit/storage/pagination-parallel-hydration.test.ts @@ -0,0 +1,94 @@ +/** + * @module tests/unit/storage/pagination-parallel-hydration + * @description The canonical enumeration walk (getNounsWithPagination) hydrated + * each item's vector + metadata ONE-AT-A-TIME — N×per-op-latency serially, the + * dominant term in an index heal (cortex heal-cost decomposition). It now + * hydrates 16-way, and a new getNounIdsWithPagination returns ids WITHOUT + * hydration (zero per-entity reads when unfiltered). Both must preserve the exact + * pagination contract: same order, cursor continuation, filters, totalCount. + */ +import { describe, it, expect, beforeEach, vi } from 'vitest' +import { Brainy, NounType } from '../../../src/index.js' + +describe('paginated enumeration — parallel hydration + id-only (cortex heal-cost)', () => { + let brain: any + let storage: any + const N = 30 + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, dimensions: 384 }) + await brain.init() + for (let i = 0; i < N; i++) { + await brain.add({ + data: `n${i}`, + type: i % 3 === 0 ? NounType.Task : NounType.Concept, + metadata: { i } + }) + } + await brain.flush() + storage = brain.storage + }) + + /** Page the whole dataset through a small limit via cursor and collect ordered ids. */ + const pageAll = async (fn: (opts: any) => Promise, key: 'items' | 'ids') => { + const out: string[] = [] + let cursor: string | undefined + for (let guard = 0; guard < 1000; guard++) { + const page = await fn({ limit: 4, cursor }) + const batch = key === 'items' ? page.items.map((n: any) => n.id) : page.ids + out.push(...batch) + if (!page.hasMore) break + cursor = page.nextCursor + } + return out + } + + it('parallel hydration yields the SAME ordered pages as one big page', async () => { + const big = await storage.getNounsWithPagination({ limit: 1000, offset: 0 }) + const bigIds = big.items.map((n: any) => n.id) + // At least the N we added (a brain also has its VFS root entity). + expect(bigIds.length).toBeGreaterThanOrEqual(N) + expect(big.totalCount).toBe(bigIds.length) + + const paged = await pageAll((o) => storage.getNounsWithPagination(o), 'items') + expect(paged).toEqual(bigIds) // identical order, no dupes, no gaps across pages + }) + + it('getNounIdsWithPagination returns exactly the same ids, in the same order', async () => { + const idsPaged = await pageAll((o) => storage.getNounIdsWithPagination(o), 'ids') + const itemsPaged = await pageAll((o) => storage.getNounsWithPagination(o), 'items') + expect(idsPaged).toEqual(itemsPaged) + expect(new Set(idsPaged).size).toBe(idsPaged.length) // every id exactly once + expect(idsPaged.length).toBeGreaterThanOrEqual(N) + }) + + it('id-only enumeration does ZERO per-entity hydration reads when unfiltered', async () => { + const readSpy = vi.spyOn(storage as any, 'readCanonicalObject') + await storage.getNounIdsWithPagination({ limit: 1000, offset: 0 }) + expect(readSpy).not.toHaveBeenCalled() + readSpy.mockRestore() + + // The hydrating walk, by contrast, DOES read each entity. + const readSpy2 = vi.spyOn(storage as any, 'readCanonicalObject') + await storage.getNounsWithPagination({ limit: 1000, offset: 0 }) + expect(readSpy2.mock.calls.length).toBeGreaterThan(0) + readSpy2.mockRestore() + }) + + it('a type filter matches between the hydrating and id-only walks', async () => { + const taskItems = await storage.getNounsWithPagination({ + limit: 1000, + offset: 0, + filter: { nounType: NounType.Task } + }) + const taskIds = await storage.getNounIdsWithPagination({ + limit: 1000, + offset: 0, + filter: { nounType: NounType.Task } + }) + const expected = Math.ceil(N / 3) // every 3rd is a Task + expect(taskItems.items.length).toBe(expected) + expect(new Set(taskIds.ids)).toEqual(new Set(taskItems.items.map((n: any) => n.id))) + }) +}) diff --git a/tests/unit/storage/registered-blob-contract.test.ts b/tests/unit/storage/registered-blob-contract.test.ts new file mode 100644 index 00000000..6ad51ae7 --- /dev/null +++ b/tests/unit/storage/registered-blob-contract.test.ts @@ -0,0 +1,130 @@ +/** + * @module tests/unit/storage/registered-blob-contract + * @description Declared derived-index blob FAMILIES are + * undeletable through the storage layer — an in-process GC/sweeper cannot remove + * a load-bearing index file (the lost-main.dkann class). Covers declare → + * protected-delete-throws; unregister → delete-ok; transient passthrough; + * namespace prefix protects children; removeRawPrefix refuses a protected + * intersection; persistence across reopen; missing-on-open detection. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' +import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js' +import { ProtectedArtifactError } from '../../../src/index.js' + +describe('registered-blob family contract', () => { + let storage: any + + beforeEach(async () => { + storage = new MemoryStorage() + await storage.init() + }) + + const seedVectorFamily = async () => { + await storage.saveBinaryBlob('_system/vector-index/main.dkann', Buffer.from([1])) + await storage.saveBinaryBlob('_system/vector-index/main.slotmap', Buffer.from([2])) + await storage.saveBinaryBlob('_system/vector-index/main.slotrev', Buffer.from([3])) + await storage.registerDerivedFamily({ + name: 'vector-base', + members: [ + '_system/vector-index/main.dkann', + '_system/vector-index/main.slotmap', + '_system/vector-index/main.slotrev' + ] + }) + } + + it('a declared member is undeletable (throws ProtectedArtifactError)', async () => { + await seedVectorFamily() + await expect(storage.deleteBinaryBlob('_system/vector-index/main.dkann')).rejects.toBeInstanceOf( + ProtectedArtifactError + ) + // The blob is still there. + expect(await storage.loadBinaryBlob('_system/vector-index/main.dkann')).not.toBeNull() + }) + + it('unregistering the family makes its members deletable again', async () => { + await seedVectorFamily() + await storage.unregisterDerivedFamily('vector-base') + await expect(storage.deleteBinaryBlob('_system/vector-index/main.dkann')).resolves.toBeUndefined() + expect(await storage.loadBinaryBlob('_system/vector-index/main.dkann')).toBeNull() + }) + + it('an undeclared blob is still deletable (contract does not lock everything)', async () => { + await seedVectorFamily() + await storage.saveBinaryBlob('graph-lsm/source/sstable-1', Buffer.from([9])) + await expect(storage.deleteBinaryBlob('graph-lsm/source/sstable-1')).resolves.toBeUndefined() + }) + + it('a transient (*.tmp.*) is deletable even if it sits under a protected namespace', async () => { + await storage.registerDerivedFamily({ + name: 'vector-ns', + members: ['_system/vector-index/'], + namespace: true + }) + await storage.saveBinaryBlob('_system/vector-index/main.dkann.tmp.123', Buffer.from([1])) + await expect( + storage.deleteBinaryBlob('_system/vector-index/main.dkann.tmp.123') + ).resolves.toBeUndefined() + }) + + it('a namespace family protects a growing child key', async () => { + await storage.registerDerivedFamily({ + name: 'vector-segments', + members: ['_system/vector-index/'], + namespace: true + }) + await storage.saveBinaryBlob('_system/vector-index/seg-42', Buffer.from([7])) + await expect(storage.deleteBinaryBlob('_system/vector-index/seg-42')).rejects.toBeInstanceOf( + ProtectedArtifactError + ) + }) + + it('checkDerivedFamiliesPresent() names a member deleted outside the write path', async () => { + await seedVectorFamily() + // Simulate an EXTERNAL deleter (bypasses the in-process refusal): drop a + // member straight from the underlying store. + ;(storage as any).blobStore.delete('_system/vector-index/main.slotmap') + const incomplete = await storage.checkDerivedFamiliesPresent() + expect(incomplete).toHaveLength(1) + expect(incomplete[0].name).toBe('vector-base') + expect(incomplete[0].missing).toContain('_system/vector-index/main.slotmap') + expect(incomplete[0].rebuildable).toBe(true) + }) + + it('no families registered → deleteBinaryBlob behaves exactly as before (no enforcement)', async () => { + await storage.saveBinaryBlob('some/blob', Buffer.from([1])) + await expect(storage.deleteBinaryBlob('some/blob')).resolves.toBeUndefined() + }) +}) + +describe('registered-blob family protection survives a reopen (FileSystemStorage)', () => { + let dir: string + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-regblob-')) + }) + afterEach(() => fs.rmSync(dir, { recursive: true, force: true })) + + it('a family declared in one session protects members after reopen', async () => { + const s1: any = new FileSystemStorage(dir) + await s1.init() + await s1.saveBinaryBlob('_system/vector-index/main.dkann', Buffer.from([1])) + await s1.registerDerivedFamily({ name: 'vector-base', members: ['_system/vector-index/main.dkann'] }) + + // Reopen — the registry is loaded from _system/derived-artifacts.json. + const s2: any = new FileSystemStorage(dir) + await s2.init() + expect(await s2.listDerivedFamilies()).toHaveLength(1) + await expect(s2.deleteBinaryBlob('_system/vector-index/main.dkann')).rejects.toBeInstanceOf( + ProtectedArtifactError + ) + // removeRawPrefix nuking the vector dir is also refused. + await expect(s2.removeRawPrefix('_blobs/_system/vector-index')).rejects.toBeInstanceOf( + ProtectedArtifactError + ) + }) +}) diff --git a/tests/unit/storage/verb-cursor-pagination.test.ts b/tests/unit/storage/verb-cursor-pagination.test.ts index 985a650f..cff41d02 100644 --- a/tests/unit/storage/verb-cursor-pagination.test.ts +++ b/tests/unit/storage/verb-cursor-pagination.test.ts @@ -95,9 +95,15 @@ describe('verb cursor pagination (graph-perf #2)', () => { expect(new Set(cursorSeen)).toEqual(new Set(offsetSeen)) }) - it('a foreign/malformed cursor falls back gracefully (no throw, starts from the beginning)', async () => { - const page = await storage.getVerbs({ pagination: { limit: 5, cursor: 'not-a-cv1-token' } }) - expect(page.items.length).toBe(5) - expect(page.hasMore).toBe(true) + it('a foreign/malformed cursor FAILS LOUDLY — never a silent restart from page 1', async () => { + // The old behavior (decode-null → silent offset-0 fallback) re-served page 1 + // forever to any while(hasMore) walker: an unbounded CPU loop with no log + // line. An undecodable resume token now refuses the walk instead. + await expect( + storage.getVerbs({ pagination: { limit: 5, cursor: 'not-a-cv1-token' } }) + ).rejects.toThrow('invalid pagination cursor') + await expect( + storage.getNouns({ pagination: { limit: 5, cursor: 'not-a-cv1-token' } }) + ).rejects.toThrow('invalid pagination cursor') }) }) diff --git a/tests/unit/transaction/budget-commit-completed-work.test.ts b/tests/unit/transaction/budget-commit-completed-work.test.ts new file mode 100644 index 00000000..a32489de --- /dev/null +++ b/tests/unit/transaction/budget-commit-completed-work.test.ts @@ -0,0 +1,149 @@ +/** + * @module tests/unit/transaction/budget-commit-completed-work + * @description Regression coverage for the "commit-completed-work" transaction + * budget semantics: the budget gates STARTING the next operation; it never + * converts already-completed work into a rollback. Concretely: + * + * - A single-op transaction can never time out post-hoc — its one operation + * either runs (and the transaction commits, however long it took) or it + * never gets to start (which cannot happen: there is no operation before it + * to have overrun the budget). + * - A multi-op transaction whose op `i` overruns the budget stops op `i+1` + * from starting: everything applied so far rolls back atomically and a + * `TransactionTimeoutError` is thrown — the mid-batch zero-loss guarantee is + * unchanged. + * - `transactTimeoutBudget`'s scaling floor (`max(floorMs, opCount * 2000)`) + * is overridable per call, the seam `BrainyConfig.transactionBudgetFloorMs` + * feeds at the brainy.ts read sites. + * + * Uses a real class implementing the `Operation` interface (not an anonymous + * object literal, not a mock of Transaction internals) so the exercised path + * is exactly what a real caller's operation looks like. + */ +import { describe, it, expect } from 'vitest' +import { Transaction, transactTimeoutBudget } from '../../../src/transaction/Transaction.js' +import type { Operation, RollbackAction } from '../../../src/transaction/types.js' +import { TransactionTimeoutError } from '../../../src/transaction/errors.js' + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +/** + * A real `Operation` implementation that sleeps for `delayMs` before applying + * a write to `log`, and returns a rollback action that removes it again. Used + * to deterministically make one operation "slow" relative to a transaction's + * budget without touching any Transaction internals. + */ +class SlowOperation implements Operation { + readonly name: string + executed = false + rolledBack = false + + constructor( + private readonly log: string[], + label: string, + private readonly delayMs: number + ) { + this.name = label + } + + async execute(): Promise { + this.executed = true + if (this.delayMs > 0) await sleep(this.delayMs) + this.log.push(this.name) + return async () => { + this.rolledBack = true + const idx = this.log.indexOf(this.name) + if (idx >= 0) this.log.splice(idx, 1) + } + } +} + +describe('Transaction budget — commit-completed-work semantics', () => { + it('(a) single-op transaction whose op overruns the budget COMMITS — no rollback, no throw', async () => { + const log: string[] = [] + const op = new SlowOperation(log, 'slow-single-op', 40) + + // Budget (5ms) is far smaller than the op's 40ms — under the OLD + // (post-completion-check) semantics this would have thrown and rolled + // back after the op finished. Under the new semantics there is no + // operation after it to gate, so it commits. + const tx = new Transaction({ timeout: 5 }) + tx.addOperation(op) + + await expect(tx.execute()).resolves.toBeUndefined() + + expect(tx.getState()).toBe('committed') + expect(op.executed).toBe(true) + expect(op.rolledBack).toBe(false) + expect(log).toEqual(['slow-single-op']) + }) + + it('(b) two-op transaction: op0 overruns the budget → op1 never starts, op0 rolls back, throws TransactionTimeoutError', async () => { + const log: string[] = [] + const op0 = new SlowOperation(log, 'op0-overruns', 40) + const op1 = new SlowOperation(log, 'op1-never-starts', 0) + + const tx = new Transaction({ timeout: 5 }) + tx.addOperation(op0) + tx.addOperation(op1) + + const error = await tx.execute().catch((e) => e) + + expect(error).toBeInstanceOf(TransactionTimeoutError) + expect(op0.executed).toBe(true) + expect(op0.rolledBack).toBe(true) + expect(op1.executed).toBe(false) + expect(op1.rolledBack).toBe(false) + expect(log).toEqual([]) // op0's write was undone; op1 never wrote + expect(tx.getState()).toBe('rolled_back') + }) + + it('a single-op transaction never checks the budget before its only operation starts', async () => { + // Budget of 0ms: under the old "check before every op including 0" code + // this could theoretically trip before op 0 even started (if any + // measurable time elapsed between startTime capture and the check). The + // documented contract is stronger: operation 0 ALWAYS gets to start. + const log: string[] = [] + const op = new SlowOperation(log, 'only-op', 5) + + const tx = new Transaction({ timeout: 0 }) + tx.addOperation(op) + + await expect(tx.execute()).resolves.toBeUndefined() + expect(tx.getState()).toBe('committed') + expect(op.executed).toBe(true) + }) + + it('(c) transactTimeoutBudget: an explicit floorMs overrides the 30s default floor', () => { + // Below the per-op scaling term, a raised floor wins. + expect(transactTimeoutBudget(1, undefined, 5_000)).toBe(5_000) // 1 op × 2000 = 2000 < 5000 floor + expect(transactTimeoutBudget(1)).toBe(30_000) // unchanged default when floorMs omitted + + // Above the floor, opCount * 2000 still wins over a smaller floor. + expect(transactTimeoutBudget(10, undefined, 5_000)).toBe(20_000) // 10 × 2000 = 20000 > 5000 floor + + // A full `override` always wins, regardless of floorMs. + expect(transactTimeoutBudget(10, 999, 5_000)).toBe(999) + }) + + it('(c) a configured floor changes real Transaction behavior end-to-end via TransactionManager-style options', async () => { + // Simulates how brainy.ts wires `this.config.transactionBudgetFloorMs` + // into `transactTimeoutBudget()`: opCount 0 isolates the floor term + // (0 × 2000 = 0), so a lowered floor makes a normally-generous budget + // fail-fast for a real 2-op Transaction. + const lowFloorBudget = transactTimeoutBudget(0, undefined, 10) // 10ms floor + expect(lowFloorBudget).toBe(10) + + const log: string[] = [] + const op0 = new SlowOperation(log, 'op0', 30) + const op1 = new SlowOperation(log, 'op1-gated-by-lowered-floor', 0) + + const tx = new Transaction({ timeout: lowFloorBudget }) + tx.addOperation(op0) + tx.addOperation(op1) + + const error = await tx.execute().catch((e) => e) + expect(error).toBeInstanceOf(TransactionTimeoutError) + expect(op1.executed).toBe(false) + }) +}) diff --git a/tests/unit/transaction/vectorIndexOperations-rename.test.ts b/tests/unit/transaction/vectorIndexOperations-rename.test.ts new file mode 100644 index 00000000..bdfe58a3 --- /dev/null +++ b/tests/unit/transaction/vectorIndexOperations-rename.test.ts @@ -0,0 +1,176 @@ +/** + * @module tests/unit/transaction/vectorIndexOperations-rename + * @description Coverage for the backend-neutral vector-index transaction + * operations (formerly `AddToHNSWOperation` / `RemoveFromHNSWOperation`). + * These op-name strings surface directly in consumer-visible transaction + * journals and timings — a fossil "HNSW" name misdirects an operator running + * a native (non-HNSW) vector provider. Verifies: + * 1. rollback wiring stayed byte-identical under the rename, + * 2. the emitted `name` stamps the active backend — `'hnsw-js'` for + * Brainy's own JS index, a provider's own required `name` when it + * self-identifies, and the tolerant-loud `'unknown-provider'` literal + * (plus one console.warn) when a runtime instance lacks `name` + * altogether (an older native provider compiled against the previous, + * optional `providerId` contract) — never a silently-wrong guess. + */ +import { describe, it, expect, vi, afterEach } from 'vitest' +import { + AddToVectorIndexOperation, + RemoveFromVectorIndexOperation, + BatchAddToVectorIndexOperation +} from '../../../src/transaction/operations/IndexOperations.js' +import type { VectorIndexProvider } from '../../../src/plugin.js' +import type { VectorDocument, Vector } from '../../../src/coreTypes.js' +import { JsHnswVectorIndex } from '../../../src/hnsw/hnswIndex.js' + +/** + * A minimal, real (not mocked) VectorIndexProvider implementation backed by + * a plain Map — exercises the exact contract the operations call through, + * without any Brainy internals. + */ +class FakeVectorProvider implements VectorIndexProvider { + readonly items = new Map() + constructor(readonly name: string) {} + + async addItem(item: VectorDocument): Promise { + this.items.set(item.id, item.vector) + return item.id + } + async removeItem(id: string): Promise { + return this.items.delete(id) + } + async getItem(id: string): Promise { + const vector = this.items.get(id) + return vector ? { id, vector } : undefined + } + async search(): Promise> { + return [] + } + size(): number { + return this.items.size + } + clear(): void { + this.items.clear() + } + async rebuild(): Promise {} + async flush(): Promise { + return 0 + } + getPersistMode(): 'immediate' | 'deferred' { + return 'immediate' + } +} + +describe('Vector index transaction operations — backend-neutral rename', () => { + describe('rollback wiring (byte-identical behavior under the rename)', () => { + it('AddToVectorIndexOperation rollback removes a newly-added item', async () => { + const provider = new FakeVectorProvider('fake-provider') + const op = new AddToVectorIndexOperation(provider, 'id-1', [1, 2, 3]) + + const rollback = await op.execute() + expect(provider.items.has('id-1')).toBe(true) + + await rollback() + expect(provider.items.has('id-1')).toBe(false) + }) + + it('AddToVectorIndexOperation rollback is a no-op when the item pre-existed (update semantics)', async () => { + const provider = new FakeVectorProvider('fake-provider') + await provider.addItem({ id: 'id-1', vector: [9, 9, 9] }) + + const op = new AddToVectorIndexOperation(provider, 'id-1', [1, 2, 3]) + const rollback = await op.execute() + expect(provider.items.get('id-1')).toEqual([1, 2, 3]) + + await rollback() + // Pre-existing item is NOT removed by rollback — it stays (the update + // itself is not undone by this op; that's RemoveFromVectorIndexOperation's job). + expect(provider.items.has('id-1')).toBe(true) + }) + + it('RemoveFromVectorIndexOperation rollback re-adds the item with its original vector', async () => { + const provider = new FakeVectorProvider('fake-provider') + await provider.addItem({ id: 'id-1', vector: [4, 5, 6] }) + + const op = new RemoveFromVectorIndexOperation(provider, 'id-1', [4, 5, 6]) + const rollback = await op.execute() + expect(provider.items.has('id-1')).toBe(false) + + await rollback() + expect(provider.items.get('id-1')).toEqual([4, 5, 6]) + }) + + it('BatchAddToVectorIndexOperation rolls back every item in reverse order on undo', async () => { + const provider = new FakeVectorProvider('fake-provider') + const op = new BatchAddToVectorIndexOperation(provider, [ + { id: 'a', vector: [1] }, + { id: 'b', vector: [2] }, + { id: 'c', vector: [3] } + ]) + + const rollback = await op.execute() + expect([...provider.items.keys()].sort()).toEqual(['a', 'b', 'c']) + + await rollback() + expect(provider.items.size).toBe(0) + }) + }) + + describe('backend stamping in the emitted op name', () => { + afterEach(() => { + vi.restoreAllMocks() + }) + + it('stamps the real JS HNSW index as hnsw-js (self-identifying name)', () => { + const index = new JsHnswVectorIndex() + const addOp = new AddToVectorIndexOperation(index, 'id-1', [1, 2, 3]) + const removeOp = new RemoveFromVectorIndexOperation(index, 'id-1', [1, 2, 3]) + + expect(addOp.name).toBe('AddToVectorIndex(hnsw-js)') + expect(removeOp.name).toBe('RemoveFromVectorIndex(hnsw-js)') + }) + + it('stamps a self-identifying provider\'s own name, never "HNSW"', () => { + const provider = new FakeVectorProvider('acme-diskann') + const addOp = new AddToVectorIndexOperation(provider, 'id-1', [1, 2, 3]) + const removeOp = new RemoveFromVectorIndexOperation(provider, 'id-1', [1, 2, 3]) + const batchOp = new BatchAddToVectorIndexOperation(provider, [{ id: 'a', vector: [1] }]) + + expect(addOp.name).toBe('AddToVectorIndex(acme-diskann)') + expect(removeOp.name).toBe('RemoveFromVectorIndex(acme-diskann)') + expect(batchOp.name).toBe('BatchAddToVectorIndex(acme-diskann)') + expect(addOp.name).not.toContain('HNSW') + expect(removeOp.name).not.toContain('HNSW') + }) + + it('tolerant-loud: stamps "unknown-provider" and warns exactly once when a runtime provider lacks the required `name` (an older native provider compiled against the previous optional `providerId` contract)', () => { + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) + // Simulate a native provider compiled before `name` was required — the + // TypeScript contract requires `name`, but `as unknown as` mirrors what + // actually reaches this code at runtime from an un-rebuilt native addon. + const legacyProvider = { + addItem: async (item: VectorDocument) => item.id, + removeItem: async () => true, + search: async () => [], + size: () => 0, + clear: () => {}, + rebuild: async () => {}, + flush: async () => 0, + getPersistMode: () => 'immediate' as const + } as unknown as VectorIndexProvider + + const addOp = new AddToVectorIndexOperation(legacyProvider, 'id-1', [1, 2, 3]) + const removeOp = new RemoveFromVectorIndexOperation(legacyProvider, 'id-1', [1, 2, 3]) + + expect(addOp.name).toBe('AddToVectorIndex(unknown-provider)') + expect(removeOp.name).toBe('RemoveFromVectorIndex(unknown-provider)') + // Never silently falls back to the JS engine's name for a provider it + // knows nothing about — that's the exact fossil-naming bug being fixed. + expect(addOp.name).not.toContain('hnsw-js') + // Loud, not silent — but exactly once per provider instance, not once + // per op stamped against it. + expect(warnSpy).toHaveBeenCalledTimes(1) + expect(warnSpy.mock.calls[0][0]).toContain('name') + }) + }) +}) diff --git a/tests/unit/utils/osLimits.test.ts b/tests/unit/utils/osLimits.test.ts new file mode 100644 index 00000000..a56d593e --- /dev/null +++ b/tests/unit/utils/osLimits.test.ts @@ -0,0 +1,76 @@ +/** + * @module tests/unit/utils/osLimits + * @description OS-limit detection for pool-scale use. Laws: + * (1) the /proc/self/limits parser reads soft/hard NOFILE exactly, including + * 'unlimited'; (2) assessment warns ONLY below the pool floors and NEVER + * on an unreadable (null) limit — no measurement, no claim; (3) the full + * check composes both sources and survives unreadable /proc silently. + */ +import { describe, it, expect } from 'vitest' +import { + parseProcLimits, + assessOsLimits, + checkOsLimits, + NOFILE_POOL_FLOOR, + MAX_MAP_COUNT_POOL_FLOOR +} from '../../../src/utils/osLimits.js' + +const SAMPLE_LIMITS = [ + 'Limit Soft Limit Hard Limit Units', + 'Max cpu time unlimited unlimited seconds', + 'Max open files 1024 1048576 files', + 'Max locked memory 8388608 8388608 bytes' +].join('\n') + +describe('osLimits — detect + warn at pool scale', () => { + it('parses soft/hard NOFILE from /proc/self/limits, including unlimited', () => { + expect(parseProcLimits(SAMPLE_LIMITS)).toEqual({ soft: 1024, hard: 1048576 }) + expect( + parseProcLimits('Max open files unlimited unlimited files') + ).toEqual({ soft: Infinity, hard: Infinity }) + expect(parseProcLimits('no such row here')).toEqual({ soft: null, hard: null }) + }) + + it('warns below the floors, stays quiet at or above them', () => { + const low = assessOsLimits({ nofileSoft: 1024, nofileHard: 1048576, maxMapCount: 65530 }) + expect(low).toHaveLength(2) + expect(low[0]).toContain('RLIMIT_NOFILE soft limit is 1024') + expect(low[0]).toContain(`ulimit -n ${NOFILE_POOL_FLOOR}`) + expect(low[0]).toContain('raise the soft limit only') // hard already allows it + expect(low[1]).toContain('vm.max_map_count is 65530') + expect(low[1]).toContain(`vm.max_map_count=${MAX_MAP_COUNT_POOL_FLOOR}`) + + expect( + assessOsLimits({ + nofileSoft: NOFILE_POOL_FLOOR, + nofileHard: Infinity, + maxMapCount: MAX_MAP_COUNT_POOL_FLOOR + }) + ).toEqual([]) + }) + + it('an unreadable limit makes NO claim — nulls never warn', () => { + expect(assessOsLimits({ nofileSoft: null, nofileHard: null, maxMapCount: null })).toEqual([]) + }) + + it('checkOsLimits composes both sources and survives unreadable /proc silently', async () => { + const report = await checkOsLimits(async (p) => { + if (p === '/proc/self/limits') return SAMPLE_LIMITS + if (p === '/proc/sys/vm/max_map_count') return '65530\n' + throw new Error('unexpected path') + }) + expect(report.nofileSoft).toBe(1024) + expect(report.maxMapCount).toBe(65530) + expect(report.warnings).toHaveLength(2) + + const offLinux = await checkOsLimits(async () => { + throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) + }) + expect(offLinux).toEqual({ + nofileSoft: null, + nofileHard: null, + maxMapCount: null, + warnings: [] + }) + }) +}) diff --git a/tests/unit/utils/paramValidation.test.ts b/tests/unit/utils/paramValidation.test.ts index e7113506..4dc83554 100644 --- a/tests/unit/utils/paramValidation.test.ts +++ b/tests/unit/utils/paramValidation.test.ts @@ -270,27 +270,24 @@ describe('Zero-Config Parameter Validation', () => { expect(config.availableMemory).toBeGreaterThan(0) }) - it('should adapt limits based on query performance', () => { - const initialConfig = getValidationConfig() - const initialLimit = initialConfig.maxLimit - - // Simulate fast queries with large results + it('never mutates the cap from query timing (telemetry only)', () => { + const initialLimit = getValidationConfig().maxLimit + + // Fast queries with large results: no silent growth. for (let i = 0; i < 10; i++) { recordQueryPerformance(50, initialLimit * 0.9) } - - const updatedConfig = getValidationConfig() - // Limit might increase if performance is good - expect(updatedConfig.maxLimit).toBeGreaterThanOrEqual(initialLimit) - - // Simulate slow queries - for (let i = 0; i < 10; i++) { - recordQueryPerformance(2000, 100) + expect(getValidationConfig().maxLimit).toBe(initialLimit) + + // A burst of catastrophically slow queries must not strangle the cap. + // The removed "learning" ratchet shrank it 20% per recorded query down + // to a floor of 1000 — below the documented MIN_AUTO_QUERY_LIMIT — and + // the error message blamed "available free memory" (a production + // incident: every find({ limit: 5000 }) failed on an idle 23GB-free box). + for (let i = 0; i < 50; i++) { + recordQueryPerformance(90_000, 100) } - - const finalConfig = getValidationConfig() - // Limit should decrease if performance is poor - expect(finalConfig.maxLimit).toBeLessThanOrEqual(updatedConfig.maxLimit) + expect(getValidationConfig().maxLimit).toBe(initialLimit) }) }) }) \ No newline at end of file diff --git a/tests/unit/validate-invariants-delegation.test.ts b/tests/unit/validate-invariants-delegation.test.ts new file mode 100644 index 00000000..45e12ccd --- /dev/null +++ b/tests/unit/validate-invariants-delegation.test.ts @@ -0,0 +1,99 @@ +/** + * @module tests/unit/validate-invariants-delegation + * @description validateIndexConsistency() was blind to native + * providers — it only saw the JS metadata index, so a native manifest↔segments↔count + * divergence read as "healthy". It now feature-detects + aggregates each provider's + * validateInvariants(), and repairIndex() maps a failing invariant with heal:'rebuild' + * to that provider's rebuild(). "healthy-while-broken must be impossible." + */ +import { describe, it, expect, beforeEach } from 'vitest' +import { Brainy, NounType } from '../../src/index.js' +import type { ProviderInvariantReport } from '../../src/index.js' + +const healthyReport = (provider: string): ProviderInvariantReport => ({ + provider, + healthy: true, + serving: true, + invariants: [{ name: 'manifest-residency', holds: true, detail: 'ok', heal: 'none' }], + checkedAt: 1, + durationMs: 1 +}) + +const brokenReport = (provider: string): ProviderInvariantReport => ({ + provider, + healthy: false, + serving: true, + invariants: [ + { name: 'manifest-residency', holds: true, detail: 'ok', heal: 'none' }, + { + name: 'posted-count-floor', + holds: false, + detail: 'posted 2304 < canonical 2354', + expected: 2354, + actual: 2304, + heal: 'rebuild' + } + ], + checkedAt: 1, + durationMs: 2 +}) + +describe('validateIndexConsistency delegates to provider validateInvariants() (Pass 3)', () => { + let brain: any + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, dimensions: 384 }) + await brain.init() + await brain.add({ data: 'x', type: NounType.Concept }) + await brain.flush() + }) + + it('a broken provider report makes the store unhealthy and names the failing invariant', async () => { + brain.index.validateInvariants = async () => brokenReport('vector') + const v = await brain.validateIndexConsistency() + expect(v.healthy).toBe(false) + expect(v.recommendation).toMatch(/posted-count-floor/) + expect(v.recommendation).toMatch(/posted 2304 < canonical 2354/) + expect(v.recommendation).toMatch(/repairIndex\(\)/) + expect(v.providers?.some((p: ProviderInvariantReport) => p.provider === 'vector' && !p.healthy)).toBe(true) + delete brain.index.validateInvariants + }) + + it('all-healthy provider reports do not flip the store unhealthy', async () => { + brain.index.validateInvariants = async () => healthyReport('vector') + brain.graphIndex.validateInvariants = async () => healthyReport('graph') + const v = await brain.validateIndexConsistency() + expect(v.healthy).toBe(true) + expect(v.providers?.length).toBe(2) + delete brain.index.validateInvariants + delete brain.graphIndex.validateInvariants + }) + + it('a validateInvariants() that THROWS is surfaced as unhealthy, never swallowed', async () => { + brain.index.validateInvariants = async () => { throw new Error('provider blew up') } + const v = await brain.validateIndexConsistency() + expect(v.healthy).toBe(false) + expect(v.recommendation).toMatch(/validate-invariants-threw/) + delete brain.index.validateInvariants + }) + + it('providers without validateInvariants() are omitted (JS baseline unchanged)', async () => { + const v = await brain.validateIndexConsistency() + expect(v.providers).toBeUndefined() + expect(typeof v.healthy).toBe('boolean') + }) + + it('repairIndex() rebuilds a provider whose failing invariant asks for it', async () => { + let rebuilt = false + brain.index.validateInvariants = async () => (rebuilt ? healthyReport('vector') : brokenReport('vector')) + const origRebuild = brain.index.rebuild.bind(brain.index) + brain.index.rebuild = async (...a: any[]) => { rebuilt = true; return origRebuild(...a) } + await brain.repairIndex() + expect(rebuilt).toBe(true) + // After repair, the store validates healthy again. + const v = await brain.validateIndexConsistency() + expect(v.providers?.find((p: ProviderInvariantReport) => p.provider === 'vector')?.healthy).toBe(true) + brain.index.rebuild = origRebuild + delete brain.index.validateInvariants + }) +})