diff --git a/CHANGELOG.md b/CHANGELOG.md index fd0de54e..457cbd0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,185 +2,6 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. -### [8.9.0](https://github.com/soulcraftlabs/brainy/compare/v8.8.2...v8.9.0) (2026-07-19) - -- docs: measured performance envelopes v1 (per-op p50/p95 at 1k and 10k, pure-JS floor) (5cabd78) -- fix: release drains in-flight writer-lock heartbeat — no phantom lock after unlink (70e4bc8) -- feat: flush() never compacts — history maintenance moves to close() with bounded passes (300d9f2) - - -### [8.8.2](https://github.com/soulcraftlabs/brainy/compare/v8.8.1...v8.8.2) (2026-07-19) - -- fix: one field-resolution law across aggregation hooks, source.where, removeMany, and find() spellings (945d92d) -- chore: push public docs to the soulcraft.com ingest door on release (42037d0) - - -### [8.8.1](https://github.com/soulcraftlabs/brainy/compare/v8.8.0...v8.8.1) (2026-07-18) - -- fix: O(1) adaptive retention accounting + historyStats fleet audit (6207e48) -- fix: import dedup off-switch honesty + brain-owned lifecycle for the background pass (4fcef7b) - - -### [8.8.0](https://github.com/soulcraftlabs/brainy/compare/v8.7.1...v8.8.0) (2026-07-17) - -- feat: OS-limit detection for pool-scale deployments (16a73b8) - - -### [8.7.1](https://github.com/soulcraftlabs/brainy/compare/v8.7.0...v8.7.1) (2026-07-17) - -- fix: race-proof writer-lock acquisition + machine-readable conflict through init (01a3b46) - - -### [8.7.0](https://github.com/soulcraftlabs/brainy/compare/v8.6.0...v8.7.0) (2026-07-17) - -- feat: scaled transact budgets + labeled timeout diagnostics + envelope docs (6ef9fcb) - - -### [8.6.0](https://github.com/soulcraftlabs/brainy/compare/v8.5.2...v8.6.0) (2026-07-17) - -- feat: brain.auditGraph() — read-only graph-truth audit (2a03fae) - - -### [8.5.2](https://github.com/soulcraftlabs/brainy/compare/v8.5.1...v8.5.2) (2026-07-17) - -- fix: exception-safe aggregation backfill + generation-verified adoption + loud open-path guards (a77b064) - - -### [8.5.1](https://github.com/soulcraftlabs/brainy/compare/v8.5.0...v8.5.1) (2026-07-17) - -- fix: aggregation state adoption on reopen + single-flight backfill + query-cap ratchet removal (da55be7) -- docs: external-backups/sparse-storage guide + generation fact log concept (593bb8b) - - -### [8.5.0](https://github.com/soulcraftlabs/brainy/compare/v8.4.0...v8.5.0) (2026-07-15) - -- test: tolerant timing assertion in the execution-time measure test (4dc0a92) -- feat: committedGeneration capability + pinned durability/stability contracts (d1ecee1) -- docs: RELEASES.md entry for 8.5.0 (provider fact-log access + shared verifier) (e4f37cd) -- feat: provider access to the fact log + shared stamp verifier via internals (352e356) - - -### [8.4.0](https://github.com/soulcraftlabs/brainy/compare/v8.3.3...v8.4.0) (2026-07-15) - -- docs: RELEASES.md entry for 8.4.0 (generation fact log + family stamp) (4a60b43) -- feat: entity-tree family stamp — sourceGeneration + rollup coherence at open (2888ae6) -- feat: generation fact log — after-image commit records, dual-written at every commit point (38b0041) - - -### [8.3.3](https://github.com/soulcraftlabs/brainy/compare/v8.3.2...v8.3.3) (2026-07-15) - -- docs: RELEASES.md entry for 8.3.3 (rename containment fix + repair) (c3feafd) -- test: lens-consistency regression — combined vs subtype-only vs canonical ground truth (4fb41f9) -- fix: VFS rename moves the containment edge — no ghost in the old directory (af8c179) - - -### [8.3.2](https://github.com/soulcraftlabs/brainy/compare/v8.3.1...v8.3.2) (2026-07-14) - -- docs: RELEASES.md entry for 8.3.2 (honest counters) (0932ecd) -- fix: honest counters — removal never re-reads the removed record + repairIndex recounts and persists all rollups (2e2ba9c) - - -### [8.3.1](https://github.com/soulcraftlabs/brainy/compare/v8.3.0...v8.3.1) (2026-07-14) - -- docs: RELEASES.md entry for 8.3.1 (full-removal deletes + family-scoped gate) (c0c68ac) -- fix: full-removal canonical deletes + family-scoped migration gate (366f9a9) -- docs: cite the cross-layer integrity contract generically in comments and notes (1d26988) - - -### [8.3.0](https://github.com/soulcraftlabs/brainy/compare/v8.2.8...v8.3.0) (2026-07-13) - -- docs: RELEASES.md entry for 8.3.0 (heal-cost + cross-layer integrity contract) (7692c6f) -- perf: parallel + id-only canonical enumeration (heal-cost dominant term) (ec5b933) -- feat: registered-blob family contract — declared index blobs are undeletable (bfa1762) -- feat: validateIndexConsistency delegates to provider invariants (6bcb54f) - - -### [8.2.8](https://github.com/soulcraftlabs/brainy/compare/v8.2.7...v8.2.8) (2026-07-13) - -- fix: honest index readiness — no silently-empty queries on a cold index (d0f69c7) - - -### [8.2.7](https://github.com/soulcraftlabs/brainy/compare/v8.2.6...v8.2.7) (2026-07-13) - -- fix: restore loadBinaryBlob fault-propagation (native column-store lockstep) (b6c7039) - - -### [8.2.6](https://github.com/soulcraftlabs/brainy/compare/v8.2.5...v8.2.6) (2026-07-13) - -- docs: RELEASES.md entry for 8.2.6 (write/index-spine hardening) (a873852) -- chore: hold loadBinaryBlob fault-propagation for the cortex column-store lockstep (36c10c1) -- fix: aggregation surfaces materialize/state-load failures loudly (02eff64) -- fix: surface a degraded derived index on reads instead of serving it silently (ba958d9) -- fix: saveBinaryBlob never acks a durable write that stored nothing (7feba49) -- fix: refuse writes when single-op history cannot be made durable (54c1836) -- fix: clear() wipes the full native/derived footprint, not a subset (d8301f8) -- fix: surface segment/entity read faults loudly instead of masking as absent (af5d2f3) -- fix: spine hardening pass 1 (part) — count symmetry, honest partial-load, flush durability, read-fault propagation (119087a) -- test: pin the read-your-writes contract under the single writer (eb9c4eb) - - -### [8.2.5](https://github.com/soulcraftlabs/brainy/compare/v8.2.4...v8.2.5) (2026-07-12) - -- docs: RELEASES.md entry for 8.2.5 (honest rollback-failure response) (a7c7aa5) -- fix: honest response when a transaction rollback cannot complete (711d2f0) - - -### [8.2.4](https://github.com/soulcraftlabs/brainy/compare/v8.2.3...v8.2.4) (2026-07-12) - -- docs: RELEASES.md entry for 8.2.4 (non-destructive restore) (4574695) -- fix: non-destructive, crash-resumable restore (a2f4f6a) - - -### [8.2.3](https://github.com/soulcraftlabs/brainy/compare/v8.2.2...v8.2.3) (2026-07-12) - -- docs: RELEASES.md entry for 8.2.3 (transact durability barrier) (be5ce0b) -- fix: transact durability barrier — committed transactions are durable on return (3b8fa51) - - -### [8.2.2](https://github.com/soulcraftlabs/brainy/compare/v8.2.1...v8.2.2) (2026-07-11) - -- docs: RELEASES.md entry for 8.2.2 (transaction timeout rollback) (ed97006) -- fix: transaction timeout rolls back applied operations (no torn state) (508a8e3) - - -### [8.2.1](https://github.com/soulcraftlabs/brainy/compare/v8.2.0...v8.2.1) (2026-07-10) - -- test: update graph-index operation constructors to the VerbEndpointInts signature (62a449d) -- docs: RELEASES.md entry for 8.2.1 (transact forward-ref parity fix) (7089782) -- fix: transact forward references resolve graph endpoint ints at execute time (a175406) - - -### [8.2.0](https://github.com/soulcraftlabs/brainy/compare/v8.1.0...v8.2.0) (2026-07-10) - -- docs: RELEASES.md entry for 8.2.0 (temporal VFS) (98ceadc) -- feat: temporal VFS — file content joins the Model-B immutability model (a3467e1) -- docs: pin the write-path invariant in the plugin contract (the onChange change-feed guarantee) (4af8fb3) - - -### [8.1.0](https://github.com/soulcraftlabs/brainy/compare/v8.0.17...v8.1.0) (2026-07-10) - -- docs: RELEASES.md entry for 8.1.0 (brain.onChange change feed) (4e9be08) -- feat: brain.onChange — the in-process change feed for every committed mutation (fd5edb5) - - -### [8.0.17](https://github.com/soulcraftlabs/brainy/compare/v8.0.16...v8.0.17) (2026-07-08) - -- docs: RELEASES.md entry for 8.0.17 (canonical count recovery + dead-machinery sweep) (6b8b9cb) -- fix: count recovery scans the canonical layout; remove the dead 7.x hnsw sharding machinery (352e2da) - - -### [8.0.16](https://github.com/soulcraftlabs/brainy/compare/v8.0.15...v8.0.16) (2026-07-08) - -- docs: RELEASES.md entry for 8.0.16 (atomic ifAbsent/upsert + exact blob refCounts) (54e7c0e) -- fix: atomic ifAbsent/upsert inserts + exact blob reference counts under concurrency (867939e) - - -### [8.0.15](https://github.com/soulcraftlabs/brainy/compare/v8.0.14...v8.0.15) (2026-07-08) - -- docs: RELEASES.md entry for 8.0.15 (atomic ifRev CAS) (b1fe25a) -- fix: ifRev CAS is atomic — the revision check now runs under the commit mutex (9a3d1bd) - - ### [8.0.14](https://github.com/soulcraftlabs/brainy/compare/v8.0.13...v8.0.14) (2026-07-07) - docs: RELEASES.md entry for 8.0.14 (migration preserves branch-scoped non-entity state) (64188a3) diff --git a/RELEASES.md b/RELEASES.md index 7799c6f6..018856c0 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -8,842 +8,8 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so - Debugging data, query, or storage behaviour - A new Brainy feature is available that you want to adopt -## Removed APIs — 7.x → 8.x (the complete ledger) - -Every public API removed at the 8.0 major, with its sanctioned replacement. If your code -still calls a left-column name on 8.x it throws (or the config key is rejected) — the -replacement is always a one-line change. (Standing contract from 8.9.0 forward: removals -happen only at majors, after ≥1 minor of loud runtime deprecation naming the replacement.) - -| Removed (7.x) | Replacement (8.x) | -|---|---| -| `brain.search(query, k)` | `find({ query })` — semantic; `find({ query, searchMode })` for hybrid | -| `brain.getRelations({...})` | `related(id, opts)` for adjacency; `find({ connected: {...} })` for scoped traversal | -| `brain.neural()` clustering | `find({ vector })` + aggregation `GROUP BY` | -| `Db.search()` | `db.find({ vector })` | -| Pre-8.0 storage path aliases (`directory`, `basePath`, …) | one `storage.path` key (old aliases throw) | -| Reserved keys inside `metadata` bags (silently remapped in 7.x) | top-level params (`subtype`, `visibility`, `confidence`, `weight`, …) — reserved-in-bag throws | -| 7.x COW branches layout (`branches/main/`) | generational MVCC (`asOf()`, `now()`, `db.persist(path)`) — on-disk migration is automatic at first 8.x open | - -The fork/snapshot family (`brain.snapshot()`, `createSnapshot()`, `restoreSnapshot()`) -is sometimes cited as a 7.x removal — those methods never existed on 7.x; the 8.0 Db API -(`asOf`/`persist`/`restore({confirm})`) is their first real implementation. - --- -## v8.9.0 — 2026-07-19 (flush is durability-only: history maintenance moves to close()) - -The write path stops paying maintenance costs — the last structural piece of the -flush-storm class (a production deployment measured single writes blocked 25–191s behind -history reclaim running inline on flush under memory pressure): - -- **`flush()` never compacts history.** It persists the current window's deltas and - nothing else — its cost no longer depends on history backlog or retention mode, in any - configuration. **`close()` is the auto-compaction site** (time-bounded per pass, ~5s; - an early stop is a consistent prefix and the next pass resumes). -- **`compactHistory()` gains `timeBudgetMs`** — bound your own maintenance windows; the - same resumable-prefix guarantee applies. -- **The documented trade**: a long-lived writer that never closes accumulates history - until its next explicit `compactHistory()`. Predictable writes, explicit maintenance. - If you run bounded retention on an always-on service, schedule a periodic - `compactHistory({ ...caps, timeBudgetMs })` in your maintenance window. -- **New public doc: `docs/performance-envelopes.md`** — measured per-op envelopes - (p50/p95 at stated scales, hardware, and backend, with the measuring script cited). - Refresh rule going forward: any release touching a measured path re-runs that op's - benchmark and updates the envelope in the same release. -- **New in this file: the Removed APIs 7.x→8.x table** (top of this document) — every - removal with its sanctioned replacement, one place, per the engine-currency contract. - Standing from here: removals only at majors, after ≥1 minor of loud runtime deprecation. - -## v8.8.2 — 2026-07-19 (one field-resolution law: reserved-field aggregates stop drifting) - -Four fixes from a consumer conformance audit, all rooted in the same disease — two field-resolution -regimes where there must be one: - -- **Aggregates grouped by a RESERVED field (`subtype`, `visibility`, …) now decrement on - delete.** The delete/update hooks fed the aggregation engine a partial entity view (type, - service, data, metadata only), so a reserved-field `groupBy` resolved to a nonexistent group - on the way DOWN — counts drifted upward forever after any delete, and updates that moved an - entity between reserved-field groups double-counted it. The hooks now pass the full-fidelity - entity view (every reserved field top-level, the same shape the add path uses). If your - deployment derives stats from reserved-field aggregates, re-define those aggregates once - after upgrading (a changed definition triggers one rescan) or run them fresh — the drifted - persisted counts do not self-heal retroactively. -- **Aggregation `source.where` on reserved fields now filters** instead of silently matching - nothing: the matcher resolves fields through the same resolver `groupBy` uses (top-level - standard fields + custom metadata), so `where: { subtype: 'note' }` means what it says. -- **`removeMany()` refuses empty/invalid selectors loudly.** A bare array passed positionally - (`removeMany([id])` instead of `removeMany({ ids: [id] })`), an empty params object, or - `ids: []` used to resolve successfully having deleted nothing. All three now throw. -- **`find()` accepts both where-key spellings.** Metadata is flattened at index time - (`metadata.entry.title` indexes as `entry.title`); a `metadata.`-prefixed where key now - falls back to its flattened spelling when the prefixed one isn't indexed — the - "unindexed field(s), returning []" confusion for storage-shaped spellings is gone. (A - literal nested custom key named `metadata` still wins when indexed as spelled.) - -## v8.8.1 — 2026-07-18 (flush no longer walks the whole generation history + the import dedup off-switch is now honest) - -### The flush-storm fix (production incident, reported by a long-running deployment) - -Under the default adaptive retention, **every `flush()` re-walked the entire committed -generation history** to compute total history bytes for the budget check — O(all -generations) with disk re-reads past the 4,096-entry delta-cache bound. On a brain with -70,000+ accumulated generations that turned every write into a full-tail scan (60-100s -writes), even though the budget (free-RAM-based) never tripped and nothing was ever -reclaimed. Fixed: - -- `historyBytes()` now maintains a **running total**: seeded by one walk on first use, - then updated incrementally at every commit and reclaim — the adaptive retention check - on every flush is O(1). Invariant regression-pinned (running total ≡ fresh walk through - both commit paths and compaction). -- New **`brain.historyStats()`** (read-only, exported `HistoryStats`): generation count, - total on-disk bytes, generation/timestamp range, compaction horizon, retention mode, - and the effective adaptive budget — the one-call fleet-audit for sizing retention - exposure per brain. -- Interim guidance for keep-everything deployments already affected: `retention: 'all'` - skips the adaptive accounting entirely (and is the correct policy if you never want - history reclaimed). The accumulated files are harmless at rest; this release removes - the per-write cost of their existence. - -### The import dedup off-switch (lifecycle honesty) - -The post-import background deduplication pass (a merge-DELETE writer that runs ~5 minutes -after an import, merging entities judged duplicates by id / name / vector similarity) had -three lifecycle defects, all fixed: - -- **`enableDeduplication: false` now actually disables it.** The background pass was - scheduled unconditionally — an import that explicitly opted out could still have - entities auto-removed 5 minutes later. The flag now gates BOTH the inline merge and - the background pass (regression-pinned). -- **One deduplicator per brain, owned by the brain.** Each `import()` call constructed its - own coordinator + deduplicator, so the "debounced" timer never actually debounced across - imports (N imports = N delete timers). The brain now owns a single instance — the - debounce genuinely spans imports — and `close()` cancels pending work, so a delete pass - can never fire against a closed brain. -- **The 5-minute timer is unref'd** — a pending pass no longer holds the process open - (the exit-hang class; this timer had escaped the earlier sweep). - -Retention note for keep-everything deployments: with `enableDeduplication: false` on -import calls and `retention: 'all'` in config, no engine path removes records -automatically. - -## v8.8.0 — 2026-07-17 (OS-limit detection for pool-scale deployments) - -Small minor: brains now detect the two OS limits that bite at pool scale and warn **before** -the incident instead of during it. - -- At open (once per process, Linux-only, measurement-only), Brainy reads `RLIMIT_NOFILE` - (soft/hard, from `/proc/self/limits`) and `vm.max_map_count`, and warns loudly when either - sits below the pool-scale floors (soft NOFILE < 65 536; max_map_count < 262 144) — with the - exact raise commands (`ulimit -n` / `LimitNOFILE=` / `sysctl vm.max_map_count`). On stock - defaults the failure otherwise arrives as `EMFILE` or a failed mmap deep inside an index - open, long after the real cause stopped being visible. An unreadable limit produces **no** - warning — no measurement, no claim (non-Linux platforms stay silent). -- Exported for ops doors: `checkOsLimits()` returns the full `OsLimitsReport` - (values + warnings) programmatically, with the floors exported as constants. - -## v8.7.1 — 2026-07-17 (writer-lock acquisition is race-proof + machine-readable through init) - -Two hardenings of the multi-process writer lock (the `locks/_writer.lock` lease that makes a -second writer on the same brain directory fail loudly): - -- **Lock acquisition claims atomically.** The acquire path used read-then-write, leaving a - window where two processes racing an *absent* lock could both "succeed" — and the loser - kept running unlocked, silently. The claim is now an atomic create-exclusive write - (`O_EXCL`): exactly one racer wins; the loser re-evaluates and either fails loudly with - the winner's details or performs a verified stale-takeover. Bounded retries; contention - beyond them fails loudly rather than degrading into a lockless open. -- **`BRAINY_WRITER_LOCKED` survives `init()`.** The conflict error documents a - machine-readable contract (`err.code`, `err.lockInfo` with the holder's pid/host/ - heartbeat), but init's error wrapping silently stripped both, leaving consumers a message - to regex against. The error now passes through unwrapped. - -Measured while verifying (for operators sizing audits): `brain.auditGraph()` at a -production-consumer scale of ~2,600 relationships / 800 entities costs ~0.1 s warm and -~0.5 s cold, with exact scar counting across reopen. - -## v8.7.0 — 2026-07-17 (bulk-transact ergonomics: scaled budgets + timeout telemetry) - -The bulk-import ergonomics release, from a consumer's measured production incident (a serial -import on network-attached storage at ~2 s/op met a flat 30 s transaction budget): - -- **The transact apply budget now scales with the batch** — `max(30 s, opCount × 2 s)` — or - is exactly what you pass as the new `TransactOptions.timeoutMs`. A flat 30 s cap silently - limited honest bulk work to ~15 operations on slow disks while looking generous for small - batches. Internal batch paths (e.g. `removeMany` chunks) get the same scaling. -- **`TransactionTimeoutError` is a diagnosis, not just a failure**: it now reports the - operation it stopped at as `i/N` with the operation's name, elapsed vs budgeted time, and - states the batch rolled back atomically and is retryable. Its `context` carries the same - fields programmatically. -- **The transact envelope is documented** — batch sizing, budget math, chunking with - `ifAbsent` idempotency, and the precompute pattern (`embedBatch` + per-op `vector`) that - keeps model inference out of the commit path. Guide: `docs/guides/optimistic-concurrency.md`. -- Note: `brain.embed()` / `brain.embedBatch()` (the precompute APIs) already ship — public, - with native-provider passthrough, verified end-to-end (batch and single paths produce - bit-identical vectors; a vector-supplied `add` is fully searchable). Honest measurement: - on the default WASM engine, batch throughput ≈ sequential (~160 ms/text) — the precompute - win is keeping inference out of the budgeted commit path, not raw embedding speed. - -## v8.6.0 — 2026-07-17 (brain.auditGraph — the graph-truth verification instrument) - -A minor release adding one new public API, from the fleet's graph-trust program: a read-only -audit that **proves whether relationship reads return stored truth** on a given brain. - -- **`brain.auditGraph(options?)`** walks every canonical relationship record, queries the same - read path applications use (`related()` / VFS `readdir`) with all visibility tiers included, - and classifies every discrepancy into its failure family: `missingFromReads` (records the - read path omits — a stale adjacency index), `danglingEndpoints` (relationships whose endpoint - entity no longer exists — the historical partial-delete scar class), and `readOnlyVerbIds` - (read-path edges with no stored record — ghosts). Design-hidden internal/system edges are - counted separately so intentional hiding is never misclassified as loss. Counts are exact; - example lists cap at `maxExamples` with an explicit `truncatedExamples` flag; the result is - narrated loudly on incoherence. Mutates nothing — safe on a live brain. -- The operational pairing: audit → if incoherent, `repairIndex()` → audit again. A `coherent` - report after the repair is the verified all-clear. Run it after any engine upgrade, restore, - or migration. Guide: `docs/guides/inspection.md`. -- Also: `getNounIds` pagination now refuses an undecodable resume cursor loudly (the same - contract `getNouns`/`getVerbs` gained in 8.5.2 — the third and final walk brought under it). -- Types exported: `GraphAuditReport`, `GraphAuditDiscrepancy`. - -## v8.5.2 — 2026-07-17 (aggregation backfill: exception-safe, generation-verified, and loud) - -Hardening patch from a migration incident (a byte-copied store on new hardware; the service -entered a silent full-CPU loop at boot). Four changes, all in the aggregation engine's -backfill/adoption path: - -- **Backfill walks are exception-safe and non-destructive.** A rescan now builds into a - staging map and swaps in atomically on completion; a mid-walk failure drops the staging map, - keeps the previous live state serving, and surfaces the storage error to the failing query. - Previously the walk wiped live state *before* a scan that could throw, never cleared the - pending flag on failure, and re-ran a full walk on every subsequent query — a silent - wipe/walk/throw loop at the caller's retry rate. -- **Failed walks are latched.** After a walk fails, retries within a 30-second cooldown rethrow - the recorded error instantly instead of re-walking — a tight caller-side retry loop now costs - one loud error per query, never a full store walk per query. -- **Adoption is generation-verified.** Persisted aggregation state is stamped with the store's - committed generation at flush; reopen adoption requires the stamp to equal the current - watermark. Stale state (unclean shutdown) or over-counting state (a fact-log truncation on a - copied store pulled the watermark back) triggers exactly one loud rescan — never a silent - adopt. Pre-8.5.2 state on generation-aware stores rescans once after upgrade, then is stamped. -- **The path narrates.** Adoption decisions, no-adoptable-state outcomes, walk start/finish - (entity count + duration), and walk failures all log by default; a non-advancing storage - pagination cursor aborts the walk loudly instead of looping forever. - -Plus three guards from a full audit of every loop in the open/init path: - -- **Invalid pagination cursors fail loudly.** A supplied-but-undecodable resume token to - `getNouns`/`getVerbs` used to silently restart the walk at offset 0 — to a `while(hasMore)` - caller that re-serves page 1 forever (an unbounded silent CPU loop). It now throws with a - clear message instead. -- **The graph cold-load verb walk has a stall guard.** `hasMore=true` with a missing or - non-advancing cursor aborts loudly instead of re-reading the same page forever. -- **A derived index AHEAD of the store is named at open.** Brainy already surfaced a provider - generation *behind* the committed watermark; the *ahead* direction (the signature of a - byte-copy of a live service, or a log truncation during crash recovery) now logs a loud - warning explaining what happened and that `brain.repairIndex()` forces a heal — instead of - passing unnamed into whatever the derived index does next. - -## v8.5.1 — 2026-07-17 (aggregation state survives restarts + the query-cap ratchet removed) - -Patch release from a production incident (aggregate/count paths taking 40–90 s on an idle box -while vector search stayed fast, and every `find({ limit: 5000 })` suddenly failing against an -"auto-configured query limit of 1000"). Three fixes, one cosmetic: - -- **Aggregation state is actually adopted on reopen.** The boot pattern `defineAggregate()` → - query raced the engine's async state load: the synchronous define always won, flagged a - backfill, and the first query then wiped the just-loaded persisted state and re-walked the - entire store — every restart, forever. Reopening with an unchanged definition now adopts the - persisted state directly (zero scans); a backfill runs only on a real definition change, a - missing/failed state load, or a write that landed before adoption (exactness wins). Apps that - rely on persisted definitions without re-defining at boot also no longer race a spurious - "Aggregate not defined". -- **Backfills are single-flight and batched.** Concurrent queries on a cold aggregate used to - each wipe the others' partial state and start their own full store walk — under steady query - arrival the store never converged (the 40–90 s loop). Now all concurrent queries share one - walk, and one walk fills every aggregate pending backfill (M aggregates ≠ M scans). -- **The query-cap "learning" ratchet is removed.** `maxLimit` is a memory-protection bound, but - a hidden tuner shrank it 20 % per recorded query while the lifetime-average query time - exceeded 1 s — down to a floor of 1000, below the documented 10 000 auto floor, with no - practical recovery, and the resulting error blamed "available free memory" (stale basis - label). The cap now comes from its construction-time basis (or your explicit - `maxQueryLimit` / `reservedQueryMemory`) alone and never changes at runtime; query timing is - recorded for diagnostics only. -- **Cosmetic:** the engine's own persistence keys (`__aggregation_*`, `brainy:entityIdMapper`) - no longer log `[Storage] Unknown key format` at boot — they were always routed correctly; - they're now recognized before the warning fires. - -Operationally: if a host was bitten, upgrading and restarting is the whole fix — no repair -ritual needed. Setting `maxQueryLimit` explicitly remains the valve that bypasses auto-detection -entirely. - -## v8.5.0 — 2026-07-15 (provider access to the fact log + the shared stamp verifier) - -Small additive follow-up to 8.4.0, from the native accelerator's first consumption pass: - -- **Index providers can now reach the fact log through the storage adapter** — new optional - capability `storage.scanFacts()` / `storage.factLogHeadGeneration()` / `storage.factSegmentPaths()`, - wired by the host brain at init as a closure over its live log. Providers hold only `storage` and - must never construct their own fact-log reader (the log's open path is writer-side); `null` means - "no fact log here — use the enumeration walk." -- **The family-stamp verifier is shared** via `@soulcraft/brainy/internals` - (`readFamilyStamp` / `writeFamilyStamp` / `verifyFamilyStamp` + types) so first-party native - providers run literally the same verification function, never a synchronized copy. -- **Rollup stamp invariants accept strings** (`number | string`) — content fingerprints such as a - per-tree SHA-256 are valid invariant values; a type mismatch reads as incoherence, never a pass. -- **`storage.committedGeneration()`** — the committed watermark as a capability, so a provider - compares its stamp's `sourceGeneration` against the store's truth without parsing the private - manifest format. -- **Two durability/stability contracts pinned in the suite:** fsync-before-ack (holds for - `transact()` today; the single-op path is pinned as the documented future target — group commit - becomes latency batching, never durability skipping) and scan-stability-under-rotation (a scan - snapshot yields exactly its facts — no gaps, duplicates, or bleed-in — while segments rotate - beneath it). - -No behavior change for applications; all additions. - -## v8.4.0 — 2026-07-15 (the generation fact log — a sequential, self-verifying commit stream) - -A minor release, fully backward-compatible (all additions; no behavior change for existing APIs). -This is infrastructure: it changes nothing about how you query today, and lays the substrate that -makes index heals and incremental catch-up sequential-read problems instead of directory walks. - -- **Every committed write now also appends a "fact" — an after-image commit record.** Alongside the - existing before-image history, each committed generation (single-op and `transact()` alike) appends - what each touched entity/relationship *became* — or a body-less tombstone for a removal — to an - append-only, checksummed segment log under `_generations/facts/`. Crash-safe by construction: a - torn tail is detected and ignored; on open the log is reconciled to committed truth, so an absent - generation always means "never committed." `transact()` facts are durable when `transact()` - returns; single-op facts ride the same group-commit flush as their history. - -- **New: `brain.scanFacts()`** — stream committed facts in commit order, in batches, with heal-grade - telemetry (total scope up front; per-batch generation range, byte size, and segment; a summary - cross-check at the end; loud abort on any gap — never a silent skip). **`brain.factSegmentPaths()`** - hands zero-copy consumers the immutable sealed segment files directly. New exported types: - `CommitFact`, `FactOp`, `FactScanBatch`, `FactScanHandle`. Facts accumulate from the first write - after upgrading — pre-existing history is not retroactively converted (enumeration remains the - fallback for old data). - -- **New: the entity-tree family stamp.** At every flush/close, brainy stamps which committed - generation the canonical entity files reflect plus the rollup invariants (entity/relationship - counts) that verify the tree whole. At open, coherence is a comparison — a genuine divergence is - loud and names the failing invariant; `repairIndex()` recounts from canonical and re-stamps. New - exports: `readFamilyStamp`, `verifyFamilyStamp`, `ENTITY_TREE_STAMP_PATH`, `FamilyStamp` types. - -- **Storage adapters** gain optional binary raw-byte primitives (`appendRawBytes`, `readRawBytes`, - `writeRawBytes`, `rawByteSize`) — feature-detected; the filesystem and memory adapters implement - them; an adapter without them simply hosts no fact log. The fact-log namespace is registered as a - protected family: no sweeper or GC can delete under it. - -No API breaks. 24 new tests; the full commit-path regression suite is green. - -## v8.3.3 — 2026-07-15 (rename moves the containment edge — no ghost in the old directory) - -One production-reported fix plus a repair path, completing the delete/move hygiene arc (8.3.1 fixed -deletes, 8.3.2 fixed counters, this fixes moves). - -- **A cross-directory `vfs.rename()` now MOVES the containment edge instead of accumulating one per - parent.** The old parent's `Contains` edge was never removed on a move, leaving the entity a child - of **both** directories: `readdir(oldDir)` kept listing it after the move, re-creating the old path - showed the same name twice, and any tree-walking consumer (sync engines, file browsers) saw the - file in two places. The old edge is now removed by edge id, resolved from the graph's own adjacency - — a removal never requires reading the thing being removed. Bonus fix in the same seam: a move **to - the root** now gets its containment edge (it was previously skipped, orphaning the file out of - `readdir('/')`). - -- **`repairIndex()` now also reconciles VFS containment** (new `vfs.repairContainment()`): every VFS - entity's containment edges are checked against its canonical `metadata.path` — stale old-parent - ghosts and duplicate edges are removed, a missing expected edge is restored, and user - knowledge-graph edges are never touched (only `vfs-contains` edges are candidates). Loud per - repair. Stores that performed cross-directory renames under ≤8.3.2 should run `brain.repairIndex()` - once after upgrading — the same single ritual now heals orphan directories, counters, **and** - containment edges. - -- Also ships a permanent lens-consistency regression suite (combined type+subtype vs subtype-only vs - canonical ground truth, id-for-id, warm and after a cold reopen), ported from the field - investigation that closed the historical lens-drop report. - -No API changes beyond the new optional `vfs.repairContainment()` (also invoked by `repairIndex()`). - -## v8.3.2 — 2026-07-14 (honest counters — the recount + removal-without-re-reading) - -Completes 8.3.1's delete-hygiene story at the counter layer, from a production proof chain reported -by a downstream deployment: persisted entity totals were permanently **inflated** — deletes whose -count decrement was silently skipped — and because paginated `totalCount` serves -`Math.max(persistedTotal, scanned)`, the inflated number always won and **no disk cleanup could ever -lower it**. - -- **A removal's count decrement no longer requires re-reading the record being removed.** The - decrement was sourced from re-reading the entity's metadata inside the delete; if that read - returned `null` (a replace race, or a ghost left by a pre-8.3.1 partial delete) the decrement was - silently skipped while the paired add had counted — minting drift on every write→delete→re-create - cycle. The caller's pre-delete read now rides through the whole delete path - (`remove()`/`removeMany()` → the delete operation → `deleteNoun`/`deleteVerb` → - `deleteNounMetadata`/`deleteVerbMetadata`, both sides symmetric): a null internal read falls back - to the known prior record instead of skipping. The `StorageAdapter` signatures gain an optional - `priorMetadata` parameter (additive; existing adapters unaffected). - -- **`repairIndex()` is the sanctioned counter recount — unconditional, and it actually persists.** - `rebuildTypeCounts()` previously rebuilt only the type-statistics arrays and computed the total - *just to log it* — the persisted scalar (`counts.json`) survived every "rebuild" untouched, so an - already-inflated brain could never be corrected. One canonical walk now rebuilds **every** counter - rollup — scalar totals, per-type maps, and type statistics — and persists them, and `repairIndex()` - runs it unconditionally (not only when orphan directories are found: counters can be inflated over - perfectly clean shelves). Brains with delete history should run `brain.repairIndex()` once after - upgrading; the correction survives reopen. - -No API breaks (optional-parameter additions only). Regression tests cover the drift cycle, the -null-read decrement fallback, and the persisted recount across reopen. - -## v8.3.1 — 2026-07-14 (full-removal deletes + family-scoped migration gate) - -Two production-reported fixes in the write/index spine, plus an operator repair path. No API changes; -all behavior changes make previously-wrong states honest. - -- **Deleting an entity now removes it completely — no more "ghost" leftovers on disk.** A canonical - noun delete removed the metadata (content) leg but left the entity's `vectors.json` and its `/` - directory behind. Consequences observed in a production deployment: deleted rows were - indistinguishable on disk from damage scars, enumerated counts inflated monotonically with every - delete (the leftovers were counted forever), and locator-style reads hit unreadable ghost rows. - `remove()`/`removeMany()`/`deleteNoun`/`deleteVerb` now remove **both legs and the entity - container**, with a full two-leg before-image rollback inside the transaction. The generation log - still holds the delete's before-image, so `asOf()` time-travel reconstructs deleted entities exactly - as before — this is live-HEAD hygiene, not a history change. This also fixes **duplicate `readdir` - entries for re-created VFS paths** at the root: with no ghost state, a delete always unposts its - index rows, so a delete→recreate cycle lists the path exactly once (regression-tested across - repeated cycles). - -- **`repairIndex()` prunes ghost/scar directories left by earlier versions.** Stores that deleted - entities under ≤8.3.0 may hold orphaned entity directories (a vector-only leg, or an empty dir). - `brain.repairIndex()` now sweeps them: it removes only containers with **no metadata content leg** - (never a directory that still holds content), logs every removal, and recomputes type/subtype - counts afterward so totals stop counting ghosts. - -- **Reads no longer hang behind an unrelated index migration (family-scoped gate).** During a native - provider's one-time background migration, *every* read — including plain `get()`, VFS - `readdir`/`readFile`, and metadata-only `find({ where })` — blocked on the whole-brain migration - lock until timeout, even when the migrating index was irrelevant to the read. The gate is now - scoped to the index families a read actually consults: canonical reads (`get`, `batchGet`, VFS - content) never wait; a `find` waits only on the families its query shape needs (vector for - semantic, metadata for `where`/type, graph for `connected`); graph traversals wait only on the - graph family. Writes and unclassified operations keep the conservative whole-brain wait. A read - that *does* need the migrating family still blocks (bounded by `migrationWaitTimeoutMs`) and - surfaces the retryable `MigrationInProgressError` — never a partial result. - -No breaking API change. Each fix ships with regression tests. - -## v8.3.0 — 2026-07-13 (faster index heals + the cross-layer integrity contract) - -Three additive changes. The first is an immediate, standalone performance win; the other two are the -brainy side of the write/index-spine integrity contract, inert until a native accelerator -that implements the matching hooks is present — so this release changes nothing for a JS-only brain -beyond the speedup. - -- **Canonical enumeration is up to ~16× faster — the dominant term in an index heal.** The paginated - entity walk (`getNounsWithPagination`) hydrated each entity's vector + metadata one-at-a-time; since - every index rebuild enumerates canonical storage, that serial per-item latency dominated multi-minute - heals. Hydration is now 16-way bounded-concurrency, with the pagination contract (order, cursor - resume, filters, totalCount) byte-identical to before. New **`getNounIdsWithPagination()`** returns - ids without hydrating anything (zero per-entity reads when unfiltered) for callers that own their own - IO schedule. - -- **Cross-layer integrity — `validateIndexConsistency()` is no longer blind to native providers.** It - only ran the JS metadata index's own check, so a native provider whose manifest/segments/counts had - diverged still read as "healthy". It now feature-detects and aggregates each provider's optional - `validateInvariants()` self-report, names every failing invariant with its numbers, and exposes the - per-provider reports. `repairIndex()` now reconciles native derived state from canonical too - (rebuilding any provider whose failing invariant asks for it). New exported types - `ProviderInvariantReport` / `InvariantResult` / `InvariantHeal`. - -- **Registered-blob families — declared index files are undeletable through the storage layer.** A - provider can declare a derived-index blob *family* (a set of members that are load-bearing together); - once declared, `deleteBinaryBlob` / `removeRawPrefix` refuse to remove a member (new exported - `ProtectedArtifactError`), so a stray in-process sweeper cannot delete a load-bearing index file. The - declaration persists across reopen; `checkDerivedFamiliesPresent()` names any member missing on open. - New optional `StorageAdapter` surface (`registerDerivedFamily` / `unregisterDerivedFamily` / - `listDerivedFamilies` + `DerivedFamilyDeclaration`) and exported `DerivedArtifactMissingError`. - -No breaking API change (all additions are optional/new). Each change ships with regression tests. - -## v8.2.8 — 2026-07-13 (honest index readiness — no more silently-empty queries on a cold index) - -Closes the last of the three spine anti-patterns: the "dishonest readiness proxy," where `size() > 0` -was treated as "this index actually serves queries." On a cold open (fresh boot, restart, crash -recovery) a native index can load its **count** before its **serving structure** — so for a brief -window it has data but cannot answer, and a query returned a silent empty result indistinguishable -from "no such data." Every fix here asks the index whether it can *actually serve*, and if not, -self-heals or fails loudly instead of returning `[]`. - -- **Semantic search no longer returns a silent `[]` on a cold vector index.** A pure semantic - `find({ query })` has no filter, so nothing previously guarded the vector index. A new one-shot - guard verifies the vector index serves a known persisted vector on the first semantic/proximity - search — preferring the provider's honest `isReady()` signal, else a known-vector self-match probe. - It rebuilds from canonical records if the serving structure did not load, and throws the new - **`VectorIndexNotReadyError`** only if a rebuild still cannot serve — never a silent empty result. - -- **Relationship reads fall back to the canonical scan instead of an empty result on a cold graph - index.** `getVerbsBySource`/`getVerbsByTarget` (used by relationship queries and virtual-filesystem - traversal) skipped the fast path only on `isInitialized` — which reads true once the manifest loaded - even if the source→target adjacency did not. They now consult the honest readiness signal and, when - the adjacency is not serving, take the correct-but-slower canonical shard scan. A one-shot self-heal - probe covers providers that expose no readiness signal. - -- **`getIndexStatus()` tells the truth for readiness probes.** It reported `populated: size>0` only, so - a Kubernetes readiness check could route traffic to a brain still warming up. It now folds in the - honest per-index `ready` signal (making `populated` honest), plus the degraded states already - surfaced by `checkHealth()`/`validateIndexConsistency()` (`rebuildFailed`/`rebuildError` and a - `degradedIds` count) — so a probe never reports 200-ready over a known-degraded index. - -This completes the write/index-spine hardening end to end. New export: **`VectorIndexNotReadyError`**; -`getIndexStatus()` gains additive fields (`rebuildFailed`, `rebuildError?`, `degradedIds`, per-index -`ready?`). No breaking API change. Each fix ships with a dedicated regression test. - -## v8.2.7 — 2026-07-13 (loadBinaryBlob fault-propagation — the lockstep completion of 8.2.6) - -Completes the Pass-1 spine hardening. `loadBinaryBlob` (the raw-blob read a native accelerator mmaps -for its index files) previously returned `null` on ANY read error, so a real IO fault -(EIO/EACCES/EMFILE) on a present-but-unreadable index blob masqueraded as "the blob is absent" — -driving a needless full rebuild or an empty read. It now distinguishes genuine absence (ENOENT → -`null`, the documented contract) from a real fault (→ throw), so a transient disk fault surfaces -loudly instead of silently degrading the index. - -This one change was deliberately held out of 8.2.6 and ships now, in lockstep with the native -accelerator release that hardened its two column-store read sites to handle the throw (a faulted -segment read marks the field unavailable and throws a named error, instead of relying on -null-on-error). A consumer on new brainy + an older accelerator was never at risk: 8.2.6 kept the -prior swallow-on-fault behavior for this method until the accelerator was ready. - -No API change. Regression: `tests/unit/storage/blob-save-durability.test.ts` gains the loadBinaryBlob -leg (absent → null; present → bytes; real fault → throws). - -## v8.2.6 — 2026-07-13 (write/index-spine hardening — loud errors, never quiet losses) - -Durability + integrity hardening across the write and index paths. Every fix converts a place that -could *silently* lose data, serve a partial result, or acknowledge a write that did not land into a -loud, observable failure. No breaking API changes; one new exported error. - -- **A durable blob write that stored nothing is no longer acknowledged.** The atomic blob write - (`saveBinaryBlob`, used for vector/graph index segments and the native index files) uses a unique - per-writer temp file, so a rename `ENOENT` can only mean *our* temp vanished before the rename — - the bytes never landed. It previously returned success on that path; it now retries once with a - fresh temp and, if the temp vanishes again, throws instead of acknowledging a write that persisted - nothing. A downstream deployment that mmaps these files no longer finds a "successfully written" - blob missing on the next open. - -- **Single-op history durability is enforced, not assumed.** The asynchronous group-commit flush - that persists single-op generation history previously swallowed a persist failure as a warning - while writes kept succeeding and their history piled up, undurable, in memory. It now tolerates a - transient blip (retry with capped backoff) and, after repeated failures, **refuses further writes** - with the new **`PendingFlushDurabilityError`** rather than promise a durability it cannot deliver. - Live canonical data is untouched; the latch self-heals the moment a flush succeeds. Callers needing - hard per-write durability should keep using `transact()` (or `flush()` after a single-op). - -- **A degraded derived index is surfaced on reads, not served silently.** When a non-fatal index - rebuild fails at open, or a write commits via adopt-forward recovery with an incomplete derived - index, `find()` and `get()` now emit one loud warning per degraded window (reads still return — - canonical is the source of truth), the state folds into `checkHealth()` / - `validateIndexConsistency()`, and `repairIndex()` reconciles and clears it. - -- **`clear()` removes the full derived footprint.** It previously left raw index blobs, the native - id-mapper, and column-index manifests on disk, so a cleared brain could re-read stale native state - on reopen. It now wipes them together as a set. - -- **Counts stay honest across deletes.** A delete decremented the per-type breakdown but not the - scalar total, so the total inflated permanently (and won pagination). Delete now decrements both; - the invariant `total === Σ per-type` holds across any interleaving of add / visibility-flip / - delete and across a reopen. - -- **Index-maintenance and aggregation failures are loud.** A partial LSM/segment load no longer - publishes the manifest's full count as if healthy; an HNSW flush that can't persist a node throws - (`HnswFlushError`) instead of returning a lying count and dropping the node; a corrupt or missing - manifest-listed column segment throws (`ColumnSegmentLoadError`) instead of silently dropping its - entities from every query; and aggregation materialization / state-load failures now warn instead - of vanishing into an empty catch. - -New export: **`PendingFlushDurabilityError`** (with `.cause` and `.failedAttempts`). No other public -API change. Each fix ships with a dedicated regression test. - -## v8.2.5 — 2026-07-12 (honest response when a transaction rollback can't complete) - -Data-integrity fix. When a transaction failed and its rollback then *also* failed to undo a -canonical write (retries exhausted), the old behavior logged, continued, and threw -`TransactionRollbackError` — while the record it couldn't undo stayed durable on disk, and the -transaction even reported its state as `'rolled_back'`. The caller got an error implying the write -was undone; a read-back showed the record. A failed rollback had no truthful response. - -Rollback now tells the truth, with a two-branch contract: - -- **Adopt-forward (safe case).** A single-op write (`add`/`update`/…) whose only damage is a - durably-present record — the write the caller asked for — is **adopted**: its generation is - committed, the write returns success, and a loud warning records that the derived index may be - incomplete for that id until the next rebuild/`repairIndex()`. No error, no double-write; the - record is immediately durable and retrievable by `get()`. -- **Fail loud + quarantine (unsafe case).** A multi-operation batch, or *any* case where a record - was lost (a remove/update whose restore-undo failed), throws the new **`StoreInconsistentError`** - naming every unreconciled record and its disposition (`orphan` vs `loss`), and puts the brain into - **write-quarantine**: reads keep working, but writes are refused until `repairIndex()` reconciles - the derived indexes against canonical storage and lifts the quarantine. The generation counter is - not advanced, and a transaction whose rollback failed is now `'inconsistent'`, never the - `'rolled_back'` lie. - -The decision is made by *observation*, not guesswork: both commit paths already hold byte-identical -before-images, so after a failed rollback the store compares current canonical state to them to -classify exactly which records are orphaned or lost. - -New export: `StoreInconsistentError` (with `.records` and `.cause`) and the `UnreconciledRecord` type. -No other API change; `repairIndex()` gains the quarantine-lift behavior. Regression -(`tests/integration/rollback-trapdoor.test.ts`) injects the exact failure (index add throws → -canonical delete-undo fails) and pins adopt-forward (durable, get-able, not quarantined), fail-loud -(`StoreInconsistentError` + quarantine + reads work + `repairIndex()` lifts it), and the error's -record naming. - -## v8.2.4 — 2026-07-12 (restore can no longer destroy the store it's recovering) - -Recovery-safety fix. `restore()` removed the entire live brain directory and THEN copied the -snapshot in — so any copy failure left the store destroyed with only a partial copy. The sharpest -edge: the copy (`fs.cp`) materialized the holes of sparse mmap blob files, so a snapshot that fits -on disk could balloon and `ENOSPC` mid-copy, and the recovery tool would have just destroyed the -brain it was asked to recover. Reported from a downstream incident's recovery forensics. - -Restore is now **non-destructive and crash-resumable**: - -- The snapshot is copied into a staging area (`_restore_staging/`) **before any live data is - touched**. The copy is **sparse-aware** — all-zero regions are left as holes, so a store of - mostly-hole blobs restores at its true allocated size instead of its apparent size. -- A copy failure (including `ENOSPC`) removes only the half-written staging area and throws; the - live store is left **exactly as it was**. -- Only after the copy succeeds and a completion marker is `fsync`'d does an **atomic per-entry - swap** move the staged data into place — same-filesystem renames that cannot fail for disk space. -- A crash mid-swap is finished **forward** on the next open: startup resumes a committed-but- - incomplete swap, or discards an uncommitted staging area (live data still authoritative). - -No API change — `restore(path, { confirm: true })` is unchanged. `persist()` was already safe -(hard-link snapshot). Regression (`tests/integration/restore-nondestructive.test.ts`): a forced -copy failure leaves live data fully intact, a normal restore round-trips, an interrupted-but- -committed restore completes on reopen, an uncommitted staging area is discarded, and the sparse -copy is byte-identical with allocation far below apparent size. - -## v8.2.3 — 2026-07-12 (a committed transaction is durable on return) - -Durability fix. A `transact()` reported "committed" while its canonical entity writes were still -only in the OS page cache (written via tmp+rename, not yet `fsync`'d), even though the generation -counter and manifest WERE fsync'd. A hard kill (power loss, SIGKILL) in that window could leave the -durable generation counter **ahead of** the persisted entity bytes — so a consumer resuming from the -counter would see "phantom progress": a generation that claims writes the disk never kept. Reported -from a downstream migration's crash-lifecycle forensics. - -`commitTransaction` now runs a **durability barrier**: it records every canonical write and delete -the batch's operations make, then `fsync`s that entire footprint (file contents, the rename -directory entries, and the parent directories of any deletes) **before** advancing the generation -counter and manifest. A committed transaction is therefore durable the moment `transact()` returns — -the counter can never outrun the entity bytes. - -**Durability contract, now explicit.** `transact()` is durable-on-return (above). A **single-op** -write (`add`/`update`/`remove`/`relate`/…) is Model-B group-commit: its live bytes are written but -become durable at the next `flush()` or `close()`, not the instant the call resolves — the counter -is buffered alongside the data, so a crash loses both together (never a torn counter-ahead-of-state -store). Need per-write durability? Use `transact()` (even for one op), or `flush()` after the write. -This deliberately trades single-op fsync latency (a 3-5x write regression) for throughput. - -No API change; no accelerator involvement (the barrier is in the filesystem storage adapter). In-memory -and durable-per-call (cloud object-PUT) adapters treat the barrier as a no-op. Regression: -`tests/integration/transact-durability-barrier.test.ts` proves entity writes fsync in an earlier -batch than the manifest, for single-op and multi-op (add+relate) transactions, and that a -precommit-rejected batch opens no barrier and advances nothing. - -## v8.2.2 — 2026-07-11 (P0: a timed-out transaction now rolls back — no torn state) - -Data-integrity fix. A transaction that exceeded its time budget **mid-flight** (e.g. a bulk -`transact()` on slower hardware crossing the 30s ceiling) threw a timeout error WITHOUT rolling -back the operations it had already applied. The budget check sat outside the per-operation -rollback path, so only per-operation *failures* rolled back — a timeout stranded the partial -writes in canonical storage while the generation was never stamped, leaving torn, generation-less -state. This was caught by a downstream migration that crossed the ceiling on a large batch. - -`Transaction.execute()` now has a **single rollback point**: any error that escapes the operation -loop — an operation failure OR a mid-flight timeout — rolls back every applied operation in -reverse order, then surfaces the original error (a rollback failure supersedes it, loudly, as -before). This restores the invariant the generation-store commit path already depended on: a throw -from execute means the applied operations were undone byte-identically, and the aborted -transaction leaves `generation()` unchanged and storage byte-identical to its pre-transaction -state. - -No API change. Regression pins the reported requirement — after a mid-flight timeout, -storage is byte-identical and the transaction ends in the `rolled_back` terminal state -(`tests/unit/transaction/timeout-rollback.test.ts`), plus the operation-failure and -rollback-failure paths through the same single rollback point. - -**Note on the 30s ceiling itself** (configurable/scaled timeout, batched embedding precompute, -timeout telemetry) — that ergonomics work is tracked separately; this release fixes only the -correctness bug (a timeout must never leave partial state), independent of where the ceiling sits. - -## v8.2.1 — 2026-07-11 (transact forward references work on the native accelerator) - -Parity fix. `transact()` has always promised atomic forward references — `add` an entity and -`relate` to it in one batch — but the transaction planner resolved relationship endpoint ids at -**plan** time, before the batch's adds had applied. Asking the id mapper about an entity that -doesn't exist yet made the accelerator's (correctly strict) native mapper throw in -`EntityIdMapper.getOrAssign`, so `transact([{op:'add', id:X}, {op:'relate', to:X}])` failed on -native deployments while the permissive JS mapper masked the bug — and silently leaked an id -assignment whenever a batch was later rejected by a commit precondition. - -Endpoint resolution is now **lazy** — evaluated when the graph operation executes inside the -commit, after the batch's adds have applied (the same lazy pattern the operation's generation -stamp already used). Fixed across all transact-planned graph operations: relate (including the -bidirectional reverse edge), remove's relationship cascade, and unrelate. Single-operation writes -are unchanged. Also fixed as a byproduct: a rejected batch no longer pollutes the id mapper. - -Regression suite covers the exact reported shape, both-endpoints-in-batch, bidirectional, -add+relate+remove in one batch, and the mapper-cleanliness proof on rejected batches -(`tests/integration/transact-forward-ref-graph.test.ts`). No API changes; no accelerator version -pairing required. - -## v8.2.0 — 2026-07-10 (temporal VFS — file content joins the immutability model) - -Time travel now covers Virtual Filesystem **content**. Previously the temporal model had a hole -exactly where files were concerned: every entity write was an immutable generation, but the -content *bytes* lived under an eager reference-count GC — deleting a file could physically destroy -bytes that in-window history still referenced, while overwriting never released the old content at -all (an unbounded silent leak that only *accidentally* preserved history). A production consumer's -recovery from a bad deploy succeeded only because a stale field happened to hold the good value — -luck, not a guarantee. Both directions are now fixed by making blob reclamation a **history** -decision instead of a **liveness** decision: - -- **Content blobs are retention-protected.** Each blob tracks live references AND history - references (one per persisted generation record that carries its hash). Deleting/overwriting a - file drops only the live reference; bytes are physically reclaimed in exactly one place — - history compaction — once no live reference and no retained generation references the hash. - Pinned views are exempt automatically (compaction already respects pins). Crash ordering is - over-count-only (never under-count), so a crash can leak until the built-in scrub recounts, but - can never reclaim bytes history still needs. Existing stores get a one-time, marker-gated - backfill on open; cross-file content dedup is handled exactly. -- **`vfs.readFile(path, { asOf })`** — the file's exact bytes as of a generation or `Date`, - guaranteed present within the retention window. -- **`vfs.history(path)`** — the file's versions (`{ generation, timestamp, hash, size, - mimeType? }`, ascending). Restore = read the old bytes `asOf` and write them back (a new write; - history is never rewritten). -- **Overwrites now refresh the file entity's `data`/embedding text** — previously semantic search - and the `data` field served the FIRST version's text forever. - -Lifecycle note: deleting a file no longer frees its bytes immediately — old content lives until -compaction reclaims its generations, under the same `retention` budget as all Model-B history -(`retention: 'all'` keeps every version forever). Guide: the new "Time travel for files" section in -`docs/guides/snapshots-and-time-travel.md`. Native accelerator: unaffected (canonical write path -only) — no version pairing required. - -## v8.1.0 — 2026-07-10 (`brain.onChange` — the in-process change feed) - -New public API: subscribe to every committed mutation with -`brain.onChange(cb) → unsubscribe`. One event per affected record, for **every** -canonical write regardless of origin — direct calls, batch methods, `transact()`, -imports, and Virtual Filesystem writes all funnel through the same commit point the -feed is emitted from. This is the authoritative in-process signal for live UIs, -cache invalidation, and realtime sync layers (downstream SDKs can forward it over -their own transports to make local and remote brains uniform). - -Event shape (`BrainyChangeEvent`, exported): `kind` (`entity`/`relation`/`store`), -`op` (`add`/`update`/`remove`/`relate`/`unrelate`/`updateRelation`/`clear`/`restore`), -the post-commit `entity` or `relation` view (type + full custom metadata), the -committed `generation`, and `timestamp`. Notable properties: - -- **Post-commit only** — an aborted write (a losing `ifRev` CAS, a rejected - transaction) never emits. If you got the event, the write is durable. -- **Deletes fully described** — `remove`/`unrelate` carry the record's last - committed state (from the commit's own history record), not just an id; batch - deletes included. -- **Batches emit per item**; a `transact()` batch's events share its single - generation. Entity deletes also emit `unrelate` for each cascaded relationship. -- **Commit-ordered, asynchronous, isolated** — events arrive in commit order, - dispatched in a microtask so a slow listener never delays a write and a throwing - listener never affects the write or other listeners. Zero overhead with no - subscribers. -- **`kind: 'store'`** events for `clear()`/`restore()` mean "refetch everything". -- Fire-and-forget by design; each event's `generation` composes with - `asOf()`/`transactionLog()` for catch-up after a gap. - -Guide: `docs/guides/reacting-to-changes.md`. No behavior changes for existing code. - -## v8.0.17 — 2026-07-08 (count recovery scans the real layout · ~1,100 lines of dead 7.x machinery removed) - -A cleanup of the vestigial 7.x "hnsw sharding" machinery that turned up two real fixes. - -**1 — Count recovery now scans the layout the database actually uses.** When `counts.json` is -lost or corrupted (container restarts, partial copies), the recovery scan rebuilt the entity/ -relationship counters by counting files in `entities/*/hnsw/` — directories the 8.0 write path -never populates — so an established store recovered to **zero counts** on real data: wrong -`getNounCount()`/stats, a "New installation"-style boot log, and a mis-sized rebuild-strategy -decision at open. The scan now counts the canonical `entities////` tree. -Regression-tested: delete `counts.json` from a populated store → reopen → exact counts. - -**2 — A latent init-time deadlock removed.** The recovery scan's type-distribution sampler called -a guarded accessor (`getNounMetadata` → `ensureInitialized`) from **inside** `init()`, which -re-enters `init()` and hangs the open. It was unreachable before only because the old scan never -found anything to sample; the fix reads the sampled metadata files directly. - -**3 — The dead machinery itself is gone (−1,138 lines).** Twenty-three methods with zero callers: -the 7.x hnsw-layout entity/edge CRUD, the sharding-depth prober + depth-migration engine, and an -orphaned streaming paginator. Verified by reachability analysis before deletion; no public API -touched; the full suite is green. New stores no longer carry the always-empty legacy directories' -scan cost, and the boot log keeps the truthful new-vs-established wording from v8.0.13. - -## v8.0.16 — 2026-07-08 (concurrency fast-follow: atomic `ifAbsent`/`upsert` + exact blob reference counts) - -Closes the two remaining check-then-act races found in the sweep that followed v8.0.15's CAS fix -(disclosed in that release's notes). Same bug class — a check performed before the serialization -point that guards the apply — same cure. - -**1 — `add({ ifAbsent })` and `add({ upsert })` are now atomic.** The absence check ran before the -commit mutex, so N concurrent same-id creates could all pass it and all write — the second -silently overwriting the first, violating ifAbsent's "returns the existing id **without writing**" -contract and upsert's "merge, never clobber" contract. The insert leg now carries a must-be-absent -precondition (the same conditional-commit primitive as `ifRev`), verified under the commit mutex: -exactly one concurrent create wins; every other caller takes its documented resolution — ifAbsent -returns the existing id with zero writes, upsert merges into the now-existing entity (with a -bounded retry if a concurrent delete intervenes). Verified: 8 concurrent `ifAbsent` creates -advance the store by exactly ONE generation; 8 concurrent upserts on an absent id produce one -create + seven merges (`_rev` lands at exactly 8). Note: `transact()`'s batch-level -ifAbsent/upsert keep planning-time semantics (the batch converges to a valid entity; the window is -documented, not silent). - -**2 — Blob reference counts are exact under concurrency.** The content-addressed blob store's -`write()` (dedup: exists → add a reference, absent → create) and `delete()` (decrement → remove at -zero) were unserialized read-modify-writes over the blob's metadata. Concurrent writes of -identical content could lose references — making a later delete remove bytes **another file still -referenced** (data loss), or leak unreferenced blobs. All reference-count-bearing mutations are -now serialized per content hash (distinct content never contends): N concurrent identical writes -yield exactly N references, and a blob is physically removed only when the true last reference -drops. Verified with concurrent write/delete storms. - -Both fixes are in-process complete by construction — storage enforces single-writer-per-directory, -so the process is the whole concurrency domain. No API changes. - -## v8.0.15 — 2026-07-08 (`ifRev` CAS is now atomic — exactly one winner under concurrency) - -Correctness fix for optimistic concurrency, reported from a production cutover rehearsal. N -**concurrent** `update({ ifRev })` calls carrying the same expected revision ALL succeeded — zero -`RevisionConflictError`s, last-writer-wins, the other N−1 writes silently lost. (Sequential calls -conflicted correctly.) The revision check ran before the commit mutex, so interleaved callers all -passed it before any apply landed — breaking the exactly-one-winner semantics the -[optimistic-concurrency guide](docs/guides/optimistic-concurrency.md) promises, which advisory -locks and per-entity ledgers/counters build on. - -The fix makes the check-and-apply a **conditional commit**: the generation store's commit paths -accept a precondition that runs under the commit mutex against the just-read authoritative -before-images — the per-record analogue of `ifAtGeneration`, which always ran there. `update()` -and `transact()` per-op `ifRev` both re-verify at that point (the earlier check remains as a cheap -fast-fail). A conflict aborts atomically: nothing staged, nothing applied, the same -`RevisionConflictError` as before. Two adjacent behaviors also became honest: - -- **`_rev` is now monotonic under concurrency.** The winner's stamp derives from the - authoritative before-image, so N concurrent plain updates advance `_rev` by N (previously they - could all stamp the same stale value). -- **CAS against a concurrently-deleted entity** now throws `EntityNotFoundError` instead of - silently resurrecting it (plain updates keep their last-writer-wins re-create semantics). - -Verified with a concurrency regression suite: 8 parallel same-rev updates → exactly 1 winner + 7 -conflicts; the documented read→CAS→retry ledger loop converges exactly (8 workers, 8 decrements, -0 lost) — `tests/integration/ifrev-concurrent-cas.test.ts`. If you serialized writes in your own -adapter as a workaround, it can come out after this upgrade. - ## v8.0.14 — 2026-07-07 (7→8 migration preserves branch-scoped non-entity state instead of deleting it) Defense-in-depth for the one-time 7→8 layout migration. The migration rescues the head branch's diff --git a/docs/PLUGINS.md b/docs/PLUGINS.md index d9a4d3e7..0a553d46 100644 --- a/docs/PLUGINS.md +++ b/docs/PLUGINS.md @@ -149,17 +149,6 @@ context.registerProvider('embedBatch', async (texts: string[]) => { ### Index Providers -> **Write-path invariant (the change-feed contract).** Every canonical -> mutation flows through Brainy's generation-store commit points — index -> providers are invoked *inside* that commit and never originate canonical -> writes of their own. The `brain.onChange` change feed is emitted from those -> commit points and relies on this: **a plugin must never introduce a write -> path that bypasses the generation-store commit.** If a future provider ever -> needs a direct native ingest path, it must either route through the commit -> or emit equivalent change events — otherwise every `onChange` consumer -> (live UIs, cache invalidation, realtime sync) silently develops a blind -> spot. - #### `vector` **Type:** `(config: object, distanceFunction: Function, options: object) => VectorIndexProvider-compatible` diff --git a/docs/concepts/generation-fact-log.md b/docs/concepts/generation-fact-log.md deleted file mode 100644 index f9b3e974..00000000 --- a/docs/concepts/generation-fact-log.md +++ /dev/null @@ -1,117 +0,0 @@ ---- -title: The Generation Fact Log -slug: concepts/generation-fact-log -public: true -category: concepts -template: concept -order: 6 -description: Every committed write also appends a self-verifying "fact" — an after-image commit record — to an append-only log. What facts are, the crash-safety model, the scanFacts() streaming surface, family stamps, and how index providers consume the log for sequential heals. -next: - - concepts/consistency-model - - guides/snapshots-and-time-travel ---- - -# The Generation Fact Log - -Since 8.4.0, every committed generation also appends a **fact** — a compact record of what each -touched entity or relationship *became* — to an append-only, checksummed log under -`_generations/facts/`. Where the generational history answers *"what did things look like -before?"* (before-images, powering `asOf()` and rollback), the fact log answers *"what happened, -in order?"* — one sequential, self-verifying stream of the store's present being written. - -Nothing about querying changes. The fact log exists for three consumers: - -1. **Index heals and rebuilds** — one sequential read in commit order replaces a per-entity - directory walk over millions of files. -2. **Incremental catch-up** — a derived index that knows which generation it reflects reads *just - the gap*, instead of rebuilding from scratch. -3. **Replay and audit tooling** — anything that wants the store's committed timeline as a stream. - -## What a fact is - -One fact per committed generation: - -- **`generation`** and **`timestamp`** — which commit, when. -- **`ops`** — every write in that commit: `{ kind: 'noun' | 'verb', id, record }` where `record` - holds the entity's full after-image (both stored legs), or **`null` for a tombstone** — a - removal carries no body, by design. -- **`meta`** — the transaction metadata `transact()` was submitted with, when present. -- **`blobHashes`** — content-blob references, for exact reclamation accounting. - -Facts accumulate **from the first write after upgrading** — pre-existing history is not -retroactively converted, and consumers fall back to the enumeration walk when no log exists. - -## Crash safety, in one paragraph - -Facts are appended and fsynced **inside the same durability window as the commit itself**, before -the commit point — so after a crash, the log can only ever be *ahead* of committed truth, never -behind it with a hole. On open, the store reconciles the log back to the committed watermark: -torn tails are detected by per-record checksums and cut; whole records beyond the watermark are -truncated. The invariant every reader can rely on: **an absent generation was never committed; a -present fact was.** `transact()` facts are durable the moment `transact()` returns; single-op -facts share the same group-commit flush as the rest of their generation, so a hard kill loses the -fact and the generation *together* — never a torn state. - -## Reading the log - -```typescript -const scan = brain.scanFacts({ fromGeneration: 1 }) -if (scan) { - // Telemetry up front — progress bars get a denominator from second zero. - console.log(scan.headGeneration, scan.segmentCount, scan.approxFactCount) - - for await (const batch of scan.batches()) { - // Each batch: { facts, firstGeneration, lastGeneration, factCount, byteSize, segmentId } - for (const fact of batch.facts) { - for (const op of fact.ops) { - if (op.record === null) { - // a tombstone: op.id was removed in this generation - } - } - } - } - - console.log(scan.summary()) // { factsYielded, segmentsRead } — the cross-check -} -``` - -- `scanFacts()` returns `null` when the store hosts no fact log (older store, or a storage adapter - without binary append support) — fall back to enumerating entities. -- Scans run against a **snapshot**: facts appended after the scan opens never bleed in, each fact - is yielded exactly once, and a detected gap aborts loudly — never a silent skip. -- `brain.factSegmentPaths()` returns the immutable, *sealed* segment files for zero-copy consumers - (the append-mutable tail is excluded — read it through `scanFacts()`). - -## Family stamps: how a projection proves it's current - -Anything derived from the store — an index, the entity file tree itself — carries a **family -stamp**: a small JSON record of *which committed generation the projection reflects* -(`sourceGeneration`) plus the invariants that verify it whole (exact per-file byte sizes for -bounded families; rollup invariants like entity counts for unbounded ones). At open, coherence is -a **comparison**, not a walk: - -- stamp equals the committed watermark and invariants hold → serve; -- stamp is behind → the projection reads just the gap from the fact log; -- invariants fail → loud, named divergence — `brain.repairIndex()` rebuilds from canonical and - re-stamps. - -The verifier is exported (`verifyFamilyStamp`) so every projection — TypeScript or native — runs -literally the same check. - -## For plugin authors: the storage capability - -Index providers receive the storage adapter, not the brain — so the host wires the log onto it. -Feature-detect and prefer the stream; fall back to enumeration: - -```typescript -const scan = storage.scanFacts?.({ fromGeneration: stamp.sourceGeneration + 1 }) -if (scan) { - // sequential catch-up from the log -} else { - // enumeration walk (older store or adapter) -} -const committed = storage.committedGeneration?.() // the watermark stamps compare against -``` - -Providers must never construct their own reader over the log's files — the open path belongs to -the single writer (it reconciles the log at open); the capability is the sanctioned seam. diff --git a/docs/guides/aggregation.md b/docs/guides/aggregation.md index 11d86ec8..e58eb2ce 100644 --- a/docs/guides/aggregation.md +++ b/docs/guides/aggregation.md @@ -11,13 +11,7 @@ No batch jobs. No scheduled recalculations. Aggregates stay current with every w **Defining over existing data:** if you define an aggregate on a store that already holds matching entities, Brainy backfills it from those entities on the first query (a one-time scan, then purely incremental). So `defineAggregate()` behaves the same whether you define it before -or after the data exists. - -**Reopening a persisted brain:** aggregate state persists across restarts. Re-defining the -same aggregate at boot (the normal declarative pattern) adopts the persisted state directly — -no rescan. A backfill scan runs only when the definition actually changed, when no persisted -state exists, or when the state failed to load; and however many aggregates need backfilling, -they share a single scan. +or after the data exists — including when a persisted brain reopens already populated. ## Quick Start diff --git a/docs/guides/external-backups-and-sparse-storage.md b/docs/guides/external-backups-and-sparse-storage.md deleted file mode 100644 index f28dcfd7..00000000 --- a/docs/guides/external-backups-and-sparse-storage.md +++ /dev/null @@ -1,99 +0,0 @@ ---- -title: External Backups & Sparse Storage -slug: guides/external-backups -public: true -category: guides -template: guide -order: 10 -description: How to back up a brain directory with external tools (tar, rsync, cp) without exploding sparse files — why a store can show 100+ GB "apparent" size on a small disk, which files are sparse, and how persist()/restore() handle it for you. -next: - - guides/snapshots-and-time-travel - - concepts/storage-adapters ---- - -# External Backups & Sparse Storage - -The built-in snapshot path — [`db.persist()` and `brain.restore()`](/docs/guides/snapshots-and-time-travel) — -already handles everything on this page for you. Read this when you back up a brain directory with -**external tools**: `tar`, `rsync`, `cp`, `scp`, or a filesystem-level backup agent. - -## The one-sentence rule - -> **Always use the sparse-aware flag**: `tar czSf` (capital `S`), `rsync --sparse`, -> `cp --sparse=always`. A naive copy can turn a 2 GB store into a 100+ GB one — or fail -> the disk entirely. - -## Why: some files are sparse - -When a native accelerator plugin is active, parts of the index live in **memory-mapped files** -created at a large fixed virtual size — the file's *apparent* size — while the filesystem only -allocates blocks that were actually written. A brand-new id-mapper file can report tens of -gigabytes in `ls -l` while occupying a few megabytes on disk. - -Check the difference yourself: - -```bash -ls -lh brain-data/_id_mapper/ # APPARENT size (can be huge) -du -sh brain-data/ # ALLOCATED size (the real footprint) -``` - -The sparse candidates in a brain directory: - -| Path | What it is | -|---|---| -| `_id_mapper/` | The native id-mapper's mmap files (large fixed virtual size) | -| `_blobs/` | Native index files (vector base, segments) — may be mmap-backed | - -Everything else (entities, `_system`, `_generations`, `_cas` content blobs) is ordinary dense data. - -## Doing it right - -**tar** — the `S` flag detects holes and stores only real data: - -```bash -tar czSf brain-backup.tgz /data/brain -# restore preserves the holes: -tar xzSf brain-backup.tgz -C /data/ -``` - -**rsync**: - -```bash -rsync -a --sparse /data/brain/ backup-host:/backups/brain/ -``` - -**cp**: - -```bash -cp -a --sparse=always /data/brain /backups/brain -``` - -**What goes wrong without the flag:** the copy *materializes* every hole as real zero bytes. -A store whose apparent size exceeds the target disk fails with `ENOSPC` partway through — and a -copy that *does* fit silently costs the full apparent size in storage and transfer time. - -## What the built-in paths do (so you don't have to) - -- **`db.persist(path)`** snapshots via **hard links** — instant and space-shared, since every data - file is immutable-by-rename. The handful of append-in-place files (the transaction log, the - commit fact log's tail segment) and mmap-mutated directories (`_id_mapper/`) are **byte-copied** - instead, so a post-snapshot write can never reach through a shared inode into your backup. -- **`brain.restore(path, { confirm: true })`** is **non-destructive and sparse-aware**: the snapshot - is copied into a staging area *before* any live data is touched (all-zero blocks stay holes), and - only after the copy fully succeeds does an atomic swap move it into place. A failed copy — - including `ENOSPC` — leaves the live store exactly as it was. A crash mid-swap completes forward - on the next open. - -## Live-store caveats for external tools - -1. **Prefer snapshotting a `persist()` output, not the live directory.** `persist()` produces a - crash-consistent, immutable snapshot; running `tar` against a live, actively-written directory - can capture a torn mid-write state. If you must archive live, stop writes first (or accept that - the archive is only as consistent as the moment's flush state). -2. **Never prune or "clean up" files inside a brain directory.** Index files that look stale or - redundant are load-bearing; the store protects its declared index families from in-process - deletion, but an external `rm` bypasses that fence. If space is the concern, `du -sh` first — - the allocated size is usually far smaller than it looks. -3. **Verify restores by opening them.** `Brainy.load(path)` opens any snapshot or restored - directory read-only — the store verifies its own coherence at open and reports loudly if - anything is missing or torn. diff --git a/docs/guides/find-limits.md b/docs/guides/find-limits.md index 4c7fd252..66418b80 100644 --- a/docs/guides/find-limits.md +++ b/docs/guides/find-limits.md @@ -40,10 +40,6 @@ Brainy picks `maxLimit` from the first of these that's available: Worked example: a 4 GB Cloud Run container picks priority 3 → `floor(4 GB × 0.25 / 25 KB) = floor(40 960) = 40 000` results. A 900 MB free-memory box on priority 4 gets `floor(900 MB / 25 KB) = ~36 000`. -The cap is fixed at construction and never changes at runtime. Query timing is recorded -for diagnostics only — a burst of slow queries cannot silently shrink the cap, and the -auto-detected tiers (3 and 4) never go below a floor of 10 000. - > **Calibration note.** Pre-7.30.2 used 100 KB per result instead of 25 KB, which produced caps that were 4× too tight for typical workloads (an 8 KB / result reality). 7.30.2 recalibrated to match observed entity sizes; existing `limit: 10_000` safety patterns now pass silently on any reasonably-sized box. ## What happens when you exceed the cap diff --git a/docs/guides/import-anything.md b/docs/guides/import-anything.md index b1bb15ef..e0cd94ed 100644 --- a/docs/guides/import-anything.md +++ b/docs/guides/import-anything.md @@ -300,10 +300,7 @@ await brain.import(data, { // Deduplication enableDeduplication: true, // Check for duplicate entities (default: true) deduplicationThreshold: 0.85, // Similarity threshold for duplicates (0-1, default: 0.85) - // Notes: false disables BOTH the inline merge and the background pass that - // runs ~5 min after the last import (merged duplicates are deleted). - // The inline pass auto-disables for imports >100 entities (O(n²) cost); - // the background pass still covers those unless the flag is false. + // Note: Auto-disabled for imports >100 entities // Performance chunkSize: 100, // Batch size for processing (default: varies by operation) diff --git a/docs/guides/import-quick-reference.md b/docs/guides/import-quick-reference.md index 7837d49e..5a850a86 100644 --- a/docs/guides/import-quick-reference.md +++ b/docs/guides/import-quick-reference.md @@ -86,22 +86,11 @@ await brain.import(file, { ```typescript await brain.import(file, { - enableDeduplication: true, // Check for duplicates (default: true) + enableDeduplication: true, // Check for duplicates (default: false) deduplicationThreshold: 0.85 // Similarity threshold (default: 0.85) }) ``` -Deduplication merges entities judged duplicates — the non-primary records are -**deleted**. Set `enableDeduplication: false` to disable it entirely: the flag -gates both the inline merge during import and the background pass that runs -about 5 minutes after the last import. - -```typescript -await brain.import(file, { - enableDeduplication: false // No merging, inline or background -}) -``` - ### Import Tracking Track and organize imports by project: diff --git a/docs/guides/inspection.md b/docs/guides/inspection.md index 240e81ae..015c5118 100644 --- a/docs/guides/inspection.md +++ b/docs/guides/inspection.md @@ -166,34 +166,6 @@ brainy inspect diff /data/brain-prod /data/brain-staging Sample-based — for a full diff, dump both with `inspect dump` and compare the JSONL. -## Auditing graph-read truth - -`brain.auditGraph()` (8.6.0+) proves — or disproves — that relationship reads -return canonical truth on a given brain, without mutating anything. It walks -every stored relationship record, asks the same read path your application -uses (`related()`, VFS `readdir`) with every visibility tier included, and -classifies every discrepancy: - -```typescript -const report = await brain.auditGraph() - -report.coherent // true = related()/readdir can be trusted on this brain -report.missingFromReadsCount // records the read path omits — stale index -report.danglingEndpointsCount // relationships whose endpoint entity is gone -report.readOnlyCount // read-path edges with NO stored record — ghosts -report.visibilityHiddenCount // internal/system edges hidden by design (not a fault) -``` - -Counts are always exact; the example lists (`missingFromReads`, -`danglingEndpoints`, `readOnlyVerbIds`) are capped at `maxExamples` -(default 100) and `truncatedExamples` says so when they are. - -Run it after any engine upgrade, restore, or migration. If it reports -discrepancies, run `brain.repairIndex()` and audit again — a `coherent` -report after the repair is the verified statement that the heal worked. -Cost: one relationship-record walk plus one indexed read per distinct -source entity — safe on a live brain. - ## Repairing a corrupted store If invariants fail and you suspect index corruption, `inspect repair` diff --git a/docs/guides/optimistic-concurrency.md b/docs/guides/optimistic-concurrency.md index 268bc5fa..a21a8af0 100644 --- a/docs/guides/optimistic-concurrency.md +++ b/docs/guides/optimistic-concurrency.md @@ -180,35 +180,3 @@ Brainy 8.0 has exactly two write-coordination counters, at two granularities: They compose: a `transact()` batch can carry per-entity `ifRev` checks *and* a whole-store `ifAtGeneration`; any failed check rejects the entire batch before anything is staged. Generations also power snapshots and time travel (`brain.now()`, `brain.asOf()`, `db.persist()`) — see the [consistency model](../concepts/consistency-model.md) and [Snapshots & Time Travel](./snapshots-and-time-travel.md). A snapshot or historical view captures each entity *including* its `_rev` at that moment, so reading the past and writing back with `ifRev` against the live state works exactly as you'd hope: the write fails if the entity moved since the state you copied from. - -## The transact envelope: batch size, budget, and bulk imports - -`transact()` applies its batch atomically under one commit — which means the whole batch -shares one **apply budget**. Since 8.7.0 the budget scales with the batch: -`max(30 s, opCount × 2 s)`, or exactly what you pass as `timeoutMs`. A tripped budget rolls -the entire batch back (nothing partial survives) and throws a retryable -`TransactionTimeoutError` that names the operation it stopped at, the batch size, and the -elapsed vs budgeted time — a diagnosis, not just a failure: - -``` -Transaction timed out at operation 41/120 ('add') — 246012ms elapsed, budget 240000ms. -The batch rolled back atomically; retry with a higher timeoutMs or a smaller batch. -``` - -Practical envelope guidance for bulk work: - -1. **Precompute embeddings outside the commit path.** Embedding inside `transact()` spends - the budget on model inference. Use `brain.embedBatch(texts)` and pass each vector via - the op's `vector` field — the commit then pays only storage costs, and a retried batch - never re-pays inference. (The win is *where* the inference happens, not raw embedding - throughput: on the default WASM engine, batch and sequential embedding measure - comparably, ~160 ms/text; native embedding providers may batch faster.) -2. **Chunk very large imports** into batches of a few hundred ops with one `transact()` - each. You lose whole-import atomicity but keep per-chunk atomicity, bounded memory, and - resumability — pair with `ifAbsent` upserts so a retried chunk is idempotent. -3. **Slow disks change the math, not the contract.** On network-attached storage a single - op can cost ~2 s (canonical write + fsync + index maintenance). The scaled default - absorbs that; pass an explicit `timeoutMs` only when you know better than the scale. -4. **`addMany`/`relateMany` are the convenience tier** — they chunk and batch-embed for - you, with per-item error reporting instead of batch atomicity. Choose by what you need: - atomic-all-or-nothing → `transact()`; resilient bulk load → `addMany`. diff --git a/docs/guides/reacting-to-changes.md b/docs/guides/reacting-to-changes.md deleted file mode 100644 index 7367a5e3..00000000 --- a/docs/guides/reacting-to-changes.md +++ /dev/null @@ -1,117 +0,0 @@ ---- -title: Reacting to Changes -slug: guides/reacting-to-changes -public: true -category: guides -template: guide -order: 11 -description: Subscribe to every committed mutation with `brain.onChange` — the in-process change feed behind live UIs, cache invalidation, and realtime sync. Covers the event shape, delivery guarantees, and catch-up patterns. -next: - - guides/optimistic-concurrency - - guides/snapshots-and-time-travel ---- - -# Reacting to Changes - -`brain.onChange(cb)` is Brainy's in-process change feed: subscribe once and -receive one event per committed mutation — **every** mutation, regardless of -how it happened. Direct calls, batch methods, `transact()`, imports, and -Virtual Filesystem writes all funnel through the same commit point the feed is -emitted from, so nothing slips past it. - -```ts -const off = brain.onChange((e) => { - if (e.kind === 'entity') { - console.log(`${e.op} ${e.entity?.type} ${e.id} @ generation ${e.generation}`) - } -}) - -await brain.add({ data: 'Ada Lovelace', type: 'person' }) -// → "add person 0198... @ generation 42" - -off() // unsubscribe when done -``` - -## The event - -```ts -interface BrainyChangeEvent { - kind: 'entity' | 'relation' | 'store' - op: 'add' | 'update' | 'remove' | 'relate' | 'unrelate' | 'updateRelation' - | 'clear' | 'restore' - id?: string - entity?: { id: string; type: string; subtype?: string; - metadata: Record; service?: string } - relation?: { id: string; from: string; to: string; type: string; - metadata?: Record } - generation?: number - timestamp: number -} -``` - -- **Entity events** (`add` / `update` / `remove`) carry the post-commit indexed - view — `type`, `subtype`, and the full custom `metadata`, so you can match - your own `where`-style filters against events without a read. -- **Deletes are fully described.** A `remove` or `unrelate` event carries the - record's *last committed state* (sourced from the commit's own history - record), not just an id. -- **Batches emit per item.** `addMany` / `updateMany` / `relateMany` / - `removeMany` emit one event per affected record; a `transact()` batch emits - one event per item, all sharing the batch's single `generation`. -- **Cascades are visible.** Removing an entity also emits `unrelate` for each - relationship the delete cascaded to. -- **Store-level events** (`kind: 'store'`) fire for the two wholesale - operations — `clear()` and `restore()` — and mean *"everything may have - changed; refetch what you care about."* - -## Delivery guarantees - -- **Post-commit only.** An aborted write — a losing - [`ifRev` compare-and-swap](optimistic-concurrency.md), a rejected - transaction — never emits. If you received the event, the write is durable. -- **Commit-ordered.** Events arrive in the order writes committed; - `generation` is monotonic. -- **Asynchronous, never blocking.** Delivery happens in a microtask after the - write completes. A slow listener cannot delay a write; a throwing listener - is logged and isolated from other listeners. -- **Zero overhead when unused.** With no subscribers, the write path does no - event work at all. -- **Fire-and-forget.** There is no replay or backpressure. For catch-up after - a disconnect, use the `generation` on each event together with - [`asOf()` / the transaction log](snapshots-and-time-travel.md): record the - last generation you processed, and on reconnect diff from there. For file - content specifically, `vfs.readFile(path, { asOf })` and - `vfs.history(path)` are the temporal read — see - [Snapshots & Time Travel](snapshots-and-time-travel.md). - -## Patterns - -**Cache invalidation** — drop cached reads for whatever changed: - -```ts -brain.onChange((e) => { - if (e.kind === 'store') return cache.clear() - if (e.id) cache.delete(e.id) -}) -``` - -**Live queries (notify-and-refetch)** — re-run a query when a relevant change -lands, rather than diffing incrementally: - -```ts -brain.onChange((e) => { - if (e.kind === 'entity' && e.entity?.type === 'order') { - refreshOpenOrdersView() // debounce as needed - } -}) -``` - -**Forwarding to other processes** — the feed is in-process by design. To push -changes to browsers or other services, forward events through your own -transport (WebSocket, SSE) from the process that owns the brain. - -## Lifecycle - -`onChange` returns an unsubscribe function — call it when tearing down a -subscriber (for example, when evicting a pooled instance). `brain.close()` -drops all listeners; no events are delivered for or after `close()`. diff --git a/docs/guides/snapshots-and-time-travel.md b/docs/guides/snapshots-and-time-travel.md index 490aecab..d9f0e264 100644 --- a/docs/guides/snapshots-and-time-travel.md +++ b/docs/guides/snapshots-and-time-travel.md @@ -9,7 +9,6 @@ description: Recipes for the Db API — instant backups with persist(), restore, next: - concepts/consistency-model - guides/optimistic-concurrency - - guides/external-backups --- # Snapshots & Time Travel @@ -42,10 +41,6 @@ bytes. Cross-device targets fall back to per-file byte copies, and persisting an in-memory brain serializes it to the same directory layout — a real, durable store. -> Archiving a brain directory with **external tools** (`tar`, `rsync`, `cp`)? -> Some index files are sparse and can explode to their apparent size under a -> naive copy — see [External Backups & Sparse Storage](/docs/guides/external-backups). - Two things to know: - `persist()` requires the view to still be the store's **latest** @@ -344,12 +339,8 @@ For per-entity write coordination (rather than whole-store history), the ## Keeping history bounded Under Model-B every write is a generation, so history can grow quickly — -Brainy auto-compacts at `close()` (time-bounded per pass) under the -**`retention`** knob (configured on the constructor). Since 8.9.0, `flush()` -never compacts: flushing is durability work and costs only what the current -window's writes cost, regardless of history backlog. A long-lived writer that -never closes keeps its history until its next explicit `compactHistory()` — -schedule one in your maintenance window if you run bounded retention: +Brainy auto-compacts on every `flush()`/`close()` under the **`retention`** +knob (configured on the constructor): ```typescript // Zero-config: ADAPTIVE — keep as much history as free disk/RAM allows, @@ -363,13 +354,10 @@ new Brainy({ retention: 'all' }) new Brainy({ retention: { maxGenerations: 1000, maxAge: 7 * 86_400_000, maxBytes: 512 * 1024 ** 2 } }) ``` -Reclaim manually at any time (the same caps, plus an optional per-pass time -budget for maintenance windows — an early stop is a consistent prefix and the -next pass resumes): +Reclaim manually at any time (the same caps): ```typescript await brain.compactHistory({ maxGenerations: 100, maxAge: 7 * 24 * 60 * 60 * 1000 }) -await brain.compactHistory({ maxBytes: 512 * 1024 ** 2, timeBudgetMs: 10_000 }) ``` Compaction never breaks a pinned read — record-sets are reclaimed only when @@ -378,61 +366,6 @@ are done with (including the ones `transact()` returns), and `persist()` any generation you want to keep beyond the retention window: snapshots are self-contained and unaffected by compaction. -## Time travel for files (the VFS) - -Since 8.2.0, time travel covers Virtual Filesystem **content**, not just -entity records. File bytes are retention-protected: a content blob referenced -by any generation inside the retention window is never reclaimed, so reading -the past always returns the exact bytes — never a stale field or a -dangling hash. - -**`vfs.readFile(path, { asOf })`** takes a generation number or a `Date` and -returns the file's exact bytes as they stood then. It resolves the path's -current entity, then materializes its state at the target generation — so it -answers *"what did the file at this path hold at that point?"* It bypasses -the content cache; the `encoding` option still applies. Asking about a -generation before the file existed throws the usual not-found error, and -asking past the retention window's compaction horizon throws a -compacted-generation error. - -**`vfs.history(path)`** returns the file's versions inside the retention -window, oldest first — one `FileVersion` per generation that wrote the file, -the newest entry being the current state: - -```typescript -// A CMS page evolves… -await brain.vfs.writeFile('/pages/home.json', '{"title":"Launch"}') -await brain.vfs.writeFile('/pages/home.json', '{"title":"Launch v2"}') -await brain.vfs.writeFile('/pages/home.json', '{"title":""}') // bad deploy! - -// Every version is listed and readable: -const versions = await brain.vfs.history('/pages/home.json') -// → [{ generation, timestamp, hash, size, mimeType? }, …] ascending - -const good = versions[versions.length - 2] -const bytes = await brain.vfs.readFile('/pages/home.json', { - asOf: good.generation -}) - -// Restore = write the old bytes back. This is a NEW write (a new -// generation) — history is never rewritten, so the bad version stays -// visible in the audit trail. -await brain.vfs.writeFile('/pages/home.json', bytes) -``` - -Two lifecycle consequences worth stating plainly: - -- **Deleting or overwriting a file no longer frees its bytes immediately.** - Old content lives until history compaction reclaims the generations that - reference it — the same `retention` budget that bounds all Model-B history - (and pinned views are exempt, exactly as above). Size your `retention` for - the file-version depth you want; `retention: 'all'` keeps every version of - every file forever. -- **After `compactHistory()` reclaims a generation, its file versions are - gone** and their bytes are physically reclaimed. (This also fixed a - pre-8.2.0 defect where overwritten content was never reclaimed at all — an - unbounded silent leak.) - ## From branches to values If you used the pre-8.0 `fork`/`checkout`/`commit`/`versions` surface, every diff --git a/docs/performance-envelopes.md b/docs/performance-envelopes.md deleted file mode 100644 index d29677e3..00000000 --- a/docs/performance-envelopes.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: Performance Envelopes -slug: guides/performance-envelopes -public: true -category: guides -template: guide -order: 40 -description: Measured per-operation latency envelopes at stated scales — what to expect, on what hardware, and exactly how each number was produced. -next: - - guides/find-limits ---- - -# Performance Envelopes - -Every number on this page is **measured, never projected** — produced by the script -cited at the bottom, against the built package (the artifact you install), on the stated -hardware. Each entry says what was measured, at what scale, on which storage backend. -When a release touches a measured path, that operation is re-measured and this page -updates in the same release. - -Two scopes to keep straight: - -- **These envelopes are the pure-JS engine** (no native accelerator registered) on - filesystem storage. This is the floor every deployment gets from `npm install` alone. -- **Accelerated deployments** (the optional native provider) publish their own numbers — - this page never claims them. - -## Read operations - -Reads are where the architecture pays off: after the write path has done its indexing -work, queries answer from purpose-built indexes without scanning. - -| Operation | 1,000 entities | 10,000 entities | Notes | -|---|---|---|---| -| `get(id)` (warm) | p50 < 0.1ms | p50 < 0.1ms | served from cache/metadata index | -| `find` (metadata: indexed equality + range, limit 100) | p50 1.0ms · p95 1.8ms | p50 7.0ms · p95 8.9ms | column-store bitmap paths | -| `related(id)` (per-node adjacency) | p50 < 0.1ms · p95 0.2ms | p50 < 0.1ms | LSM adjacency index — O(degree), scale-independent | -| `find` (semantic: embed + HNSW, 1k docs) | p50 178ms · p95 393ms | — | dominated by WASM query embedding (measured on a machine under concurrent load — treat the p95 as an upper bound); the vector search itself is single-digit ms | - -## Write operations - -Under Model-B **every write is its own durable generation** — a single-op `add` pays -serialization, before-image staging, and fsync before it acks. That durability is priced -into the write path visibly, by design: - -| Operation | 1,000 entities | 10,000 entities | Notes | -|---|---|---|---| -| `add` (single-op) | p50 167ms · p95 171ms | p50 165ms · p95 172ms | full durable generation per write — flat across scale | -| `addMany` (bulk) | ~163ms/entity | ~187ms/entity | **currently per-item commits** — see the honest note below | -| `relateMany` | ~0.8ms/edge | ~0.9ms/edge | edges batch efficiently today | -| `flush` (steady-state, 1 pending write) | p50 8ms · p95 10ms | p50 45ms · p95 52ms | durability-only since 8.9.0 — cost no longer depends on history backlog or retention mode | - -**The honest note on bulk writes:** `addMany` today commits each item as its own -generation (the same durability as single-op `add`, serialized by the single-writer -lock), so bulk-load cost is N × single-op cost. Batched chunk commits (one generation -and one fsync window per chunk, as `removeMany` already does) are designed into the -unified-commit work on the current roadmap. Until that ships, size bulk imports -accordingly — 10k entities is minutes, not seconds, on filesystem storage. - -## Open / close - -| Operation | 1,000 entities | 10,000 entities | Notes | -|---|---|---|---| -| `open` (empty store) | ~560ms | ~190ms | includes embedder initialization | -| `open` (warm, populated, clean shutdown) | 763ms | 4.9s | pure-JS vector index load dominates and grows with entity count; the native accelerator exists precisely to remove this | -| `close` | bounded | bounded | auto-compaction pass is time-bounded (~5s max) since 8.9.0 | - -A store that was NOT cleanly closed pays index rebuilds on top of the warm-open -number (tens of seconds at 10k) — clean shutdown is worth engineering for. - -## How these were produced - -- **Hardware**: Intel Core i9-14900HX (32 threads), 62GB RAM, NVMe, Linux, Node v22. -- **Backend**: `storage: { type: 'filesystem' }`, pure JS (no native providers). -- **Embeddings**: deterministic stub for non-semantic ops (isolates engine cost); - the real WASM embedder for the semantic row (that's what you'll run). -- **Method**: p50/p95 over 50–200 samples per op against the built `dist/`; - the measuring script ships in the repo history and re-runs per release. - -Numbers on different hardware will differ; the *shape* (sub-2ms indexed reads, -~160ms embedding-bound semantic queries, durability-priced writes) is the envelope -you should hold your deployment against. If your measurements diverge from these -shapes by an order of magnitude, something is wrong — file it. diff --git a/package-lock.json b/package-lock.json index fb9262e9..978500d1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "8.9.0", + "version": "8.0.14", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "8.9.0", + "version": "8.0.14", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 7366ce98..1074e624 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "8.9.0", + "version": "8.0.14", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", diff --git a/scripts/push-docs.js b/scripts/push-docs.js deleted file mode 100644 index 699d332b..00000000 --- a/scripts/push-docs.js +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env node -/** - * @module scripts/push-docs - * @description Push this repo's PUBLIC docs to the soulcraft.com docs ingest - * door after an npm publish (VENUE-DOCS-RELEASE-PUSH — retires the old - * build-time docs sync). - * - * Contract (mirrors the reference implementation on the serving side): - * POST {base}/api/docs/ingest - * headers: x-service-secret: $DOCS_INGEST_SECRET, Content-Type: application/json - * body: { docs: [{ slug, title, markdown, nav: { order, section } }] } - * batches of 10, idempotent per slug. - * - * A doc is public iff its frontmatter has `public: true` AND a `slug`. The - * frontmatter is stripped; `category` → nav.section, `order` → nav.order. - * - * Deliberately NOT pushed: the combined /docs landing index. It spans BOTH - * engine corpora (this repo's and the native accelerator's), so a per-repo - * push would clobber the union — the index is authored on the serving side. - * - * Env: DOCS_INGEST_SECRET (required), DOCS_INGEST_BASE (default - * https://soulcraft.com). Exits 0 with a LOUD warning when the secret is - * absent (the npm publish has already happened; the serving side runs its - * interim sync on request) and exits 1 when a push actually fails — the docs - * site would silently trail npm otherwise, and that must be visible. - */ -import * as fs from 'node:fs' -import * as path from 'node:path' - -const BASE = (process.env.DOCS_INGEST_BASE || 'https://soulcraft.com').replace(/\/+$/, '') -const SECRET = process.env.DOCS_INGEST_SECRET -const DOCS_DIR = path.join(path.dirname(new URL(import.meta.url).pathname), '..', 'docs') -const BATCH = 10 - -if (!SECRET) { - console.warn( - '⚠️ DOCS PUSH SKIPPED: DOCS_INGEST_SECRET is not set.\n' + - ' soulcraft.com/docs now TRAILS this npm release until docs are pushed.\n' + - ' Either export DOCS_INGEST_SECRET and re-run `node scripts/push-docs.js`,\n' + - ' or ping venue on VENUE-DOCS-RELEASE-PUSH for the interim sync.' - ) - process.exit(0) -} - -/** Minimal frontmatter split — returns [meta, body] or [null, raw]. */ -function parseFrontmatter(raw) { - const m = raw.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/) - if (!m) return [null, raw] - const meta = {} - for (const line of m[1].split('\n')) { - const kv = line.match(/^(\w[\w-]*):\s*(.*)$/) - if (kv) meta[kv[1]] = kv[2].trim().replace(/^["']|["']$/g, '') - } - return [meta, m[2]] -} - -const docs = [] -;(function walk(dir) { - for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const full = path.join(dir, entry.name) - if (entry.isDirectory()) walk(full) - else if (entry.name.endsWith('.md')) { - const [meta, body] = parseFrontmatter(fs.readFileSync(full, 'utf-8')) - if (!meta || meta.public !== 'true' || !meta.slug) continue - docs.push({ - slug: meta.slug, - title: meta.title || meta.slug, - markdown: body.trim(), - nav: { - order: Number.parseInt(meta.order || '99', 10) || 99, - section: meta.category || 'guides' - } - }) - } - } -})(DOCS_DIR) - -if (docs.length === 0) { - console.error('❌ DOCS PUSH FAILED: zero public docs collected — refusing to push an empty corpus.') - process.exit(1) -} -docs.sort((a, b) => a.slug.localeCompare(b.slug)) -console.log(`Pushing ${docs.length} public docs to ${BASE}/api/docs/ingest …`) - -let failed = false -for (let i = 0; i < docs.length; i += BATCH) { - const batch = docs.slice(i, i + BATCH) - try { - const res = await fetch(`${BASE}/api/docs/ingest`, { - method: 'POST', - headers: { - 'x-service-secret': SECRET, - 'Content-Type': 'application/json', - 'User-Agent': 'brainy-docs-push/1.0' - }, - body: JSON.stringify({ docs: batch }), - signal: AbortSignal.timeout(120_000) - }) - if (!res.ok) { - throw new Error(`HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`) - } - console.log(` batch ${i / BATCH + 1}: ${batch.map((d) => d.slug).join(', ')} → ok`) - } catch (err) { - failed = true - console.error(` batch ${i / BATCH + 1} FAILED: ${err instanceof Error ? err.message : err}`) - } -} - -if (failed) { - console.error( - '❌ DOCS PUSH INCOMPLETE — soulcraft.com/docs may trail npm. ' + - 'Re-run `node scripts/push-docs.js` or ping venue on VENUE-DOCS-RELEASE-PUSH.' - ) - process.exit(1) -} -console.log('✅ Docs pushed.') diff --git a/scripts/release.sh b/scripts/release.sh index 7d860564..0e6a9c43 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -196,18 +196,6 @@ else fi echo -e "${GREEN}✅ GitHub release created${NC}\n" -# Step 12: Push public docs to the soulcraft.com docs ingest door -# (VENUE-DOCS-RELEASE-PUSH). Skips with a loud warning when -# DOCS_INGEST_SECRET is unset; fails loudly (without undoing the publish — -# that already happened) when a push errors, so the docs site never -# silently trails npm. -echo -e "${BLUE}1️⃣2️⃣ Pushing public docs to soulcraft.com/docs...${NC}" -if node scripts/push-docs.js; then - echo -e "${GREEN}✅ Docs push step done${NC}\n" -else - echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n" -fi - echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}" echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" diff --git a/src/aggregation/AggregationIndex.ts b/src/aggregation/AggregationIndex.ts index f9382218..ecd16228 100644 --- a/src/aggregation/AggregationIndex.ts +++ b/src/aggregation/AggregationIndex.ts @@ -29,7 +29,6 @@ import { matchesMetadataFilter } from '../utils/metadataFilter.js' import { compareCodePoints } from '../utils/collation.js' import { bucketTimestamp } from './timeWindows.js' import { NounType } from '../types/graphTypes.js' -import { prodLog } from '../utils/logger.js' /** Persistence key for aggregate definitions */ const DEFINITIONS_KEY = '__aggregation_definitions__' @@ -88,18 +87,10 @@ function matchesSource(entity: Record, source: AggregateDefinit if (entity.service !== source.service) return false } - // Where filter — resolve each filtered field through resolveEntityField, - // the SAME single source of truth groupBy uses (top-level standard fields - // + custom metadata). Matching only the metadata sub-object made - // where:{subtype}/{visibility}/… a silent no-op: reserved fields never - // live in the custom bag, so those filters could never match anything. + // Metadata where filter — match against the entity's metadata sub-object if (source.where && Object.keys(source.where).length > 0) { - const e = entity as unknown as HNSWNounWithMetadata - const resolved: Record = {} - for (const key of Object.keys(source.where)) { - resolved[key] = resolveEntityField(e, key) - } - if (!matchesMetadataFilter(resolved, source.where)) return false + const metadata = (entity.metadata ?? entity) as Record + if (!matchesMetadataFilter(metadata, source.where)) return false } return true @@ -336,30 +327,6 @@ export class AggregationIndex { /** Track aggregates with stale MIN/MAX (need lazy recompute) */ private staleMinMax = new Map>() - /** Resolves when init() has finished loading persisted definitions/state. */ - private initPromise: Promise | null = null - - /** True once init() has settled (success or failure). */ - private initDone = false - - /** - * Aggregates registered by the app before init() finished loading persisted - * state, awaiting reconciliation: init() adopts the persisted state when the - * definition hash matches; anything left unadopted when init settles resolves - * to a backfill. Deciding backfill eagerly at define time was the boot-order - * bug that wiped valid persisted state on every restart — the synchronous - * defineAggregate() always beats the async init(). - */ - private pendingAdopt = new Set() - - /** - * In-flight rescan targets. While a name has a staging map, ALL - * contributions (the walk's and concurrent write hooks') land there instead - * of the live map; the live map keeps serving until {@link finishBackfill} - * swaps the staging map in atomically. - */ - private backfillStaging = new Map>() - constructor(storage: StorageAdapter, nativeProvider?: AggregationProvider) { this.storage = storage this.nativeProvider = nativeProvider @@ -369,128 +336,21 @@ export class AggregationIndex { /** * Initialize: load persisted definitions and state, detect changes, rebuild stale. - * - * Idempotent — repeated calls return the same promise. Definitions registered - * *before* this completes (the normal boot order: `defineAggregate()` is - * synchronous and always beats this async load) are reconciled rather than - * clobbered: the app's definition wins, and its persisted state is adopted - * when the definition hash matches — backfill happens only on a real change. */ - init(): Promise { - if (!this.initPromise) { - this.initPromise = this.loadPersisted().finally(() => { - this.resolvePendingAdoptToBackfill() - this.initDone = true - }) - } - return this.initPromise - } - - /** - * Await the persisted-state load (if one was started) and settle every - * pending adoption decision. After this resolves, `getPendingBackfills()` - * is authoritative: a name is listed iff it genuinely needs a rescan. - * Query paths must await this before consulting backfill state. - */ - async ready(): Promise { - if (this.initPromise) { - try { - await this.initPromise - } catch { - // The owner already surfaced the load failure loudly; backfill covers. - } - } - this.resolvePendingAdoptToBackfill() - } - - /** - * Any definition still awaiting state adoption has no persisted state to - * adopt (or init never ran / failed) — it must backfill. - */ - private resolvePendingAdoptToBackfill(): void { - if (this.pendingAdopt.size > 0) { - prodLog.info( - `[Aggregation] no adoptable persisted state for: ${Array.from(this.pendingAdopt).join(', ')} — flagged for backfill` - ) - } - for (const name of this.pendingAdopt) this.needsBackfill.add(name) - this.pendingAdopt.clear() - } - - /** - * May this persisted state be ADOPTED? When the store exposes its committed - * watermark, the state's `sourceGeneration` must EQUAL it: behind means - * later writes are missing from the state (unclean shutdown); ahead means - * it counts writes that no longer exist (e.g. a fact-log truncation on a - * copied store pulled the watermark back). Either way: one exact rescan, - * said out loud — never a silent adopt. Stores without the capability (and - * pre-stamp state on them) fall back to hash-only adoption. - */ - private stateGenerationAdoptable(name: string, stateData: unknown): boolean { - const committed = this.storage.committedGeneration?.() ?? null - if (committed === null) return true - const raw = (stateData as Record).sourceGeneration - const stamped = typeof raw === 'number' ? raw : null - if (stamped === committed) return true - prodLog.warn( - `[Aggregation] '${name}': persisted state is at generation ${stamped ?? 'unstamped'} ` + - `but the store's committed generation is ${committed} — rescanning instead of adopting` - ) - return false - } - - private async loadPersisted(): Promise { + async init(): Promise { // Load persisted definitions const savedDefs = await this.storage.getMetadata(DEFINITIONS_KEY) if (savedDefs && typeof savedDefs === 'object' && savedDefs.definitions) { const defs = savedDefs.definitions as Array for (const def of defs) { - const savedHash = def._hash || '' - - if (this.definitions.has(def.name)) { - // The app re-registered this aggregate before the load finished. - // The app's definition wins — never clobber it with the persisted - // copy. Adopt the persisted state when the definition is unchanged - // AND no write has landed for it yet (a landed write would be lost - // by adoption; the hook flips such names to backfill). - const appHash = this.definitionHashes.get(def.name) || '' - if (appHash === savedHash && this.pendingAdopt.has(def.name)) { - const stateData = await this.storage.getMetadata(`${STATE_KEY_PREFIX}${def.name}__`) - if ( - stateData && - stateData.groups && - this.stateGenerationAdoptable(def.name, stateData) - ) { - const groupMap = new Map() - for (const group of stateData.groups as AggregateGroupState[]) { - groupMap.set(serializeGroupKey(group.groupKey), group) - } - this.states.set(def.name, groupMap) - this.pendingAdopt.delete(def.name) - this.needsBackfill.delete(def.name) - prodLog.info( - `[Aggregation] '${def.name}': adopted persisted state (${groupMap.size} groups) — no rescan` - ) - } - // No/invalid persisted state: stays in pendingAdopt and resolves - // to backfill when init settles. - } - continue - } - - // Not registered this session — restore definition + state from - // persistence. this.definitions.set(def.name, def) const currentHash = hashDefinition(def) + const savedHash = def._hash || '' + // Load persisted state const stateData = await this.storage.getMetadata(`${STATE_KEY_PREFIX}${def.name}__`) - if ( - stateData && - stateData.groups && - savedHash === currentHash && - this.stateGenerationAdoptable(def.name, stateData) - ) { + if (stateData && stateData.groups && savedHash === currentHash) { // Definition unchanged — load state const groupMap = new Map() for (const group of stateData.groups as AggregateGroupState[]) { @@ -498,10 +358,6 @@ export class AggregationIndex { groupMap.set(serialized, group) } this.states.set(def.name, groupMap) - this.needsBackfill.delete(def.name) - prodLog.info( - `[Aggregation] '${def.name}': restored definition + adopted persisted state (${groupMap.size} groups)` - ) } else { // Definition changed or no saved state — start fresh and backfill from // existing entities (the owner drains needsBackfill on first query). @@ -542,22 +398,14 @@ export class AggregationIndex { })) await this.storage.saveMetadata(DEFINITIONS_KEY, { definitions: defsToSave }) - // Persist dirty states, stamped with the committed generation they - // reflect. The stamp is what makes reopen-adoption verifiable: state at a - // different generation than the store's committed watermark is stale (an - // unclean shutdown after later writes) or over-counts (a fact-log - // truncation on a copied store pulled the watermark BACK below the - // stamp) — either way the answer is one exact rescan, never a silent - // adopt. Read the generation after collecting groups so any racing - // commit resolves toward rescan, not wrong-adopt. + // Persist dirty states for (const name of this.dirty) { const stateMap = this.states.get(name) if (stateMap) { const groups = Array.from(stateMap.values()) - const sourceGeneration = this.storage.committedGeneration?.() ?? null await this.storage.saveMetadata( `${STATE_KEY_PREFIX}${name}__`, - sourceGeneration === null ? { groups } : { groups, sourceGeneration } + { groups } ) } } @@ -604,19 +452,10 @@ export class AggregationIndex { this.definitions.set(def.name, def) this.definitionHashes.set(def.name, newHash) - // First sight this session, before init() settled: defer the backfill - // decision — init() adopts the persisted state on hash match, and anything - // left unadopted resolves to backfill. Deciding eagerly here wiped valid - // persisted state on every restart. - if (!this.states.has(def.name) && !this.initDone) { - this.states.set(def.name, new Map()) - this.pendingAdopt.add(def.name) - } // Reset state if definition changed or doesn't exist yet, and flag it for // backfill so already-stored entities are counted (write-time hooks only see // future writes). The owner drains this on the next query via getPendingBackfills(). - else if (!this.states.has(def.name) || (oldHash && oldHash !== newHash)) { - this.pendingAdopt.delete(def.name) + if (!this.states.has(def.name) || (oldHash && oldHash !== newHash)) { this.states.set(def.name, new Map()) this.needsBackfill.add(def.name) } @@ -637,8 +476,6 @@ export class AggregationIndex { this.definitionHashes.delete(name) this.states.delete(name) this.staleMinMax.delete(name) - this.pendingAdopt.delete(name) - this.needsBackfill.delete(name) // Notify native provider if (this.nativeProvider?.removeAggregate) { @@ -676,17 +513,9 @@ export class AggregationIndex { return Array.from(this.needsBackfill) } - /** - * Begin a rescan into a STAGING map. The live state is not touched — it - * keeps serving (possibly stale, but flagged pending) until the rescan - * completes and swaps in atomically. A mid-walk failure drops the staging - * map via {@link abortBackfill} and loses nothing: wiping live state before - * a scan that could throw was the destructive-before-durable defect. - * Contributions (walk + concurrent write hooks) land in staging while it - * exists, so the swapped-in result reflects writes that raced the walk. - */ + /** Clear an aggregate's state so a full rescan cannot double-count. */ beginBackfill(name: string): void { - this.backfillStaging.set(name, new Map()) + this.states.set(name, new Map()) // Reset native provider state for this aggregate too, if present. const def = this.definitions.get(name) if (def && this.nativeProvider?.removeAggregate && this.nativeProvider?.defineAggregate) { @@ -695,15 +524,6 @@ export class AggregationIndex { } } - /** - * Abandon an in-flight rescan after a failure: drop the staging map, keep - * the live state serving, leave the aggregate flagged as pending so a later - * attempt rescans. The failure itself must be surfaced loudly by the owner. - */ - abortBackfill(name: string): void { - this.backfillStaging.delete(name) - } - /** Feed one already-stored entity into a single aggregate during backfill. */ backfillEntity(name: string, entity: Record): void { if (isAggregateEntity(entity)) return @@ -717,33 +537,14 @@ export class AggregationIndex { } } - /** Swap the rebuilt staging state in atomically; persists on next flush(). */ + /** Mark an aggregate's backfill complete; rebuilt state persists on next flush(). */ finishBackfill(name: string): void { - const staged = this.backfillStaging.get(name) - if (staged) { - this.states.set(name, staged) - this.backfillStaging.delete(name) - } this.needsBackfill.delete(name) this.dirty.add(name) } // ============= Write-Time Hooks ============= - /** - * A write is landing for an aggregate whose persisted-state adoption is still - * pending — adopting after this write would lose its contribution. Settle the - * decision now: an exact rescan instead of adoption. The window is the few - * milliseconds between a boot-time defineAggregate() and init() completing, - * so this rarely fires; when it does, correctness wins over the walk. - */ - private resolveAdoptOnWrite(name: string): void { - if (this.pendingAdopt.has(name)) { - this.pendingAdopt.delete(name) - this.needsBackfill.add(name) - } - } - /** * Called when an entity is added. Updates all matching aggregates. */ @@ -752,7 +553,6 @@ export class AggregationIndex { for (const [name, def] of this.definitions) { if (!matchesSource(entity, def.source)) continue - this.resolveAdoptOnWrite(name) if (this.nativeProvider) { const results = this.nativeProvider.incrementalUpdate(name, def, entity, 'add') @@ -779,10 +579,6 @@ export class AggregationIndex { const oldMatches = matchesSource(oldEntity, def.source) const newMatches = matchesSource(newEntity, def.source) - if (oldMatches || newMatches) { - this.resolveAdoptOnWrite(name) - } - if (this.nativeProvider && (oldMatches || newMatches)) { const results = this.nativeProvider.incrementalUpdate(name, def, newEntity, 'update', oldEntity) this.applyNativeResults(name, results) @@ -809,7 +605,6 @@ export class AggregationIndex { for (const [name, def] of this.definitions) { if (!matchesSource(entity, def.source)) continue - this.resolveAdoptOnWrite(name) if (this.nativeProvider) { const results = this.nativeProvider.incrementalUpdate(name, def, entity, 'delete') @@ -962,7 +757,7 @@ export class AggregationIndex { def: AggregateDefinition, entity: Record ): void { - const stateMap = (this.backfillStaging.get(aggName) ?? this.states.get(aggName))! + const stateMap = this.states.get(aggName)! // Fan out: an unnest dimension makes one entity contribute to several groups. for (const groupKey of computeGroupKeys(entity, def.groupBy)) { @@ -1020,7 +815,7 @@ export class AggregationIndex { def: AggregateDefinition, entity: Record ): void { - const stateMap = (this.backfillStaging.get(aggName) ?? this.states.get(aggName))! + const stateMap = this.states.get(aggName)! // Fan out: reverse the entity's contribution from every group it joined. for (const groupKey of computeGroupKeys(entity, def.groupBy)) { @@ -1076,7 +871,7 @@ export class AggregationIndex { * Apply results from native provider back into the state maps. */ private applyNativeResults(aggName: string, results: AggregateGroupState[]): void { - const stateMap = (this.backfillStaging.get(aggName) ?? this.states.get(aggName))! + const stateMap = this.states.get(aggName)! for (const group of results) { const serialized = serializeGroupKey(group.groupKey) stateMap.set(serialized, group) diff --git a/src/aggregation/materializer.ts b/src/aggregation/materializer.ts index f752648a..3a8dcb3f 100644 --- a/src/aggregation/materializer.ts +++ b/src/aggregation/materializer.ts @@ -14,7 +14,6 @@ import type { AggregateMetricDef } from '../types/brainy.types.js' import { serializeGroupKey } from './AggregationIndex.js' -import { prodLog } from '../utils/logger.js' /** * Callback interface for the materializer to create/update Brainy entities. @@ -87,15 +86,8 @@ export class AggregateMaterializer { if (existing) clearTimeout(existing) this.debounceTimers.set(key, setTimeout(() => { - this.materializeOne(key).catch((err) => { - // Materialization is derived data (rebuildable via backfill-on-query), so - // a failure is non-fatal — but it leaves the materialized Measurement - // entity STALE. Surface it loudly rather than swallow it silently. - prodLog.warn( - `[Aggregation] Failed to materialize aggregate group '${key}': ` + - `${(err as Error).message}. The materialized value is stale until the ` + - `next successful materialization or a backfill-on-query.` - ) + this.materializeOne(key).catch(() => { + // Non-fatal — materialization is derived data }) }, debounceMs)) } diff --git a/src/brainy.ts b/src/brainy.ts index 9ab3acee..2d1a9a31 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -46,10 +46,8 @@ import { pageRank, MinHeap } from './graph/analyticsFallback.js' -import { runGraphAudit, type GraphAuditReport } from './graph/graphAudit.js' import { createPipeline } from './streaming/pipeline.js' import { configureLogger, LogLevel, prodLog } from './utils/logger.js' -import { warnOnLowOsLimits } from './utils/osLimits.js' import { setGlobalCache } from './utils/unifiedCache.js' import type { UnifiedCache } from './utils/unifiedCache.js' import { rankIndicesByScore, reorderByIndices } from './utils/resultRanking.js' @@ -72,7 +70,6 @@ import type { } from './plugin.js' import { ConnectionsCodec } from './hnsw/connectionsCodec.js' import { TransactionManager } from './transaction/TransactionManager.js' -import { transactTimeoutBudget } from './transaction/Transaction.js' import { RevisionConflictError } from './transaction/RevisionConflictError.js' import { EntityNotFoundError, RelationNotFoundError } from './errors/notFound.js' import { @@ -169,30 +166,14 @@ import { type ImportOptions, type ImportResult } from './db/portableGraph.js' -import { GenerationStore, type CommitBeforeImages } from './db/generationStore.js' -import type { FactScanHandle } from './db/factLog.js' -import { - ENTITY_TREE_STAMP_PATH, - readFamilyStamp, - verifyFamilyStamp, - writeFamilyStamp, - type FamilyStamp -} from './db/familyStamp.js' -import { - ChangeFeed, - type BrainyChangeEvent, - type ChangeListener, - type PendingChangeEvent -} from './events/changeFeed.js' +import { GenerationStore } from './db/generationStore.js' import { isDeterministicEmbedMode } from './embeddings/deterministicEmbedMode.js' -import { GenerationConflictError, StoreInconsistentError } from './db/errors.js' -import { BrainyError, GraphIndexNotReadyError, MetadataIndexNotReadyError, MigrationInProgressError, VectorIndexNotReadyError } from './errors/brainyError.js' -import { assessIndexReadiness } from './utils/indexReadiness.js' +import { GenerationConflictError } from './db/errors.js' +import { BrainyError, GraphIndexNotReadyError, MetadataIndexNotReadyError, MigrationInProgressError } from './errors/brainyError.js' import { MemoryStorage } from './storage/adapters/memoryStorage.js' import type { CompactHistoryOptions, CompactHistoryResult, - HistoryStats, TransactOptions, TransactReceipt, TxLogEntry, @@ -202,7 +183,7 @@ import type { HistoryVersion } from './db/types.js' import { stableDeepEqual } from './db/stableEqual.js' -import type { VersionedIndexProvider, ProviderInvariantReport } from './plugin.js' +import type { VersionedIndexProvider } from './plugin.js' import type { Operation, TransactionFunction } from './transaction/types.js' /** @@ -333,78 +314,8 @@ interface PlannedTransact { touchedVerbs: string[] /** Aggregation-index hooks to run after the commit point. */ postCommit: Array<() => void> - /** - * One entry per staged `{ op: 'update' }`, re-verified UNDER the commit - * mutex (the generation store's `precommit`) so per-op `ifRev` CAS is - * atomic with the apply — planning-time checks can interleave with - * concurrent writers. `updatedMetadata` is the staged metadata object (held - * by reference by the staged operation), re-stamped there with the - * authoritative `_rev`. A conflict rejects the WHOLE batch. - */ - casUpdates: Array<{ - id: string - ifRev?: number - updatedMetadata: { _rev?: number } - }> - /** - * Ids whose revision baseline THIS batch resets via an `add` op (a fresh - * create, or add's overwrite semantics which restamp `_rev: 1`). Updates to - * these ids sequence against in-batch state no concurrent writer can touch, - * so their plan-time CAS checks and rev stamps are already exact — the - * commit precondition leaves them untouched. - */ - createdNouns: Set - /** - * Change-feed events, one per affected record in op order, populated by the - * planners ONLY when a listener is subscribed. Stamped with the batch's - * committed generation and emitted after `commitTransaction` returns — a - * rejected batch (CAS conflict, failed apply) emits nothing. - */ - changeEvents: PendingChangeEvent[] } -/** - * Internal control-flow signal: an insert guarded by a must-be-absent - * precondition (`add({ ifAbsent })` / `add({ upsert })`) found the entity - * already present in the authoritative before-image under the commit mutex — - * a concurrent writer created it after the planning-time absence check. - * Never escapes `add()`: the caller converts it into the documented - * resolution (ifAbsent → return the existing id without writing; upsert → - * merge into the now-existing entity via `update()`). - */ -class InsertPreconditionExistsSignal extends Error { - constructor(readonly id: string) { - super(`insert precondition: entity ${id} already exists`) - this.name = 'InsertPreconditionExistsSignal' - } -} - -/** - * @description The derived-index families a read may depend on. A read that - * consults none of them (a canonical-storage read: `get`, an entity - * enumeration, a VFS content/dir read) is index-independent and must never - * block on another family's one-time migration. Used by the family-scoped - * migration gate ({@link Brainy.awaitMigrationLock}). - */ -export type IndexFamily = 'vector' | 'metadata' | 'graph' - -/** - * How long a failed aggregation-backfill walk suppresses fresh walk attempts. - * Within the window, queries rethrow the recorded failure instantly (loud, - * cheap); after it, one new attempt is allowed. Bounds the damage of a - * caller-side tight retry loop against a deterministically-failing store. - */ -const AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS = 30_000 - -/** - * Time budget for the auto-compaction pass at close() (8.9.0). Bounds how long - * a clean shutdown spends reclaiming history backlog — an early stop is a - * consistent prefix and the next close/explicit pass resumes. Explicit - * `compactHistory()` calls are unbounded unless the caller passes their own - * `timeBudgetMs` (maintenance windows choose their own budgets). - */ -const CLOSE_COMPACTION_BUDGET_MS = 5_000 - /** * The main Brainy class - Clean, Beautiful, Powerful * REAL IMPLEMENTATION - No stubs, no mocks @@ -517,18 +428,6 @@ export class Brainy implements BrainyInterface { * during rebuild is NOT recorded here — it re-throws and aborts init() loudly. */ private _indexRebuildFailed: Error | null = null - /** - * Ids of records that committed via an adopt-forward failed-rollback recovery - * ({@link GenerationStore.commitSingleOp} `degraded`): the canonical record is - * durable but its derived index entry may be incomplete until - * {@link repairIndex}. Non-empty = a queryable degraded state, folded into - * {@link checkHealth}/{@link validateIndexConsistency} and warned on the read - * paths. Cleared by {@link repairIndex}. - */ - private _indexDegradedIds: Set = new Set() - /** One-shot guard so the degraded-reads warning fires once per degraded window - * (reset when the degraded state clears). See {@link warnIfReadsDegraded}. */ - private _degradedReadWarned = false /** One-shot guard so the metadata cold-open consistency probe runs once per brain. */ private _metadataConsistencyProbed = false /** Graph-adjacency cold-load consistency: verified-live this session (one-shot). */ @@ -539,10 +438,6 @@ export class Brainy implements BrainyInterface { private _metadataVerified = false /** Re-entrancy guard for {@link verifyMetadataLive}. */ private _metadataVerifying = false - /** Vector-index cold-read guard: verified-serving this session (one-shot). */ - private _vectorVerified = false - /** Re-entrancy guard for {@link verifyVectorLive}. */ - private _vectorVerifying = false /** * Coordinated migration LOCK (#18): dedup guards so the "upgrading, blocking" * and "upgrade complete, resumed" lines each log once per migration window, @@ -559,15 +454,6 @@ export class Brainy implements BrainyInterface { * upgrade verifies + stamps; retained on failure. See {@link createMigrationBackupIfNeeded}. */ private _migrationBackupPath: string | null = null - /** - * The in-process change feed behind {@link onChange}. Emitted from the - * commit seam ({@link persistSingleOp} / {@link transact}), so every - * canonical mutation — any origin — produces exactly one post-commit event - * per affected record. See src/events/changeFeed.ts for the delivery - * contract. - */ - private readonly _changeFeed = new ChangeFeed() - /** Set when the on-open VFS-blob adoption left one or more `_cow/` blobs it * could not fully adopt (bytes or metadata missing). While true, the * pre-upgrade backup is NOT auto-removed — the upgrade is not verifiably @@ -619,11 +505,6 @@ export class Brainy implements BrainyInterface { private _hub?: IntegrationHub // Integration Hub for external tools private _pendingMigrationRunner?: MigrationRunner // Deferred migration runner for large datasets private _aggregationIndex?: AggregationIndex // Incremental aggregation engine - private _aggregationBackfillFlight: Promise | null = null // Single-flight backfill walk - // A failed walk latches its error: retries within the cooldown rethrow it - // instantly instead of re-walking, so a tight caller-side retry loop costs - // one loud error per query, never a full store walk per query. - private _aggregationBackfillFailure: { at: number; error: Error } | null = null private _materializer?: AggregateMaterializer // Debounced materialization of aggregate results /** * Fields registered via `brain.trackField()` — drives optional value validation on @@ -653,13 +534,6 @@ export class Brainy implements BrainyInterface { // set this to ReaderMode so historical instances are protected the same way. private operationalMode: BaseOperationalMode - // Write-quarantine after a failed transaction rollback left the store - // inconsistent (see StoreInconsistentError). Set at the moment the failure is - // surfaced; every subsequent mutation is refused via assertWritable until - // repairIndex() reconciles canonical vs derived state and clears it. Reads are - // never affected. null = healthy. - private storeInconsistency: StoreInconsistentError | null = null - // Ready Promise state (Unified readiness API) // Allows consumers to await brain.ready for initialization completion private _readyPromise: Promise | null = null @@ -825,17 +699,6 @@ export class Brainy implements BrainyInterface { `Open in writer mode to modify data.` ) } - // Write-quarantine: a prior transaction's rollback failed and left the store - // inconsistent. Refuse further writes (which would compound the damage) until - // repairIndex() reconciles and lifts the quarantine. Reads still work. - if (this.storeInconsistency) { - throw new Error( - `Cannot call ${method}() — the store is WRITE-QUARANTINED after a failed ` + - `transaction rollback left it inconsistent. Reads still work; run repairIndex() ` + - `to reconcile the derived indexes against canonical storage and lift the ` + - `quarantine. Original inconsistency: ${this.storeInconsistency.message}` - ) - } } /** @@ -946,13 +809,6 @@ export class Brainy implements BrainyInterface { this.storage = await this.setupStorage() await this.storage.init() - // OS-limit detection (once per process, Linux-only, measurement-only): - // warn NOW about RLIMIT_NOFILE / vm.max_map_count values that will bite - // at pool scale, instead of letting the operator meet them as EMFILE or - // a failed mmap deep inside an index open. Fire-and-forget — the check - // never affects open. - void warnOnLowOsLimits() - // Acquire the writer lock for filesystem (and other locking-capable) backends. // Skipped in reader mode and on backends that don't support multi-process locking. // Throws if another live writer holds the directory (unless force: true). @@ -1014,44 +870,6 @@ export class Brainy implements BrainyInterface { readOnly: this.config.mode === 'reader' }) - // The generation fact log is CANONICAL state, not a derived index — no - // sweeper, GC, or blob-lifecycle path may ever delete under it. Declare - // its namespace as a protected family (rebuildable: false — a lost fact - // segment is NOT reconstructable) so the storage layer REFUSES such - // deletes; refusal beats trust. Feature-detected + idempotent per name. - if ( - this.config.mode !== 'reader' && - this.generationStore.getFactLog() && - typeof this.storage.registerDerivedFamily === 'function' - ) { - await this.storage.registerDerivedFamily({ - name: 'generation-facts', - members: ['_generations/facts/'], - namespace: true, - rebuildable: false - }) - } - - // Fact-scan capability: wire the storage seam through which index - // providers (which hold only `storage`) reach the fact log. A closure - // over the LIVE log — restore/reopen swaps the instance transparently — - // so a provider's heal can switch from the enumeration walk to one - // sequential fact scan whenever the log exists. - if (typeof (this.storage as BaseStorage).setFactScanSource === 'function') { - ;(this.storage as BaseStorage).setFactScanSource({ - factLog: () => this.generationStore?.getFactLog() ?? null, - // The committed watermark, exposed as a capability so providers - // never parse the store's private manifest format. - committedGeneration: () => this.generationStore?.committedGeneration() ?? 0 - }) - } - - // Entity-tree stamp coherence: compare the stamped sourceGeneration + - // rollup invariants against the log head + live counters. Loud on - // genuine incoherence (repairIndex heals), silent on absent/coherent, - // benign-behind refreshes at the next flush. Never blocks open. - await this.verifyEntityTreeStamp() - // 8.0 ⇄ native-provider version handshake: load the on-disk brain-format // marker (`_system/brain-format.json`) into an in-memory field NOW — // after the store-open phase, but BEFORE any derived index or native @@ -1214,22 +1032,6 @@ export class Brainy implements BrainyInterface { `(storage committed: ${committed}) — provider replays the gap per ` + `the post-commit applier contract` ) - } else if (providerGen > committed) { - // The AHEAD direction is incoherence, not a replay gap: the provider's - // persisted index claims writes the store no longer has — the signature - // of a torn copy or a log truncation that pulled the committed - // watermark back (crash recovery, byte-copy of a live store). A replay - // can never converge on it and index answers may reference vanished - // writes. Name it loudly at open so it is never diagnosed from a - // silent journal; the provider's own coherence check / heal walk (or - // brain.repairIndex()) is the cure. - prodLog.warn( - `[Brainy] Versioned index provider is AHEAD of the store: provider ` + - `generation ${providerGen} vs committed ${committed}. This store was ` + - `likely copied from a live service or truncated during crash recovery. ` + - `Derived-index answers may reference rolled-back writes until the ` + - `provider heals from canonical (brain.repairIndex() forces it).` - ) } } @@ -1241,19 +1043,6 @@ export class Brainy implements BrainyInterface { // already healed. See adoptLegacyCowBlobs. await this.autoAdoptLegacyVfsBlobsIfNeeded() - // Temporal-blob contract: one-time (marker-gated) backfill of blob - // history reference counts for stores whose generation history predates - // the contract — after it, compaction reclaims blob bytes exactly (zero - // live AND zero history references). On a scrub failure the store runs - // leak-safe (no blob reclamation) rather than risk a premature delete. - if (typeof (this.storage as unknown as { - backfillBlobHistoryRefCountsIfNeeded?: () => Promise - }).backfillBlobHistoryRefCountsIfNeeded === 'function') { - await (this.storage as unknown as { - backfillBlobHistoryRefCountsIfNeeded: () => Promise - }).backfillBlobHistoryRefCountsIfNeeded() - } - // Rebuild indexes if needed for existing data await this.rebuildIndexesIfNeeded() @@ -1391,13 +1180,6 @@ export class Brainy implements BrainyInterface { if (this._readyReject) { this._readyReject(error instanceof Error ? error : new Error(String(error))) } - // Machine-readable init failures pass through UNWRAPPED — the writer-lock - // conflict documents an err.code/err.lockInfo contract ("callers detect - // this case via err.code"), and wrapping in a fresh Error silently - // stripped both, leaving consumers only a message to regex against. - if (error instanceof Error && (error as Error & { code?: string }).code === 'BRAINY_WRITER_LOCKED') { - throw error - } throw new Error(`Failed to initialize Brainy: ${error}`) } } @@ -1550,10 +1332,7 @@ export class Brainy implements BrainyInterface { * re-initializing — using a closed Brainy is a consumer bug, not a lazy-init * opportunity. */ - private async ensureInitialized(opts?: { - bypassMigrationLock?: boolean - needs?: IndexFamily[] - }): Promise { + private async ensureInitialized(opts?: { bypassMigrationLock?: boolean }): Promise { if (this.closed) { throw new Error('Brainy instance is not initialized: it was closed via close(). Create a new instance.') } @@ -1563,17 +1342,12 @@ export class Brainy implements BrainyInterface { // Coordinated migration LOCK (#18): every data-plane read and write funnels // through here, so this is the single choke point that holds operations while // a native provider runs its one-time 7.x → 8.0 rebuild-from-canonical — no - // op touches a half-built index. The gate is FAMILY-SCOPED: `needs` names the - // derived-index families this operation actually consults, so a read served - // entirely from canonical storage (`needs: []`) or from a healthy family - // never blocks on an UNRELATED family's migration. `needs` omitted = the - // conservative whole-brain wait (writes, and any read not yet classified). - // Observability (`health`/`checkHealth`) and the lock-clearing path - // (`stampBrainFormat`, which does not route through here) opt out entirely so - // an operator can always watch progress and a native provider can stamp. + // op touches a half-built index. Observability (`health`/`checkHealth`) and + // the lock-clearing path (`stampBrainFormat`, which does not route through + // here) opt out so an operator can always watch progress and cor can stamp. // A brain that never migrates pays one boolean check (see awaitMigrationLock). if (!opts?.bypassMigrationLock) { - await this.awaitMigrationLock(opts?.needs) + await this.awaitMigrationLock() } } @@ -1741,178 +1515,21 @@ export class Brainy implements BrainyInterface { */ private async persistSingleOp( touched: { nouns?: string[]; verbs?: string[] }, - run: TransactionFunction, - precommit?: (before: CommitBeforeImages) => void, - pendingEvents?: PendingChangeEvent[] - ): Promise<{ generation?: number; timestamp: number; degraded?: string[] }> { - // Change-feed capture: when this write will emit, hold a reference to the - // commit's before-images so `remove` events can carry the record's last - // committed state (free — the commit reads them anyway for Model B). - let capturedBefore: CommitBeforeImages | undefined - const captureAndCheck = - pendingEvents && pendingEvents.length > 0 - ? (before: CommitBeforeImages): void => { - capturedBefore = before - precommit?.(before) - } - : precommit - + run: TransactionFunction + ): Promise { if (!this._generationStampingActive) { // Init-time / infrastructure baseline write (e.g. the VFS root): apply // WITHOUT creating a generation. Generation 0 is the freshly-materialized // brain (bootstrap included); the first USER write is generation 1. - // Bootstrap writes are single-threaded, so a supplied precondition is - // honored against directly-read before-images (no commit mutex exists - // on this path — nothing to race). - if (captureAndCheck) { - const nouns = new Map() - for (const id of touched.nouns ?? []) { - const prev = await this.storage.readNounRaw(id) - nouns.set(id, { kind: 'noun', metadata: prev.metadata, vector: prev.vector }) - } - const verbs = new Map() - for (const id of touched.verbs ?? []) { - const prev = await this.storage.readVerbRaw(id) - verbs.set(id, { kind: 'verb', metadata: prev.metadata, vector: prev.vector }) - } - captureAndCheck({ nouns, verbs } as CommitBeforeImages) - } await this.generationStore.runWithoutGeneration(() => - this.transactionManager.executeTransaction(run, { - timeout: transactTimeoutBudget( - (touched.nouns?.length ?? 0) + (touched.verbs?.length ?? 0) - ) - }) + this.transactionManager.executeTransaction(run) ) - const timestamp = Date.now() - // Bootstrap writes are not generation-stamped; emit without one. - this.emitCommitted(pendingEvents, capturedBefore, undefined, timestamp) - return { timestamp } + return } - let receipt - try { - receipt = await this.generationStore.commitSingleOp({ - touched, - precommit: captureAndCheck, - execute: () => - this.transactionManager.executeTransaction(run, { - timeout: transactTimeoutBudget( - (touched.nouns?.length ?? 0) + (touched.verbs?.length ?? 0) - ) - }) - }) - } catch (err) { - // A failed rollback that left the store inconsistent (a remove/update - // whose restore-undo failed) quarantines writes until repairIndex(). - if (err instanceof StoreInconsistentError) this.storeInconsistency = err - throw err - } - // POST-COMMIT ONLY: an aborted commit (CAS conflict, failed apply) throws - // above and never reaches this line — the feed cannot announce a write - // that did not become durable. (A degraded adopt-forward write DID commit — - // it carries a generation and emits normally.) - this.emitCommitted(pendingEvents, capturedBefore, receipt.generation, receipt.timestamp) - // An adopt-forward failed-rollback recovery committed the record but may have - // left its derived index incomplete (generationStore already warned once). - // Record the ids so reads and checkHealth() surface the incompleteness rather - // than silently returning partial data; repairIndex() clears them. - if (receipt.degraded && receipt.degraded.length > 0) { - for (const degradedId of receipt.degraded) this._indexDegradedIds.add(degradedId) - this._degradedReadWarned = false // re-arm the read-path warning - if (!this.config.silent) { - prodLog.warn( - `[Brainy] A single-op write committed in a DEGRADED state — the derived ` + - `index may be incomplete for id(s) ${receipt.degraded.join(', ')}. ` + - `Reads may return partial results until repairIndex() reconciles them.` - ) - } - } - return receipt - } - - /** - * @description Stamp pending change events with their commit receipt, - * enrich entity `remove` events with the record's last committed state - * (from the commit's before-images), and hand them to the change feed for - * post-mutex dispatch. No-op when the write produced no events. - * @param pending - Events the mutation method constructed pre-commit - * (only when a listener is subscribed — see {@link ChangeFeed.hasListeners}). - * @param before - The commit's before-images (delete-payload source). - * @param generation - The committed generation (absent for bootstrap writes). - * @param timestamp - The commit timestamp. - */ - private emitCommitted( - pending: PendingChangeEvent[] | undefined, - before: CommitBeforeImages | undefined, - generation: number | undefined, - timestamp: number - ): void { - if (!pending || pending.length === 0) return - const events: BrainyChangeEvent[] = pending.map((p) => { - let entity = p.entity - if (!entity && p.kind === 'entity' && p.op === 'remove' && p.id && before) { - const record = before.nouns.get(p.id)?.metadata as - | Record - | null - | undefined - if (record) { - entity = this.entityViewFromRawRecord(p.id, record) - } - } - return { - ...p, - ...(entity && { entity }), - ...(generation !== undefined && { generation }), - timestamp - } + await this.generationStore.commitSingleOp({ + touched, + execute: () => this.transactionManager.executeTransaction(run) }) - this._changeFeed.emit(events) - } - - /** - * @description Build the change-event entity view from a stored flat - * metadata record (the shape before-images and pre-delete reads carry), - * using THE canonical reserved/custom split so the view can never drift - * from the live read paths. - * @param id - The entity's canonical UUID. - * @param record - The stored flat metadata record. - * @returns The `ChangeEventEntity` view (type, subtype, service, custom metadata). - */ - private entityViewFromRawRecord( - id: string, - record: Record - ): NonNullable { - const { reserved, custom } = splitNounMetadataRecord(record) - return { - id, - type: String(reserved.noun ?? 'unknown'), - ...(reserved.subtype !== undefined && { subtype: String(reserved.subtype) }), - metadata: custom, - ...(reserved.service !== undefined && { service: String(reserved.service) }) - } - } - - /** - * @description Build the AGGREGATION view of an entity from a stored flat - * metadata record — EVERY reserved field mapped to its top-level entity - * name (stored `noun` → `type`), custom metadata in `metadata`. This must - * mirror the add-path `entityForIndexing` shape exactly: the aggregation - * engine resolves groupBy/where fields via `resolveEntityField` - * (top-level standard fields + custom metadata), so a view that drops a - * reserved field makes every aggregate grouped by that field decrement a - * group that does not exist — counts then drift upward forever after - * deletes (SELF-AGGREGATE-DELETE-DRIFT). Do not hand-roll subsets of this. - * @param record - The stored flat metadata record (before-image or pre-delete read). - * @returns The full-fidelity entity view for aggregation hooks. - */ - private entityForAggFromRawRecord(record: Record): Record { - const { reserved, custom } = splitNounMetadataRecord(record) - const { noun, ...rest } = reserved - return { - type: noun, - ...rest, - metadata: custom - } } /** @@ -1983,10 +1600,7 @@ export class Brainy implements BrainyInterface { // ifAbsent (7.31.0) — by-ID idempotent insert. Only meaningful when a custom id // is supplied; a freshly generated UUID can never collide. Returns the existing // (canonical) id without writing if the entity is already present (no throw, - // no overwrite). This check is a FAST-FAIL (skips the embedding cost); the - // authoritative absence check runs in the insert's commit precondition - // below, atomic with the apply, so a concurrent same-id create cannot make - // both callers write. + // no overwrite). if (params.id && params.ifAbsent) { const existing = await this.storage.getNounMetadata(id) if (existing) return id @@ -1997,12 +1611,21 @@ export class Brainy implements BrainyInterface { // delegate to update() so the supplied fields MERGE into the existing entity // (metadata merge, re-embed on changed data, _rev bump, createdAt preserved) // instead of the default destructive overwrite. When the id is absent, fall - // through to the guarded insert below (whose commit precondition converts a - // concurrent same-id create into this same merge — never an overwrite). + // through to the normal insert below. if (params.id && params.upsert) { const existing = await this.storage.getNounMetadata(id) if (existing) { - await this.update(this.upsertMergeParams(id, params)) + await this.update({ + id, + ...(params.data !== undefined && { data: params.data }), + ...(params.type !== undefined && { type: params.type }), + ...(params.subtype !== undefined && { subtype: params.subtype }), + ...(params.visibility !== undefined && { visibility: params.visibility }), + ...(params.metadata !== undefined && { metadata: params.metadata }), + ...(params.vector !== undefined && { vector: params.vector }), + ...(params.confidence !== undefined && { confidence: params.confidence }), + ...(params.weight !== undefined && { weight: params.weight }) + }) return id } } @@ -2074,22 +1697,7 @@ export class Brainy implements BrainyInterface { // Execute atomically with transaction system, generation-stamped as one // immutable Model-B generation (before-image = the absent/create sentinel). // All operations succeed or all rollback - prevents partial failures - // ifAbsent/upsert insert leg: must-be-absent commit precondition, run - // under the commit mutex against the authoritative before-image — the - // planning-time absence checks above are fast-fails that can interleave - // with a concurrent same-id create. Exactly one concurrent create wins; - // every other caller takes its documented resolution instead of - // silently overwriting the winner. - const requireAbsent = Boolean(params.id && (params.ifAbsent || params.upsert)) - const insertPrecommit = requireAbsent - ? (before: CommitBeforeImages): void => { - if (before.nouns.get(id)?.metadata) { - throw new InsertPreconditionExistsSignal(id) - } - } - : undefined - - const runInsert: TransactionFunction = async (tx) => { + await this.persistSingleOp({ nouns: [id] }, async (tx) => { // Operation 1: Save metadata FIRST (TypeAwareStorage caching) // isNew=true: skip pre-read for rollback (entity doesn't exist yet) tx.addOperation( @@ -2116,62 +1724,7 @@ export class Brainy implements BrainyInterface { tx.addOperation( new AddToMetadataIndexOperation(this.metadataIndex, id, entityForIndexing) ) - } - - // Bounded retry closes the remaining interleavings without ever holding - // a lock across the loop: a lost insert race resolves to skip (ifAbsent) - // or merge (upsert); a merge that then hits a concurrent delete retries - // the insert. Each persistSingleOp call constructs fresh operations, so - // re-running the callback is safe. - // Change feed: built only when someone is listening (zero-cost gate). - const addEvents: PendingChangeEvent[] | undefined = this._changeFeed.hasListeners - ? [ - { - kind: 'entity', - op: 'add', - id, - entity: { - id, - type: String(params.type), - ...(params.subtype !== undefined && { subtype: String(params.subtype) }), - metadata: (params.metadata as Record) ?? {}, - ...(params.service !== undefined && { service: String(params.service) }) - } - } - ] - : undefined - - const MAX_UPSERT_ATTEMPTS = 10 - for (let attempt = 0; ; attempt++) { - try { - await this.persistSingleOp({ nouns: [id] }, runInsert, insertPrecommit, addEvents) - break - } catch (err) { - if (!(err instanceof InsertPreconditionExistsSignal)) { - throw err - } - // A concurrent writer created the entity between our absence check - // and the commit. - if (params.ifAbsent) { - // Documented contract: return the existing id without writing. - return id - } - // upsert: merge into the now-existing entity. A concurrent delete - // between this conflict and the merge throws EntityNotFoundError — - // loop back and retry the insert. - try { - await this.update(this.upsertMergeParams(id, params)) - return id - } catch (mergeErr) { - if ( - !(mergeErr instanceof EntityNotFoundError) || - attempt >= MAX_UPSERT_ATTEMPTS - ) { - throw mergeErr - } - } - } - } + }) // Aggregation hook (outside transaction — derived data, can be reconstructed) if (this._aggregationIndex) { @@ -2181,31 +1734,6 @@ export class Brainy implements BrainyInterface { return id } - /** - * @description The `update()` param mapping `add({ upsert })` delegates to - * when the target id already exists: every supplied field merges into the - * existing entity (metadata merge, re-embed on changed data, `_rev` bump, - * `createdAt` preserved); absent fields are left untouched. Shared by the - * planning-time upsert branch and the commit-conflict retry so the two - * paths can never drift. - * @param id - The canonical entity id the upsert resolved to. - * @param params - The original `add()` params carrying the fields to merge. - * @returns The `UpdateParams` for the merging `update()` call. - */ - private upsertMergeParams(id: string, params: AddParams): UpdateParams { - return { - id, - ...(params.data !== undefined && { data: params.data }), - ...(params.type !== undefined && { type: params.type }), - ...(params.subtype !== undefined && { subtype: params.subtype }), - ...(params.visibility !== undefined && { visibility: params.visibility }), - ...(params.metadata !== undefined && { metadata: params.metadata }), - ...(params.vector !== undefined && { vector: params.vector }), - ...(params.confidence !== undefined && { confidence: params.confidence }), - ...(params.weight !== undefined && { weight: params.weight }) - } as UpdateParams - } - /** * Get an entity by ID * @@ -2339,10 +1867,7 @@ export class Brainy implements BrainyInterface { * */ async get(id: string, options?: GetOptions): Promise | null> { - // Canonical read: a get resolves an entity by id straight from storage and - // consults no derived index — it must not wait on any family's migration. - await this.ensureInitialized({ needs: [] }) - this.warnIfReadsDegraded('get') + await this.ensureInitialized() // Id normalization (8.0): a caller may read by their natural key — resolve // it to the same canonical UUID add() stored. A real UUID passes through. @@ -2400,8 +1925,7 @@ export class Brainy implements BrainyInterface { * ``` */ async batchGet(ids: string[], options?: GetOptions): Promise>> { - // Canonical read (see get): resolves by id from storage, no derived index. - await this.ensureInitialized({ needs: [] }) + await this.ensureInitialized() // Id normalization (8.0): resolve each id to its canonical UUID so callers // can batch-read by natural key. resolveEntityId is idempotent on real @@ -2968,11 +2492,7 @@ export class Brainy implements BrainyInterface { // ifRev (7.31.0) — optimistic concurrency. If caller supplied ifRev, the persisted // _rev must match exactly. `_rev` is reserved: every read path surfaces it ONLY // top-level (entities without one are read as rev 1), so that is the one place - // to look. This check is purely a FAST-FAIL (avoids paying the embedding - // cost on an obviously stale expectation) — the authoritative check runs - // in the commit precondition below, under the commit mutex, where it is - // atomic with the apply. Concurrent same-rev updates all pass HERE but - // exactly one survives THERE. + // to look. const currentRev = typeof existing._rev === 'number' ? existing._rev : 1 if (typeof params.ifRev === 'number' && params.ifRev !== currentRev) { throw new RevisionConflictError(params.id, params.ifRev, currentRev) @@ -3048,35 +2568,6 @@ export class Brainy implements BrainyInterface { metadata: newMetadata } - // Authoritative CAS + honest rev stamp, run by the generation store - // UNDER the commit mutex against the just-read before-image — the one - // point where check-and-apply is atomic (the fast-fail above can - // interleave with concurrent writers; this cannot). Also re-stamps - // `_rev` from the authoritative base so the counter stays monotonic - // even for concurrent non-CAS updates. The staged operations below - // capture `updatedMetadata` by reference, so the re-stamp lands. - const casPrecommit = (before: CommitBeforeImages): void => { - const beforeMeta = before.nouns.get(params.id)?.metadata as - | { _rev?: unknown } - | null - | undefined - if (!beforeMeta) { - // A concurrent remove() deleted the entity after our read. A CAS - // caller gets the truth; a plain update keeps its long-standing - // last-writer-wins semantics (the staged write re-creates it). - if (typeof params.ifRev === 'number') { - throw new EntityNotFoundError(params.id) - } - return - } - const authoritativeRev = - typeof beforeMeta._rev === 'number' ? beforeMeta._rev : 1 - if (typeof params.ifRev === 'number' && params.ifRev !== authoritativeRev) { - throw new RevisionConflictError(params.id, params.ifRev, authoritativeRev) - } - updatedMetadata._rev = authoritativeRev + 1 - } - // Execute atomically with transaction system, generation-stamped as one // immutable Model-B generation (before-image = the entity's prior state). await this.persistSingleOp({ nouns: [params.id] }, async (tx) => { @@ -3136,37 +2627,17 @@ export class Brainy implements BrainyInterface { tx.addOperation( new AddToMetadataIndexOperation(this.metadataIndex, params.id, entityForIndexing) ) - }, casPrecommit, this._changeFeed.hasListeners - ? [ - { - kind: 'entity', - op: 'update', - id: params.id, - entity: { - id: params.id, - type: String(entityForIndexing.type), - ...(entityForIndexing.subtype !== undefined && { - subtype: String(entityForIndexing.subtype) - }), - metadata: (newMetadata as Record) ?? {}, - ...(entityForIndexing.service !== undefined && { - service: String(entityForIndexing.service) - }) - } - } - ] - : undefined) + }) - // Aggregation hook (outside transaction — derived data). `existing` is - // the full get() view — every reserved field top-level — and must be - // passed whole: a subset view makes the old-side decrement miss any - // reserved-field group (update would then double-count it). + // Aggregation hook (outside transaction — derived data) if (this._aggregationIndex) { - this._aggregationIndex.onEntityUpdated( - params.id, - entityForIndexing, - existing as unknown as Record - ) + const oldEntityForAgg = { + type: existing.type, + service: existing.service, + data: existing.data, + metadata: existing.metadata + } + this._aggregationIndex.onEntityUpdated(params.id, entityForIndexing, oldEntityForAgg) } } @@ -3228,11 +2699,9 @@ export class Brainy implements BrainyInterface { ) } - // Operation 3: Delete noun (full removal). The pre-read metadata rides - // along so the count decrement never depends on re-reading the record - // being removed (a null re-read must not silently skip it). + // Operation 3: Delete noun metadata tx.addOperation( - new DeleteNounMetadataOperation(this.storage, id, metadata) + new DeleteNounMetadataOperation(this.storage, id) ) // Operations 4+: Delete all related verbs atomically @@ -3241,52 +2710,28 @@ export class Brainy implements BrainyInterface { // rollback can re-add through the BigInt addVerb contract) const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb) tx.addOperation( - new RemoveFromGraphIndexOperation(this.graphIndex, verb, { sourceInt, targetInt }, this.graphWriteGeneration) + new RemoveFromGraphIndexOperation(this.graphIndex, verb, sourceInt, targetInt, this.graphWriteGeneration) ) // Delete verb metadata tx.addOperation( new DeleteVerbMetadataOperation(this.storage, verb.id) ) } - }, - undefined, - this._changeFeed.hasListeners - ? [ - // The entity delete (payload = last state, from the pre-delete read) - // plus one unrelate per cascade-deleted relationship. - { - kind: 'entity', - op: 'remove', - id, - ...(metadata && { - entity: this.entityViewFromRawRecord(id, metadata as Record) - }) - }, - ...allVerbs.map( - (v): PendingChangeEvent => ({ - kind: 'relation', - op: 'unrelate', - id: v.id, - relation: { - id: v.id, - from: v.sourceId, - to: v.targetId, - type: String(v.verb) - } - }) - ) - ] - : undefined) + }) - // Aggregation hook (outside transaction — derived data). The view must - // carry EVERY reserved field top-level (not a subset): a groupBy on - // subtype/visibility/etc. otherwise decrements a nonexistent group and - // the real count never comes down. + // Aggregation hook (outside transaction — derived data) if (this._aggregationIndex && metadata) { - this._aggregationIndex.onEntityDeleted( - id, - this.entityForAggFromRawRecord(metadata as Record) - ) + // Reconstruct entity-like object from stored metadata via the + // canonical reserved/custom split (the hand-rolled destructure here + // missed subtype/_rev, leaking them into the aggregation view). + const { reserved, custom } = splitNounMetadataRecord(metadata) + const entityForAgg = { + type: reserved.noun, + service: reserved.service, + data: reserved.data, + metadata: custom + } + this._aggregationIndex.onEntityDeleted(id, entityForAgg) } } @@ -3681,142 +3126,6 @@ export class Brainy implements BrainyInterface { return null } - /** - * @description The vector-index counterpart of {@link verifyGraphAdjacencyLive} - * / {@link verifyMetadataLive}. On a cold open a native vector provider can - * report a non-zero `size()` (its persisted COUNT loaded) yet not have loaded - * its serving structure (the mmap/DiskANN graph) — so a pure semantic - * `find({ query })` silently returns `[]`. A pure semantic query has - * `hasFilterCriteria === false`, so the metadata guard never fires; this guard - * closes that gap. Run one-shot on the first vector/proximity search: - * - **Preferred (honest signal):** the provider exposes `isReady()`. `false` - * → rebuild from storage, re-check; if still `false`, throw - * {@link VectorIndexNotReadyError} rather than serving `[]`. - * - **Fallback (no `isReady()`):** a KNOWN persisted vector (sampled + - * hydrated) is searched against the index; if it does not self-match, the - * serving structure did not load — rebuild + re-probe, else throw. - * Inconclusive cases (empty store, no probeable vector, `size()===0` — where - * the JS baseline's cold load is `ensureIndexesLoaded`'s job) are treated as - * live: never a false rebuild. A migrating provider is skipped (it owns its - * locked rebuild). - * @returns `'live'` when the index serves, `'rebuilt'` when a rebuild restored it. - */ - private async verifyVectorLive(): Promise<'live' | 'rebuilt'> { - if (this._vectorVerified) return 'live' - // Migration LOCK (#18): a migrating provider owns its in-place rebuild. - if (this.providerIsMigrating(this.index)) return 'live' - // Re-entrancy: rebuild() can trigger reads that call back into this guard. - if (this._vectorVerifying) return 'live' - this._vectorVerifying = true - try { - // ── Strategy 1: honest isReady() signal (native provider) ────────────── - const readiness = assessIndexReadiness(this.index) - if (readiness !== 'unknown') { - if (readiness === 'ready') { - this._vectorVerified = true - return 'live' - } - // Not ready: the serving structure did not load on open. Rebuild. - if (!this.config.silent) { - console.warn( - `[Brainy] Vector index reports not-ready (isReady() === false) — the persisted ` + - `vector index did not load on open. Rebuilding from storage…` - ) - } - await this.index.rebuild() - if (assessIndexReadiness(this.index) === 'ready') { - this._vectorVerified = true - return 'rebuilt' - } - throw new VectorIndexNotReadyError( - `Vector index reports not-ready even after a rebuild — semantic find({ query }) and ` + - `proximity search cannot be served reliably for this brain (a silent empty result ` + - `would misrepresent existing data).` - ) - } - - // ── Strategy 2: known-vector probe (providers without isReady()) ─────── - const claimed = this.index.size() - if (!claimed || claimed <= 0) return 'live' // JS cold path is ensureIndexesLoaded's job - - const probe = await this.pickVectorProbe() - if (!probe) { - // Empty store, or nothing with a probeable vector — inconclusive. - this._vectorVerified = true - return 'live' - } - const p = probe - - const probeServes = async (): Promise => { - // The failure mode we guard is the SILENT EMPTY result: a cold index that - // loaded its COUNT but not its serving structure returns `[]` for a - // known-present vector, while a warm index returns at least one hit. We - // check for a NON-EMPTY result, NOT an exact self-match — HNSW is - // approximate and `get()` may return a re-hydrated/normalized vector, so - // demanding the exact self as top-1 would false-positive on a perfectly - // healthy index (and wrongly rebuild → throw). - const hits = await this.index.search(p.vector, 1) - return hits.length > 0 - } - void p.id // probe keyed on the vector; id retained for diagnostics only - - if (await probeServes()) { - this._vectorVerified = true - return 'live' // serving structure is live — the common case - } - - if (!this.config.silent) { - console.warn( - `[Brainy] Vector index reports ${claimed} vector(s) but a known persisted vector ` + - `returns no results — the serving structure did not load on open. Rebuilding…` - ) - } - await this.index.rebuild() - - if (await probeServes()) { - this._vectorVerified = true - return 'rebuilt' - } - throw new VectorIndexNotReadyError( - `Vector index reports ${claimed} vector(s) but a known persisted vector returns no ` + - `results even after a rebuild — semantic find({ query }) cannot be served reliably ` + - `for this brain (a silent empty result would misrepresent existing data).` - ) - } catch (err) { - if (err instanceof VectorIndexNotReadyError) throw err - // A transient probe/rebuild failure must not break the query NOR mask as - // "no data". Allow a re-check on the next vector read and fall through. - this._vectorVerified = false - if (!this.config.silent) { - console.warn(`[Brainy] Vector consistency check skipped (transient): ${err}`) - } - return 'live' - } finally { - this._vectorVerifying = false - } - } - - /** - * @description Sample a KNOWN persisted noun and hydrate its vector, to probe - * the vector index with. `get()` omits vectors by default, so this passes - * `{ includeVectors: true }`. Samples a few (a system-only / vectorless entity - * must not make every open inconclusive). Returns `null` when nothing has a - * probeable vector. - */ - private async pickVectorProbe(): Promise<{ id: string; vector: number[] } | null> { - const sample = await this.storage.getNouns({ pagination: { limit: 5, offset: 0 } }) - for (const noun of sample.items ?? []) { - const id = (noun as { id?: string }).id - if (!id) continue - const full = await this.get(id, { includeVectors: true }) - const vector = (full as { vector?: number[] } | null)?.vector - if (Array.isArray(vector) && vector.length > 0) { - return { id, vector } - } - } - return null - } - // ------------------------------------------------------------------------- /** @@ -4083,7 +3392,7 @@ export class Brainy implements BrainyInterface { // Operation 3: Add to graph index for O(1) lookups tx.addOperation( new AddToGraphIndexOperation( - this.graphIndex, verb, { sourceInt, targetInt }, + this.graphIndex, verb, sourceInt, targetInt, this.graphWriteGeneration, (verbInt) => this.cacheVerbInt(verbInt, id) ) @@ -4121,50 +3430,13 @@ export class Brainy implements BrainyInterface { // Operation 6: Add reverse relationship to graph index tx.addOperation( new AddToGraphIndexOperation( - this.graphIndex, reverseVerb, { sourceInt: targetInt, targetInt: sourceInt }, + this.graphIndex, reverseVerb, targetInt, sourceInt, this.graphWriteGeneration, (verbInt) => this.cacheVerbInt(verbInt, reverseId) ) ) } - }, - undefined, - this._changeFeed.hasListeners - ? [ - { - kind: 'relation', - op: 'relate', - id, - relation: { - id, - from: params.from, - to: params.to, - type: String(params.type), - ...(params.metadata && { - metadata: params.metadata as Record - }) - } - }, - ...(reverseId - ? [ - { - kind: 'relation', - op: 'relate', - id: reverseId, - relation: { - id: reverseId, - from: params.to, - to: params.from, - type: String(params.type), - ...(params.metadata && { - metadata: params.metadata as Record - }) - } - } as PendingChangeEvent - ] - : []) - ] - : undefined) + }) return id } @@ -4203,7 +3475,7 @@ export class Brainy implements BrainyInterface { if (verb && endpointInts) { tx.addOperation( new RemoveFromGraphIndexOperation( - this.graphIndex, verb, endpointInts, this.graphWriteGeneration + this.graphIndex, verb, endpointInts.sourceInt, endpointInts.targetInt, this.graphWriteGeneration ) ) } @@ -4212,26 +3484,7 @@ export class Brainy implements BrainyInterface { tx.addOperation( new DeleteVerbMetadataOperation(this.storage, id) ) - }, - undefined, - this._changeFeed.hasListeners && verb - ? [ - { - kind: 'relation', - op: 'unrelate', - id, - relation: { - id, - from: verb.sourceId, - to: verb.targetId, - type: String(verb.verb), - ...(verb.metadata && { - metadata: verb.metadata as Record - }) - } - } - ] - : undefined) + }) } /** @@ -4355,37 +3608,18 @@ export class Brainy implements BrainyInterface { if (typeChanged && reindexInts) { tx.addOperation( new RemoveFromGraphIndexOperation( - this.graphIndex, existing, reindexInts, this.graphWriteGeneration + this.graphIndex, existing, reindexInts.sourceInt, reindexInts.targetInt, this.graphWriteGeneration ) ) tx.addOperation( new AddToGraphIndexOperation( - this.graphIndex, verbForIndex, reindexInts, + this.graphIndex, verbForIndex, reindexInts.sourceInt, reindexInts.targetInt, this.graphWriteGeneration, (verbInt) => this.cacheVerbInt(verbInt, params.id) ) ) } - }, - undefined, - this._changeFeed.hasListeners - ? [ - { - kind: 'relation', - op: 'updateRelation', - id: params.id, - relation: { - id: params.id, - from: verbForIndex.sourceId, - to: verbForIndex.targetId, - type: String(verbForIndex.verb ?? verbForIndex.type), - ...(verbForIndex.metadata && { - metadata: verbForIndex.metadata as Record - }) - } - } - ] - : undefined) + }) } /** @@ -4424,9 +3658,7 @@ export class Brainy implements BrainyInterface { async related( paramsOrId?: string | RelatedParams ): Promise[]> { - // Graph read: relationship traversal consults only the graph adjacency - // family, so it waits on a graph migration but not on vector/metadata. - await this.ensureInitialized({ needs: ['graph'] }) + await this.ensureInitialized() await this.verifyGraphAdjacencyLive() // Handle string ID shorthand: related(id) -> related({ from: id }) @@ -5868,10 +5100,6 @@ export class Brainy implements BrainyInterface { ): Promise { await this.ensureInitialized() this.ensureAggregationIndex() - // Persisted definitions load asynchronously — wait for them before deciding - // the aggregate doesn't exist (an app that relies on persisted definitions - // without re-defining at boot would otherwise race a spurious throw here). - await this._aggregationIndex!.ready() if (!this._aggregationIndex!.hasAggregate(name)) { throw new Error(`Aggregate '${name}' is not defined. Call defineAggregate() first.`) } @@ -5890,17 +5118,8 @@ export class Brainy implements BrainyInterface { this._aggregationIndex = new AggregationIndex(this.storage, nativeProvider) // Note: init() is async but definitions can be registered synchronously. // State loading happens lazily on first query. - this._aggregationIndex.init().catch((err) => { - // Non-fatal — definitions still work and state backfills on first query — - // but a failed state load means aggregates read empty/stale until then, so - // surface it loudly rather than swallow it. - if (!this.config.silent) { - prodLog.warn( - `[Brainy] Aggregation index state failed to load at init ` + - `(${(err as Error).message}). Aggregates may read empty until a ` + - `backfill-on-query repopulates them.` - ) - } + this._aggregationIndex.init().catch(() => { + // Non-fatal — aggregation state will be empty but definitions still work }) } @@ -5969,11 +5188,7 @@ export class Brainy implements BrainyInterface { * }) */ async find(query: string | FindParams): Promise[]> { - // Init only here — the migration gate is applied family-scoped below, once - // the query params reveal which index families this read actually consults - // (a string query may parse to a where / connected / vector shape). The lazy - // loader and cold-read probes below already defer to a migrating provider. - await this.ensureInitialized({ needs: [] }) + await this.ensureInitialized() // Ensure indexes are loaded (lazy loading when disableAutoRebuild: true) // This is a production-safe, concurrency-controlled lazy load @@ -5984,12 +5199,6 @@ export class Brainy implements BrainyInterface { // metadata counterpart of the graph cold-load guard. No-op for the JS index. await this.ensureMetadataConsistencyProbed() - // Loudly flag a degraded derived index (failed init rebuild, or an - // adopt-forward degraded commit) so a partial result is never mistaken for - // authoritative. The streaming search() generator delegates to find(), so it - // is covered here. - this.warnIfReadsDegraded('find') - // Parse natural language queries let params: FindParams = typeof query === 'string' ? await this.parseNaturalQuery(query) : query @@ -6012,13 +5221,6 @@ export class Brainy implements BrainyInterface { // Zero-config validation (static import for performance) validateFindParams(params) - // Family-scoped migration gate: wait only on the index families THIS query - // consults, so a filter or graph query served from a healthy family never - // blocks on an unrelated family's one-time 7.x → 8.0 migration. Placed once - // params are final (after natural-language parse + connected-id resolution) - // and before the aggregate path, which carries no gate of its own. - await this.awaitMigrationLock(this.queryIndexFamilies(params)) - // Aggregate query path — early return when params.aggregate is set if (params.aggregate) { return this.findAggregate(params) @@ -6710,13 +5912,10 @@ export class Brainy implements BrainyInterface { /** * Add multiple entities in a single batch operation * - * Uses batch embedding (embedBatch) to pre-compute all vectors before any - * storage write, keeping model inference out of the per-item write path. - * (On the default WASM engine, batch throughput is comparable to sequential - * embed() calls — measured ~160 ms/text either way; a native embedding - * provider may batch faster.) Automatically adapts batch size and - * parallelism to the storage adapter (e.g., smaller batches for cloud - * storage). + * Uses batch embedding (embedBatch) to pre-compute all vectors in a single + * WASM forward pass instead of N individual embed() calls, providing 5-10x + * speedup on bulk inserts. Automatically adapts batch size and parallelism + * to the storage adapter (e.g., smaller batches for cloud storage). * * @param params - Batch add parameters * @param params.items - Array of AddParams (same shape as brain.add()) @@ -6914,30 +6113,6 @@ export class Brainy implements BrainyInterface { this.assertWritable('removeMany') await this.ensureInitialized() - // Loud selector validation: a call with no usable selector used to - // resolve successfully having deleted NOTHING (total: 0) — the classic - // silent no-op being a bare array passed positionally - // (removeMany([id]) instead of removeMany({ ids: [id] })). The caller - // believes the delete happened; every count derived afterwards is "wrong" - // while the engine was never even asked. Refuse instead. - if (Array.isArray(params)) { - throw new Error( - `removeMany() takes a params object, not a bare array — use removeMany({ ids: [...] })` - ) - } - if (!params || (!params.ids && !params.type && !params.where)) { - throw new Error( - `removeMany() requires a selector: { ids } and/or { type, where }. ` + - `An empty selector would silently delete nothing — refusing.` - ) - } - if (params.ids && params.ids.length === 0) { - throw new Error( - `removeMany() received ids: [] — an empty id list deletes nothing. ` + - `Pass the ids to delete, or omit ids and select by { type, where }.` - ) - } - // Determine what to delete let idsToDelete: string[] = [] @@ -7001,16 +6176,6 @@ export class Brainy implements BrainyInterface { } try { - // Change feed: the BUILDER populates this (per id actually staged), so - // an id whose builder failed never emits — the array is read only - // after the chunk's commit succeeds. Verb events dedupe within the - // chunk (two related chunk-members see the same cascade verb twice). - const chunkEvents: PendingChangeEvent[] | undefined = this._changeFeed - .hasListeners - ? [] - : undefined - const seenVerbEvents = new Set() - // Process chunk as ONE atomic Model-B generation (entities + cascade verbs). await this.persistSingleOp({ nouns: touchedNouns, verbs: touchedVerbs }, async (tx) => { for (const id of chunk) { @@ -7035,51 +6200,20 @@ export class Brainy implements BrainyInterface { ) } - // Pre-read metadata rides along: the count decrement must not - // depend on re-reading the record being removed (see remove()). tx.addOperation( - new DeleteNounMetadataOperation(this.storage, id, metadata) + new DeleteNounMetadataOperation(this.storage, id) ) for (const verb of allVerbs) { const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb) tx.addOperation( - new RemoveFromGraphIndexOperation(this.graphIndex, verb, { sourceInt, targetInt }, this.graphWriteGeneration) + new RemoveFromGraphIndexOperation(this.graphIndex, verb, sourceInt, targetInt, this.graphWriteGeneration) ) tx.addOperation( new DeleteVerbMetadataOperation(this.storage, verb.id) ) } - if (chunkEvents) { - chunkEvents.push({ - kind: 'entity', - op: 'remove', - id, - ...(metadata && { - entity: this.entityViewFromRawRecord( - id, - metadata as Record - ) - }) - }) - for (const verb of allVerbs) { - if (seenVerbEvents.has(verb.id)) continue - seenVerbEvents.add(verb.id) - chunkEvents.push({ - kind: 'relation', - op: 'unrelate', - id: verb.id, - relation: { - id: verb.id, - from: verb.sourceId, - to: verb.targetId, - type: String(verb.verb) - } - }) - } - } - chunkQueued.push(id) } catch (error) { chunkBuilderFailed.push({ @@ -7091,7 +6225,7 @@ export class Brainy implements BrainyInterface { } } } - }, undefined, chunkEvents) + }) // Transaction committed — queued IDs were actually deleted result.successful.push(...chunkQueued) @@ -7397,12 +6531,6 @@ export class Brainy implements BrainyInterface { // VFS was never used, reset flag for clean state this._vfsInitialized = false } - - // Change feed: clear() wipes the store wholesale (raw, not per-record) — - // one store-level event tells subscribers everything they held is gone. - this._changeFeed.emit([ - { kind: 'store', op: 'clear', timestamp: Date.now() } - ]) } // ─── Migration API ─────────────────────────────────────────────── @@ -7596,59 +6724,6 @@ export class Brainy implements BrainyInterface { await this.removeMigrationBackupSafe() } - /** - * @description Open a sequential scan over the generation FACT LOG — the - * append-only record of every committed generation as an AFTER-IMAGE fact - * (what each touched entity/relationship became, or a body-less tombstone - * for a removal). The scan is the streaming substrate for index heals and - * incremental replays: one sequential read in commit order replaces a - * per-entity directory walk. The handle carries heal-narration telemetry - * (`headGeneration` / `segmentCount` / `approxFactCount` up front; ordered - * batches each stamped with their generation range, byte size, and segment; - * a `summary()` cross-check after iteration). A detected gap or damaged - * segment ABORTS the scan loudly — never a silent skip. - * - * Returns `null` when this store hosts no fact log: the storage adapter - * lacks the binary append primitives, or the brain predates the fact log - * (its history began before dual-write — facts exist only from the first - * write after upgrade; callers fall back to the canonical enumeration walk). - * - * @param options - `fromGeneration`/`toGeneration` bound the scan (inclusive - * both ends); `kinds` filters ops to `'noun'`/`'verb'`; `batchSize` caps - * facts per yielded batch (default 256). - * @returns The scan handle, or `null` when no fact log exists. - * @example - * const scan = brain.scanFacts({ fromGeneration: 1 }) - * if (scan) { - * for await (const batch of scan.batches()) { - * for (const fact of batch.facts) { - * // fact.ops: [{ kind, id, record | null (tombstone) }, ...] - * } - * } - * } - */ - scanFacts(options?: { - fromGeneration?: number - toGeneration?: number - kinds?: Array<'noun' | 'verb'> - batchSize?: number - }): FactScanHandle | null { - const factLog = this.generationStore?.getFactLog() - return factLog ? factLog.scanFacts(options) : null - } - - /** - * @description The immutable, sealed fact-log segment files covering - * `fromGeneration` — the zero-copy handoff for consumers that map segment - * files directly instead of streaming {@link scanFacts} batches. The - * append-mutable TAIL segment is deliberately excluded (read it via - * `scanFacts`). Paths are storage-root-relative. Empty when no fact log - * exists or nothing is sealed yet. - */ - factSegmentPaths(options?: { fromGeneration?: number }): string[] { - return this.generationStore?.getFactLog()?.segmentPaths(options) ?? [] - } - /** * @description Read the reified transaction log — one entry per committed * generation, carrying the committed generation, the commit timestamp, and @@ -7826,78 +6901,18 @@ export class Brainy implements BrainyInterface { const plan = await this.planTransact(ops) - // Authoritative per-op ifRev CAS, run by the generation store UNDER the - // commit mutex against the just-read before-images — atomic with the - // apply (the planning-time checks above are fast-fails that can - // interleave with concurrent writers). Any conflict rejects the WHOLE - // batch before anything is staged or applied. Sequencing rules: - // - runningRev tracks multiple updates to the SAME entity in one batch - // (each op CASes against — and bumps — its predecessor's result). - // - An entity the batch itself creates (null before-image, forward-ref - // add+update) keeps its plan-sequenced stamps: no concurrent writer - // can have moved a rev that did not exist, so plan-time was exact. - const casPrecommit = (before: CommitBeforeImages): void => { - const runningRev = new Map() - for (const cas of plan.casUpdates) { - // An id this batch adds owns its revision baseline (the add restamps - // `_rev: 1` even on overwrite) — plan-time sequencing is exact, no - // concurrent writer can move a baseline the batch itself sets. - if (plan.createdNouns.has(cas.id)) { - continue - } - const beforeMeta = before.nouns.get(cas.id)?.metadata as - | { _rev?: unknown } - | null - | undefined - if (!beforeMeta && !runningRev.has(cas.id)) { - // A pre-existing entity a concurrent remove() deleted after - // planning: a CAS caller gets the truth; a plain update keeps its - // last-writer-wins re-create semantics (plan-stamped rev). - if (typeof cas.ifRev === 'number') { - throw new EntityNotFoundError(cas.id) + const { generation, timestamp } = await this.generationStore.commitTransaction({ + touched: { nouns: plan.touchedNouns, verbs: plan.touchedVerbs }, + meta: options?.meta, + ifAtGeneration: options?.ifAtGeneration, + execute: async () => { + await this.transactionManager.executeTransaction(async (tx) => { + for (const operation of plan.operations) { + tx.addOperation(operation) } - continue - } - const base = - runningRev.get(cas.id) ?? - (typeof beforeMeta?._rev === 'number' ? beforeMeta._rev : 1) - if (typeof cas.ifRev === 'number' && cas.ifRev !== base) { - throw new RevisionConflictError(cas.id, cas.ifRev, base) - } - const next = base + 1 - cas.updatedMetadata._rev = next - runningRev.set(cas.id, next) + }) } - } - - let generation: number - let timestamp: number - try { - ;({ generation, timestamp } = await this.generationStore.commitTransaction({ - touched: { nouns: plan.touchedNouns, verbs: plan.touchedVerbs }, - meta: options?.meta, - ifAtGeneration: options?.ifAtGeneration, - precommit: casPrecommit, - execute: async () => { - await this.transactionManager.executeTransaction( - async (tx) => { - for (const operation of plan.operations) { - tx.addOperation(operation) - } - }, - // Budget scales with the batch (or the caller's explicit - // timeoutMs): a flat 30s cap silently limited honest bulk work - // to ~15 ops on network disks (~2s/op measured in the field). - { timeout: transactTimeoutBudget(plan.operations.length, options?.timeoutMs) } - ) - } - })) - } catch (err) { - // A batch whose rollback failed and left canonical records unreconciled - // quarantines writes until repairIndex() reconciles and lifts it. - if (err instanceof StoreInconsistentError) this.storeInconsistency = err - throw err - } + }) // Aggregation-index maintenance: derived data, applied after the commit // point — exactly where the single-operation methods apply it. @@ -7905,10 +6920,6 @@ export class Brainy implements BrainyInterface { hook() } - // Change feed: the batch's events share its single committed generation. - // A rejected batch throws at commitTransaction and never reaches here. - this.emitCommitted(plan.changeEvents, undefined, generation, timestamp) - const receipt: TransactReceipt = { generation, timestamp, ids: plan.ids } return this.createPinnedDb({ generation, timestamp, receipt }) } @@ -8265,72 +7276,6 @@ export class Brainy implements BrainyInterface { return this.generationStore.compact(options) } - /** - * @description Repack cold generation history into sealed segments — - * re-representation, never deletion: every record and delta stays readable - * (`asOf()` unchanged); the physical file count drops by orders of - * magnitude. Runs automatically (time-bounded) at `close()`; call this for - * explicit maintenance windows on long-lived writers. The ONLY history - * transform permitted under the archival profile (`retention: 'all'`). - * @param options - `timeBudgetMs` bounds the pass (early stop = consistent - * prefix, next pass resumes); `batchGenerations` sizes each fold. - * @returns Folded generation count and segments created. - */ - async repackHistory(options?: { - timeBudgetMs?: number - batchGenerations?: number - }): Promise<{ foldedGenerations: number; segmentsCreated: number }> { - this.assertWritable('repackHistory') - await this.ensureInitialized() - await this.generationStore.flushPendingSingleOps() - return this.generationStore.repackHistory(options) - } - - /** - * @description Read-only generational-history footprint for fleet audits: - * generation count, total on-disk bytes, generation/timestamp range, the - * compaction horizon, and the retention policy in force. Touches no data and - * changes nothing. First call pays one walk over committed deltas to seed - * the running byte total (subsequent calls — and every adaptive retention - * check — are then O(1)). - * - * A pool operator's exposure check is one call per brain: - * @example - * const stats = await brain.historyStats() - * console.log(`${stats.generations} generations, ${stats.bytes} bytes, mode=${stats.retentionMode}`) - */ - async historyStats(): Promise { - await this.ensureInitialized() - const stats = await this.generationStore.historyStats() - const policy = this.resolveRetentionPolicy() - return { - ...stats, - retentionMode: policy.mode, - effectiveBudgetBytes: - policy.mode === 'adaptive' - ? this.adaptiveHistoryBudgetBytes(policy.budgetBytes) - : null - } - } - - /** - * @description A deterministic content digest of the generation log through - * `g` (D8 — gate-to-generation provenance): identical history produces the - * identical digest on any machine; divergence produces a different one. - * Release gates and suite verdicts pin `{generation, digest}` and verify - * both at execution time instead of pinning a git commit. O(segments + - * live-tier window), never O(all generations). Throws `RangeError` out of - * range and `GenerationCompactedError` below the horizon — a gate can - * never silently pin reclaimed history. - * @example - * const gate = { generation: brain.generation(), digest: await brain.generationDigest(brain.generation()) } - */ - async generationDigest(g: number): Promise { - await this.ensureInitialized() - await this.generationStore.flushPendingSingleOps() - return this.generationStore.generationDigest(g) - } - /** * @description Drive the adaptive retention byte budget at runtime — the * settable input a machine-level coordinator (e.g. cor's `ResourceManager`, @@ -8410,24 +7355,19 @@ export class Brainy implements BrainyInterface { /** * @description Run history compaction under the resolved `retention` policy - * when `autoCompact` is on (the default). Invoked from `close()` ONLY - * (8.9.0) — flush() is durability work and never pays maintenance costs; a - * production deployment measured reclaim-on-flush blocking single writes - * for 25-191s under memory pressure. Long-lived writers that never close - * accumulate history until their next explicit `compactHistory()` — the - * documented trade: predictable writes, explicit maintenance. + * when `autoCompact` is on (the default). Invoked from `flush()` and + * `close()` so generational record-sets cannot accumulate unbounded across a + * long-lived writer's lifetime. * * - `'all'` → returns without reclaiming (index compaction for speed still * runs elsewhere; history is decoupled and kept). * - `'adaptive'` → reclaim oldest-unpinned history down to the byte budget. * - `'explicit'` → apply the supplied `maxGenerations`/`maxAge`/`maxBytes` caps. * - * Every auto pass is TIME-BOUNDED ({@link CLOSE_COMPACTION_BUDGET_MS}) so a - * large backlog can never stall a clean shutdown; the next pass resumes. * Read-only instances and an explicit `autoCompact: false` skip silently. * Pinned generations are never reclaimed ({@link GenerationStore.compact}). * Failures are logged and swallowed — compaction is housekeeping and must - * never fail a clean shutdown. + * never fail a flush or a clean shutdown. */ private async autoCompactHistory(): Promise { // Nothing to compact on a read-only instance or before init wired up the @@ -8443,16 +7383,12 @@ export class Brainy implements BrainyInterface { if (policy.mode === 'adaptive') { const budget = this.adaptiveHistoryBudgetBytes(policy.budgetBytes) if (budget === Infinity) return // no pressure signal → keep everything this pass - await this.generationStore.compact({ - maxBytes: budget, - timeBudgetMs: CLOSE_COMPACTION_BUDGET_MS - }) + await this.generationStore.compact({ maxBytes: budget }) } else { await this.generationStore.compact({ maxGenerations: policy.maxGenerations, maxAge: policy.maxAge, - maxBytes: policy.maxBytes, - timeBudgetMs: CLOSE_COMPACTION_BUDGET_MS + maxBytes: policy.maxBytes }) } } catch (error) { @@ -8477,14 +7413,6 @@ export class Brainy implements BrainyInterface { * replacement. Release them first; a warning is logged when live pins * exist. * - * NON-DESTRUCTIVE STAGING + SPARSE-AWARE: the snapshot is copied into a - * staging area BEFORE any live data is touched (all-zero blocks stay - * holes, so a sparse mmap store restores at its true allocated size — not - * its apparent size). Any copy failure, ENOSPC included, removes only the - * staging and throws; the live store is untouched. Only after the copy - * succeeds does an atomic per-entry swap move it into place; a crash - * mid-swap completes FORWARD on the next open. - * * @param path - Snapshot directory to restore from. * @param options - Must be `{ confirm: true }` — an explicit acknowledgment * that current state is destroyed. @@ -8535,12 +7463,6 @@ export class Brainy implements BrainyInterface { this.index.rebuild(), this.graphIndex.rebuild() ]) - - // Change feed: a restore is a wholesale raw-state replacement, not a - // per-record commit — one store-level event tells subscribers to refetch. - this._changeFeed.emit([ - { kind: 'store', op: 'restore', timestamp: Date.now() } - ]) } /** @@ -9123,10 +8045,7 @@ export class Brainy implements BrainyInterface { ids: [], touchedNouns: [], touchedVerbs: [], - postCommit: [], - casUpdates: [], - createdNouns: new Set(), - changeEvents: [] + postCommit: [] } for (const op of ops) { @@ -9261,11 +8180,6 @@ export class Brainy implements BrainyInterface { ? !state.nouns.has(id) && (await this.storage.getNounMetadata(id)) === null : true - // Either way (fresh create OR overwrite) this add resets the id's revision - // baseline (`_rev: 1` below), so later in-batch updates to it sequence - // against batch-local state — see PlannedTransact.createdNouns. - plan.createdNouns.add(id) - const now = Date.now() const storageMetadata = { ...params.metadata, @@ -9320,20 +8234,6 @@ export class Brainy implements BrainyInterface { this._aggregationIndex.onEntityAdded(id, entityForIndexing) } }) - if (this._changeFeed.hasListeners) { - plan.changeEvents.push({ - kind: 'entity', - op: 'add', - id, - entity: { - id, - type: String(params.type), - ...(params.subtype !== undefined && { subtype: String(params.subtype) }), - metadata: (params.metadata as Record) ?? {}, - ...(params.service !== undefined && { service: String(params.service) }) - } - }) - } state.nouns.set(id, { metadata: storageMetadata, vector }) state.removedNouns.delete(id) @@ -9373,10 +8273,8 @@ export class Brainy implements BrainyInterface { } // ifRev CAS — identical resolution to update(); a conflict rejects the - // WHOLE batch. This planning-time check is purely a FAST-FAIL (it can - // interleave with concurrent writers); the authoritative re-verify runs in - // transact()'s commit precondition under the commit mutex, via the - // plan.casUpdates entry registered below. + // WHOLE batch before anything is staged or applied. `_rev` is reserved: + // every read path surfaces it ONLY top-level. const currentRev = typeof existing._rev === 'number' ? existing._rev : 1 if (typeof params.ifRev === 'number' && params.ifRev !== currentRev) { throw new RevisionConflictError(params.id, params.ifRev, currentRev) @@ -9426,17 +8324,6 @@ export class Brainy implements BrainyInterface { visibility: params.visibility ?? existing.visibility }) } - - // Register for the authoritative under-mutex CAS re-verify + rev re-stamp - // (see PlannedTransact.casUpdates). The staged UpdateNounMetadataOperation - // holds `updatedMetadata` by reference, so the precommit re-stamp lands in - // what execute() writes. - plan.casUpdates.push({ - id: params.id, - ...(typeof params.ifRev === 'number' && { ifRev: params.ifRev }), - updatedMetadata - }) - const entityForIndexing = { id: params.id, vector, @@ -9489,32 +8376,17 @@ export class Brainy implements BrainyInterface { ) plan.touchedNouns.push(params.id) - // The full planGetEntity view, passed whole — a subset view makes the - // old-side decrement miss reserved-field groups (double-count on update). - const oldEntityForAgg = existing as unknown as Record + const oldEntityForAgg = { + type: existing.type, + service: existing.service, + data: existing.data, + metadata: existing.metadata + } plan.postCommit.push(() => { if (this._aggregationIndex) { this._aggregationIndex.onEntityUpdated(params.id, entityForIndexing, oldEntityForAgg) } }) - if (this._changeFeed.hasListeners) { - plan.changeEvents.push({ - kind: 'entity', - op: 'update', - id: params.id, - entity: { - id: params.id, - type: String(entityForIndexing.type), - ...(entityForIndexing.subtype !== undefined && { - subtype: String(entityForIndexing.subtype) - }), - metadata: (newMetadata as Record) ?? {}, - ...(entityForIndexing.service !== undefined && { - service: String(entityForIndexing.service) - }) - } - }) - } state.nouns.set(params.id, { metadata: updatedMetadata, vector }) return params.id @@ -9570,51 +8442,28 @@ export class Brainy implements BrainyInterface { if (metadata) { plan.operations.push(new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata)) } - // Pre-read metadata rides along: the count decrement must not depend on - // re-reading the record being removed (see remove()). - plan.operations.push(new DeleteNounMetadataOperation(this.storage, id, metadata)) + plan.operations.push(new DeleteNounMetadataOperation(this.storage, id)) for (const verb of cascade.values()) { + const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb) plan.operations.push( - // Endpoint ints resolve at EXECUTE time — a cascade verb (or its - // endpoints) may have been created earlier in this same batch, so a - // plan-time resolution would ask the id mapper about entities that do - // not exist yet (the native mapper rightly refuses). - new RemoveFromGraphIndexOperation(this.graphIndex, verb, () => this.resolveVerbEndpointInts(verb), this.graphWriteGeneration), + new RemoveFromGraphIndexOperation(this.graphIndex, verb, sourceInt, targetInt, this.graphWriteGeneration), new DeleteVerbMetadataOperation(this.storage, verb.id) ) plan.touchedVerbs.push(verb.id) state.verbs.delete(verb.id) state.removedVerbs.add(verb.id) - if (this._changeFeed.hasListeners) { - plan.changeEvents.push({ - kind: 'relation', - op: 'unrelate', - id: verb.id, - relation: { - id: verb.id, - from: verb.sourceId, - to: verb.targetId, - type: String(verb.verb) - } - }) - } } plan.touchedNouns.push(id) - if (this._changeFeed.hasListeners) { - plan.changeEvents.push({ - kind: 'entity', - op: 'remove', - id, - ...(metadata && { - entity: this.entityViewFromRawRecord(id, metadata as Record) - }) - }) - } if (metadata) { - // Mirror of remove()'s aggregation hook — the FULL reserved view, so - // reserved-field groupBy decrements find their group. - const entityForAgg = this.entityForAggFromRawRecord(metadata as Record) + // Canonical reserved/custom split — mirror of remove()'s aggregation hook. + const { reserved, custom } = splitNounMetadataRecord(metadata) + const entityForAgg = { + type: reserved.noun, + service: reserved.service, + data: reserved.data, + metadata: custom + } plan.postCommit.push(() => { if (this._aggregationIndex) { this._aggregationIndex.onEntityDeleted(id, entityForAgg) @@ -9721,6 +8570,8 @@ export class Brainy implements BrainyInterface { data: params.data, createdAt: now } + const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb) + plan.operations.push( new SaveVerbOperation(this.storage, { id, @@ -9731,31 +8582,13 @@ export class Brainy implements BrainyInterface { targetId: params.to }), new SaveVerbMetadataOperation(this.storage, id, verbMetadata), - // Endpoint ints resolve at EXECUTE time — after any same-batch add of - // an endpoint has applied. Plan-time resolution was a consumer-reported - // native/JS parity bug: transact([add X, relate →X]) asked the native - // id mapper to assign an int for an entity that did not exist yet. - new AddToGraphIndexOperation(this.graphIndex, verb, () => this.resolveVerbEndpointInts(verb), this.graphWriteGeneration, (verbInt) => + new AddToGraphIndexOperation(this.graphIndex, verb, sourceInt, targetInt, this.graphWriteGeneration, (verbInt) => this.cacheVerbInt(verbInt, id) ) ) plan.touchedVerbs.push(id) state.verbs.set(id, verb) state.removedVerbs.delete(id) - if (this._changeFeed.hasListeners) { - plan.changeEvents.push({ - kind: 'relation', - op: 'relate', - id, - relation: { - id, - from: params.from, - to: params.to, - type: String(params.type), - ...(params.metadata && { metadata: params.metadata as Record }) - } - }) - } if (params.bidirectional) { const reverseId = uuidv4() @@ -9763,9 +8596,9 @@ export class Brainy implements BrainyInterface { ...verb, id: reverseId, sourceId: params.to, - targetId: params.from - // sourceInt/targetInt are mirrored onto this object when the graph - // operation resolves endpoints at execute time. + targetId: params.from, + sourceInt: targetInt, + targetInt: sourceInt } plan.operations.push( new SaveVerbOperation(this.storage, { @@ -9777,27 +8610,13 @@ export class Brainy implements BrainyInterface { targetId: params.from }), new SaveVerbMetadataOperation(this.storage, reverseId, verbMetadata), - new AddToGraphIndexOperation(this.graphIndex, reverseVerb, () => this.resolveVerbEndpointInts(reverseVerb), this.graphWriteGeneration, (verbInt) => + new AddToGraphIndexOperation(this.graphIndex, reverseVerb, targetInt, sourceInt, this.graphWriteGeneration, (verbInt) => this.cacheVerbInt(verbInt, reverseId) ) ) plan.touchedVerbs.push(reverseId) state.verbs.set(reverseId, reverseVerb) state.removedVerbs.delete(reverseId) - if (this._changeFeed.hasListeners) { - plan.changeEvents.push({ - kind: 'relation', - op: 'relate', - id: reverseId, - relation: { - id: reverseId, - from: params.to, - to: params.from, - type: String(params.type), - ...(params.metadata && { metadata: params.metadata as Record }) - } - }) - } } return id @@ -9819,29 +8638,13 @@ export class Brainy implements BrainyInterface { : (state.verbs.get(id) ?? (await this.storage.getVerb(id))) if (verb) { + const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb) plan.operations.push( - // Endpoint ints resolve at EXECUTE time — the verb (or its endpoints) - // may have been created earlier in this same batch (forward refs). - new RemoveFromGraphIndexOperation(this.graphIndex, verb, () => this.resolveVerbEndpointInts(verb), this.graphWriteGeneration) + new RemoveFromGraphIndexOperation(this.graphIndex, verb, sourceInt, targetInt, this.graphWriteGeneration) ) } plan.operations.push(new DeleteVerbMetadataOperation(this.storage, id)) plan.touchedVerbs.push(id) - if (this._changeFeed.hasListeners) { - plan.changeEvents.push({ - kind: 'relation', - op: 'unrelate', - id, - ...(verb && { - relation: { - id, - from: verb.sourceId, - to: verb.targetId, - type: String(verb.verb) - } - }) - }) - } state.verbs.delete(id) state.removedVerbs.add(id) @@ -10249,30 +9052,6 @@ export class Brainy implements BrainyInterface { return await coordinator.import(source as Buffer | string | object, options) } - /** Brain-owned background deduplicator (lazy; see getBackgroundDeduplicator). */ - private _backgroundDedup?: import('./import/BackgroundDeduplicator.js').BackgroundDeduplicator - - /** - * The single brain-owned BackgroundDeduplicator, lazily constructed. - * - * Ownership matters here: the post-import dedup timer must outlive the - * per-call ImportCoordinator but never the brain. One instance per brain - * restores the intended cross-import debounce (per-coordinator instances - * each armed their own timer, so the "debounce" never spanned imports) and - * gives close() a handle to cancel pending work — a delete pass must never - * fire against a closed brain. - * @internal - */ - async getBackgroundDeduplicator(): Promise< - import('./import/BackgroundDeduplicator.js').BackgroundDeduplicator - > { - if (!this._backgroundDedup) { - const { BackgroundDeduplicator } = await import('./import/BackgroundDeduplicator.js') - this._backgroundDedup = new BackgroundDeduplicator(this) - } - return this._backgroundDedup - } - /** * Virtual File System API - Knowledge Operating System * @@ -10355,43 +9134,6 @@ export class Brainy implements BrainyInterface { return this._hub } - /** - * @description Subscribe to this brain's committed mutations — the - * authoritative in-process change feed. Fires exactly once per affected - * record for EVERY canonical write, regardless of origin: direct API calls, - * batch methods, `transact()`, imports, the Virtual Filesystem, and - * native-accelerated deployments all funnel through the same commit point - * this feed is emitted from. - * - * Delivery contract (see {@link BrainyChangeEvent}): - * - **Post-commit only** — an aborted write (e.g. a losing `ifRev` - * compare-and-swap) never emits. - * - **Commit-ordered, asynchronous** — events arrive in the order writes - * became durable, dispatched in a microtask so a slow listener never - * delays a write. A throwing listener is isolated (logged, others - * unaffected). - * - **Fully described** — `remove`/`unrelate` events carry the record's - * LAST committed state, and batch operations emit one event per item. - * - **Fire-and-forget** — no replay or backpressure. Each event carries its - * committed `generation`, so catch-up after a gap can be built on - * {@link transactionLog} / {@link asOf}. - * - **Zero overhead when unused** — with no subscribers the write path does - * no event work at all. - * - * @param listener - Called once per committed mutation. - * @returns An unsubscribe function — call it when done (e.g. on teardown of - * a pooled instance) to stop delivery. - * @example - * const off = brain.onChange((e) => { - * if (e.kind === 'entity') console.log(e.op, e.entity?.type, e.id) - * }) - * await brain.add({ data: 'hello', type: 'document' }) // → "add document " - * off() - */ - onChange(listener: ChangeListener): () => void { - return this._changeFeed.subscribe(listener) - } - /** * Get Triple Intelligence System * Advanced pattern recognition and relationship analysis @@ -10597,101 +9339,16 @@ export class Brainy implements BrainyInterface { this.generationStore.persistCounterNow() ]) - // NOTE (8.9.0): flush() no longer compacts history. Flush is DURABILITY - // work — it must cost what this window's deltas cost, never what the - // history backlog costs. Auto-compaction (a MAINTENANCE concern) runs at - // close() and via explicit compactHistory(); a production deployment - // measured reclaim-on-flush stalling single writes for 25-191s under - // memory pressure, which is exactly the class this separation ends. - - // 6. Stamp the entity tree: which source generation the canonical tree - // reflects + the rollup invariants that verify it whole (the counters - // persisted in step 1). Written at flush boundaries — the tree tracks - // every commit by construction, so the stamp is a durable checkpoint, - // not a per-commit cost. Open compares stamp vs log head + rollups. - await this.stampEntityTree() + // 6. Auto-compact generational history per config.retention (default on). + // Runs after the flush so durable state is in place; respects live + // Db pins and an explicit autoCompact: false. + await this.autoCompactHistory() const elapsed = Date.now() - startTime console.log(`All indexes flushed to disk in ${elapsed}ms`) } - /** - * @description Write the entity tree's FAMILY STAMP: `sourceGeneration` (the - * committed generation the canonical tree reflects — equal by construction, - * since the tree is written by the commit itself) plus the rollup invariants - * (entity/relationship counts) that verify the tree whole where per-file - * checks cannot scale. Verified at open by {@link verifyEntityTreeStamp}; - * healed by `repairIndex()`, whose unconditional recount rebuilds the - * rollups from a canonical walk and re-stamps. Best-effort: a stamp-write - * fault warns loudly but never fails the flush that carried real data. - */ - private async stampEntityTree(): Promise { - if (this.isReadOnly) return - try { - const [nounCount, verbCount] = await Promise.all([ - this.storage.getNounCount(), - this.storage.getVerbCount() - ]) - await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, { - family: 'entity-tree', - sourceGeneration: this.generationStore.generation(), - members: { mode: 'rollup', invariants: { nounCount, verbCount } } - }) - } catch (error) { - prodLog.warn( - `[Brainy] entity-tree stamp write failed (coherence checking degrades until the ` + - `next successful flush): ${(error as Error).message}` - ) - } - } - - /** - * @description Open-time coherence check for the entity tree's family stamp: - * compare `sourceGeneration` against the log head and the stamped rollup - * invariants against the live counters. Verdicts: - * - `coherent` / `absent` (legacy store; first flush stamps) → silent. - * - `behind` → benign for the tree (it is written BY the commit; only the - * stamp is stale — a crash landed between commit and flush). Refreshed at - * the next flush. - * - `incoherent` → LOUD: the tree or its counters diverged from what was - * stamped — `repairIndex()` recounts from canonical and re-stamps. - * Never blocks open; a fault reading the stamp is surfaced as unverifiable, - * never conflated with absence. - */ - private async verifyEntityTreeStamp(): Promise { - let stamp: FamilyStamp | null - try { - stamp = await readFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH) - } catch (error) { - prodLog.warn( - `[Brainy] entity-tree stamp is UNVERIFIABLE (read fault, not absence): ` + - `${(error as Error).message}` - ) - return - } - const [nounCount, verbCount] = await Promise.all([ - this.storage.getNounCount(), - this.storage.getVerbCount() - ]) - const verdict = verifyFamilyStamp(stamp, this.generationStore.generation(), { - nounCount, - verbCount - }) - if (verdict.state === 'incoherent') { - prodLog.warn( - `[Brainy] entity-tree stamp INCOHERENT at open: ${verdict.failures.join('; ')}. ` + - `The canonical tree or its counters diverged from the stamped state — run ` + - `brain.repairIndex() to recount from canonical and re-stamp.` - ) - } else if (verdict.state === 'behind') { - prodLog.debug( - `[Brainy] entity-tree stamp is behind the log head (${verdict.stampSource} < ` + - `${verdict.head}) — benign (stamped at last flush); refreshes at the next flush.` - ) - } - } - /** * Ask the writer process serving this data directory to flush its in-memory * indexes to disk, so a read-only inspector can observe fresh state. @@ -10760,32 +9417,17 @@ export class Brainy implements BrainyInterface { migrating: boolean /** Structured progress while {@link migrating}; absent otherwise. */ migration?: MigrationProgress - /** `true` when a non-fatal index rebuild failed at init() (degraded — queries - * may be incomplete). Folds in the same `_indexRebuildFailed` signal that - * {@link validateIndexConsistency} / {@link checkHealth} already expose. */ - rebuildFailed: boolean - /** The rebuild failure message when {@link rebuildFailed}; absent otherwise. */ - rebuildError?: string - /** Count of records committed via adopt-forward failed-rollback recovery whose - * derived index may be incomplete until repairIndex() (the 8.2.6 degraded - * set). Non-zero = degraded — a readiness probe should not report 200-ready. */ - degradedIds: number hnswIndex: { size: number - /** Honest "serving" — the provider's `isReady()` when exposed, else `size>0`. */ populated: boolean - /** The provider's honest `isReady()` signal; absent when unexposed. */ - ready?: boolean } metadataIndex: { entries: number populated: boolean - ready?: boolean } graphIndex: { relationships: number populated: boolean - ready?: boolean } storage: { totalEntities: number @@ -10800,9 +9442,6 @@ export class Brainy implements BrainyInterface { lazyRebuildCompleted: this.lazyRebuildCompleted, disableAutoRebuild: this.config.disableAutoRebuild || false, migrating: false, - rebuildFailed: this._indexRebuildFailed != null, - ...(this._indexRebuildFailed ? { rebuildError: this._indexRebuildFailed.message } : {}), - degradedIds: this._indexDegradedIds.size, hnswIndex: { size: 0, populated: false }, metadataIndex: { entries: 0, populated: false }, graphIndex: { relationships: 0, populated: false }, @@ -10814,20 +9453,6 @@ export class Brainy implements BrainyInterface { const hnswSize = this.index.size() const graphSize = await this.graphIndex.size() - // Honest readiness: when a provider exposes isReady(), it is the truth of - // whether the index actually SERVES (a native index can report a non-zero - // size/count yet not have loaded its serving structure). `populated` reflects - // that honest signal when present, falling back to size>0 only for providers - // that do not expose isReady(). Mirrors validateIndexConsistency/checkHealth, - // which already fold in the same signals. - const readyOf = (p: unknown): boolean | undefined => { - const r = assessIndexReadiness(p) - return r === 'unknown' ? undefined : r === 'ready' - } - const hnswReady = readyOf(this.index) - const metadataReady = readyOf(this.metadataIndex) - const graphReady = readyOf(this.graphIndex) - // Check storage entity count let storageEntityCount = 0 try { @@ -10841,29 +9466,18 @@ export class Brainy implements BrainyInterface { initialized: this.initialized, lazyRebuildCompleted: this.lazyRebuildCompleted, disableAutoRebuild: this.config.disableAutoRebuild || false, - // A non-fatal index-rebuild failure recorded at init(), or adopt-forward - // degraded ids, are degraded states (queries may be incomplete) — surface - // them here alongside the same signals validateIndexConsistency()/ - // checkHealth() already expose, so a readiness probe never reports 200-ready - // over a known-degraded index. - rebuildFailed: this._indexRebuildFailed != null, - ...(this._indexRebuildFailed ? { rebuildError: this._indexRebuildFailed.message } : {}), - degradedIds: this._indexDegradedIds.size, ...this.migrationSnapshot(), hnswIndex: { size: hnswSize, - populated: hnswReady ?? hnswSize > 0, - ...(hnswReady !== undefined ? { ready: hnswReady } : {}) + populated: hnswSize > 0 }, metadataIndex: { entries: metadataStats.totalEntries, - populated: metadataReady ?? metadataStats.totalEntries > 0, - ...(metadataReady !== undefined ? { ready: metadataReady } : {}) + populated: metadataStats.totalEntries > 0 }, graphIndex: { relationships: graphSize, - populated: graphReady ?? graphSize > 0, - ...(graphReady !== undefined ? { ready: graphReady } : {}) + populated: graphSize > 0 }, storage: { totalEntities: storageEntityCount @@ -12974,96 +11588,22 @@ export class Brainy implements BrainyInterface { entityCount: number indexEntryCount: number recommendation: string | null - /** Cross-layer: each provider's own invariant self-report, when - * it exposes validateInvariants(). Absent providers are simply not listed. */ - providers?: ProviderInvariantReport[] }> { await this.ensureInitialized() const result = await this.metadataIndex.validateConsistency() - - // Cross-layer integrity: validateConsistency() above only sees the - // JS metadata index — it is BLIND to a native provider whose manifest ↔ - // segments ↔ counts have diverged. Delegate to each provider's own - // validateInvariants() and aggregate, so "healthy-while-broken" is impossible. - const providers = await this.collectProviderInvariants() - const brokenProviders = providers.filter((p) => !p.healthy) - - let healthy = result.healthy - const notes: string[] = [] - if (result.recommendation) notes.push(result.recommendation) - + // Fold in a non-fatal index-rebuild failure recorded at init() so the degraded + // state is observable through the same health surface (not just the console). if (this._indexRebuildFailed) { - healthy = false - notes.unshift( - `Index rebuild failed at init() (degraded — queries may be incomplete): ` + - `${this._indexRebuildFailed.message}. Rebuild the indexes and re-open.` - ) - } - if (this._indexDegradedIds.size > 0) { - healthy = false - notes.unshift( - `${this._indexDegradedIds.size} record(s) committed via adopt-forward ` + - `failed-rollback recovery — the derived index may be incomplete for them. ` + - `Run repairIndex() to reconcile.` - ) - } - for (const p of brokenProviders) { - healthy = false - const failing = p.invariants.filter((i) => !i.holds) - notes.unshift( - `Provider '${p.provider}' reports ${failing.length} failing invariant(s): ` + - failing.map((i) => `${i.name} (${i.detail}; heal=${i.heal})`).join('; ') + - `. Run repairIndex() to reconcile from canonical.` - ) - } - - return { - ...result, - healthy, - recommendation: notes.length > 0 ? notes.join(' ') : null, - ...(providers.length > 0 ? { providers } : {}) - } - } - - /** - * @description Feature-detect + call each index provider's OPTIONAL - * `validateInvariants()` and collect the reports. The contract is - * that `validateInvariants()` NEVER throws and is bounded <50ms — but if a - * provider violates that, the throw is turned into an UNHEALTHY synthetic - * report (loud), never swallowed into "healthy". Providers that do not expose - * the hook are simply omitted (the JS baseline validates via - * `metadataIndex.validateConsistency()`). - * @returns One report per provider that exposes `validateInvariants()`. - */ - private async collectProviderInvariants(): Promise { - const reports: ProviderInvariantReport[] = [] - const providers: unknown[] = [this.metadataIndex, this.index, this.graphIndex] - for (const provider of providers) { - const fn = (provider as { validateInvariants?: () => Promise } | null) - ?.validateInvariants - if (typeof fn !== 'function') continue - try { - const report = await fn.call(provider) - if (report && Array.isArray(report.invariants)) reports.push(report) - } catch (err) { - reports.push({ - provider: 'unknown', - healthy: false, - serving: false, - invariants: [ - { - name: 'validate-invariants-threw', - holds: false, - detail: `validateInvariants() threw (contract violation — it must never throw): ${(err as Error).message}`, - heal: 'rebuild' - } - ], - checkedAt: Date.now(), - durationMs: 0 - }) + return { + ...result, + healthy: false, + recommendation: + `Index rebuild failed at init() (degraded — queries may be incomplete): ` + + `${this._indexRebuildFailed.message}. Rebuild the indexes and re-open.` + + (result.recommendation ? ` Also: ${result.recommendation}` : '') } } - return reports + return result } /** @@ -13117,8 +11657,7 @@ export class Brainy implements BrainyInterface { limit?: number } ): Promise { - // Graph read (see related): adjacency traversal only. - await this.ensureInitialized({ needs: ['graph'] }) + await this.ensureInitialized() await this.verifyGraphAdjacencyLive() const direction = options?.direction || 'both' @@ -13555,14 +12094,6 @@ export class Brainy implements BrainyInterface { candidateIds?: string[], allowedIds?: OpaqueIdSet ): Promise[]> { - // Vector cold-read guard: before trusting a semantic/vector result, verify the - // vector index actually SERVES a known persisted vector (one-shot per brain). - // A pure semantic find({ query }) has no filter, so verifyMetadataLive never - // fires — a cold native index that loaded its COUNT but not its serving - // structure would return a silent []. This self-heals (rebuild) or throws - // VectorIndexNotReadyError instead. - await this.verifyVectorLive() - const vector = params.vector || (await this.embed(params.query!)) const limit = params.limit || 10 @@ -13603,10 +12134,6 @@ export class Brainy implements BrainyInterface { private async executeProximitySearch(params: FindParams): Promise[]> { if (!params.near) return [] - // Vector cold-read guard (see executeVectorSearch): proximity search also - // hits this.index.search — verify it serves before trusting an empty result. - await this.verifyVectorLive() - // Teaching error: without an anchor id the constraint is meaningless, and // letting it fall through produces an opaque storage-layer sharding error. if (!params.near.id) { @@ -14861,15 +13388,8 @@ export class Brainy implements BrainyInterface { return } - // If indexes already populated AND honestly serving, mark complete and skip. - // Honest gate: when the provider exposes isReady(), that REPLACES the size()>0 - // proxy (a native index can report a non-zero size while its serving structure - // is not loaded — the silent-empty cold-load class). A not-ready provider falls - // through so the rebuild path can load it; verifyVectorLive() is the query-time - // backstop either way. Providers without isReady() keep the size() heuristic - // (the JS index's size()>0 genuinely means loaded). - const vectorReadiness = assessIndexReadiness(this.index) - if (vectorReadiness === 'ready' || (vectorReadiness === 'unknown' && this.index.size() > 0)) { + // If indexes already populated, mark as complete and skip + if (this.index.size() > 0) { this.lazyRebuildCompleted = true return } @@ -15289,58 +13809,6 @@ export class Brainy implements BrainyInterface { ) } - /** - * @description The provider instance backing a given index {@link IndexFamily}. - * The single place the family label maps to the concrete provider, so the - * scoped migration gate and any future family-aware routing agree. - */ - private providerForFamily(family: IndexFamily): unknown { - switch (family) { - case 'vector': - return this.index - case 'metadata': - return this.metadataIndex - case 'graph': - return this.graphIndex - } - } - - /** - * @description Whether the migration LOCK should hold for an operation that - * depends on the given index families: - * - `needs === undefined` → WHOLE-BRAIN: any migrating provider counts (the - * conservative default for writes and unclassified reads — a write touches - * every index). - * - `needs === []` → NEVER: a canonical-storage read consults no derived index, - * so a migration elsewhere is irrelevant; it serves immediately. - * - `needs = [families]` → SCOPED: only those families' providers count, so a - * read served from a healthy family is not blocked by an unrelated family's - * one-time migration. - */ - private neededFamiliesMigrating(needs?: IndexFamily[]): boolean { - if (needs === undefined) return this.anyProviderMigrating() - for (const family of needs) { - if (this.providerIsMigrating(this.providerForFamily(family))) return true - } - return false - } - - /** - * @description The index families a {@link find} query consults, derived from - * its params — the read-side input to the family-scoped migration gate. A - * vector / near / semantic query needs `vector`; a `where` / type / subtype / - * service filter or an aggregate needs `metadata`; a `connected` traversal - * needs `graph`. A query combining shapes needs the union. Mirrors find()'s - * own criteria split so the gate and the executor never disagree. - */ - private queryIndexFamilies(params: FindParams): IndexFamily[] { - const needs: IndexFamily[] = [] - if ((params.query && params.query.trim() !== '') || params.vector || params.near) needs.push('vector') - if (params.where || params.type || params.subtype || params.service || params.aggregate) needs.push('metadata') - if (params.connected) needs.push('graph') - return needs - } - /** * @description Rich migration progress relayed verbatim from whichever provider * exposes the OPTIONAL `migrationStatus(): { phase?, index?, percent?, … } | null`. @@ -15369,19 +13837,11 @@ export class Brainy implements BrainyInterface { /** * @description Block-and-queue on the coordinated migration LOCK (#18). Called * at the {@link ensureInitialized} choke point (and once inside `init()` before - * VFS bootstrap), so a data-plane operation waits here while a native provider - * runs its one-time 7.x → 8.0 rebuild-from-canonical. The caller gets the - * correct answer, never a partial read of a half-built index and never a lost - * write. A brain that is not migrating pays a single boolean check and returns - * immediately. - * - * FAMILY-SCOPED: `needs` names the index families the caller depends on, so the - * wait holds ONLY for a migration of one of those families. A read served - * entirely from canonical storage (`needs: []`) or from a healthy family never - * blocks on an unrelated family's migration. `needs` omitted = the conservative - * whole-brain wait (writes touch every index; startup wants all of them ready). - * @param needs - Index families this operation consults, or `undefined` for the - * whole-brain wait. + * VFS bootstrap), so every data-plane read and write — and startup itself — + * waits here while a native provider runs its one-time 7.x → 8.0 + * rebuild-from-canonical. The caller gets the correct answer, never a partial + * read of a half-built index and never a lost write. A brain that is not + * migrating pays a single boolean check and returns immediately. * * IMPORTANT: this timeout bounds THE CALLER'S WAIT, not the migration. The * migration itself is unbounded — the native provider rebuilds a billion-scale @@ -15395,8 +13855,8 @@ export class Brainy implements BrainyInterface { * for a very large brain, runs the offline migrator. The block/release lines * log once per window; the flags reset on release so a later upgrade re-logs. */ - private async awaitMigrationLock(needs?: IndexFamily[]): Promise { - if (!this.neededFamiliesMigrating(needs)) return // fast path — the overwhelming common case + private async awaitMigrationLock(): Promise { + if (!this.anyProviderMigrating()) return // fast path — the overwhelming common case const startedAt = Date.now() const timeoutMs = this.config.migrationWaitTimeoutMs ?? 30_000 @@ -15409,7 +13869,7 @@ export class Brainy implements BrainyInterface { ) } - while (this.neededFamiliesMigrating(needs)) { + while (this.anyProviderMigrating()) { const remaining = timeoutMs - (Date.now() - startedAt) if (remaining <= 0) { const status = this.providerMigrationStatus() @@ -15571,17 +14031,6 @@ export class Brainy implements BrainyInterface { (result.recommendation ? ` Also: ${result.recommendation}` : '') } } - if (this._indexDegradedIds.size > 0) { - return { - ...result, - healthy: false, - recommendation: - `${this._indexDegradedIds.size} record(s) committed via adopt-forward ` + - `failed-rollback recovery — the derived index may be incomplete for them. ` + - `Run repairIndex() to reconcile.` + - (result.recommendation ? ` Also: ${result.recommendation}` : '') - } - } return result } @@ -15597,13 +14046,6 @@ export class Brainy implements BrainyInterface { */ private async ensureMetadataConsistencyProbed(): Promise { if (this._metadataConsistencyProbed) return - // Defer while the metadata provider runs its one-time in-place migration: - // probing (and self-healing via rebuild) an index the provider is mid-rebuild - // would collide with the provider that owns it. Mirrors the vector deference - // in ensureIndexesLoaded. Do NOT latch — once the migration clears, the next - // read runs the probe. (The family-scoped find() gate waits on the metadata - // family separately before any actual filter read.) - if (this.providerIsMigrating(this.metadataIndex)) return this._metadataConsistencyProbed = true const provider = this.metadataIndex as { probeConsistency?: () => Promise @@ -15629,233 +14071,15 @@ export class Brainy implements BrainyInterface { } /** - * Detect and repair corrupted metadata indexes. + * Detect and repair corrupted metadata indexes * - * Runs corruption detection and auto-rebuilds if corruption is found. This is - * the equivalent of the old init()-time corruption check, now available as an - * explicit operation. - * - * It is ALSO the recovery path for a write-quarantine: when a transaction's - * rollback fails and leaves the store inconsistent ({@link StoreInconsistentError}), - * writes are refused until this method reconciles the derived indexes against - * canonical storage (a forced rebuild — orphaned/lost records reflected - * consistently) and lifts the quarantine. Canonical is the source of truth: a - * genuinely lost record cannot be resurrected here (restore from a snapshot for - * that), but the store is made internally consistent and writes re-enabled. + * Runs corruption detection and auto-rebuilds if corruption is found. + * This is the equivalent of the old init()-time corruption check, + * now available as an explicit operation. */ - /** - * Emit ONE loud warning per degraded window when a read is served while the - * derived index is known-incomplete — either a non-fatal init rebuild failure - * ({@link _indexRebuildFailed}) or an adopt-forward degraded commit - * ({@link _indexDegradedIds}). Reads still return (canonical is the source of - * truth), but the caller is told the result may be partial and how to heal it. - * No-op when healthy or when the brain is configured `silent`. - * @param op - The read method name for the message (e.g. `'find'`, `'get'`). - */ - private warnIfReadsDegraded(op: string): void { - const degraded = - this._indexRebuildFailed !== null || this._indexDegradedIds.size > 0 - if (!degraded) { - this._degradedReadWarned = false - return - } - if (this._degradedReadWarned || this.config.silent) return - this._degradedReadWarned = true - const reason = this._indexRebuildFailed - ? `index rebuild failed at init() (${this._indexRebuildFailed.message})` - : `${this._indexDegradedIds.size} record(s) committed in a degraded state` - prodLog.warn( - `[Brainy] ${op}() is serving reads while the derived index is INCOMPLETE ` + - `(${reason}). Results may be missing entries — run repairIndex() to ` + - `reconcile the derived indexes against canonical storage.` - ) - } - - /** - * Read-only graph-truth audit — proves (or disproves) that relationship - * reads return canonical truth on THIS brain, and classifies every - * discrepancy into its failure family: - * - * - `missingFromReads` — canonical verb records the read path omits - * (PRESENT BUT INVISIBLE: adjacency/membership staleness) - * - `danglingEndpoints` — verbs whose endpoint entity is gone (SCAR class) - * - `readOnlyVerbIds` — read-path edges with no canonical record (GHOSTS) - * - * Design-hidden edges (internal/system visibility) are counted separately — - * the audit reads with every visibility tier included, so intentional - * hiding is never misclassified as index loss. Mutates nothing; safe on a - * live brain (cost: one canonical walk + one indexed read per source). - * Run it after any engine upgrade, restore, or migration; a `coherent` - * report is the verified statement that `related()`/`readdir` can be - * trusted. If it reports discrepancies, `repairIndex()` is the sanctioned - * heal — re-run the audit afterwards to prove the repair. - * - * @param options.maxExamples - Cap per example list (counts stay exact). Default 100. - * @returns The full audit report; also narrated via logs (loud on incoherence). - * @since 8.6.0 - */ - async auditGraph(options: { maxExamples?: number } = {}): Promise { - await this.ensureInitialized({ needs: ['graph'] }) - - const PAGE = 1000 - return runGraphAudit( - { - eachNounId: async (consume) => { - let cursor: string | undefined - let offset = 0 - for (;;) { - const page = await (this.storage as unknown as { - getNounIdsWithPagination(o: { - limit: number - offset?: number - cursor?: string - }): Promise<{ ids: string[]; hasMore: boolean; nextCursor?: string }> - }).getNounIdsWithPagination( - cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset } - ) - for (const id of page.ids) consume(id) - if (!page.hasMore || page.ids.length === 0) break - if (page.nextCursor) cursor = page.nextCursor - else offset += page.ids.length - } - }, - eachVerb: async (consume) => { - let cursor: string | undefined - let offset = 0 - for (;;) { - const page = await this.storage.getVerbs({ - pagination: cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset } - }) - for (const verb of page.items) { - const v = verb as unknown as Record - consume({ - id: String(v.id), - type: String(v.verb ?? 'unknown'), - sourceId: String(v.sourceId), - targetId: String(v.targetId), - visibility: typeof v.visibility === 'string' ? v.visibility : undefined - }) - } - if (!page.hasMore || page.items.length === 0) break - if (page.nextCursor) cursor = page.nextCursor - else offset += page.items.length - } - }, - readRelationsFrom: async (sourceId) => - this.related({ - from: sourceId, - includeInternal: true, - includeSystem: true, - limit: 100000 - }) - }, - options - ) - } - async repairIndex(): Promise { await this.ensureInitialized() - - // Prune orphaned canonical containers left by the pre-8.3.1 partial-delete - // defect: a delete that removed the metadata (content) leg but left the - // vector leg + the entity directory (a "ghost"), or left an empty directory - // (a "scar"). These are not live entities (getNoun needs the content leg) - // yet inflate enumerated counts and confuse locator resolution. Feature- - // detected (filesystem-only — key/prefix stores have no orphan containers); - // recompute counts afterward so the totals stop counting the ghosts. - const pruner = this.storage as { - pruneOrphanedEntities?: () => Promise<{ nouns: string[]; verbs: string[] }> - rebuildTypeCounts?: () => Promise - rebuildSubtypeCounts?: () => Promise - } - if (typeof pruner.pruneOrphanedEntities === 'function') { - const orphans = await pruner.pruneOrphanedEntities() - if (orphans.nouns.length + orphans.verbs.length > 0) { - prodLog.warn( - `[Brainy] repairIndex() pruned ${orphans.nouns.length} orphaned noun + ` + - `${orphans.verbs.length} orphaned verb container(s) left by a pre-8.3.1 ` + - `partial delete.` - ) - } - } - // SANCTIONED RECOUNT — unconditional, not gated on orphans found: the - // persisted counters can be inflated over perfectly clean shelves (deletes - // whose decrement was skipped by the removed-record re-read), and - // Math.max(totalNounCount, scanned) means an inflated scalar can never - // correct itself. rebuildTypeCounts() recomputes EVERY counter rollup - // (scalar totals + per-type maps + type-statistics arrays) from one - // canonical walk and persists them. - await pruner.rebuildTypeCounts?.() - await pruner.rebuildSubtypeCounts?.() - - // The recount changed the rollup truth — re-stamp the entity tree so the - // stamp's invariants match the healed counters (repair leaves a coherent - // stamp, not a stale one that warns on the next open). - await this.stampEntityTree() - - // VFS containment reconciliation: heal "cosmetic ghost" edges left by - // pre-fix renames (an entity Contains-linked from BOTH its old and new - // directory — readdir listed it in two places) and duplicate edges from - // concurrent writers. Canonical metadata.path is the truth; only VFS - // containment edges are touched. Loud per repair. - if (this._vfsInitialized && this._vfs) { - const containment = await this._vfs.repairContainment() - if (containment.removed + containment.restored > 0) { - prodLog.warn( - `[Brainy] repairIndex() reconciled VFS containment: removed ${containment.removed} ` + - `stale/duplicate edge(s), restored ${containment.restored} missing edge(s).` - ) - } - } - await this.metadataIndex.detectAndRepairCorruption() - // Lift a failed-rollback write-quarantine: force a full rebuild so the - // derived indexes are provably reconciled with canonical, then clear the - // flag so writes resume. - if (this.storeInconsistency) { - await this.rebuildIndexesIfNeeded(true) - const cleared = this.storeInconsistency - this.storeInconsistency = null - prodLog.warn( - `[Brainy] repairIndex() reconciled the store and LIFTED the write-quarantine ` + - `set by a failed transaction rollback (${cleared.records.length} record(s) affected). ` + - `Writes are re-enabled.` - ) - } - // Cross-layer repair: repairIndex must reconcile NATIVE derived - // state from canonical, not just the JS metadata index. Consult each provider's - // own validateInvariants() and rebuild any whose failing invariant asks for it - // (heal: 'rebuild') — the native counterpart of detectAndRepairCorruption(). - for (const provider of [this.metadataIndex, this.index, this.graphIndex]) { - const p = provider as { - validateInvariants?: () => Promise - rebuild?: () => Promise - } | null - if (!p || typeof p.validateInvariants !== 'function' || typeof p.rebuild !== 'function') continue - let report: ProviderInvariantReport - try { - report = await p.validateInvariants() - } catch { - continue // a throwing validateInvariants is surfaced by validateIndexConsistency; skip repair here - } - if (report.healthy) continue - if (report.invariants.some((i) => !i.holds && i.heal === 'rebuild')) { - prodLog.warn( - `[Brainy] repairIndex(): provider '${report.provider}' has a failing invariant ` + - `requiring a rebuild — reconciling its derived state from canonical.` - ) - await p.rebuild() - } - } - // detectAndRepairCorruption() above rebuilt the derived indexes from - // canonical, so any adopt-forward degraded ids and a non-fatal init - // rebuild failure are now reconciled — clear the queryable degraded state - // and re-arm the read-path warning. - if (this._indexDegradedIds.size > 0 || this._indexRebuildFailed) { - this._indexDegradedIds.clear() - this._indexRebuildFailed = null - this._degradedReadWarned = false - } } /** @@ -16107,107 +14331,29 @@ export class Brainy implements BrainyInterface { */ private async backfillAggregateIfNeeded(name: string): Promise { const index = this._aggregationIndex - if (!index) return + if (!index || !index.getPendingBackfills().includes(name)) return - // Persisted-state adoption happens inside ready(); after it resolves the - // pending-backfill set is authoritative (an unchanged definition with valid - // persisted state is NOT listed — no walk at all on a clean reopen). - await index.ready() + index.beginBackfill(name) - // Single-flight: concurrent queries share ONE walk instead of each wiping - // the others' partial state and starting their own (the stampede that kept - // a busy store from ever converging). The loop covers the rare case where - // the in-flight walk snapshotted its batch before `name` became pending — - // the next iteration starts a fresh walk that includes it. - while (index.getPendingBackfills().includes(name)) { - // Failure latch: a deterministically-failing walk must not be re-run at - // the caller's retry rate — that is a silent CPU loop wearing a retry - // loop's clothes. Within the cooldown, rethrow the recorded failure - // immediately; after it, one fresh attempt is allowed. - const failure = this._aggregationBackfillFailure - if (failure && Date.now() - failure.at < AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS) { - throw new Error( - `Aggregation backfill for '${name}' is in failure cooldown (retry in ` + - `${Math.ceil((AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS - (Date.now() - failure.at)) / 1000)}s). ` + - `Last failure: ${failure.error.message}` - ) + const PAGE = 500 + let offset = 0 + let cursor: string | undefined + for (;;) { + const page = await this.storage.getNouns({ + pagination: cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset } + }) + for (const noun of page.items) { + index.backfillEntity(name, noun as unknown as Record) } - if (!this._aggregationBackfillFlight) { - this._aggregationBackfillFlight = this.runAggregationBackfillWalk() - .finally(() => { - this._aggregationBackfillFlight = null - }) + if (!page.hasMore || page.items.length === 0) break + if (page.nextCursor) { + cursor = page.nextCursor + } else { + offset += page.items.length } - await this._aggregationBackfillFlight - } - } - - /** - * One store walk fills EVERY aggregate currently pending backfill — M pending - * aggregates cost one enumeration, not M. Only reached when an aggregate - * genuinely needs a rescan (new definition over a populated store, changed - * definition, or failed state load); a clean reopen adopts persisted state - * and never walks. - */ - private async runAggregationBackfillWalk(): Promise { - const index = this._aggregationIndex! - const names = index.getPendingBackfills() - if (names.length === 0) return - - prodLog.info(`[Aggregation] backfill walk starting for: ${names.join(', ')}`) - const startedAt = Date.now() - for (const n of names) index.beginBackfill(n) - - let scanned = 0 - try { - const PAGE = 500 - let offset = 0 - let cursor: string | undefined - for (;;) { - const page = await this.storage.getNouns({ - pagination: cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset } - }) - for (const noun of page.items) { - const record = noun as unknown as Record - for (const n of names) { - index.backfillEntity(n, record) - } - } - scanned += page.items.length - if (!page.hasMore || page.items.length === 0) break - if (page.nextCursor) { - if (page.nextCursor === cursor) { - // A non-advancing cursor with hasMore=true would loop this walk at - // CPU speed forever, silently. That is a storage pagination defect — - // fail the waiting queries loudly instead of spinning. - throw new Error( - `Aggregation backfill aborted: storage pagination returned a non-advancing cursor ` + - `after ${scanned} entities with hasMore=true — the storage adapter's getNouns cursor is broken.` - ) - } - cursor = page.nextCursor - } else { - offset += page.items.length - } - } - } catch (err) { - // Non-destructive failure: drop the staging maps (live state keeps - // serving), keep the aggregates flagged pending, latch the error so - // retries within the cooldown fail fast, and say all of it out loud. - for (const n of names) index.abortBackfill(n) - this._aggregationBackfillFailure = { at: Date.now(), error: err as Error } - prodLog.warn( - `[Aggregation] backfill walk FAILED after ${scanned} entities: ${(err as Error).message} — ` + - `prior aggregate state preserved; retries suppressed for ${AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS / 1000}s` - ) - throw err } - for (const n of names) index.finishBackfill(n) - this._aggregationBackfillFailure = null - prodLog.info( - `[Aggregation] backfill walk finished: ${scanned} entities → ${names.length} aggregate(s) in ${Date.now() - startedAt}ms` - ) + index.finishBackfill(name) } /** @@ -16217,13 +14363,6 @@ export class Brainy implements BrainyInterface { * This ensures deferred persistence mode data is saved */ async close(): Promise { - // Cancel any pending post-import background deduplication FIRST — it is a - // writer (merge-deletes), and no delete pass may start mid- or post-close. - this._backgroundDedup?.cancelPending() - - // Change-feed teardown: no events are delivered for or after close(). - this._changeFeed.close() - // Phase 0a: Persist buffered single-op generation history (async // group-commit) before anything else, so a clean close never drops history // the caller already observed. No-op when nothing is pending or read-only. @@ -16231,21 +14370,9 @@ export class Brainy implements BrainyInterface { await this.generationStore.flushPendingSingleOps() } - // Phase 0b: REPACK cold history into sealed segments (D1+D3 — - // re-representation, never deletion; the only history transform under the - // archival profile), then auto-compact per config.retention. Repack runs - // FIRST so bounded-retention reclaim can drop whole segments. Both are - // time-bounded maintenance passes (8.9.0 law: flush() never pays these); - // both are housekeeping — failures warn, never fail a clean shutdown. - if (!this.isReadOnly && this.generationStore) { - try { - await this.generationStore.repackHistory({ timeBudgetMs: 5_000 }) - } catch (error) { - console.warn( - `History repacking failed (non-fatal): ${error instanceof Error ? error.message : String(error)}` - ) - } - } + // Phase 0b: Auto-compact generational history per config.retention (default + // on) BEFORE the generation store closes below. Respects live Db pins and + // an explicit autoCompact: false; no-op on read-only instances. await this.autoCompactHistory() // Phase 1: Flush ALL components in parallel to persist buffered data @@ -16289,13 +14416,6 @@ export class Brainy implements BrainyInterface { })() ]) - // Stamp the entity tree at the close boundary (counters + counter now - // durable from Phase 1/0a), so a cleanly-closed store reopens COHERENT - // instead of benign-behind. Best-effort, never blocks the close. - if (this.generationStore && !this.isReadOnly) { - await this.stampEntityTree() - } - // Phase 2: Close components to release resources (timers, file handles) // Data is already safe on disk from Phase 1 await Promise.all([ diff --git a/src/coreTypes.ts b/src/coreTypes.ts index e0248d17..cce06e8a 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -754,39 +754,6 @@ export interface Change { data?: HNSWNounWithMetadata | HNSWVerbWithMetadata } -/** - * @description A declared derived-index blob FAMILY (the - * registered-blob contract). A family names the set of on-disk blobs that a - * derived index needs AS A SET (e.g. the vector base = `main.dkann` + - * `main.slotmap` + `main.slotrev`): losing ANY member corrupts the index. Once a - * family is declared: - * - its members are UNDELETABLE through the storage layer — `deleteBinaryBlob` / - * `removeRawPrefix` refuse with a `ProtectedArtifactError`, so an in-process - * GC/sweeper cannot remove a load-bearing file (intentional retirement = - * `unregisterDerivedFamily` first); - * - a member missing on open is a loud `DerivedArtifactMissingError` → rebuild. - * `COLD ≠ DEAD`: a write-once segment or a recovery-critical archive is - * load-bearing even when it has not been touched in a long time. - */ -export interface DerivedFamilyDeclaration { - /** Stable family id, e.g. `'vector-base'` / `'metadata-sstables'`. */ - name: string - /** - * The logical blob keys that make up the family. When {@link namespace} is - * set, each entry is a PREFIX protecting every key beneath it (for growing - * sets like `seg-*`); otherwise each entry is an exact member key. - */ - members: string[] - /** When true, {@link members} are prefixes (protect all keys beneath each). */ - namespace?: boolean - /** - * Whether a missing family can be rebuilt from the canonical records (default - * `true`). `false` marks an irreplaceable family (a missing member is data - * loss, not a rebuild). - */ - rebuildable?: boolean -} - export interface StorageAdapter { init(): Promise @@ -848,17 +815,7 @@ export interface StorageAdapter { */ getNounsByNounType(nounType: string): Promise - /** - * Delete a noun — FULL canonical removal (both legs + the entity container). - * - * @param id The entity id. - * @param priorMetadata OPTIONAL already-known metadata of the entity being - * removed (the caller's pre-delete read). The count decrement must never - * REQUIRE re-reading the record being removed: when the internal read - * returns `null` (replace race, or a ghost left by an earlier version) the - * decrement falls back to this record instead of being silently skipped. - */ - deleteNoun(id: string, priorMetadata?: NounMetadata | null): Promise + deleteNoun(id: string): Promise /** * Save verb - Pure HNSW verb with core fields only @@ -915,15 +872,7 @@ export interface StorageAdapter { */ getVerbsByType(type: string): Promise - /** - * Delete a verb — FULL canonical removal (both legs + the container). - * - * @param id The relationship id. - * @param priorMetadata OPTIONAL already-known metadata of the edge being - * removed (the caller's pre-delete read); keeps the count decrement honest - * when the internal read returns `null` (see `deleteNoun`). - */ - deleteVerb(id: string, priorMetadata?: VerbMetadata | null): Promise + deleteVerb(id: string): Promise /** * Save metadata @@ -1100,104 +1049,6 @@ export interface StorageAdapter { */ getBinaryBlobPath(key: string): string | null - /** - * @description OPTIONAL (registered-blob contract). Declare a - * derived-index blob {@link DerivedFamilyDeclaration | family} whose members - * become UNDELETABLE through this adapter — a subsequent `deleteBinaryBlob` / - * `removeRawPrefix` that would remove a declared member throws a - * `ProtectedArtifactError`. Providers declare their families on create; the - * declaration is persisted so protection survives a reopen. Idempotent per - * `name` (re-declaring replaces). - * @param family - The family to protect. - */ - registerDerivedFamily?(family: DerivedFamilyDeclaration): Promise - - /** - * @description OPTIONAL. Remove a family's protection so its members can be - * deleted again — the explicit, auditable step for intentional retirement of a - * derived index (the ONLY way a declared member becomes deletable). - * @param name - The {@link DerivedFamilyDeclaration.name} to unregister. - */ - unregisterDerivedFamily?(name: string): Promise - - /** - * @description OPTIONAL. List the currently-declared derived-index families — - * the source of truth for what `clear()` must wipe and what a - * missing-on-open check verifies. - */ - listDerivedFamilies?(): Promise - - /** - * @description OPTIONAL binary raw-byte primitives — the substrate for - * append-only log-structured files (the generation fact log's CRC-framed - * segments). Feature-detected: an adapter that omits them simply hosts no - * fact log (readers fall back to canonical enumeration). Paths are - * storage-root-relative and used VERBATIM (no `.gz`/`.bin` suffixing — - * unlike the JSON object and blob primitives). - * - * Append to a raw binary file, creating it (and parent directories) when - * absent. Append durability is the CALLER's job via `syncRawObjects` — - * matching the commit protocol, which batches fsyncs at its barrier. - */ - appendRawBytes?(path: string, bytes: Uint8Array): Promise - - /** - * Read a raw binary file whole. Absent → `null`; a real IO fault throws - * (never masked as absence). - */ - readRawBytes?(path: string): Promise - - /** - * Replace a raw binary file atomically (write-new → fsync → rename) — the - * reconcile primitive (e.g. truncating a fact-log tail back to committed - * truth after a crash). - */ - writeRawBytes?(path: string, bytes: Uint8Array): Promise - - /** - * Byte size of a raw binary file, or `null` when absent. - */ - rawByteSize?(path: string): Promise - - /** - * @description OPTIONAL fact-scan capability — how an index provider that - * holds only `storage` reaches the generation fact log (the host brain - * wires it at init; providers must never construct their own fact-log - * reader — the log's open path is writer-side). Present ⟺ this store hosts - * a fact log AND the host wired the capability. Returns a scan handle over - * committed facts (heal-grade telemetry included), or `null` when no fact - * log exists — callers fall back to the canonical enumeration walk. - */ - scanFacts?(options?: { - fromGeneration?: number - toGeneration?: number - kinds?: Array<'noun' | 'verb'> - batchSize?: number - }): import('./db/factLog.js').FactScanHandle | null - - /** - * @description OPTIONAL (rides the fact-scan capability): the fact log's - * head generation — the replay target for `stamp.sourceGeneration + 1` - * catch-ups. `null` when no fact log exists. - */ - factLogHeadGeneration?(): number | null - - /** - * @description OPTIONAL (rides the fact-scan capability): the COMMITTED - * generation watermark — the manifest truth a projection's - * `sourceGeneration` compares against. Exposed as a capability so a - * provider never parses the store's private manifest format. `null` when - * the capability is unwired. - */ - committedGeneration?(): number | null - - /** - * @description OPTIONAL (rides the fact-scan capability): the immutable, - * sealed fact-segment file paths covering `fromGeneration` — the zero-copy - * handoff. The mutable tail is excluded (read it via `scanFacts`). - */ - factSegmentPaths?(options?: { fromGeneration?: number }): string[] - /** * Save statistics data * @param statistics The statistics data to save diff --git a/src/db/db.ts b/src/db/db.ts index ac927fc5..62c3de0e 100644 --- a/src/db/db.ts +++ b/src/db/db.ts @@ -945,13 +945,6 @@ export class Db { * {@link SpeculativeOverlayError} (commit them with `brain.transact()` * first). * - * SPARSE FILES: a native accelerator's mmap index files can be sparse — - * huge apparent size, small allocated size. `persist()` handles them - * correctly (hard links share the allocation). But if you then archive the - * snapshot with EXTERNAL tools, use the sparse-aware flags (`tar czSf`, - * `rsync --sparse`, `cp --sparse=always`) or the copy materializes every - * hole — see docs/guides/external-backups-and-sparse-storage.md. - * * @param path - Absolute directory for the snapshot (created; must be * empty or absent). * @throws GenerationConflictError when this view is no longer the latest diff --git a/src/db/errors.ts b/src/db/errors.ts index 22f405be..e5f70add 100644 --- a/src/db/errors.ts +++ b/src/db/errors.ts @@ -159,133 +159,3 @@ export class GenerationCompactedError extends Error { this.horizon = horizon } } - -/** One entity/relationship left in an unreconciled state by a failed rollback. */ -export interface UnreconciledRecord { - /** The entity or relationship id. */ - id: string - /** `'noun'` (entity) or `'verb'` (relationship). */ - kind: 'noun' | 'verb' - /** - * `'orphan'` — the record is durably PRESENT but the transaction aborted - * (an add whose delete-undo failed); `'loss'` — the record is durably GONE - * or wrong when it should have been restored (a remove/update whose - * restore-undo failed — actual data loss). - */ - disposition: 'orphan' | 'loss' -} - -/** - * @description Thrown when a transaction's rollback could not be fully applied - * and the resulting inconsistency cannot be safely adopted forward — i.e. a - * multi-operation batch, or ANY case where a record was lost (a remove/update - * whose restore-undo failed). The store is left in a known-inconsistent state: - * the {@link records} name every entity/relationship whose canonical state no - * longer matches what the aborted transaction should have produced. - * - * On throw, the brain enters **write-quarantine** — reads continue to work, but - * further writes are refused until {@link } `repairIndex()` reconciles the - * derived indexes against canonical storage and lifts the quarantine. This is - * the honest, loud failure: a visible inconsistency the operator must repair, - * never a silent partial write. The generation counter is NOT advanced. - * - * (A SINGLE-op add whose only damage is a durably-present orphan is NOT this - * error: it is adopted forward as a committed generation and returned as a - * degraded-but-successful write — the record the caller asked for exists.) - * - * @example - * try { - * await brain.transact(ops) - * } catch (err) { - * if (err instanceof StoreInconsistentError) { - * console.error('store inconsistent:', err.records) // ids + dispositions - * await brain.repairIndex() // reconcile + lift the write-quarantine - * } - * } - */ -export class StoreInconsistentError extends Error { - /** Every record left in an unreconciled state by the failed rollback. */ - public readonly records: UnreconciledRecord[] - /** The original error that triggered the (then-failed) rollback. */ - public override readonly cause: Error - - /** - * @param records - The unreconciled entities/relationships (ids + disposition). - * @param cause - The error that triggered the rollback. - */ - constructor(records: UnreconciledRecord[], cause: Error) { - const orphans = records.filter((r) => r.disposition === 'orphan').length - const losses = records.filter((r) => r.disposition === 'loss').length - super( - `Store left inconsistent by a failed transaction rollback: ` + - `${records.length} record(s) could not be reconciled ` + - `(${orphans} orphaned, ${losses} lost). ` + - `The brain is now WRITE-QUARANTINED (reads still work) — run repairIndex() ` + - `to reconcile the derived indexes against canonical storage and lift the ` + - `quarantine. Ids: ${records.map((r) => `${r.id}:${r.disposition}`).join(', ')}. ` + - `Cause: ${cause.message}` - ) - this.name = 'StoreInconsistentError' - this.records = records - this.cause = cause - } -} - -/** - * @description Thrown by a write when the store cannot make single-op - * generation **history** durable: the asynchronous group-commit flush that - * persists buffered before-images to disk has failed repeatedly (a real - * storage fault — a full disk, an EIO, a permission loss — not a transient - * blip). Rather than keep accepting writes whose history silently piles up in - * memory and is never persisted — an invisible durability loss, and an - * unbounded leak — the store LATCHES this failure and refuses further writes - * until the pending tier drains. - * - * This is the loud, honest response to a broken history-durability path: - * - Live canonical data is unaffected (single-op live bytes are written before - * the history flush; only the immutable before-image history is stuck). - * - The background flush keeps retrying with backoff; when the underlying fault - * clears and a flush succeeds, the latch lifts and writes resume - * automatically. An explicit `flush()` also clears it on success. - * - {@link cause} is the underlying storage error from the last failed flush. - * - * A caller seeing this should treat it exactly like a full disk: stop writing, - * resolve the storage fault, and retry. It is NOT a data-corruption error — no - * committed generation is lost — it is a refusal to *promise* durability the - * store currently cannot deliver. - * - * @example - * try { - * await brain.add({ ... }) - * } catch (err) { - * if (err instanceof PendingFlushDurabilityError) { - * // History can't be persisted right now (disk fault). Resolve storage, - * // then retry — the store self-heals once a flush succeeds. - * console.error('history durability stalled:', err.cause) - * } - * } - */ -export class PendingFlushDurabilityError extends Error { - /** The underlying storage error from the most recent failed flush. */ - public override readonly cause: Error - /** How many consecutive flush attempts had failed when the latch tripped. */ - public readonly failedAttempts: number - - /** - * @param cause - The storage error from the last failed pending-tier flush. - * @param failedAttempts - Consecutive failed flush attempts at latch time. - */ - constructor(cause: Error, failedAttempts: number) { - super( - `Write refused: single-op generation history could not be made durable ` + - `after ${failedAttempts} consecutive flush attempts (${cause.message}). ` + - `Live data is intact, but buffered history is not yet persisted — the ` + - `store refuses further writes rather than silently accumulate ` + - `un-durable history. Resolve the underlying storage fault; the store ` + - `self-heals and resumes writes once a flush succeeds. Cause: ${cause.message}` - ) - this.name = 'PendingFlushDurabilityError' - this.cause = cause - this.failedAttempts = failedAttempts - } -} diff --git a/src/db/factLog.ts b/src/db/factLog.ts deleted file mode 100644 index 94e79700..00000000 --- a/src/db/factLog.ts +++ /dev/null @@ -1,721 +0,0 @@ -/** - * @module db/factLog - * @description The generation FACT LOG — an append-only, CRC-framed record of - * every committed generation as an AFTER-IMAGE "fact": what each touched - * entity/relationship BECAME (or a body-less tombstone when it was removed). - * This is the dual-write half of the log-canonical transition: today the - * before-image history + canonical tree remain authoritative; the fact log is - * appended at the same commit points and reconciled to committed truth at - * open, so consumers (index heals, replays, scans) can read one sequential, - * self-verifying stream instead of walking the entity tree. - * - * ## Wire format (frozen; additive-only within a major) - * - * Fact (msgpack, POSITIONAL array — the segment header's formatVersion - * governs the schema): - * - * fact := [ generation:u64, timestamp:u64, ops, meta|nil, blobHashes|nil ] - * op := [ kind:u8 (0=noun, 1=verb), id:bin16 (raw uuid bytes), - * record:[metaLeg, vecLeg] | nil ] // nil = TOMBSTONE - * - * Segment file (`_generations/facts/seg-.bfl`): - * - * header := magic "BFACTS\0\0" (8B) | formatVersion:u32 LE | - * firstGeneration:u64 LE | reserved 12B (ZEROED, verified) - * frame := length:u32 LE | crc32c:u32 LE (of payload) | payload - * - * A fact is never split across segments; a torn tail (length overrun or CRC - * mismatch) terminates that segment's scan — everything before it is intact. - * Zero-padded names make lexicographic order == generation order. - * - * ## Invariant - * - * After {@link FactLog.open}, the log contains EXACTLY the committed prefix: - * facts are appended BEFORE the commit point (inside the same durability - * window), so a crash can only leave the log AHEAD of committed truth — open - * truncates any fact beyond the committed generation. Absent generation = - * never committed; present = committed. A scan can never see an uncommitted - * fact. - * - * The manifest (`_generations/facts/manifest.json`, JSON — forensics stay - * terminal-readable) is the single source of truth for the segment SET; - * rotation flips it atomically (write-new → fsync → rename) BEFORE the new - * tail's first byte exists, so no segment file is ever unaccounted for. - */ -import { encode as defaultEncode, decode as defaultDecode } from '@msgpack/msgpack' -import { crc32c } from '../utils/crc32c.js' -import { prodLog } from '../utils/logger.js' - -// Swappable msgpack implementation — defaults to the JS codec; a native -// provider (registered via the plugin registry's 'msgpack' key) may replace -// it. Byte-compatibility is the contract (positional arrays, bin16 ids). -let msgpackEncode: (value: unknown) => Uint8Array = defaultEncode -let msgpackDecode: (bytes: Uint8Array) => unknown = defaultDecode - -/** Replace the msgpack encode/decode implementation at runtime. */ -export function setFactCodec(impl: { - encode: (value: unknown) => Uint8Array - decode: (bytes: Uint8Array) => unknown -}): void { - msgpackEncode = impl.encode - msgpackDecode = impl.decode -} - -/** Storage-root-relative home of the fact log. */ -export const FACTS_PREFIX = '_generations/facts' -/** The facts manifest path (JSON). */ -export const FACTS_MANIFEST_PATH = `${FACTS_PREFIX}/manifest.json` -/** Current segment format version (header field; additive-only within a major). */ -export const FACTS_FORMAT_VERSION = 1 -/** Rotation threshold: seal the tail segment once it exceeds this many bytes. */ -const SEGMENT_ROTATE_BYTES = 8 * 1024 * 1024 -/** Segment header: magic(8) + formatVersion(4) + firstGeneration(8) + reserved(12). */ -const HEADER_BYTES = 32 -const MAGIC = new Uint8Array([0x42, 0x46, 0x41, 0x43, 0x54, 0x53, 0x00, 0x00]) // "BFACTS\0\0" -/** Frame prefix: length(4) + crc32c(4). */ -const FRAME_PREFIX_BYTES = 8 - -/** One write inside a fact: what the id became (or a tombstone). */ -export interface FactOp { - kind: 'noun' | 'verb' - id: string - /** The AFTER-IMAGE legs, or `null` for a tombstone (the id was removed). */ - record: { metadata: unknown | null; vector: unknown | null } | null -} - -/** One committed generation, as scanned back out of the log. */ -export interface CommitFact { - generation: number - timestamp: number - ops: FactOp[] - meta?: Record - blobHashes?: string[] -} - -/** The telemetry a scan batch carries (frozen shape). */ -export interface FactScanBatch { - facts: CommitFact[] - firstGeneration: number - lastGeneration: number - factCount: number - byteSize: number - segmentId: string -} - -/** - * Liveness bound on a scan's FIRST batch (Stage-2 co-freeze, D1 contract): - * `batches()` must yield its first batch — or fail loudly — within this many - * ms of the first pull. A backlogged or damaged store may be SLOW, but it may - * never be SILENT: a consumer awaiting the first batch is otherwise - * indistinguishable from a wedge (the exact failure shape a production heal - * hit against a generations-backlogged brain). - */ -export const SCANFACTS_FIRST_BATCH_MS = 10_000 - -/** The telemetry a scan OPEN returns (frozen shape). */ -export interface FactScanHandle { - headGeneration: number - segmentCount: number - approxFactCount: number - /** - * Ordered batches; a detected gap aborts LOUDLY, never a silent skip. - * Liveness contract: the FIRST batch resolves or rejects within - * {@link SCANFACTS_FIRST_BATCH_MS} of the first pull — never a silent hang. - */ - batches: () => AsyncGenerator - /** Close telemetry — the invariant cross-check, valid after iteration ends. */ - summary: () => { factsYielded: number; segmentsRead: number } -} - -/** Manifest entry for a sealed segment. */ -interface SegmentEntry { - file: string - firstGeneration: number - lastGeneration: number - facts: number - bytes: number -} - -/** The facts manifest (JSON on disk). */ -interface FactsManifest { - formatVersion: number - segments: SegmentEntry[] - /** The append target. Its true content is established by scanning (crash tolerance). */ - tailSegment: string | null - updatedAt: string -} - -/** The narrow byte-level storage surface the fact log rides. */ -export interface FactLogStorage { - appendRawBytes(path: string, bytes: Uint8Array): Promise - readRawBytes(path: string): Promise - writeRawBytes(path: string, bytes: Uint8Array): Promise - rawByteSize(path: string): Promise - readRawObject(path: string): Promise - writeRawObject(path: string, data: any): Promise - syncRawObjects(paths: string[]): Promise - deleteRawObject(path: string): Promise -} - -/** True when the storage adapter exposes every primitive the fact log needs. */ -export function storageSupportsFactLog(storage: unknown): storage is FactLogStorage { - const s = storage as Record - return ( - typeof s.appendRawBytes === 'function' && - typeof s.readRawBytes === 'function' && - typeof s.writeRawBytes === 'function' && - typeof s.rawByteSize === 'function' - ) -} - -/** uuid string → 16 raw bytes (bin16 on the wire). */ -function uuidToBytes(id: string): Uint8Array { - const hex = id.replace(/-/g, '') - if (hex.length !== 32) { - // Non-uuid ids (legacy/natural keys) ride as UTF-8 with a length prefix - // marker impossible for uuids: we refuse instead — the write API has - // guaranteed uuid ids since 8.0, so anything else is a corruption signal. - throw new Error(`fact log: id is not a uuid: ${id}`) - } - const bytes = new Uint8Array(16) - for (let i = 0; i < 16; i++) { - bytes[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16) - } - return bytes -} - -/** 16 raw bytes → canonical lowercase uuid string. */ -function bytesToUuid(bytes: Uint8Array): string { - let hex = '' - for (let i = 0; i < 16; i++) hex += bytes[i].toString(16).padStart(2, '0') - return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}` -} - -/** Zero-padded segment filename: lexicographic order == generation order. */ -function segmentFileName(firstGeneration: number): string { - return `seg-${String(firstGeneration).padStart(20, '0')}.bfl` -} - -/** Build a segment header. Reserved bytes are ZEROED (and verified on open). */ -function buildHeader(firstGeneration: number): Uint8Array { - const header = new Uint8Array(HEADER_BYTES) - header.set(MAGIC, 0) - const view = new DataView(header.buffer) - view.setUint32(8, FACTS_FORMAT_VERSION, true) - view.setBigUint64(12, BigInt(firstGeneration), true) - // bytes 20..31 stay zero (reserved) - return header -} - -/** Encode one fact into a framed record (length + crc32c + msgpack payload). */ -function encodeFrame(fact: CommitFact): Uint8Array { - const payload = msgpackEncode([ - fact.generation, - fact.timestamp, - fact.ops.map((op) => [ - op.kind === 'noun' ? 0 : 1, - uuidToBytes(op.id), - op.record === null ? null : [op.record.metadata, op.record.vector] - ]), - fact.meta ?? null, - fact.blobHashes && fact.blobHashes.length > 0 ? fact.blobHashes : null - ]) - const frame = new Uint8Array(FRAME_PREFIX_BYTES + payload.length) - const view = new DataView(frame.buffer) - view.setUint32(0, payload.length, true) - view.setUint32(4, crc32c(payload), true) - frame.set(payload, FRAME_PREFIX_BYTES) - return frame -} - -/** Decode one msgpack payload back into a CommitFact. */ -function decodeFact(payload: Uint8Array): CommitFact { - const raw = msgpackDecode(payload) as unknown[] - const [generation, timestamp, ops, meta, blobHashes] = raw as [ - number, - number, - Array<[number, Uint8Array, [unknown, unknown] | null]>, - Record | null, - string[] | null - ] - return { - generation: Number(generation), - timestamp: Number(timestamp), - ops: ops.map(([kind, idBytes, record]) => ({ - kind: kind === 0 ? ('noun' as const) : ('verb' as const), - id: bytesToUuid(idBytes), - record: record === null ? null : { metadata: record[0] ?? null, vector: record[1] ?? null } - })), - ...(meta ? { meta } : {}), - ...(blobHashes && blobHashes.length > 0 ? { blobHashes } : {}) - } -} - -/** - * Parse a segment's bytes: verify the header, then walk frames until the end - * or a torn tail (length overrun / CRC mismatch), which terminates the walk — - * everything before it is intact. Returns the decoded facts plus the byte - * length of the VALID prefix (header + intact frames), which reconciliation - * uses to cut a torn tail without re-encoding. - */ -function parseSegment( - file: string, - bytes: Uint8Array -): { facts: CommitFact[]; validBytes: number } { - if (bytes.length < HEADER_BYTES) { - prodLog.warn(`[FactLog] segment ${file} shorter than its header — treating as empty`) - return { facts: [], validBytes: 0 } - } - for (let i = 0; i < MAGIC.length; i++) { - if (bytes[i] !== MAGIC[i]) { - throw new Error(`fact log: segment ${file} has a bad magic — not a fact segment`) - } - } - const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) - const version = view.getUint32(8, true) - if (version !== FACTS_FORMAT_VERSION) { - throw new Error( - `fact log: segment ${file} has formatVersion ${version}; this build reads ${FACTS_FORMAT_VERSION}` - ) - } - for (let i = 20; i < HEADER_BYTES; i++) { - if (bytes[i] !== 0) { - // Non-zero reserved bytes = a future format this build cannot verify. - throw new Error(`fact log: segment ${file} has non-zero reserved header bytes — unverifiable`) - } - } - - const facts: CommitFact[] = [] - let offset = HEADER_BYTES - while (offset + FRAME_PREFIX_BYTES <= bytes.length) { - const length = view.getUint32(offset, true) - const expectedCrc = view.getUint32(offset + 4, true) - const start = offset + FRAME_PREFIX_BYTES - const end = start + length - if (end > bytes.length) break // torn tail: frame length overruns the file - const payload = bytes.subarray(start, end) - if (crc32c(payload) !== expectedCrc) break // torn tail: payload CRC mismatch - facts.push(decodeFact(payload)) - offset = end - } - return { facts, validBytes: offset } -} - -/** - * The generation fact log. One instance per open store; every method assumes - * the single-writer discipline the generation store already enforces (calls - * arrive under its commit mutex). - */ -export class FactLog { - private readonly storage: FactLogStorage - /** Rotation threshold (bytes); tests may lower it to exercise rotation. */ - private readonly rotateBytes: number - private manifest: FactsManifest = { - formatVersion: FACTS_FORMAT_VERSION, - segments: [], - tailSegment: null, - updatedAt: new Date(0).toISOString() - } - /** Decoded facts of the TAIL segment (bounded by the rotation threshold). */ - private tailFacts: CommitFact[] = [] - /** Byte size of the tail segment file (valid prefix). */ - private tailBytes = 0 - /** Highest generation in the log (0 = empty). */ - private head = 0 - /** Segment paths appended since the last sync (the fsync batch). */ - private readonly dirtySegments = new Set() - - constructor(storage: FactLogStorage, options?: { rotateBytes?: number }) { - this.storage = storage - this.rotateBytes = options?.rotateBytes ?? SEGMENT_ROTATE_BYTES - } - - /** The highest committed generation the log holds (0 = empty). */ - headGeneration(): number { - return this.head - } - - /** - * Open the log and reconcile it to committed truth: read the manifest, - * establish the tail's intact content (torn-tail scan), then TRUNCATE any - * fact with `generation > committedGeneration` — those never committed (a - * crash between fact-append and the commit point). After open, the log is - * exactly the committed prefix. - */ - async open(committedGeneration: number): Promise { - const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null - if (stored && typeof stored === 'object' && Array.isArray(stored.segments)) { - if (stored.formatVersion !== FACTS_FORMAT_VERSION) { - throw new Error( - `fact log: manifest formatVersion ${stored.formatVersion}; this build reads ${FACTS_FORMAT_VERSION}` - ) - } - this.manifest = stored - } - - // Drop sealed segments that sit ENTIRELY beyond committed truth (a crash - // right after a rotation whose facts never committed), newest first. - while (this.manifest.segments.length > 0) { - const last = this.manifest.segments[this.manifest.segments.length - 1] - if (last.firstGeneration > committedGeneration) { - prodLog.warn( - `[FactLog] dropping sealed segment ${last.file} (generations ${last.firstGeneration}..` + - `${last.lastGeneration} never committed)` - ) - await this.storage.deleteRawObject(`${FACTS_PREFIX}/${last.file}`) - this.manifest.segments.pop() - await this.persistManifest() - } else if (last.lastGeneration > committedGeneration) { - // A sealed segment STRADDLING committed truth: cut it back. - await this.truncateSegmentTo(last.file, committedGeneration) - const cut = await this.reloadSegmentEntry(last.file) - this.manifest.segments[this.manifest.segments.length - 1] = cut - await this.persistManifest() - break - } else { - break - } - } - - // Establish the tail: scan its intact prefix, then truncate beyond - // committed truth (the common crash shape: buffered single-op facts whose - // counter never went durable). - if (this.manifest.tailSegment) { - const tailPath = `${FACTS_PREFIX}/${this.manifest.tailSegment}` - const bytes = await this.storage.readRawBytes(tailPath) - if (bytes === null) { - // Manifest named a tail whose first byte never landed — an empty tail. - this.tailFacts = [] - this.tailBytes = 0 - } else { - const { facts, validBytes } = parseSegment(this.manifest.tailSegment, bytes) - const kept = facts.filter((f) => f.generation <= committedGeneration) - if (kept.length !== facts.length || validBytes !== bytes.length) { - const dropped = facts.length - kept.length - if (dropped > 0) { - prodLog.warn( - `[FactLog] truncating ${dropped} uncommitted fact(s) beyond generation ` + - `${committedGeneration} from the tail (never committed)` - ) - } - await this.rewriteTail(kept) - } else { - this.tailFacts = facts - this.tailBytes = validBytes - } - } - } - - this.head = this.computeHead() - } - - /** - * Append one committed generation's fact. NOT durable until {@link sync} — - * the caller batches durability at its commit barrier (transact syncs in - * the same call; Model-B group-commit syncs at flush). - */ - async append(fact: CommitFact): Promise { - if (fact.generation <= this.head) { - throw new Error( - `fact log: non-monotonic append (generation ${fact.generation} ≤ head ${this.head})` - ) - } - if (this.manifest.tailSegment === null) { - await this.startTail(fact.generation) - } else if (this.tailBytes >= this.rotateBytes) { - await this.rotate(fact.generation) - } - const frame = encodeFrame(fact) - const tailPath = `${FACTS_PREFIX}/${this.manifest.tailSegment}` - await this.storage.appendRawBytes(tailPath, frame) - this.tailFacts.push(fact) - this.tailBytes += frame.length - this.head = fact.generation - this.dirtySegments.add(tailPath) - } - - /** Fsync every segment appended since the last sync. */ - async sync(): Promise { - if (this.dirtySegments.size === 0) return - const paths = [...this.dirtySegments] - this.dirtySegments.clear() - await this.storage.syncRawObjects(paths) - } - - /** - * Open a scan over committed facts. The scan runs against a MANIFEST - * SNAPSHOT (sealed segments + the tail's decoded facts at open) — exactly- - * once per fact, inclusive bounds, stable under concurrent appends. Gaps - * abort LOUDLY: a missing generation inside a segment's declared range is - * corruption, never silently skipped. - */ - scanFacts(options?: { - fromGeneration?: number - toGeneration?: number - kinds?: Array<'noun' | 'verb'> - batchSize?: number - /** Test override for the first-batch liveness bound (default {@link SCANFACTS_FIRST_BATCH_MS}). */ - firstBatchTimeoutMs?: number - }): FactScanHandle { - const from = options?.fromGeneration ?? 1 - const to = options?.toGeneration ?? this.head - const kinds = options?.kinds - const batchSize = Math.max(1, options?.batchSize ?? 256) - - // Snapshot: the segment list + tail content as of NOW. - const segments = this.manifest.segments.filter( - (s) => s.lastGeneration >= from && s.firstGeneration <= to - ) - const tailSnapshot = this.tailFacts.filter((f) => f.generation >= from && f.generation <= to) - const tailId = this.manifest.tailSegment ?? 'tail' - const approxFactCount = - segments.reduce((sum, s) => sum + s.facts, 0) + tailSnapshot.length - - let factsYielded = 0 - let segmentsRead = 0 - const storage = this.storage - - async function* batches(this: void): AsyncGenerator { - let expectedNext = 0 // gap detection: generations are monotonic, not necessarily dense - const emit = (facts: CommitFact[], segmentId: string, byteSize: number): FactScanBatch => ({ - facts, - firstGeneration: facts[0].generation, - lastGeneration: facts[facts.length - 1].generation, - factCount: facts.length, - byteSize, - segmentId - }) - const filterOps = (fact: CommitFact): CommitFact => - kinds - ? { ...fact, ops: fact.ops.filter((op) => kinds.includes(op.kind)) } - : fact - - for (const entry of segments) { - const bytes = await storage.readRawBytes(`${FACTS_PREFIX}/${entry.file}`) - if (bytes === null) { - throw new Error( - `fact log: sealed segment ${entry.file} is MISSING — the log is damaged; aborting scan` - ) - } - const { facts } = parseSegment(entry.file, bytes) - segmentsRead++ - const inRange = facts.filter((f) => f.generation >= from && f.generation <= to) - for (const f of inRange) { - if (f.generation <= expectedNext - 1) { - throw new Error(`fact log: out-of-order fact ${f.generation} in ${entry.file} — aborting scan`) - } - expectedNext = f.generation + 1 - } - for (let i = 0; i < inRange.length; i += batchSize) { - const slice = inRange.slice(i, i + batchSize).map(filterOps) - if (slice.length === 0) continue - factsYielded += slice.length - yield emit(slice, entry.file, slice.reduce((n, f) => n + encodeFrame(f).length, 0)) - } - } - - if (tailSnapshot.length > 0) { - segmentsRead++ - for (const f of tailSnapshot) { - if (f.generation <= expectedNext - 1) { - throw new Error(`fact log: out-of-order fact ${f.generation} in the tail — aborting scan`) - } - expectedNext = f.generation + 1 - } - for (let i = 0; i < tailSnapshot.length; i += batchSize) { - const slice = tailSnapshot.slice(i, i + batchSize).map(filterOps) - factsYielded += slice.length - yield emit(slice, tailId, slice.reduce((n, f) => n + encodeFrame(f).length, 0)) - } - } - } - - // Liveness wrapper: the FIRST pull races the contract deadline. Only the - // first — the bound is time-to-first-batch (proof the producer is alive), - // not per-batch pacing; and it runs only while a pull is actually pending, - // so consumer think-time between pulls never counts against the producer. - const firstBatchTimeoutMs = options?.firstBatchTimeoutMs ?? SCANFACTS_FIRST_BATCH_MS - async function* batchesWithLiveness(this: void): AsyncGenerator { - const inner = batches() - let timer: NodeJS.Timeout | undefined - try { - const deadline = new Promise((_, reject) => { - timer = setTimeout( - () => - reject( - new Error( - `fact log: scanFacts produced no first batch within ${firstBatchTimeoutMs}ms ` + - `(liveness contract) — the store is wedged or unreadably slow; aborting scan LOUDLY ` + - `instead of hanging the consumer.` - ) - ), - firstBatchTimeoutMs - ) - timer.unref?.() - }) - const first = await Promise.race([inner.next(), deadline]) - if (first.done) return - yield first.value - } finally { - clearTimeout(timer) - } - yield* inner - } - - return { - headGeneration: this.head, - segmentCount: segments.length + (tailSnapshot.length > 0 ? 1 : 0), - approxFactCount, - batches: batchesWithLiveness, - summary: () => ({ factsYielded, segmentsRead }) - } - } - - /** - * The mmap fast path (capability handoff): the immutable sealed segment - * files covering `fromGeneration`, in order. The TAIL is deliberately NOT - * included — it is append-mutable; consumers read it via {@link scanFacts}. - */ - segmentPaths(options?: { fromGeneration?: number }): string[] { - const from = options?.fromGeneration ?? 1 - return this.manifest.segments - .filter((s) => s.lastGeneration >= from) - .map((s) => `${FACTS_PREFIX}/${s.file}`) - } - - /** - * Drop every fact with `generation > keepThrough` — the in-session abort - * compensation: a transact appends its fact BEFORE the commit point, so a - * real (non-crash) abort after the append must take the fact back out. The - * dropped facts can only live in the TAIL (they were just appended); the - * rewrite is atomic and bounded by the rotation threshold. - */ - async dropAbove(keepThrough: number): Promise { - if (this.head <= keepThrough) return - const kept = this.tailFacts.filter((f) => f.generation <= keepThrough) - if (kept.length === this.tailFacts.length) { - throw new Error( - `fact log: dropAbove(${keepThrough}) found no droppable facts in the tail ` + - `(head ${this.head}) — the fact to drop was already sealed; the log needs reopen` - ) - } - await this.rewriteTail(kept) - this.head = this.computeHead() - } - - // -- internals ------------------------------------------------------------- - - private computeHead(): number { - if (this.tailFacts.length > 0) return this.tailFacts[this.tailFacts.length - 1].generation - const sealed = this.manifest.segments - if (sealed.length > 0) return sealed[sealed.length - 1].lastGeneration - return 0 - } - - /** Create the very first tail segment (manifest-first, then header bytes). */ - private async startTail(firstGeneration: number): Promise { - const file = segmentFileName(firstGeneration) - this.manifest.tailSegment = file - await this.persistManifest() - await this.storage.appendRawBytes(`${FACTS_PREFIX}/${file}`, buildHeader(firstGeneration)) - this.tailFacts = [] - this.tailBytes = HEADER_BYTES - } - - /** - * Seal the tail into the manifest and start a new one. Manifest-first: the - * flip both seals the old tail AND names the new one atomically, so no - * segment file ever exists unaccounted for. - */ - private async rotate(nextGeneration: number): Promise { - const sealedFile = this.manifest.tailSegment - if (!sealedFile) return - // Seal what the tail actually holds. - await this.sync() // sealed segments are always fully durable - const entry: SegmentEntry = { - file: sealedFile, - firstGeneration: this.tailFacts[0]?.generation ?? nextGeneration, - lastGeneration: this.tailFacts[this.tailFacts.length - 1]?.generation ?? nextGeneration - 1, - facts: this.tailFacts.length, - bytes: this.tailBytes - } - const newFile = segmentFileName(nextGeneration) - this.manifest.segments.push(entry) - this.manifest.tailSegment = newFile - await this.persistManifest() - await this.storage.appendRawBytes(`${FACTS_PREFIX}/${newFile}`, buildHeader(nextGeneration)) - this.tailFacts = [] - this.tailBytes = HEADER_BYTES - } - - /** Atomically persist the manifest (write-new → fsync → rename downstream). */ - private async persistManifest(): Promise { - this.manifest.updatedAt = new Date().toISOString() - await this.storage.writeRawObject(FACTS_MANIFEST_PATH, this.manifest) - await this.storage.syncRawObjects([FACTS_MANIFEST_PATH]) - } - - /** Rewrite the tail segment to hold exactly `facts` (atomic replace). */ - private async rewriteTail(facts: CommitFact[]): Promise { - const file = this.manifest.tailSegment - if (!file) return - const first = facts[0]?.generation ?? this.segmentFirstGenerationFromName(file) - const parts: Uint8Array[] = [buildHeader(first)] - for (const f of facts) parts.push(encodeFrame(f)) - const total = parts.reduce((n, p) => n + p.length, 0) - const merged = new Uint8Array(total) - let offset = 0 - for (const p of parts) { - merged.set(p, offset) - offset += p.length - } - await this.storage.writeRawBytes(`${FACTS_PREFIX}/${file}`, merged) - this.tailFacts = facts - this.tailBytes = total - } - - /** Cut a SEALED segment back to `committedGeneration` (atomic replace). */ - private async truncateSegmentTo(file: string, committedGeneration: number): Promise { - const path = `${FACTS_PREFIX}/${file}` - const bytes = await this.storage.readRawBytes(path) - if (bytes === null) return - const { facts } = parseSegment(file, bytes) - const kept = facts.filter((f) => f.generation <= committedGeneration) - prodLog.warn( - `[FactLog] truncating sealed segment ${file} to generation ${committedGeneration} ` + - `(${facts.length - kept.length} uncommitted fact(s) dropped)` - ) - const first = kept[0]?.generation ?? this.segmentFirstGenerationFromName(file) - const parts: Uint8Array[] = [buildHeader(first)] - for (const f of kept) parts.push(encodeFrame(f)) - const total = parts.reduce((n, p) => n + p.length, 0) - const merged = new Uint8Array(total) - let offset = 0 - for (const p of parts) { - merged.set(p, offset) - offset += p.length - } - await this.storage.writeRawBytes(path, merged) - } - - /** Re-derive a sealed segment's manifest entry from its actual bytes. */ - private async reloadSegmentEntry(file: string): Promise { - const bytes = await this.storage.readRawBytes(`${FACTS_PREFIX}/${file}`) - const { facts, validBytes } = bytes - ? parseSegment(file, bytes) - : { facts: [] as CommitFact[], validBytes: 0 } - return { - file, - firstGeneration: facts[0]?.generation ?? this.segmentFirstGenerationFromName(file), - lastGeneration: facts[facts.length - 1]?.generation ?? 0, - facts: facts.length, - bytes: validBytes - } - } - - /** Parse the zero-padded firstGeneration back out of a segment filename. */ - private segmentFirstGenerationFromName(file: string): number { - const match = /^seg-(\d{20})\.bfl$/.exec(file) - return match ? Number(match[1]) : 0 - } -} diff --git a/src/db/familyStamp.ts b/src/db/familyStamp.ts deleted file mode 100644 index 98342884..00000000 --- a/src/db/familyStamp.ts +++ /dev/null @@ -1,152 +0,0 @@ -/** - * @module db/familyStamp - * @description The generalized FAMILY STAMP — one JSON shape that declares, - * for any derived projection, WHICH source state it reflects and HOW to verify - * it is whole. The entity tree (canonical current-state files) carries the - * first brainy-side stamp; native index families carry the same shape. One - * verifier reads both member modes: - * - * - `enumerated` — bounded families: exact byte size per member file, - * verified at open. - * - `rollup` — unbounded families (the entity tree: millions of files): - * the verified surface is a small set of rollup invariants (entity/ - * relationship counts) plus `sourceGeneration`. - * - * `sourceGeneration` is the generation of the source-of-truth log this - * projection reflects — open-time coherence becomes a COMPARISON (stamp vs - * log head), not a walk: - * - * - equal + invariants hold → coherent, serve. - * - behind → the projection missed the tail (crash between commit and stamp); - * for the Stage-1 tree this is benign by construction (the tree is written - * BY the commit), so the stamp refreshes; a DERIVED projection would replay - * the gap instead. - * - invariants FAIL at equal generation → genuine incoherence: loud, and the - * repair ritual (`repairIndex()`, whose recount rebuilds the rollups from a - * canonical walk) heals it. - * - * Stamps are JSON on purpose — every incident gets debugged by reading a - * stamp in a terminal. - */ - -/** Storage-root-relative directory holding family stamps. */ -export const FAMILY_STAMPS_PREFIX = '_system/family-stamps' - -/** The entity tree's stamp path. */ -export const ENTITY_TREE_STAMP_PATH = `${FAMILY_STAMPS_PREFIX}/entity-tree.json` - -/** One enumerated member: a file and its exact expected byte size. */ -export interface EnumeratedMember { - path: string - bytes: number -} - -/** - * The stamp's verified surface, in one of the two member modes. Rollup - * invariant values may be numbers (counts, byte sizes) or strings (content - * fingerprints, e.g. a per-tree SHA-256) — the verifier compares by strict - * equality either way, so a type mismatch reads as incoherence, never a pass. - */ -export type StampMembers = - | { mode: 'enumerated'; files: EnumeratedMember[] } - | { mode: 'rollup'; invariants: Record } - -/** The generalized family stamp (one shape, one verifier, both engines). */ -export interface FamilyStamp { - /** Which projection this stamps (e.g. `'entity-tree'`). */ - family: string - /** Monotonic per-family stamp generation — bumps on every committed stamp. */ - generation: number - /** ISO timestamp of the stamp write. */ - committedAt: string - /** The source-of-truth generation this projection reflects. */ - sourceGeneration: number - /** The verified surface. */ - members: StampMembers -} - -/** The verdict of an open-time stamp verification. */ -export type StampVerdict = - | { state: 'coherent' } - | { state: 'absent' } // legacy store — first stamp writes at the next flush - | { state: 'behind'; stampSource: number; head: number } - | { state: 'incoherent'; failures: string[] } - | { state: 'unverifiable'; reason: string } // a FAULT reading the stamp — never conflated with absence - -/** The narrow storage surface stamps ride (JSON objects + fsync). */ -export interface StampStorage { - readRawObject(path: string): Promise - writeRawObject(path: string, data: any): Promise - syncRawObjects(paths: string[]): Promise -} - -/** Read a family's stamp; `null` when none was ever written. */ -export async function readFamilyStamp( - storage: StampStorage, - path: string -): Promise { - const stored = (await storage.readRawObject(path)) as FamilyStamp | null - if (!stored || typeof stored !== 'object' || typeof stored.family !== 'string') return null - return stored -} - -/** Write a family's stamp durably (atomic object write + fsync). */ -export async function writeFamilyStamp( - storage: StampStorage, - path: string, - stamp: Omit & { generation?: number } -): Promise { - const prior = await readFamilyStamp(storage, path) - const full: FamilyStamp = { - ...stamp, - generation: (prior?.generation ?? 0) + 1, - committedAt: new Date().toISOString() - } - await storage.writeRawObject(path, full) - await storage.syncRawObjects([path]) -} - -/** - * The ONE verifier, both member modes. `actual` supplies the observed rollup - * values (rollup mode) or file sizes (enumerated mode, keyed by path); - * `head` is the source-of-truth generation now. - */ -export function verifyFamilyStamp( - stamp: FamilyStamp | null, - head: number, - actual: Record -): StampVerdict { - if (stamp === null) return { state: 'absent' } - if (stamp.sourceGeneration > head) { - // A stamp AHEAD of the log claims state that never committed — the - // projection was stamped against truth that a crash rolled back. - return { - state: 'incoherent', - failures: [`sourceGeneration ${stamp.sourceGeneration} is ahead of the log head ${head}`] - } - } - if (stamp.sourceGeneration < head) { - return { state: 'behind', stampSource: stamp.sourceGeneration, head } - } - const failures: string[] = [] - if (stamp.members.mode === 'rollup') { - for (const [name, expected] of Object.entries(stamp.members.invariants)) { - const observed = actual[name] - if (observed === undefined) { - failures.push(`rollup invariant '${name}' has no observed value`) - } else if (observed !== expected) { - failures.push(`rollup invariant '${name}': stamped ${expected}, observed ${observed}`) - } - } - } else { - for (const member of stamp.members.files) { - const observed = actual[member.path] - if (observed === undefined) { - failures.push(`member '${member.path}' is missing`) - } else if (observed !== member.bytes) { - failures.push(`member '${member.path}': stamped ${member.bytes} bytes, observed ${observed}`) - } - } - } - return failures.length > 0 ? { state: 'incoherent', failures } : { state: 'coherent' } -} diff --git a/src/db/generationSegments.ts b/src/db/generationSegments.ts deleted file mode 100644 index 0c14b60c..00000000 --- a/src/db/generationSegments.ts +++ /dev/null @@ -1,459 +0,0 @@ -/** - * @module db/generationSegments - * @description The generation-segment store — Stage-2 D1+D3+repacking's file - * format (co-frozen 2026-07-19; design: the d1-d3-repacking spec). - * - * Packs CONSECUTIVE cold generations' record-sets (before-images + delta) - * into append-once segment files with derived sidecar indexes, so history - * scales in SEGMENTS (tens) instead of FILES-PER-GENERATION (hundreds of - * thousands), and cold-open reads ONE manifest instead of listing the - * backlog. Layout under `_generations/segments/`: - * - * - `seg-.bgs` — magic "BGS1", then one frame per - * generation: `u32 payloadLen | u32 crc32c | msgpack payload`. Payload is - * POSITIONAL: `[generation, timestamp, delta, records[], flags]` with - * records `[kindByte, id, record]`. `flags` reserves encoding evolution - * (bit 0 = compressed payload — v1 always 0; a future writer upgrade, - * never a format break). Sealed segments are IMMUTABLE — the fact log's - * own law, generalized. - * - `seg-.idx` — DERIVED sidecar (msgpack): per-generation frame - * offsets (point reads = one ranged read, never a listing) + per-id - * generation postings (per-id chain rebuilds read only what they need). - * Corrupt/missing → rebuilt from its segment in one sequential read, - * loudly. - * - `manifest.json` — the segment catalogue + `compactedBelow` (D3's - * horizon marker). Cold-open reads THIS; the packed backlog is never - * listed. - * - * D3 semantics carried here: bounded-retention reclaim drops WHOLE segments - * at boundaries (O(1) per segment, no rewrite); under the archival profile - * (`retention: 'all'`) nothing here is ever dropped — folding is the only - * transform (re-representation, never deletion). - */ - -import { encode as msgpackEncode, decode as msgpackDecode } from '@msgpack/msgpack' -import { crc32c } from '../utils/crc32c.js' -import type { FactLogStorage } from './factLog.js' -import { prodLog } from '../utils/logger.js' - -/** Directory for segment files + manifest, under the generations prefix. */ -export const SEGMENTS_PREFIX = '_generations/segments' - -/** Target sealed-segment size (co-freeze proposal; tunable on evidence). */ -export const SEGMENT_TARGET_BYTES = 64 * 1024 * 1024 - -const MAGIC = new TextEncoder().encode('BGS1') -const FRAME_PREFIX_BYTES = 8 // u32 payloadLen + u32 crc32c -const MANIFEST_PATH = `${SEGMENTS_PREFIX}/manifest.json` - -/** One generation's fold input — exactly what the live tier holds for it. */ -export interface FoldGeneration { - generation: number - timestamp: number - /** The tx.json delta object, carried verbatim. */ - delta: unknown - /** The before-image record-set (empty for record-less generations). */ - records: Array<{ kind: 'noun' | 'verb'; id: string; record: unknown }> -} - -/** Manifest entry for one sealed segment. */ -export interface SegmentMeta { - file: string - firstGeneration: number - lastGeneration: number - frames: number - bytes: number - /** crc32c of the full segment byte stream — the digest chain's link. */ - checksum: number -} - -interface SegmentManifest { - version: 1 - compactedBelow: number - segments: SegmentMeta[] -} - -interface SidecarIndex { - version: 1 - /** [generation, frameOffset, frameLen] ascending by generation. */ - generations: Array<[number, number, number]> - /** `${kindByte}:${id}` → ascending generations holding a record for it. */ - ids: Record -} - -const segmentFileName = (firstGeneration: number): string => - `seg-${String(firstGeneration).padStart(20, '0')}.bgs` -const sidecarFileName = (firstGeneration: number): string => - `seg-${String(firstGeneration).padStart(20, '0')}.idx` - -/** - * The generation-segment store. Owns the packed tier ONLY — the live - * per-generation tier and the routing between tiers belong to - * `GenerationStore`. All mutating entry points here are called under the - * generation store's commit mutex. - */ -export class GenerationSegmentStore { - private readonly storage: FactLogStorage - private manifest: SegmentManifest = { version: 1, compactedBelow: 0, segments: [] } - /** Sidecar cache — segments are immutable, so entries never invalidate. */ - private readonly sidecars = new Map() - - constructor(storage: FactLogStorage) { - this.storage = storage - } - - /** Load the manifest (ONE read — never a directory listing). */ - async open(): Promise { - const raw = (await this.storage.readRawObject(MANIFEST_PATH)) as SegmentManifest | null - if (raw) { - if (raw.version !== 1) { - throw new Error( - `[GenerationSegments] manifest version ${String(raw.version)} is newer than this ` + - `engine understands — refusing to serve partial history. Upgrade the engine.` - ) - } - this.manifest = raw - } - } - - /** The packed tier's catalogue (ascending, immutable snapshot). */ - segments(): readonly SegmentMeta[] { - return this.manifest.segments - } - - /** D3's horizon marker: generations below this were reclaimed (bounded profiles only). */ - compactedBelow(): number { - return this.manifest.compactedBelow - } - - /** The covering sealed segment for `gen`, or null if it lives outside the packed tier. */ - private coveringSegment(gen: number): SegmentMeta | null { - // Manifest is ascending and ranges never overlap — binary search. - const segs = this.manifest.segments - let lo = 0 - let hi = segs.length - 1 - while (lo <= hi) { - const mid = (lo + hi) >> 1 - const s = segs[mid] - if (gen < s.firstGeneration) hi = mid - 1 - else if (gen > s.lastGeneration) lo = mid + 1 - else return s - } - return null - } - - /** True when `gen` is packed (readable from this tier). */ - hasGeneration(gen: number): boolean { - return this.coveringSegment(gen) !== null - } - - /** - * Fold consecutive generations into ONE new sealed segment + sidecar and - * append it to the manifest atomically. Caller guarantees: `gens` is - * ascending, contiguous with the packed tier (first = last packed + 1 when - * segments exist), and already durable in the live tier. Crash between the - * segment write and the caller's live-tier delete leaves a DUPLICATE - * representation — resolved live-tier-wins by the reader; never a gap. - */ - async fold(gens: FoldGeneration[]): Promise { - if (gens.length === 0) { - throw new Error('[GenerationSegments] fold() requires at least one generation') - } - for (let i = 1; i < gens.length; i++) { - if (gens[i].generation <= gens[i - 1].generation) { - throw new Error('[GenerationSegments] fold() input must be strictly ascending') - } - } - const last = this.manifest.segments[this.manifest.segments.length - 1] - if (last && gens[0].generation <= last.lastGeneration) { - throw new Error( - `[GenerationSegments] fold() overlaps the packed tier: ${gens[0].generation} ≤ ` + - `sealed ${last.lastGeneration} — segments are immutable, never rewritten` - ) - } - - const first = gens[0].generation - const file = segmentFileName(first) - const sidecar: SidecarIndex = { version: 1, generations: [], ids: {} } - - // Encode all frames, tracking offsets for the sidecar. - const parts: Uint8Array[] = [MAGIC] - let offset = MAGIC.length - for (const g of gens) { - const payload = msgpackEncode([ - g.generation, - g.timestamp, - g.delta, - g.records.map((r) => [r.kind === 'noun' ? 0 : 1, r.id, r.record]), - 0 // flags: v1 = uncompressed - ]) - const frame = new Uint8Array(FRAME_PREFIX_BYTES + payload.length) - const view = new DataView(frame.buffer) - view.setUint32(0, payload.length, true) - view.setUint32(4, crc32c(payload), true) - frame.set(payload, FRAME_PREFIX_BYTES) - sidecar.generations.push([g.generation, offset, frame.length]) - for (const r of g.records) { - const key = `${r.kind === 'noun' ? 0 : 1}:${r.id}` - ;(sidecar.ids[key] ??= []).push(g.generation) - } - parts.push(frame) - offset += frame.length - } - const total = parts.reduce((n, p) => n + p.length, 0) - const bytes = new Uint8Array(total) - let at = 0 - for (const p of parts) { - bytes.set(p, at) - at += p.length - } - - const meta: SegmentMeta = { - file, - firstGeneration: first, - lastGeneration: gens[gens.length - 1].generation, - frames: gens.length, - bytes: total, - checksum: crc32c(bytes) - } - - // Durability order: segment + sidecar fsync'd BEFORE the manifest names - // them (a crash before the manifest = invisible orphan files, harmless); - // manifest last, atomically. - const segPath = `${SEGMENTS_PREFIX}/${file}` - const idxPath = `${SEGMENTS_PREFIX}/${sidecarFileName(first)}` - await this.storage.writeRawBytes(segPath, bytes) - await this.storage.writeRawBytes(idxPath, msgpackEncode(sidecar)) - await this.storage.syncRawObjects([segPath, idxPath]) - const next: SegmentManifest = { - ...this.manifest, - segments: [...this.manifest.segments, meta] - } - await this.storage.writeRawObject(MANIFEST_PATH, next) - await this.storage.syncRawObjects([MANIFEST_PATH]) - this.manifest = next - this.sidecars.set(file, sidecar) - return meta - } - - /** Load (or rebuild, loudly) a segment's sidecar. */ - private async sidecarFor(meta: SegmentMeta): Promise { - const cached = this.sidecars.get(meta.file) - if (cached) return cached - const idxPath = `${SEGMENTS_PREFIX}/${sidecarFileName(meta.firstGeneration)}` - const raw = await this.storage.readRawBytes(idxPath) - if (raw) { - try { - const idx = msgpackDecode(raw) as SidecarIndex - if (idx.version === 1) { - this.sidecars.set(meta.file, idx) - return idx - } - } catch { - // fall through to rebuild - } - } - // Sidecars are DERIVED: rebuild from the segment, loudly — never serve - // wrong offsets silently. - prodLog.warn( - `[GenerationSegments] sidecar for ${meta.file} missing or unreadable — rebuilding from the segment` - ) - const rebuilt = await this.rebuildSidecar(meta) - await this.storage.writeRawBytes(idxPath, msgpackEncode(rebuilt)) - this.sidecars.set(meta.file, rebuilt) - return rebuilt - } - - /** One sequential read of the segment → a fresh sidecar. Verifies every frame CRC. */ - private async rebuildSidecar(meta: SegmentMeta): Promise { - const frames = await this.readAllFrames(meta) - const idx: SidecarIndex = { version: 1, generations: [], ids: {} } - for (const f of frames) { - idx.generations.push([f.generation, f.offset, f.frameLen]) - for (const r of f.records) { - const key = `${r.kind === 'noun' ? 0 : 1}:${r.id}` - ;(idx.ids[key] ??= []).push(f.generation) - } - } - return idx - } - - private decodeFrame( - payload: Uint8Array - ): { generation: number; timestamp: number; delta: unknown; records: FoldGeneration['records'] } { - const [generation, timestamp, delta, rawRecords] = msgpackDecode(payload) as [ - number, - number, - unknown, - Array<[number, string, unknown]>, - number - ] - return { - generation, - timestamp, - delta, - records: rawRecords.map(([kindByte, id, record]) => ({ - kind: kindByte === 0 ? ('noun' as const) : ('verb' as const), - id, - record - })) - } - } - - private async readAllFrames(meta: SegmentMeta): Promise< - Array & { offset: number; frameLen: number }> - > { - const bytes = await this.storage.readRawBytes(`${SEGMENTS_PREFIX}/${meta.file}`) - if (!bytes) { - throw new Error( - `[GenerationSegments] sealed segment ${meta.file} is MISSING — packed history is damaged; ` + - `refusing to continue silently` - ) - } - const out: Array & { offset: number; frameLen: number }> = [] - let at = MAGIC.length - const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) - while (at + FRAME_PREFIX_BYTES <= bytes.length) { - const payloadLen = view.getUint32(at, true) - const crc = view.getUint32(at + 4, true) - const payload = bytes.subarray(at + FRAME_PREFIX_BYTES, at + FRAME_PREFIX_BYTES + payloadLen) - if (payload.length !== payloadLen || crc32c(payload) !== crc) { - throw new Error( - `[GenerationSegments] frame CRC mismatch in ${meta.file} at offset ${at} — ` + - `packed history is damaged; refusing to serve it` - ) - } - out.push({ ...this.decodeFrame(payload), offset: at, frameLen: FRAME_PREFIX_BYTES + payloadLen }) - at += FRAME_PREFIX_BYTES + payloadLen - } - return out - } - - /** Read one packed generation's frame via its sidecar offset (one ranged read). */ - private async readFrame( - gen: number - ): Promise | null> { - const meta = this.coveringSegment(gen) - if (!meta) return null - const idx = await this.sidecarFor(meta) - // generations ascending → binary search. - const gens = idx.generations - let lo = 0 - let hi = gens.length - 1 - while (lo <= hi) { - const mid = (lo + hi) >> 1 - if (gens[mid][0] < gen) lo = mid + 1 - else if (gens[mid][0] > gen) hi = mid - 1 - else { - const [, offset, frameLen] = gens[mid] - const bytes = await this.storage.readRawBytes(`${SEGMENTS_PREFIX}/${meta.file}`) - if (!bytes) { - throw new Error(`[GenerationSegments] sealed segment ${meta.file} is MISSING`) - } - const frame = bytes.subarray(offset, offset + frameLen) - const view = new DataView(frame.buffer, frame.byteOffset, frame.byteLength) - const payloadLen = view.getUint32(0, true) - const crc = view.getUint32(4, true) - const payload = frame.subarray(FRAME_PREFIX_BYTES, FRAME_PREFIX_BYTES + payloadLen) - if (payload.length !== payloadLen || crc32c(payload) !== crc) { - throw new Error( - `[GenerationSegments] frame CRC mismatch for generation ${gen} in ${meta.file} — ` + - `packed history is damaged; refusing to serve it` - ) - } - return this.decodeFrame(payload) - } - } - // In the covering range but not present: the packed tier is dense by - // construction (fold packs every generation it is handed, including - // record-less ones) — absence inside a sealed range is damage. - throw new Error( - `[GenerationSegments] generation ${gen} is inside sealed segment ${meta.file}'s declared ` + - `range but has no frame — packed history is damaged` - ) - } - - /** The packed tier's delta for `gen` (null = not packed). */ - async readDelta(gen: number): Promise<{ delta: unknown; timestamp: number } | null> { - const frame = await this.readFrame(gen) - return frame ? { delta: frame.delta, timestamp: frame.timestamp } : null - } - - /** The packed tier's full record-set for `gen` (null = not packed). */ - async readRecords(gen: number): Promise { - const frame = await this.readFrame(gen) - return frame ? frame.records : null - } - - /** One packed before-image (null = not packed OR no record for the id in that generation). */ - async readRecord(gen: number, kind: 'noun' | 'verb', id: string): Promise { - const frame = await this.readFrame(gen) - if (!frame) return null - const hit = frame.records.find((r) => r.kind === kind && r.id === id) - return hit ? hit.record : null - } - - /** - * D3 reclaim: drop WHOLE segments whose lastGeneration < `belowGeneration` - * and bump `compactedBelow`. Partial segments are never dropped — the - * boundary waits. NEVER called under the archival profile (the caller - * enforces retention semantics; this method only executes boundary drops). - */ - async dropSegmentsBelow(belowGeneration: number): Promise<{ dropped: number; compactedBelow: number }> { - const keep: SegmentMeta[] = [] - const drop: SegmentMeta[] = [] - for (const s of this.manifest.segments) { - ;(s.lastGeneration < belowGeneration ? drop : keep).push(s) - } - if (drop.length === 0) { - return { dropped: 0, compactedBelow: this.manifest.compactedBelow } - } - const compactedBelow = Math.max( - this.manifest.compactedBelow, - drop[drop.length - 1].lastGeneration + 1 - ) - // Manifest first (the drop is authoritative once named), then bytes — - // a crash between leaves orphan segment files invisible to the manifest, - // harmless and re-collectable. - const next: SegmentManifest = { ...this.manifest, compactedBelow, segments: keep } - await this.storage.writeRawObject(MANIFEST_PATH, next) - await this.storage.syncRawObjects([MANIFEST_PATH]) - this.manifest = next - for (const s of drop) { - await this.storage.deleteRawObject(`${SEGMENTS_PREFIX}/${s.file}`) - await this.storage.deleteRawObject(`${SEGMENTS_PREFIX}/${sidecarFileName(s.firstGeneration)}`) - this.sidecars.delete(s.file) - } - return { dropped: drop.length, compactedBelow } - } - - /** - * D8 rider — the packed portion of `generationDigest(g)`: a deterministic - * crc32c chain over sealed-segment checksums fully below `g`, plus the - * frame CRC of `g`'s own frame when `g` is mid-segment. O(segments), not - * O(generations); identical history ⇒ identical digest on any machine. - * The live-tier portion is composed by the caller. - */ - async digestThroughPacked(g: number): Promise { - let digest = 0 - let covered = false - for (const s of this.manifest.segments) { - if (s.lastGeneration <= g) { - digest = crc32c(new TextEncoder().encode(`${digest}:${s.checksum}`)) - if (s.lastGeneration === g) covered = true - } else if (s.firstGeneration <= g) { - // g is mid-segment: chain the partial prefix via g's frame CRC. - const frame = await this.readFrame(g) - if (frame === null) return null - const idx = await this.sidecarFor(s) - const upTo = idx.generations.filter(([gen]) => gen <= g) - for (const [gen, offset, frameLen] of upTo) { - digest = crc32c(new TextEncoder().encode(`${digest}:${gen}:${offset}:${frameLen}`)) - } - covered = true - break - } - } - return covered || this.manifest.segments.length > 0 ? digest : null - } -} diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index aede17a4..bd83684d 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -32,9 +32,7 @@ */ import { prodLog } from '../utils/logger.js' -import { GenerationCompactedError, GenerationConflictError, PendingFlushDurabilityError, StoreInconsistentError } from './errors.js' -import type { UnreconciledRecord } from './errors.js' -import { TransactionRollbackError } from '../transaction/errors.js' +import { GenerationCompactedError, GenerationConflictError } from './errors.js' import type { ChangedIds, CompactHistoryOptions, @@ -45,23 +43,6 @@ import type { GenerationStorage, TxLogEntry } from './types.js' -import { FactLog, storageSupportsFactLog, type CommitFact, type FactOp } from './factLog.js' -import { GenerationSegmentStore, type FoldGeneration } from './generationSegments.js' -import { crc32c } from '../utils/crc32c.js' - -/** - * The byte-identical before-images of every id a commit touches, read UNDER - * the commit mutex immediately before the write applies. Passed to a commit's - * optional `precommit` hook so callers can enforce compare-and-swap - * preconditions (e.g. the per-entity `ifRev` check) atomically with the apply - * — the same conditional-commit idea as `ifAtGeneration`, generalized to - * arbitrary per-record predicates. An absent id maps to the create sentinel - * (`metadata: null, vector: null`). - */ -export interface CommitBeforeImages { - nouns: Map - verbs: Map -} /** Storage-root-relative path of the persisted generation counter. */ export const GENERATION_COUNTER_PATH = '_system/generation.json' @@ -124,16 +105,6 @@ export interface GenerationStoreOpenResult { export class GenerationStore { private readonly storage: GenerationStorage - /** - * The generation FACT LOG (dual-write transition) — an append-only, - * CRC-framed record of every committed generation as an AFTER-IMAGE fact. - * `null` when the storage layer lacks the binary raw-byte primitives. - * Appends ride the same commit protocol: a fact-append failure FAILS the - * write (loud — a silent fact gap would make the log a lie that a later - * replay discovers), and open() reconciles the log to committed truth. - */ - private factLog: FactLog | null = null - /** Latest reserved/observed generation (≥ {@link committed}). */ private counter = 0 /** Committed-transaction watermark (manifest generation). */ @@ -259,30 +230,6 @@ export class GenerationStore { */ private deltaCacheMax = 4096 - /** - * Running total of on-disk history bytes across committed generations — - * `null` until {@link historyBytes} pays its one seeding walk. Maintained - * incrementally at commit/reclaim so the adaptive retention check on every - * flush() is O(1), never a tail re-walk. Never updated by cache re-reads - * ({@link setDelta} inserts are cache population, not new history). - */ - private historyBytesTotal: number | null = null - - /** - * The packed tier (D1+D3): sealed segments holding folded cold - * generations. Null until {@link open} wires it (and on storage adapters - * without raw-byte primitives — the live tier then carries everything, - * exactly as before the packed tier existed). - */ - private segments: GenerationSegmentStore | null = null - - /** - * Live-tier window: generations newer than `committed - REPACK_LIVE_WINDOW` - * are never folded — the hot tail stays in the per-generation layout the - * write path owns. Matches the resident chain window's scale. - */ - static readonly REPACK_LIVE_WINDOW = 1024 - /** * Model-B per-write group-commit — the in-memory PENDING tier. * @@ -321,26 +268,6 @@ export class GenerationStore { /** Coalescing window (ms) before an idle pending tier is flushed. */ private static readonly PENDING_FLUSH_DELAY_MS = 50 - /** - * Latched pending-flush durability failure. Set once background flushes have - * failed {@link PENDING_FLUSH_FAILURE_THRESHOLD} consecutive times (a real, - * persistent storage fault — not a blip); while set, writes are REFUSED with - * a {@link PendingFlushDurabilityError} rather than silently accumulate - * un-durable history in memory. Cleared the moment a flush finally succeeds - * (the store self-heals). `null` = history-durability path is healthy. - */ - private pendingFlushError: Error | null = null - /** Consecutive failed background pending-flush attempts (resets on success). */ - private pendingFlushFailures = 0 - /** - * Consecutive failed flush attempts before the durability latch trips and - * writes start refusing. A small tolerance so a single transient fault does - * not trip the alarm, while a genuinely stuck disk stops the bleed fast. - */ - private static readonly PENDING_FLUSH_FAILURE_THRESHOLD = 3 - /** Upper bound (ms) on the exponential retry backoff between failed flushes. */ - private static readonly PENDING_FLUSH_RETRY_CAP_MS = 30_000 - /** Live pin refcounts, keyed by pinned generation. */ private readonly pins = new Map() @@ -437,46 +364,6 @@ export class GenerationStore { this.opened = true - // Generation FACT LOG (dual-write transition): when the storage layer - // exposes the binary raw-byte primitives, open the after-image fact log - // and reconcile it to committed truth — facts are appended BEFORE the - // commit point, so a crash can only leave the log AHEAD; open truncates - // any fact beyond `committed`. Storage without the primitives simply - // hosts no fact log (readers fall back to canonical enumeration). - if (storageSupportsFactLog(this.storage)) { - this.factLog = new FactLog(this.storage) - await this.factLog.open(this.committed) - } else { - this.factLog = null - } - - // PACKED TIER (D1+D3): same capability gate as the fact log. Opening - // reads ONE manifest — never a listing of the packed backlog — and seeds - // committedRanges with the sealed ranges so packed generations resolve - // exactly like live ones. - if (storageSupportsFactLog(this.storage)) { - this.segments = new GenerationSegmentStore(this.storage) - await this.segments.open() - const packedRanges = this.segments - .segments() - .map((s): [number, number] => [s.firstGeneration, Math.min(s.lastGeneration, this.committed)]) - .filter(([lo, hi]) => lo <= hi) - if (packedRanges.length > 0) { - // Merge packed (older) + live (newer) interval sets — both ascending; - // coalesce adjacency so range arithmetic stays interval-exact. - const merged: Array<[number, number]> = [] - for (const r of [...packedRanges, ...this.committedRanges].sort((a, b) => a[0] - b[0])) { - const last = merged[merged.length - 1] - if (last && r[0] <= last[1] + 1) last[1] = Math.max(last[1], r[1]) - else merged.push([r[0], r[1]]) - } - this.committedRanges = merged - } - this.horizonGen = Math.max(this.horizonGen, this.segments.compactedBelow() - 1) - } else { - this.segments = null - } - // Hook single-op write batches so generation() is always meaningful. // Suppressed while a transact batch executes (the batch is ONE generation). if (!options?.readOnly) { @@ -536,110 +423,6 @@ export class GenerationStore { return this.horizonGen } - /** - * @description Read-only history footprint for fleet audits: how much - * generational history this store holds on disk. `bytes` pays (and seeds) - * the one-time {@link historyBytes} walk on first call — subsequent calls - * are O(1). The oldest/newest timestamps come from those generations' - * deltas (cache-bounded reads). - * @returns Counts, bytes, generation range, and the compaction horizon. - */ - /** - * @description D8 (gate-to-generation provenance): a deterministic content - * digest of the generation log THROUGH `g` — identical history ⇒ identical - * digest on any machine; any divergence (different records, different - * order, reclaimed range) ⇒ different digest. Composed from the packed - * tier's sealed-segment checksum chain (O(segments)) plus the live tier's - * per-generation delta digests (O(live window at most)). Release gates pin - * {generation, digest} and verify both at execution time. - * @param g - The generation to digest through (≤ committed). - * @returns A hex digest string, stable across reopen and repacking states - * ONLY for fully-packed prefixes — repacking changes representation, so - * the composed digest is defined over CONTENT: live-tier gens hash their - * delta + record ids, packed gens hash via frame CRCs. A gate should pin - * after a repack pass for long-term stability, or re-pin on repack. - */ - async generationDigest(g: number): Promise { - if (!Number.isInteger(g) || g < 1 || g > this.committed) { - throw new RangeError( - `generationDigest(): generation ${g} is out of range [1, ${this.committed}]` - ) - } - if (g <= this.horizonGen) { - throw new GenerationCompactedError(g, this.horizonGen) - } - let digest = 0 - const enc = new TextEncoder() - if (this.segments) { - const packed = await this.segments.digestThroughPacked(g) - if (packed !== null) digest = packed - } - // Live-tier composition: every committed gen ≤ g not covered by a sealed - // segment hashes its delta content in ascending order. - for (const gen of this.committedGensAsc()) { - if (gen > g) break - if (this.segments?.hasGeneration(gen)) continue - const delta = await this.getDelta(gen) - digest = crc32c( - enc.encode( - `${digest}:${gen}:${delta.timestamp}:${[...delta.nouns].sort().join(',')}:${[...delta.verbs].sort().join(',')}` - ) - ) - } - return digest.toString(16).padStart(8, '0') - } - - async historyStats(): Promise<{ - generations: number - bytes: number - oldestGeneration: number | null - newestGeneration: number | null - oldestTimestamp: number | null - newestTimestamp: number | null - horizon: number - }> { - let oldest: number | null = null - let newest: number | null = null - for (const gen of this.committedGensAsc()) { - if (oldest === null) oldest = gen - newest = gen - } - return { - generations: this.committedCount(), - bytes: await this.historyBytes(), - oldestGeneration: oldest, - newestGeneration: newest, - oldestTimestamp: oldest !== null ? (await this.getDelta(oldest)).timestamp : null, - newestTimestamp: newest !== null ? (await this.getDelta(newest)).timestamp : null, - horizon: this.horizonGen - } - } - - /** - * @description Read one generation's persisted before-image records — the - * compaction fallback for generations written before deltas carried - * `blobHashes`. O(that generation's records). - * @param gen - The generation whose `prev/` records to read. - * @returns The record-set (empty when absent). - */ - private async readGenerationRecords(gen: number): Promise { - let paths: string[] = [] - try { - paths = await this.storage.listRawObjects(`${GENERATIONS_PREFIX}/${gen}/prev`) - } catch { - paths = [] - } - const records: GenerationRecord[] = [] - for (const p of paths) { - const record = (await this.storage.readRawObject(p)) as GenerationRecord | null - if (record) records.push(record) - } - if (records.length > 0) return records - // Two-tier: folded generations serve their record-set from the segment. - const packed = await this.segments?.readRecords(gen) - return packed ? (packed.map((r) => r.record) as GenerationRecord[]) : [] - } - /** * @description Single-operation write hook (registered with the storage * layer in {@link open}). Bumps the in-memory counter and schedules a @@ -759,75 +542,13 @@ export class GenerationStore { * @returns The committed generation and its commit timestamp. * @throws GenerationConflictError when the CAS expectation fails. */ - /** - * The generation fact log, or `null` when the storage layer cannot host one. - * Consumers scan committed facts through it (`scanFacts` / `segmentPaths`). - */ - getFactLog(): FactLog | null { - return this.factLog - } - - /** - * @description Build one commit's AFTER-IMAGE fact by reading canonical - * state back for every touched id — under the commit mutex, immediately - * after the operations applied, so canonical IS the after-image (and the - * reads are page-cache-warm: the operations just wrote these files). An - * absent id (both legs null) becomes a body-less TOMBSTONE — the delete - * fact needs no body, so removal never requires reading the removed thing. - * The fact's blobHashes are extracted from the AFTER records (the content - * this generation's state references), unlike the history path's - * before-image hashes. - */ - private async buildCommitFact(args: { - generation: number - timestamp: number - nouns: string[] - verbs: string[] - meta?: Record - }): Promise { - const ops: FactOp[] = [] - const afterRecords: GenerationRecord[] = [] - for (const id of args.nouns) { - const after = await this.storage.readNounRaw(id) - const absent = after.metadata === null && after.vector === null - ops.push({ kind: 'noun', id, record: absent ? null : after }) - if (!absent) afterRecords.push({ kind: 'noun', metadata: after.metadata, vector: after.vector }) - } - for (const id of args.verbs) { - const after = await this.storage.readVerbRaw(id) - const absent = after.metadata === null && after.vector === null - ops.push({ kind: 'verb', id, record: absent ? null : after }) - if (!absent) afterRecords.push({ kind: 'verb', metadata: after.metadata, vector: after.vector }) - } - const blobHashes = this.storage.extractBlobHashesFromRecords - ? this.storage.extractBlobHashesFromRecords(afterRecords) - : [] - return { - generation: args.generation, - timestamp: args.timestamp, - ops, - ...(args.meta ? { meta: args.meta } : {}), - ...(blobHashes.length > 0 ? { blobHashes } : {}) - } - } - async commitTransaction(args: { touched: TouchedIds meta?: Record ifAtGeneration?: number - /** Optional compare-and-swap precondition over the {@link CommitBeforeImages}, - * run under the commit mutex BEFORE anything is staged or applied — the - * per-record analogue of `ifAtGeneration`. A throw aborts the whole batch: - * the generation reservation is returned and no staging I/O has happened. */ - precommit?: (before: CommitBeforeImages) => void execute: () => Promise }): Promise<{ generation: number; timestamp: number }> { return this.withMutex(async () => { - // A latched history-durability failure compromises the whole generation - // spine — refuse a transact too (advancing the manifest past stuck, - // un-durable single-op generations would be inconsistent). Same loud - // error; self-clears when the pending tier drains. - this.assertHistoryDurable() if (args.ifAtGeneration !== undefined && args.ifAtGeneration !== this.counter) { throw new GenerationConflictError(args.ifAtGeneration, this.counter) } @@ -852,59 +573,22 @@ export class GenerationStore { } } - // Temporal-blob contract: hashes this record-set references, recorded - // before staging; scoped outside the try so an abort can compensate. - let txBlobHashes: string[] = [] - - // Hoisted so the catch can reconcile canonical state against them after a - // failed rollback (the trapdoor). Empty until populated below. - const nounBefore = new Map() - const verbBefore = new Map() - try { // -- 3. Before-images + delta (the durable undo log) ------------------ - // Read every before-image FIRST, then run the caller's CAS - // precondition against them, and only then stage to disk — so a - // conflicting batch aborts with zero staging I/O. The maps hold the - // byte-identical records the staged files are written from. - for (const id of nouns) { - const prev = await this.storage.readNounRaw(id) - nounBefore.set(id, { kind: 'noun', metadata: prev.metadata, vector: prev.vector }) - } - for (const id of verbs) { - const prev = await this.storage.readVerbRaw(id) - verbBefore.set(id, { kind: 'verb', metadata: prev.metadata, vector: prev.vector }) - } - - // Conditional commit: the per-record CAS analogue of the - // `ifAtGeneration` check above, but against authoritative - // before-images under the mutex. A throw lands in the catch below, - // which returns the generation reservation; nothing was applied. - args.precommit?.({ nouns: nounBefore, verbs: verbBefore }) - - // Temporal-blob contract: count this record-set's content-hash - // references BEFORE it is staged (over-count-only crash ordering — - // see flushPendingSingleOps' matching note). - txBlobHashes = this.storage.extractBlobHashesFromRecords - ? this.storage.extractBlobHashesFromRecords([ - ...nounBefore.values(), - ...verbBefore.values() - ]) - : [] - if (txBlobHashes.length > 0 && this.storage.recordHistoryBlobReferences) { - await this.storage.recordHistoryBlobReferences(txBlobHashes) - } - const stagedPaths: string[] = [] let recordBytes = 0 // serialized record-set size, for retention accounting - for (const [id, record] of nounBefore) { + for (const id of nouns) { + const prev = await this.storage.readNounRaw(id) const recordPath = `${dir}/prev/${id}.json` + const record: GenerationRecord = { kind: 'noun', metadata: prev.metadata, vector: prev.vector } await this.storage.writeRawObject(recordPath, record) recordBytes += serializedBytes(record) stagedPaths.push(recordPath) } - for (const [id, record] of verbBefore) { + for (const id of verbs) { + const prev = await this.storage.readVerbRaw(id) const recordPath = `${dir}/prev/${id}.json` + const record: GenerationRecord = { kind: 'verb', metadata: prev.metadata, vector: prev.vector } await this.storage.writeRawObject(recordPath, record) recordBytes += serializedBytes(record) stagedPaths.push(recordPath) @@ -914,10 +598,7 @@ export class GenerationStore { timestamp, ...(args.meta && { meta: args.meta }), nouns, - verbs, - // Always present on new deltas (empty = "no blobs"), so compaction - // only falls back to record reads for pre-contract generations. - blobHashes: txBlobHashes + verbs } delta.bytes = recordBytes + serializedBytes(delta) const deltaPath = `${dir}/tx.json` @@ -927,41 +608,14 @@ export class GenerationStore { faultPoint('after-staging') // -- 4. Execute the planned batch ------------------------------------- - // Durability barrier: record every canonical write/delete the operations - // make, then fsync them BELOW (before the counter advances). Without - // this, a hard kill after commitTransaction returns could leave the - // fsync'd generation counter ahead of still-page-cached entity bytes — - // phantom progress for any generation-based consumer. - this.storage.beginWriteBarrier?.() this.inTransact = true try { await args.execute() } finally { this.inTransact = false } - // The transaction's entire canonical footprint is now durable, so the - // counter/manifest advance below can never outrun the entity bytes. - await this.storage.flushWriteBarrier?.() faultPoint('after-execute') - // Fact log (dual-write): append + fsync this generation's AFTER-IMAGE - // fact BEFORE the commit point, inside the same durability window — - // so a crash can only leave the log AHEAD (open truncates), never a - // committed generation without its fact. A real abort below this point - // compensates via dropAbove in the catch. An append failure fails the - // write, loudly — a silent fact gap would be a lie a replay discovers. - if (this.factLog) { - const fact = await this.buildCommitFact({ - generation: gen, - timestamp, - nouns, - verbs, - ...(args.meta ? { meta: args.meta } : {}) - }) - await this.factLog.append(fact) - await this.factLog.sync() - } - // -- 5. Counter + manifest rename (COMMIT POINT) ---------------------- await this.persistCounterUnlocked() faultPoint('before-manifest-rename') @@ -984,9 +638,6 @@ export class GenerationStore { timestamp, bytes: delta.bytes ?? 0 }) - if (this.historyBytesTotal !== null) { - this.historyBytesTotal += delta.bytes ?? 0 - } this.extendChains(gen, nouns, verbs) const logEntry: TxLogEntry = { generation: gen, timestamp, ...(args.meta && { meta: args.meta }) } await this.storage.appendTxLogLine(JSON.stringify(logEntry)) @@ -999,16 +650,6 @@ export class GenerationStore { if (crashSimulated) { throw err } - // The trapdoor for a batch: if rollback FAILED to fully apply, canonical - // storage may be inconsistent. A batch is never adopted forward (its - // other ops were rolled back — partial commit would break atomicity), so - // any unreconciled record is a fail-loud StoreInconsistentError. Compute - // it here (before the staging cleanup, which is always safe to run) and - // throw it in place of the raw error at the end. - let inconsistent: UnreconciledRecord[] = [] - if (err instanceof TransactionRollbackError) { - inconsistent = await this.reconcileFailedRollback(nounBefore, verbBefore) - } // Failed before the manifest rename: nothing is committed. Remove the // staging directory; the TransactionManager already restored any // applied operation byte-identically. @@ -1020,90 +661,14 @@ export class GenerationStore { `${(cleanupErr as Error).message} (recovery will remove it on next open)` ) } - // Compensate the pre-staging history-reference increments. Best - // effort — a failure here only over-counts (a leak the scrub - // repairs). Reclaim inside is safe on an abort: the before-image - // hashes are the entities' still-live content, so live references - // block any physical delete. - if (txBlobHashes.length > 0 && this.storage.releaseHistoryBlobReferences) { - try { - await this.storage.releaseHistoryBlobReferences(txBlobHashes) - } catch { - // over-count-safe; the scrub restores exactness - } - } - // Fact-log compensation: a real (non-crash) abort after the fact was - // appended must take the fact back out — the generation never - // committed. A crash instead reaches open(), whose truncation does the - // same reconcile from disk. - if (this.factLog && this.factLog.headGeneration() >= gen) { - await this.factLog.dropAbove(gen - 1) - } // Return the reservation when no concurrent bump consumed a later // number, so a failed transaction leaves generation() unchanged. if (this.counter === gen) this.counter = gen - 1 - // A failed rollback that left canonical records unreconciled surfaces as - // a loud StoreInconsistentError (brainy quarantines writes until repair); - // otherwise the raw error (clean abort, or a derived-only undo failure - // the egress guard + rebuild handle). - if (inconsistent.length > 0) { - throw new StoreInconsistentError( - inconsistent, - (err as TransactionRollbackError).originalError - ) - } throw err } }) } - /** - * @description After a transaction's rollback FAILED to fully apply - * (`TransactionRollbackError`), determine which touched records are now - * unreconciled by comparing current canonical state to the before-images the - * commit captured. This observes reality rather than trusting the opaque undo - * closures — the authoritative signal for adopt-forward vs fail-loud. - * - * @param nounBefore - Byte-identical entity before-images captured pre-execute. - * @param verbBefore - Byte-identical relationship before-images. - * @returns Every record whose canonical state no longer matches its - * before-image, tagged `'orphan'` (durably present when it should be gone — - * an add whose delete-undo failed) or `'loss'` (durably gone/wrong when it - * should have been restored — a remove/update whose restore-undo failed). An - * empty array means canonical storage is cleanly rolled back (only a derived - * index undo failed — the egress guard + a rebuild handle that). - */ - private async reconcileFailedRollback( - nounBefore: Map, - verbBefore: Map - ): Promise { - const out: UnreconciledRecord[] = [] - const classify = ( - before: GenerationRecord, - current: { metadata: unknown | null; vector: unknown | null } - ): 'orphan' | 'loss' | null => { - const beforeEmpty = before.metadata == null && before.vector == null - const currentEmpty = current.metadata == null && current.vector == null - if (beforeEmpty && currentEmpty) return null // add cleanly undone - if (beforeEmpty) return 'orphan' // add's delete-undo failed → still present - if (currentEmpty) return 'loss' // remove's restore-undo failed → gone - // Both present: same value = reconciled; different = restore left a wrong value. - const same = - JSON.stringify({ m: before.metadata, v: before.vector }) === - JSON.stringify({ m: current.metadata, v: current.vector }) - return same ? null : 'loss' - } - for (const [id, before] of nounBefore) { - const d = classify(before, await this.storage.readNounRaw(id)) - if (d) out.push({ id, kind: 'noun', disposition: d }) - } - for (const [id, before] of verbBefore) { - const d = classify(before, await this.storage.readVerbRaw(id)) - if (d) out.push({ id, kind: 'verb', disposition: d }) - } - return out - } - // ========================================================================== // Single-operation commit (Model-B per-write group-commit) // ========================================================================== @@ -1132,20 +697,6 @@ export class GenerationStore { * mid-flush is recovered by drop-without-restore * (see {@link GenerationDelta.groupCommit}). * - * **Durability contract (single-op vs transact).** A single-op write's live - * canonical bytes are written via tmp+rename but NOT individually fsync'd — - * they are durable at the next {@link flushPendingSingleOps} or `close()`, not - * the instant the write resolves. On a hard kill before that flush, a - * single-op write can be lost even though the call returned; the group-commit - * counter is buffered alongside, so counter and data are lost together (no - * counter-ahead-of-state torn store). {@link commitTransaction} is the - * stronger contract: it runs a write barrier that fsyncs the whole batch's - * canonical footprint BEFORE advancing the generation counter, so a committed - * transact is durable on return. Callers that need per-write durability should - * use `transact()` (or `flush()` after a single-op); Model-B group-commit - * deliberately trades single-op fsync latency (a 3-5x write regression) for - * throughput. - * * The per-write generation is read by graph-index ops through the same * `generation()` watermark `transact()` uses — because this method, like * {@link commitTransaction}, holds the mutex across the counter bump AND @@ -1154,25 +705,13 @@ export class GenerationStore { * * @param args.touched - The ids this write creates/updates/deletes, by kind. * @param args.execute - Runs the single-op's existing operation batch. - * @param args.precommit - Optional compare-and-swap precondition, invoked - * under the commit mutex with the just-read {@link CommitBeforeImages} and - * BEFORE `execute()`. A throw aborts the commit atomically: the generation - * reservation is returned and nothing is applied or buffered. This is the - * one place a per-record CAS (e.g. `ifRev`) is race-free — any check done - * before this mutex can interleave with a concurrent writer. * @returns The reserved generation and its timestamp. */ async commitSingleOp(args: { touched: { nouns?: string[]; verbs?: string[] } execute: () => Promise - precommit?: (before: CommitBeforeImages) => void - }): Promise<{ generation: number; timestamp: number; degraded?: string[] }> { + }): Promise<{ generation: number; timestamp: number }> { return this.withMutex(async () => { - // Refuse to accept a write whose history we cannot make durable: if the - // pending-tier flush has latched a persistent failure, accepting more - // writes would silently pile un-durable before-images into memory. Fail - // loud instead (the latch self-clears when a flush finally succeeds). - this.assertHistoryDurable() const nouns = args.touched.nouns ? [...new Set(args.touched.nouns)] : [] const verbs = args.touched.verbs ? [...new Set(args.touched.verbs)] : [] const gen = ++this.counter @@ -1193,19 +732,6 @@ export class GenerationStore { verbBefore.set(id, { kind: 'verb', metadata: prev.metadata, vector: prev.vector }) } - // Conditional commit: the caller's CAS precondition runs here — under - // the mutex, against the authoritative before-images, before any write. - // A throw aborts cleanly: return the generation reservation (nothing was - // applied or buffered) and surface the conflict to the caller. - if (args.precommit) { - try { - args.precommit({ nouns: nounBefore, verbs: verbBefore }) - } catch (err) { - if (this.counter === gen) this.counter = gen - 1 - throw err - } - } - // Execute the live write. inTransact suppresses the storage bump hook so // the counter is not double-advanced (this generation already reserved // it) — the same suppression transact() uses. @@ -1214,50 +740,9 @@ export class GenerationStore { await args.execute() } catch (err) { this.inTransact = false - // A failed rollback (TransactionRollbackError) may have left canonical - // storage inconsistent — the trapdoor. Reconcile against the - // before-images to decide the honest response (David's ruling: - // adopt-forward when safe, else fail loud). - if (err instanceof TransactionRollbackError) { - const records = await this.reconcileFailedRollback(nounBefore, verbBefore) - if (records.length > 0 && records.every((r) => r.disposition === 'orphan')) { - // ADOPT FORWARD: the durably-present orphan(s) are exactly what this - // single-op add wrote. Keep + buffer the generation so the record is - // legitimately committed and readable; the derived index may be - // incomplete for these ids until the next rebuild/repairIndex (the - // egress guard prevents wrong results meanwhile). Loud, honest, - // no double-write. - this.pendingBuffer.set(gen, { nouns: nounBefore, verbs: verbBefore, timestamp }) - this.pendingGens.push(gen) - this.extendChains(gen, nouns, verbs) - // The adopted generation is committed — it gets its fact like any - // other (durability rides the group-commit flush, same as the - // buffered history). - if (this.factLog) { - await this.factLog.append( - await this.buildCommitFact({ generation: gen, timestamp, nouns, verbs }) - ) - } - prodLog.warn( - `[GenerationStore] Recovered a failed rollback FORWARD: single-op write ` + - `committed as generation ${gen} because its canonical undo could not be ` + - `applied (record(s) ${records.map((r) => r.id).join(', ')} are durable). ` + - `The derived index may be incomplete for these ids — run repairIndex() to heal.` - ) - return { generation: gen, timestamp, degraded: records.map((r) => r.id) } - } - if (records.length > 0) { - // FAIL LOUD: a loss (or mixed) cannot be safely adopted. Return the - // reservation and throw — brainy quarantines writes until repair. - if (this.counter === gen) this.counter = gen - 1 - throw new StoreInconsistentError(records, err.originalError) - } - // records.length === 0: canonical is cleanly rolled back (only a - // derived undo failed) — fall through to the clean-abort path below. - } - // Live write failed with canonical cleanly rolled back: nothing buffered, - // nothing durable. Return the reservation (when no concurrent bump - // consumed a later number) so generation() is unchanged. + // Live write failed: nothing buffered, nothing on disk. Return the + // reservation (when no concurrent bump consumed a later number) so + // generation() is unchanged. if (this.counter === gen) this.counter = gen - 1 throw err } @@ -1267,17 +752,6 @@ export class GenerationStore { this.pendingBuffer.set(gen, { nouns: nounBefore, verbs: verbBefore, timestamp }) this.pendingGens.push(gen) this.extendChains(gen, nouns, verbs) - // Fact log (dual-write): the acked write's AFTER-IMAGE fact, appended - // now (read back warm, under the mutex — group-commit means flush-time - // canonical only holds the LATEST state, so each generation's after-image - // exists only here). Durability rides the group-commit flush, exactly - // like the buffered before-image history: a crash before the flush loses - // the fact AND the generation together — never a torn state. - if (this.factLog) { - await this.factLog.append( - await this.buildCommitFact({ generation: gen, timestamp, nouns, verbs }) - ) - } this.schedulePendingFlush() return { generation: gen, timestamp } }) @@ -1318,22 +792,6 @@ export class GenerationStore { */ async flushPendingSingleOps(): Promise { if (this.pendingGens.length === 0) return - // Centralize durability accounting here so EVERY failure path — the - // background scheduler AND an explicit flush()/close() — latches - // consistently, and success (from any caller) clears the latch. The - // scheduler owns only the retry cadence. - try { - await this.flushPendingSingleOpsUnlocked() - this.onPendingFlushSuccess() - } catch (err) { - this.recordPendingFlushFailure(err as Error) - throw err - } - } - - /** The actual pending-tier flush under the mutex (see {@link flushPendingSingleOps}, - * which wraps this with durability accounting). */ - private async flushPendingSingleOpsUnlocked(): Promise { return this.withMutex(async () => { if (this.pendingGens.length === 0) return this.clearPendingFlushTimer() @@ -1346,21 +804,6 @@ export class GenerationStore { const buf = this.pendingBuffer.get(gen) if (!buf) continue const dir = `${GENERATIONS_PREFIX}/${gen}` - - // Temporal-blob contract: count this record-set's content-hash - // references BEFORE persisting it (a crash between the two only - // over-counts — the scrub repairs a leak; under-counting could let - // compaction reclaim bytes a retained generation still needs). - const blobHashes = this.storage.extractBlobHashesFromRecords - ? this.storage.extractBlobHashesFromRecords([ - ...buf.nouns.values(), - ...buf.verbs.values() - ]) - : [] - if (blobHashes.length > 0 && this.storage.recordHistoryBlobReferences) { - await this.storage.recordHistoryBlobReferences(blobHashes) - } - const nounIds: string[] = [] const verbIds: string[] = [] let recordBytes = 0 @@ -1383,10 +826,7 @@ export class GenerationStore { timestamp: buf.timestamp, groupCommit: true, nouns: nounIds, - verbs: verbIds, - // Always present on new deltas (empty = "no blobs"), so compaction - // only falls back to record reads for pre-contract generations. - blobHashes + verbs: verbIds } delta.bytes = recordBytes + serializedBytes(delta) genBytes.set(gen, delta.bytes) @@ -1399,12 +839,6 @@ export class GenerationStore { // ONE fsync for the whole window — the durability-batching win. await this.storage.syncRawObjects(stagedPaths) - // Fact log (dual-write): make the window's buffered facts durable in the - // same batch, BEFORE the commit point below — so a crash can only leave - // the log AHEAD of the counter (open truncates), never a committed - // generation without its durable fact. - await this.factLog?.sync() - // Test-only crash simulation: a throwing injector here leaves the staged // group-commit generation dirs on disk with NO manifest advance — the // exact "crashed mid-flush" state recovery must DROP-WITHOUT-RESTORE @@ -1440,9 +874,6 @@ export class GenerationStore { timestamp: buf.timestamp, bytes: genBytes.get(gen) ?? 0 }) - if (this.historyBytesTotal !== null) { - this.historyBytesTotal += genBytes.get(gen) ?? 0 - } this.pendingBuffer.delete(gen) } this.pendingGens = [] @@ -1453,105 +884,24 @@ export class GenerationStore { }) } - /** - * @description Reset the durability-failure state after a successful flush. - * If a failure was latched (writes were being refused), log the recovery so - * the transition out of the loud state is as visible as the transition in. - */ - private onPendingFlushSuccess(): void { - if (this.pendingFlushError) { - prodLog.info( - `[GenerationStore] pending single-op history flush recovered after ` + - `${this.pendingFlushFailures} failed attempt(s); writes resume.` - ) - } - this.pendingFlushFailures = 0 - this.pendingFlushError = null - } - - /** - * @description Account for a failed pending-tier flush (called from - * {@link flushPendingSingleOps}' catch, so it fires on EVERY failure path — - * background scheduler and explicit flush()/close() alike). Escalates a - * transient blip (a warn, keep retrying) into a latched durability failure - * once {@link PENDING_FLUSH_FAILURE_THRESHOLD} consecutive attempts fail — - * from that point writes are refused with a {@link PendingFlushDurabilityError} - * until a flush finally succeeds. Always ensures a backoff retry is scheduled - * so a recovering disk self-heals without operator action, regardless of who - * triggered the failing flush. The pending before-images are retained (the - * failed flush never cleared them), so no history is lost — only not-yet-durable. - */ - private recordPendingFlushFailure(err: Error): void { - this.pendingFlushFailures++ - const attempts = this.pendingFlushFailures - if (attempts >= GenerationStore.PENDING_FLUSH_FAILURE_THRESHOLD) { - // Latch: refuse further writes rather than pile un-durable history. - this.pendingFlushError = err - prodLog.error( - `[GenerationStore] pending single-op history flush has FAILED ${attempts} ` + - `consecutive times (${err.message}). Generation history is not durable; ` + - `writes are now REFUSED until it drains. Live canonical data is intact. ` + - `Retrying with backoff; the store resumes writes when a flush succeeds.` - ) - } else { - prodLog.warn( - `[GenerationStore] pending single-op flush failed (attempt ${attempts}/` + - `${GenerationStore.PENDING_FLUSH_FAILURE_THRESHOLD}): ${err.message}; retrying with backoff.` - ) - } - this.scheduleFlushRetry(attempts) - } - - /** - * @description (Re)arm the pending-flush timer at a capped exponential - * backoff after a failure, so a persistent fault is retried at a decaying - * cadence instead of hot-looping. Shares the single {@link pendingFlushTimer} - * slot with {@link schedulePendingFlush} (only one flush is ever scheduled). - * The retry's own failure is re-accounted inside {@link flushPendingSingleOps}. - */ - private scheduleFlushRetry(attempt: number): void { - if (this.pendingFlushTimer !== null) return - const backoff = Math.min( - GenerationStore.PENDING_FLUSH_DELAY_MS * 2 ** Math.min(attempt, 6), - GenerationStore.PENDING_FLUSH_RETRY_CAP_MS - ) - this.pendingFlushTimer = setTimeout(() => { - this.pendingFlushTimer = null - // flushPendingSingleOps records the failure + reschedules internally. - void this.flushPendingSingleOps().catch(() => {}) - }, backoff) - const t = this.pendingFlushTimer as unknown as { unref?: () => void } - if (t && typeof t.unref === 'function') t.unref() - } - - /** - * @description Throw if the store has a latched history-durability failure. - * Called at the top of every write commit so a broken durability path fails - * loud and immediately instead of silently accumulating un-durable history. - */ - private assertHistoryDurable(): void { - if (this.pendingFlushError) { - throw new PendingFlushDurabilityError(this.pendingFlushError, this.pendingFlushFailures) - } - } - /** Schedule a coalesced pending-tier flush (size trigger fires immediately on * the next microtask; otherwise a {@link PENDING_FLUSH_DELAY_MS} timer). Both - * defer outside the current mutex section so the flush can re-acquire it. A - * failed flush is accounted + rescheduled inside {@link flushPendingSingleOps} - * (latch-on-persistent-failure), never swallowed as a bare log line. */ + * defer outside the current mutex section so the flush can re-acquire it. */ private schedulePendingFlush(): void { if (this.pendingGens.length >= this.pendingFlushThreshold) { - // Failure is recorded + retried inside flushPendingSingleOps. void Promise.resolve() .then(() => this.flushPendingSingleOps()) - .catch(() => {}) + .catch((err) => + prodLog.warn(`[GenerationStore] pending single-op flush failed: ${(err as Error).message}`) + ) return } if (this.pendingFlushTimer !== null) return this.pendingFlushTimer = setTimeout(() => { this.pendingFlushTimer = null - void this.flushPendingSingleOps().catch(() => {}) + void this.flushPendingSingleOps().catch((err) => + prodLog.warn(`[GenerationStore] pending single-op flush failed: ${(err as Error).message}`) + ) }, GenerationStore.PENDING_FLUSH_DELAY_MS) // A background flush must not keep the process alive (Node only). const t = this.pendingFlushTimer as unknown as { unref?: () => void } @@ -1875,15 +1225,9 @@ export class GenerationStore { if (pending) { return (kind === 'noun' ? pending.nouns : pending.verbs).get(id) ?? null } - const live = (await this.storage.readRawObject( + return (await this.storage.readRawObject( `${GENERATIONS_PREFIX}/${gen}/prev/${id}.json` )) as GenerationRecord | null - if (live) return live - // Two-tier: the packed tier serves folded generations (live-tier-wins). - if (this.segments?.hasGeneration(gen)) { - return (await this.segments.readRecord(gen, kind, id)) as GenerationRecord | null - } - return null } /** @@ -2230,21 +1574,6 @@ export class GenerationStore { `${GENERATIONS_PREFIX}/${gen}/tx.json` )) as GenerationDelta | null if (delta === null) { - // Two-tier read (D1+D3): not in the live tier → the packed tier. - // Live-tier-wins ordering (a crash mid-fold leaves a duplicate, never - // a gap), so the segment lookup runs only after the live miss. - const packed = await this.segments?.readDelta(gen) - if (packed) { - const d = packed.delta as GenerationDelta - const entry = { - nouns: new Set(d.nouns), - verbs: new Set(d.verbs), - timestamp: packed.timestamp, - bytes: d.bytes ?? 0 - } - this.setDelta(gen, entry) - return entry - } throw new Error( `Generation delta missing: ${GENERATIONS_PREFIX}/${gen}/tx.json ` + `(store corrupted or records removed outside compactHistory())` @@ -2284,26 +1613,17 @@ export class GenerationStore { /** * @description Total serialized bytes of the ON-DISK generational history — * the sum of every committed generation's recorded `bytes`. Backs the - * `maxBytes` and adaptive retention caps. O(1) after the first call: the - * total is computed by ONE walk over committed deltas, then maintained - * incrementally at every commit (+bytes) and reclaim (−bytes) and dropped on - * a wholesale state replacement (restore). Without the running total, the - * adaptive auto-compaction on every flush() re-walked the ENTIRE history — - * O(committed generations) file reads per flush past the delta-cache bound — - * which is how a 70k-generation production brain turned every write into a - * full-tail scan (SELF-GENERATIONS-GROWTH). Pending (un-flushed) generations - * are excluded (they are not on disk). + * `maxBytes` and adaptive retention caps. Reads each committed generation's + * delta (cached; a re-read only for cache-evicted ones) — O(committed + * generations), bounded by retention itself and invoked only at compaction + * time. Pending (un-flushed) generations are excluded (they are not on disk). * @returns The total on-disk history byte count. */ async historyBytes(): Promise { - if (this.historyBytesTotal !== null) { - return this.historyBytesTotal - } let total = 0 for (const gen of this.committedGensAsc()) { total += (await this.getDelta(gen)).bytes } - this.historyBytesTotal = total return total } @@ -2326,94 +1646,6 @@ export class GenerationStore { * @param options - Retention caps (see {@link CompactHistoryOptions}). * @returns Count of removed record-sets and the new horizon. */ - /** - * @description The REPACKER (D1+D3+repacking): fold cold live-tier - * generations into sealed segments — re-representation, never deletion. - * Every record and delta stays readable (asOf/chains unchanged); the - * per-generation directories are deleted only AFTER their segment is - * durable (crash between = duplicate representation, resolved - * live-tier-wins by every reader; never a gap). This is the transform that - * takes a 70k-file history to tens of segment files, and the ONLY history - * transform permitted under the archival profile. - * - * Folds oldest-first, contiguous from the packed boundary, in batches, and - * stops at the live window ({@link GenerationStore.REPACK_LIVE_WINDOW}) - * or when `timeBudgetMs` is spent — an early stop is a consistent prefix; - * the next pass resumes. - */ - async repackHistory(options?: { timeBudgetMs?: number; batchGenerations?: number }): Promise<{ - foldedGenerations: number - segmentsCreated: number - }> { - if (!this.segments) return { foldedGenerations: 0, segmentsCreated: 0 } - const segments = this.segments - return this.withMutex(async () => { - const deadline = - options?.timeBudgetMs !== undefined ? Date.now() + options.timeBudgetMs : undefined - const batchSize = options?.batchGenerations ?? 512 - const coldCeiling = this.committed - GenerationStore.REPACK_LIVE_WINDOW - const packedThrough = - segments.segments().length > 0 - ? segments.segments()[segments.segments().length - 1].lastGeneration - : 0 - - // Cold, unpacked, committed generations — ascending, contiguous scan. - const eligible: number[] = [] - for (const gen of this.committedGensAsc()) { - if (gen > coldCeiling) break - if (gen <= packedThrough) continue // already packed (dup fold barred) - if (this.pendingBuffer.has(gen)) continue // un-flushed = live by definition - eligible.push(gen) - } - - let folded = 0 - let segmentsCreated = 0 - for (let i = 0; i < eligible.length; i += batchSize) { - if (deadline !== undefined && Date.now() >= deadline) break - const batch = eligible.slice(i, i + batchSize) - const foldInput: FoldGeneration[] = [] - for (const gen of batch) { - const delta = (await this.storage.readRawObject( - `${GENERATIONS_PREFIX}/${gen}/tx.json` - )) as GenerationDelta | null - if (delta === null) { - // Already folded by a prior crashed pass whose dirs were removed, - // or damage — getDelta's two-tier read decides which, loudly, - // when someone asks. Skip; never fold a generation we cannot read. - continue - } - const records: FoldGeneration['records'] = [] - for (const [kind, ids] of [ - ['noun', delta.nouns] as const, - ['verb', delta.verbs] as const - ]) { - for (const id of ids) { - const record = await this.storage.readRawObject( - `${GENERATIONS_PREFIX}/${gen}/prev/${id}.json` - ) - if (record) records.push({ kind, id, record }) - } - } - foldInput.push({ generation: gen, timestamp: delta.timestamp, delta, records }) - } - if (foldInput.length === 0) continue - await segments.fold(foldInput) - segmentsCreated++ - // Segment + manifest durable → the live copies retire. - for (const g of foldInput) { - await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`) - } - folded += foldInput.length - } - if (folded > 0) { - prodLog.info( - `[GenerationStore] repacked ${folded} cold generation(s) into ${segmentsCreated} segment(s) — history preserved, file count reduced` - ) - } - return { foldedGenerations: folded, segmentsCreated } - }) - } - async compact(options?: CompactHistoryOptions): Promise { return this.withMutex(async () => { const minPinned = this.minPinnedGeneration() @@ -2421,11 +1653,6 @@ export class GenerationStore { const maxAge = options?.maxAge const maxBytes = options?.maxBytes const ageCutoff = maxAge !== undefined ? Date.now() - maxAge : undefined - // Bounded maintenance pass (8.9.0): stop reclaiming once the budget is - // spent. Safe mid-loop — reclamation is oldest-first, so an early stop - // leaves a consistent contiguous prefix and the next pass resumes. - const deadline = - options?.timeBudgetMs !== undefined ? Date.now() + options.timeBudgetMs : undefined const noCaps = maxGenerations === undefined && maxAge === undefined && maxBytes === undefined @@ -2439,49 +1666,20 @@ export class GenerationStore { for (const gen of [...this.committedGensAsc()]) { // Pins are always exempt: never reclaim a generation a live pin needs. if (gen > minPinned) break // committedGensAsc ascending → nothing newer is eligible either - if (deadline !== undefined && Date.now() >= deadline) break // budget spent — resume next pass - const delta = await this.getDelta(gen) if (!noCaps) { const violatesCount = maxGenerations !== undefined && remainingCount > maxGenerations const violatesBytes = maxBytes !== undefined && remainingBytes > maxBytes - const violatesAge = ageCutoff !== undefined && delta.timestamp < ageCutoff + let violatesAge = false + if (ageCutoff !== undefined) { + const delta = await this.getDelta(gen) + violatesAge = delta.timestamp < ageCutoff + } // Oldest-first: once the oldest unpinned gen trips no cap, none newer do. if (!violatesCount && !violatesBytes && !violatesAge) break } - const genBytes = maxBytes !== undefined ? delta.bytes : 0 - - // Temporal-blob contract: resolve the content-blob hashes this - // generation's record-set references BEFORE deleting it. New - // generations carry the multiset in their persisted delta (empty - // array when none — distinguishing "new format, no blobs" from a - // pre-contract delta); legacy generations fall back to reading the - // records themselves. Skipped entirely on non-blob-aware storage. - let blobHashes: string[] | undefined - if (this.storage.releaseHistoryBlobReferences) { - const rawDelta = (await this.storage.readRawObject( - `${GENERATIONS_PREFIX}/${gen}/tx.json` - )) as GenerationDelta | null - blobHashes = rawDelta?.blobHashes - if (blobHashes === undefined && this.storage.extractBlobHashesFromRecords) { - blobHashes = this.storage.extractBlobHashesFromRecords( - await this.readGenerationRecords(gen) - ) - } - } - + const genBytes = maxBytes !== undefined ? (await this.getDelta(gen)).bytes : 0 await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${gen}`) this.deltaCache.delete(gen) - if (this.historyBytesTotal !== null) { - this.historyBytesTotal -= delta.bytes - } - - // AFTER the record-set is gone (over-count-only crash ordering): - // release its history references and reclaim any blob left with zero - // live AND zero history references — the system's one byte-reclaim point. - if (blobHashes && blobHashes.length > 0 && this.storage.releaseHistoryBlobReferences) { - await this.storage.releaseHistoryBlobReferences(blobHashes) - } - remainingCount-- remainingBytes -= genBytes removed.push(gen) @@ -2505,16 +1703,6 @@ export class GenerationStore { // Reclaimed generations leave the per-id chains stale → rebuild on next read. this.invalidateChains() this.horizonGen = Math.max(this.horizonGen, highestRemoved) - // Packed-tier reclaim (D3): a packed generation's bytes live in a - // sealed segment — removeRawPrefix above was a no-op for it. Drop - // WHOLE segments now fully below the horizon; a partially-reclaimed - // segment keeps its bytes until the boundary passes it (the frozen - // partial-segments-wait rule; logical reclamation above still holds — - // the generations left committedRanges and asOf below the horizon - // throws regardless). - if (this.segments) { - await this.segments.dropSegmentsBelow(this.horizonGen + 1) - } const manifest: GenerationManifest = { version: 1, generation: this.committed, @@ -2544,9 +1732,6 @@ export class GenerationStore { async reopenAfterRestore(floorGeneration: number): Promise { await this.withMutex(async () => { this.deltaCache.clear() - // The running history-byte total describes the REPLACED store — drop it; - // the next historyBytes() re-seeds with one walk over the new state. - this.historyBytesTotal = null // A wholesale state replacement invalidates any buffered single-op // history — discard the pending tier (its live writes are gone with the // replaced store). diff --git a/src/db/types.ts b/src/db/types.ts index 4c8a4957..80a1959d 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -116,16 +116,6 @@ export interface TransactOptions { * record is staged. */ ifAtGeneration?: number - /** - * Budget (ms) for the atomic apply phase. When omitted, the budget SCALES - * with the batch: `max(30 000, opCount × 2 000)` — production imports on - * network-attached disks measure ~2 s per operation, so a flat 30 s budget - * silently capped honest bulk work at ~15 operations. A tripped budget - * rolls the whole batch back and throws a retryable - * `TransactionTimeoutError` naming the operation it stopped at, the batch - * size, and the elapsed/budget times. - */ - timeoutMs?: number } /** @@ -176,15 +166,6 @@ export interface CompactHistoryOptions { * of each surviving generation's serialized record set (`GenerationDelta.bytes`). */ maxBytes?: number - /** - * Stop reclaiming after this many milliseconds even if caps are still - * exceeded (8.9.0). Compaction is maintenance — a bounded pass keeps - * `close()` (and any explicit maintenance window) from stalling on a large - * backlog; the next pass resumes where this one stopped (reclamation is - * oldest-first, so an early stop is always a consistent prefix). Unset = - * run to completion. - */ - timeBudgetMs?: number } /** @@ -202,36 +183,6 @@ export interface CompactHistoryResult { horizon: number } -/** - * @description Result of `brain.historyStats()` — the read-only generational - * history footprint, for fleet audits and ops doors. A pool operator runs this - * per brain to size retention exposure (how much MVCC history each brain - * carries and under which policy) without touching any data. - */ -export interface HistoryStats { - /** Committed generation record-sets currently on disk. */ - generations: number - /** Total on-disk history bytes across those record-sets. */ - bytes: number - /** Oldest committed generation still on disk (null when history is empty). */ - oldestGeneration: number | null - /** Newest committed generation (null when history is empty). */ - newestGeneration: number | null - /** Commit timestamp (ms) of the oldest on-disk generation. */ - oldestTimestamp: number | null - /** Commit timestamp (ms) of the newest on-disk generation. */ - newestTimestamp: number | null - /** Compaction horizon — generations below it were reclaimed. */ - horizon: number - /** The effective retention mode this brain runs under. */ - retentionMode: 'all' | 'adaptive' | 'explicit' - /** - * The adaptive byte budget in force (coordinator-driven or the local - * free-memory probe); null under 'all' or explicit caps. - */ - effectiveBudgetBytes: number | null -} - // ============================================================================ // Db surfaces // ============================================================================ @@ -350,16 +301,6 @@ export interface GenerationDelta { nouns: string[] /** Relationship ids touched by this generation. */ verbs: string[] - /** - * Content-blob hashes referenced by this generation's before-image records - * (a MULTISET — one entry per referencing record occurrence), captured at - * persist time so compaction can release the exact history references it - * reclaims without re-reading the records. Always present (possibly empty) - * on deltas written under the temporal-blob contract; absent only on - * pre-contract generations, for which compaction falls back to reading the - * record-set itself. - */ - blobHashes?: string[] /** * `true` for a Model-B single-operation generation persisted by the async * group-commit flush (`GenerationStore.flushPendingSingleOps`). It marks the @@ -440,24 +381,6 @@ export interface GenerationStorage { /** Durability barrier: fsync the given object paths (no-op in memory). */ syncRawObjects(paths: string[]): Promise - /** - * OPTIONAL transaction durability barrier. `commitTransaction` calls - * {@link beginWriteBarrier} immediately before running the planned operations - * and {@link flushWriteBarrier} immediately after — BEFORE the generation - * counter and manifest are advanced. An adapter whose canonical writes are - * not synchronously durable (the filesystem adapter's tmp+rename lands in the - * page cache) MUST implement these so a transaction reported "committed" is - * durable before its generation stamp: otherwise a hard kill can leave the - * fsync'd counter ahead of the still-buffered entity bytes (phantom progress - * for any generation-based consumer). Adapters whose writes are already - * durable per-call (cloud object PUT) may leave these undefined — the store - * treats them as no-ops. `beginWriteBarrier` also resets any tracking left by - * an aborted prior transaction. - */ - beginWriteBarrier?(): void - /** @see beginWriteBarrier — fsync every canonical write since begin. */ - flushWriteBarrier?(): Promise - /** Read an entity's raw stored metadata+vector objects. */ readNounRaw(id: string): Promise<{ metadata: any | null; vector: any | null }> /** Restore an entity's raw stored objects (`null` part ⇒ delete that file). */ @@ -472,44 +395,6 @@ export interface GenerationStorage { /** Read all lines of `_system/tx-log.jsonl` (empty array if absent). */ readTxLogLines(): Promise - /** - * OPTIONAL binary raw-byte primitives — the substrate for the generation - * fact log's append-only CRC-framed segments. Feature-detected: a storage - * layer that omits them hosts no fact log (dual-write is skipped; readers - * fall back to canonical enumeration). Paths are used VERBATIM (no - * suffixing). Append durability rides `syncRawObjects` at the commit - * barrier, exactly like the staged history files. - */ - appendRawBytes?(path: string, bytes: Uint8Array): Promise - /** Read a raw binary file whole; absent → null; a real fault throws. */ - readRawBytes?(path: string): Promise - /** Replace a raw binary file atomically (tmp → fsync → rename). */ - writeRawBytes?(path: string, bytes: Uint8Array): Promise - /** Byte size of a raw binary file, or null when absent. */ - rawByteSize?(path: string): Promise - - /** - * OPTIONAL temporal-blob contract (implemented by blob-aware storage; the - * generation store treats the hashes as opaque strings). Extract the - * content-blob hashes a record-set references — a pure multiset extraction, - * no side effects. - */ - extractBlobHashesFromRecords?(records: GenerationRecord[]): string[] - /** - * OPTIONAL: record history references for the given hashes (one increment - * per occurrence). Called BEFORE the referencing record-set is persisted — - * a crash between the two can only over-count (a leak the scrub repairs), - * never under-count (which would risk reclaiming bytes history still needs). - */ - recordHistoryBlobReferences?(hashes: string[]): Promise - /** - * OPTIONAL: release history references for the given hashes (one decrement - * per occurrence) and physically reclaim any hash left with zero live AND - * zero history references. Called AFTER the referencing record-set is - * deleted by compaction — same over-count-only crash ordering. - */ - releaseHistoryBlobReferences?(hashes: string[]): Promise - /** * Register the generation-bump hook invoked on every entity-visible * single-operation write (see `BaseStorage.setGenerationBumpHook`). diff --git a/src/errors/brainyError.ts b/src/errors/brainyError.ts index a58236e3..f627ba00 100644 --- a/src/errors/brainyError.ts +++ b/src/errors/brainyError.ts @@ -14,9 +14,6 @@ export type BrainyErrorType = | 'FIELD_NOT_INDEXED' | 'GRAPH_INDEX_NOT_READY' | 'METADATA_INDEX_NOT_READY' - | 'VECTOR_INDEX_NOT_READY' - | 'PROTECTED_ARTIFACT' - | 'DERIVED_ARTIFACT_MISSING' | 'MIGRATION_IN_PROGRESS' /** @@ -283,86 +280,6 @@ export class MetadataIndexNotReadyError extends BrainyError { } } -/** - * Thrown when the vector index reports vectors (`size() > 0` or, on a native - * provider, `isReady() === false`) but cannot return a KNOWN persisted vector - * even after a rebuild — i.e. the semantic serving structure did not load on a - * cold open and could not be restored. The vector-search counterpart of - * {@link GraphIndexNotReadyError} / {@link MetadataIndexNotReadyError}: it - * replaces the silent-empty failure mode (a cold `find({ query })` returning - * `[]` indistinguishable from "no similar data") with a loud, catchable error, - * so a consumer never renders "nothing found" over data that is simply - * not-yet-warm. - * - * Detected once per brain by a known-vector serving probe on the first - * semantic / proximity `find()`; brainy self-heals (rebuilds the index from the - * canonical records) first and only raises this if the rebuild still cannot - * serve the known vector. - */ -export class VectorIndexNotReadyError extends BrainyError { - constructor(message: string, originalError?: Error) { - super(message, 'VECTOR_INDEX_NOT_READY', false, originalError) - this.name = 'VectorIndexNotReadyError' - if (Error.captureStackTrace) { - Error.captureStackTrace(this, VectorIndexNotReadyError) - } - } -} - -/** - * Thrown when a delete (`deleteBinaryBlob` / `removeRawPrefix`) would remove a - * blob that is a declared member of a protected derived-index FAMILY (the - * registered-blob contract). Declared derived artifacts are undeletable through - * the storage layer — this makes an in-process GC / sweeper INCAPABLE of removing - * a load-bearing index file (the lost-`main.dkann` class). Intentional retirement - * is the explicit `unregisterDerivedFamily(name)` step, then the delete. - */ -export class ProtectedArtifactError extends BrainyError { - /** The blob key the delete targeted. */ - public readonly key: string - /** The protected family the key belongs to. */ - public readonly family: string - constructor(key: string, family: string) { - super( - `Refused to delete '${key}': it is a declared member of the protected ` + - `derived-index family '${family}'. Declared derived artifacts are undeletable ` + - `through the storage layer (COLD ≠ DEAD) — unregisterDerivedFamily('${family}') ` + - `first if retirement is intentional.`, - 'PROTECTED_ARTIFACT', - false - ) - this.name = 'ProtectedArtifactError' - this.key = key - this.family = family - } -} - -/** - * Raised (loudly) when a declared derived-index family is missing one or more of - * its members on open — i.e. a load-bearing blob was deleted OUTSIDE the write - * path (an external sweeper the in-process refusal cannot stop). The index must - * be rebuilt from canonical; "healthy-while-broken" is impossible because the - * missing member is named, not silently tolerated. - */ -export class DerivedArtifactMissingError extends BrainyError { - /** The family with missing members. */ - public readonly family: string - /** The member keys that are absent. */ - public readonly missing: string[] - constructor(family: string, missing: string[]) { - super( - `Derived-index family '${family}' is missing ${missing.length} declared ` + - `member(s) on open (${missing.join(', ')}) — deleted outside the write path. ` + - `The index must be rebuilt from canonical.`, - 'DERIVED_ARTIFACT_MISSING', - false - ) - this.name = 'DerivedArtifactMissingError' - this.family = family - this.missing = missing - } -} - /** * Thrown when a data-plane read or write is issued against a brain that is * running its one-time, automatic 7.x → 8.0 on-disk upgrade — the coordinated diff --git a/src/events/changeFeed.ts b/src/events/changeFeed.ts deleted file mode 100644 index 9b9fda6f..00000000 --- a/src/events/changeFeed.ts +++ /dev/null @@ -1,171 +0,0 @@ -/** - * @module events/changeFeed - * @description The in-process change feed behind {@link Brainy.onChange} — the - * authoritative "something committed" signal for every canonical mutation, - * regardless of origin (direct API calls, batches, transactions, imports, the - * VFS, or an accelerated native deployment: all of them funnel through the - * same generation-store commit points this feed is emitted from). - * - * Design properties: - * - **Post-commit only.** Events are enqueued after the commit succeeds, so an - * aborted commit (a losing `ifRev` CAS, a failed transaction) never emits. - * - **Commit-ordered.** Events are enqueued in commit order and dispatched - * FIFO, so a subscriber observes mutations in the order they became durable. - * - **Never blocks the write path.** Dispatch happens in a microtask after the - * committing call returns; a slow or throwing listener cannot delay or fail - * a write. Listener errors are isolated per listener and per event. - * - **Zero cost when unused.** Callers consult {@link ChangeFeed.hasListeners} - * before constructing event payloads; with no subscribers the write path - * does no event work at all. - * - **Fire-and-forget.** No acknowledgement, backpressure, or replay. Events - * carry the committed `generation`, so a consumer that needs catch-up - * semantics can pair the live feed with `transactionLog()` / `asOf()`. - */ - -/** The post-commit view of an entity carried by entity change events. */ -export interface ChangeEventEntity { - /** The entity's canonical UUID. */ - id: string - /** The entity's NounType string (e.g. `'person'`, `'document'`). */ - type: string - /** The entity's subtype, when set. */ - subtype?: string - /** The entity's custom (indexed) metadata fields. */ - metadata: Record - /** The writing service, when set. */ - service?: string -} - -/** The post-commit view of a relationship carried by relation change events. */ -export interface ChangeEventRelation { - /** The relationship's canonical UUID. */ - id: string - /** Source entity UUID. */ - from: string - /** Target entity UUID. */ - to: string - /** The relationship's VerbType string (e.g. `'contains'`). */ - type: string - /** The relationship's custom metadata fields, when present. */ - metadata?: Record -} - -/** - * One committed mutation, as delivered to {@link Brainy.onChange} listeners. - * - * Exactly one of `entity` / `relation` is populated for `kind: 'entity'` / - * `kind: 'relation'` events. `kind: 'store'` events (`clear` / `restore`) - * carry neither — they mean "the whole store changed; refetch what you care - * about". - * - * For `op: 'remove'` / `op: 'unrelate'` the payload is the record's LAST - * committed state (sourced from the commit's own before-image), so deletes are - * fully described rather than id-only. - */ -export interface BrainyChangeEvent { - /** What changed: one record, one relationship, or the whole store. */ - kind: 'entity' | 'relation' | 'store' - /** The mutation, in Brainy's own API vocabulary. */ - op: - | 'add' - | 'update' - | 'remove' - | 'relate' - | 'unrelate' - | 'updateRelation' - | 'clear' - | 'restore' - /** The mutated record's id (absent for store-level events). */ - id?: string - /** Post-commit entity view (entity events only). */ - entity?: ChangeEventEntity - /** Post-commit relation view (relation events only). */ - relation?: ChangeEventRelation - /** - * The committed generation this mutation belongs to (Model B: every write - * is a generation; a transaction's items share one). Absent for store-level - * events and init-time bootstrap writes, which are not generation-stamped. - */ - generation?: number - /** Commit timestamp (ms since epoch). */ - timestamp: number -} - -/** Listener signature for {@link Brainy.onChange}. */ -export type ChangeListener = (event: BrainyChangeEvent) => void - -/** - * A change event as constructed by a mutation method BEFORE its commit: the - * commit seam stamps `generation`/`timestamp` after the write becomes - * durable. An entity `remove` descriptor may omit `entity` — the seam fills - * it from the commit's own before-image (the record's last committed state). - */ -export type PendingChangeEvent = Omit - -/** - * @description Listener registry + commit-ordered async dispatcher for - * {@link BrainyChangeEvent}s. One instance per {@link Brainy}. - */ -export class ChangeFeed { - private listeners = new Set() - private queue: BrainyChangeEvent[] = [] - private draining = false - - /** Whether any listener is subscribed — the write path's zero-cost gate. */ - get hasListeners(): boolean { - return this.listeners.size > 0 - } - - /** - * @description Subscribe to committed mutations. - * @param listener - Called once per committed mutation, in commit order. - * @returns An unsubscribe function. - */ - subscribe(listener: ChangeListener): () => void { - this.listeners.add(listener) - return () => { - this.listeners.delete(listener) - } - } - - /** - * @description Enqueue committed events and schedule dispatch. Call ONLY - * after the commit has succeeded — this feed must never announce a write - * that did not become durable. Safe to call with an empty array. - * @param events - The committed mutations, in commit order. - */ - emit(events: BrainyChangeEvent[]): void { - if (events.length === 0 || this.listeners.size === 0) return - this.queue.push(...events) - if (!this.draining) { - this.draining = true - queueMicrotask(() => this.drain()) - } - } - - /** Deliver everything queued, FIFO, isolating listener errors. */ - private drain(): void { - try { - while (this.queue.length > 0) { - const event = this.queue.shift()! - for (const listener of this.listeners) { - try { - listener(event) - } catch (err) { - // A subscriber's bug must never affect the write path or its - // sibling subscribers. - console.error('[Brainy] onChange listener threw:', err) - } - } - } - } finally { - this.draining = false - } - } - - /** Drop all listeners (brain close/teardown). Queued events are discarded. */ - close(): void { - this.listeners.clear() - this.queue.length = 0 - } -} diff --git a/src/graph/graphAdjacencyIndex.ts b/src/graph/graphAdjacencyIndex.ts index b37391aa..816856b1 100644 --- a/src/graph/graphAdjacencyIndex.ts +++ b/src/graph/graphAdjacencyIndex.ts @@ -282,16 +282,6 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { } hasMore = result.hasMore - if (hasMore && (!result.nextCursor || result.nextCursor === cursor)) { - // A stalled cursor with hasMore=true would re-read the same page - // forever — a silent full-CPU loop at cold open. Abort loudly; a - // graph read failing beats a process that spins without a log line. - throw new Error( - `GraphAdjacencyIndex: verb walk stalled after ${count} verbs — storage returned ` + - `hasMore=true with ${result.nextCursor ? 'a non-advancing' : 'no'} cursor. ` + - `Aborting the cold-load; run brain.repairIndex() if this persists.` - ) - } cursor = result.nextCursor } diff --git a/src/graph/graphAudit.ts b/src/graph/graphAudit.ts deleted file mode 100644 index d0e44cd9..00000000 --- a/src/graph/graphAudit.ts +++ /dev/null @@ -1,214 +0,0 @@ -/** - * @module graph/graphAudit - * @description Read-only graph-truth audit — the graph sibling of `repairIndex()`'s - * diagnosis half. Verifies three layers against each other without mutating anything: - * - * 1. CANONICAL verb records (the storage walk — the source of truth) - * 2. the RELATIONSHIP READ PATH (`related()` with all visibility tiers — exactly - * what application reads like a VFS `readdir` consult) - * 3. ENTITY ENDPOINTS (does each verb's source/target still exist?) - * - * and classifies every discrepancy into the three failure families production - * incidents have shown: - * - * - `missingFromReads` — a canonical verb record the read path does NOT return - * for its source: PRESENT BUT INVISIBLE (adjacency/membership staleness). - * - `danglingEndpoints` — a canonical verb whose endpoint entity is gone: - * the SCAR class (write-path loss / partial delete). - * - `readOnlyVerbIds` — the read path returns an edge with NO canonical - * record: GHOST edges (stale index entries). - * - * `visibilityHiddenCount` is reported separately: an internal/system edge that is - * indexed and present but hidden from DEFAULT reads is working as designed — the - * audit reads with all tiers included so design-hiding is never misclassified as - * index loss. - * - * Full counts are always exact; only the example LISTS are capped (`maxExamples`) - * — a capped report says so via `truncatedExamples`, never silently. - */ - -import { prodLog } from '../utils/logger.js' - -/** One discrepant relationship, identified fully enough to inspect by hand. */ -export interface GraphAuditDiscrepancy { - verbId: string - from: string - to: string - type: string -} - -export interface GraphAuditReport { - /** True iff every discrepancy count is zero — `related()` returns canonical truth. */ - coherent: boolean - verbsInCanonical: number - entitiesInCanonical: number - /** Distinct source entities whose read path was actually consulted (coverage honesty). */ - sourcesChecked: number - - /** PRESENT BUT INVISIBLE: canonical records the read path omits. */ - missingFromReadsCount: number - missingFromReads: GraphAuditDiscrepancy[] - - /** SCAR CLASS: canonical verbs with a missing endpoint entity. */ - danglingEndpointsCount: number - danglingEndpoints: Array - - /** GHOST EDGES: read-path verb ids with no canonical record. */ - readOnlyCount: number - readOnlyVerbIds: string[] - - /** Canonical verbs hidden from DEFAULT reads by design (internal/system visibility). */ - visibilityHiddenCount: number - - /** Example lists above were capped at maxExamples; counts remain exact. */ - truncatedExamples: boolean - durationMs: number -} - -/** A canonical verb record, as the audit needs it. */ -export interface AuditVerbRecord { - id: string - type: string - sourceId: string - targetId: string - visibility?: string -} - -/** The seams the audit runs over — injected so the walk is testable in isolation. */ -export interface GraphAuditDeps { - /** Stream every canonical entity id (id-only; no per-entity reads needed). */ - eachNounId(consume: (id: string) => void): Promise - /** Stream every canonical verb record. */ - eachVerb(consume: (verb: AuditVerbRecord) => void): Promise - /** - * The END-TO-END relationship read for one source, ALL visibility tiers - * included — must be the same path application reads consult. - */ - readRelationsFrom(sourceId: string): Promise> -} - -export interface GraphAuditOptions { - /** Cap on entries per example list (counts stay exact). Default 100. */ - maxExamples?: number -} - -export async function runGraphAudit( - deps: GraphAuditDeps, - options: GraphAuditOptions = {} -): Promise { - const maxExamples = options.maxExamples ?? 100 - const started = Date.now() - - // 1. Canonical entity ids — endpoint existence oracle. - const entityIds = new Set() - await deps.eachNounId((id) => entityIds.add(id)) - - // 2. Canonical verb walk: group by source, check endpoints, note visibility. - const canonicalVerbIds = new Set() - const bySource = new Map() - let verbsInCanonical = 0 - let visibilityHiddenCount = 0 - let danglingEndpointsCount = 0 - const danglingEndpoints: GraphAuditReport['danglingEndpoints'] = [] - - await deps.eachVerb((verb) => { - verbsInCanonical++ - canonicalVerbIds.add(verb.id) - const list = bySource.get(verb.sourceId) - if (list) list.push(verb) - else bySource.set(verb.sourceId, [verb]) - - if (verb.visibility === 'internal' || verb.visibility === 'system') { - visibilityHiddenCount++ - } - - const fromMissing = !entityIds.has(verb.sourceId) - const toMissing = !entityIds.has(verb.targetId) - if (fromMissing || toMissing) { - danglingEndpointsCount++ - if (danglingEndpoints.length < maxExamples) { - danglingEndpoints.push({ - verbId: verb.id, - from: verb.sourceId, - to: verb.targetId, - type: verb.type, - missingEnd: fromMissing && toMissing ? 'both' : fromMissing ? 'from' : 'to' - }) - } - } - }) - - // 3. Per-source read-path comparison. A verb must be returned by the read - // path of ITS OWN source — the exact consult a readdir/traversal makes. - let missingFromReadsCount = 0 - const missingFromReads: GraphAuditDiscrepancy[] = [] - let readOnlyCount = 0 - const readOnlyVerbIds: string[] = [] - const readOnlySeen = new Set() - - for (const [sourceId, verbs] of bySource) { - const readIds = new Set((await deps.readRelationsFrom(sourceId)).map((r) => r.id)) - - for (const verb of verbs) { - if (!readIds.has(verb.id)) { - missingFromReadsCount++ - if (missingFromReads.length < maxExamples) { - missingFromReads.push({ - verbId: verb.id, - from: verb.sourceId, - to: verb.targetId, - type: verb.type - }) - } - } - } - - for (const readId of readIds) { - if (!canonicalVerbIds.has(readId) && !readOnlySeen.has(readId)) { - readOnlySeen.add(readId) - readOnlyCount++ - if (readOnlyVerbIds.length < maxExamples) { - readOnlyVerbIds.push(readId) - } - } - } - } - - const coherent = - missingFromReadsCount === 0 && danglingEndpointsCount === 0 && readOnlyCount === 0 - - const report: GraphAuditReport = { - coherent, - verbsInCanonical, - entitiesInCanonical: entityIds.size, - sourcesChecked: bySource.size, - missingFromReadsCount, - missingFromReads, - danglingEndpointsCount, - danglingEndpoints, - readOnlyCount, - readOnlyVerbIds, - visibilityHiddenCount, - truncatedExamples: - missingFromReadsCount > missingFromReads.length || - danglingEndpointsCount > danglingEndpoints.length || - readOnlyCount > readOnlyVerbIds.length, - durationMs: Date.now() - started - } - - if (coherent) { - prodLog.info( - `[GraphAudit] coherent: ${verbsInCanonical} verbs across ${bySource.size} sources — ` + - `the read path returns canonical truth (${report.durationMs}ms)` - ) - } else { - prodLog.warn( - `[GraphAudit] INCOHERENT: ${missingFromReadsCount} present-but-invisible, ` + - `${danglingEndpointsCount} dangling-endpoint, ${readOnlyCount} ghost ` + - `(of ${verbsInCanonical} canonical verbs, ${bySource.size} sources, ` + - `${visibilityHiddenCount} visibility-hidden by design) — ${report.durationMs}ms` - ) - } - - return report -} diff --git a/src/graph/lsm/LSMTree.ts b/src/graph/lsm/LSMTree.ts index e19ec145..bfd48df4 100644 --- a/src/graph/lsm/LSMTree.ts +++ b/src/graph/lsm/LSMTree.ts @@ -574,7 +574,6 @@ export class LSMTree { * Load SSTables from storage based on manifest */ private async loadSSTables(): Promise { - const failures: string[] = [] const loadPromises: Promise[] = [] this.manifest.sstables.forEach((level, sstableId) => { @@ -599,12 +598,7 @@ export class LSMTree { } } } catch (error) { - // A per-SSTable load failure means the persisted adjacency is INCOMPLETE. - // Record it and fail the whole load closed (below): a partially-loaded - // tree that still publishes its full manifest count via size() would - // serve silent-empty traversals as truth (the cold-load swallow class). prodLog.warn(`LSMTree: Failed to load SSTable ${sstableId}`, error) - failures.push(sstableId) } })() @@ -612,20 +606,6 @@ export class LSMTree { }) await Promise.all(loadPromises) - - if (failures.length > 0) { - // Fail closed. loadManifest()'s catch resets sstables/totalRelationships/ - // sstablesByLevel to honest-empty, so size() reports 0 and the graph - // self-heal (_initializeGraphIndex size()===0 → rebuild) restores the index - // from the canonical records. Honest-partial is never published. - throw new Error( - `LSMTree(${this.config.storagePrefix}): ${failures.length} of ` + - `${this.manifest.sstables.size} SSTable(s) failed to load ` + - `(${failures.join(', ')}) — failing the load closed so size() reports 0 ` + - `and the graph self-heal rebuilds from canonical.` - ) - } - prodLog.info(`LSMTree: Loaded ${this.manifest.sstables.size} SSTables`) } diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index 605d2a0b..b4948bf8 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -25,29 +25,6 @@ const DEFAULT_CONFIG: HNSWConfig = { ml: 16 // Max level } -/** - * @description Thrown by {@link JsHnswVectorIndex.flush} when one or more dirty - * nodes (or the system record) could not be persisted. The failed nodes remain - * in the dirty set for the next flush; this error tells the caller the flush did - * NOT achieve durability instead of a node-count that lies. Mandate: loud - * errors, never quiet losses. - */ -export class HnswFlushError extends Error { - constructor( - public readonly failedNodeCount: number, - public readonly systemFailed: boolean, - public override readonly cause?: Error - ) { - super( - `HNSW flush did not achieve durability: ${failedNodeCount} node(s) failed to ` + - `persist${systemFailed ? ' and the system record (entryPoint/maxLevel) failed' : ''}. ` + - `Failed nodes remain dirty for retry.` + - (cause ? ` First error: ${cause.message}` : '') - ) - this.name = 'HnswFlushError' - } -} - /** * Implements {@link VectorIndexProvider}: the vector-index surface Brainy calls * on whatever the `'vector'` factory returns (its own `JsHnswVectorIndex`, or a native @@ -173,69 +150,41 @@ export class JsHnswVectorIndex implements VectorIndexProvider { const startTime = Date.now() const nodeCount = this.dirtyNodes.size - // Batch persist all dirty nodes concurrently. A node whose connections FAIL - // to persist must stay dirty (retried on the next flush) — clearing it would - // silently drop the write forever. Track failures; only successfully- - // persisted (or deleted) nodes leave the dirty set, so nodes added to it - // during this flush are preserved. - const failedNodes = new Set() - let firstError: Error | null = null - + // Batch persist all dirty nodes concurrently if (this.dirtyNodes.size > 0) { const batchSize = 50 // Reasonable batch size for cloud storage const nodeIds = Array.from(this.dirtyNodes) for (let i = 0; i < nodeIds.length; i += batchSize) { const batch = nodeIds.slice(i, i + batchSize) - const promises = batch.map(async nodeId => { + const promises = batch.map(nodeId => { const noun = this.nouns.get(nodeId) - if (!noun) return // Node was deleted — drop it from the dirty set. - try { - await this.persistNodeConnections(nodeId, noun) - } catch (error) { - failedNodes.add(nodeId) - if (firstError === null) firstError = error as Error - prodLog.error(`[HNSW flush] Failed to persist node ${nodeId}: ${(error as Error).message}`) - } + if (!noun) return Promise.resolve() // Node was deleted + + return this.persistNodeConnections(nodeId, noun).catch(error => { + console.error(`[HNSW flush] Failed to persist node ${nodeId}:`, error) + }) }) - await Promise.all(promises) + await Promise.allSettled(promises) } - // Remove only nodes that were persisted (or deleted mid-flush); keep the - // failed ones dirty for the next attempt. - for (const nodeId of nodeIds) { - if (!failedNodes.has(nodeId)) this.dirtyNodes.delete(nodeId) - } + this.dirtyNodes.clear() } - // Persist system data if dirty — keep it dirty on failure so the next flush - // retries rather than losing the entry-point/maxLevel update. - let systemFailed = false + // Persist system data if dirty if (this.dirtySystem) { - try { - await this.storage.saveHNSWSystem({ - entryPointId: this.entryPointId, - maxLevel: this.maxLevel - }) - this.dirtySystem = false - } catch (error) { - systemFailed = true - if (firstError === null) firstError = error as Error - prodLog.error(`[HNSW flush] Failed to persist system data: ${(error as Error).message}`) - } + await this.storage.saveHNSWSystem({ + entryPointId: this.entryPointId, + maxLevel: this.maxLevel + }).catch(error => { + console.error('[HNSW flush] Failed to persist system data:', error) + }) + + this.dirtySystem = false } const duration = Date.now() - startTime - - // Loud failure: if ANY node or the system record could not be persisted the - // flush did not achieve durability. Throw so callers (close(), explicit - // flush(), the flush-request watcher) see the failure instead of a success - // count that lies. The failed nodes/system stay dirty for retry. - if (failedNodes.size > 0 || systemFailed) { - throw new HnswFlushError(failedNodes.size, systemFailed, firstError ?? undefined) - } - if (nodeCount > 0) { prodLog.info(`[HNSW] Flushed ${nodeCount} dirty nodes in ${duration}ms`) } @@ -447,14 +396,13 @@ export class JsHnswVectorIndex implements VectorIndexProvider { this.maxLevel = nounLevel this.nouns.set(id, noun) - // Persist system data for first noun (previously skipped). Surface a - // persist failure loudly — the entry point is the root of the whole index; - // silently dropping it while addItem() returns the id would strand every - // future search on a rootless index. Mandate: never a quiet loss. + // Persist system data for first noun (previously skipped) if (this.storage && this.persistMode === 'immediate') { await this.storage.saveHNSWSystem({ entryPointId: this.entryPointId, maxLevel: this.maxLevel + }).catch(error => { + console.error('Failed to persist initial HNSW system data:', error) }) } else if (this.persistMode === 'deferred') { this.dirtySystem = true diff --git a/src/import/BackgroundDeduplicator.ts b/src/import/BackgroundDeduplicator.ts index 073fbd9e..53b28262 100644 --- a/src/import/BackgroundDeduplicator.ts +++ b/src/import/BackgroundDeduplicator.ts @@ -41,14 +41,6 @@ export interface DeduplicationStats { * - Import-scoped deduplication (no cross-contamination) * - 3-tier strategy (ID → Name → Similarity) * - Uses existing indexes (EntityIdMapper, MetadataIndexManager, TypeAware HNSW) - * - * Lifecycle: ONE instance per brain, owned by Brainy (getBackgroundDeduplicator) - * so the debounce genuinely spans imports and brain.close() cancels pending - * work via cancelPending() — this pass merge-DELETES duplicate entities, so it - * must never fire against a closed brain. The enableDeduplication gate lives - * at the scheduling call site (ImportCoordinator); scheduleDedup itself is - * unconditional. The timer is unref'd — a pending pass never holds the - * process open. */ export class BackgroundDeduplicator { private brain: Brainy @@ -75,15 +67,12 @@ export class BackgroundDeduplicator { clearTimeout(this.debounceTimer) } - // Schedule for 5 minutes from now. unref'd: a pending dedup pass must - // never hold the process open (exit-hang class) — if the process exits - // first, the pass simply never runs; imports are already durable. + // Schedule for 5 minutes from now this.debounceTimer = setTimeout(() => { this.runBatchDedup().catch(error => { prodLog.error('[BackgroundDedup] Batch dedup failed:', error) }) }, 5 * 60 * 1000) - this.debounceTimer.unref?.() } /** diff --git a/src/import/ImportCoordinator.ts b/src/import/ImportCoordinator.ts index 1e1316b7..21d09410 100644 --- a/src/import/ImportCoordinator.ts +++ b/src/import/ImportCoordinator.ts @@ -13,6 +13,7 @@ import { Brainy } from '../brainy.js' import { FormatDetector, SupportedFormat } from './FormatDetector.js' import { ImportHistory, type ImportHistoryEntry } from './ImportHistory.js' +import { BackgroundDeduplicator } from './BackgroundDeduplicator.js' import { SmartExcelImporter } from '../importers/SmartExcelImporter.js' import { SmartPDFImporter } from '../importers/SmartPDFImporter.js' import { SmartCSVImporter } from '../importers/SmartCSVImporter.js' @@ -111,12 +112,7 @@ export interface ValidImportOptions { /** Confidence threshold for entities */ confidenceThreshold?: number - /** - * Enable entity deduplication (default: true). Gates BOTH passes: the - * inline merge during import AND the debounced background pass that runs - * ~5 minutes after the last import (which merge-DELETES duplicate entities). - * Set false for deployments that must never auto-remove records. - */ + /** Enable entity deduplication across imports */ enableDeduplication?: boolean /** Similarity threshold for deduplication (0-1) */ @@ -290,6 +286,7 @@ export class ImportCoordinator { private brain: Brainy private detector: FormatDetector private history: ImportHistory + private backgroundDedup: BackgroundDeduplicator private excelImporter: SmartExcelImporter private pdfImporter: SmartPDFImporter private csvImporter: SmartCSVImporter @@ -303,6 +300,7 @@ export class ImportCoordinator { this.brain = brain this.detector = new FormatDetector() this.history = new ImportHistory(brain) + this.backgroundDedup = new BackgroundDeduplicator(brain) this.excelImporter = new SmartExcelImporter(brain) this.pdfImporter = new SmartPDFImporter(brain) this.csvImporter = new SmartCSVImporter(brain) @@ -1461,16 +1459,9 @@ export class ImportCoordinator { } } - // Schedule background deduplication (debounced 5 minutes, brain-owned so - // close() can cancel it). Honors the same enableDeduplication gate as the - // inline pass — false means NO dedup, inline or background. - if ( - trackingContext && - trackingContext.importId && - options.enableDeduplication !== false - ) { - const backgroundDedup = await this.brain.getBackgroundDeduplicator() - backgroundDedup.scheduleDedup(trackingContext.importId) + // Schedule background deduplication (debounced 5 minutes) + if (trackingContext && trackingContext.importId) { + this.backgroundDedup.scheduleDedup(trackingContext.importId) } return { diff --git a/src/index.ts b/src/index.ts index b978b9fd..d9acb524 100644 --- a/src/index.ts +++ b/src/index.ts @@ -15,29 +15,8 @@ import { Brainy } from './brainy.js' export { Brainy } -// The in-process change feed (brain.onChange) — event + listener types. -export type { - BrainyChangeEvent, - ChangeEventEntity, - ChangeEventRelation, - ChangeListener -} from './events/changeFeed.js' - -// Temporal VFS — a file version entry (vfs.history / readFile({ asOf })). -export type { FileVersion } from './vfs/types.js' - // Export diagnostics result type export type { DiagnosticsResult } from './brainy.js' -export type { - GraphAuditReport, - GraphAuditDiscrepancy -} from './graph/graphAudit.js' -export { - checkOsLimits, - NOFILE_POOL_FLOOR, - MAX_MAP_COUNT_POOL_FLOOR -} from './utils/osLimits.js' -export type { OsLimitsReport } from './utils/osLimits.js' // Export Brainy configuration and types export type { @@ -160,7 +139,7 @@ export { EntityNotFoundError, RelationNotFoundError } from './errors/notFound.js // Base error + typed migration-lock error — thrown by any data-plane call while a // brain runs its one-time 7.x→8.0 upgrade; catch to answer HTTP 503 + Retry-After. -export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError, ProtectedArtifactError, DerivedArtifactMissingError } from './errors/brainyError.js' +export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError } from './errors/brainyError.js' export type { BrainyErrorType } from './errors/brainyError.js' // ============= 8.0 Db API — generational MVCC ============= @@ -184,11 +163,8 @@ export type { export { GenerationConflictError, SpeculativeOverlayError, - GenerationCompactedError, - StoreInconsistentError, - PendingFlushDurabilityError + GenerationCompactedError } from './db/errors.js' -export type { UnreconciledRecord } from './db/errors.js' export type { TxOperation, TxAddOperation, @@ -201,30 +177,14 @@ export type { TxLogEntry, CompactHistoryOptions, CompactHistoryResult, - HistoryStats, ChangedIds, DiffResult, HistoryVersion, EntityHistory } from './db/types.js' -// The generation fact log — sequential after-image scan surface -// (brain.scanFacts / brain.factSegmentPaths) for index heals and replays. -export type { - CommitFact, - FactOp, - FactScanBatch, - SCANFACTS_FIRST_BATCH_MS, - FactScanHandle -} from './db/factLog.js' -// The generalized family stamp — which source generation a projection -// reflects + the surface that verifies it whole; one verifier, both member -// modes (enumerated byte-exact / rollup invariants). -export { readFamilyStamp, verifyFamilyStamp, ENTITY_TREE_STAMP_PATH } from './db/familyStamp.js' -export type { FamilyStamp, StampMembers, StampVerdict } from './db/familyStamp.js' // Optional provider capability for generation-aware native indexes export { isVersionedIndexProvider } from './plugin.js' export type { VersionedIndexProvider } from './plugin.js' -export type { ProviderInvariantReport, InvariantResult, InvariantHeal } from './plugin.js' // Optional native graph-acceleration engine (cor 3.0) — the published provider // contract + its columnar wire types. Brainy feature-detects an implementation // and falls back to its pure-TS adjacency when absent. @@ -326,8 +286,7 @@ import type { HNSWNoun, HNSWVerb, HNSWConfig, - StorageAdapter, - DerivedFamilyDeclaration + StorageAdapter } from './coreTypes.js' // Export vector index implementation (the JS HNSW path) @@ -345,8 +304,7 @@ export type { HNSWNoun, HNSWVerb, HNSWConfig, - StorageAdapter, - DerivedFamilyDeclaration + StorageAdapter } // Export graph types diff --git a/src/indexes/columnStore/ColumnStore.ts b/src/indexes/columnStore/ColumnStore.ts index d33c05c4..2c07d433 100644 --- a/src/indexes/columnStore/ColumnStore.ts +++ b/src/indexes/columnStore/ColumnStore.ts @@ -69,34 +69,6 @@ interface HeapEntry { * await store.flush() * const newest = await store.sortTopK('createdAt', 'desc', 10) // bigint[] */ -/** - * @description Thrown when a segment the manifest lists cannot be loaded — - * either it yields no bytes (missing/empty on disk) or its bytes are - * undecodable (corruption). Previously such a segment was silently skipped, - * dropping ALL of its entities from every `filter`/`rangeQuery`/`sortTopK` with - * no error — a manifest↔segments divergence that looked like a short result. - * Surfacing it loudly makes the divergence visible and repairable. (A genuine - * storage IO fault is a different class — it propagates as the underlying error, - * not wrapped in this.) - */ -export class ColumnSegmentLoadError extends Error { - /** The indexed field whose segment failed to load. */ - public readonly field: string - /** The manifest-listed segment id that could not be loaded. */ - public readonly segmentId: number | string - constructor(field: string, segmentId: number | string, reason: string) { - super( - `ColumnStore segment ${field}:${segmentId} is listed in the manifest but ` + - `could not be loaded (${reason}). The index is inconsistent with its ` + - `manifest — rebuild/repair the metadata index rather than trusting a ` + - `short query result.` - ) - this.name = 'ColumnSegmentLoadError' - this.field = field - this.segmentId = segmentId - } -} - export class ColumnStore implements ColumnStoreProvider { private storage!: StorageAdapter private idMapper!: EntityIdMapper @@ -622,15 +594,14 @@ export class ColumnStore implements ColumnStoreProvider { let cursor = this.segmentCache.get(cacheKey) if (!cursor) { - // loadSegmentCursor either returns a cursor or THROWS — a corrupt / - // missing manifest-listed segment raises ColumnSegmentLoadError and a - // real storage fault propagates, so a listed segment is never silently - // dropped from the result set. - cursor = await this.loadSegmentCursor(field, seg) - this.segmentCache.set(cacheKey, cursor) + const loaded = await this.loadSegmentCursor(field, seg) + if (loaded) { + cursor = loaded + this.segmentCache.set(cacheKey, cursor) + } } - cursors.push(cursor) + if (cursor) cursors.push(cursor) } return cursors @@ -644,51 +615,39 @@ export class ColumnStore implements ColumnStoreProvider { * `.cidx` object-path so indexes written before the format unification keep * loading correctly. Mirror of cortex's 2.3.1 read-side fallback. */ - private async loadSegmentCursor(field: string, seg: SegmentMeta): Promise { + private async loadSegmentCursor(field: string, seg: SegmentMeta): Promise { const manifest = this.manifests.get(field) - if (!manifest) { - // Defensive: getSegmentCursors guards on the manifest before calling. - throw new ColumnSegmentLoadError(field, seg.id, 'no manifest for field') - } + if (!manifest) return null - let buf: Buffer | null = null + try { + let buf: Buffer | null = null - const storage = this.storage as unknown as { - loadBinaryBlob?: (key: string) => Promise - readObjectFromPath: (path: string) => Promise - } + const storage = this.storage as unknown as { + loadBinaryBlob?: (key: string) => Promise + readObjectFromPath: (path: string) => Promise + } - // Preferred: raw blob at the cor-shared key. A real IO fault PROPAGATES — - // loadBinaryBlob throws on a fault and returns null only for genuine absence - // (a present-but-unreadable segment must not read as "missing"). - if (typeof storage.loadBinaryBlob === 'function') { - const key = `${this.basePath}/${field}/L${seg.level}-${String(seg.id).padStart(6, '0')}` - const blob = await storage.loadBinaryBlob(key) - if (blob && blob.length > 0) buf = blob - } + // Preferred: raw blob at the cor-shared key. + if (typeof storage.loadBinaryBlob === 'function') { + const key = `${this.basePath}/${field}/L${seg.level}-${String(seg.id).padStart(6, '0')}` + const blob = await storage.loadBinaryBlob(key) + if (blob && blob.length > 0) buf = blob + } - // Legacy fallback: `{ _binary, base64 }` envelope at the .cidx object-path. - if (!buf) { - const segPath = manifest.segmentPath(seg.level, seg.id) - const stored = await storage.readObjectFromPath(segPath) - if (stored) { + // Legacy fallback: `{ _binary, base64 }` envelope at the .cidx object-path. + if (!buf) { + const segPath = manifest.segmentPath(seg.level, seg.id) + const stored = await storage.readObjectFromPath(segPath) + if (!stored) return null if (stored._binary && stored.data) { buf = Buffer.from(stored.data, 'base64') } else if (Buffer.isBuffer(stored)) { buf = stored + } else { + return null } } - } - // A manifest-listed segment that yields NO loadable bytes is corruption, not - // benign absence: swallowing it silently dropped all of the segment's - // entities from every filter/rangeQuery/sortTopK with no error. Surface it - // loudly so the manifest↔segments divergence is visible (and repairable). - if (!buf) { - throw new ColumnSegmentLoadError(field, seg.id, 'manifest-listed segment has no loadable bytes') - } - - try { const parsed = readSegmentFromBuffer(buf) return new ColumnSegmentCursor( parsed.header, @@ -696,13 +655,8 @@ export class ColumnStore implements ColumnStoreProvider { parsed.entityIds, parsed.tombstones ) - } catch (err) { - // Undecodable bytes for a listed segment — corruption, not a short result. - throw new ColumnSegmentLoadError( - field, - seg.id, - `segment decode failed: ${(err as Error).message}` - ) + } catch { + return null } } diff --git a/src/internals.ts b/src/internals.ts index 009124fe..00a24638 100644 --- a/src/internals.ts +++ b/src/internals.ts @@ -19,29 +19,3 @@ export type { MemoryInfo, CacheAllocationStrategy } from './utils/memoryDetectio // HNSWNounWithMetadata. First-party plugins (Cor) use this to stay in // lockstep with the entity shape contract. export { resolveEntityField, STANDARD_ENTITY_FIELDS } from './coreTypes.js' - -// The generalized family stamp — ONE verifier, both member modes, shared -// verbatim with native providers so stamp verification is literally one -// function, never two synchronized copies. Providers write the same shape -// (their set-swap rewrites stamps, so adoption is migration-free). -export { - readFamilyStamp, - writeFamilyStamp, - verifyFamilyStamp, - FAMILY_STAMPS_PREFIX, - ENTITY_TREE_STAMP_PATH -} from './db/familyStamp.js' -export type { - FamilyStamp, - StampMembers, - StampVerdict, - EnumeratedMember, - StampStorage -} from './db/familyStamp.js' - -// Generation fact-log types — the scan surface a provider reaches through the -// storage capability (`storage.scanFacts` / `storage.factLogHeadGeneration` / -// `storage.factSegmentPaths`, wired by the host brain at init). Providers -// never construct a FactLog themselves: open() is writer-side (it reconciles -// by truncating/rewriting) and there is exactly one writer. -export type { CommitFact, FactOp, FactScanBatch, FactScanHandle } from './db/factLog.js' diff --git a/src/plugin.ts b/src/plugin.ts index df7c7224..12455b3a 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -115,62 +115,6 @@ export interface BrainyPluginContext { // implementation then fails to compile until it provides the member. // =========================================================================== -/** - * @description How a failed provider invariant should be remediated: - * - `'none'` — informational; the invariant held or nothing to do. - * - `'repair'` — a targeted, cheap fix exists (e.g. re-derive a count/manifest field). - * - `'rebuild'` — the derived state must be rebuilt from canonical (`provider.rebuild()`). - */ -export type InvariantHeal = 'none' | 'repair' | 'rebuild' - -/** - * @description The result of ONE provider invariant check. A failure - * (`holds === false`) NAMES what diverged, with numbers, so it is diagnosable - * from the report alone — never a bare boolean. `name` is a stable kebab-case id - * for telemetry / remediation routing. - */ -export interface InvariantResult { - /** Stable kebab-case id, e.g. `'manifest-residency'` / `'posted-count-floor'`. */ - name: string - /** `true` when the invariant holds. */ - holds: boolean - /** Human-readable detail; on failure, names the divergence WITH numbers. */ - detail: string - /** The value the invariant expected (optional, for diagnosis). */ - expected?: unknown - /** The value actually observed (optional, for diagnosis). */ - actual?: unknown - /** How a failure should be remediated. Ignored when `holds === true`. */ - heal: InvariantHeal -} - -/** - * @description A provider's self-report of its own cross-layer invariants - * (the `validateInvariants()` hook). Contract: - * - It NEVER throws — a failure is DATA (`healthy: false` + a failing invariant), - * not an exception. - * - It is BOUNDED (<50ms): residency checks + O(1) counts only, NO canonical - * walks — safe to call on a live brain, repeatedly. - * - `serving` = can the provider answer queries right now (the `isReady()` truth); - * `healthy` = do ALL invariants hold. A provider can be `serving` while an - * invariant flags a latent divergence, or `healthy` but not-yet-`serving` on a - * cold open. - */ -export interface ProviderInvariantReport { - /** Which provider produced this report, e.g. `'vector'` / `'graph'` / `'metadata'` / `'column'`. */ - provider: string - /** `true` iff every invariant in {@link invariants} holds. */ - healthy: boolean - /** `true` iff the provider can serve queries now (the `isReady()` truth). */ - serving: boolean - /** Each checked invariant and its verdict. */ - invariants: InvariantResult[] - /** Epoch millis when the check ran. */ - checkedAt: number - /** How long the check took (must stay well under 50ms). */ - durationMs: number -} - /** * The `'metadataIndex'` provider — a drop-in for `MetadataIndexManager`. * Brainy calls this surface via `this.metadataIndex.*` (see `brainy.ts`) and @@ -195,18 +139,6 @@ export interface MetadataIndexProvider { */ isReady?(): boolean - /** - * @description OPTIONAL. The provider's self-report of its own - * cross-layer invariants (manifest ↔ segments ↔ counts residency/coherence). - * MUST NOT throw — a failure is DATA (`healthy: false` + a failing invariant). - * MUST be bounded (<50ms): residency + O(1) counts only, NO canonical walks, so - * brainy's {@link } `validateIndexConsistency()` can call it on a live brain. - * Absent → brainy skips this provider in the cross-layer check (feature-detected). - * `repairIndex()` maps any failing invariant with `heal: 'rebuild'` to this - * provider's `rebuild()`. - */ - validateInvariants?(): Promise - /** * @description OPTIONAL. A native provider returns true from the moment its * `init()` detects a large epoch-drift until its background @@ -357,18 +289,6 @@ export interface GraphIndexProvider { */ isReady?(): boolean - /** - * @description OPTIONAL. The provider's self-report of its own - * cross-layer invariants (manifest ↔ segments ↔ counts residency/coherence). - * MUST NOT throw — a failure is DATA (`healthy: false` + a failing invariant). - * MUST be bounded (<50ms): residency + O(1) counts only, NO canonical walks, so - * brainy's {@link } `validateIndexConsistency()` can call it on a live brain. - * Absent → brainy skips this provider in the cross-layer check (feature-detected). - * `repairIndex()` maps any failing invariant with `heal: 'rebuild'` to this - * provider's `rebuild()`. - */ - validateInvariants?(): Promise - /** * @description OPTIONAL eager cold-load. Called once during brain init — AFTER * the metadata provider's `init()` (so the id-mapper is hydrated; a native int @@ -1023,18 +943,6 @@ export interface VectorIndexProvider { */ isReady?(): boolean - /** - * @description OPTIONAL. The provider's self-report of its own - * cross-layer invariants (manifest ↔ segments ↔ counts residency/coherence). - * MUST NOT throw — a failure is DATA (`healthy: false` + a failing invariant). - * MUST be bounded (<50ms): residency + O(1) counts only, NO canonical walks, so - * brainy's {@link } `validateIndexConsistency()` can call it on a live brain. - * Absent → brainy skips this provider in the cross-layer check (feature-detected). - * `repairIndex()` maps any failing invariant with `heal: 'rebuild'` to this - * provider's `rebuild()`. - */ - validateInvariants?(): Promise - /** * @description OPTIONAL. A native provider returns true from the moment its * `init()` detects a large epoch-drift until its background diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 5eb4785a..81390eab 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -5,7 +5,9 @@ import { HNSWNoun, + HNSWVerb, HNSWNounWithMetadata, + HNSWVerbWithMetadata, StatisticsData, NounType } from '../../coreTypes.js' @@ -17,7 +19,10 @@ import { WriterLockInfo } from '../baseStorage.js' import { getBrainyVersion } from '../../utils/index.js' -import { isAbsentError } from '../../utils/errorClassification.js' + +// Type aliases for better readability +type HNSWNode = HNSWNoun +type Edge = HNSWVerb // Node.js modules - dynamically imported to avoid issues in browser environments let fs: any @@ -70,6 +75,8 @@ export class FileSystemStorage extends BaseStorage { // - Handles 2.5M+ entities with < 10K files per shard // - Eliminates dynamic depth changes that cause path mismatch bugs private readonly SHARDING_DEPTH = 1 as const + private readonly MAX_SHARDS = 256 // Hex range: 00-ff + private cachedShardingDepth: number = this.SHARDING_DEPTH // Always use fixed depth protected rootDir: string private nounsDir!: string private verbsDir!: string @@ -96,14 +103,6 @@ export class FileSystemStorage extends BaseStorage { private static readonly WRITER_STALE_THRESHOLD_MS = 60_000 private writerLockHeartbeat?: NodeJS.Timeout private writerLockInfo?: WriterLockInfo - /** - * The currently-executing heartbeat refresh, if any. `releaseWriterLock()` - * awaits it before unlinking: clearInterval() stops FUTURE ticks but not a - * tick already in flight, and a straggler landing after the unlink would - * RE-CREATE the lock file — a phantom lock blocking the next writer until - * the stale TTL expires (the pool-eviction reopen case). - */ - private writerHeartbeatInFlight?: Promise // Flush-request RPC state. The writer polls `locks/_flush_requests/` for // new `.req` files and emits `.ack` files in `locks/_flush_responses/` after @@ -128,18 +127,6 @@ export class FileSystemStorage extends BaseStorage { private compressionEnabled: boolean = true // Enable gzip compression by default for 60-80% disk savings private compressionLevel: number = 6 // zlib compression level (1-9, default: 6 = balanced) - // Transaction durability barrier (see GenerationStorage.beginWriteBarrier). - // Non-null ONLY between beginWriteBarrier() and flushWriteBarrier(). Because - // canonical writes are tmp+rename (durable only in the page cache until an - // fsync), the generation store fsyncs everything a transaction wrote before - // it advances the generation counter. `writeBarrierPaths` collects the - // root-relative object paths written; `writeBarrierDeleteDirs` the parent - // dirs of deleted objects (an unlink is durable only once its directory is - // fsync'd). Both are null outside a transaction, so the tracking `add`s below - // are free on the single-op and non-transactional write paths. - private writeBarrierPaths: Set | null = null - private writeBarrierDeleteDirs: Set | null = null - /** * Initialize the storage adapter * @param rootDirectory The root directory for storage @@ -232,11 +219,6 @@ export class FileSystemStorage extends BaseStorage { // Create the root directory if it doesn't exist await this.ensureDirectoryExists(this.rootDir) - // Finish any restore interrupted by a crash (resume the staged swap, or - // discard an uncommitted staging area) BEFORE counts/derived state load, - // so the rest of startup sees the completed store. - await this.completeInterruptedRestore() - // Create the nouns directory if it doesn't exist await this.ensureDirectoryExists(this.nounsDir) @@ -269,20 +251,40 @@ export class FileSystemStorage extends BaseStorage { this.countsFilePath = path.join(this.systemDir, 'counts.json') await this.initializeCounts() - // Boot log: new-vs-established, decided from the canonical layout the - // database actually reads and writes (`entities/nouns///`) - // plus the known noun count. The legacy hnsw sharding-depth probe and - // its depth-migration machinery are gone: the 8.0 write path never - // populated the directory they inspected, so the probe concluded "new - // installation" for every store on every boot and the migration branch - // was unreachable. - const established = - this.totalNounCount > 0 || (await this.hasCanonicalEntities()) - console.log( - established - ? `📁 Using depth ${this.SHARDING_DEPTH} sharding (${this.totalNounCount} entities)` - : `📁 New installation: using depth ${this.SHARDING_DEPTH} sharding (optimal for 1-2.5M entities)` - ) + // Detect existing sharding structure and migrate if needed + const detectedDepth = await this.detectExistingShardingDepth() + + if (detectedDepth !== null && detectedDepth !== this.SHARDING_DEPTH) { + // Migration needed: existing structure doesn't match our fixed depth + console.log(`📦 Brainy Storage Migration`) + console.log(` Current structure: depth ${detectedDepth}`) + console.log(` Target structure: depth ${this.SHARDING_DEPTH}`) + console.log(` Entities to migrate: ${this.totalNounCount}`) + + await this.migrateShardingStructure(detectedDepth, this.SHARDING_DEPTH) + + console.log(`✅ Migration complete - now using depth ${this.SHARDING_DEPTH} sharding`) + } else if (detectedDepth === null) { + // The legacy sharding probe inspects `entities/nouns/hnsw` — a 7.x + // directory the 8.0 write path never populates (8.0 stores entities in + // the canonical `entities/nouns///` layout), so it returns + // null for EVERY 8.0 store, new or not. Decide new-vs-existing from the + // canonical layout the DB actually reads/writes so an established brain + // is not mislabeled "New installation" on every boot. + const established = + this.totalNounCount > 0 || (await this.hasCanonicalEntities()) + console.log( + established + ? `📁 Using depth ${this.SHARDING_DEPTH} sharding (${this.totalNounCount} entities)` + : `📁 New installation: using depth ${this.SHARDING_DEPTH} sharding (optimal for 1-2.5M entities)` + ) + } else { + // Already using correct depth + console.log(`📁 Using depth ${this.SHARDING_DEPTH} sharding (${this.totalNounCount} entities)`) + } + + // Always use fixed depth after migration/detection + this.cachedShardingDepth = this.SHARDING_DEPTH // Initialize GraphAdjacencyIndex and type statistics await super.init() @@ -318,15 +320,390 @@ export class FileSystemStorage extends BaseStorage { } } + /** + * Save a node to storage + * CRITICAL FIX: Added atomic write pattern to prevent file corruption during concurrent imports + */ + protected async saveNode(node: HNSWNode): Promise { + await this.ensureInitialized() + + // Convert connections Map to a serializable format + // CRITICAL: Only save lightweight vector data (no metadata) + // Metadata is saved separately via saveNounMetadata() (2-file system) + const serializableNode = { + id: node.id, + vector: node.vector, + connections: this.mapToObject(node.connections, (set) => + Array.from(set as Set) + ), + level: node.level || 0 + // NO metadata field - saved separately for scalability + } + + const filePath = this.getNodePath(node.id) + const tempPath = `${filePath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}` + + try { + // ATOMIC WRITE SEQUENCE: + // 1. Write to temp file + await this.ensureDirectoryExists(path.dirname(tempPath)) + await fs.promises.writeFile(tempPath, JSON.stringify(serializableNode, null, 2)) + + // 2. Atomic rename temp → final (crash-safe, prevents truncation during concurrent writes) + await fs.promises.rename(tempPath, filePath) + } catch (error: any) { + // Clean up temp file on any error + try { + await fs.promises.unlink(tempPath) + } catch (cleanupError) { + // Ignore cleanup errors + } + throw error + } + + // Count tracking happens in baseStorage.saveNounMetadata_internal + // This fixes the race condition where metadata didn't exist yet + } + + /** + * Get a node from storage + */ + protected async getNode(id: string): Promise { + await this.ensureInitialized() + + // Clean, predictable path - no backward compatibility needed + const filePath = this.getNodePath(id) + try { + const data = await fs.promises.readFile(filePath, 'utf-8') + const parsedNode = JSON.parse(data) + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(parsedNode.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + // CRITICAL: Only return lightweight vector data (no metadata) + // Metadata is retrieved separately via getNounMetadata() (2-file system) + return { + id: parsedNode.id, + vector: parsedNode.vector, + connections, + level: parsedNode.level || 0 + // NO metadata field - retrieved separately for scalability + } + } catch (error: any) { + if (error.code !== 'ENOENT') { + console.error(`Error reading node ${id}:`, error) + } + return null + } + } + + /** + * Get all nodes from storage + * CRITICAL FIX: Now scans sharded subdirectories (depth=1) + * Previously only scanned flat directory, causing rebuild to find 0 entities + */ + protected async getAllNodes(): Promise { + await this.ensureInitialized() + + const allNodes: HNSWNode[] = [] + try { + // FIX: Use sharded file discovery instead of flat directory read + // This scans all 256 shard subdirectories (00-ff) to find actual files + const files = await this.getAllShardedFiles(this.nounsDir) + + for (const file of files) { + // Extract ID from filename and use sharded path + const id = file.replace('.json', '') + const filePath = this.getNodePath(id) + + const data = await fs.promises.readFile(filePath, 'utf-8') + const parsedNode = JSON.parse(data) + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries( + parsedNode.connections + )) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + allNodes.push({ + id: parsedNode.id, + vector: parsedNode.vector, + connections, + level: parsedNode.level || 0 + }) + } + } catch (error: any) { + if (error.code !== 'ENOENT') { + console.error(`Error reading directory ${this.nounsDir}:`, error) + } + } + return allNodes + } + + /** + * Get nodes by noun type + * CRITICAL FIX: Now scans sharded subdirectories (depth=1) + * @param nounType The noun type to filter by + * @returns Promise that resolves to an array of nodes of the specified noun type + */ + protected async getNodesByNounType(nounType: string): Promise { + await this.ensureInitialized() + + const nouns: HNSWNode[] = [] + try { + // FIX: Use sharded file discovery instead of flat directory read + const files = await this.getAllShardedFiles(this.nounsDir) + + for (const file of files) { + // Extract ID from filename and use sharded path + const nodeId = file.replace('.json', '') + const filePath = this.getNodePath(nodeId) + + const data = await fs.promises.readFile(filePath, 'utf-8') + const parsedNode = JSON.parse(data) + + // Filter by noun type using metadata + const metadata = await this.getMetadata(nodeId) + if (metadata && metadata.noun === nounType) { + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries( + parsedNode.connections + )) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + nouns.push({ + id: parsedNode.id, + vector: parsedNode.vector, + connections, + level: parsedNode.level || 0 + }) + } + } + } catch (error: any) { + if (error.code !== 'ENOENT') { + console.error(`Error reading directory ${this.nounsDir}:`, error) + } + } + + return nouns + } + + /** + * Delete a node from storage + */ + protected async deleteNode(id: string): Promise { + await this.ensureInitialized() + + const filePath = this.getNodePath(id) + + // Load metadata to get type for count update (separate storage) + try { + const metadata = await this.getNounMetadata(id) + if (metadata) { + const type = metadata.noun || 'default' + this.decrementEntityCount(type) + } + } catch { + // Metadata might not exist, that's ok + } + + try { + await fs.promises.unlink(filePath) + + // Persist counts periodically + if (this.totalNounCount % 10 === 0) { + await this.persistCounts() + } + } catch (error: any) { + if (error.code !== 'ENOENT') { + console.error(`Error deleting node file ${filePath}:`, error) + throw error + } + } + } + + /** + * Save an edge to storage + * CRITICAL FIX: Added atomic write pattern to prevent file corruption during concurrent imports + */ + protected async saveEdge(edge: Edge): Promise { + await this.ensureInitialized() + + // Convert connections Map to a serializable format + // ARCHITECTURAL FIX: Include core relational fields in verb vector file + // These fields are essential for 90% of operations - no metadata lookup needed + const serializableEdge = { + id: edge.id, + vector: edge.vector, + connections: this.mapToObject(edge.connections, (set) => + Array.from(set as Set) + ), + + // CORE RELATIONAL DATA + verb: edge.verb, + sourceId: edge.sourceId, + targetId: edge.targetId, + + // User metadata (if any) - saved separately for scalability + // metadata field is saved separately via saveVerbMetadata() + } + + const filePath = this.getVerbPath(edge.id) + const tempPath = `${filePath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}` + + try { + // ATOMIC WRITE SEQUENCE: + // 1. Write to temp file + await this.ensureDirectoryExists(path.dirname(tempPath)) + await fs.promises.writeFile(tempPath, JSON.stringify(serializableEdge, null, 2)) + + // 2. Atomic rename temp → final (crash-safe, prevents truncation during concurrent writes) + await fs.promises.rename(tempPath, filePath) + } catch (error: any) { + // Clean up temp file on any error + try { + await fs.promises.unlink(tempPath) + } catch (cleanupError) { + // Ignore cleanup errors + } + throw error + } + + // Count tracking happens in baseStorage.saveVerbMetadata_internal + // This fixes the race condition where metadata didn't exist yet + } + + /** + * Get an edge from storage + */ + protected async getEdge(id: string): Promise { + await this.ensureInitialized() + + const filePath = this.getVerbPath(id) + try { + const data = await fs.promises.readFile(filePath, 'utf-8') + const parsedEdge = JSON.parse(data) + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + // Return HNSWVerb with core relational fields (NO metadata field) + return { + id: parsedEdge.id, + vector: parsedEdge.vector, + connections, + + // CORE RELATIONAL DATA (read from vector file) + verb: parsedEdge.verb, + sourceId: parsedEdge.sourceId, + targetId: parsedEdge.targetId + + // ✅ NO metadata field + // User metadata retrieved separately via getVerbMetadata() + } + } catch (error: any) { + if (error.code !== 'ENOENT') { + console.error(`Error reading edge ${id}:`, error) + } + return null + } + } + + /** + * Get all edges from storage + * CRITICAL FIX: Now scans sharded subdirectories (depth=1) + * Previously only scanned flat directory, causing rebuild to find 0 relationships + */ + protected async getAllEdges(): Promise { + await this.ensureInitialized() + + const allEdges: Edge[] = [] + try { + // FIX: Use sharded file discovery instead of flat directory read + // This scans all 256 shard subdirectories (00-ff) to find actual files + const files = await this.getAllShardedFiles(this.verbsDir) + + for (const file of files) { + // Extract ID from filename and use sharded path + const id = file.replace('.json', '') + const filePath = this.getVerbPath(id) + + const data = await fs.promises.readFile(filePath, 'utf-8') + const parsedEdge = JSON.parse(data) + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nodeIds] of Object.entries( + parsedEdge.connections + )) { + connections.set(Number(level), new Set(nodeIds as string[])) + } + + // Include core relational fields (NO metadata field) + allEdges.push({ + id: parsedEdge.id, + vector: parsedEdge.vector, + connections, + + // CORE RELATIONAL DATA + verb: parsedEdge.verb, + sourceId: parsedEdge.sourceId, + targetId: parsedEdge.targetId + + // ✅ NO metadata field + // User metadata retrieved separately via getVerbMetadata() + }) + } + } catch (error: any) { + if (error.code !== 'ENOENT') { + console.error(`Error reading directory ${this.verbsDir}:`, error) + } + } + return allEdges + } + /** + * Delete an edge from storage + */ + protected async deleteEdge(id: string): Promise { + await this.ensureInitialized() + // Delete the HNSWVerb file using sharded path + const filePath = this.getVerbPath(id) + try { + await fs.promises.unlink(filePath) + } catch (error: any) { + if (error.code !== 'ENOENT') { + console.error(`Error deleting edge file ${filePath}:`, error) + throw error + } + } - - - - - + // CRITICAL: Also delete verb metadata - this is what getVerbs() uses to find verbs + // Without this, getVerbsBySource() will still find "deleted" verbs via their metadata + try { + const metadata = await this.getVerbMetadata(id) + if (metadata) { + const verbType = (metadata.verb || metadata.type || 'default') as string + this.decrementVerbCount(verbType) + await this.deleteVerbMetadata(id) + } + } catch (error) { + // Ignore metadata deletion errors - verb file is already deleted + console.warn(`Failed to delete verb metadata for ${id}:`, error) + } + } /** * Primitive operation: Write object to path @@ -399,12 +776,6 @@ export class FileSystemStorage extends BaseStorage { throw error } } - - // Transaction durability barrier: record the write so the generation store - // can fsync it before advancing the counter. Reached only on a successful - // rename; the logical (non-`.gz`) path is stored — flushWriteBarrier's - // syncRawObjects resolves the compressed variant. - this.writeBarrierPaths?.add(pathStr) } /** @@ -455,12 +826,8 @@ export class FileSystemStorage extends BaseStorage { return null } - // A real storage fault (EIO/EACCES/EMFILE/…) is NOT "object absent". The - // ENOENT branch (above) already returns null, and the corrupted-JSON - // branch (above) is a deliberate concurrent-write tolerance; a genuine - // fault reaching here must propagate loudly rather than masquerade as a - // missing object — which would corrupt reads and drive needless rebuilds. - throw error + console.error(`Error reading object from ${pathStr}:`, error) + return null } } @@ -503,109 +870,6 @@ export class FileSystemStorage extends BaseStorage { if (deletedCount === 0) { // File doesn't exist - this is fine } - - // Transaction durability barrier: an unlink is durable only once its parent - // directory is fsync'd. Record the dir (root-relative; '.' for a top-level - // object) so flushWriteBarrier can sync it before the counter advances. - this.writeBarrierDeleteDirs?.add(path.dirname(pathStr)) - } - - /** - * @description Remove the entity container directory after both canonical legs - * were deleted (full-removal delete). `objectLegPath` is one leg's - * storage-root-relative path (e.g. `entities/nouns///vectors.json`); - * its parent is the `/` entity directory. `rmdir` removes it ONLY if empty - * — both legs are gone by this point, so it should be. A non-empty dir means an - * unexpected leftover (a leg whose delete faulted — already surfaced upstream — - * or foreign data): we do NOT recursively nuke it (loud errors, never quiet - * losses); we log and leave it for the orphan repair sweep (`repairIndex`). - */ - protected override async removeCanonicalContainer(objectLegPath: string): Promise { - const relDir = path.dirname(objectLegPath) - const absDir = path.join(this.rootDir, relDir) - try { - await fs.promises.rmdir(absDir) - // The dir removal is durable once its PARENT (the shard dir) is fsync'd. - this.writeBarrierDeleteDirs?.add(path.dirname(relDir)) - } catch (error: any) { - if (error?.code === 'ENOENT') return // container already gone — fine - if (error?.code === 'ENOTEMPTY' || error?.code === 'EEXIST') { - console.warn( - `[FileSystemStorage] entity container ${relDir} not empty after delete — ` + - `leaving it for the orphan repair sweep (brain.repairIndex()).` - ) - return - } - throw error - } - } - - /** - * @description Prune orphaned entity directories left by the pre-8.3.1 - * partial-delete defect. A delete used to remove the metadata (content) leg - * but leave the `vectors.json` leg + the `/` directory (a "ghost"), or — - * on the direct storage path — remove both legs but leave an empty directory - * (a "scar"). Neither is a live entity (`getNoun` needs the metadata content - * leg) yet each lingers on disk, inflating enumerated counts and confusing - * locator resolution. - * - * CONSERVATIVE by design: removes ONLY dirs with NO metadata content leg — a - * directory that still holds its content is left untouched. LOUD: logs every - * removed orphan. Operator-invoked via {@link Brainy.repairIndex}; never runs - * automatically. Returns the pruned container ids so the caller can recompute - * counts. - */ - public async pruneOrphanedEntities(): Promise<{ nouns: string[]; verbs: string[] }> { - await this.ensureInitialized() - const pruned: { nouns: string[]; verbs: string[] } = { nouns: [], verbs: [] } - - for (const [kind, root] of [ - ['nouns', 'entities/nouns'], - ['verbs', 'entities/verbs'] - ] as const) { - const rootAbs = path.join(this.rootDir, root) - let shards: string[] - try { - shards = await fs.promises.readdir(rootAbs) - } catch (error: any) { - if (error?.code === 'ENOENT') continue // no entities of this kind yet - throw error - } - - for (const shard of shards) { - const shardAbs = path.join(rootAbs, shard) - let entries: import('fs').Dirent[] - try { - entries = await fs.promises.readdir(shardAbs, { withFileTypes: true }) - } catch (error: any) { - if (error?.code === 'ENOENT') continue - throw error - } - - for (const entry of entries) { - if (!entry.isDirectory()) continue - const idAbs = path.join(shardAbs, entry.name) - let legs: string[] - try { - legs = await fs.promises.readdir(idAbs) - } catch (error: any) { - if (error?.code === 'ENOENT') continue - throw error - } - // A live entity has its metadata content leg. No content leg → a - // vector-only ghost or an empty scar → prune the whole container. - if (legs.some((f) => f.startsWith('metadata.json'))) continue - await fs.promises.rm(idAbs, { recursive: true, force: true }) - pruned[kind].push(entry.name) - console.warn( - `[FileSystemStorage] pruned orphaned ${kind === 'nouns' ? 'noun' : 'verb'} ` + - `container ${root}/${shard}/${entry.name} (no metadata content leg)` - ) - } - } - } - - return pruned } /** @@ -701,43 +965,11 @@ export class FileSystemStorage extends BaseStorage { */ private static readonly SNAPSHOT_BYTE_COPY_DIRS = new Set(['_id_mapper']) - /** - * Nested path PREFIXES whose files are byte-copied into snapshots, not - * hard-linked — for append-in-place files below the top level. The - * generation fact log's tail segment is appended in place between rotations; - * a hard-linked tail would let post-snapshot appends reach through into the - * snapshot. (Sealed segments are immutable and would be link-safe, but the - * prefix rule keeps the discipline simple; segments are bounded by the - * rotation threshold, so the copy cost is small.) - */ - private static readonly SNAPSHOT_BYTE_COPY_PREFIXES: string[] = ['_generations/facts/'] - /** * Top-level directories excluded from snapshots: process-local lock state - * (writer lock, flush-request RPC files) must never travel with the data, and - * the restore staging area ({@link RESTORE_STAGING_DIR}) is transient scratch - * that must never be captured or restored. + * (writer lock, flush-request RPC files) must never travel with the data. */ - private static readonly SNAPSHOT_EXCLUDED_TOP_DIRS = new Set(['locks', '_restore_staging']) - - /** - * Transient top-level directory holding a restore-in-progress: the snapshot is - * fully copied here (sparse-aware) BEFORE any live data is touched, then an - * atomic per-entry swap moves it into place. Its presence + the completion - * marker let {@link completeInterruptedRestore} resume a crashed restore. - */ - private static readonly RESTORE_STAGING_DIR = '_restore_staging' - /** - * Written+fsync'd inside the staging dir ONLY after the whole snapshot has - * copied successfully. Its presence authorizes the swap (and its resume): a - * staging dir WITHOUT this marker is an interrupted copy — discardable debris, - * live data still authoritative. - */ - private static readonly RESTORE_MARKER = '.restore-manifest.json' - /** Chunk size for sparse-aware copying (holes are preserved at this grain). */ - private static readonly SPARSE_CHUNK_BYTES = 4 * 1024 * 1024 - /** A zero buffer the size of one sparse chunk, for all-zero (hole) detection. */ - private static readonly SPARSE_ZERO_CHUNK = Buffer.alloc(4 * 1024 * 1024) + private static readonly SNAPSHOT_EXCLUDED_TOP_DIRS = new Set(['locks']) /** * Remove every object under a storage-root-relative prefix — one recursive @@ -747,9 +979,6 @@ export class FileSystemStorage extends BaseStorage { */ public override async removeRawPrefix(prefix: string): Promise { await this.ensureInitialized() - // Registered-blob contract: a prefix-nuke must not take out a protected - // family member. Throws if the prefix intersects one. - await this.assertPrefixNotProtected(prefix) await fs.promises.rm(path.join(this.rootDir, prefix), { recursive: true, force: true }) } @@ -804,61 +1033,6 @@ export class FileSystemStorage extends BaseStorage { } } - /** - * Begin a transaction durability barrier: start recording every canonical - * object write and delete so {@link flushWriteBarrier} can fsync them before - * the generation counter advances. Resets unconditionally, discarding any - * tracking left by a transaction that aborted without flushing. - * - * @see GenerationStorage.beginWriteBarrier - */ - public beginWriteBarrier(): void { - this.writeBarrierPaths = new Set() - this.writeBarrierDeleteDirs = new Set() - } - - /** - * Flush the transaction durability barrier: fsync every canonical write since - * {@link beginWriteBarrier} (file contents AND the rename directory entries, - * via {@link syncRawObjects}), then fsync the parent directory of every - * canonical delete so the unlinks are durable too. Clears the tracking. After - * this resolves, the transaction's entire canonical footprint is on disk, so - * the generation counter/manifest can be advanced without risking a - * counter-ahead-of-state torn store on a hard kill. - * - * @see GenerationStorage.flushWriteBarrier - */ - public async flushWriteBarrier(): Promise { - const paths = this.writeBarrierPaths - const deleteDirs = this.writeBarrierDeleteDirs - this.writeBarrierPaths = null - this.writeBarrierDeleteDirs = null - - if (paths && paths.size > 0) { - // syncRawObjects fsyncs each file and its parent directory. - await this.syncRawObjects([...paths]) - } - - if (deleteDirs && deleteDirs.size > 0) { - for (const relDir of deleteDirs) { - const dirPath = path.join(this.rootDir, relDir) - let handle: any - try { - handle = await fs.promises.open(dirPath, 'r') - } catch { - continue // directory vanished or platform disallows opening dirs - } - try { - await handle.sync() - } catch { - // Some platforms/filesystems reject directory fsync — best effort. - } finally { - await handle.close() - } - } - } - } - /** * Append one line to `_system/tx-log.jsonl`. Plain `appendFile` — the * tx-log is the one append-in-place file in the store (and is byte-copied, @@ -889,75 +1063,6 @@ export class FileSystemStorage extends BaseStorage { } } - // ========================================================================== - // Binary raw-byte primitives — the substrate for append-only log-structured - // files (the generation fact log's CRC-framed segments). Paths are used - // VERBATIM (no .gz/.bin suffixing). Append durability rides syncRawObjects - // at the commit barrier, like every other staged write. - // ========================================================================== - - /** - * Append bytes to a raw binary file, creating it (and parent directories) - * when absent. NOT fsync'd here — the caller batches durability via - * `syncRawObjects` at its commit barrier. - */ - public async appendRawBytes(rawPath: string, bytes: Uint8Array): Promise { - await this.ensureInitialized() - const fullPath = path.join(this.rootDir, rawPath) - await fs.promises.mkdir(path.dirname(fullPath), { recursive: true }) - await fs.promises.appendFile(fullPath, bytes) - } - - /** - * Read a raw binary file whole. Absent → `null`; a real IO fault throws — - * a present-but-unreadable log segment must never read as "no facts". - */ - public async readRawBytes(rawPath: string): Promise { - await this.ensureInitialized() - try { - const buf: Buffer = await fs.promises.readFile(path.join(this.rootDir, rawPath)) - return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength) - } catch (error: any) { - if (isAbsentError(error)) return null - throw error - } - } - - /** - * Replace a raw binary file atomically: write-new → fsync → rename. The - * reconcile primitive (e.g. truncating a fact-log tail back to committed - * truth after a crash) — a crash mid-replace leaves either the old file or - * the new one, never a mix. - */ - public async writeRawBytes(rawPath: string, bytes: Uint8Array): Promise { - await this.ensureInitialized() - const fullPath = path.join(this.rootDir, rawPath) - await fs.promises.mkdir(path.dirname(fullPath), { recursive: true }) - const tmpPath = `${fullPath}.tmp.${Date.now()}.${Math.random().toString(36).slice(2)}` - const handle = await fs.promises.open(tmpPath, 'w') - try { - await handle.writeFile(bytes) - await handle.sync() - } finally { - await handle.close() - } - await fs.promises.rename(tmpPath, fullPath) - } - - /** - * Byte size of a raw binary file, or `null` when absent. - */ - public async rawByteSize(rawPath: string): Promise { - await this.ensureInitialized() - try { - const stat = await fs.promises.stat(path.join(this.rootDir, rawPath)) - return stat.size - } catch (error: any) { - if (isAbsentError(error)) return null - throw error - } - } - /** * Snapshot the entire store into `targetPath` as a hard-link farm * (Cassandra-style: instant, space-shared). Safe because every data file @@ -1007,8 +1112,7 @@ export class FileSystemStorage extends BaseStorage { // msync/truncate reach through into the snapshot. if ( FileSystemStorage.SNAPSHOT_BYTE_COPY_PATHS.has(normalized) || - FileSystemStorage.SNAPSHOT_BYTE_COPY_DIRS.has(normalized.split('/')[0]) || - FileSystemStorage.SNAPSHOT_BYTE_COPY_PREFIXES.some((p) => normalized.startsWith(p)) + FileSystemStorage.SNAPSHOT_BYTE_COPY_DIRS.has(normalized.split('/')[0]) ) { await fs.promises.copyFile(sourceFile, targetFile) continue @@ -1120,16 +1224,6 @@ export class FileSystemStorage extends BaseStorage { * * @param sourcePath - Absolute path of a directory produced by * {@link FileSystemStorage.snapshotToDirectory}. - * - * Non-destructive: the snapshot is copied into a staging area (sparse-aware, - * so a store of mostly-hole mmap blobs cannot balloon and ENOSPC) BEFORE any - * live data is touched. Only once the full copy has succeeded and a completion - * marker is fsync'd does an atomic per-entry swap move it into place. A copy - * failure (including ENOSPC) leaves the live store exactly as it was; a crash - * mid-swap is resumed forward on the next {@link init} by - * {@link completeInterruptedRestore}. The previous implementation removed the - * live store first and then `fs.cp`'d — a copy failure destroyed the brain it - * was meant to recover. */ public async restoreFromDirectory(sourcePath: string): Promise { await this.ensureInitialized() @@ -1139,194 +1233,23 @@ export class FileSystemStorage extends BaseStorage { throw new Error(`restoreFromDirectory: ${sourcePath} is not a directory`) } - const staging = path.join(this.rootDir, FileSystemStorage.RESTORE_STAGING_DIR) - - // Discard any staging left by a prior aborted restore, then stage fresh. - await fs.promises.rm(staging, { recursive: true, force: true }) - await fs.promises.mkdir(staging, { recursive: true }) - - // Copy the snapshot into staging (sparse-aware). ANY failure here — most - // importantly ENOSPC — leaves the live store untouched: we remove only the - // half-written staging area and re-throw. - const staged: string[] = [] - try { - const sourceEntries = await fs.promises.readdir(sourcePath) - for (const entry of sourceEntries) { - if (FileSystemStorage.SNAPSHOT_EXCLUDED_TOP_DIRS.has(entry)) continue - await this.copyTreeSparse( - path.join(sourcePath, entry), - path.join(staging, entry) - ) - staged.push(entry) - } - // Commit point: fsync a marker naming the fully-staged entries. Only after - // this does the swap (and its resume) become authorized. - const markerPath = path.join(staging, FileSystemStorage.RESTORE_MARKER) - await fs.promises.writeFile(markerPath, JSON.stringify({ entries: staged })) - const mfh = await fs.promises.open(markerPath, 'r') - try { - await mfh.sync() - } finally { - await mfh.close() - } - const dfh = await fs.promises.open(staging, 'r').catch(() => null) - if (dfh) { - try { - await dfh.sync() - } catch { - // platform may reject directory fsync — best effort - } finally { - await dfh.close() - } - } - } catch (error: any) { - await fs.promises.rm(staging, { recursive: true, force: true }).catch(() => {}) - throw new Error( - `restoreFromDirectory: staging copy failed, live store left untouched: ${error.message}` - ) - } - - // Atomic per-entry swap (metadata-only renames within rootDir — cannot ENOSPC). - await this.swapStagedRestoreIn() - await this.reloadDerivedState() - } - - /** - * Move a fully-staged, marker-committed restore into place, then clear the - * staging area. Idempotent and resumable: driven by the marker's entry list - * and by which staged entries remain, so a crash at any point is completed by - * simply calling it again (from {@link completeInterruptedRestore} on the next - * open). Every step is a same-filesystem rename or a remove — no operation can - * fail for disk space, so once the marker exists the store is guaranteed to - * reach the restored state. - */ - private async swapStagedRestoreIn(): Promise { - const staging = path.join(this.rootDir, FileSystemStorage.RESTORE_STAGING_DIR) - const markerPath = path.join(staging, FileSystemStorage.RESTORE_MARKER) - - let staged: string[] - try { - const marker = JSON.parse(await fs.promises.readFile(markerPath, 'utf-8')) - staged = Array.isArray(marker?.entries) ? marker.entries : [] - } catch { - return // no committed marker — nothing to swap - } - const stagedSet = new Set(staged) - - // 1. Remove stale live entries the snapshot does not contain (excluding - // process-local dirs and the staging area itself). An already-placed - // staged entry is in stagedSet, so it is kept. - for (const entry of await fs.promises.readdir(this.rootDir)) { + // Clear current contents, preserving live lock state. + const currentEntries = await fs.promises.readdir(this.rootDir) + for (const entry of currentEntries) { if (FileSystemStorage.SNAPSHOT_EXCLUDED_TOP_DIRS.has(entry)) continue - if (entry === FileSystemStorage.RESTORE_STAGING_DIR) continue - if (stagedSet.has(entry)) continue await fs.promises.rm(path.join(this.rootDir, entry), { recursive: true, force: true }) } - // 2. Place each staged entry (idempotent: a prior attempt that already moved - // it leaves staging/entry absent, so we skip). `rm` the old first — a - // rename onto an existing non-empty directory is not allowed; the staged - // copy is the durable source until it is placed, so a crash between the - // rm and the rename is recovered forward on the next call. - for (const entry of staged) { - const from = path.join(staging, entry) - const to = path.join(this.rootDir, entry) - const exists = await fs.promises.lstat(from).then(() => true, () => false) - if (!exists) continue - await fs.promises.rm(to, { recursive: true, force: true }) - await fs.promises.rename(from, to) + // Byte-copy the snapshot in (defensively skipping lock dirs in old snapshots). + const sourceEntries = await fs.promises.readdir(sourcePath) + for (const entry of sourceEntries) { + if (FileSystemStorage.SNAPSHOT_EXCLUDED_TOP_DIRS.has(entry)) continue + await fs.promises.cp(path.join(sourcePath, entry), path.join(this.rootDir, entry), { + recursive: true + }) } - // 3. Clear the staging area (marker last-standing entry). - await fs.promises.rm(staging, { recursive: true, force: true }) - } - - /** - * On open, finish any restore interrupted by a crash. A staging dir WITH the - * completion marker means the copy had succeeded — resume the swap forward - * (loudly). A staging dir WITHOUT the marker is an interrupted copy — pure - * debris; the live store is authoritative, so discard it. Called from - * {@link init} before counts/derived state load, so recovery is invisible to - * the rest of startup. Returns `true` if a swap was resumed. - */ - private async completeInterruptedRestore(): Promise { - const staging = path.join(this.rootDir, FileSystemStorage.RESTORE_STAGING_DIR) - const stagingStat = await fs.promises.stat(staging).catch(() => null) - if (!stagingStat || !stagingStat.isDirectory()) return false - - const markerPath = path.join(staging, FileSystemStorage.RESTORE_MARKER) - const hasMarker = await fs.promises - .stat(markerPath) - .then(() => true, () => false) - - if (!hasMarker) { - // Interrupted before the copy committed — live data untouched, discard. - await fs.promises.rm(staging, { recursive: true, force: true }).catch(() => {}) - return false - } - - console.log('♻️ Resuming an interrupted restore (completing the staged swap)') - await this.swapStagedRestoreIn() - return true - } - - /** - * Recursively copy `src` to `dest`, preserving holes (sparse regions). Files - * are copied chunk-by-chunk skipping all-zero chunks, so a store of - * mostly-hole mmap blobs restores at its true allocated size instead of - * materializing every hole (the failure that made `fs.cp` ENOSPC a restore - * that would otherwise fit). Directories recurse; symlinks are recreated. - */ - private async copyTreeSparse(src: string, dest: string): Promise { - const stat = await fs.promises.lstat(src) - if (stat.isDirectory()) { - await fs.promises.mkdir(dest, { recursive: true }) - for (const child of await fs.promises.readdir(src)) { - await this.copyTreeSparse(path.join(src, child), path.join(dest, child)) - } - } else if (stat.isSymbolicLink()) { - await fs.promises.symlink(await fs.promises.readlink(src), dest) - } else if (stat.isFile()) { - await this.copyFileSparse(src, dest, stat.size, stat.mode) - } - // Other node types (sockets, devices) do not occur in a brain store. - } - - /** Sparse-aware single-file copy — see {@link copyTreeSparse}. */ - private async copyFileSparse( - src: string, - dest: string, - size: number, - mode: number - ): Promise { - const CHUNK = FileSystemStorage.SPARSE_CHUNK_BYTES - const srcFh = await fs.promises.open(src, 'r') - try { - const destFh = await fs.promises.open(dest, 'w', mode) - try { - // Pre-size the destination so unwritten regions are holes. - await destFh.truncate(size) - const buf = Buffer.allocUnsafe(CHUNK) - let pos = 0 - while (pos < size) { - const { bytesRead } = await srcFh.read(buf, 0, CHUNK, pos) - if (bytesRead === 0) break - const chunk = buf.subarray(0, bytesRead) - // Skip all-zero chunks: leaving them unwritten preserves the hole. - const isHole = chunk.equals( - FileSystemStorage.SPARSE_ZERO_CHUNK.subarray(0, bytesRead) - ) - if (!isHole) { - await destFh.write(chunk, 0, bytesRead, pos) - } - pos += bytesRead - } - } finally { - await destFh.close() - } - } finally { - await srcFh.close() - } + await this.reloadDerivedState() } // =========================================================================== @@ -1368,44 +1291,27 @@ export class FileSystemStorage extends BaseStorage { await this.ensureInitialized() const filePath = this.blobPath(key) await this.ensureDirectoryExists(path.dirname(filePath)) - - // Atomic write via a UNIQUE per-writer temp suffix — matches the pattern - // used at every other atomic-write site in this file (lines 336, 551, 744, - // 781, 1529, 2908). The unique suffix (pid+time+random) means NO other - // writer ever touches this temp, so two concurrent saveBinaryBlob() calls - // for the same key can never collide on the temp path (the shared-`.tmp` - // race that once threw ENOENT — column-store compaction vs an explicit - // flush() on `_column_index//DELETED.bin` — is gone). - // - // Crucially, BECAUSE the temp is unique, a rename ENOENT can no longer mean - // "a concurrent idempotent writer already renamed it": nobody else has this - // temp. It means OUR just-written temp vanished before the rename, so the - // bytes did NOT land — returning success would acknowledge a write that - // stored nothing (the native provider mmaps these blobs; a phantom-acked - // blob is exactly the silent-loss class). The only way that happens with a - // unique temp is an external sweeper / crash-cleanup removing it mid-write, - // so retry ONCE with a fresh temp; if it vanishes again, FAIL LOUD. - const writeOnce = async (): Promise<'ok' | 'temp-vanished'> => { - const tmpPath = `${filePath}.tmp.${process.pid}.${Date.now()}.${Math.random().toString(36).slice(2)}` - await fs.promises.writeFile(tmpPath, data) - try { - await fs.promises.rename(tmpPath, filePath) - return 'ok' - } catch (err) { - // Clean up our own temp (best-effort) so no failure path orphans it. - await fs.promises.unlink(tmpPath).catch(() => {}) - if ((err as NodeJS.ErrnoException).code === 'ENOENT') return 'temp-vanished' - throw err - } - } - - if ((await writeOnce()) === 'temp-vanished' && (await writeOnce()) === 'temp-vanished') { - throw new Error( - `saveBinaryBlob('${key}'): the temp file was removed before rename on two ` + - `successive attempts — the blob did NOT persist. Some external process is ` + - `deleting files under ${this.blobsDir} mid-write. Failing loud rather than ` + - `acknowledging a durable write that stored nothing.` - ) + // Unique per-writer temp suffix — matches the pattern used at every other + // atomic-write site in this file (lines 336, 551, 744, 781, 1529, 2908). + // Without a unique suffix, two concurrent saveBinaryBlob() calls for the + // same key collide on `${filePath}.tmp`: both writeFile, the first rename + // succeeds, the second fires against a missing temp and throws ENOENT. + // Reproduced in production: column-store compaction running alongside an + // explicit flush() repeatedly raced on `_column_index//DELETED.bin`. + const tmpPath = `${filePath}.tmp.${process.pid}.${Date.now()}.${Math.random().toString(36).slice(2)}` + await fs.promises.writeFile(tmpPath, data) + try { + await fs.promises.rename(tmpPath, filePath) + } catch (err) { + const code = (err as NodeJS.ErrnoException).code + // The writes are idempotent for a given key — every caller persists the + // same logical bytes for that key — so ENOENT on rename means the temp + // is already gone (rare with the unique suffix, but defensive against + // crash-resume cleanup paths and external file-system sweepers). + if (code === 'ENOENT') return + // Any other failure: clean up our own temp to avoid orphans before rethrow. + await fs.promises.unlink(tmpPath).catch(() => {}) + throw err } } @@ -1419,17 +1325,8 @@ export class FileSystemStorage extends BaseStorage { await this.ensureInitialized() try { return await fs.promises.readFile(this.blobPath(key)) - } catch (err) { - // Absent blob → null (the documented contract). A real fault - // (EIO/EACCES/EMFILE/…) must NOT be masked as "absent": doing so makes a - // present-but-unreadable blob look missing and drives a needless rebuild - // or an empty read (the native provider consumes this). Mandate: loud - // errors, never quiet losses. Fault-propagation restored lockstep with - // cortex 3.0.13, whose two column-store call sites now handle the throw - // (they mark the field unavailable + throw a named error) instead of - // relying on null-on-error. - if (isAbsentError(err)) return null - throw err + } catch { + return null } } @@ -1440,11 +1337,6 @@ export class FileSystemStorage extends BaseStorage { */ public async deleteBinaryBlob(key: string): Promise { await this.ensureInitialized() - // Registered-blob contract: refuse to delete a declared - // derived-index family member — an in-process GC/sweeper cannot remove a - // load-bearing index file. Throws ProtectedArtifactError; no-op when no - // families are registered. - await this.assertBlobKeyDeletable(key) try { await fs.promises.unlink(this.blobPath(key)) } catch { @@ -1564,24 +1456,6 @@ export class FileSystemStorage extends BaseStorage { await fs.promises.rm(casDir, { recursive: true, force: true }) } - // Remove the raw-blob + native-shared + column-index footprint. These - // top-level trees were NOT wiped before, so a cleared brain re-read stale - // native blobs (HNSW/LSM segments, the native dkann index), a stale native - // id-mapper, and — worst — orphaned column manifests. `_column_index` holds - // the column-store MANIFEST.json files while their segment bytes live under - // `_blobs/_column_index/...`; removing one without the other would strand a - // manifest listing segments that no longer exist, which the load path now - // (correctly) refuses with ColumnSegmentLoadError. They must fall together, - // as a set, exactly as `_cas` does — the complete derived footprint, not a - // subset. (`locks/` is deliberately left: it is live coordination state, - // not data.) - for (const nativeDir of ['_blobs', '_id_mapper', '_column_index']) { - const dir = path.join(this.rootDir, nativeDir) - if (await this.directoryExists(dir)) { - await fs.promises.rm(dir, { recursive: true, force: true }) - } - } - // Reset ALL shared derived state via the same path restore uses: the // write-through cache, id→type/subtype caches, per-type/subtype count // rollups, statistics cache, graph-index singleton, and the BlobStorage @@ -1772,158 +1646,79 @@ export class FileSystemStorage extends BaseStorage { const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE) const os = await import('node:os') const hostname = os.hostname() - const myPid = typeof process !== 'undefined' && process.pid ? process.pid : 0 + const now = new Date().toISOString() - // Bounded acquire loop. The CLAIM itself is an atomic create-exclusive - // write (O_EXCL) — two processes racing an ABSENT lock can never both - // succeed, which closes the read-then-write window where the loser used - // to keep running unlocked, silently. An EEXIST loser loops, re-reads, - // and handles whatever it finds honestly (fresh foreign lock → loud - // throw; stale/forced → verified takeover). - const MAX_ATTEMPTS = 3 - for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) { - const now = new Date().toISOString() - const existing = await this.readWriterLock() - - if (existing) { - // Same-process re-open: a second Brainy instance in this Node process - // (e.g. test "simulate server restart" patterns, or a consumer that - // explicitly re-instantiates without closing first). This isn't the - // dangerous cross-process case the lock exists to prevent — the two - // instances share a memory space and can't silently diverge from each - // other beyond what their callers already see. Warn and take over. - if (existing.pid === myPid && existing.hostname === hostname && !options?.force) { - console.warn( - `[brainy] Re-acquiring writer lock for ${this.rootDir} held by the same process (PID ${existing.pid}). ` + - `If you intended to keep the previous Brainy instance alive, this is a bug — close it first.` - ) - const info: WriterLockInfo = { - pid: myPid, - hostname, - startedAt: now, - lastHeartbeat: now, - version: getBrainyVersion(), - rootDir: this.rootDir - } - await this.writeFileAtomic(lockFile, JSON.stringify(info, null, 2)) - this.installWriterLock(info) - return info - } - - const stale = !options?.force && (await this.isWriterLockStale(existing)) - if (!options?.force && !stale) { + const existing = await this.readWriterLock() + if (existing && !options?.force) { + // Same-process re-open: a second Brainy instance in this Node process + // (e.g. test "simulate server restart" patterns, or a consumer that + // explicitly re-instantiates without closing first). This isn't the + // dangerous cross-process case the lock exists to prevent — the two + // instances share a memory space and can't silently diverge from each + // other beyond what their callers already see. Warn and take over the lock. + if (existing.pid === (typeof process !== 'undefined' ? process.pid : 0) && + existing.hostname === hostname) { + console.warn( + `[brainy] Re-acquiring writer lock for ${this.rootDir} held by the same process (PID ${existing.pid}). ` + + `If you intended to keep the previous Brainy instance alive, this is a bug — close it first.` + ) + } else { + const stale = await this.isWriterLockStale(existing) + if (!stale) { // Consumer-facing error contract: callers detect this case via // err.code and read the holder's details from err.lockInfo. - throw this.writerLockedError(existing) + const err = new Error( + `Another writer holds this Brainy directory.\n` + + ` PID: ${existing.pid} on host ${existing.hostname}\n` + + ` Started: ${existing.startedAt}\n` + + ` Heartbeat: ${existing.lastHeartbeat}\n` + + ` Version: ${existing.version}\n` + + ` Directory: ${this.rootDir}\n\n` + + `For diagnostic queries against this live store, use:\n` + + ` const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: '${this.rootDir}' } })\n\n` + + `If you have verified the existing lock is stale (e.g. a crashed writer on a different host that PID liveness cannot reach), pass { force: true }.` + ) as Error & { code: string; lockInfo: WriterLockInfo } + err.code = 'BRAINY_WRITER_LOCKED' + err.lockInfo = existing + throw err } - console.warn( - options?.force - ? `[brainy] Force-overwriting writer lock for ${this.rootDir} ` + - `(was held by PID ${existing.pid} on ${existing.hostname}).` - : `[brainy] Overwriting stale writer lock for ${this.rootDir} ` + - `(PID ${existing.pid} on ${existing.hostname} appears dead).` + `[brainy] Overwriting stale writer lock for ${this.rootDir} ` + + `(PID ${existing.pid} on ${existing.hostname} appears dead).` ) - // Takeover: verify the file still holds the lock we judged (a live - // successor may have claimed meanwhile), then remove it and fall - // through to the atomic claim below. A racing claimer who beats us to - // the create simply wins — our next loop iteration reads their fresh - // lock and throws honestly. (Advisory file locking has no - // compare-and-delete; staleness requiring a 60s-old heartbeat keeps - // the residual verify-to-unlink window practically unreachable.) - const recheck = await this.readWriterLock() - if ( - recheck && - (recheck.pid !== existing.pid || - recheck.startedAt !== existing.startedAt || - recheck.lastHeartbeat !== existing.lastHeartbeat) - ) { - continue // the lock changed hands while we deliberated — re-evaluate - } - try { - await fs.promises.unlink(lockFile) - } catch (err: any) { - if (err.code !== 'ENOENT') throw err - } } - - const info: WriterLockInfo = { - pid: myPid, - hostname, - startedAt: existing && options?.force ? existing.startedAt : now, - lastHeartbeat: now, - version: getBrainyVersion(), - rootDir: this.rootDir - } - - // The atomic claim: create-exclusive, so exactly ONE racer wins. - try { - await fs.promises.writeFile(lockFile, JSON.stringify(info, null, 2), { flag: 'wx' }) - } catch (err: any) { - if (err.code === 'EEXIST') { - continue // someone else claimed between our read and create — re-evaluate - } - throw err - } - - this.installWriterLock(info) - return info + } else if (existing && options?.force) { + console.warn( + `[brainy] Force-overwriting writer lock for ${this.rootDir} ` + + `(was held by PID ${existing.pid} on ${existing.hostname}).` + ) } - // Attempts exhausted: something is claiming this directory faster than we - // can evaluate it. Read whoever holds it now and fail loudly with their - // details rather than degrading into a lockless open. - const holder = await this.readWriterLock() - if (holder) throw this.writerLockedError(holder) - throw new Error( - `Failed to acquire the writer lock for ${this.rootDir} after ${MAX_ATTEMPTS} attempts — ` + - `the lock file is being contended. Retry, or inspect ${lockFile}.` - ) - } + const info: WriterLockInfo = { + pid: typeof process !== 'undefined' && process.pid ? process.pid : 0, + hostname, + startedAt: existing && options?.force ? existing.startedAt : now, + lastHeartbeat: now, + version: getBrainyVersion(), + rootDir: this.rootDir + } - /** Record lock ownership + start the unref'd heartbeat. */ - private installWriterLock(info: WriterLockInfo): void { + await this.writeFileAtomic(lockFile, JSON.stringify(info, null, 2)) this.writerLockInfo = info // Heartbeat — rewrite lastHeartbeat every WRITER_HEARTBEAT_MS so other // processes can tell a live writer from one that crashed without releasing. this.writerLockHeartbeat = setInterval(() => { - const tick = this.refreshWriterLockHeartbeat().catch((err) => { - // ENOENT = the lock (or its directory) vanished mid-refresh — the - // store was released or removed under us; the next acquire recreates - // it. Benign by construction; anything else stays loud. - if ((err as NodeJS.ErrnoException)?.code !== 'ENOENT') { - console.warn('[brainy] Failed to refresh writer lock heartbeat:', err) - } - }) - this.writerHeartbeatInFlight = tick.finally(() => { - if (this.writerHeartbeatInFlight === tick) { - this.writerHeartbeatInFlight = undefined - } + this.refreshWriterLockHeartbeat().catch((err) => { + console.warn('[brainy] Failed to refresh writer lock heartbeat:', err) }) }, FileSystemStorage.WRITER_HEARTBEAT_MS) if (typeof this.writerLockHeartbeat.unref === 'function') { // Don't keep the event loop alive just for the heartbeat. this.writerLockHeartbeat.unref() } - } - /** The consumer-facing BRAINY_WRITER_LOCKED error, holder details attached. */ - private writerLockedError(existing: WriterLockInfo): Error { - const err = new Error( - `Another writer holds this Brainy directory.\n` + - ` PID: ${existing.pid} on host ${existing.hostname}\n` + - ` Started: ${existing.startedAt}\n` + - ` Heartbeat: ${existing.lastHeartbeat}\n` + - ` Version: ${existing.version}\n` + - ` Directory: ${this.rootDir}\n\n` + - `For diagnostic queries against this live store, use:\n` + - ` const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: '${this.rootDir}' } })\n\n` + - `If you have verified the existing lock is stale (e.g. a crashed writer on a different host that PID liveness cannot reach), pass { force: true }.` - ) as Error & { code: string; lockInfo: WriterLockInfo } - err.code = 'BRAINY_WRITER_LOCKED' - err.lockInfo = existing - return err + return info } public override async releaseWriterLock(): Promise { @@ -1931,15 +1726,6 @@ export class FileSystemStorage extends BaseStorage { clearInterval(this.writerLockHeartbeat) this.writerLockHeartbeat = undefined } - // Drain an in-flight heartbeat tick BEFORE unlinking: clearInterval stops - // future ticks only, and a straggler write landing after the unlink would - // re-create the lock as a phantom (blocking the next writer until the - // stale TTL). After the drain, any refresh is either fully landed (we - // unlink its output below) or not started (it sees writerLockInfo - // undefined and returns). - if (this.writerHeartbeatInFlight) { - await this.writerHeartbeatInFlight - } if (!this.writerLockInfo) { return } @@ -2459,33 +2245,39 @@ export class FileSystemStorage extends BaseStorage { */ private async initializeCountsFromDisk(): Promise { try { - // Count the CANONICAL 8.0 layout (`entities////…`) — - // the tree saveNoun/getNouns actually read and write. The previous scan - // counted the vestigial `entities/*/hnsw` directories, which the 8.0 - // write path never populates, so a store recovering from a lost or - // corrupted counts.json re-initialized every counter to ZERO on real - // data (wrong stats/boot logs and a mis-sized rebuild-strategy - // decision at open). - const nouns = await this.scanCanonicalEntities('nouns') - this.totalNounCount = nouns.count - const verbs = await this.scanCanonicalEntities('verbs') - this.totalVerbCount = verbs.count + // CRITICAL: Detect existing depth before counting + // Can't use getAllShardedFiles() which assumes depth=1 + const existingDepth = await this.detectExistingShardingDepth() + const depthToUse = existingDepth !== null ? existingDepth : this.SHARDING_DEPTH - // Sample some entities for the type distribution (don't read all). - // Read the metadata files DIRECTLY with fs — this runs inside init(), - // and every guarded accessor (getNounMetadata → ensureInitialized) - // re-enters init() from here, deadlocking the open. (Latent in the old - // code too: its scan of the empty hnsw dirs just never sampled.) - for (const entityDir of nouns.sampleDirs) { - const metadata = await this.readEntityMetadataRaw(entityDir) - if (metadata) { - const type = metadata.noun || 'default' - this.entityCounts.set(type, (this.entityCounts.get(type) || 0) + 1) + // Count nouns using detected depth + const validNounFiles = await this.getAllFilesAtDepth(this.nounsDir, depthToUse) + this.totalNounCount = validNounFiles.length + + // Count verbs using detected depth + const validVerbFiles = await this.getAllFilesAtDepth(this.verbsDir, depthToUse) + this.totalVerbCount = validVerbFiles.length + + // Sample some files to get type distribution (don't read all) + // Load metadata separately for type information + const sampleSize = Math.min(100, validNounFiles.length) + for (let i = 0; i < sampleSize; i++) { + try { + const file = validNounFiles[i] + const id = file.replace('.json', '') + + // Load metadata from separate storage for type info + const metadata = await this.getNounMetadata(id) + if (metadata) { + const type = metadata.noun || 'default' + this.entityCounts.set(type, (this.entityCounts.get(type) || 0) + 1) + } + } catch { + // Skip invalid files or missing metadata } } // Extrapolate counts if we sampled - const sampleSize = nouns.sampleDirs.length if (sampleSize < this.totalNounCount && sampleSize > 0) { const multiplier = this.totalNounCount / sampleSize for (const [type, count] of this.entityCounts.entries()) { @@ -2499,62 +2291,6 @@ export class FileSystemStorage extends BaseStorage { } } - /** - * Walk the canonical `entities//<2-hex-shard>//` tree, counting - * one entity per id directory (the layout `getNounVectorPath`/`getNouns` - * use). Returns up to 100 sampled entity directories (absolute paths) — - * nouns feed the type-distribution estimate above. An absent tree (fresh - * store) counts zero. - */ - private async scanCanonicalEntities( - kind: 'nouns' | 'verbs' - ): Promise<{ count: number; sampleDirs: string[] }> { - const base = path.join(this.rootDir, 'entities', kind) - const SAMPLE_MAX = 100 - let count = 0 - const sampleDirs: string[] = [] - try { - const shards = await fs.promises.readdir(base, { withFileTypes: true }) - for (const shard of shards) { - if (!shard.isDirectory() || !/^[0-9a-f]{2}$/i.test(shard.name)) continue - const shardPath = path.join(base, shard.name) - const ids = await fs.promises.readdir(shardPath, { withFileTypes: true }) - for (const entry of ids) { - if (!entry.isDirectory()) continue - count++ - if (sampleDirs.length < SAMPLE_MAX) { - sampleDirs.push(path.join(shardPath, entry.name)) - } - } - } - } catch (error: any) { - if (error?.code !== 'ENOENT') throw error - } - return { count, sampleDirs } - } - - /** - * Read one canonical entity's `metadata.json` (or `.json.gz`) directly with - * fs — NO guarded accessors. Used only by the init-time count recovery, - * where `getNounMetadata`'s `ensureInitialized()` would re-enter `init()`. - * @param entityDir - Absolute `entities///` directory. - * @returns The parsed metadata, or null when absent/unreadable. - */ - private async readEntityMetadataRaw(entityDir: string): Promise { - const base = path.join(entityDir, 'metadata.json') - try { - return JSON.parse(await fs.promises.readFile(base, 'utf-8')) - } catch { - // fall through to the compressed variant - } - try { - const gz = await fs.promises.readFile(`${base}.gz`) - return JSON.parse(zlib.gunzipSync(gz).toString('utf-8')) - } catch { - return null - } - } - /** * Persist counts to filesystem storage */ @@ -2585,9 +2321,307 @@ export class FileSystemStorage extends BaseStorage { // Intelligent Directory Sharding // ============================================= + /** + * Migrate files from one sharding depth to another + * Handles: 0→1 (flat to single-level), 2→1 (deep to single-level) + * Uses atomic file operations and comprehensive error handling + * + * @param fromDepth - Source sharding depth + * @param toDepth - Target sharding depth (must be 1) + */ + private async migrateShardingStructure(fromDepth: number, toDepth: number): Promise { + // Validation + if (fromDepth === toDepth) { + throw new Error(`Migration not needed: already at depth ${toDepth}`) + } + if (toDepth !== 1) { + throw new Error(`Migration only supports target depth 1 (got ${toDepth})`) + } + if (fromDepth !== 0 && fromDepth !== 2) { + throw new Error(`Migration only supports source depth 0 or 2 (got ${fromDepth})`) + } + // Create migration lock to prevent concurrent migrations + const lockFile = path.join(this.systemDir, '.migration-lock') + const lockExists = await this.fileExists(lockFile) + + if (lockExists) { + // Check if lock is stale (> 1 hour old) + try { + const stats = await fs.promises.stat(lockFile) + const lockAge = Date.now() - stats.mtimeMs + const ONE_HOUR = 60 * 60 * 1000 + + if (lockAge < ONE_HOUR) { + throw new Error( + 'Migration already in progress. If this is incorrect, delete .migration-lock file.' + ) + } + + // Lock is stale, remove it + console.log('⚠️ Removing stale migration lock (> 1 hour old)') + await fs.promises.unlink(lockFile) + } catch (error: any) { + if (error.code !== 'ENOENT') { + throw error + } + } + } + + try { + // Create lock file + await fs.promises.writeFile(lockFile, JSON.stringify({ + startedAt: new Date().toISOString(), + fromDepth, + toDepth, + pid: process.pid + })) + + // Discover all files to migrate + console.log('📊 Discovering files to migrate...') + const filesToMigrate = await this.discoverFilesForMigration(fromDepth) + + if (filesToMigrate.length === 0) { + console.log('ℹ️ No files to migrate') + return + } + + console.log(`📦 Migrating ${filesToMigrate.length} files...`) + + // Create all target shard directories upfront + await this.createAllShardDirectories(this.nounsDir) + await this.createAllShardDirectories(this.verbsDir) + + // Migrate files with progress tracking + let migratedCount = 0 + let skippedCount = 0 + const errors: Array<{ file: string; error: string }> = [] + + for (const fileInfo of filesToMigrate) { + try { + await this.migrateFile(fileInfo, fromDepth, toDepth) + migratedCount++ + + // Progress update every 1000 files + if (migratedCount % 1000 === 0) { + const percent = ((migratedCount / filesToMigrate.length) * 100).toFixed(1) + console.log(` 📊 Progress: ${migratedCount}/${filesToMigrate.length} (${percent}%)`) + } + + // Yield to event loop every 100 files to prevent blocking + if (migratedCount % 100 === 0) { + await new Promise(resolve => setImmediate(resolve)) + } + } catch (error: any) { + skippedCount++ + errors.push({ + file: fileInfo.oldPath, + error: error.message + }) + + // Log first few errors + if (errors.length <= 5) { + console.warn(`⚠️ Skipped ${fileInfo.oldPath}: ${error.message}`) + } + } + } + + // Final summary + console.log(`\n✅ Migration Results:`) + console.log(` Migrated: ${migratedCount} files`) + console.log(` Skipped: ${skippedCount} files`) + + if (errors.length > 0) { + console.warn(`\n⚠️ ${errors.length} files could not be migrated`) + if (errors.length > 5) { + console.warn(` (First 5 errors shown above, ${errors.length - 5} more occurred)`) + } + } + + // Cleanup: Remove empty old directories + if (fromDepth === 0) { + // No subdirectories to clean for flat structure + } else if (fromDepth === 2) { + await this.cleanupEmptyDirectories(this.nounsDir, fromDepth) + await this.cleanupEmptyDirectories(this.verbsDir, fromDepth) + } + + // Verification: Count files in new structure + const verifyCount = await this.countFilesInStructure(toDepth) + console.log(`\n🔍 Verification: ${verifyCount} files in new structure`) + + if (verifyCount < migratedCount) { + console.warn(`⚠️ Warning: Verification count (${verifyCount}) < migrated count (${migratedCount})`) + } + } finally { + // Always remove lock file + try { + await fs.promises.unlink(lockFile) + } catch (error) { + // Ignore error if lock file doesn't exist + } + } + } + + /** + * Discover all files that need to be migrated + * Constructs correct oldPath based on source depth + */ + private async discoverFilesForMigration(fromDepth: number): Promise> { + const files: Array<{ oldPath: string; id: string; type: 'noun' | 'verb' }> = [] + + // Discover noun files + const nounFiles = await this.getAllFilesAtDepth(this.nounsDir, fromDepth) + for (const filename of nounFiles) { + const id = filename.replace('.json', '') + + // Construct correct oldPath based on fromDepth + let oldPath: string + switch (fromDepth) { + case 0: + // Flat: nouns/uuid.json + oldPath = path.join(this.nounsDir, `${id}.json`) + break + case 1: + // Single-level: nouns/ab/uuid.json + oldPath = path.join(this.nounsDir, id.substring(0, 2), `${id}.json`) + break + case 2: + // Deep: nouns/ab/cd/uuid.json + oldPath = path.join(this.nounsDir, id.substring(0, 2), id.substring(2, 4), `${id}.json`) + break + default: + throw new Error(`Unsupported fromDepth: ${fromDepth}`) + } + + files.push({ oldPath, id, type: 'noun' }) + } + + // Discover verb files + const verbFiles = await this.getAllFilesAtDepth(this.verbsDir, fromDepth) + for (const filename of verbFiles) { + const id = filename.replace('.json', '') + + // Construct correct oldPath based on fromDepth + let oldPath: string + switch (fromDepth) { + case 0: + // Flat: verbs/uuid.json + oldPath = path.join(this.verbsDir, `${id}.json`) + break + case 1: + // Single-level: verbs/ab/uuid.json + oldPath = path.join(this.verbsDir, id.substring(0, 2), `${id}.json`) + break + case 2: + // Deep: verbs/ab/cd/uuid.json + oldPath = path.join(this.verbsDir, id.substring(0, 2), id.substring(2, 4), `${id}.json`) + break + default: + throw new Error(`Unsupported fromDepth: ${fromDepth}`) + } + + files.push({ oldPath, id, type: 'verb' }) + } + + return files + } + + /** + * Get all files at a specific depth + */ + private async getAllFilesAtDepth(baseDir: string, depth: number): Promise { + const allFiles: string[] = [] + + try { + const dirExists = await this.directoryExists(baseDir) + if (!dirExists) { + return [] + } + + switch (depth) { + case 0: + // Flat: files directly in baseDir + const entries = await fs.promises.readdir(baseDir) + for (const entry of entries) { + if (entry.endsWith('.json')) { + allFiles.push(entry) + } + } + break + + case 1: + // Single-level: baseDir/ab/uuid.json + const shardDirs = await fs.promises.readdir(baseDir) + for (const shard of shardDirs) { + const shardPath = path.join(baseDir, shard) + try { + const stat = await fs.promises.stat(shardPath) + if (stat.isDirectory()) { + const shardFiles = await fs.promises.readdir(shardPath) + for (const file of shardFiles) { + if (file.endsWith('.json')) { + allFiles.push(file) + } + } + } + } catch (error) { + // Skip inaccessible directories + } + } + break + + case 2: + // Deep: baseDir/ab/cd/uuid.json + const level1Dirs = await fs.promises.readdir(baseDir) + for (const level1 of level1Dirs) { + const level1Path = path.join(baseDir, level1) + try { + const level1Stat = await fs.promises.stat(level1Path) + if (level1Stat.isDirectory()) { + const level2Dirs = await fs.promises.readdir(level1Path) + for (const level2 of level2Dirs) { + const level2Path = path.join(level1Path, level2) + try { + const level2Stat = await fs.promises.stat(level2Path) + if (level2Stat.isDirectory()) { + const files = await fs.promises.readdir(level2Path) + for (const file of files) { + if (file.endsWith('.json')) { + allFiles.push(file) + } + } + } + } catch (error) { + // Skip inaccessible directories + } + } + } + } catch (error) { + // Skip inaccessible directories + } + } + break + } + } catch (error) { + // Directory doesn't exist or not accessible + } + + return allFiles + } + + /** + * Create all 256 shard directories (00-ff) + */ + private async createAllShardDirectories(baseDir: string): Promise { + for (let i = 0; i < this.MAX_SHARDS; i++) { + const shard = i.toString(16).padStart(2, '0') + const shardDir = path.join(baseDir, shard) + await this.ensureDirectoryExists(shardDir) + } + } /** * Migrate a single file atomically @@ -2616,20 +2650,116 @@ export class FileSystemStorage extends BaseStorage { await fs.promises.rename(oldPath, newPath) } + /** + * Clean up empty directories after migration + */ + private async cleanupEmptyDirectories(baseDir: string, depth: number): Promise { + try { + if (depth === 2) { + // Clean up level2 and level1 directories + const level1Dirs = await fs.promises.readdir(baseDir) + for (const level1 of level1Dirs) { + const level1Path = path.join(baseDir, level1) + try { + const level1Stat = await fs.promises.stat(level1Path) + if (level1Stat.isDirectory()) { + const level2Dirs = await fs.promises.readdir(level1Path) + for (const level2 of level2Dirs) { + const level2Path = path.join(level1Path, level2) + try { + // Try to remove level2 directory (will fail if not empty) + await fs.promises.rmdir(level2Path) + } catch (error) { + // Directory not empty or other error - ignore + } + } + // Try to remove level1 directory + await fs.promises.rmdir(level1Path) + } + } catch (error) { + // Directory not empty or other error - ignore + } + } + } + } catch (error) { + // Cleanup is best-effort, don't throw + } + } + /** + * Count files in the current structure + */ + private async countFilesInStructure(depth: number): Promise { + let count = 0 + + count += (await this.getAllFilesAtDepth(this.nounsDir, depth)).length + count += (await this.getAllFilesAtDepth(this.verbsDir, depth)).length + + return count + } + + /** + * Detect the actual sharding depth used by existing files + * Examines directory structure to determine current sharding strategy + * Returns null if no files exist yet (new installation) + */ + private async detectExistingShardingDepth(): Promise { + try { + // Check if nouns directory exists and has content + const dirExists = await this.directoryExists(this.nounsDir) + if (!dirExists) { + return null // New installation + } + + const entries = await fs.promises.readdir(this.nounsDir, { withFileTypes: true }) + + // Check if there are any .json files directly in nounsDir (flat structure) + const hasDirectJsonFiles = entries.some((e: any) => e.isFile() && e.name.endsWith('.json')) + if (hasDirectJsonFiles) { + return 0 // Flat structure: nouns/uuid.json + } + + // Check for subdirectories with hex names (sharding directories) + const subdirs = entries.filter((e: any) => e.isDirectory() && /^[0-9a-f]{2}$/i.test(e.name)) + if (subdirs.length === 0) { + return null // No files yet + } + + // Check first subdir to see if it has files or more subdirs + const firstSubdir = subdirs[0].name + const subdirPath = path.join(this.nounsDir, firstSubdir) + const subdirEntries = await fs.promises.readdir(subdirPath, { withFileTypes: true }) + + const hasJsonFiles = subdirEntries.some((e: any) => e.isFile() && e.name.endsWith('.json')) + if (hasJsonFiles) { + return 1 // Single-level sharding: nouns/ab/uuid.json + } + + const hasSubSubdirs = subdirEntries.some((e: any) => e.isDirectory() && /^[0-9a-f]{2}$/i.test(e.name)) + if (hasSubSubdirs) { + return 2 // Deep sharding: nouns/ab/cd/uuid.json + } + + return 1 // Default to single-level if structure is unclear + } catch (error) { + // If we can't read the directory, assume new installation + return null + } + } /** * Whether this store already holds canonical 8.0 entities. * * 8.0 writes nouns to `entities/nouns///vectors.json` (see - * `getNounVectorPath`). This checks that canonical shard tree — the one the - * DB actually reads and writes (the same `entities/nouns/` shards - * `getNounsWithPagination` walks) — so the new-vs-established boot log is - * truthful. (The 7.x hnsw sharding probe this replaced inspected a directory - * the 8.0 write path never populated, so it mislabeled every established - * store "New installation" on every boot; that probe and its depth-migration - * machinery are removed.) + * `getNounVectorPath`), but the legacy sharding probe + * ({@link detectExistingShardingDepth}) inspects `entities/nouns/hnsw` — a 7.x + * directory the 8.0 write path never populates. The probe therefore returns + * null for every 8.0 store and cannot tell an established brain from a fresh + * one, which mislabels established stores "New installation" on every boot. + * This checks the canonical shard tree the DB actually reads and writes (the + * same `entities/nouns/` shards `getNounsWithPagination` walks) so boot + * logs are truthful. * * @returns true if at least one 2-hex shard directory (00–ff) exists under * `entities/nouns/`, i.e. the store has previously persisted entities. @@ -2651,6 +2781,305 @@ export class FileSystemStorage extends BaseStorage { } } + /** + * Get sharding depth + * Always returns 1 (single-level sharding) for optimal balance of + * simplicity, performance, and reliability across all dataset sizes + * + * Single-level sharding (depth=1): + * - 256 shard directories (00-ff) + * - Handles 2.5M+ entities with excellent performance + * - No dynamic depth changes = no path mismatch bugs + * - Industry standard approach (Git uses similar) + */ + private getOptimalShardingDepth(): number { + return this.SHARDING_DEPTH + } + + /** + * Get the path for a node with consistent sharding strategy + * Clean, predictable path generation + */ + private getNodePath(id: string): string { + return this.getShardedPath(this.nounsDir, id) + } + + /** + * Get the path for a verb with consistent sharding strategy + */ + private getVerbPath(id: string): string { + return this.getShardedPath(this.verbsDir, id) + } + + /** + * Universal sharded path generator + * Always uses depth=1 (single-level sharding) for consistency + * + * Format: baseDir/ab/uuid.json + * Where 'ab' = first 2 hex characters of UUID (lowercase) + * + * Validates UUID format and throws descriptive errors + */ + private getShardedPath(baseDir: string, id: string): string { + // Extract first 2 characters for shard directory + const shard = id.substring(0, 2).toLowerCase() + + // Validate shard is valid hex (00-ff) + if (!/^[0-9a-f]{2}$/.test(shard)) { + throw new Error( + `Invalid entity ID format: ${id}. ` + + `Expected UUID starting with 2 hex characters, got '${shard}'. ` + + `IDs must be UUIDs or hex strings.` + ) + } + + // Single-level sharding: baseDir/ab/uuid.json + return path.join(baseDir, shard, `${id}.json`) + } + + /** + * Get all JSON files from the single-level sharded directory structure + * Traverses all shard subdirectories (00-ff) + */ + private async getAllShardedFiles(baseDir: string): Promise { + const allFiles: string[] = [] + + try { + const shardDirs = await fs.promises.readdir(baseDir) + + for (const shardDir of shardDirs) { + const shardPath = path.join(baseDir, shardDir) + + try { + const stat = await fs.promises.stat(shardPath) + + if (stat.isDirectory()) { + const shardFiles = await fs.promises.readdir(shardPath) + for (const file of shardFiles) { + if (file.endsWith('.json')) { + allFiles.push(file) + } + } + } + } catch (shardError) { + // Skip inaccessible shard directories + continue + } + } + + // Sort for consistent ordering + allFiles.sort() + return allFiles + + } catch (error: any) { + if (error.code === 'ENOENT') { + // Directory doesn't exist yet + return [] + } + throw error + } + } + + /** + * Production-scale streaming pagination for very large datasets + * Avoids loading all filenames into memory + */ + private async getVerbsWithPaginationStreaming( + options: { + limit?: number + cursor?: string + filter?: { + verbType?: string | string[] + sourceId?: string | string[] + targetId?: string | string[] + service?: string | string[] + metadata?: Record + } + }, + startIndex: number, + limit: number + ): Promise<{ + items: HNSWVerbWithMetadata[] + totalCount?: number + hasMore: boolean + nextCursor?: string + }> { + const verbs: HNSWVerbWithMetadata[] = [] + let processedCount = 0 + let skippedCount = 0 + let resultCount = 0 + + const depth = this.cachedShardingDepth ?? this.getOptimalShardingDepth() + + try { + // Stream through sharded directories efficiently + // hasMore=false means we reached the end of files, hasMore=true means streaming stopped early + const streamingHasMore = await this.streamShardedFiles( + this.verbsDir, + depth, + async (filename: string, filePath: string) => { + // Skip files until we reach start index + if (skippedCount < startIndex) { + skippedCount++ + return true // continue + } + + // Stop if we have enough results + if (resultCount >= limit) { + return false // stop streaming - more files exist + } + + try { + const id = filename.replace('.json', '') + + // Read verb data and metadata + const data = await fs.promises.readFile(filePath, 'utf-8') + const edge = JSON.parse(data) + const metadata = await this.getVerbMetadata(id) + + // Don't skip verbs without metadata - metadata is optional + // FIX: This was the root cause of the VFS bug (11 versions) + // Verbs can exist without metadata files (e.g., from imports/migrations) + + // Convert connections if needed + let connections = edge.connections + if (connections && typeof connections === 'object' && !(connections instanceof Map)) { + const connectionsMap = new Map>() + for (const [level, nodeIds] of Object.entries(connections)) { + connectionsMap.set(Number(level), new Set(nodeIds as string[])) + } + connections = connectionsMap + } + + // Canonical hydration (single source of truth: + // src/types/reservedFields.ts) — reserved fields top-level, ONLY + // custom fields in `metadata`. The previous hand-rolled + // destructure here missed `subtype` (so streamed verbs lost it) + // and `verb` (so the type key echoed inside custom metadata). + const verbWithMetadata: HNSWVerbWithMetadata = this.hydrateVerbWithMetadata( + { + id: edge.id, + vector: edge.vector, + connections: connections || new Map(), + verb: edge.verb, + sourceId: edge.sourceId, + targetId: edge.targetId + }, + metadata + ) + + // Apply filters + if (options.filter) { + const filter = options.filter + + if (filter.verbType) { + const types = Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType] + if (!types.includes(verbWithMetadata.verb)) return true // continue + } + + if (filter.sourceId) { + const sources = Array.isArray(filter.sourceId) ? filter.sourceId : [filter.sourceId] + if (!sources.includes(verbWithMetadata.sourceId)) return true // continue + } + + if (filter.targetId) { + const targets = Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId] + if (!targets.includes(verbWithMetadata.targetId)) return true // continue + } + } + + verbs.push(verbWithMetadata) + resultCount++ + processedCount++ + return true // continue + + } catch (error) { + console.warn(`Failed to read verb from ${filePath}:`, error) + processedCount++ + return true // continue + } + } + ) + + // CRITICAL FIX: Use streaming result for hasMore, not cached totalVerbCount + // streamingHasMore=false means we exhausted all files + // Also verify we loaded items to prevent infinite loops + const finalHasMore = streamingHasMore && (resultCount > 0 || startIndex === 0) + + return { + items: verbs, + totalCount: this.totalVerbCount || undefined, // Return cached count as hint only + hasMore: finalHasMore, + nextCursor: finalHasMore ? String(startIndex + resultCount) : undefined + } + + } catch (error: any) { + if (error.code === 'ENOENT') { + return { + items: [], + totalCount: 0, + hasMore: false + } + } + throw error + } + } + + /** + * Stream through sharded files without loading all names into memory + * Production-scale implementation for millions of files + */ + /** + * Stream through files in single-level sharded structure + * Calls processor for each file until processor returns false + * Returns true if more files exist (processor stopped early), false if all processed + */ + private async streamShardedFiles( + baseDir: string, + depth: number, + processor: (filename: string, fullPath: string) => Promise + ): Promise { + let hasMore = true + + // Single-level sharding (depth=1): baseDir/ab/uuid.json + try { + const shardDirs = await fs.promises.readdir(baseDir) + const sortedShardDirs = shardDirs.sort() + + for (const shardDir of sortedShardDirs) { + const shardPath = path.join(baseDir, shardDir) + + try { + const stat = await fs.promises.stat(shardPath) + + if (stat.isDirectory()) { + const files = await fs.promises.readdir(shardPath) + const sortedFiles = files.filter((f: string) => f.endsWith('.json')).sort() + + for (const file of sortedFiles) { + const shouldContinue = await processor(file, path.join(shardPath, file)) + + if (!shouldContinue) { + hasMore = false + break + } + } + + if (!hasMore) break + } + } catch (shardError) { + // Skip inaccessible shard directories + continue + } + } + } catch (error: any) { + if (error.code === 'ENOENT') { + hasMore = false + } + } + + return hasMore + } /** * Check if a file exists (handles both sharded and non-sharded) diff --git a/src/storage/adapters/memoryStorage.ts b/src/storage/adapters/memoryStorage.ts index 1b1f412e..05dfb0b4 100644 --- a/src/storage/adapters/memoryStorage.ts +++ b/src/storage/adapters/memoryStorage.ts @@ -133,11 +133,6 @@ export class MemoryStorage extends BaseStorage { */ protected async deleteObjectFromPath(path: string): Promise { this.objectStore.delete(path) - // Filesystem parity: on disk, objects and raw BYTE files are both just - // files — unlink removes whichever exists. Without this, deleteRawObject - // on a raw-bytes path (fact-log/generation segments) silently no-ops on - // memory storage: the delete "succeeds" and the bytes remain. - this.rawBytesStore.delete(path) } /** @@ -188,9 +183,6 @@ export class MemoryStorage extends BaseStorage { * @param key - The blob key. */ public async deleteBinaryBlob(key: string): Promise { - // Registered-blob contract — parity with the filesystem adapter: - // a declared family member is undeletable (throws ProtectedArtifactError). - await this.assertBlobKeyDeletable(key) this.blobStore.delete(key) } @@ -227,45 +219,6 @@ export class MemoryStorage extends BaseStorage { return [...this.txLogLines] } - // =========================================================================== - // Binary raw-byte primitives — in-memory mirror of the filesystem adapter's - // append-only substrate (the generation fact log's segments), so memory - // brains dual-write facts too and the compat suite runs on both adapters. - // =========================================================================== - - /** Raw binary files, keyed by verbatim path. */ - private rawBytesStore: Map = new Map() - - /** Append bytes to a raw binary file, creating it when absent. */ - public async appendRawBytes(rawPath: string, bytes: Uint8Array): Promise { - const existing = this.rawBytesStore.get(rawPath) - if (!existing) { - this.rawBytesStore.set(rawPath, bytes.slice()) - return - } - const merged = new Uint8Array(existing.length + bytes.length) - merged.set(existing, 0) - merged.set(bytes, existing.length) - this.rawBytesStore.set(rawPath, merged) - } - - /** Read a raw binary file whole (a copy); absent → null. */ - public async readRawBytes(rawPath: string): Promise { - const bytes = this.rawBytesStore.get(rawPath) - return bytes ? bytes.slice() : null - } - - /** Replace a raw binary file (atomic by construction in memory). */ - public async writeRawBytes(rawPath: string, bytes: Uint8Array): Promise { - this.rawBytesStore.set(rawPath, bytes.slice()) - } - - /** Byte size of a raw binary file, or null when absent. */ - public async rawByteSize(rawPath: string): Promise { - const bytes = this.rawBytesStore.get(rawPath) - return bytes ? bytes.length : null - } - /** * Serialize the entire in-memory store to a directory in the exact layout * the filesystem adapter uses (uncompressed JSON objects, `_blobs/*.bin` @@ -398,7 +351,6 @@ export class MemoryStorage extends BaseStorage { public async clear(): Promise { this.objectStore.clear() this.blobStore.clear() - this.rawBytesStore.clear() this.txLogLines = [] this.statistics = null diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index 6daf09c0..a678a000 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -5,7 +5,6 @@ import { GraphAdjacencyIndex } from '../graph/graphAdjacencyIndex.js' import type { GraphEntityIdResolver } from '../graph/graphAdjacencyIndex.js' -import { assessIndexReadiness } from '../utils/indexReadiness.js' import { GraphVerb, @@ -15,8 +14,7 @@ import { VerbMetadata, HNSWNounWithMetadata, HNSWVerbWithMetadata, - StatisticsData, - DerivedFamilyDeclaration + StatisticsData } from '../coreTypes.js' import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js' import { validateNounType, validateVerbType } from '../utils/typeValidation.js' @@ -31,8 +29,7 @@ import { getShardId } from './sharding.js' import { BlobStorage, type BlobStoreAdapter } from './blobStorage.js' import { unwrapBinaryData } from './binaryDataCodec.js' import { prodLog } from '../utils/logger.js' -import { isAbsentError } from '../utils/errorClassification.js' -import { BrainyError, ProtectedArtifactError, DerivedArtifactMissingError } from '../errors/brainyError.js' +import { BrainyError } from '../errors/brainyError.js' import { MetadataWriteBuffer } from '../utils/metadataWriteBuffer.js' import { splitNounMetadataRecord, @@ -250,27 +247,6 @@ function isCountedVisibility(visibility: unknown): boolean { */ export abstract class BaseStorage extends BaseStorageAdapter { protected isInitialized = false - /** One-shot guard so the graph fast-path cold-load probe runs once per adapter. */ - private _graphFastPathProbed = false - /** - * Registered-blob contract: declared derived-index families, keyed - * by name. Members are undeletable through the blob delete seams. Loaded lazily - * from `_system/derived-artifacts.json` and re-persisted on every change. - */ - private _derivedFamilies = new Map() - /** One-shot guard for loading the persisted family registry. */ - private _derivedFamiliesLoaded = false - /** Storage-root-relative path of the persisted family registry. */ - private static readonly DERIVED_FAMILIES_KEY = '_system/derived-artifacts.json' - /** - * Bounded concurrency for hydrating enumerated nouns during a pagination walk - * (readCanonicalObject + getNounMetadata per item). A canonical enumeration — - * which every index heal performs — otherwise pays N×per-op-latency serially; - * 16-way matches the wave the native rebuild uses so heal wall-clock is - * ~2×N/16×per-op instead of N×per-op. Bounded so a huge dataset can't spawn a - * read per entity at once. - */ - private static readonly HYDRATE_CONCURRENCY = 16 protected graphIndex?: GraphAdjacencyIndex protected graphIndexPromise?: Promise /** @@ -377,14 +353,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { id.startsWith('statistics_') || id === 'statistics' || id.startsWith('__chunk__') || // Metadata index chunks (roaring bitmap data) - id.startsWith('__sparse_index__') || // Metadata sparse indices (zone maps + bloom filters) - id.startsWith('__aggregation_') || // Aggregation engine definitions + state (routing is - // identical to the unknown-key fallback these keys hit - // before being listed here — this only kills the - // per-boot "Unknown key format" warning) - isSingletonSystemKey(id) // Known singletons (e.g. brainy:entityIdMapper) hit the - // same warn-then-route fallback without this — the - // routing below already handles them identically + id.startsWith('__sparse_index__') // Metadata sparse indices (zone maps + bloom filters) if (isSystemKey) { if (isSingletonSystemKey(id)) { @@ -849,82 +818,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { return this.deleteObjectFromPath(path) } - // ========================================================================== - // Fact-scan capability — the seam through which an index provider holding - // only `storage` reaches the generation fact log. The HOST brain wires the - // source at init (a closure over its live fact log, so restore/reopen stays - // transparent); providers call storage.scanFacts?.(...) and fall back to the - // enumeration walk when it returns null. Providers never construct a - // fact-log reader themselves — the log's open path is writer-side. - // ========================================================================== - - /** The host-wired fact-scan source (closures over the live generation state). */ - private _factScanSource: { - factLog: () => import('../db/factLog.js').FactLog | null - committedGeneration: () => number - } | null = null - - /** HOST-ONLY: wire (or clear) the fact-scan capability's source. */ - public setFactScanSource( - source: { - factLog: () => import('../db/factLog.js').FactLog | null - committedGeneration: () => number - } | null - ): void { - this._factScanSource = source - } - - /** Open a scan over committed facts, or `null` when no fact log exists. */ - public scanFacts(options?: { - fromGeneration?: number - toGeneration?: number - kinds?: Array<'noun' | 'verb'> - batchSize?: number - }): import('../db/factLog.js').FactScanHandle | null { - const log = this._factScanSource?.factLog() - return log ? log.scanFacts(options) : null - } - - /** The fact log's head generation, or `null` when no fact log exists. */ - public factLogHeadGeneration(): number | null { - const log = this._factScanSource?.factLog() - return log ? log.headGeneration() : null - } - - /** - * The COMMITTED generation watermark (the manifest truth) — the value a - * projection's `sourceGeneration` compares against. Exposed here so a - * provider never parses the store's private manifest format. `null` when - * the capability is unwired (no host, or a bare adapter). - */ - public committedGeneration(): number | null { - return this._factScanSource ? this._factScanSource.committedGeneration() : null - } - - /** Sealed fact-segment paths (zero-copy handoff); empty when none. */ - public factSegmentPaths(options?: { fromGeneration?: number }): string[] { - const log = this._factScanSource?.factLog() - return log ? log.segmentPaths(options) : [] - } - - /** - * @description Remove the container that held a canonical entity's leg files, - * called after both legs are deleted so a delete leaves NOTHING behind — no - * orphan "scar" directory. `objectLegPath` is any one of the entity's leg - * paths (e.g. its `vectors.json`); the container is that path's parent. - * - * Default: a no-op. Key/prefix-addressed stores (in-memory, cloud object - * stores) have no empty-container concept — deleting the leg keys already - * removes the entity entirely. The filesystem adapter overrides this to - * `rmdir` the emptied entity directory. - * - * @param objectLegPath - Storage-root-relative path of one of the entity's legs. - * @protected - */ - protected async removeCanonicalContainer(objectLegPath: string): Promise { - void objectLegPath - } - /** * @description List canonical objects under a storage-root-relative prefix. * @@ -976,148 +869,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { this.generationBumpHook = hook } - // ========================================================================== - // Temporal-blob contract (the GenerationStorage optional methods) - // - // Content blobs join the Model-B immutability model through these hooks: - // the generation store counts a history reference per before-image record - // that carries a content hash, and compaction — the ONE reclamation point — - // releases those references and physically deletes bytes only at zero live - // AND zero history references. Crash ordering is over-count-only (record - // BEFORE the record-set persists, release AFTER it is deleted), so a crash - // can leak bytes until the scrub recounts but can never reclaim bytes a - // retained generation still needs. - // ========================================================================== - - /** Set when the open-time backfill/scrub could not verify history reference - * counts. While true, the temporal-blob hooks stop mutating counts and - * compaction stops reclaiming blob bytes — pure leak-safe mode until a - * successful {@link scrubBlobHistoryRefCounts} restores exactness. */ - private blobHistoryRefsUnverified = false - - /** - * @description Extract the content-blob hashes a generation record-set - * references — a pure MULTISET extraction (one entry per referencing record - * occurrence), no side effects. Only entity records can reference VFS - * content (`metadata.storage.type === 'blob'`). - * @param records - The record-set's before-image records. - * @returns The referenced hashes, duplicates preserved. - */ - public extractBlobHashesFromRecords( - records: Array<{ kind: string; metadata: unknown }> - ): string[] { - const hashes: string[] = [] - for (const record of records) { - if (record.kind !== 'noun') continue - const storage = (record.metadata as { storage?: { type?: string; hash?: unknown } } | null) - ?.storage - if (storage?.type === 'blob' && typeof storage.hash === 'string') { - hashes.push(storage.hash) - } - } - return hashes - } - - /** - * @description Record one history reference per hash occurrence (see the - * contract note above — called BEFORE the referencing record-set persists). - * No-op without a blob store or while counts are unverified. - * @param hashes - Hash multiset from {@link extractBlobHashesFromRecords}. - */ - public async recordHistoryBlobReferences(hashes: string[]): Promise { - if (!this.blobStorage || this.blobHistoryRefsUnverified || hashes.length === 0) return - for (const hash of hashes) { - await this.blobStorage.recordHistoryReference(hash) - } - } - - /** - * @description Release one history reference per hash occurrence and - * physically reclaim any hash left with zero live AND zero history - * references — compaction's blob-reclamation step (called AFTER the - * referencing record-set is deleted). No-op without a blob store or while - * counts are unverified (leak-safe: nothing is reclaimed on guesses). - * @param hashes - Hash multiset recorded when the record-set was persisted. - */ - public async releaseHistoryBlobReferences(hashes: string[]): Promise { - if (!this.blobStorage || this.blobHistoryRefsUnverified || hashes.length === 0) return - for (const hash of hashes) { - await this.blobStorage.releaseHistoryReference(hash) - } - for (const hash of new Set(hashes)) { - await this.blobStorage.reclaimIfUnreferenced(hash) - } - } - - /** - * @description One-time (marker-gated) backfill of blob history reference - * counts for stores whose generation history predates the temporal-blob - * contract. Runs the scrub, then stamps `_system/blob-history-refs.json` so - * later opens skip the walk. On scrub failure the store enters leak-safe - * mode (counts untouched, reclamation disabled) rather than risking a - * premature delete on wrong counts. - */ - public async backfillBlobHistoryRefCountsIfNeeded(): Promise { - if (!this.blobStorage) return - const MARKER = '_system/blob-history-refs.json' - try { - const marker = (await this.readObjectFromPath(MARKER)) as { version?: number } | null - if (marker?.version === 1) return - } catch { - // no marker — proceed to scrub - } - try { - await this.scrubBlobHistoryRefCounts() - await this.writeObjectToPath(MARKER, { version: 1, verifiedAt: new Date().toISOString() }) - } catch (err) { - this.blobHistoryRefsUnverified = true - console.error( - '[Brainy] blob history-reference backfill failed — temporal-blob ' + - 'reclamation disabled for this session (leak-safe); history reads ' + - 'are unaffected. Re-open to retry.', - err - ) - } - } - - /** - * @description Recount every blob's history references from the actual - * generation record-sets and set the counts ABSOLUTELY (uncounted blobs are - * zeroed) — the idempotent repair that restores exactness after any crash - * that over-counted. O(history records + stored blobs). - * @returns Blobs counted and records walked, for observability. - */ - public async scrubBlobHistoryRefCounts(): Promise<{ blobs: number; records: number }> { - if (!this.blobStorage) return { blobs: 0, records: 0 } - const counts = new Map() - let records = 0 - let paths: string[] = [] - try { - paths = await this.listObjectsUnderPath('_generations') - } catch { - paths = [] // no history yet - } - for (const p of paths) { - if (!p.includes('/prev/')) continue - const record = (await this.readObjectFromPath(p)) as - | { kind?: string; metadata?: unknown } - | null - if (!record) continue - records++ - for (const hash of this.extractBlobHashesFromRecords([ - { kind: record.kind ?? '', metadata: record.metadata } - ])) { - counts.set(hash, (counts.get(hash) ?? 0) + 1) - } - } - const allHashes = await this.blobStorage.listHashes() - for (const hash of allHashes) { - await this.blobStorage.setHistoryRefCount(hash, counts.get(hash) ?? 0) - } - this.blobHistoryRefsUnverified = false - return { blobs: allHashes.length, records } - } - /** * Read a raw object at a storage-root-relative path. Bypasses the write * cache (record-layer files are written through @@ -1144,154 +895,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { await this.writeObjectToPath(path, data) } - // ========================================================================== - // Registered-blob contract — declared derived-index families are - // undeletable through the blob delete seams. Shared here so every adapter that - // extends BaseStorage inherits the same enforcement; the concrete - // deleteBinaryBlob / removeRawPrefix call assertBlobKeyDeletable / - // assertPrefixNotProtected before removing anything. - // ========================================================================== - - /** Load the persisted family registry once (lazy). */ - private async ensureDerivedFamiliesLoaded(): Promise { - if (this._derivedFamiliesLoaded) return - const stored = await this.readRawObject(BaseStorage.DERIVED_FAMILIES_KEY).catch(() => null) - const families = (stored as { families?: DerivedFamilyDeclaration[] } | null)?.families - if (Array.isArray(families)) { - for (const f of families) { - if (f && typeof f.name === 'string' && Array.isArray(f.members)) this._derivedFamilies.set(f.name, f) - } - } - this._derivedFamiliesLoaded = true - } - - /** Persist the current family registry (fsync'd via the raw-object write). */ - private async persistDerivedFamilies(): Promise { - await this.writeRawObject(BaseStorage.DERIVED_FAMILIES_KEY, { - families: [...this._derivedFamilies.values()] - }) - } - - public async registerDerivedFamily(family: DerivedFamilyDeclaration): Promise { - await this.ensureInitialized() - await this.ensureDerivedFamiliesLoaded() - this._derivedFamilies.set(family.name, { - name: family.name, - members: [...family.members], - ...(family.namespace !== undefined ? { namespace: family.namespace } : {}), - ...(family.rebuildable !== undefined ? { rebuildable: family.rebuildable } : {}) - }) - await this.persistDerivedFamilies() - } - - public async unregisterDerivedFamily(name: string): Promise { - await this.ensureInitialized() - await this.ensureDerivedFamiliesLoaded() - if (this._derivedFamilies.delete(name)) await this.persistDerivedFamilies() - } - - public async listDerivedFamilies(): Promise { - await this.ensureInitialized() - await this.ensureDerivedFamiliesLoaded() - return [...this._derivedFamilies.values()] - } - - /** - * @description A blob key that is a transient write-scratch file (a `*.tmp.*` - * temp, a `*.rebuild-tmp`, a `*.rotate-tmp`). Its OWNER renames/removes it as - * part of an atomic write; it is never a protected family member. - */ - private isTransientBlobKey(key: string): boolean { - return /\.rebuild-tmp$|\.rotate-tmp$|\.tmp(\.|$)/.test(key) - } - - /** - * @description The name of the protected family a blob key belongs to, or - * `null`. A `namespace` member protects every key beneath it; a plain member - * protects that exact key. - */ - private protectingFamilyOf(key: string): string | null { - for (const family of this._derivedFamilies.values()) { - for (const member of family.members) { - if (family.namespace ? key === member || key.startsWith(member) : key === member) { - return family.name - } - } - } - return null - } - - /** - * @description Enforcement point for `deleteBinaryBlob`: refuse (throw - * {@link ProtectedArtifactError}) when the key is a declared family member. - * Transients pass through. An undeclared blob delete under an ACTIVE contract - * (families are registered) is logged loudly — nothing under `_blobs/` should - * vanish unremarked once the contract is in force. - */ - protected async assertBlobKeyDeletable(key: string): Promise { - await this.ensureDerivedFamiliesLoaded() - if (this._derivedFamilies.size === 0) return // contract inactive — no enforcement - if (this.isTransientBlobKey(key)) return - const family = this.protectingFamilyOf(key) - if (family) throw new ProtectedArtifactError(key, family) - prodLog.warn( - `[BaseStorage] deleteBinaryBlob('${key}') removes an UNDECLARED blob while the ` + - `registered-blob contract is active — permitted, but surfaced so no _blobs/ file ` + - `disappears silently.` - ) - } - - /** - * @description Enforcement point for `removeRawPrefix`: refuse when the prefix - * would take out a protected family member (a prefix-nuke must not remove a - * declared blob). Transients are ignored. - */ - protected async assertPrefixNotProtected(prefix: string): Promise { - await this.ensureDerivedFamiliesLoaded() - if (this._derivedFamilies.size === 0) return - for (const family of this._derivedFamilies.values()) { - for (const member of family.members) { - // Under the shared `_blobs/` root, member keys resolve beneath it; a - // prefix intersects a member when either contains the other. - const memberBlobPath = `_blobs/${member}` - if ( - memberBlobPath.startsWith(prefix) || - prefix.startsWith(memberBlobPath) || - member.startsWith(prefix) || - prefix.startsWith(member) - ) { - if (!this.isTransientBlobKey(member)) throw new ProtectedArtifactError(member, family.name) - } - } - } - } - - /** - * @description Verify every declared family has all its members present on - * disk (missing-on-open catch for an EXTERNAL deleter that bypasses - * the in-process refusal). Returns the incomplete families (name + missing - * members) — the caller decides how to heal (rebuild from canonical). Loud by - * construction: a missing load-bearing blob is named, never silently tolerated. - */ - public async checkDerivedFamiliesPresent(): Promise> { - await this.ensureInitialized() - await this.ensureDerivedFamiliesLoaded() - const incomplete: Array<{ name: string; missing: string[]; rebuildable: boolean }> = [] - for (const family of this._derivedFamilies.values()) { - if (family.namespace) continue // a growing prefix has no fixed member set to verify - const missing: string[] = [] - for (const member of family.members) { - const blob = await this.loadBinaryBlob(member).catch(() => null) - if (!blob) missing.push(member) - } - if (missing.length > 0) { - prodLog.warn(new DerivedArtifactMissingError(family.name, missing).message) - incomplete.push({ name: family.name, missing, rebuildable: family.rebuildable !== false }) - } - } - return incomplete - } - /** * Delete a raw object at a storage-root-relative path (no-op if absent). * @@ -1475,12 +1078,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { */ protected async reloadDerivedState(): Promise { this.clearWriteCache() - // Registered-blob registry: clear() wipes the persisted `_system/` copy and - // the family members under `_blobs/`, so drop the in-memory cache too — a - // reset brain must not keep stale family protection or report their members - // "missing" on the next check. - this._derivedFamilies.clear() - this._derivedFamiliesLoaded = false this.nounCountsByType.fill(0) this.verbCountsByType.fill(0) this.subtypeCountsByType.clear() @@ -1681,27 +1278,19 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Delete a noun from storage */ - public async deleteNoun(id: string, priorMetadata?: NounMetadata | null): Promise { + public async deleteNoun(id: string): Promise { await this.ensureInitialized() - // FULL removal (live-HEAD hygiene): remove BOTH canonical legs AND the - // entity's container, so a delete leaves nothing behind — no orphan/scar - // directory to inflate the enumerated count or ghost resolveLocator. The - // generation store holds the immutable before-image, so asOf() still - // reconstructs the deleted entity until retention expires. - await this.deleteNoun_internal(id) // vectors.json leg + // Delete both the vector file and metadata file (2-file system) + await this.deleteNoun_internal(id) - // Metadata leg + count decrement. A genuine "already absent" is a no-op - // downstream (readCanonicalObject / deleteObjectFromPath both treat ENOENT as - // success); a REAL fault must surface loudly (loud errors, never quiet - // losses) and must never silently skip the count decrement — so this is NO - // LONGER wrapped in a blind catch that masked faults as "file didn't exist". - // `priorMetadata` (the caller's pre-delete read) keeps the decrement honest - // even when the canonical read inside returns null (replace race / ghost). - await this.deleteNounMetadata(id, priorMetadata) - - // Remove the now-empty entity container (a no-op for key/prefix stores). - await this.removeCanonicalContainer(getNounVectorPath(id)) + // Delete metadata file (if it exists) + try { + await this.deleteNounMetadata(id) + } catch (error) { + // Ignore if metadata file doesn't exist + prodLog.debug(`No metadata file to delete for noun ${id}`) + } } /** @@ -2068,20 +1657,11 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Cursor (8.0): resume token carrying the (shard, nounId) of the last returned // noun — the noun mirror of getVerbsWithPagination. When present it supersedes // `offset` and resumes the shard walk immediately AFTER that position, so a full - // walk is O(N) instead of the O(N²) of offset paging. (Previously the cursor was - // ignored, which was latent — the only multi-page consumer used a single big - // page — until small chunk sizes needed page 2 and an offset-0-on-every-call - // walk never terminated.) + // walk is O(N) instead of the O(N²) of offset paging. Malformed/foreign tokens + // decode to null → offset fallback. (Previously the cursor was ignored, which + // was latent — the only multi-page consumer used a single big page — until small + // chunk sizes needed page 2 and an offset-0-on-every-call walk never terminated.) const cursor = this.decodeNounWalkCursor(options.cursor) - if (options.cursor && cursor === null) { - // A supplied-but-undecodable resume token must FAIL, not silently restart - // at offset 0 — to a while(hasMore) caller the silent fallback re-serves - // page 1 forever: an unbounded CPU loop wearing pagination's clothes. - throw BrainyError.storage( - `getNouns: invalid pagination cursor '${options.cursor}' — cannot resume this walk. ` + - `Restart it without a cursor.` - ) - } const collected: Array<{ noun: HNSWNounWithMetadata; shard: number }> = [] // Peek one past the window so `hasMore` is decidable. Cursor mode collects one @@ -2104,63 +1684,39 @@ export abstract class BaseStorage extends BaseStorageAdapter { .map((p) => ({ path: p, id: idFromVectorPath(p) })) .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)) - // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor - // id BEFORE hydrating — a cheap id compare (from the path), so skipped - // nouns are never read. - const toHydrate = - cursor && shard === cursor.shard - ? entries.filter((e) => e.id > cursor.id) - : entries + for (const { path: nounPath, id: nounId } of entries) { + if (collected.length >= peekCount) break + // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor id. + if (cursor && shard === cursor.shard && nounId <= cursor.id) continue - // Hydrate in bounded-concurrency batches (16-way) instead of one-at-a-time. - // Every canonical enumeration — and every index heal enumerates canonical — - // otherwise pays N×per-op-latency SERIALLY (the dominant heal-time term). - // Order is preserved (the batch is a slice of the sorted entries and its - // results are consumed in order), so offset windows / cursor resume stay - // deterministic. peekCount stops the walk; the final batch over-hydrates by - // at most BASE_STORAGE_HYDRATE_CONCURRENCY entries (bounded, acceptable). - for ( - let i = 0; - i < toHydrate.length && collected.length < peekCount; - i += BaseStorage.HYDRATE_CONCURRENCY - ) { - const batch = toHydrate.slice(i, i + BaseStorage.HYDRATE_CONCURRENCY) - const hydrated = await Promise.all( - batch.map(async ({ path: nounPath }) => { - try { - const noun = await this.readCanonicalObject(nounPath) - if (!noun) return null - const deserialized = this.deserializeNoun(noun) - const metadata = await this.getNounMetadata(deserialized.id) - if (!metadata) return null - return { deserialized, metadata } - } catch (error) { - // Skip nouns that fail to load - return null - } - }) - ) - - for (const h of hydrated) { - if (collected.length >= peekCount) break - if (!h) continue - const { deserialized, metadata } = h + try { + const noun = await this.readCanonicalObject(nounPath) + if (!noun) continue + const deserialized = this.deserializeNoun(noun) + const metadata = await this.getNounMetadata(deserialized.id) + if (!metadata) continue // Apply type filter if (filter?.nounType && metadata.noun) { const types = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType] - if (!types.includes(metadata.noun)) continue + if (!types.includes(metadata.noun)) { + continue + } } // Apply service filter if (filter?.service) { const services = Array.isArray(filter.service) ? filter.service : [filter.service] - if (metadata.service && !services.includes(metadata.service)) continue + if (metadata.service && !services.includes(metadata.service)) { + continue + } } // Combine noun + metadata via the canonical hydration helper — // reserved fields top-level, ONLY custom fields in `metadata`. collected.push({ noun: this.hydrateNounWithMetadata(deserialized, metadata), shard }) + } catch (error) { + // Skip nouns that fail to load } } } catch (error) { @@ -2200,119 +1756,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { } } - /** - * @description Enumerate noun IDS ONLY, without hydrating each entity's vector - * and metadata — the opt-out for callers (e.g. an index heal) that own their - * own IO schedule and index straight from a stream. Shares the exact - * shard-walk, cursor, offset and `nextCursor` contract of - * {@link getNounsWithPagination}, so the two are page-compatible. - * - * - UNFILTERED (the heal case): ids come straight from the shard paths — ZERO - * per-entity reads. A full enumeration is O(entries listed), not O(N reads). - * - FILTERED: the type/service filter needs metadata, so only the metadata is - * hydrated (16-way bounded concurrency), never the full noun. - * - * @param options - `limit`/`offset`/`cursor`/`filter` — same semantics as - * {@link getNounsWithPagination}. - * @returns The page of ids plus `totalCount` / `hasMore` / `nextCursor`. - */ - public async getNounIdsWithPagination(options: { - limit: number - offset?: number - cursor?: string - filter?: { - nounType?: string | string[] - service?: string | string[] - metadata?: Record - } - }): Promise<{ ids: string[]; totalCount: number; hasMore: boolean; nextCursor?: string }> { - await this.ensureInitialized() - - const { limit, offset = 0, filter } = options - const cursor = this.decodeNounWalkCursor(options.cursor) - if (options.cursor && cursor === null) { - // Same law as getNouns/getVerbs: an undecodable resume token FAILS - // instead of silently restarting the walk at offset 0. - throw BrainyError.storage( - `getNounIds: invalid pagination cursor '${options.cursor}' — cannot resume this walk. ` + - `Restart it without a cursor.` - ) - } - const collected: Array<{ id: string; shard: number }> = [] - const peekCount = cursor ? limit + 1 : offset + limit + 1 - const startShard = cursor ? cursor.shard : 0 - - for (let shard = startShard; shard < 256 && collected.length < peekCount; shard++) { - const shardHex = shard.toString(16).padStart(2, '0') - const shardDir = `entities/nouns/${shardHex}` - try { - const nounFiles = await this.listCanonicalObjects(shardDir) - const entries = nounFiles - .filter((p) => p.includes('/vectors.json')) - .map((p) => idFromVectorPath(p)) - .sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)) - const toWalk = - cursor && shard === cursor.shard ? entries.filter((id) => id > cursor.id) : entries - - if (!filter) { - // Unfiltered — ids straight from the paths, no reads at all. - for (const id of toWalk) { - if (collected.length >= peekCount) break - collected.push({ id, shard }) - } - } else { - // Filtered — hydrate metadata ONLY (16-way) to apply the filter. - for ( - let i = 0; - i < toWalk.length && collected.length < peekCount; - i += BaseStorage.HYDRATE_CONCURRENCY - ) { - const batch = toWalk.slice(i, i + BaseStorage.HYDRATE_CONCURRENCY) - const metas = await Promise.all( - batch.map(async (id) => { - try { - return { id, metadata: await this.getNounMetadata(id) } - } catch { - return null - } - }) - ) - for (const m of metas) { - if (collected.length >= peekCount) break - if (!m || !m.metadata) continue - const metadata = m.metadata - if (filter.nounType && metadata.noun) { - const types = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType] - if (!types.includes(metadata.noun)) continue - } - if (filter.service) { - const services = Array.isArray(filter.service) ? filter.service : [filter.service] - if (metadata.service && !services.includes(metadata.service)) continue - } - collected.push({ id: m.id, shard }) - } - } - } - } catch (error) { - // Skip shards with no data - } - } - - const windowStart = cursor ? 0 : offset - const pagePairs = collected.slice(windowStart, windowStart + limit) - const ids = pagePairs.map((p) => p.id) - const hasMore = collected.length > windowStart + limit - const totalCount = filter ? collected.length : Math.max(this.totalNounCount, collected.length) - - let nextCursor: string | undefined = undefined - if (hasMore && pagePairs.length > 0) { - const lastPair = pagePairs[pagePairs.length - 1] - nextCursor = this.encodeNounWalkCursor(lastPair.shard, lastPair.id) - } - - return { ids, totalCount, hasMore, nextCursor } - } - /** * @description Encode a noun-walk resume cursor — the `(shard, nounId)` of the * last returned noun — as an opaque, version-tagged token (`cn1:` prefix lets @@ -2383,17 +1826,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { // (shard, verbId) of the last returned verb. When present it SUPERSEDES `offset` // and resumes the shard walk immediately AFTER that position, so a full walk is // O(N) total instead of the O(N²) of offset paging (which re-scans from shard 0 - // every page). + // every page). Malformed / foreign tokens decode to null → offset fallback. const cursor = this.decodeVerbWalkCursor(options.cursor) - if (options.cursor && cursor === null) { - // A supplied-but-undecodable resume token must FAIL, not silently restart - // at offset 0 — to a while(hasMore) caller the silent fallback re-serves - // page 1 forever: an unbounded CPU loop wearing pagination's clothes. - throw BrainyError.storage( - `getVerbs: invalid pagination cursor '${options.cursor}' — cannot resume this walk. ` + - `Restart it without a cursor.` - ) - } // Each collected entry remembers its shard so nextCursor can point at the exact // (shard, id) resume position. @@ -3029,16 +2463,19 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Delete a verb from storage */ - public async deleteVerb(id: string, priorMetadata?: VerbMetadata | null): Promise { + public async deleteVerb(id: string): Promise { await this.ensureInitialized() - // FULL removal (see deleteNoun): both canonical legs + the verb container. - // The blind catch that masked real faults as "no metadata file" is gone — a - // genuine absence is already a no-op downstream; a real fault surfaces loudly. - // `priorMetadata` keeps the count decrement honest on a null internal read. - await this.deleteVerb_internal(id) // vectors.json leg - await this.deleteVerbMetadata(id, priorMetadata) // metadata leg + count decrement - await this.removeCanonicalContainer(getVerbVectorPath(id)) + // Delete both the vector file and metadata file (2-file system) + await this.deleteVerb_internal(id) + + // Delete metadata file (if it exists) + try { + await this.deleteVerbMetadata(id) + } catch (error) { + // Ignore if metadata file doesn't exist + prodLog.debug(`No metadata file to delete for verb ${id}`) + } } /** * Get graph index (lazy initialization with concurrent access protection) @@ -3101,61 +2538,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { return this.graphIndex } - - /** - * @description One-shot cold-load self-heal for a graph provider that does NOT - * expose the honest `isReady()` signal. A native provider wired via - * {@link setGraphIndex} bypasses `_initializeGraphIndex`'s `size()===0` - * self-heal, so a cold-open that loaded the COUNT but not the source→target - * adjacency would let the fast path return a silent `[]`. This probe (run once, - * before the fast path) samples ONE known persisted edge: if the index claims - * relationships (`size() > 0`) yet that edge's source resolves to no verb ints, - * the adjacency did not load → rebuild once from storage. Providers that expose - * `isReady()` are covered by the honest gate in - * getVerbsBy{Source,Target}_internal and skip this probe; the JS index (which - * self-heals in `_initializeGraphIndex`) resolves its known edge and no-ops. - */ - private async ensureGraphFastPathProbed(): Promise { - if (this._graphFastPathProbed) return - const index = this.graphIndex - const resolver = this.graphEntityIdResolver - if (!index || !index.isInitialized || !resolver) return - // isReady()-capable providers report serving honestly — the gate handles them. - if (assessIndexReadiness(index) !== 'unknown') { - this._graphFastPathProbed = true - return - } - if (index.size() <= 0) { - this._graphFastPathProbed = true - return // no edges claimed — nothing to verify - } - try { - const sample = await this.getVerbs({ pagination: { limit: 1 } }) - const verb = sample.items?.[0] - if (!verb || !verb.sourceId) { - this._graphFastPathProbed = true - return // no edges in storage — stale count, harmless - } - const sourceInt = resolver.getInt(verb.sourceId) - if (sourceInt === undefined) { - this._graphFastPathProbed = true - return // foreign / unmapped sample — inconclusive, never a false rebuild - } - const verbInts = await index.getVerbIdsBySource(BigInt(sourceInt)) - if (verbInts.length === 0) { - prodLog.warn( - `[BaseStorage] Graph fast path: index reports ${index.size()} relationship(s) but a ` + - `known persisted edge resolves to none — the persisted adjacency did not load. ` + - `Rebuilding once from storage.` - ) - await index.rebuild() - } - this._graphFastPathProbed = true - } catch (error) { - // Transient probe/rebuild failure must not break the read; re-arm for next call. - prodLog.debug(`[BaseStorage] Graph fast-path probe skipped (transient): ${error}`) - } - } /** * Clear all data from storage * This method should be implemented by each specific adapter @@ -3690,32 +3072,19 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Delete noun metadata from storage (ID-first, O(1) delete). - * - * @param id - The entity id. - * @param priorRecord - OPTIONAL already-known metadata of the entity being - * removed (e.g. the pre-delete read `remove()` performs, or a captured - * before-image). The count decrement must never REQUIRE re-reading the - * thing being removed: when the canonical read here returns `null` (a - * replace race, or a partial-delete ghost from an earlier version) the - * decrement falls back to this record instead of being silently skipped — - * the skip permanently inflated the persisted totals (adds counted, paired - * removals not decremented), and `Math.max(totalNounCount, scanned)` made - * the inflation unfixable by any disk cleanup. + * Delete noun metadata from storage (ID-first, O(1) delete) */ - public async deleteNounMetadata(id: string, priorRecord?: NounMetadata | null): Promise { + public async deleteNounMetadata(id: string): Promise { await this.ensureInitialized() // Direct O(1) delete with ID-first path. Read the canonical record BEFORE // removing it: the per-type and subtype decrements are sourced from the // entity's own metadata (`noun` type, `subtype`, `visibility`) rather than an // id-keyed cache, keeping type-statistics honest across deletes — symmetric - // with the increments in `saveNounMetadata_internal()`. A null read falls - // back to the caller-provided prior record (see @param priorRecord). + // with the increments in `saveNounMetadata_internal()`. const path = getNounMetadataPath(id) - const read = await this.readCanonicalObject(path) + const record = await this.readCanonicalObject(path) await this.deleteCanonicalObject(path) - const record = read ?? priorRecord const priorType = record?.noun as NounType | undefined // 8.0 visibility: an internal/system entity was never added to `nounCountsByType` @@ -3723,22 +3092,10 @@ export abstract class BaseStorage extends BaseStorageAdapter { const priorCounted = isCountedVisibility(record?.visibility) if (priorType) { if (priorCounted) { - // Symmetric with the counted-add increment in saveNounMetadata_internal(): - // decrement BOTH the user-facing scalar total (getNounCount / counts.json) AND - // the per-type bucket, then persist. The scalar decrement was previously - // omitted here, so deletes permanently inflated getNounCount() — the stale - // scalar wins pagination via Math.max(totalNounCount, collected.length) and is - // persisted by scheduleCountPersist(). With this, the invariant - // `totalNounCount === Σ nounCountsByType` holds across add / update-flip / delete. - this.decrementEntityCount(priorType) const idx = TypeUtils.getNounIndex(priorType) if (this.nounCountsByType[idx] > 0) { this.nounCountsByType[idx]-- } - this.scheduleCountPersist().catch(() => { - // Ignore persist errors — the in-memory count is authoritative; a later - // operation retries the persist. - }) } // Symmetric subtype decrement — same non-empty-string guard as the write path. @@ -3889,38 +3246,20 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Delete verb metadata from storage (ID-first, O(1) delete) */ - public async deleteVerbMetadata(id: string, priorRecord?: VerbMetadata | null): Promise { + public async deleteVerbMetadata(id: string): Promise { await this.ensureInitialized() // Direct O(1) delete with ID-first path. Read the canonical record BEFORE - // removing it so every decrement is sourced from the edge's own metadata - // (`verb` type, `subtype`, `visibility`) rather than an id-keyed cache — - // symmetric with the increments in `saveVerbMetadata_internal()`. A null - // read falls back to the caller-provided prior record: the decrement must - // never REQUIRE re-reading the thing being removed (a silent skip minted - // permanent counter inflation — see deleteNounMetadata). + // removing it so the verb-subtype decrement is sourced from the edge's own + // metadata (`verb` type + `subtype`) rather than an id-keyed cache — symmetric + // with the increment in `saveVerbMetadata_internal()`. Verb deletes do not + // touch `verbCountsByType` in this path (matching prior behavior), so no + // visibility read is needed here. const path = getVerbMetadataPath(id) - const read = await this.readCanonicalObject(path) + const record = await this.readCanonicalObject(path) await this.deleteCanonicalObject(path) - const record = read ?? priorRecord const priorVerb = record?.verb as VerbType | undefined - // Symmetric count decrement (previously OMITTED — verb deletes touched neither the - // scalar total nor the per-type bucket, so both inflated permanently). A COUNTED - // edge bumped BOTH the scalar (incrementVerbCount in saveVerbMetadata_internal) and - // the per-type bucket (the unconditional bump in saveVerb_internal that the metadata - // path keeps for counted edges). Delete must undo both, gated on the SAME visibility - // as the add, so `totalVerbCount === Σ verbCountsByType` holds across delete. - const priorCounted = isCountedVisibility(record?.visibility) - if (priorVerb && priorCounted) { - this.decrementVerbCount(priorVerb) - const idx = TypeUtils.getVerbIndex(priorVerb) - if (this.verbCountsByType[idx] > 0) this.verbCountsByType[idx]-- - this.scheduleCountPersist().catch(() => { - // Ignore persist errors — in-memory count is authoritative; a later op retries. - }) - } - const priorSubtype = typeof record?.subtype === 'string' && (record.subtype as string).length > 0 ? (record.subtype as string) : undefined @@ -4339,17 +3678,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { this.nounCountsByType = new Uint32Array(NOUN_TYPE_COUNT) this.verbCountsByType = new Uint32Array(VERB_TYPE_COUNT) - // The SAME walk also rebuilds the user-facing scalar totals + per-type maps - // persisted in counts.json (totalNounCount / totalVerbCount / entityCounts / - // verbCounts). Previously only the type-statistics arrays were rebuilt and - // the total was computed just to LOG it — so a drifted persisted scalar - // (deletes whose decrement was skipped) survived every "rebuild" forever, - // and Math.max(totalNounCount, scanned) made the inflation unfixable by any - // disk cleanup. This method is now the SANCTIONED RECOUNT: one canonical - // walk, every counter rollup rebuilt and persisted from it. - const countedNouns = new Map() - const countedVerbs = new Map() - // Scan noun shards for (let shard = 0; shard < 256; shard++) { const shardHex = shard.toString(16).padStart(2, '0') @@ -4370,7 +3698,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { if (typeIndex >= 0 && typeIndex < NOUN_TYPE_COUNT) { this.nounCountsByType[typeIndex]++ } - countedNouns.set(metadata.noun, (countedNouns.get(metadata.noun) || 0) + 1) } } } catch (error) { @@ -4402,7 +3729,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { if (typeIndex >= 0 && typeIndex < VERB_TYPE_COUNT) { this.verbCountsByType[typeIndex]++ } - countedVerbs.set(metadata.verb, (countedVerbs.get(metadata.verb) || 0) + 1) } } } catch (error) { @@ -4419,19 +3745,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { const totalVerbs = this.verbCountsByType.reduce((sum, count) => sum + count, 0) const totalNouns = this.nounCountsByType.reduce((sum, count) => sum + count, 0) - - // The sanctioned recount half: replace the user-facing scalar totals + the - // per-type maps with the walk's truth and PERSIST them (counts.json), so an - // inflated persisted counter is actually corrected — not merely out-voted - // in memory until the next reopen rehydrates the stale file. - this.entityCounts = countedNouns - this.verbCounts = countedVerbs - this.totalNounCount = totalNouns - this.totalVerbCount = totalVerbs - this.countCache.clear() - await this.persistCounts() - - prodLog.info(`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs (scalar + per-type persisted)`) + prodLog.info(`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs`) } /** @@ -4582,12 +3896,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { return this.deserializeNoun(noun) } } catch (error) { - // A real storage/deserialize fault is NOT "entity not found": - // readCanonicalObject returns null for genuine absence and only throws on - // a real fault, so masking it here reports a present-but-unreadable entity - // as missing. Absence → null, fault → propagate loudly. - if (isAbsentError(error)) return null - throw error + // Entity not found + return null } return null @@ -4697,12 +4007,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { return this.deserializeVerb(verb) } } catch (error) { - // A real storage/deserialize fault is NOT "relationship not found": - // readCanonicalObject returns null for genuine absence and only throws on - // a real fault, so masking it here reports a present-but-unreadable edge - // as missing. Absence → null, fault → propagate loudly. - if (isAbsentError(error)) return null - throw error + // Entity not found + return null } return null @@ -4716,22 +4022,13 @@ export abstract class BaseStorage extends BaseStorageAdapter { sourceId: string ): Promise { await this.ensureInitialized() - await this.ensureGraphFastPathProbed() prodLog.debug(`[BaseStorage] getVerbsBySource_internal: sourceId=${sourceId}, graphIndex=${!!this.graphIndex}, isInitialized=${this.graphIndex?.isInitialized}`) // Fast path - use GraphAdjacencyIndex if available (lazy-loaded). // 8.0 BigInt boundary: convert the UUID to an entity int up front and // resolve returned verb ints back to verb-id strings. - // Honest gate: a provider that exposes isReady() and reports not-ready - // (count/manifest loaded but source→target edges NOT) is SKIPPED so we fall - // to the correct-but-slower canonical shard scan below instead of a silent []. - if ( - this.graphIndex && - this.graphIndex.isInitialized && - this.graphEntityIdResolver && - assessIndexReadiness(this.graphIndex) !== 'not-ready' - ) { + if (this.graphIndex && this.graphIndex.isInitialized && this.graphEntityIdResolver) { try { const sourceInt = this.graphEntityIdResolver.getInt(sourceId) if (sourceInt === undefined) { @@ -4950,19 +4247,11 @@ export abstract class BaseStorage extends BaseStorageAdapter { targetId: string ): Promise { await this.ensureInitialized() - await this.ensureGraphFastPathProbed() // Fast path - use GraphAdjacencyIndex if available (lazy-loaded). // 8.0 BigInt boundary: convert the UUID to an entity int up front and // resolve returned verb ints back to verb-id strings. - // Honest gate: a not-ready provider (count loaded but edges NOT) is SKIPPED so - // we fall to the correct-but-slower canonical shard scan instead of a silent []. - if ( - this.graphIndex && - this.graphIndex.isInitialized && - this.graphEntityIdResolver && - assessIndexReadiness(this.graphIndex) !== 'not-ready' - ) { + if (this.graphIndex && this.graphIndex.isInitialized && this.graphEntityIdResolver) { try { const targetInt = this.graphEntityIdResolver.getInt(targetId) if (targetInt === undefined) { diff --git a/src/storage/blobStorage.ts b/src/storage/blobStorage.ts index f83511f9..4d98c0bc 100644 --- a/src/storage/blobStorage.ts +++ b/src/storage/blobStorage.ts @@ -15,7 +15,6 @@ import { createHash } from 'crypto' import { unwrapBinaryData } from './binaryDataCodec.js' -import { InMemoryMutex } from '../utils/mutex.js' /** * @description Key-value bridge the blob store persists through. Implemented @@ -48,19 +47,8 @@ export interface BlobMetadata { compression: 'none' | 'zstd' /** Creation timestamp (epoch ms). */ createdAt: number - /** Number of LIVE logical references to this blob (deduplicated writes). */ + /** Number of logical references to this blob (deduplicated writes). */ refCount: number - /** - * Number of persisted generation record-sets (Model-B before-images) that - * reference this hash — the blob's membership in the temporal history. - * Bytes are physically reclaimed only when BOTH counts are zero, and only - * by history compaction: live references protect the present, history - * references protect every `asOf` read inside the retention window (pins - * ride generation pinning, which compaction already respects). Absent on - * metas written before the temporal contract existed (treated as 0; the - * one-time open-time backfill makes legacy stores exact). - */ - historyRefCount?: number } /** @@ -159,9 +147,7 @@ interface CacheEntry { * @example * const hash = await blobStorage.write(buffer, { mimeType: 'image/png' }) * const bytes = await blobStorage.read(hash) // verified against the hash - * await blobStorage.release(hash) // drop one LIVE reference - * // bytes are physically reclaimed only by history compaction, once no - * // live reference AND no in-window generation references the hash + * await blobStorage.delete(hash) // decrements refCount first */ export class BlobStorage { private adapter: BlobStoreAdapter @@ -178,19 +164,6 @@ export class BlobStorage { private readonly CACHE_MAX_SIZE = 100 * 1024 * 1024 // 100MB default private readonly COMPRESSION_THRESHOLD = 1024 // 1KB - don't compress smaller - /** - * Per-hash write serialization. Every reference-count-bearing mutation - * (`write`'s dedup check-then-act, `delete`'s decrement-then-maybe-remove) - * is a read-modify-write over `blob-meta:` — unserialized, two - * concurrent writes of identical content both saw "absent" and both wrote - * `refCount: 1` (one reference lost → a later delete removed bytes another - * file still referenced), and concurrent increments/decrements could drop - * counts. Keyed by hash, so distinct content never contends; the process - * is the whole concurrency domain (storage enforces single-writer per - * directory). - */ - private readonly hashLocks = new InMemoryMutex() - /** * @param adapter - Key-value bridge to persist through. * @param options - `cacheMaxSize` bounds the LRU read cache (bytes, @@ -249,53 +222,45 @@ export class BlobStorage { async write(data: Buffer, options: BlobWriteOptions = {}): Promise { const hash = BlobStorage.hash(data) - // The dedup decision (exists → add a reference; absent → create with - // refCount 1) is check-then-act over the same metadata a concurrent - // same-content write mutates — serialized per hash so N concurrent - // writes of identical content yield exactly N references, never a lost - // count (a lost reference turns a later delete into premature removal - // of bytes another file still needs). - return this.hashLocks.runExclusive(hash, async () => { - // Deduplication: identical content already stored — just add a reference. - if (await this.has(hash)) { - await this.incrementRefCount(hash) - return hash - } - - await this.ensureCompressionReady() - - // Determine compression strategy - const compression = this.selectCompression(data, options) - - // Compress if needed - let finalData = data - let compressedSize = data.length - - if (compression === 'zstd' && this.zstdCompress) { - finalData = await this.zstdCompress(data) - compressedSize = finalData.length - } - - // Record the ACTUAL compression state, not the intended one — prevents - // corruption if compression failed to initialize. - const actualCompression = finalData === data ? 'none' : compression - const metadata: BlobMetadata = { - hash, - size: data.length, - compressedSize, - compression: actualCompression, - createdAt: Date.now(), - refCount: 1 - } - - await this.adapter.put(`blob:${hash}`, finalData) - await this.adapter.put(`blob-meta:${hash}`, Buffer.from(JSON.stringify(metadata))) - - // Write-through cache (caches the ORIGINAL bytes, not the compressed form) - this.addToCache(hash, data, metadata) - + // Deduplication: identical content already stored — just add a reference. + if (await this.has(hash)) { + await this.incrementRefCount(hash) return hash - }) + } + + await this.ensureCompressionReady() + + // Determine compression strategy + const compression = this.selectCompression(data, options) + + // Compress if needed + let finalData = data + let compressedSize = data.length + + if (compression === 'zstd' && this.zstdCompress) { + finalData = await this.zstdCompress(data) + compressedSize = finalData.length + } + + // Record the ACTUAL compression state, not the intended one — prevents + // corruption if compression failed to initialize. + const actualCompression = finalData === data ? 'none' : compression + const metadata: BlobMetadata = { + hash, + size: data.length, + compressedSize, + compression: actualCompression, + createdAt: Date.now(), + refCount: 1 + } + + await this.adapter.put(`blob:${hash}`, finalData) + await this.adapter.put(`blob-meta:${hash}`, Buffer.from(JSON.stringify(metadata))) + + // Write-through cache (caches the ORIGINAL bytes, not the compressed form) + this.addToCache(hash, data, metadata) + + return hash } /** @@ -369,105 +334,23 @@ export class BlobStorage { } /** - * @description Drop one LIVE reference to the blob. Never deletes bytes — - * blob content is immutable under the temporal model, exactly like every - * other record: a past generation's `asOf` read may still need these bytes - * even when no live file references them. Physical reclamation happens in - * ONE place only — history compaction via {@link reclaimIfUnreferenced}, - * once no live reference AND no retained generation references the hash. + * @description Drop one reference to the blob. The stored bytes and + * metadata are physically deleted only when the reference count reaches + * zero — deduplicated content shared by other writers survives. * * @param hash - The blob's SHA-256 hash. */ - async release(hash: string): Promise { - await this.hashLocks.runExclusive(hash, async () => { - await this.decrementRefCount(hash) - }) - } + async delete(hash: string): Promise { + const refCount = await this.decrementRefCount(hash) - /** - * @description Record that one persisted generation record-set references - * this hash (called by the commit path BEFORE the record-set is written — - * a crash between the two can only over-count, which leaks until the scrub - * recounts; it can never under-count, which would risk premature deletion). - * A missing meta (bytes never stored or already gone) is skipped with a - * warning — counting it could not make its bytes readable. - * @param hash - The blob's SHA-256 hash. - */ - async recordHistoryReference(hash: string): Promise { - await this.hashLocks.runExclusive(hash, async () => { - const metadata = await this.getMetadata(hash) - if (!metadata) { - console.warn( - `[BlobStorage] history reference recorded for absent blob ${hash} — skipped` - ) - return - } - metadata.historyRefCount = (metadata.historyRefCount ?? 0) + 1 - await this.adapter.put(`blob-meta:${hash}`, Buffer.from(JSON.stringify(metadata))) - }) - } + // Only delete if no references remain + if (refCount > 0) { + return + } - /** - * @description Drop one history reference (called by compaction AFTER the - * referencing generation record-set is deleted — the safe ordering: a crash - * between the two over-counts, never under-counts). Floored at zero. - * @param hash - The blob's SHA-256 hash. - */ - async releaseHistoryReference(hash: string): Promise { - await this.hashLocks.runExclusive(hash, async () => { - const metadata = await this.getMetadata(hash) - if (!metadata) return - metadata.historyRefCount = Math.max(0, (metadata.historyRefCount ?? 0) - 1) - await this.adapter.put(`blob-meta:${hash}`, Buffer.from(JSON.stringify(metadata))) - }) - } - - /** - * @description Physically delete the blob's bytes + metadata IFF nothing - * references it: zero live references AND zero history references. The one - * reclamation point in the system, invoked by history compaction after it - * releases the reclaimed generations' references. Atomic per hash. - * @param hash - The blob's SHA-256 hash. - * @returns `true` when the bytes were reclaimed. - */ - async reclaimIfUnreferenced(hash: string): Promise { - return this.hashLocks.runExclusive(hash, async () => { - const metadata = await this.getMetadata(hash) - if (!metadata) return false - if ((metadata.refCount ?? 0) > 0 || (metadata.historyRefCount ?? 0) > 0) { - return false - } - await this.adapter.delete(`blob:${hash}`) - await this.adapter.delete(`blob-meta:${hash}`) - this.removeFromCache(hash) - return true - }) - } - - /** - * @description Set the history reference count to an absolute value — the - * backfill/scrub primitive (recounts derived from the actual generation - * records replace whatever the incremental counters hold). Idempotent. - * @param hash - The blob's SHA-256 hash. - * @param count - The exact history reference count. - */ - async setHistoryRefCount(hash: string, count: number): Promise { - await this.hashLocks.runExclusive(hash, async () => { - const metadata = await this.getMetadata(hash) - if (!metadata) return - metadata.historyRefCount = Math.max(0, count) - await this.adapter.put(`blob-meta:${hash}`, Buffer.from(JSON.stringify(metadata))) - }) - } - - /** - * @description Enumerate every stored blob hash (from the metadata keys) — - * the backfill/scrub walk. O(stored blobs). - * @returns All hashes with a stored metadata record. - */ - async listHashes(): Promise { - const keys = await this.adapter.list('blob-meta:') - return keys.map((k) => k.slice('blob-meta:'.length)) + await this.adapter.delete(`blob:${hash}`) + await this.adapter.delete(`blob-meta:${hash}`) + this.removeFromCache(hash) } /** @@ -520,8 +403,6 @@ export class BlobStorage { /** * Increment the reference count for an existing blob. - * Caller MUST hold the per-hash lock ({@link hashLocks}) — this is a raw - * read-modify-write with no serialization of its own. */ private async incrementRefCount(hash: string): Promise { const metadata = await this.getMetadata(hash) @@ -536,8 +417,6 @@ export class BlobStorage { /** * Decrement the reference count for a blob (floored at zero). - * Caller MUST hold the per-hash lock ({@link hashLocks}) — this is a raw - * read-modify-write with no serialization of its own. */ private async decrementRefCount(hash: string): Promise { const metadata = await this.getMetadata(hash) diff --git a/src/transaction/Transaction.ts b/src/transaction/Transaction.ts index 092a2a25..ff19b571 100644 --- a/src/transaction/Transaction.ts +++ b/src/transaction/Transaction.ts @@ -37,24 +37,6 @@ const DEFAULT_OPTIONS: Required = { maxRollbackRetries: 3 } -/** - * The apply-phase budget for a batch of `opCount` operations. - * - * An explicit override wins untouched. Otherwise the budget SCALES with the - * batch: `max(30 000 ms, opCount × 2 000 ms)`. The per-op term is calibrated - * from field data — bulk imports on network-attached disks measure ~2 s per - * operation (each op pays canonical writes + fsync + index maintenance) — so - * a flat 30 s budget silently capped honest work at ~15 operations while - * looking generous for small batches. Scaling keeps small transacts - * fast-failing and gives bulk ones a budget proportional to the work they - * actually asked for; a trip still rolls back atomically and throws a - * retryable, fully-labeled TransactionTimeoutError. - */ -export function transactTimeoutBudget(opCount: number, override?: number): number { - if (override !== undefined) return override - return Math.max(30_000, opCount * 2_000) -} - /** * Transaction class */ @@ -116,64 +98,49 @@ export class Transaction implements TransactionContext { } try { - // Execute each operation in order. This loop is the sole rollback-guarded - // region: ANY error that escapes it — an operation failure OR a - // mid-flight timeout — is caught below and rolls back every operation - // applied so far, in reverse order. That single guarantee is the - // transaction's atomicity contract, and the generation-store commit path - // depends on it: its abort cleanup assumes a throw from here already - // restored the applied operations byte-identically (it only discards the - // uncommitted staging directory). A rollback per exit path — the previous - // design — let the timeout throw slip past rollback and strand the - // already-applied writes as torn, generation-less state in canonical - // storage. + // Execute each operation in order for (let i = 0; i < this.operations.length; i++) { - // Budget check BEFORE starting the next operation. A trip here throws - // into the catch below and rolls back like any other failure — it must - // never bypass rollback. + // Check timeout if (Date.now() - this.startTime > this.options.timeout) { - throw new TransactionTimeoutError(this.options.timeout, i, { - elapsedMs: Date.now() - this.startTime, - totalOperations: this.operations.length, - operationName: this.operations[i]?.name - }) + throw new TransactionTimeoutError(this.options.timeout, i) } const operation = this.operations[i] - if (this.options.logging) { - prodLog.info(`[Transaction] Executing operation ${i}: ${operation.name || 'unnamed'}`) - } - - let rollbackAction: RollbackAction | undefined try { - rollbackAction = await operation.execute() + if (this.options.logging) { + prodLog.info(`[Transaction] Executing operation ${i}: ${operation.name || 'unnamed'}`) + } + + // Execute operation + const rollbackAction = await operation.execute() + + // Record rollback action (if provided) + if (rollbackAction) { + this.rollbackActions.push(rollbackAction) + } + } catch (error) { - // Normalize an operation failure to a TransactionExecutionError; the - // outer catch performs the (single) rollback and surfaces it. - throw new TransactionExecutionError( + // Operation failed - rollback and re-throw + const executionError = new TransactionExecutionError( `Operation ${i} failed: ${(error as Error).message}`, i, operation.name, error as Error ) - } - // Record rollback action (if the operation provided one) - if (rollbackAction) { - this.rollbackActions.push(rollbackAction) + await this.rollback(executionError) + throw executionError } } + + // All operations succeeded - commit + this.commit() + } catch (error) { - // Single rollback point: undo everything applied so far, in reverse - // order, then surface the original error. A rollback failure supersedes - // it (rollback() throws TransactionRollbackError wrapping this error). - await this.rollback(error as Error) + // Error already handled in rollback throw error } - - // All operations succeeded — commit. - this.commit() } /** @@ -241,22 +208,15 @@ export class Transaction implements TransactionContext { } } - // Honest terminal state: only claim 'rolled_back' when EVERY undo applied. - // If any undo failed, the store is not cleanly rolled back — mark - // 'inconsistent' so the commit orchestration reconciles rather than trusts - // a false 'rolled_back'. (Fixes the state half of the post-commit response - // lie: the record whose undo failed is still durable.) - this.state = rollbackErrors.length > 0 ? 'inconsistent' : 'rolled_back' + this.state = 'rolled_back' this.endTime = Date.now() if (this.options.logging) { const duration = this.endTime - (this.startTime || this.endTime) - prodLog.info(`[Transaction] ${this.state === 'inconsistent' ? 'Rollback INCOMPLETE' : 'Rolled back'} in ${duration}ms`) + prodLog.info(`[Transaction] Rolled back in ${duration}ms`) } - // If rollback encountered errors, wrap them with original error. The - // orchestration keys on `instanceof TransactionRollbackError` to know the - // undo did not fully apply and reconciliation is required. + // If rollback encountered errors, wrap them with original error if (rollbackErrors.length > 0) { throw new TransactionRollbackError( `Transaction rollback encountered ${rollbackErrors.length} errors during cleanup`, diff --git a/src/transaction/errors.ts b/src/transaction/errors.ts index c270d0ed..c9f6eee1 100644 --- a/src/transaction/errors.ts +++ b/src/transaction/errors.ts @@ -77,24 +77,11 @@ export class InvalidTransactionStateError extends TransactionError { export class TransactionTimeoutError extends TransactionError { constructor( timeoutMs: number, - operationIndex: number, - telemetry?: { - elapsedMs?: number - totalOperations?: number - operationName?: string - } + operationIndex: number ) { - const progress = - telemetry?.totalOperations !== undefined - ? `${operationIndex}/${telemetry.totalOperations}` - : String(operationIndex) - const name = telemetry?.operationName ? ` ('${telemetry.operationName}')` : '' - const elapsed = - telemetry?.elapsedMs !== undefined ? `${telemetry.elapsedMs}ms elapsed, ` : '' super( - `Transaction timed out at operation ${progress}${name} — ${elapsed}budget ${timeoutMs}ms. ` + - `The batch rolled back atomically; retry with a higher timeoutMs or a smaller batch.`, - { timeoutMs, operationIndex, ...telemetry } + `Transaction timed out after ${timeoutMs}ms at operation ${operationIndex}`, + { timeoutMs, operationIndex } ) this.name = 'TransactionTimeoutError' } diff --git a/src/transaction/operations/IndexOperations.ts b/src/transaction/operations/IndexOperations.ts index 97c39270..f0bede05 100644 --- a/src/transaction/operations/IndexOperations.ts +++ b/src/transaction/operations/IndexOperations.ts @@ -176,36 +176,14 @@ export class RemoveFromMetadataIndexOperation implements Operation { * generation is reused for the rollback removal, so an add and its undo * reference one watermark in a provider's per-generation edge chain. */ -/** - * A verb's interned endpoint ints — eager (already resolved), or a thunk - * evaluated when the operation EXECUTES. The lazy form exists for - * `transact()` forward references: a relate whose endpoint is added in the - * SAME batch cannot resolve ints at plan time (the entity does not exist yet - * — a strict native id mapper rightly refuses to assign, and even a permissive - * one would leak the assignment if the batch is rejected at precommit). - * Deferring to execute time resolves after the batch's add operations have - * applied, mirroring how `generationFn` is already evaluated lazily. - */ -export type VerbEndpointInts = - | { sourceInt: bigint; targetInt: bigint } - | (() => { sourceInt: bigint; targetInt: bigint }) - -/** Resolve a {@link VerbEndpointInts} at execution time. */ -function resolveEndpointInts( - endpoints: VerbEndpointInts -): { sourceInt: bigint; targetInt: bigint } { - return typeof endpoints === 'function' ? endpoints() : endpoints -} - export class AddToGraphIndexOperation implements Operation { readonly name = 'AddToGraphIndex' /** * @param index - The graph-index provider (JS baseline or native). * @param verb - The verb to index (`sourceInt`/`targetInt` mirrored on it). - * @param endpointInts - The endpoints' interned ints — eager, or a thunk - * evaluated at execute time (REQUIRED for transact forward references; - * see {@link VerbEndpointInts}). + * @param sourceInt - The source entity's interned int. + * @param targetInt - The target entity's interned int. * @param generationFn - Resolves the commit generation to stamp this edge at, * evaluated when the operation executes (see class note). * @param onVerbInt - Optional hook invoked with the interned verb int @@ -214,18 +192,17 @@ export class AddToGraphIndexOperation implements Operation { constructor( private readonly index: GraphIndexProvider, private readonly verb: GraphVerb, - private readonly endpointInts: VerbEndpointInts, + private readonly sourceInt: bigint, + private readonly targetInt: bigint, private readonly generationFn: () => bigint, private readonly onVerbInt?: (verbInt: bigint) => void ) {} async execute(): Promise { // Stamp this edge at the in-flight commit generation; reuse it for the - // rollback so add + undo reference the same watermark. Endpoint ints - // resolve HERE — after any same-batch adds have applied. + // rollback so add + undo reference the same watermark. const generation = this.generationFn() - const { sourceInt, targetInt } = resolveEndpointInts(this.endpointInts) - const verbInt = await this.index.addVerb(this.verb, sourceInt, targetInt, generation) + const verbInt = await this.index.addVerb(this.verb, this.sourceInt, this.targetInt, generation) this.onVerbInt?.(verbInt) // Return rollback action @@ -254,32 +231,28 @@ export class RemoveFromGraphIndexOperation implements Operation { /** * @param index - The graph-index provider (JS baseline or native). * @param verb - The verb being removed (required for rollback re-add). - * @param endpointInts - The endpoints' interned ints for the rollback - * re-add — eager, or a thunk evaluated at execute time (required when the - * verb or its endpoints were created in the SAME transact batch; see - * {@link VerbEndpointInts}). + * @param sourceInt - The source entity's interned int (rollback re-add). + * @param targetInt - The target entity's interned int (rollback re-add). * @param generationFn - Resolves the commit generation for this removal, * evaluated when the operation executes. */ constructor( private readonly index: GraphIndexProvider, private readonly verb: GraphVerb, // Required for rollback - private readonly endpointInts: VerbEndpointInts, + private readonly sourceInt: bigint, + private readonly targetInt: bigint, private readonly generationFn: () => bigint ) {} async execute(): Promise { // Resolve the removal generation once; reuse it for the rollback re-add. - // Endpoint ints resolve HERE (after any same-batch adds applied) and are - // captured for the rollback, whose re-add must use the same mappings. const generation = this.generationFn() - const { sourceInt, targetInt } = resolveEndpointInts(this.endpointInts) await this.index.removeVerb(this.verb.id, generation) // Return rollback action return async () => { // Re-add verb with original data - await this.index.addVerb(this.verb, sourceInt, targetInt, generation) + await this.index.addVerb(this.verb, this.sourceInt, this.targetInt, generation) } } } diff --git a/src/transaction/operations/StorageOperations.ts b/src/transaction/operations/StorageOperations.ts index 316f1ac0..ca513cd4 100644 --- a/src/transaction/operations/StorageOperations.ts +++ b/src/transaction/operations/StorageOperations.ts @@ -104,72 +104,35 @@ export class SaveNounOperation implements Operation { } /** - * Delete a noun — FULL canonical removal, with rollback support. - * - * Despite the historical name, this removes the WHOLE entity: both canonical - * legs (metadata + vector) AND the entity's container. Previously it deleted - * only the metadata leg via `deleteNounMetadata`, leaving the canonical - * `vectors.json` leg and the `/` directory orphaned on disk — a "ghost" - * that reads as absent (getNoun needs both legs) yet inflates the enumerated - * count and can never be told apart from a damage scar. Routing through - * `storage.deleteNoun` removes both legs + the container in one place. - * - * Immutability is preserved: this cleans only the live-HEAD projection; the - * generation store retains the before-image so `asOf()` still reconstructs the - * deleted entity until retention expires. + * Delete noun metadata with rollback support * * Rollback strategy: - * - Restore BOTH legs from the before-image (vector leg raw, metadata leg via - * the count-aware save so deleteNoun()'s decrement is reversed). + * - Restore deleted metadata */ export class DeleteNounMetadataOperation implements Operation { - readonly name = 'DeleteNoun' + readonly name = 'DeleteNounMetadata' constructor( private readonly storage: StorageAdapter, - private readonly id: string, - /** - * OPTIONAL already-known metadata of the entity being removed (the caller's - * pre-delete read). Removal must never REQUIRE re-reading the thing being - * removed: if the reads here return null (replace race, or a ghost left by - * an earlier version), the count decrement downstream falls back to this - * record instead of being silently skipped — the skip minted permanent - * counter inflation (adds counted, paired removals not decremented). - */ - private readonly priorMetadata?: NounMetadata | null + private readonly id: string ) {} async execute(): Promise { - // Capture the FULL before-image (both legs) so the undo restores the whole - // entity — a metadata-only rollback would leave the vector leg unrestored. - // A null metadata read falls back to the caller's pre-delete read. - const previousNoun = await this.storage.getNoun(this.id) - const previousMetadata = (await this.storage.getNounMetadata(this.id)) ?? this.priorMetadata ?? null + // Get metadata before deletion (for rollback) + const previousMetadata = await this.storage.getNounMetadata(this.id) - if (!previousNoun && !previousMetadata) { + if (!previousMetadata) { // Nothing to delete - no rollback needed return async () => {} } - // Full removal: both canonical legs + the entity container + count decrement - // (the prior record keeps the decrement honest on a null canonical read). - await this.storage.deleteNoun(this.id, previousMetadata) + // Delete metadata + await this.storage.deleteNounMetadata(this.id) // Return rollback action return async () => { - // Restore the vector leg, then the metadata leg through the count-aware - // save so deleteNoun()'s decrement is reversed. - if (previousNoun) { - await this.storage.saveNoun({ - id: previousNoun.id, - vector: previousNoun.vector, - connections: previousNoun.connections || new Map(), - level: previousNoun.level || 0 - }) - } - if (previousMetadata) { - await this.storage.saveNounMetadata(this.id, previousMetadata) - } + // Restore deleted metadata + await this.storage.saveNounMetadata(this.id, previousMetadata) } } } @@ -279,9 +242,9 @@ export class DeleteVerbMetadataOperation implements Operation { return async () => {} } - // Delete verb (metadata + vector). The pre-read rides along so the count - // decrement never depends on re-reading the record being removed. - await this.storage.deleteVerb(this.id, previousMetadata) + // Delete verb (metadata + vector) + // Note: StorageAdapter has deleteVerb but not deleteVerbMetadata + await this.storage.deleteVerb(this.id) // Return rollback action return async () => { diff --git a/src/transaction/types.ts b/src/transaction/types.ts index 9a3a2eaa..5917ea4c 100644 --- a/src/transaction/types.ts +++ b/src/transaction/types.ts @@ -14,10 +14,6 @@ export type TransactionState = | 'committed' // Successfully committed | 'rolling_back' // Rolling back due to failure | 'rolled_back' // Successfully rolled back - | 'inconsistent' // Rollback ran but ≥1 undo could not be applied — the store - // is NOT cleanly rolled back; the commit orchestration must - // reconcile (adopt-forward or fail-loud). Never claim - // 'rolled_back' when an undo failed. /** * Rollback action - undoes an operation diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index 1ec4c4c3..c2344375 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -1737,10 +1737,7 @@ export interface BrainyConfig { * Under Model-B EVERY write (`transact()` AND single-op `add`/`update`/ * `remove`/`relate`) produces an immutable generation record-set serving * historical reads (`asOf()`, pinned `Db` values). Without compaction those - * accumulate, so Brainy **auto-compacts at `close()`** (time-bounded per - * pass; 8.9.0 removed compaction from `flush()` — flush is durability work - * and never pays maintenance costs). A long-lived writer that never closes - * accumulates history until its next explicit `compactHistory()` call. + * accumulate, so Brainy **auto-compacts on every `flush()` and `close()`**. * Live `Db` pins are ALWAYS exempt from reclamation, in every mode. * * Modes: @@ -1757,7 +1754,7 @@ export interface BrainyConfig { * the oldest unpinned generations while ANY supplied cap is exceeded * (predictable ops). `maxAge` in ms; `maxBytes` total history bytes. * - * `autoCompact: false` disables the automatic close() compaction (manage + * `autoCompact: false` disables the automatic flush/close compaction (manage * manually via `brain.compactHistory()`). `budgetBytes` is the settable * adaptive byte budget a coordinator drives (also via `brain.setRetentionBudget()`). * Long-term archives belong in `db.persist(path)` snapshots, which compaction @@ -1775,7 +1772,7 @@ export interface BrainyConfig { maxBytes?: number /** Adaptive byte budget for this brain, driven by a coordinator (e.g. cor). */ budgetBytes?: number - /** Run compaction automatically at close() (default: true; 8.9.0 — flush() never compacts). */ + /** Run compaction automatically on flush()/close() (default: true). */ autoCompact?: boolean } diff --git a/src/utils/crc32c.ts b/src/utils/crc32c.ts deleted file mode 100644 index 0c2a1c14..00000000 --- a/src/utils/crc32c.ts +++ /dev/null @@ -1,43 +0,0 @@ -/** - * @module utils/crc32c - * @description CRC-32C (Castagnoli, polynomial 0x1EDC6F41, reflected 0x82F63B78) - * — the storage-industry frame checksum (ext4, iSCSI, SCTP, LSM segment files). - * Used to frame generation-fact segments: every appended record carries the - * CRC-32C of its payload, so a torn tail (crash mid-append) or bit rot is - * DETECTED at scan time and never silently read as data. - * - * Table-driven, dependency-free reference implementation. Native providers may - * substitute a hardware-accelerated (SSE4.2 / ARMv8 CRC) implementation — the - * polynomial is the contract, byte-identical results required. - */ - -/** The 256-entry lookup table for the reflected CRC-32C polynomial. */ -const TABLE: Uint32Array = (() => { - const table = new Uint32Array(256) - for (let n = 0; n < 256; n++) { - let c = n - for (let k = 0; k < 8; k++) { - c = c & 1 ? 0x82f63b78 ^ (c >>> 1) : c >>> 1 - } - table[n] = c >>> 0 - } - return table -})() - -/** - * Compute the CRC-32C checksum of a byte buffer. - * - * Known-answer vectors (RFC 3720 appendix / the standard test suite): - * - ASCII "123456789" → 0xE3069283 - * - 32 zero bytes → 0x8A9136AA - * - * @param bytes - The payload to checksum. - * @returns The CRC-32C as an unsigned 32-bit integer. - */ -export function crc32c(bytes: Uint8Array): number { - let crc = 0xffffffff - for (let i = 0; i < bytes.length; i++) { - crc = TABLE[(crc ^ bytes[i]) & 0xff] ^ (crc >>> 8) - } - return (crc ^ 0xffffffff) >>> 0 -} diff --git a/src/utils/errorClassification.ts b/src/utils/errorClassification.ts deleted file mode 100644 index a3e7dad8..00000000 --- a/src/utils/errorClassification.ts +++ /dev/null @@ -1,37 +0,0 @@ -/** - * @module utils/errorClassification - * @description Shared classification of caught errors into "genuine absence" vs - * "real fault" — the antidote to blind `catch { return null }` handlers that - * cannot tell ENOENT (the object is legitimately not on disk) from EIO / EACCES - * / EMFILE / … (a transient or permission fault on data that IS on disk). - * Masking a fault as absence yields wrong results (a present record read as - * "not found") or a needless rebuild. Mandate: loud errors, never quiet losses. - */ - -/** - * The error `code`s that denote GENUINE absence of a file/object. Only `ENOENT` - * ("no such file or directory") qualifies — on every platform Node maps a - * missing file/directory to ENOENT, and no other errno means "simply not - * there". Every other errno (EIO, EACCES, EPERM, EMFILE, ENFILE, EBUSY, - * ENOTDIR, EISDIR, ELOOP) is a real fault and must propagate, as must any error - * without an errno `code` (parse/decompress failures, generic Errors). - * `ENOTDIR`/`EISDIR` are deliberately faults: a path component of the wrong - * type is corruption, not benign absence. Named constant so a future - * genuine-absence code can be added in one reviewed place. - */ -const ABSENCE_CODES: ReadonlySet = new Set(['ENOENT']) - -/** - * @description True IFF `e` represents genuine absence (an ENOENT-class errno), - * for which returning `null`/`[]`/`undefined` is the correct answer. Returns - * `false` for every real fault, so the canonical call site is: - * `catch (e) { if (isAbsentError(e)) return null; throw e }`. - * - * @param e - The caught value (typed `unknown`; non-objects are never absence). - * @returns Whether the error means "the thing is simply not there". - */ -export function isAbsentError(e: unknown): boolean { - if (e === null || typeof e !== 'object') return false - const code = (e as { code?: unknown }).code - return typeof code === 'string' && ABSENCE_CODES.has(code) -} diff --git a/src/utils/indexReadiness.ts b/src/utils/indexReadiness.ts deleted file mode 100644 index 16266bec..00000000 --- a/src/utils/indexReadiness.ts +++ /dev/null @@ -1,38 +0,0 @@ -/** - * @module indexReadiness - * @description The single honest-readiness classifier shared by the vector, - * graph and metadata index sites. It exists to kill "Pattern A" — the dishonest - * readiness proxy where `size() > 0` / `isInitialized` is treated as "this index - * actually serves queries." A cold native index that loaded its COUNT but not its - * SERVING structure passes those proxies and silently returns `[]`. - * - * This classifier reads ONLY the provider's OPTIONAL, honest `isReady()` signal - * (see {@link import('../plugin.js').VectorIndexProvider.isReady}, - * {@link import('../plugin.js').GraphIndexProvider.isReady}, - * {@link import('../plugin.js').MetadataIndexProvider.isReady}). It NEVER inspects - * `size()` or `isInitialized`. When `isReady()` is absent, callers must fall back - * to a KNOWN-ITEM PROBE (a real search/lookup that must return a known-present - * datum) before trusting an empty result — never a `size()` proxy. - */ - -/** A provider that MAY expose the honest cold-load readiness signal. */ -export interface MaybeReadyProvider { - isReady?: () => boolean -} - -/** Three-valued honest-readiness verdict. */ -export type IndexReadiness = 'ready' | 'not-ready' | 'unknown' - -/** - * @description Classify an index provider's honest readiness. - * @param provider - Any index provider (vector / graph / metadata) or `null`. - * @returns - * - `'ready'` when `isReady() === true` (serving structure loaded — trust it); - * - `'not-ready'` when `isReady() === false` (count/manifest loaded, NOT serving — rebuild); - * - `'unknown'` when the provider exposes no `isReady()` (caller must probe / keep the JS heuristic). - */ -export function assessIndexReadiness(provider: unknown): IndexReadiness { - const p = provider as MaybeReadyProvider | null | undefined - if (p == null || typeof p.isReady !== 'function') return 'unknown' - return p.isReady() ? 'ready' : 'not-ready' -} diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index c772bce4..fd325319 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -1867,26 +1867,9 @@ export class MetadataIndexManager implements MetadataIndexProvider { // not once per AND-clause inside it. const unindexedFields: string[] = [] - for (const [rawField, condition] of Object.entries(filter)) { + for (const [field, condition] of Object.entries(filter)) { // Skip logical operators - if (rawField === 'allOf' || rawField === 'anyOf' || rawField === 'not') continue - - // Metadata is FLATTENED at index time (metadata.entry.title indexes as - // entry.title), so a `metadata.`-prefixed where key is almost always - // the caller spelling the STORAGE shape rather than the index shape. - // Accept both spellings: when the key as spelled is unindexed but its - // stripped spelling is, query the stripped one. A literal nested - // custom key named `metadata` still wins when indexed as spelled - // (checked first), so that rare shape keeps working. - let field = rawField - if ( - rawField.startsWith('metadata.') && - this.columnStore && - !this.columnStore.hasField(rawField) && - this.columnStore.hasField(rawField.slice('metadata.'.length)) - ) { - field = rawField.slice('metadata.'.length) - } + if (field === 'allOf' || field === 'anyOf' || field === 'not') continue let fieldResults: string[] = [] diff --git a/src/utils/osLimits.ts b/src/utils/osLimits.ts deleted file mode 100644 index 57e1ccf1..00000000 --- a/src/utils/osLimits.ts +++ /dev/null @@ -1,141 +0,0 @@ -/** - * @module utils/osLimits - * @description Detect-and-warn for OS resource limits that bite at POOL scale. - * - * A single brain rarely notices them, but a pool of brains — especially with a - * native accelerator memory-mapping many index files per brain — consumes file - * descriptors and memory mappings multiplicatively. On stock Linux defaults - * (RLIMIT_NOFILE soft 1024, vm.max_map_count 65530) the failure arrives as - * EMFILE or a failed mmap deep inside an index open, long after the real cause - * (the limit) stopped being visible. This module reads the limits at open and - * WARNS ONCE per process with the exact raise commands, so the operator learns - * the fix before the incident instead of from it. - * - * Read-only and Linux-only by construction: both sources are `/proc` files. - * On platforms where they are absent the check reports nulls and stays silent — - * no limit read means no claim made, never a guessed warning. - */ - -import * as fs from 'node:fs' -import { prodLog } from './logger.js' - -/** Soft-NOFILE floor below which pool-scale use is at EMFILE risk. */ -export const NOFILE_POOL_FLOOR = 65536 - -/** vm.max_map_count floor below which mmap-heavy native indexes are at risk. */ -export const MAX_MAP_COUNT_POOL_FLOOR = 262144 - -export interface OsLimitsReport { - /** RLIMIT_NOFILE soft limit (null when unreadable; Infinity for 'unlimited'). */ - nofileSoft: number | null - /** RLIMIT_NOFILE hard limit (null when unreadable; Infinity for 'unlimited'). */ - nofileHard: number | null - /** vm.max_map_count (null when unreadable). */ - maxMapCount: number | null - /** Human-actionable warnings for limits below the pool floors. Empty = fine. */ - warnings: string[] -} - -/** - * Parse the `Max open files` row of a `/proc//limits` document into - * soft/hard values. Returns nulls when the row is absent or malformed. - */ -export function parseProcLimits(content: string): { soft: number | null; hard: number | null } { - const line = content.split('\n').find((l) => l.startsWith('Max open files')) - if (!line) return { soft: null, hard: null } - const m = line.match(/^Max open files\s+(\S+)\s+(\S+)/) - if (!m) return { soft: null, hard: null } - const parse = (v: string): number | null => { - if (v === 'unlimited') return Infinity - const n = Number.parseInt(v, 10) - return Number.isNaN(n) ? null : n - } - return { soft: parse(m[1]), hard: parse(m[2]) } -} - -/** - * Assess readable limits against the pool floors. Pure — feed it any values. - * A null (unreadable) limit produces NO warning: no measurement, no claim. - */ -export function assessOsLimits(limits: { - nofileSoft: number | null - nofileHard: number | null - maxMapCount: number | null -}): string[] { - const warnings: string[] = [] - - if (limits.nofileSoft !== null && limits.nofileSoft < NOFILE_POOL_FLOOR) { - const hardNote = - limits.nofileHard !== null && limits.nofileHard >= NOFILE_POOL_FLOOR - ? ` (the hard limit ${limits.nofileHard === Infinity ? 'unlimited' : limits.nofileHard} already allows it — raise the soft limit only)` - : '' - warnings.push( - `RLIMIT_NOFILE soft limit is ${limits.nofileSoft} — below the ${NOFILE_POOL_FLOOR} recommended ` + - `for pool-scale use (a pool of brains with a native accelerator opens many index files per brain; ` + - `the failure mode is EMFILE deep inside an index open). Raise with \`ulimit -n ${NOFILE_POOL_FLOOR}\` ` + - `or LimitNOFILE=${NOFILE_POOL_FLOOR} in the service unit${hardNote}.` - ) - } - - if (limits.maxMapCount !== null && limits.maxMapCount < MAX_MAP_COUNT_POOL_FLOOR) { - warnings.push( - `vm.max_map_count is ${limits.maxMapCount} — below the ${MAX_MAP_COUNT_POOL_FLOOR} recommended ` + - `for mmap-heavy native indexes at pool scale (each mapped index segment consumes map entries; ` + - `the failure mode is a failed mmap mid-heal). Raise with ` + - `\`sysctl -w vm.max_map_count=${MAX_MAP_COUNT_POOL_FLOOR}\` (persist in /etc/sysctl.d/).` - ) - } - - return warnings -} - -/** - * Read the limits from /proc and assess them. `readFile` is injectable for - * tests; absent/unreadable sources yield nulls (and therefore no warnings). - */ -export async function checkOsLimits( - readFile: (path: string) => Promise = async (p) => fs.promises.readFile(p, 'utf-8') -): Promise { - let nofileSoft: number | null = null - let nofileHard: number | null = null - let maxMapCount: number | null = null - - try { - const parsed = parseProcLimits(await readFile('/proc/self/limits')) - nofileSoft = parsed.soft - nofileHard = parsed.hard - } catch { - // Not Linux (or /proc unavailable) — no measurement, no claim. - } - - try { - const raw = (await readFile('/proc/sys/vm/max_map_count')).trim() - const n = Number.parseInt(raw, 10) - maxMapCount = Number.isNaN(n) ? null : n - } catch { - // Not Linux — same rule. - } - - const warnings = assessOsLimits({ nofileSoft, nofileHard, maxMapCount }) - return { nofileSoft, nofileHard, maxMapCount, warnings } -} - -/** Once-per-process latch so a brain pool warns once, not once per brain. */ -let osLimitsWarned = false - -/** - * Run the check and warn (once per process) about limits below the pool - * floors. Called from brain open; safe everywhere (silent off-Linux). - */ -export async function warnOnLowOsLimits(): Promise { - if (osLimitsWarned) return - osLimitsWarned = true - try { - const report = await checkOsLimits() - for (const warning of report.warnings) { - prodLog.warn(`[Brainy] OS limit check: ${warning}`) - } - } catch { - // The check must never affect open — measurement-only. - } -} diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index ca439524..57a9a1c0 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -209,10 +209,8 @@ export interface ValidationConfigOptions { } /** - * Auto-configured limits based on system resources. - * Derived from memory (explicit overrides > reserved memory > container limit > - * free memory). Query timing is recorded for diagnostics only — it never - * changes the cap (see `recordQuery`). + * Auto-configured limits based on system resources + * These adapt to available memory and observed performance */ export class ValidationConfig { private static instance: ValidationConfig | null = null @@ -325,23 +323,24 @@ export class ValidationConfig { } /** - * Record query timing for diagnostics. Telemetry ONLY — never mutates the cap. - * - * `maxLimit` is a MEMORY-protection bound; query duration says nothing about - * memory-per-result, so duration must never drive it. An earlier version - * "learned" here: while the lifetime-average query time exceeded 1s it shrank - * `maxLimit` by 20% per recorded query down to a floor of 1000 — below the - * documented `MIN_AUTO_QUERY_LIMIT` (10 000) and with no recovery once slow - * samples poisoned the cumulative average. On a production host a burst of - * slow aggregate queries silently strangled every consumer's `find()` to - * 1000 while the error message blamed "available free memory" — exactly the - * silent throttling this module's own contract forbids. The cap now comes - * from its construction-time basis (or explicit overrides) alone. + * Learn from actual usage to adjust limits */ recordQuery(duration: number, resultCount: number) { - void resultCount this.queryCount++ this.avgQueryTime = (this.avgQueryTime * (this.queryCount - 1) + duration) / this.queryCount + + // Only auto-adjust if not using explicit overrides + if (this.limitBasis !== 'override') { + // If queries are consistently fast with large results, increase limits + if (this.avgQueryTime < 100 && resultCount > this.maxLimit * 0.8) { + this.maxLimit = Math.min(this.maxLimit * 1.5, 100000) + } + + // If queries are slow, reduce limits + if (this.avgQueryTime > 1000) { + this.maxLimit = Math.max(this.maxLimit * 0.8, 1000) + } + } } } diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts index 00bddefb..bbf8328e 100644 --- a/src/vfs/VirtualFileSystem.ts +++ b/src/vfs/VirtualFileSystem.ts @@ -36,7 +36,6 @@ import { VFSErrorCode, WriteOptions, ReadOptions, - FileVersion, MkdirOptions, ReaddirOptions, CopyOptions, @@ -407,14 +406,6 @@ export class VirtualFileSystem implements IVirtualFileSystem { async readFile(path: string, options?: ReadOptions): Promise { await this.ensureInitialized() - // Temporal read: the file's exact bytes as of a past generation or - // instant. Materializes the entity from the generation history — content - // blobs referenced by any in-window generation are retention-protected, - // so the bytes are guaranteed present. Bypasses the content cache. - if (options?.asOf !== undefined) { - return this.readFileAt(path, options.asOf, options) - } - // Check cache first if (options?.cache !== false && this.contentCache.has(path)) { const cached = this.contentCache.get(path)! @@ -481,122 +472,6 @@ export class VirtualFileSystem implements IVirtualFileSystem { } } - /** - * @description The temporal read behind `readFile(path, { asOf })`: resolve - * the path's CURRENT entity, materialize its state at the target - * generation/instant from the Model-B history, and read that version's - * content blob (retention-protected, so present for every in-window - * generation). The pinned view is always released so compaction is never - * blocked by a read. - * @param path - The file path (resolved against the live tree). - * @param asOf - A generation number or wall-clock `Date`. - * @param options - `encoding` applies; caching does not (never cached). - * @returns The exact bytes the file held at that generation. - * @throws VFSError ENOENT when the file did not exist at that generation; - * the generation store's compacted-generation error when `asOf` is past - * the retention window's horizon. - */ - private async readFileAt( - path: string, - asOf: number | Date, - options?: ReadOptions - ): Promise { - const entityId = await this.pathResolver.resolve(path) - const db = await this.brain.asOf(asOf) - try { - const entity = await db.get(entityId) - if (!entity) { - throw new VFSError( - VFSErrorCode.ENOENT, - `File did not exist as of ${String(asOf)}: ${path}`, - path, - 'readFile' - ) - } - const storage = (entity.metadata as Record | undefined)?.storage - if (storage?.type !== 'blob' || typeof storage.hash !== 'string') { - throw new VFSError( - VFSErrorCode.EIO, - `File had no blob storage as of ${String(asOf)}: ${path}`, - path, - 'readFile' - ) - } - const content = await this.blobStorage.read(storage.hash) - if (options?.encoding) { - return Buffer.from(content.toString(options.encoding)) - } - return content - } finally { - await db.release() - } - } - - /** - * @description A file's version history within the retention window: one - * entry per generation that wrote the file, ascending — the newest entry is - * the current state. Pair with `readFile(path, { asOf: entry.generation })` - * to fetch any version's exact bytes, and restore by writing those bytes - * back (a NEW write — history is never rewritten). Bounded by the retention - * window: versions whose generations were compacted are not listed. - * @param path - The file path (resolved against the live tree). - * @returns The versions, oldest first. - * @example - * const versions = await brain.vfs.history('/pages/home.json') - * const before = await brain.vfs.readFile('/pages/home.json', { - * asOf: versions[versions.length - 2].generation - * }) - */ - async history(path: string): Promise { - await this.ensureInitialized() - const entityId = await this.pathResolver.resolve(path) - - // Boundary note: reaches Brainy's internal generation store (the same - // bracket-access precedent as the blobStorage getter) — the per-id - // generation chain and horizon are not on the public Brainy surface. - const generationStore = (this.brain as unknown as { - generationStore: { - horizon(): number - generation(): number - generationsTouching( - kind: 'noun' | 'verb', - id: string, - fromGen: number, - toGen: number - ): Promise - } - }).generationStore - - const gens = await generationStore.generationsTouching( - 'noun', - entityId, - generationStore.horizon(), - generationStore.generation() - ) - - const versions: FileVersion[] = [] - for (const gen of gens) { - const db = await this.brain.asOf(gen) - try { - const entity = await db.get(entityId) - const meta = entity?.metadata as Record | undefined - const storage = meta?.storage - if (storage?.type === 'blob' && typeof storage.hash === 'string') { - versions.push({ - generation: gen, - timestamp: db.timestamp, - hash: storage.hash, - size: typeof meta?.size === 'number' ? meta.size : (storage.size ?? 0), - ...(typeof meta?.mimeType === 'string' && { mimeType: meta.mimeType }) - }) - } - } finally { - await db.release() - } - } - return versions - } - /** * Write a file */ @@ -618,18 +493,14 @@ export class VirtualFileSystem implements IVirtualFileSystem { // Ensure parent directory exists const parentId = await this.ensureDirectory(parentPath) - // Check if file already exists (and capture its current content hash so - // the overwrite can release the superseded live reference afterwards). + // Check if file already exists let existingId: string | null = null - let previousHash: string | undefined try { existingId = await this.pathResolver.resolve(path, { cache: false }) // Verify the entity still exists in the brain const existing = await this.brain.get(existingId) if (!existing) { existingId = null // Entity was deleted but cache wasn't cleared - } else if (existing.metadata?.storage?.type === 'blob') { - previousHash = existing.metadata.storage.hash as string | undefined } } catch (err) { // File doesn't exist, which is fine @@ -680,32 +551,14 @@ export class VirtualFileSystem implements IVirtualFileSystem { Object.assign(metadata, await this.extractMetadata(buffer, mimeType)) } - // For embedding: use text content, for storage: use raw data. Computed - // for BOTH branches — an overwrite must refresh the entity's `data` (and - // therefore its embedding), or semantic search and any `data` read keep - // serving the FIRST version's text forever. - const embeddingData = mimeDetector.isTextFile(mimeType) - ? buffer.toString('utf-8') - : `File: ${name} (${mimeType}, ${buffer.length} bytes)` - if (existingId) { - // Update existing file — content bytes live in BlobStorage; `data` - // carries the embedding text and MUST track the new content. + // Update existing file + // No entity.data - content is in BlobStorage await this.brain.update({ id: existingId, - data: embeddingData, metadata }) - // The update committed: drop the superseded live reference. When the - // content is unchanged this cancels the dedup increment the write() - // above just made (net: one live reference per referencing file); when - // it changed, the old bytes stay retention-protected for asOf reads - // via the update's own generation record. - if (previousHash) { - await this.blobStorage.release(previousHash) - } - // Ensure Contains relationship exists (fix for missing relationships) const existingRelations = await this.brain.related({ from: parentId, @@ -725,6 +578,9 @@ export class VirtualFileSystem implements IVirtualFileSystem { } } else { // Create new file entity + // For embedding: use text content, for storage: use raw data + const embeddingData = mimeDetector.isTextFile(mimeType) ? buffer.toString('utf-8') : `File: ${name} (${mimeType}, ${buffer.length} bytes)` + const entity = await this.brain.add({ data: embeddingData, // Always provide string for embeddings type: this.getFileNounType(mimeType), @@ -793,21 +649,14 @@ export class VirtualFileSystem implements IVirtualFileSystem { throw new VFSError(VFSErrorCode.EISDIR, `Is a directory: ${path}`, path, 'unlink') } - // Delete the entity FIRST, then drop its live blob reference — never - // release a reference while the referencing entity might survive (a - // failed remove must not leave a live file whose bytes compaction could - // later reclaim). - await this.brain.remove(entityId) - - // Drop the file's LIVE reference to its content blob. The bytes are NOT - // deleted — the remove's own generation record references this hash, so - // `readFile(path, { asOf })` keeps serving it inside the retention - // window; history compaction reclaims the bytes when the last referencing - // generation is reclaimed. + // Delete blob from BlobStorage (decrements ref count) if (entity.metadata.storage?.type === 'blob') { - await this.blobStorage.release(entity.metadata.storage.hash) + await this.blobStorage.delete(entity.metadata.storage.hash) } + // Delete the entity + await this.brain.remove(entityId) + // Invalidate caches this.pathResolver.invalidatePath(path) this.invalidateCaches(path) @@ -1183,26 +1032,23 @@ export class VirtualFileSystem implements IVirtualFileSystem { // Phase 1: Gather all descendants in ONE batch fetch const descendants = await this.gatherDescendants(entityId, Infinity) - // Phase 2: Batch delete all entities (including root directory) FIRST — - // never release a blob reference while its referencing entity might - // survive a failed delete (see unlink's ordering note). - const allIds = [...descendants.map(d => d.id), entityId] - await this.brain.removeMany({ ids: allIds, continueOnError: false }) - - // Phase 3: Drop the deleted files' LIVE blob references (chunked). - // Live-reference drops only — the bytes stay for in-window asOf reads - // and are reclaimed by history compaction (see unlink's note). + // Phase 2: Parallel blob cleanup (chunked to avoid overwhelming storage) + // Blob deletion is reference-counted, so safe to call for all files const blobFiles = descendants.filter(d => d.metadata.vfsType === 'file' && d.metadata.storage?.type === 'blob' ) - const BLOB_CHUNK_SIZE = 20 // Parallel release 20 blob references at a time + const BLOB_CHUNK_SIZE = 20 // Parallel delete 20 blobs at a time for (let i = 0; i < blobFiles.length; i += BLOB_CHUNK_SIZE) { const chunk = blobFiles.slice(i, i + BLOB_CHUNK_SIZE) await Promise.all(chunk.map(f => - this.blobStorage.release(f.metadata.storage!.hash) + this.blobStorage.delete(f.metadata.storage!.hash) )) } + + // Phase 3: Batch delete all entities (including root directory) + const allIds = [...descendants.map(d => d.id), entityId] + await this.brain.removeMany({ ids: allIds, continueOnError: false }) } else { // No children or not recursive - just delete the directory entity await this.brain.remove(entityId) @@ -1954,27 +1800,15 @@ export class VirtualFileSystem implements IVirtualFileSystem { const newParentPath = this.getParentPath(newPath) if (oldParentPath !== newParentPath) { - // Remove the OLD parent's containment edge(s) — by edge id, resolved from - // the graph's own adjacency (the removal law: a removal never requires - // reading the thing being removed). This step used to be skipped as "not - // critical", which left the moved entity a child of BOTH directories: - // readdir(oldDir) kept listing it, re-creating the old path showed the - // name twice, and tree-walking consumers saw the file in two places. + // Remove from old parent if (oldParentPath) { const oldParentId = await this.pathResolver.resolve(oldParentPath) - const staleEdges = await this.brain.related({ - from: oldParentId, - to: entityId, - type: VerbType.Contains - }) - for (const edge of staleEdges) { - await this.brain.unrelate(edge.id) - } + // unrelate takes the relation ID, not params - need to find and remove relation + // For now, skip unrelate as it's not critical for rename } - // Add to the new parent. The root ('/') is a REAL parent — skipping it - // orphaned a move-to-root out of readdir('/') entirely. - if (newParentPath) { + // Add to new parent + if (newParentPath && newParentPath !== '/') { const newParentId = await this.pathResolver.resolve(newParentPath) await this.brain.relate({ from: newParentId, @@ -2015,95 +1849,6 @@ export class VirtualFileSystem implements IVirtualFileSystem { this.triggerWatchers(newPath, 'rename') } - /** - * Reconcile every VFS entity's containment edges against its canonical - * `metadata.path` — the path is the truth (maintained by write/rename); the - * `Contains` edges are a projection of it. Heals the "cosmetic ghost" class - * left by pre-fix renames that added the new parent's edge without removing - * the old one (an entity listed in TWO directories; a re-created old path - * showing its name twice), plus duplicate edges from the same parent left by - * concurrent writers. - * - * CONSERVATIVE by design: only VFS containment edges (subtype - * `'vfs-contains'` or `metadata.isVFS`) are ever touched — a user's own - * knowledge-graph `Contains` edge between the same entities is never - * removed. An entity whose expected parent path has no entity is logged - * loudly and left alone (never orphaned further). Operator-invoked via - * `brain.repairIndex()`. - * - * The canonical pagination walk (not an index query) is deliberate: this is - * a repair op — the projections are the thing under suspicion, so the walk - * reads the source of truth. - * - * @returns Counts of stale edges removed and missing expected edges restored. - */ - async repairContainment(): Promise<{ removed: number; restored: number }> { - await this.ensureInitialized() - - // Pass 1: canonical walk → every VFS entity's id + path. - const idByPath = new Map() - const vfsEntities: Array<{ id: string; path: string }> = [] - let cursor: string | undefined - for (;;) { - const page = await (this.brain as any).storage.getNounsWithPagination({ limit: 500, cursor }) - for (const noun of page.items) { - const meta = (noun as any).metadata ?? noun - const p = meta?.path - if (meta?.vfsType && typeof p === 'string') { - idByPath.set(p, noun.id) - vfsEntities.push({ id: noun.id, path: p }) - } - } - if (!page.hasMore) break - cursor = page.nextCursor - } - - let removed = 0 - let restored = 0 - for (const { id, path } of vfsEntities) { - if (path === '/') continue // the root has no parent - const expectedParentId = idByPath.get(this.getParentPath(path)) - if (!expectedParentId) { - console.warn( - `[VFS] repairContainment: no entity found for parent of ${path} — leaving its edges untouched.` - ) - continue - } - - const incoming = await this.brain.related({ to: id, type: VerbType.Contains }) - let expectedSeen = false - for (const edge of incoming) { - const isVfsEdge = edge.subtype === 'vfs-contains' || (edge.metadata as any)?.isVFS === true - if (!isVfsEdge) continue // never touch user knowledge edges - if (edge.from === expectedParentId && !expectedSeen) { - expectedSeen = true // keep exactly one correct edge - continue - } - // Stale parent (a pre-fix rename ghost) or a duplicate of the correct - // edge (concurrent-writer artifact) — remove it, loudly. - await this.brain.unrelate(edge.id) - removed++ - console.warn( - `[VFS] repairContainment: removed ${edge.from === expectedParentId ? 'duplicate' : 'stale'} ` + - `containment edge ${edge.from} -> ${id} (${path})` - ) - } - if (!expectedSeen) { - await this.brain.relate({ - from: expectedParentId, - to: id, - type: VerbType.Contains, - subtype: 'vfs-contains', - metadata: { isVFS: true } - }) - restored++ - console.warn(`[VFS] repairContainment: restored missing containment edge for ${path}`) - } - } - - return { removed, restored } - } - /** * Copy a file or directory to a new path. * diff --git a/src/vfs/types.ts b/src/vfs/types.ts index 9188476b..54e9902b 100644 --- a/src/vfs/types.ts +++ b/src/vfs/types.ts @@ -195,36 +195,6 @@ export interface ReadOptions { // VFS-specific options cache?: boolean // Use cache if available (default: true) decompress?: boolean // Auto-decompress (default: true) - - /** - * Read the file's content as it stood at a past generation (a number) or - * wall-clock instant (a Date) — the temporal read. Serves the exact bytes - * the file held then: the historical entity is materialized from the - * generation history, and its content blob is retention-protected (bytes - * referenced by any in-window generation are never reclaimed). Bounded by - * the retention window: reading past the compaction horizon throws. - * Bypasses the content cache. - */ - asOf?: number | Date -} - -/** - * One entry of a file's version history (see `vfs.history(path)`): the state - * the file held immediately after `generation` committed. - * `readFile(path, { asOf: generation })` returns these exact bytes while the - * generation remains inside the retention window. - */ -export interface FileVersion { - /** The generation whose write produced this version. */ - generation: number - /** Commit timestamp of that generation (ms since epoch). */ - timestamp: number - /** Content hash of this version's bytes in the content-addressed store. */ - hash: string - /** Content size in bytes. */ - size: number - /** Detected MIME type at that version. */ - mimeType?: string } export interface MkdirOptions { diff --git a/tests/integration/aggregate-reserved-fields.test.ts b/tests/integration/aggregate-reserved-fields.test.ts deleted file mode 100644 index e81692d6..00000000 --- a/tests/integration/aggregate-reserved-fields.test.ts +++ /dev/null @@ -1,169 +0,0 @@ -/** - * @module tests/integration/aggregate-reserved-fields - * @description One field-resolution law across the whole aggregation + query - * surface (SELF-AGGREGATE-DELETE-DRIFT). Laws: - * (1) aggregates grouped by a RESERVED field (subtype) decrement on delete — - * the delete-side entity view carries every reserved field, so the - * decrement finds its group (counts must never drift from ground truth); - * (2) same for update: moving an entity between reserved-field groups - * decrements the old group and increments the new one (no double-count); - * (3) aggregation source.where on a reserved field (subtype) FILTERS instead - * of silently matching nothing; - * (4) removeMany refuses empty/invalid selectors loudly (bare array, empty - * object, ids: []) instead of resolving as a silent no-op; - * (5) find() accepts both where spellings: flattened (entry.title) and - * storage-shaped (metadata.entry.title) resolve to the same rows. - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' - -const stubEmbedding = async (text: string): Promise => { - const hash = text.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0) - return new Array(384).fill(0).map((_, i) => Math.sin(hash + i)) -} - -describe('aggregation + query field-resolution law', () => { - let brain: Brainy - - beforeEach(async () => { - brain = new Brainy({ - requireSubtype: false, - storage: { type: 'memory' as const }, - embeddingFunction: stubEmbedding - }) - await brain.init() - }) - - afterEach(async () => { - await brain.close() - }) - - it('reserved-field groupBy decrements on delete (the drift bug)', async () => { - brain.defineAggregate({ - name: 'by_subtype', - source: { type: NounType.Document }, - groupBy: ['subtype'], - metrics: { count: { op: 'count' } } - }) - - const ids: string[] = [] - for (let i = 0; i < 5; i++) { - ids.push( - await brain.add({ - data: `doc-${i}`, - type: NounType.Document, - subtype: 'note', - metadata: { team: 'alpha' } - }) - ) - } - let groups = await brain.queryAggregate('by_subtype') - expect(groups).toHaveLength(1) - expect(groups[0].groupKey).toEqual({ subtype: 'note' }) - expect(groups[0].metrics.count).toBe(5) - - await brain.remove(ids[0]) - await brain.flush() - - groups = await brain.queryAggregate('by_subtype') - expect(groups[0].metrics.count).toBe(4) - const live = await brain.find({ type: NounType.Document, limit: 100 }) - expect(groups[0].metrics.count).toBe(live.length) - }) - - it('reserved-field groupBy moves between groups on update (no double-count)', async () => { - brain.defineAggregate({ - name: 'by_subtype', - source: { type: NounType.Document }, - groupBy: ['subtype'], - metrics: { count: { op: 'count' } } - }) - const id = await brain.add({ - data: 'doc-move', - type: NounType.Document, - subtype: 'draft' - }) - await brain.update({ id, subtype: 'published' }) - - const groups = await brain.queryAggregate('by_subtype') - const byKey = Object.fromEntries( - groups.map((g) => [String(g.groupKey.subtype), g.metrics.count]) - ) - expect(byKey['published']).toBe(1) - // The old group must be gone or zero — never still counting the entity. - expect(byKey['draft'] ?? 0).toBe(0) - }) - - it('source.where on a reserved field filters instead of matching nothing', async () => { - brain.defineAggregate({ - name: 'notes_only', - source: { type: NounType.Document, where: { subtype: 'note' } }, - groupBy: ['team'], - metrics: { count: { op: 'count' } } - }) - await brain.add({ - data: 'n1', - type: NounType.Document, - subtype: 'note', - metadata: { team: 'alpha' } - }) - await brain.add({ - data: 'd1', - type: NounType.Document, - subtype: 'draft', - metadata: { team: 'alpha' } - }) - - const groups = await brain.queryAggregate('notes_only') - expect(groups).toHaveLength(1) - expect(groups[0].metrics.count).toBe(1) // the note, never the draft - }) - - it('removeMany refuses empty/invalid selectors loudly', async () => { - const id = await brain.add({ data: 'keep-me', type: NounType.Document }) - - // Bare array passed positionally — the classic silent no-op. - await expect( - brain.removeMany([id] as unknown as Parameters[0]) - ).rejects.toThrow(/bare array/) - // Empty selector object. - await expect( - brain.removeMany({} as Parameters[0]) - ).rejects.toThrow(/requires a selector/) - // Explicit empty id list. - await expect(brain.removeMany({ ids: [] })).rejects.toThrow(/ids: \[\]/) - - // Nothing was deleted by any of the refused calls. - expect(await brain.get(id)).toBeTruthy() - }) - - it('find() accepts both flattened and metadata.-prefixed where spellings', async () => { - await brain.add({ - data: 'nested-doc', - type: NounType.Document, - metadata: { entry: { title: 'T1' }, classifier: { contextHints: { vfsPath: '/n/a.md' } } } - }) - await brain.flush() - - const flat = await brain.find({ - type: NounType.Document, - where: { 'entry.title': 'T1' }, - limit: 10 - }) - const prefixed = await brain.find({ - type: NounType.Document, - where: { 'metadata.entry.title': 'T1' }, - limit: 10 - }) - const deepPrefixed = await brain.find({ - type: NounType.Document, - where: { 'metadata.classifier.contextHints.vfsPath': '/n/a.md' }, - limit: 10 - }) - expect(flat).toHaveLength(1) - expect(prefixed).toHaveLength(1) - expect(prefixed[0].id).toBe(flat[0].id) - expect(deepPrefixed).toHaveLength(1) - }) -}) diff --git a/tests/integration/aggregation-state-persistence.test.ts b/tests/integration/aggregation-state-persistence.test.ts deleted file mode 100644 index db8fc088..00000000 --- a/tests/integration/aggregation-state-persistence.test.ts +++ /dev/null @@ -1,311 +0,0 @@ -/** - * @module tests/integration/aggregation-state-persistence - * @description The boot-order contract for the aggregation engine. Five laws: - * (1) STATE ADOPTION — a reopen with an unchanged defineAggregate() adopts the - * persisted state and performs NO store walk. (The pre-fix behavior: the - * synchronous define always beat the async init, flagged a backfill, and - * the first query wiped the just-loaded state and re-walked the whole - * store — every restart, forever.) - * (2) SINGLE-FLIGHT + BATCH — concurrent cold queries across multiple pending - * aggregates share exactly ONE store walk; a query never wipes another's - * partial progress and M pending aggregates cost one enumeration, not M. - * (3) CHANGED DEFINITION — a real definition change still backfills, exactly. - * (4) PERSISTED-ONLY DEFINITIONS — an app that does not re-define at boot can - * query a persisted aggregate without racing a spurious "not defined". - * (5) QUIET KEYS — the engine's persistence keys (__aggregation_*) are - * recognized system keys: no "Unknown key format" warning at boot. - */ -import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/index.js' -import { NounType } from '../../src/types/graphTypes.js' -import type { AggregateDefinition } from '../../src/types/brainy.types.js' -import { prodLog } from '../../src/utils/logger.js' - -const SPENDING: AggregateDefinition = { - name: 'spending', - source: { type: NounType.Event, where: { domain: 'financial' } }, - groupBy: ['category'], - metrics: { - total: { op: 'sum', field: 'amount' }, - count: { op: 'count' } - } -} - -/** Same name, different metrics — a REAL definition change (hash differs). */ -const SPENDING_CHANGED: AggregateDefinition = { - ...SPENDING, - metrics: { - total: { op: 'sum', field: 'amount' }, - count: { op: 'count' }, - average: { op: 'avg', field: 'amount' } - } -} - -describe('aggregation state persistence — boot-order contract', () => { - let dir: string - - beforeEach(() => { - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-agg-persist-')) - }) - - afterEach(() => { - fs.rmSync(dir, { recursive: true, force: true }) - }) - - const open = async (): Promise => { - const b: any = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true - }) - await b.init() - return b - } - - /** Count store walks by intercepting the storage adapter's getNouns. */ - const countWalks = (brain: any): { count: () => number } => { - const storage = brain.storage - const orig = storage.getNouns.bind(storage) - let calls = 0 - storage.getNouns = async (opts: unknown) => { - calls++ - return orig(opts) - } - return { count: () => calls } - } - - const seed = async (brain: any): Promise => { - for (let i = 0; i < 12; i++) { - await brain.add({ - data: `tx ${i}`, - type: NounType.Event, - metadata: { - domain: 'financial', - category: i % 2 === 0 ? 'food' : 'transport', - amount: 10 + i - } - }) - } - } - - it('adopts persisted state on reopen with an unchanged definition — zero walks', async () => { - const brain1 = await open() - brain1.defineAggregate(SPENDING) - await seed(brain1) - const before = await brain1.queryAggregate('spending') - expect(before.length).toBe(2) - await brain1.close() - - const brain2 = await open() - brain2.defineAggregate(SPENDING) // the standard declarative boot pattern - await brain2.getNounCount() // settle init paths before counting walks - const walks = countWalks(brain2) - - const after = await brain2.queryAggregate('spending') - - expect(walks.count()).toBe(0) - const key = (r: any) => r.groupKey.category - expect( - after.map((r: any) => [key(r), r.metrics.total, r.metrics.count]).sort() - ).toEqual( - before.map((r: any) => [key(r), r.metrics.total, r.metrics.count]).sort() - ) - await brain2.close() - }) - - it('adopted state keeps accumulating: post-reopen writes land on top of it', async () => { - const brain1 = await open() - brain1.defineAggregate(SPENDING) - await seed(brain1) - await brain1.queryAggregate('spending') - await brain1.close() - - const brain2 = await open() - brain2.defineAggregate(SPENDING) - // A write BEFORE the first query: if it lands before state adoption the - // engine must choose an exact rescan over adoption — either way the - // result must include all 13 entities. - await brain2.add({ - data: 'late tx', - type: NounType.Event, - metadata: { domain: 'financial', category: 'food', amount: 100 } - }) - const rows = await brain2.queryAggregate('spending') - const food = rows.find((r: any) => r.groupKey.category === 'food') - expect(food.metrics.count).toBe(7) // 6 seeded + 1 late - await brain2.close() - }) - - it('a changed definition still backfills — exactly once, with correct results', async () => { - const brain1 = await open() - brain1.defineAggregate(SPENDING) - await seed(brain1) - await brain1.queryAggregate('spending') - await brain1.close() - - const brain2 = await open() - brain2.defineAggregate(SPENDING_CHANGED) - await brain2.getNounCount() - const walks = countWalks(brain2) - - const rows = await brain2.queryAggregate('spending') - - expect(walks.count()).toBe(1) // 12 entities = one page = one getNouns call - const food = rows.find((r: any) => r.groupKey.category === 'food') - expect(food.metrics.count).toBe(6) - expect(food.metrics.average).toBeCloseTo(food.metrics.total / 6) - await brain2.close() - }) - - it('concurrent cold queries across two aggregates share exactly ONE walk', async () => { - const brain = await open() - brain.defineAggregate(SPENDING) - brain.defineAggregate({ - ...SPENDING, - name: 'by_category_count', - metrics: { count: { op: 'count' } } - }) - await seed(brain) - await brain.getNounCount() - const walks = countWalks(brain) - - const results = await Promise.all([ - brain.queryAggregate('spending'), - brain.queryAggregate('by_category_count'), - brain.queryAggregate('spending'), - brain.queryAggregate('by_category_count'), - brain.queryAggregate('spending'), - brain.queryAggregate('by_category_count') - ]) - - expect(walks.count()).toBe(1) // 12 entities = one page; one walk fills both - for (const rows of results) { - const total = rows.reduce((s: number, r: any) => s + r.metrics.count, 0) - expect(total).toBe(12) - } - - // Warm re-query: converged, no further walks. - await brain.queryAggregate('spending') - expect(walks.count()).toBe(1) - await brain.close() - }) - - it('persisted-only definitions are queryable without re-defining at boot', async () => { - const brain1 = await open() - brain1.defineAggregate(SPENDING) - await seed(brain1) - await brain1.queryAggregate('spending') - await brain1.close() - - const brain2 = await open() - // NO defineAggregate — the app relies on the persisted definition. - const walks = countWalks(brain2) - const rows = await brain2.queryAggregate('spending') - expect(walks.count()).toBe(0) // persisted state adopted here too - expect(rows.length).toBe(2) - await brain2.close() - }) - - it('generation-mismatched persisted state is rescanned once, loudly — never adopted', async () => { - const brain1 = await open() - brain1.defineAggregate(SPENDING) - await seed(brain1) - await brain1.queryAggregate('spending') - await brain1.close() - - // Simulate the copied-store incident class: a fact-log truncation (or an - // unclean shutdown) leaves the committed watermark different from the - // generation the flushed state was stamped with. - const tamper: any = await open() - const key = '__aggregation_state_spending__' - const stored = await tamper.storage.getMetadata(key) - expect(typeof stored.sourceGeneration).toBe('number') // the stamp is really persisted - await tamper.storage.saveMetadata(key, { - ...stored, - sourceGeneration: stored.sourceGeneration + 5 - }) - await tamper.close() - - const warnSpy = vi.spyOn(prodLog, 'warn') - const brain2 = await open() - brain2.defineAggregate(SPENDING) - await brain2.getNounCount() - const walks = countWalks(brain2) - - const rows = await brain2.queryAggregate('spending') - - expect(walks.count()).toBe(1) // exactly ONE rescan — no silent adopt, no spin - const food = rows.find((r: any) => r.groupKey.category === 'food') - expect(food.metrics.count).toBe(6) // rescan produced exact results - expect( - warnSpy.mock.calls.some(args => String(args[0]).includes('rescanning instead of adopting')) - ).toBe(true) // and it said so out loud - warnSpy.mockRestore() - await brain2.close() - }) - - it('a failing walk is loud, non-destructive, and latched — never a silent retry loop', async () => { - // Fresh define + seeded writes: the write hooks have populated LIVE state, - // and the first-query rescan is still pending. The incident shape - // (wipe-before-scan + no try/catch + per-query re-walk) would have wiped - // that live state and silently re-walked on every query. - const brain: any = await open() - brain.defineAggregate(SPENDING) - await seed(brain) - expect(brain._aggregationIndex.queryAggregate({ name: 'spending' }).length).toBe(2) - - const storage = brain.storage - const origGetNouns = storage.getNouns.bind(storage) - let walkAttempts = 0 - storage.getNouns = async () => { - walkAttempts++ - throw new Error('injected storage failure') - } - - // First query: the walk fails LOUDLY with the storage error. - await expect(brain.queryAggregate('spending')).rejects.toThrow('injected storage failure') - expect(walkAttempts).toBe(1) - - // Live state was NOT destroyed by the failed walk (staging was dropped). - expect(brain._aggregationIndex.queryAggregate({ name: 'spending' }).length).toBe(2) - - // Second query inside the cooldown: instant loud failure, NO new walk. - await expect(brain.queryAggregate('spending')).rejects.toThrow('failure cooldown') - expect(walkAttempts).toBe(1) - - // Heal the storage + expire the cooldown: one fresh walk succeeds exactly. - storage.getNouns = origGetNouns - brain._aggregationBackfillFailure.at = Date.now() - 60_000 - const rows = await brain.queryAggregate('spending') - const food = rows.find((r: any) => r.groupKey.category === 'food') - expect(food.metrics.count).toBe(6) - await brain.close() - }) - - it('aggregation persistence keys never log "Unknown key format"', async () => { - const warnSpy = vi.spyOn(prodLog, 'warn') - const brain1 = await open() - brain1.defineAggregate(SPENDING) - await seed(brain1) - await brain1.queryAggregate('spending') - await brain1.close() - - const brain2 = await open() - brain2.defineAggregate(SPENDING) - await brain2.queryAggregate('spending') - await brain2.close() - - const offenders = warnSpy.mock.calls - .map(args => String(args[0])) - .filter( - msg => - msg.includes('Unknown key format') && - (msg.includes('__aggregation_') || msg.includes('brainy:entityIdMapper')) - ) - expect(offenders).toEqual([]) - warnSpy.mockRestore() - }) -}) diff --git a/tests/integration/background-dedup-lifecycle.test.ts b/tests/integration/background-dedup-lifecycle.test.ts deleted file mode 100644 index 48c7a4a7..00000000 --- a/tests/integration/background-dedup-lifecycle.test.ts +++ /dev/null @@ -1,85 +0,0 @@ -/** - * @module tests/integration/background-dedup-lifecycle - * @description The post-import background deduplication pass (a merge-DELETE - * writer) obeys the same contract as the inline pass. Laws: - * (1) enableDeduplication:false schedules NO background pass — the brain-owned - * deduplicator is never even constructed; - * (2) by default the pass IS scheduled, brain-owned, with an unref'd timer - * (a pending pass never holds the process open); - * (3) repeated imports debounce into ONE pending batch on ONE instance - * (per-coordinator instances used to arm one timer per import); - * (4) close() cancels pending work — no delete pass can fire after close. - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../src/brainy.js' - -const ROWS = [ - { name: 'Alice Zephyr', role: 'engineer' }, - { name: 'Bob Quill', role: 'writer' } -] - -// Keep imports fast and deterministic — dedup scheduling is what's under test. -const FAST = { - enableNeuralExtraction: false, - enableRelationshipInference: false, - enableConceptExtraction: false -} as const - -// Deterministic stub embedder (hnsw-rebuild.test.ts pattern) — dedup -// scheduling never inspects vector CONTENT, so skip the WASM model load. -const stubEmbedding = async (text: string): Promise => { - const hash = text.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0) - const vector = new Array(384).fill(0).map((_, i) => Math.sin(hash + i)) - return vector -} - -describe('background dedup lifecycle', () => { - let brain: Brainy - - beforeEach(async () => { - brain = new Brainy({ - requireSubtype: false, - storage: { type: 'memory' as const }, - embeddingFunction: stubEmbedding - }) - await brain.init() - }) - - afterEach(async () => { - await brain.close() - }) - - it('enableDeduplication:false schedules no background pass at all', async () => { - await brain.import(ROWS, { ...FAST, enableDeduplication: false }) - expect((brain as any)._backgroundDedup).toBeUndefined() - }) - - it('default schedules a brain-owned pass with an unref-ed timer', async () => { - await brain.import(ROWS, { ...FAST }) - const dedup = (brain as any)._backgroundDedup - expect(dedup).toBeDefined() - expect(dedup.pendingImports.size).toBe(1) - const timer = dedup.debounceTimer - expect(timer).toBeDefined() - // Node timers expose hasRef(); an unref'd timer must not hold the process. - expect(typeof timer.hasRef).toBe('function') - expect(timer.hasRef()).toBe(false) - }) - - it('imports debounce into one pending batch on one brain-owned instance', async () => { - await brain.import(ROWS, { ...FAST }) - const first = (brain as any)._backgroundDedup - await brain.import([{ name: 'Cara Vex', role: 'analyst' }], { ...FAST }) - expect((brain as any)._backgroundDedup).toBe(first) - expect(first.pendingImports.size).toBe(2) - }) - - it('close() cancels pending background dedup', async () => { - await brain.import(ROWS, { ...FAST }) - const dedup = (brain as any)._backgroundDedup - expect(dedup.debounceTimer).toBeDefined() - await brain.close() - expect(dedup.debounceTimer).toBeUndefined() - expect(dedup.pendingImports.size).toBe(0) - }) -}) diff --git a/tests/integration/count-invariant.test.ts b/tests/integration/count-invariant.test.ts deleted file mode 100644 index 3896b625..00000000 --- a/tests/integration/count-invariant.test.ts +++ /dev/null @@ -1,109 +0,0 @@ -/** - * @module tests/integration/count-invariant - * @description Pattern-C acceptance: the user-facing scalar total and the - * per-type array are ONE consistent projection — `getNounCount() === Σ - * nounCountsByType` (and the verb mirror) after any interleaving of add / - * update-visibility-flip / delete, AND across a reopen. - * - * The bug (finding 5): delete decremented the per-type array but never the - * scalar for nouns, and neither for verbs — so `getNounCount()`/`getVerbCount()` - * inflated permanently (the stale scalar wins pagination via - * `Math.max(total, collected.length)` and is persisted). The fix restores the - * symmetric decrement the code always intended (its own comments say "symmetric - * with the increments"). This test would report an inflated count before the - * fix and the exact count after. - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType, VerbType } from '../../src/types/graphTypes.js' - -const sum = (a: Uint32Array): number => a.reduce((s, c) => s + c, 0) - -describe('count invariant — scalar total === Σ per-type, across delete + reopen (finding 5)', () => { - let dir: string - let brain: any - - const open = async (d: string) => { - const b = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: d }, - dimensions: 384, - silent: true - }) - await b.init() - return b - } - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-count-inv-')) - brain = await open(dir) - }) - - afterEach(async () => { - try { await brain.close() } catch { /* already closed */ } - fs.rmSync(dir, { recursive: true, force: true }) - }) - - it('noun delete decrements the scalar total, not just the per-type bucket', async () => { - const things: string[] = [] - for (let i = 0; i < 5; i++) things.push(await brain.add({ data: `thing ${i}`, type: NounType.Thing })) - for (let i = 0; i < 3; i++) await brain.add({ data: `concept ${i}`, type: NounType.Concept }) - - const storage = brain['storage'] - expect(await storage.getNounCount()).toBe(8) - expect(sum(storage.getNounCountsByType())).toBe(8) - - // Delete 2 Things — the scalar must drop with the bucket (pre-fix it stayed 8). - await brain.remove(things[0]) - await brain.remove(things[1]) - - expect(await storage.getNounCount()).toBe(6) - expect(sum(storage.getNounCountsByType())).toBe(6) - // The invariant: scalar === Σ per-type. - expect(await storage.getNounCount()).toBe(sum(storage.getNounCountsByType())) - }) - - it('verb delete decrements BOTH the scalar total AND the per-type bucket', async () => { - const ids: string[] = [] - for (let i = 0; i < 4; i++) ids.push(await brain.add({ data: `node ${i}`, type: NounType.Thing })) - const edges: string[] = [] - edges.push(await brain.relate({ from: ids[0], to: ids[1], type: VerbType.RelatedTo })) - edges.push(await brain.relate({ from: ids[1], to: ids[2], type: VerbType.RelatedTo })) - edges.push(await brain.relate({ from: ids[2], to: ids[3], type: VerbType.Contains })) - edges.push(await brain.relate({ from: ids[0], to: ids[3], type: VerbType.Contains })) - - const storage = brain['storage'] - expect(await storage.getVerbCount()).toBe(4) - expect(sum(storage.getVerbCountsByType())).toBe(4) - - // Unrelate one edge — pre-fix, verb delete touched NEITHER counter (both stayed 4). - await brain.unrelate(edges[0]) - - expect(await storage.getVerbCount()).toBe(3) - expect(sum(storage.getVerbCountsByType())).toBe(3) - expect(await storage.getVerbCount()).toBe(sum(storage.getVerbCountsByType())) - }) - - it('the corrected counts persist and survive a reopen', async () => { - const things: string[] = [] - for (let i = 0; i < 6; i++) things.push(await brain.add({ data: `t${i}`, type: NounType.Thing })) - const a = await brain.relate({ from: things[0], to: things[1], type: VerbType.RelatedTo }) - await brain.relate({ from: things[1], to: things[2], type: VerbType.RelatedTo }) - await brain.remove(things[5]) - await brain.unrelate(a) - await brain.flush() - await brain.close() - - // Reopen from disk — counts.json must carry the corrected totals. - brain = await open(dir) - const storage = brain['storage'] - expect(await storage.getNounCount()).toBe(5) - expect(await storage.getVerbCount()).toBe(1) - expect(await storage.getNounCount()).toBe(sum(storage.getNounCountsByType())) - expect(await storage.getVerbCount()).toBe(sum(storage.getVerbCountsByType())) - }) -}) diff --git a/tests/integration/counter-recount.test.ts b/tests/integration/counter-recount.test.ts deleted file mode 100644 index 310ea4d5..00000000 --- a/tests/integration/counter-recount.test.ts +++ /dev/null @@ -1,122 +0,0 @@ -/** - * @module tests/integration/counter-recount - * @description Counter honesty. Two laws under test: - * (1) REMOVAL NEVER REQUIRES RE-READING THE REMOVED RECORD — the count - * decrement falls back to the caller's pre-delete read when the canonical - * metadata re-read returns null (replace race / ghost), instead of being - * silently skipped. The skip minted permanent inflation: adds counted, - * paired removals not decremented, and Math.max(totalNounCount, scanned) - * pinned the inflated scalar forever. - * (2) THE SANCTIONED RECOUNT — repairIndex() unconditionally recomputes and - * PERSISTS every counter rollup (scalar totals + per-type maps + - * type-statistics) from one canonical walk, so an already-inflated brain - * is permanently corrected (survives reopen). - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/index.js' - -/** Absolute path of one entity's `` directory (or null if not present). */ -function entityDir(root: string, id: string): string | null { - const base = path.join(root, 'entities', 'nouns') - if (!fs.existsSync(base)) return null - for (const shard of fs.readdirSync(base)) { - const candidate = path.join(base, shard, id) - if (fs.existsSync(candidate)) return candidate - } - return null -} - -describe('counter honesty — removal without re-reading + the sanctioned recount', () => { - let dir: string - let brain: any - - const open = async () => { - const b: any = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true, - dimensions: 384 - }) - await b.init() - return b - } - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-recount-')) - brain = await open() - }) - afterEach(async () => { - await brain.close?.().catch(() => {}) - fs.rmSync(dir, { recursive: true, force: true }) - }) - - it('the counter returns to baseline across add→remove→re-create cycles (no drift)', async () => { - const baseline = await brain.storage.getNounCount() - for (let i = 0; i < 4; i++) { - const id = await brain.add({ data: `cycle ${i}`, type: 'document', metadata: { i } }) - expect(await brain.storage.getNounCount()).toBe(baseline + 1) - await brain.remove(id) - expect(await brain.storage.getNounCount()).toBe(baseline) - } - }) - - it('deleteNoun decrements from the provided prior record when the canonical re-read is null', async () => { - const id = await brain.add({ data: 'to be ghosted', type: 'document', metadata: { g: 1 } }) - await brain.flush() - const baseline = await brain.storage.getNounCount() - - // Capture the pre-delete read (what remove() holds), then simulate the - // replace-race / ghost shape: the metadata leg vanishes before the delete's - // internal re-read. - const prior = await brain.storage.getNounMetadata(id) - expect(prior).not.toBeNull() - const eDir = entityDir(dir, id)! - for (const f of fs.readdirSync(eDir)) { - if (f.startsWith('metadata.json')) fs.rmSync(path.join(eDir, f)) - } - - // Without the prior record this decrement used to be silently skipped. - await brain.storage.deleteNoun(id, prior) - expect(await brain.storage.getNounCount()).toBe(baseline - 1) - // Full removal still holds: nothing left on disk. - expect(entityDir(dir, id)).toBeNull() - }) - - it('repairIndex() recounts an inflated persisted scalar over CLEAN shelves — and it survives reopen', async () => { - for (let i = 0; i < 3; i++) { - await brain.add({ data: `real ${i}`, type: 'document', metadata: { i } }) - } - await brain.flush() - const honest = await brain.storage.getNounCount() - // Pagination's totalCount has its own baseline: it enumerates EVERYTHING - // (including the internal VFS root), while the user-facing scalar counts - // only public entities — so the two legitimately differ by the internals. - const pageHonest = (await brain.storage.getNounsWithPagination({ limit: 1000, offset: 0 })).totalCount - - // Simulate the historical drift: an inflated persisted scalar (deletes whose - // decrement was skipped). Persist it so a reopen rehydrates the lie. - ;(brain.storage as any).totalNounCount = honest + 52 - await (brain.storage as any).persistCounts() - await brain.close() - brain = await open() - expect(await brain.storage.getNounCount()).toBe(honest + 52) // the lie survived reopen - - // The sanctioned recount — unconditional in repairIndex (no orphans needed). - await brain.repairIndex() - expect(await brain.storage.getNounCount()).toBe(honest) - - // Permanently: the corrected counter survives another reopen. - await brain.close() - brain = await open() - expect(await brain.storage.getNounCount()).toBe(honest) - - // And the paginated totalCount (the Math.max consumer) is honest too — - // back to ITS baseline, no longer pinned high by the inflated scalar. - const page = await brain.storage.getNounsWithPagination({ limit: 1000, offset: 0 }) - expect(page.totalCount).toBe(pageHonest) - }) -}) diff --git a/tests/integration/db-mvcc.test.ts b/tests/integration/db-mvcc.test.ts index 959d0053..764a755a 100644 --- a/tests/integration/db-mvcc.test.ts +++ b/tests/integration/db-mvcc.test.ts @@ -515,15 +515,7 @@ describe('8.0 Db API — generational MVCC', () => { await brain.transact([{ op: 'update', id: uid('compact-e'), metadata: { v: 4 } }]) ).release() - // History record-sets only — the generation FACT LOG also lives under - // `_generations/` (at `facts/`) and is deliberately NOT reclaimed by - // history compaction (facts are the future canonical, not undo history). - const historyRecords = async (): Promise => - (await storage.listRawObjects('_generations')).filter( - (p: string) => !p.startsWith('_generations/facts/') - ).length - - const recordsBefore = await historyRecords() + const recordsBefore = (await storage.listRawObjects('_generations')).length expect(recordsBefore).toBeGreaterThan(0) // Compact while pinned: record-sets above the pin survive, pinned reads stay correct. @@ -536,7 +528,7 @@ describe('8.0 Db API — generational MVCC', () => { const second = await brain.compactHistory() expect(first.removedGenerations + second.removedGenerations).toBeGreaterThan(0) - const recordsAfter = await historyRecords() + const recordsAfter = (await storage.listRawObjects('_generations')).length expect(recordsAfter).toBeLessThan(recordsBefore) expect(recordsAfter).toBe(0) @@ -1280,32 +1272,24 @@ describe('8.0 Db API — generational MVCC', () => { await expect(reopened.asOf(1)).rejects.toBeInstanceOf(GenerationCompactedError) }) - it('Model-B retention — flush() NEVER compacts (8.9.0); adaptive reclaim runs at close()', async () => { - // Default brain → ADAPTIVE retention with a driven byte budget far below - // the accumulated history (~13 generations of full-vector before-images). - // The 8.9.0 law: flush() is durability-only — it must not reclaim even - // when the budget is exceeded (reclaim-on-flush blocked production writes - // for 25-191s). Maintenance runs at close(), time-bounded. - const { brain, dir } = await openFsBrain() + it('Model-B retention — setRetentionBudget drives adaptive reclaim on flush; live data intact', async () => { + // Default brain → ADAPTIVE retention. A coordinator (e.g. cor's ResourceManager) + // pushes a byte budget via setRetentionBudget(); auto-compaction on flush() reclaims + // oldest history down toward it. Each update's before-image carries the full prior + // 384-dim vector (~KBs), so ~13 generations far exceed a few-KB budget. + const { brain } = await openFsBrain() const a = uid('ret-budget') await brain.add({ id: a, type: NounType.Document, data: 'v0', vector: vec(1), metadata: { v: 0 } }) for (let v = 1; v <= 12; v++) await brain.update({ id: a, metadata: { v } }) brain.setRetentionBudget(6000) // ~6 KB — well below the accumulated history - await brain.flush() + await brain.flush() // group-commit + adaptive auto-compaction under the budget - // flush() paid durability only: nothing reclaimed, all history readable. - expect(generationStoreOf(brain).horizon()).toBe(0) - const probe = await brain.asOf(1) // readable proves nothing was reclaimed… - await probe.release() // …and MUST be released: a held pin would (correctly) - // protect every newer generation through the close() compaction below. - await brain.close() // ← THE auto-compaction site now - - // close() reclaimed under the budget; live record intact; horizon durable. - const { brain: reopened } = await openFsBrain(dir) - expect(generationStoreOf(reopened).horizon()).toBeGreaterThan(0) - await expect(reopened.asOf(1)).rejects.toBeInstanceOf(GenerationCompactedError) - expect((await reopened.get(a))?.metadata?.v).toBe(12) + // History was reclaimed (the horizon advanced past the oldest generations)… + expect(generationStoreOf(brain).horizon()).toBeGreaterThan(0) + await expect(brain.asOf(1)).rejects.toBeInstanceOf(GenerationCompactedError) + // …but the budget reclaims HISTORY only — the live record is untouched. + expect((await brain.get(a))?.metadata?.v).toBe(12) }) // ========================================================================== diff --git a/tests/integration/delete-full-removal.test.ts b/tests/integration/delete-full-removal.test.ts deleted file mode 100644 index 83675a13..00000000 --- a/tests/integration/delete-full-removal.test.ts +++ /dev/null @@ -1,161 +0,0 @@ -/** - * @module tests/integration/delete-full-removal - * @description Canonical noun/verb deletes are FULL removals: both legs - * (metadata + vectors) AND the entity's `/` container are removed, so a - * delete leaves NOTHING behind. Regression for the ghost/scar defect where - * remove() deleted the vector INDEX entry + the canonical metadata leg but never - * the canonical vectors.json leg or the directory — leaving an orphan that reads - * as absent (getNoun needs both legs) yet inflated the enumerated count and - * confused locator resolution. Also covers the operator repair sweep - * (brain.repairIndex()) that prunes orphans left by the pre-fix behavior. - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/index.js' - -/** All entity-directory names (the `` dirs) under entities/. */ -function entityDirs(root: string, kind: 'nouns' | 'verbs'): string[] { - const base = path.join(root, 'entities', kind) - if (!fs.existsSync(base)) return [] - const ids: string[] = [] - for (const shard of fs.readdirSync(base)) { - const shardDir = path.join(base, shard) - if (!fs.statSync(shardDir).isDirectory()) continue - for (const id of fs.readdirSync(shardDir)) { - if (fs.statSync(path.join(shardDir, id)).isDirectory()) ids.push(id) - } - } - return ids -} - -/** Absolute path of one entity's `` directory (or null if not present). */ -function entityDir(root: string, kind: 'nouns' | 'verbs', id: string): string | null { - const base = path.join(root, 'entities', kind) - if (!fs.existsSync(base)) return null - for (const shard of fs.readdirSync(base)) { - const candidate = path.join(base, shard, id) - if (fs.existsSync(candidate)) return candidate - } - return null -} - -describe('canonical delete is a full removal (no ghost/scar directory)', () => { - let dir: string - let brain: any - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-del-')) - brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 }) - await brain.init() - }) - afterEach(async () => { - await brain.close?.().catch(() => {}) - fs.rmSync(dir, { recursive: true, force: true }) - }) - - it('remove() deletes BOTH legs and the container — nothing left on disk', async () => { - const ids: string[] = [] - for (let i = 0; i < 3; i++) ids.push(await brain.add({ data: `doc ${i}`, type: 'document', metadata: { i } })) - await brain.flush() - - // (A filesystem brain also has its VFS-root noun, so don't assume an exact set.) - const before = entityDirs(dir, 'nouns') - expect(before).toEqual(expect.arrayContaining(ids)) - - await brain.remove(ids[1]) - await brain.flush() - - // getNoun is null AND the on-disk container is fully gone (no orphan), and - // EXACTLY one directory disappeared (the removed entity's). - expect(await brain.get(ids[1])).toBeNull() - expect(entityDir(dir, 'nouns', ids[1])).toBeNull() - const after = entityDirs(dir, 'nouns') - expect(after).toHaveLength(before.length - 1) - expect(after).toEqual(expect.arrayContaining([ids[0], ids[2]])) - expect(after).not.toContain(ids[1]) - }) - - it('the enumerated count is honest after a delete (no monotonic inflation)', async () => { - const ids: string[] = [] - for (let i = 0; i < 4; i++) ids.push(await brain.add({ data: `n${i}`, type: 'document', metadata: { i } })) - await brain.flush() - - const before = await brain.storage.getNounsWithPagination({ limit: 100, offset: 0 }) - - await brain.remove(ids[0]) - await brain.remove(ids[1]) - await brain.flush() - - // The count drops by EXACTLY the two removed entities — no ghost lingers to - // hold the total up (the pre-fix defect left it monotonic). - const after = await brain.storage.getNounsWithPagination({ limit: 100, offset: 0 }) - expect(after.totalCount).toBe(before.totalCount - 2) - const remaining = after.items.map((n: any) => n.id) - expect(remaining).toEqual(expect.arrayContaining([ids[2], ids[3]])) - expect(remaining).not.toContain(ids[0]) - expect(remaining).not.toContain(ids[1]) - }) -}) - -describe('repairIndex() prunes orphan containers left by the pre-fix partial delete', () => { - let dir: string - let brain: any - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-orphan-')) - brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 }) - await brain.init() - }) - afterEach(async () => { - await brain.close?.().catch(() => {}) - fs.rmSync(dir, { recursive: true, force: true }) - }) - - it('a vector-only "ghost" (metadata leg deleted out-of-band) is pruned', async () => { - const ids: string[] = [] - for (let i = 0; i < 3; i++) ids.push(await brain.add({ data: `g${i}`, type: 'document', metadata: { i } })) - await brain.flush() - - // Simulate the pre-fix defect on ids[0]: remove ONLY its metadata leg, - // leaving vectors.json + the directory (the exact ghost shape). - const ghostDir = entityDir(dir, 'nouns', ids[0])! - for (const f of fs.readdirSync(ghostDir)) { - if (f.startsWith('metadata.json')) fs.rmSync(path.join(ghostDir, f)) - } - // The ghost dir (vectors.json, no metadata content leg) still sits on disk. - expect(entityDir(dir, 'nouns', ids[0])).not.toBeNull() - - await brain.repairIndex() - - // The sweep removes the ghost container; the two healthy entities survive. - expect(entityDir(dir, 'nouns', ids[0])).toBeNull() - expect(entityDir(dir, 'nouns', ids[1])).not.toBeNull() - expect(entityDir(dir, 'nouns', ids[2])).not.toBeNull() - }) - - it('an empty "scar" directory is pruned', async () => { - const id = await brain.add({ data: 'lonely', type: 'document', metadata: {} }) - await brain.flush() - const scarDir = entityDir(dir, 'nouns', id)! - for (const f of fs.readdirSync(scarDir)) fs.rmSync(path.join(scarDir, f)) // empty the dir, keep it - expect(fs.existsSync(scarDir)).toBe(true) - - await brain.repairIndex() - - expect(fs.existsSync(scarDir)).toBe(false) - }) - - it('a healthy entity (both legs present) is NEVER pruned', async () => { - const id = await brain.add({ data: 'keep me', type: 'document', metadata: { keep: true } }) - await brain.flush() - - await brain.repairIndex() - - expect(entityDir(dir, 'nouns', id)).not.toBeNull() - expect(await brain.get(id)).not.toBeNull() - }) -}) diff --git a/tests/integration/entity-tree-stamp.test.ts b/tests/integration/entity-tree-stamp.test.ts deleted file mode 100644 index deefc5e6..00000000 --- a/tests/integration/entity-tree-stamp.test.ts +++ /dev/null @@ -1,163 +0,0 @@ -/** - * @module tests/integration/entity-tree-stamp - * @description The entity tree's FAMILY STAMP: written at flush/close with - * `sourceGeneration` (the committed generation the canonical tree reflects) - * plus rollup invariants (entity/relationship counts); verified at open by - * comparison — coherent on a clean close, LOUD on genuine incoherence - * (tampered counters), healed by repairIndex()'s recount + re-stamp. One - * verifier reads both member modes. - */ -import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { - Brainy, - readFamilyStamp, - verifyFamilyStamp, - ENTITY_TREE_STAMP_PATH, - type FamilyStamp -} from '../../src/index.js' -import { prodLog } from '../../src/utils/logger.js' - -describe('entity-tree family stamp', () => { - let dir: string - let brain: any - - const open = async () => { - const b: any = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true, - dimensions: 384 - }) - await b.init() - return b - } - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-stamp-')) - brain = await open() - }) - afterEach(async () => { - vi.restoreAllMocks() - await brain.close?.().catch(() => {}) - fs.rmSync(dir, { recursive: true, force: true }) - }) - - it('flush() writes the stamp: rollup mode, counts + sourceGeneration match live state', async () => { - for (let i = 0; i < 3; i++) await brain.add({ data: `s${i}`, type: 'document', metadata: { i } }) - await brain.flush() - - const stamp = (await readFamilyStamp(brain.storage, ENTITY_TREE_STAMP_PATH)) as FamilyStamp - expect(stamp).not.toBeNull() - expect(stamp.family).toBe('entity-tree') - expect(stamp.members.mode).toBe('rollup') - const invariants = (stamp.members as any).invariants - expect(invariants.nounCount).toBe(await brain.storage.getNounCount()) - expect(invariants.verbCount).toBe(await brain.storage.getVerbCount()) - expect(stamp.sourceGeneration).toBe(brain.generation()) - expect(stamp.generation).toBeGreaterThanOrEqual(1) - }) - - it('a cleanly-closed store reopens COHERENT (no incoherence warning)', async () => { - await brain.add({ data: 'clean', type: 'document', metadata: {} }) - await brain.close() - - const warn = vi.spyOn(prodLog, 'warn') - brain = await open() - const stampWarnings = warn.mock.calls.filter((c) => String(c[0]).includes('entity-tree stamp')) - expect(stampWarnings).toEqual([]) - }) - - it('tampered counters surface as INCOHERENT at open; repairIndex() heals + re-stamps', async () => { - for (let i = 0; i < 3; i++) await brain.add({ data: `t${i}`, type: 'document', metadata: { i } }) - await brain.close() - - // Simulate counter drift AFTER the stamp was written: inflate the - // persisted scalar the way the historical decrement-skip did. - brain = await open() - ;(brain.storage as any).totalNounCount += 52 - await (brain.storage as any).persistCounts() - await brain.close() - // The close boundary re-stamps with the inflated counter — so tamper the - // STAMP instead for a deterministic mismatch: stamped counts differ from - // the (inflated) live ones at the NEXT open only if the stamp is older. - // Rewrite the stamp with the honest counts + current sourceGeneration. - const raw = JSON.parse( - require('node:zlib') - .gunzipSync(fs.readFileSync(path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`))) - .toString('utf-8') - ) as FamilyStamp - const honest = { ...raw } - ;(honest.members as any).invariants.nounCount -= 52 - fs.writeFileSync( - path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`), - require('node:zlib').gzipSync(JSON.stringify(honest)) - ) - - const warn = vi.spyOn(prodLog, 'warn') - brain = await open() - const incoherent = warn.mock.calls.filter((c) => String(c[0]).includes('INCOHERENT')) - expect(incoherent.length).toBeGreaterThanOrEqual(1) - expect(String(incoherent[0][0])).toMatch(/nounCount/) - - // The heal: recount from canonical + re-stamp → next open is quiet. - await brain.repairIndex() - await brain.close() - const warn2 = vi.spyOn(prodLog, 'warn') - brain = await open() - const stillIncoherent = warn2.mock.calls.filter((c) => String(c[0]).includes('INCOHERENT')) - expect(stillIncoherent).toEqual([]) - }) - - it('the one verifier handles both member modes', () => { - const rollup: FamilyStamp = { - family: 'x', - generation: 1, - committedAt: new Date().toISOString(), - sourceGeneration: 5, - members: { mode: 'rollup', invariants: { nounCount: 10 } } - } - expect(verifyFamilyStamp(rollup, 5, { nounCount: 10 })).toEqual({ state: 'coherent' }) - expect(verifyFamilyStamp(rollup, 5, { nounCount: 11 }).state).toBe('incoherent') - expect(verifyFamilyStamp(rollup, 9, { nounCount: 10 })).toEqual({ - state: 'behind', - stampSource: 5, - head: 9 - }) - expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 }).state).toBe('incoherent') // ahead of head - expect(verifyFamilyStamp(null, 5, {})).toEqual({ state: 'absent' }) - - const enumerated: FamilyStamp = { - family: 'y', - generation: 1, - committedAt: new Date().toISOString(), - sourceGeneration: 2, - members: { mode: 'enumerated', files: [{ path: 'a.bin', bytes: 128 }] } - } - expect(verifyFamilyStamp(enumerated, 2, { 'a.bin': 128 })).toEqual({ state: 'coherent' }) - expect(verifyFamilyStamp(enumerated, 2, { 'a.bin': 64 }).state).toBe('incoherent') - expect(verifyFamilyStamp(enumerated, 2, {}).state).toBe('incoherent') // missing member - - // Rollup invariants may be STRING fingerprints (e.g. a per-tree SHA-256): - // strict equality either way; a type mismatch reads as incoherence. - const fingerprinted: FamilyStamp = { - family: 'z', - generation: 1, - committedAt: new Date().toISOString(), - sourceGeneration: 3, - members: { mode: 'rollup', invariants: { treeDigest: 'abc123', rows: 42 } } - } - expect(verifyFamilyStamp(fingerprinted, 3, { treeDigest: 'abc123', rows: 42 })).toEqual({ - state: 'coherent' - }) - expect(verifyFamilyStamp(fingerprinted, 3, { treeDigest: 'deadbeef', rows: 42 }).state).toBe( - 'incoherent' - ) - expect(verifyFamilyStamp(fingerprinted, 3, { treeDigest: 'abc123', rows: '42' }).state).toBe( - 'incoherent' // type mismatch never passes - ) - }) -}) diff --git a/tests/integration/fact-log-contracts.test.ts b/tests/integration/fact-log-contracts.test.ts deleted file mode 100644 index eb579da9..00000000 --- a/tests/integration/fact-log-contracts.test.ts +++ /dev/null @@ -1,125 +0,0 @@ -/** - * @module tests/integration/fact-log-contracts - * @description Pinned durability + stability contracts for the fact log. - * - * (1) FSYNC-BEFORE-ACK: an acknowledged write's fact survives an abrupt - * process end (no flush, no close — reopen from disk). - * - transact(): HOLDS TODAY — the fact is fsync'd before transact returns. - * - single-op: PINNED AS `it.fails` — today's group-commit batches - * DURABILITY (ack precedes the group fsync; a hard kill loses the fact - * AND the generation together, coherently — the documented Model-B - * contract, fine while the tree is authoritative). The destination - * (ack-at-log) requires group commit to become LATENCY batching: the - * ack waits for the shared fsync. When that lands, this pin flips red — - * remove `.fails` and the contract is permanent. No cliff to discover. - * - * (2) SCAN STABILITY UNDER ROTATION: a scan handle opened before segment - * rotation yields exactly its snapshot — byte-identical facts, no gaps, - * no duplicates, and no bleed-in of facts appended after the snapshot. - * (The reclaim-during-scan variant lands with fact-log compaction, which - * does not exist yet — segments only rotate today, never reclaim.) - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy, type CommitFact } from '../../src/index.js' -import { MemoryStorage } from '../../src/storage/adapters/memoryStorage.js' -import { FactLog, type FactLogStorage } from '../../src/db/factLog.js' - -describe('fsync-before-ack contract (fact durability at the ack boundary)', () => { - let dir: string - let brain: any - - const open = async () => { - const b: any = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true, - dimensions: 384 - }) - await b.init() - return b - } - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-factack-')) - brain = await open() - }) - afterEach(async () => { - await brain.close?.().catch(() => {}) - fs.rmSync(dir, { recursive: true, force: true }) - }) - - it('transact(): the fact is durable the moment the ack returns (kill-after-ack safe)', async () => { - const receipt = await brain.transact([ - { op: 'add', type: 'document', metadata: { durable: 1 }, data: 'ack-at-commit' } - ]) - // Abrupt end: no flush(), no close() — a new instance reads only disk. - brain = await open() - const facts: CommitFact[] = [] - for await (const b of brain.scanFacts()!.batches()) facts.push(...b.facts) - expect(facts.some((f) => f.generation === receipt.generation)).toBe(true) - }) - - // PINNED (flips red when group commit becomes latency batching — then - // remove `.fails` and the ack-at-log contract is permanent on every path). - it.fails('single-op: the fact is durable the moment the ack returns (the ack-at-log target)', async () => { - await brain.add({ data: 'acked single-op', type: 'document', metadata: { n: 1 } }) - const ackedHead = brain.scanFacts()!.headGeneration - // Abrupt end immediately after the ack — before any flush window. - brain = await open() - const facts: CommitFact[] = [] - for await (const b of brain.scanFacts()!.batches()) facts.push(...b.facts) - expect(facts.some((f) => f.generation === ackedHead)).toBe(true) - }) -}) - -describe('scan stability under rotation (the snapshot contract)', () => { - const UUID = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, '0')}` - const fact = (generation: number): CommitFact => ({ - generation, - timestamp: 1_700_000_000_000 + generation, - ops: [ - { - kind: 'noun', - id: UUID(generation), - // Padding makes each frame ~1KB so a small rotateBytes forces rotations. - record: { metadata: { noun: 'document', pad: 'x'.repeat(900), g: generation }, vector: null } - } - ] - }) - - it('a scan opened before rotations yields its exact snapshot — no gaps, dups, or bleed-in', async () => { - const mem: any = new MemoryStorage() - await mem.init() - const log = new FactLog(mem as FactLogStorage, { rotateBytes: 4096 }) // ~4 facts per segment - await log.open(0) - for (let g = 1; g <= 10; g++) await log.append(fact(g)) - await log.sync() - - // Open the snapshot, THEN keep appending — forcing further rotations. - const scan = log.scanFacts() - expect(scan.headGeneration).toBe(10) - for (let g = 11; g <= 25; g++) await log.append(fact(g)) - await log.sync() - expect(log.headGeneration()).toBe(25) - - const seen: number[] = [] - for await (const batch of scan.batches()) { - for (const f of batch.facts) seen.push(f.generation) - } - // Exactly the snapshot: 1..10 in order, nothing appended-after bleeds in. - expect(seen).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) - expect(scan.summary().factsYielded).toBe(10) - - // And a fresh scan sees everything, across all rotated segments. - const all: number[] = [] - for await (const batch of log.scanFacts().batches()) { - for (const f of batch.facts) all.push(f.generation) - } - expect(all).toEqual(Array.from({ length: 25 }, (_, i) => i + 1)) - expect(log.segmentPaths().length).toBeGreaterThanOrEqual(2) // rotations actually happened - }) -}) diff --git a/tests/integration/fact-log-dual-write.test.ts b/tests/integration/fact-log-dual-write.test.ts deleted file mode 100644 index 5ec66273..00000000 --- a/tests/integration/fact-log-dual-write.test.ts +++ /dev/null @@ -1,219 +0,0 @@ -/** - * @module tests/integration/fact-log-dual-write - * @description The generation fact log end-to-end through real commits: every - * committed generation (single-op AND transact) appends its AFTER-IMAGE fact - * at the commit point; removals append body-less tombstones; an aborted - * transaction leaves no fact; facts survive reopen and continue monotonically; - * the scan surface (brain.scanFacts) carries the frozen telemetry shape; and - * the fact-log namespace is protected against prefix-nuking. - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy, ProtectedArtifactError, type CommitFact } from '../../src/index.js' - -async function allFacts(brain: any): Promise { - const scan = brain.scanFacts() - expect(scan).not.toBeNull() - const facts: CommitFact[] = [] - for await (const batch of scan!.batches()) facts.push(...batch.facts) - return facts -} - -describe('fact log dual-write (memory adapter)', () => { - let brain: any - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true, dimensions: 384 }) - await brain.init() - }) - afterEach(async () => { - await brain.close?.().catch(() => {}) - }) - - it('every single-op write appends its after-image fact; a remove appends a tombstone', async () => { - const id = await brain.add({ data: 'first', type: 'document', metadata: { rev: 1 } }) - await brain.update({ id, metadata: { rev: 2 } }) - await brain.remove(id) - - const facts = await allFacts(brain) - // add + update + remove each committed a generation (the remove may span - // cascade ops but is ONE generation). Facts are monotonic. - const gens = facts.map((f) => f.generation) - expect([...gens].sort((a, b) => a - b)).toEqual(gens) - expect(facts.length).toBeGreaterThanOrEqual(3) - - // The add fact carries the after-image of the new entity. - const addFact = facts.find((f) => f.ops.some((op) => op.id === id && op.record !== null)) - expect(addFact).toBeDefined() - - // The remove fact carries a body-less tombstone for the id. - const removeFact = facts[facts.length - 1] - const tombstone = removeFact.ops.find((op) => op.id === id) - expect(tombstone).toBeDefined() - expect(tombstone!.record).toBeNull() - expect(tombstone!.kind).toBe('noun') - }) - - it('the update fact holds the NEW state (after-image, not before)', async () => { - const id = await brain.add({ data: 'versioned', type: 'document', metadata: { v: 'old' } }) - await brain.update({ id, metadata: { v: 'new' } }) - - const facts = await allFacts(brain) - const updateFact = facts[facts.length - 1] - const op = updateFact.ops.find((o) => o.id === id)! - expect(op.record).not.toBeNull() - expect((op.record!.metadata as any).v).toBe('new') - }) - - it('a transact commits ONE fact carrying all its ops, with meta', async () => { - const receipt = await brain.transact( - [ - { op: 'add', type: 'document', metadata: { part: 1 }, data: 'a' }, - { op: 'add', type: 'document', metadata: { part: 2 }, data: 'b' } - ], - { meta: { source: 'batch-import' } } - ) - - const facts = await allFacts(brain) - const txFact = facts.find((f) => f.generation === receipt.generation) - expect(txFact).toBeDefined() - expect(txFact!.ops.filter((op) => op.kind === 'noun').length).toBeGreaterThanOrEqual(2) - expect(txFact!.meta).toEqual({ source: 'batch-import' }) - }) - - it('an aborted transact leaves NO fact (absent = never committed)', async () => { - const id = await brain.add({ data: 'cas target', type: 'document', metadata: { n: 1 } }) - const before = (await allFacts(brain)).length - - await expect( - brain.transact([{ op: 'update', id, ifRev: 999, metadata: { n: 2 } }]) - ).rejects.toThrow() - - const after = await allFacts(brain) - expect(after.length).toBe(before) - }) - - it('fact generations line up with the transaction log', async () => { - await brain.add({ data: 'x', type: 'document', metadata: {} }) - await brain.add({ data: 'y', type: 'document', metadata: {} }) - await brain.flush() - - const facts = await allFacts(brain) - const logGens = new Set((await brain.transactionLog()).map((e: any) => e.generation)) - for (const f of facts) { - expect(logGens.has(f.generation)).toBe(true) - } - }) - - it('the storage fact-scan capability serves a provider holding only `storage`', async () => { - // An index provider receives `storage` — never the brain — and reaches the - // fact log through the host-wired capability (it must never construct its - // own fact-log reader: the log's open path is writer-side). - const id = await brain.add({ data: 'via storage', type: 'document', metadata: { s: 1 } }) - await brain.remove(id) - - const storage = brain.storage - expect(typeof storage.scanFacts).toBe('function') - expect(storage.factLogHeadGeneration()).toBe(brain.scanFacts()!.headGeneration) - - const viaStorage: CommitFact[] = [] - for await (const b of storage.scanFacts()!.batches()) viaStorage.push(...b.facts) - const viaBrain: CommitFact[] = [] - for await (const b of brain.scanFacts()!.batches()) viaBrain.push(...b.facts) - expect(viaStorage.map((f) => f.generation)).toEqual(viaBrain.map((f) => f.generation)) - expect(storage.factSegmentPaths()).toEqual(brain.factSegmentPaths()) - }) - - it('scan telemetry carries the frozen shape end-to-end', async () => { - for (let i = 0; i < 5; i++) await brain.add({ data: `t${i}`, type: 'document', metadata: { i } }) - - const scan = brain.scanFacts({ batchSize: 2 })! - expect(scan.headGeneration).toBeGreaterThanOrEqual(5) - expect(scan.approxFactCount).toBeGreaterThanOrEqual(5) - let batches = 0 - for await (const b of scan.batches()) { - batches++ - expect(b.factCount).toBe(b.facts.length) - expect(b.firstGeneration).toBe(b.facts[0].generation) - expect(b.lastGeneration).toBe(b.facts[b.facts.length - 1].generation) - expect(b.byteSize).toBeGreaterThan(0) - expect(typeof b.segmentId).toBe('string') - } - expect(batches).toBeGreaterThan(1) - expect(scan.summary().factsYielded).toBe(scan.approxFactCount) - }) -}) - -describe('fact log dual-write (filesystem adapter — durability + protection)', () => { - let dir: string - let brain: any - - const open = async () => { - const b: any = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true, - dimensions: 384 - }) - await b.init() - return b - } - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-factlog-')) - brain = await open() - }) - afterEach(async () => { - await brain.close?.().catch(() => {}) - fs.rmSync(dir, { recursive: true, force: true }) - }) - - it('facts survive close + reopen and appends continue monotonically', async () => { - const id = await brain.add({ data: 'persist me', type: 'document', metadata: { k: 1 } }) - await brain.remove(id) - await brain.close() - - brain = await open() - const facts = await allFacts(brain) - expect(facts.length).toBeGreaterThanOrEqual(2) - const headBefore = facts[facts.length - 1].generation - - await brain.add({ data: 'after reopen', type: 'document', metadata: { k: 2 } }) - const facts2 = await allFacts(brain) - expect(facts2[facts2.length - 1].generation).toBeGreaterThan(headBefore) - }) - - it('the fact segments exist on disk under _generations/facts/ with zero-padded names', async () => { - await brain.add({ data: 'on disk', type: 'document', metadata: {} }) - await brain.flush() - const factsDir = path.join(dir, '_generations', 'facts') - const files = fs.readdirSync(factsDir) - // The manifest rides the store's JSON object discipline (gzip on disk). - expect(files.some((f) => f.startsWith('manifest.json'))).toBe(true) - const segs = files.filter((f) => /^seg-\d{20}\.bfl$/.test(f)) - expect(segs.length).toBeGreaterThanOrEqual(1) - }) - - it('the fact-log namespace is PROTECTED: a prefix-nuke is refused', async () => { - await brain.add({ data: 'protected', type: 'document', metadata: {} }) - await expect(brain.storage.removeRawPrefix('_generations/facts')).rejects.toBeInstanceOf( - ProtectedArtifactError - ) - // Per-generation history cleanup remains unaffected (no false intersect). - await expect(brain.storage.removeRawPrefix('_generations/999999')).resolves.toBeUndefined() - }) - - it('transact facts are durable-on-return (no flush needed before reopen)', async () => { - const receipt = await brain.transact([ - { op: 'add', type: 'document', metadata: { durable: true }, data: 'tx' } - ]) - // Simulate an abrupt end: no flush(), no close() — reopen from disk. - brain = await open() - const facts = await allFacts(brain) - expect(facts.some((f) => f.generation === receipt.generation)).toBe(true) - }) -}) diff --git a/tests/integration/graph-audit.test.ts b/tests/integration/graph-audit.test.ts deleted file mode 100644 index 74fcc96a..00000000 --- a/tests/integration/graph-audit.test.ts +++ /dev/null @@ -1,164 +0,0 @@ -/** - * @module tests/integration/graph-audit - * @description brain.auditGraph() — the read-only graph-truth instrument. - * Laws under test: - * (1) a healthy brain audits COHERENT: every canonical verb is returned by the - * read path of its source, endpoints exist, no ghosts; - * (2) design-hidden edges (internal/system visibility) are counted separately - * and never misclassified as index loss; - * (3) a verb whose endpoint entity was destroyed at the storage layer (the - * scar class) is flagged as a dangling endpoint, loudly; - * (4) the classification core flags present-but-invisible and ghost edges - * exactly (exercised via injected seams — manufacturing a genuinely stale - * adjacency index end-to-end would require corrupting internals the - * public API rightly refuses to corrupt). - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/index.js' -import { NounType, VerbType } from '../../src/types/graphTypes.js' -import { runGraphAudit, type AuditVerbRecord } from '../../src/graph/graphAudit.js' - -describe('brain.auditGraph() — graph-truth audit', () => { - let dir: string - let brain: any - - beforeEach(async () => { - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-graph-audit-')) - brain = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true - }) - await brain.init() - }) - - afterEach(async () => { - await brain.close().catch(() => {}) - fs.rmSync(dir, { recursive: true, force: true }) - }) - - async function seedGraph(): Promise<{ ids: string[]; verbIds: string[] }> { - const ids: string[] = [] - for (let i = 0; i < 5; i++) { - ids.push( - await brain.add({ - data: `entity ${i}`, - type: NounType.Concept, - metadata: { n: i } - }) - ) - } - const verbIds: string[] = [] - verbIds.push(await brain.relate({ from: ids[0], to: ids[1], type: VerbType.RelatedTo })) - verbIds.push(await brain.relate({ from: ids[0], to: ids[2], type: VerbType.Contains })) - verbIds.push(await brain.relate({ from: ids[1], to: ids[3], type: VerbType.DependsOn })) - verbIds.push(await brain.relate({ from: ids[3], to: ids[4], type: VerbType.RelatedTo })) - return { ids, verbIds } - } - - it('audits a healthy brain as coherent, with exact counts', async () => { - const { ids } = await seedGraph() - const report = await brain.auditGraph() - - expect(report.coherent).toBe(true) - expect(report.verbsInCanonical).toBe(4) - expect(report.entitiesInCanonical).toBeGreaterThanOrEqual(ids.length) // VFS root etc. may add system nouns - expect(report.sourcesChecked).toBe(3) // ids[0], ids[1], ids[3] - expect(report.missingFromReadsCount).toBe(0) - expect(report.danglingEndpointsCount).toBe(0) - expect(report.readOnlyCount).toBe(0) - expect(report.truncatedExamples).toBe(false) - }) - - it('counts design-hidden edges separately and stays coherent', async () => { - const { ids } = await seedGraph() - await brain.relate({ - from: ids[2], - to: ids[4], - type: VerbType.RelatedTo, - visibility: 'internal' - }) - - const report = await brain.auditGraph() - expect(report.coherent).toBe(true) // hidden-by-design is NOT a discrepancy - expect(report.verbsInCanonical).toBe(5) - expect(report.visibilityHiddenCount).toBeGreaterThanOrEqual(1) - }) - - it('flags a destroyed endpoint as a dangling verb (the scar class)', async () => { - const { ids } = await seedGraph() - // Destroy ids[4] at the STORAGE layer (bypassing remove(), which would - // also delete its verbs) — the historical partial-delete scar shape. - await brain.storage.deleteNoun(ids[4]) - - const report = await brain.auditGraph() - expect(report.coherent).toBe(false) - expect(report.danglingEndpointsCount).toBe(1) - expect(report.danglingEndpoints[0].to).toBe(ids[4]) - expect(report.danglingEndpoints[0].missingEnd).toBe('to') - }) -}) - -describe('runGraphAudit classification core (injected seams)', () => { - const verb = (id: string, from: string, to: string): AuditVerbRecord => ({ - id, - type: 'relatedTo', - sourceId: from, - targetId: to - }) - - const deps = (opts: { - nouns: string[] - verbs: AuditVerbRecord[] - reads: Record // sourceId -> verb ids the read path returns - }) => ({ - eachNounId: async (consume: (id: string) => void) => { - for (const id of opts.nouns) consume(id) - }, - eachVerb: async (consume: (v: AuditVerbRecord) => void) => { - for (const v of opts.verbs) consume(v) - }, - readRelationsFrom: async (sourceId: string) => - (opts.reads[sourceId] ?? []).map((id) => ({ id })) - }) - - it('flags a canonical verb the read path omits — present but invisible', async () => { - const report = await runGraphAudit( - deps({ - nouns: ['A', 'B', 'C'], - verbs: [verb('v1', 'A', 'B'), verb('v2', 'A', 'C')], - reads: { A: ['v1'] } // v2 exists canonically but reads miss it - }) - ) - expect(report.coherent).toBe(false) - expect(report.missingFromReadsCount).toBe(1) - expect(report.missingFromReads[0].verbId).toBe('v2') - }) - - it('flags a read-path edge with no canonical record — a ghost', async () => { - const report = await runGraphAudit( - deps({ - nouns: ['A', 'B'], - verbs: [verb('v1', 'A', 'B')], - reads: { A: ['v1', 'ghost-9'] } - }) - ) - expect(report.coherent).toBe(false) - expect(report.readOnlyCount).toBe(1) - expect(report.readOnlyVerbIds).toEqual(['ghost-9']) - }) - - it('caps example lists but keeps counts exact, and says so', async () => { - const verbs = Array.from({ length: 10 }, (_, i) => verb(`v${i}`, 'A', 'B')) - const report = await runGraphAudit( - deps({ nouns: ['A', 'B'], verbs, reads: { A: [] } }), - { maxExamples: 3 } - ) - expect(report.missingFromReadsCount).toBe(10) - expect(report.missingFromReads.length).toBe(3) - expect(report.truncatedExamples).toBe(true) - }) -}) diff --git a/tests/integration/history-repacking.test.ts b/tests/integration/history-repacking.test.ts deleted file mode 100644 index 2bcee038..00000000 --- a/tests/integration/history-repacking.test.ts +++ /dev/null @@ -1,186 +0,0 @@ -/** - * @module tests/integration/history-repacking - * @description The D1+D3 two-tier history lifecycle end-to-end on a real - * brain. Laws: (1) repacking is RE-REPRESENTATION — after folding, every - * asOf() read below the fold boundary answers exactly as before, across a - * cold reopen; (2) folded per-generation directories are physically gone - * (the file-count cure is real, not cosmetic); (3) repack + reclaim compose: - * bounded retention after repacking drops whole segments and asOf below the - * horizon throws GenerationCompactedError; (4) repackHistory is explicit - * API and time-bounded (spent budget = consistent no-op). - * - * Uses a tiny REPACK_LIVE_WINDOW override so a small history has a cold - * tier at all (the production window is 1024). - */ -import { describe, it, expect, afterEach } from 'vitest' -import * as fs from 'node:fs' -import * as path from 'node:path' -import * as os from 'node:os' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' -import { GenerationStore } from '../../src/db/generationStore.js' -import { GenerationCompactedError } from '../../src/db/errors.js' -import { SEGMENTS_PREFIX } from '../../src/db/generationSegments.js' - -const stub = async (text: string): Promise => { - const h = text.split('').reduce((a, c) => a + c.charCodeAt(0), 0) - return new Array(384).fill(0).map((_, i) => Math.sin(h + i)) -} - -const openBrain = async (dir: string): Promise => { - const brain = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - embeddingFunction: stub - }) - await brain.init() - return brain -} - -describe('history repacking — the two-tier lifecycle', () => { - const dirs: string[] = [] - const tempDir = (): string => { - const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-repack-')) - dirs.push(d) - return d - } - const originalWindow = GenerationStore.REPACK_LIVE_WINDOW - - afterEach(() => { - ;(GenerationStore as any).REPACK_LIVE_WINDOW = originalWindow - for (const d of dirs.splice(0)) { - try { - fs.rmSync(d, { recursive: true, force: true }) - } catch { - /* best effort */ - } - } - }) - - it('repack preserves every historical read across cold reopen; folded dirs are gone', async () => { - ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3 - const dir = tempDir() - const brain = await openBrain(dir) - - const id = await brain.add({ - data: 'versioned-entity', - type: NounType.Document, - metadata: { v: 0 } - }) - for (let v = 1; v <= 10; v++) await brain.update({ id, metadata: { v } }) - await brain.flush() - - // Ground truth BEFORE repacking: capture asOf views for early generations. - const before: Record = {} - for (const g of [2, 4, 6]) { - const db = await brain.asOf(g) - before[g] = (await db.get(id))?.metadata?.v as number - await db.release() - } - - const result = await brain.repackHistory() - expect(result.foldedGenerations).toBeGreaterThan(0) - expect(result.segmentsCreated).toBeGreaterThan(0) - - // The folded per-generation directories are PHYSICALLY gone… - const genDirs = fs - .readdirSync(path.join(dir, '_generations'), { withFileTypes: true }) - .filter((e) => e.isDirectory() && /^\d+$/.test(e.name)).length - expect(genDirs).toBeLessThanOrEqual(4) // live window (3) + at most the newest - // …and the segment tier exists (the filesystem adapter stores objects - // gzipped, so the manifest may live at either spelling). - const segDir = path.join(dir, SEGMENTS_PREFIX) - expect( - fs.existsSync(path.join(segDir, 'manifest.json')) || - fs.existsSync(path.join(segDir, 'manifest.json.gz')) - ).toBe(true) - expect(fs.readdirSync(segDir).some((f) => f.endsWith('.bgs'))).toBe(true) - - // Same asOf answers from the packed tier, same process… - for (const g of [2, 4, 6]) { - const db = await brain.asOf(g) - expect((await db.get(id))?.metadata?.v).toBe(before[g]) - await db.release() - } - await brain.close() - - // …and across a COLD REOPEN (manifest discovery, no live dirs to list). - const reopened = await openBrain(dir) - for (const g of [2, 4, 6]) { - const db = await reopened.asOf(g) - expect((await db.get(id))?.metadata?.v).toBe(before[g]) - await db.release() - } - expect((await reopened.get(id))?.metadata?.v).toBe(10) // live state untouched - await reopened.close() - }) - - it('repack + bounded reclaim compose: whole segments drop, horizon is loud', async () => { - ;(GenerationStore as any).REPACK_LIVE_WINDOW = 2 - const dir = tempDir() - const brain = await openBrain(dir) - const id = await brain.add({ data: 'reclaim-probe', type: NounType.Document, metadata: { v: 0 } }) - for (let v = 1; v <= 8; v++) await brain.update({ id, metadata: { v } }) - await brain.flush() - await brain.repackHistory() - - // Reclaim down to the 3 newest generations — packed segments below the - // horizon drop whole; asOf below throws loudly. - const res = await brain.compactHistory({ maxGenerations: 3 }) - expect(res.removedGenerations).toBeGreaterThan(0) - await expect(brain.asOf(1)).rejects.toBeInstanceOf(GenerationCompactedError) - expect((await brain.get(id))?.metadata?.v).toBe(8) - await brain.close() - }) - - it('generationDigest: reopen-stable, divergence-sensitive, loud below the horizon', async () => { - ;(GenerationStore as any).REPACK_LIVE_WINDOW = 2 - const dir = tempDir() - const brain = await openBrain(dir) - const id = await brain.add({ data: 'digest-probe', type: NounType.Document, metadata: { v: 0 } }) - for (let v = 1; v <= 6; v++) await brain.update({ id, metadata: { v } }) - await brain.flush() - await brain.repackHistory() - - const gen = brain.generation() - const atHead = await brain.generationDigest(gen) - const atMid = await brain.generationDigest(3) - expect(atHead).toMatch(/^[0-9a-f]{8}$/) - expect(atMid).not.toBe(atHead) // more history ⇒ different digest - await brain.close() - - // Reopen-stable: same history, same digests (packed prefix stability). - const reopened = await openBrain(dir) - expect(await reopened.generationDigest(gen)).toBe(atHead) - expect(await reopened.generationDigest(3)).toBe(atMid) - - // New history diverges the head digest. - await reopened.update({ id, metadata: { v: 7 } }) - await reopened.flush() - expect(await reopened.generationDigest(reopened.generation())).not.toBe(atHead) - - // Below the horizon: LOUD, never a silent pin of reclaimed history. - await reopened.compactHistory({ maxGenerations: 2 }) - await expect(reopened.generationDigest(1)).rejects.toBeInstanceOf(GenerationCompactedError) - await reopened.close() - }) - - it('a spent time budget is a consistent no-op; the next pass resumes', async () => { - ;(GenerationStore as any).REPACK_LIVE_WINDOW = 2 - const dir = tempDir() - const brain = await openBrain(dir) - const id = await brain.add({ data: 'budget-probe', type: NounType.Document, metadata: { v: 0 } }) - for (let v = 1; v <= 6; v++) await brain.update({ id, metadata: { v } }) - await brain.flush() - - const bounded = await brain.repackHistory({ timeBudgetMs: 0 }) - expect(bounded).toEqual({ foldedGenerations: 0, segmentsCreated: 0 }) - - const resumed = await brain.repackHistory() - expect(resumed.foldedGenerations).toBeGreaterThan(0) - const db = await brain.asOf(3) - expect((await db.get(id))?.metadata?.v).toBeDefined() - await db.release() - await brain.close() - }) -}) diff --git a/tests/integration/ifabsent-upsert-blob-concurrency.test.ts b/tests/integration/ifabsent-upsert-blob-concurrency.test.ts deleted file mode 100644 index 119a5a97..00000000 --- a/tests/integration/ifabsent-upsert-blob-concurrency.test.ts +++ /dev/null @@ -1,212 +0,0 @@ -/** - * @module tests/integration/ifabsent-upsert-blob-concurrency - * @description The 8.0.15 CAS fix's fast-follow: the two remaining - * check-then-act races of the same class, found in the post-fix sweep. - * - * 1. `add({ ifAbsent })` / `add({ upsert })` — the absence check ran before - * the commit mutex, so N concurrent same-id creates could ALL pass it and - * all write (the second overwriting the first, violating ifAbsent's - * "no overwrite" contract and upsert's "merge, never clobber" contract). - * Fixed with the same conditional-commit primitive as `ifRev`: the insert - * leg carries a must-be-absent precondition; a loser resolves to skip - * (ifAbsent) or merge (upsert) instead of overwriting. - * - * 2. `BlobStorage` reference counts — `write()`'s dedup decision and - * `delete()`'s decrement-then-remove were unserialized read-modify-writes - * over `blob-meta:`; concurrent same-content writes could lose - * references, turning a later delete into premature removal of bytes - * another file still needs. Fixed with a per-hash mutex. - * - * Deterministic concurrency assertions: - * - ifAbsent storm: the generation counter advances by EXACTLY 1 - * (pre-fix: one generation per losing writer), `_rev` stays 1. - * - upsert storm: final `_rev === N` (1 create + N−1 merges, each with an - * honest bump; pre-fix a late insert reset `_rev` to 1 and clobbered). - * - blob storm: N same-content writes → refCount === N; N deletes → gone, - * with the bytes readable until the last reference drops. - */ -import { describe, it, expect, beforeAll, afterAll } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' -import { BlobStorage } from '../../src/storage/blobStorage.js' - -describe('ifAbsent/upsert insert race + blob refCount (conditional commit fast-follow)', () => { - let dir: string - let brain: any - let seq = 0 - const freshId = (): string => - `00000000-0000-4000-8000-${(++seq).toString(16).padStart(12, '0')}` - - beforeAll(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-upsert-race-')) - brain = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - dimensions: 384, - silent: true - }) - await brain.init() - }) - - afterAll(async () => { - await brain.close() - fs.rmSync(dir, { recursive: true, force: true }) - }) - - it('N concurrent add({ifAbsent}) → exactly ONE write (generation +1), no overwrite', async () => { - const id = freshId() - const genBefore = brain.generationStore.generation() - - const ids = await Promise.all( - Array.from({ length: 8 }, (_, i) => - brain.add({ - id, - ifAbsent: true, - data: `contender ${i}`, - type: NounType.Thing, - metadata: { writer: i } - }) - ) - ) - - // Every caller resolves to the same canonical id. - expect(new Set(ids).size).toBe(1) - - // THE contract: exactly one write landed. Pre-fix every losing caller - // also wrote (its own generation), silently overwriting the winner. - expect(brain.generationStore.generation()).toBe(genBefore + 1) - - const after = await brain.get(id) - expect(after._rev).toBe(1) - expect(typeof after.metadata.writer).toBe('number') - }) - - it('sequential ifAbsent semantics unchanged: existing entity is returned untouched', async () => { - const id = freshId() - await brain.add({ id, data: 'original', type: NounType.Thing, metadata: { keep: true } }) - const returned = await brain.add({ - id, - ifAbsent: true, - data: 'impostor', - type: NounType.Thing, - metadata: { keep: false } - }) - expect(returned).toBe(id) - const after = await brain.get(id) - expect(after.metadata.keep).toBe(true) - expect(after.data).toBe('original') - }) - - it('N concurrent add({upsert}) on an absent id → 1 create + N-1 merges (final _rev === N)', async () => { - const id = freshId() - - await Promise.all( - Array.from({ length: 8 }, (_, i) => - brain.add({ - id, - upsert: true, - data: 'shared upsert target', - type: NounType.Thing, - metadata: { [`k${i}`]: true } - }) - ) - ) - - const after = await brain.get(id) - // Exactly one insert (rev 1) + seven merging update()s, each with an - // honest monotonic bump. Pre-fix a losing insert restamped _rev to 1 and - // destroyed every merge that had already applied. - expect(after._rev).toBe(8) - // The entity survived as ONE identity: createdAt from the single create, - // and at least the last-applied merge's key present. - expect(Object.keys(after.metadata).filter((k) => k.startsWith('k')).length) - .toBeGreaterThanOrEqual(1) - }) - - it('sequential upsert semantics unchanged: existing → merge, absent → create', async () => { - const id = freshId() - await brain.add({ id, upsert: true, data: 'v1', type: NounType.Thing, metadata: { a: 1 } }) - expect((await brain.get(id))._rev).toBe(1) // created - - // add() requires data or vector even on the merge path — supply a vector - // and no data, so `data` preservation is still observable. - const vec = Array.from({ length: 384 }, (_, i) => (i % 7) / 7 - 0.5) - await brain.add({ id, upsert: true, vector: vec, type: NounType.Thing, metadata: { b: 2 } }) - const after = await brain.get(id) - expect(after._rev).toBe(2) // merged, not overwritten - expect(after.metadata.a).toBe(1) - expect(after.metadata.b).toBe(2) - expect(after.data).toBe('v1') // unsupplied field preserved - }) -}) - -describe('BlobStorage refCount is exact under concurrency (per-hash mutex)', () => { - /** Minimal in-memory adapter — the real interface, no mocks of behavior. */ - function memAdapter() { - const kv = new Map() - return { - get: async (k: string) => kv.get(k), - put: async (k: string, v: Buffer) => void kv.set(k, v), - delete: async (k: string) => void kv.delete(k), - list: async (prefix: string) => - [...kv.keys()].filter((k) => k.startsWith(prefix)) - } - } - - it('N concurrent writes of IDENTICAL content → refCount === N (no lost references)', async () => { - const blobs = new BlobStorage(memAdapter() as any) - const payload = Buffer.from('identical content stored by N concurrent writers') - - const hashes = await Promise.all( - Array.from({ length: 10 }, () => blobs.write(payload)) - ) - expect(new Set(hashes).size).toBe(1) - const meta = await blobs.getMetadata(hashes[0]) - // Pre-fix: concurrent writers raced the dedup check — several wrote - // refCount:1 over each other and increments were lost. - expect(meta?.refCount).toBe(10) - }) - - it('references drop exactly; bytes survive live-zero (temporal immutability) until reclaim', async () => { - const blobs = new BlobStorage(memAdapter() as any) - const payload = Buffer.from('shared bytes, two referencing files') - - const hash = await blobs.write(payload) - await blobs.write(payload) // second reference (concurrent-equivalent path) - - await blobs.release(hash) // drop one reference - expect((await blobs.read(hash)).toString()).toBe(payload.toString()) // still readable - expect((await blobs.getMetadata(hash))?.refCount).toBe(1) - - await blobs.release(hash) // last LIVE reference — bytes still exist (history may need them) - expect(await blobs.has(hash)).toBe(true) - expect((await blobs.getMetadata(hash))?.refCount).toBe(0) - - // Reclamation is compaction's job: zero-zero → physically removed. - expect(await blobs.reclaimIfUnreferenced(hash)).toBe(true) - expect(await blobs.has(hash)).toBe(false) - await expect(blobs.read(hash)).rejects.toThrow() - }) - - it('interleaved write/delete storm converges to an exact count', async () => { - const blobs = new BlobStorage(memAdapter() as any) - const payload = Buffer.from('storm payload') - const hash = BlobStorage.hash(payload) - - // 12 writes and 5 releases racing: net 7 references, blob alive. - await Promise.all([ - ...Array.from({ length: 12 }, () => blobs.write(payload)), - ...Array.from({ length: 5 }, () => blobs.release(hash)) - ]) - const meta = await blobs.getMetadata(hash) - // Releases against a not-yet-written hash floor at zero, so the net can - // only be >= 12 - 5. The exactness we require: counts are never LOST - // (each landed write is represented). - expect(meta?.refCount).toBeGreaterThanOrEqual(7) - expect(await blobs.has(hash)).toBe(true) - }) -}) diff --git a/tests/integration/ifrev-concurrent-cas.test.ts b/tests/integration/ifrev-concurrent-cas.test.ts deleted file mode 100644 index 4b04c58f..00000000 --- a/tests/integration/ifrev-concurrent-cas.test.ts +++ /dev/null @@ -1,187 +0,0 @@ -/** - * @module tests/integration/ifrev-concurrent-cas - * @description Regression for the P0 report that `update({ ifRev })` CAS was - * not atomic under concurrent in-process calls: N concurrent updates all - * carrying the same `ifRev` ALL fulfilled (0 conflicts, last-writer-wins) — - * the check ran before the commit mutex, so interleaved callers all passed it - * before any apply landed. In production that silently lost 7 of 8 - * ifRev-guarded ledger writes and let two workers both "acquire" an advisory - * lock built on exactly-one-winner semantics. - * - * The fix is a conditional commit: the generation store's `precommit` hook - * re-verifies `ifRev` against the authoritative before-image UNDER the commit - * mutex, atomically with the apply (the per-record analogue of - * `ifAtGeneration`, which always ran there). These tests pin: - * - exactly-one-winner for N concurrent same-rev `update({ ifRev })` - * - the same for `transact()` per-op ifRev (whole batch rejected) - * - `_rev` monotonicity under concurrent plain updates (no ifRev) - * - the CAS retry loop converging exactly (the ledger/credit-consume shape) - * - sequential conflict behavior unchanged - */ -import { describe, it, expect, beforeAll, afterAll } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' - -const isRevConflict = (e: unknown): boolean => - (e as Error)?.name === 'RevisionConflictError' || - String(e).includes('RevisionConflict') - -describe('ifRev CAS is atomic under concurrency (conditional commit)', () => { - let dir: string - let brain: any - let seq = 0 - const freshId = (): string => - `00000000-0000-4000-8000-${(++seq).toString(16).padStart(12, '0')}` - - beforeAll(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-cas-')) - brain = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - dimensions: 384, - silent: true - }) - await brain.init() - }) - - afterAll(async () => { - await brain.close() - fs.rmSync(dir, { recursive: true, force: true }) - }) - - it('N concurrent update({ifRev}) → exactly 1 winner, N-1 RevisionConflictError', async () => { - const id = await brain.add({ - id: freshId(), - data: 'contended entity', - type: NounType.Thing, - metadata: { credits: 8 } - }) - const rev = (await brain.get(id))._rev - - const results = await Promise.allSettled( - Array.from({ length: 8 }, (_, i) => - brain.update({ id, metadata: { writer: i }, merge: false, ifRev: rev }) - ) - ) - - const wins = results.filter((r) => r.status === 'fulfilled') - const conflicts = results.filter( - (r) => r.status === 'rejected' && isRevConflict(r.reason) - ) - expect(wins.length).toBe(1) - expect(conflicts.length).toBe(7) - - // Exactly one bump; the surviving metadata belongs to the single winner. - const after = await brain.get(id) - expect(after._rev).toBe(rev + 1) - expect(typeof after.metadata.writer).toBe('number') - }) - - it('transact() per-op ifRev → exactly 1 batch wins, others rejected whole', async () => { - const id = await brain.add({ - id: freshId(), - data: 'transact contended', - type: NounType.Thing, - metadata: { state: 'initial' } - }) - const rev = (await brain.get(id))._rev - - const results = await Promise.allSettled( - Array.from({ length: 6 }, (_, i) => - brain.transact([ - { op: 'update', id, metadata: { batch: i }, merge: false, ifRev: rev } - ]) - ) - ) - expect(results.filter((r) => r.status === 'fulfilled').length).toBe(1) - expect( - results.filter((r) => r.status === 'rejected' && isRevConflict(r.reason)) - .length - ).toBe(5) - expect((await brain.get(id))._rev).toBe(rev + 1) - }) - - it('_rev is monotonic under concurrent plain updates (no ifRev): N updates → +N', async () => { - const id = await brain.add({ - id: freshId(), - data: 'plain concurrent updates', - type: NounType.Thing, - metadata: { v: 0 } - }) - const rev = (await brain.get(id))._rev - - await Promise.all( - Array.from({ length: 8 }, (_, i) => brain.update({ id, metadata: { v: i } })) - ) - // Every applied update gets its own honest bump (previously all stamped - // the same stale rev+1 they computed before the commit). - expect((await brain.get(id))._rev).toBe(rev + 8) - }) - - it('the CAS retry loop converges exactly (the ledger shape: 8 workers, 8 consumes)', async () => { - const id = await brain.add({ - id: freshId(), - data: 'credit budget', - type: NounType.Thing, - metadata: { credits: 8 } - }) - - // Each worker: read → CAS-decrement → retry on conflict. The docs' documented - // pattern; with atomic CAS this MUST land on exactly 0 with zero lost updates. - const consumeOne = async (): Promise => { - for (let attempt = 0; attempt < 50; attempt++) { - const cur = await brain.get(id) - if (cur.metadata.credits <= 0) throw new Error('budget exhausted') - try { - await brain.update({ - id, - metadata: { ...cur.metadata, credits: cur.metadata.credits - 1 }, - merge: false, - ifRev: cur._rev - }) - return - } catch (e) { - if (!isRevConflict(e)) throw e - } - } - throw new Error('no convergence after 50 attempts') - } - - await Promise.all(Array.from({ length: 8 }, () => consumeOne())) - expect((await brain.get(id)).metadata.credits).toBe(0) - }) - - it('sequential conflict behavior is unchanged (stale ifRev throws, fresh succeeds)', async () => { - const id = await brain.add({ - id: freshId(), - data: 'sequential control', - type: NounType.Thing, - metadata: { n: 1 } - }) - const rev = (await brain.get(id))._rev - - await brain.update({ id, metadata: { n: 2 }, ifRev: rev }) - await expect( - brain.update({ id, metadata: { n: 3 }, ifRev: rev }) - ).rejects.toMatchObject({ name: 'RevisionConflictError' }) - // Fresh rev succeeds. - const fresh = (await brain.get(id))._rev - await brain.update({ id, metadata: { n: 3 }, ifRev: fresh }) - expect((await brain.get(id)).metadata.n).toBe(3) - }) - - it('forward-ref add+update of the same entity in one transact() batch still works', async () => { - const id = freshId() - await brain.transact([ - { op: 'add', id, data: 'created in batch', type: NounType.Thing, metadata: { step: 1 } }, - { op: 'update', id, metadata: { step: 2 } } - ]) - const after = await brain.get(id) - expect(after.metadata.step).toBe(2) - expect(after._rev).toBe(2) // add stamps 1, in-batch update sequences to 2 - }) -}) diff --git a/tests/integration/lens-consistency.test.ts b/tests/integration/lens-consistency.test.ts deleted file mode 100644 index 64484a38..00000000 --- a/tests/integration/lens-consistency.test.ts +++ /dev/null @@ -1,138 +0,0 @@ -/** - * @module tests/integration/lens-consistency - * @description The three metadata "lenses" over one corpus must agree with - * canonical ground truth id-for-id, warm AND after a cold reopen: - * - combined: find({ type: T, where: { subtype: S } }) - * - subtype-only: find({ where: { subtype: S } }) - * - type-only: find({ type: T }) - * Ported from the fresh-brain probe that closed the type+subtype lens-drop - * investigation (a restored pre-8.2.2 torn capture had entities visible to the - * subtype-only lens but dropped by the combined lens — "0 of 2 migrated, all - * gates green"). The corpus is seeded through the REAL write API — never - * restored bytes — which is what made the original datapoint decisive. The - * invariants: every lens matches an unfiltered canonical scan exactly (no - * missing ids, no extras) and combined ⊆ subtype-only always holds. - */ -import { describe, it, expect, beforeAll, afterAll } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/index.js' - -/** The corpus: 7 (type, subtype) pairs, uneven counts, incl. the incident's 2-of-a-pair shape. */ -const CORPUS: Array<{ type: string; subtype: string; count: number }> = [ - { type: 'proposition', subtype: 'decision', count: 2 }, // the incident shape: "0 of 2" - { type: 'concept', subtype: 'decision', count: 3 }, - { type: 'task', subtype: 'decision', count: 2 }, - { type: 'concept', subtype: 'action', count: 4 }, - { type: 'message', subtype: 'note', count: 5 }, - { type: 'message', subtype: 'ship', count: 3 }, - { type: 'document', subtype: 'guide', count: 4 } -] - -/** Canonical ground truth: unfiltered enumeration, post-filtered IN THE TEST. */ -async function groundTruth( - brain: any, - match: { type?: string; subtype?: string } -): Promise> { - const ids = new Set() - let cursor: string | undefined - for (;;) { - const page = await brain.storage.getNounsWithPagination({ limit: 500, cursor }) - for (const noun of page.items) { - // Hydrated shape: `type`/`subtype` are TOP-LEVEL; `metadata` holds only - // custom user fields (vfsType is one — the VFS plumbing marker). - const n = noun as any - if (n.metadata?.vfsType) continue // VFS plumbing is not corpus - if (!n.type || !n.subtype) continue - if (match.type && n.type !== match.type) continue - if (match.subtype && n.subtype !== match.subtype) continue - ids.add(n.id) - } - if (!page.hasMore) break - cursor = page.nextCursor - } - return ids -} - -const idSet = (results: Array<{ id: string }>): Set => new Set(results.map((r) => r.id)) - -/** Every lens vs ground truth, id-for-id, for every pair in the corpus. */ -async function assertAllLenses(brain: any): Promise { - const types = [...new Set(CORPUS.map((c) => c.type))] - const subtypes = [...new Set(CORPUS.map((c) => c.subtype))] - - for (const { type, subtype } of CORPUS) { - const combined = idSet(await brain.find({ type, where: { subtype }, limit: 1000 })) - const subtypeOnly = idSet(await brain.find({ where: { subtype }, limit: 1000 })) - const truthPair = await groundTruth(brain, { type, subtype }) - const truthSubtype = await groundTruth(brain, { subtype }) - - expect([...combined].sort()).toEqual([...truthPair].sort()) // no drops, no extras - expect([...subtypeOnly].sort()).toEqual([...truthSubtype].sort()) - for (const id of combined) expect(subtypeOnly.has(id)).toBe(true) // combined ⊆ subtype-only - } - - for (const type of types) { - const typeOnly = idSet(await brain.find({ type, limit: 1000 })) - const truthType = await groundTruth(brain, { type }) - expect([...typeOnly].sort()).toEqual([...truthType].sort()) - } - - // Count cross-check against the corpus definition itself. - for (const subtype of subtypes) { - const expected = CORPUS.filter((c) => c.subtype === subtype).reduce((s, c) => s + c.count, 0) - const got = (await brain.find({ where: { subtype }, limit: 1000 })).length - expect(got).toBe(expected) - } -} - -describe('lens consistency — combined vs subtype-only vs canonical ground truth', () => { - let dir: string - let brain: any - - beforeAll(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-lens-')) - brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 }) - await brain.init() - // Seed through the REAL write API — never restored bytes. - let i = 0 - for (const { type, subtype, count } of CORPUS) { - for (let k = 0; k < count; k++) { - await brain.add({ data: `${type} ${subtype} ${i++}`, type, subtype, metadata: { k } }) - } - } - await brain.flush() - }) - afterAll(async () => { - await brain.close?.().catch(() => {}) - fs.rmSync(dir, { recursive: true, force: true }) - }) - - it('WARM: all lenses agree with ground truth id-for-id', async () => { - await assertAllLenses(brain) - }) - - it('COLD REOPEN: all lenses still agree after close + reopen from disk', async () => { - await brain.close() - brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 }) - await brain.init() - await assertAllLenses(brain) - }) - - it('after an update() flips type AND subtype, every lens tracks the move exactly', async () => { - // The historical cross-bucket-staleness path: change (concept, action) -> (task, review). - const victims = await brain.find({ type: 'concept', where: { subtype: 'action' }, limit: 1 }) - expect(victims.length).toBe(1) - const id = victims[0].id - await brain.update({ id, type: 'task', subtype: 'review' }) - - const oldCombined = idSet(await brain.find({ type: 'concept', where: { subtype: 'action' }, limit: 1000 })) - expect(oldCombined.has(id)).toBe(false) // unposted from the old buckets - const newCombined = idSet(await brain.find({ type: 'task', where: { subtype: 'review' }, limit: 1000 })) - expect(newCombined.has(id)).toBe(true) // posted to the new buckets - const subtypeOnly = idSet(await brain.find({ where: { subtype: 'review' }, limit: 1000 })) - expect(subtypeOnly.has(id)).toBe(true) - }) -}) diff --git a/tests/integration/multi-process-safety.test.ts b/tests/integration/multi-process-safety.test.ts index 592d7969..6ac5265d 100644 --- a/tests/integration/multi-process-safety.test.ts +++ b/tests/integration/multi-process-safety.test.ts @@ -110,89 +110,6 @@ describe('Multi-process safety + read-only mode', () => { // Don't track `blocked` for afterEach cleanup since init failed. }) - it('takes over a STALE foreign lock (dead PID + old heartbeat) and claims atomically', async () => { - const { mkdirSync, writeFileSync, readFileSync } = await import('node:fs') - const { join } = await import('node:path') - const os = await import('node:os') - mkdirSync(join(dir, 'locks'), { recursive: true }) - const tenMinutesAgo = new Date(Date.now() - 10 * 60 * 1000).toISOString() - writeFileSync(join(dir, 'locks', '_writer.lock'), JSON.stringify({ - pid: 999999999, // no such process — provably dead - hostname: os.hostname(), - startedAt: tenMinutesAgo, - lastHeartbeat: tenMinutesAgo, - version: '8.0.0', - rootDir: dir - })) - - writer = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await writer.init() // stale takeover must succeed - - const lock = JSON.parse(readFileSync(join(dir, 'locks', '_writer.lock'), 'utf-8')) - expect(lock.pid).toBe(process.pid) // the atomic wx claim installed OUR lock - }) - - it('the writer-locked error carries the machine-readable contract (code + lockInfo)', async () => { - const { mkdirSync, writeFileSync } = await import('node:fs') - const { join } = await import('node:path') - const os = await import('node:os') - mkdirSync(join(dir, 'locks'), { recursive: true }) - const otherPid = (process as any).ppid || 1 - writeFileSync(join(dir, 'locks', '_writer.lock'), JSON.stringify({ - pid: otherPid, - hostname: os.hostname(), - startedAt: new Date().toISOString(), - lastHeartbeat: new Date().toISOString(), - version: '8.7.0', - rootDir: dir - })) - - const blocked = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - const err: any = await blocked.init().catch((e) => e) - expect(err.code).toBe('BRAINY_WRITER_LOCKED') - expect(err.lockInfo?.pid).toBe(otherPid) - }) - - it('release drains an in-flight heartbeat — no phantom lock re-created after unlink', async () => { - // The race (8.9.0): clearInterval stops FUTURE heartbeat ticks, but a - // tick already in flight could land its lock rewrite AFTER release's - // unlink — re-creating the lock as a phantom that blocks the next - // writer until the stale TTL. Simulate the in-flight tick explicitly - // and prove release waits for it. - writer = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await writer.init() - const storage: any = (writer as any).storage - - // An in-flight refresh that is ALREADY PAST its ownership guards - // (captured the lock info before release ran) and lands its atomic - // rewrite slowly — the exact straggler shape; absent the drain it - // writes after the unlink. - const { join: joinPath } = await import('node:path') - const capturedInfo = { ...storage.writerLockInfo } - const lockPath = joinPath(dir, 'locks', '_writer.lock') - const slowTick = (async () => { - await new Promise((r) => setTimeout(r, 100)) - await storage.writeFileAtomic( - lockPath, - JSON.stringify({ ...capturedInfo, lastHeartbeat: new Date().toISOString() }) - ) - })() - storage.writerHeartbeatInFlight = slowTick.catch(() => {}) - - await writer.close() // → releaseWriterLock must drain slowTick first - await slowTick.catch(() => {}) // both paths fully settled either way - writer = null - - const { existsSync } = await import('node:fs') - const { join } = await import('node:path') - expect(existsSync(join(dir, 'locks', '_writer.lock'))).toBe(false) - - // And the directory is immediately claimable — no stale-TTL wait. - const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await expect(next.init()).resolves.toBeUndefined() - await next.close() - }) - it('allows a second in-process writer with a warning (same PID)', async () => { // Two Brainy instances in the same Node process: not the dangerous // cross-process case. Should succeed (with a console warning). diff --git a/tests/integration/onchange-feed.test.ts b/tests/integration/onchange-feed.test.ts deleted file mode 100644 index 8470b59c..00000000 --- a/tests/integration/onchange-feed.test.ts +++ /dev/null @@ -1,228 +0,0 @@ -/** - * @module tests/integration/onchange-feed - * @description The `brain.onChange` in-process change feed — the authoritative - * post-commit signal for every canonical mutation, regardless of origin. - * Pins the delivery contract: - * - every operation fires exactly once per affected record with the - * post-commit payload (deletes carry the record's LAST state); - * - batch operations emit one event per item; transact items share one - * generation; - * - VFS writes (a Tier-1 blind spot for router-synthesized events) emit; - * - events arrive in commit order with monotonic generations; - * - a losing `ifRev` CAS emits NOTHING (aborted commits are invisible); - * - a throwing listener is isolated; unsubscribe stops delivery; - * - clear()/restore() emit one store-level "refetch everything" event. - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/brainy.js' -import type { BrainyChangeEvent } from '../../src/events/changeFeed.js' -import { NounType, VerbType } from '../../src/types/graphTypes.js' - -let seq = 0 -const freshId = (): string => - `00000000-0000-4000-8000-${(++seq).toString(16).padStart(12, '0')}` - -/** Let the microtask-dispatched feed drain. */ -const drained = (): Promise => new Promise((r) => setTimeout(r, 0)) - -describe('brain.onChange — the in-process change feed', () => { - let dir: string - let brain: any - let events: BrainyChangeEvent[] - let off: () => void - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-onchange-')) - brain = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - dimensions: 384, - silent: true - }) - await brain.init() - events = [] - off = brain.onChange((e: BrainyChangeEvent) => events.push(e)) - }) - - afterEach(async () => { - off() - await brain.close() - fs.rmSync(dir, { recursive: true, force: true }) - }) - - it('add → update → remove: one fully-described event each; remove carries the last state', async () => { - const id = await brain.add({ - id: freshId(), - data: 'a document', - type: NounType.Document, - metadata: { status: 'draft' } - }) - await brain.update({ id, metadata: { status: 'published' } }) - await brain.remove(id) - await drained() - - expect(events.map((e) => e.op)).toEqual(['add', 'update', 'remove']) - expect(events.every((e) => e.kind === 'entity' && e.id === id)).toBe(true) - - expect(events[0].entity).toMatchObject({ id, type: 'document', metadata: { status: 'draft' } }) - expect(events[1].entity).toMatchObject({ id, metadata: { status: 'published' } }) - // The delete payload is the record's LAST committed state. - expect(events[2].entity).toMatchObject({ id, type: 'document', metadata: { status: 'published' } }) - - // Post-commit ordering: generations are monotonic. - const gens = events.map((e) => e.generation!) - expect([...gens].sort((a, b) => a - b)).toEqual(gens) - expect(new Set(gens).size).toBe(3) - }) - - it('relate → updateRelation → unrelate: relation events with endpoints + type', async () => { - const a = await brain.add({ id: freshId(), data: 'a', type: NounType.Thing }) - const b = await brain.add({ id: freshId(), data: 'b', type: NounType.Thing }) - events.length = 0 - - const rel = await brain.relate({ from: a, to: b, type: VerbType.RelatedTo, metadata: { w: 1 } }) - await brain.updateRelation({ id: rel, metadata: { w: 2 } }) - await brain.unrelate(rel) - await drained() - - expect(events.map((e) => e.op)).toEqual(['relate', 'updateRelation', 'unrelate']) - expect(events.every((e) => e.kind === 'relation' && e.id === rel)).toBe(true) - for (const e of events) { - expect(e.relation).toMatchObject({ id: rel, from: a, to: b }) - } - }) - - it('remove() cascades: deleting an entity emits unrelate for its relationships too', async () => { - const a = await brain.add({ id: freshId(), data: 'src', type: NounType.Thing }) - const b = await brain.add({ id: freshId(), data: 'tgt', type: NounType.Thing }) - const rel = await brain.relate({ from: a, to: b, type: VerbType.Contains }) - events.length = 0 - - await brain.remove(a) - await drained() - - const removeEvent = events.find((e) => e.op === 'remove') - const unrelateEvent = events.find((e) => e.op === 'unrelate') - expect(removeEvent).toMatchObject({ kind: 'entity', id: a }) - expect(unrelateEvent).toMatchObject({ - kind: 'relation', - id: rel, - relation: { from: a, to: b } - }) - // Entity + cascaded verb share the one remove() generation. - expect(removeEvent!.generation).toBe(unrelateEvent!.generation) - }) - - it('batches emit one event per item; removeMany deletes carry payloads', async () => { - const ids = [freshId(), freshId(), freshId()] - await brain.addMany({ - items: ids.map((id, i) => ({ - id, - data: `item ${i}`, - type: NounType.Thing, - metadata: { n: i } - })) - }) - await drained() - expect(events.filter((e) => e.op === 'add').length).toBe(3) - - events.length = 0 - await brain.removeMany({ ids }) - await drained() - const removes = events.filter((e) => e.op === 'remove') - expect(removes.length).toBe(3) - // Every delete is fully described (enriched, not id-only). - for (const e of removes) { - expect(e.entity).toBeDefined() - expect(e.entity!.type).toBe('thing') - expect(typeof e.entity!.metadata.n).toBe('number') - } - }) - - it('transact(): per-item events sharing ONE generation; a rejected batch emits nothing', async () => { - const x = freshId() - const y = freshId() - await brain.transact([ - { op: 'add', id: x, data: 'x', type: NounType.Thing, metadata: { k: 1 } }, - { op: 'add', id: y, data: 'y', type: NounType.Thing }, - { op: 'relate', from: x, to: y, type: VerbType.RelatedTo } - ]) - await drained() - - expect(events.map((e) => e.op)).toEqual(['add', 'add', 'relate']) - expect(new Set(events.map((e) => e.generation)).size).toBe(1) - - // Rejected batch (stale per-op CAS) → zero events. - events.length = 0 - await expect( - brain.transact([{ op: 'update', id: x, metadata: { k: 2 }, ifRev: 999 }]) - ).rejects.toMatchObject({ name: 'RevisionConflictError' }) - await drained() - expect(events.length).toBe(0) - }) - - it('a losing ifRev CAS update emits NOTHING; the winner emits once', async () => { - const id = await brain.add({ id: freshId(), data: 'contended', type: NounType.Thing }) - const rev = (await brain.get(id))._rev - events.length = 0 - - const results = await Promise.allSettled( - Array.from({ length: 4 }, (_, i) => - brain.update({ id, metadata: { w: i }, merge: false, ifRev: rev }) - ) - ) - await drained() - - expect(results.filter((r) => r.status === 'fulfilled').length).toBe(1) - expect(events.filter((e) => e.op === 'update').length).toBe(1) // exactly the winner - }) - - it('VFS writes emit (the router-synthesis blind spot)', async () => { - await brain.vfs.writeFile('/notes/hello.txt', 'hello feed') - await drained() - // A VFS write is entities + containment relations under the hood — the - // feed sees it because VFS delegates to canonical brain methods. - expect(events.length).toBeGreaterThan(0) - expect(events.some((e) => e.kind === 'entity' && e.op === 'add')).toBe(true) - }) - - it('listener errors are isolated; unsubscribe stops delivery', async () => { - const good: BrainyChangeEvent[] = [] - const offBad = brain.onChange(() => { - throw new Error('subscriber bug') - }) - const offGood = brain.onChange((e: BrainyChangeEvent) => good.push(e)) - - const id = await brain.add({ id: freshId(), data: 'p', type: NounType.Thing }) - await drained() - expect(good.length).toBeGreaterThan(0) // sibling unaffected by the throwing listener - - offBad() - offGood() - good.length = 0 - events.length = 0 - off() // unsubscribe the outer listener too - await brain.update({ id, metadata: { after: true } }) - await drained() - expect(events.length).toBe(0) - expect(good.length).toBe(0) - - // Re-subscribe for afterEach symmetry. - off = brain.onChange((e: BrainyChangeEvent) => events.push(e)) - }) - - it('clear() emits one store-level event meaning "refetch everything"', async () => { - await brain.add({ id: freshId(), data: 'doomed', type: NounType.Thing }) - events.length = 0 - await brain.clear() - await drained() - const store = events.filter((e) => e.kind === 'store') - expect(store).toHaveLength(1) - expect(store[0].op).toBe('clear') - expect(store[0].generation).toBeUndefined() - }) -}) diff --git a/tests/integration/read-your-writes-contract.test.ts b/tests/integration/read-your-writes-contract.test.ts deleted file mode 100644 index aa0c93e5..00000000 --- a/tests/integration/read-your-writes-contract.test.ts +++ /dev/null @@ -1,133 +0,0 @@ -/** - * @module tests/integration/read-your-writes-contract - * @description The read-your-writes contract — brainy's spine guarantee that a - * write which has RETURNED is immediately readable, under the single writer, by - * BOTH id and every index (metadata filter, vector search, graph traversal), - * with NO await, delay, or retry between the write returning and the read. - * - * Why it holds by construction on the canonical + JS-index path: every mutation - * runs its canonical writes AND all derived-index projections (HNSW, metadata - * index, graph index) as operations inside ONE transaction that commits before - * the method returns — there is no post-return "eventual" indexing window. get() - * additionally rides the storage write-through cache (read-after-write within - * the process). The generation counter is the {seq} a caller can pin: a write - * returns generation N and every subsequent live read observes ≥ N. - * - * This pins the guarantee so no future change can quietly move index maintenance - * off the commit path (which would turn a returned write into a not-yet-queryable - * one — the "a 200 is still not durability/queryability" failure the spine - * program exists to make impossible). The native accelerator must honor the same - * in-commit contract; a background consolidation/optimization must never gate - * queryability of an acknowledged write. - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType, VerbType } from '../../src/types/graphTypes.js' - -let seq = 0 -const freshId = (): string => - `00000000-0000-4000-8000-${(++seq).toString(16).padStart(12, '0')}` - -describe('read-your-writes contract (single-writer, no await between write and read)', () => { - let dir: string - let brain: any - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-ryw-')) - brain = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - dimensions: 384, - silent: true - }) - await brain.init() - }) - - afterEach(async () => { - await brain.close() - fs.rmSync(dir, { recursive: true, force: true }) - }) - - it('add(): the entity is immediately readable by id, metadata filter, AND vector search', async () => { - const id = freshId() - const marker = `ryw-${id}` - await brain.add({ id, data: 'read your writes probe alpha', type: NounType.Concept, metadata: { marker } }) - - // id read (write-through cache / canonical) - expect(await brain.get(id)).not.toBeNull() - // metadata-filter read (metadata index committed in-transaction) - const byMeta = await brain.find({ where: { marker }, limit: 10 }) - expect(byMeta.map((r: any) => r.id)).toContain(id) - // vector read (HNSW committed in-transaction) - const byVector = await brain.find({ query: 'read your writes probe alpha', limit: 10 }) - expect(byVector.map((r: any) => r.id)).toContain(id) - // type-filter read - const byType = await brain.find({ type: NounType.Concept, limit: 100 }) - expect(byType.map((r: any) => r.id)).toContain(id) - }) - - it('update(): the new metadata is immediately queryable and the old value is immediately gone', async () => { - const id = freshId() - await brain.add({ id, data: 'updatable', type: NounType.Thing, metadata: { phase: 'before' } }) - await brain.update({ id, metadata: { phase: 'after' } }) - - const after = await brain.find({ where: { phase: 'after' }, limit: 10 }) - expect(after.map((r: any) => r.id)).toContain(id) - const before = await brain.find({ where: { phase: 'before' }, limit: 10 }) - expect(before.map((r: any) => r.id)).not.toContain(id) - }) - - it('relate(): the edge is immediately traversable', async () => { - const a = await brain.add({ id: freshId(), data: 'A', type: NounType.Thing }) - const b = await brain.add({ id: freshId(), data: 'B', type: NounType.Thing }) - await brain.relate({ from: a, to: b, type: VerbType.RelatedTo }) - expect((await brain.related(a)).map((r: any) => r.to)).toContain(b) - }) - - it('remove(): the entity is immediately gone from id, metadata, and vector reads', async () => { - const id = freshId() - const marker = `gone-${id}` - await brain.add({ id, data: 'to be removed promptly', type: NounType.Thing, metadata: { marker } }) - expect(await brain.get(id)).not.toBeNull() - - await brain.remove(id) - expect(await brain.get(id)).toBeNull() - expect((await brain.find({ where: { marker }, limit: 10 })).map((r: any) => r.id)).not.toContain(id) - expect((await brain.find({ query: 'to be removed promptly', limit: 10 })).map((r: any) => r.id)).not.toContain(id) - }) - - it('transact(): every item of a committed batch is immediately readable by id, index, and traversal', async () => { - const a = freshId() - const b = freshId() - const marker = `batch-${a}` - const db = await brain.transact([ - { op: 'add', id: a, data: 'batch node A', type: NounType.Thing, metadata: { marker } }, - { op: 'add', id: b, data: 'batch node B', type: NounType.Thing, metadata: { marker } }, - { op: 'relate', from: a, to: b, type: VerbType.Contains } - ]) - expect(db.generation).toBeGreaterThan(0) - - expect(await brain.get(a)).not.toBeNull() - expect(await brain.get(b)).not.toBeNull() - const byMeta = (await brain.find({ where: { marker }, limit: 10 })).map((r: any) => r.id) - expect(byMeta).toContain(a) - expect(byMeta).toContain(b) - expect((await brain.related(a)).map((r: any) => r.to)).toContain(b) - }) - - it('the write returns a generation ({seq}) and a live read observes it — the awaitable-read primitive', async () => { - const id = freshId() - const before = brain.generationStore.generation() - await brain.add({ id, data: 'generation stamped', type: NounType.Thing }) - const after = brain.generationStore.generation() - // The write advanced the generation… - expect(after).toBeGreaterThan(before) - // …and a read pinned to that generation observes the write (asOf ≥ N sees it). - const pinned = await brain.asOf(after) - expect(await pinned.get(id)).not.toBeNull() - }) -}) diff --git a/tests/integration/readdir-no-duplicate-replace.test.ts b/tests/integration/readdir-no-duplicate-replace.test.ts deleted file mode 100644 index 8043bd40..00000000 --- a/tests/integration/readdir-no-duplicate-replace.test.ts +++ /dev/null @@ -1,56 +0,0 @@ -/** - * @module tests/integration/readdir-no-duplicate-replace - * @description A re-created path must NOT show up twice in readdir. The reported - * defect (memory-vm) was readdir serving DUPLICATE entries for re-created paths, - * rooted in a delete that left a ghost (partial removal) whose next delete read - * null metadata and skipped unposting the stale Contains edge. With full-removal - * deletes there is no ghost, so the edge is always unposted and the path appears - * exactly once after any number of delete→recreate cycles. - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/index.js' - -describe('readdir shows a re-created path exactly once (no duplicate on replace)', () => { - let dir: string - let brain: any - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-readdir-')) - brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 }) - await brain.init() - }) - afterEach(async () => { - await brain.close?.().catch(() => {}) - fs.rmSync(dir, { recursive: true, force: true }) - }) - - it('delete then re-create the same file path — readdir lists it once', async () => { - await brain.vfs.mkdir('/d', { recursive: true }) - await brain.vfs.writeFile('/d/x.txt', 'v1') - expect(await brain.vfs.readdir('/d')).toEqual(['x.txt']) - - await brain.vfs.unlink('/d/x.txt') - expect(await brain.vfs.readdir('/d')).toEqual([]) - - await brain.vfs.writeFile('/d/x.txt', 'v2') - const listing = (await brain.vfs.readdir('/d')) as string[] - expect(listing).toEqual(['x.txt']) // exactly once — no ghost duplicate - expect(listing.filter((n) => n === 'x.txt')).toHaveLength(1) - }) - - it('survives several delete→recreate cycles without accumulating duplicates', async () => { - await brain.vfs.mkdir('/c', { recursive: true }) - for (let i = 0; i < 4; i++) { - await brain.vfs.writeFile('/c/f.txt', `gen ${i}`) - await brain.vfs.unlink('/c/f.txt') - } - await brain.vfs.writeFile('/c/f.txt', 'final') - const listing = (await brain.vfs.readdir('/c')) as string[] - expect(listing).toEqual(['f.txt']) - expect(await brain.vfs.readFile('/c/f.txt')).toBeDefined() - }) -}) diff --git a/tests/integration/restore-nondestructive.test.ts b/tests/integration/restore-nondestructive.test.ts deleted file mode 100644 index b36d2620..00000000 --- a/tests/integration/restore-nondestructive.test.ts +++ /dev/null @@ -1,168 +0,0 @@ -/** - * @module tests/integration/restore-nondestructive - * @description Regression for a consumer-reported recovery hazard: `restore()` - * removed the entire live brain directory and THEN `fs.cp`'d the snapshot in, so - * a copy failure (most dangerously ENOSPC — `fs.cp` also materializes the holes - * of sparse mmap blobs, ballooning a snapshot that would otherwise fit) left the - * store destroyed with only a partial copy: the recovery tool could destroy the - * brain it was meant to recover. - * - * Fix: the snapshot is copied into a staging area (sparse-aware) BEFORE any live - * data is touched; only after the copy succeeds and a marker is fsync'd does an - * atomic per-entry swap move it into place. A copy failure leaves the live store - * exactly as it was; a crash mid-swap is resumed forward on the next open. - * - * These tests exercise the property directly: a forced copy failure leaves live - * data intact, a normal restore round-trips correctly, an interrupted-but- - * committed staging area is completed on reopen, and the sparse copy preserves - * holes byte-for-byte. - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' - -const STAGING = '_restore_staging' -const MARKER = '.restore-manifest.json' - -describe('non-destructive restore (BRAINY-RESTORE-DESTRUCTIVE)', () => { - let root: string - let snap: string - let brain: any - - const openBrain = async (dir: string) => { - const b = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - dimensions: 384, - silent: true - }) - await b.init() - return b - } - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - const base = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-restore-')) - root = path.join(base, 'store') - snap = path.join(base, 'snapshot') - brain = await openBrain(root) - }) - - afterEach(async () => { - try { await brain.close() } catch { /* already closed in a test */ } - fs.rmSync(path.dirname(root), { recursive: true, force: true }) - }) - - it('a copy failure leaves the LIVE store completely intact (the core property)', async () => { - const keep = await brain.add({ id: '00000000-0000-4000-8000-0000000000c1', data: 'in snapshot', type: NounType.Thing }) - await brain.flush() - await brain['storage'].snapshotToDirectory(snap) - // Mutate live AFTER the snapshot — this state must survive a failed restore. - const extra = await brain.add({ id: '00000000-0000-4000-8000-0000000000c2', data: 'live only', type: NounType.Thing }) - - // Force the staging copy to fail on its second entry (partial staging). - const storage = brain['storage'] - const realCopy = storage.copyTreeSparse.bind(storage) - let calls = 0 - storage.copyTreeSparse = async (src: string, dest: string) => { - if (++calls === 2) throw new Error('simulated ENOSPC') - return realCopy(src, dest) - } - - await expect(brain.restore(snap, { confirm: true })).rejects.toThrow(/untouched/) - - // Live data — BOTH the snapshot entity and the live-only mutation — survives. - expect(await brain.get(keep)).not.toBeNull() - expect(await brain.get(extra)).not.toBeNull() - // The half-written staging area was cleaned up. - expect(fs.existsSync(path.join(root, STAGING))).toBe(false) - }) - - it('a normal restore round-trips to the snapshot state', async () => { - const a = await brain.add({ id: '00000000-0000-4000-8000-0000000000c3', data: 'A', type: NounType.Thing }) - await brain.flush() - await brain['storage'].snapshotToDirectory(snap) - // Diverge from the snapshot: add B, remove A. - const b = await brain.add({ id: '00000000-0000-4000-8000-0000000000c4', data: 'B', type: NounType.Thing }) - await brain.remove(a) - expect(await brain.get(a)).toBeNull() - expect(await brain.get(b)).not.toBeNull() - - await brain.restore(snap, { confirm: true }) - - // Back to exactly the snapshot: A present, B gone. - expect(await brain.get(a)).not.toBeNull() - expect(await brain.get(b)).toBeNull() - expect(fs.existsSync(path.join(root, STAGING))).toBe(false) - }) - - it('an interrupted-but-committed restore is completed on the next open (resume forward)', async () => { - const a = await brain.add({ id: '00000000-0000-4000-8000-0000000000c5', data: 'A', type: NounType.Thing }) - await brain.flush() - await brain['storage'].snapshotToDirectory(snap) - const b = await brain.add({ id: '00000000-0000-4000-8000-0000000000c6', data: 'B', type: NounType.Thing }) - await brain.flush() - await brain.close() - - // Simulate the state right after the staging copy committed but before the - // swap ran: a _restore_staging holding the snapshot's entries + the marker. - const staging = path.join(root, STAGING) - fs.mkdirSync(staging, { recursive: true }) - const entries: string[] = [] - for (const entry of fs.readdirSync(snap)) { - if (entry === 'locks' || entry === STAGING) continue - fs.cpSync(path.join(snap, entry), path.join(staging, entry), { recursive: true }) - entries.push(entry) - } - fs.writeFileSync(path.join(staging, MARKER), JSON.stringify({ entries })) - - // Reopen → init resumes the swap → the store is the snapshot (A, no B). - brain = await openBrain(root) - expect(await brain.get(a)).not.toBeNull() - expect(await brain.get(b)).toBeNull() - expect(fs.existsSync(staging)).toBe(false) - }) - - it('an uncommitted staging area (no marker) is discarded on open, live untouched', async () => { - const a = await brain.add({ id: '00000000-0000-4000-8000-0000000000c7', data: 'A live', type: NounType.Thing }) - await brain.flush() - await brain.close() - - // A staging dir with NO marker = a copy that never committed → debris. - const staging = path.join(root, STAGING) - fs.mkdirSync(staging, { recursive: true }) - fs.writeFileSync(path.join(staging, 'entities'), 'garbage partial copy') - - brain = await openBrain(root) - expect(await brain.get(a)).not.toBeNull() // live authoritative - expect(fs.existsSync(staging)).toBe(false) // debris removed - }) - - it('the sparse copy preserves holes: content byte-identical, allocation far below apparent size', async () => { - const base = path.dirname(root) - const src = path.join(base, 'sparse-src.bin') - const dest = path.join(base, 'sparse-dest.bin') - const SIZE = 32 * 1024 * 1024 // 32 MiB apparent - const DATA = Buffer.from('hello sparse world') - - // Create a sparse source: 32 MiB of holes with a little real data at the start. - const fh = fs.openSync(src, 'w') - fs.ftruncateSync(fh, SIZE) - fs.writeSync(fh, DATA, 0, DATA.length, 0) - fs.closeSync(fh) - - const storage = brain['storage'] - const stat = fs.statSync(src) - await storage.copyFileSparse(src, dest, stat.size, stat.mode) - - // Same apparent size and byte-identical content… - const destStat = fs.statSync(dest) - expect(destStat.size).toBe(SIZE) - expect(fs.readFileSync(dest).equals(fs.readFileSync(src))).toBe(true) - // …but the destination is actually sparse (allocated bytes far below size). - expect(destStat.blocks * 512).toBeLessThan(SIZE / 2) - }) -}) diff --git a/tests/integration/rollback-trapdoor.test.ts b/tests/integration/rollback-trapdoor.test.ts deleted file mode 100644 index cf2245b8..00000000 --- a/tests/integration/rollback-trapdoor.test.ts +++ /dev/null @@ -1,156 +0,0 @@ -/** - * @module tests/integration/rollback-trapdoor - * @description Regression for a consumer-reported data-integrity bug: a failed - * CANONICAL rollback undo left the record durable while the request errored (a - * post-commit response lie). Reproduced by a failure-injection coherence harness: - * a late index op throws → rollback runs → an earlier op's canonical undo - * (storage.deleteNounMetadata) fails persistently → the record stays on disk - * while Transaction.rollback threw TransactionRollbackError AND lied with - * state='rolled_back'. - * - * Fix (David's ruling — adopt-forward when safe, else fail loud): - * - SINGLE-op add whose only damage is a durably-present orphan → adopt it - * forward: commit the generation (the record IS what add() wanted), return - * success + a loud warn; the derived index heals on rebuild/repairIndex. - * - MULTI-op batch, OR any restorative LOSS → StoreInconsistentError naming the - * unreconciled ids; the brain enters write-quarantine (reads OK, writes - * refused) until repairIndex() reconciles and lifts it. Transaction state is - * 'inconsistent', never the 'rolled_back' lie. - * - * The injection mirrors that harness: fault index.addToIndex (trigger rollback) and - * storage.deleteNounMetadata (fail the canonical undo). - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' -import { StoreInconsistentError } from '../../src/db/errors.js' - -let seq = 0 -const freshId = (): string => - `00000000-0000-4000-8000-${(++seq).toString(16).padStart(12, '0')}` - -describe('rollback trapdoor — failed canonical undo (data integrity)', () => { - let dir: string - let brain: any - let restore: Array<() => void> - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-trapdoor-')) - brain = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - dimensions: 384, - silent: true - }) - await brain.init() - restore = [] - }) - - afterEach(async () => { - for (const r of restore) r() - try { await brain.close() } catch { /* ignore */ } - fs.rmSync(dir, { recursive: true, force: true }) - }) - - /** Fault the metadata-index add so a write's rollback is triggered. */ - const faultIndexAdd = () => { - const idx = brain['metadataIndex'] - const orig = idx.addToIndex.bind(idx) - idx.addToIndex = async () => { throw new Error('injected: index add failed') } - restore.push(() => { idx.addToIndex = orig }) - } - /** Fault the canonical additive undo (delete of a just-created record). */ - const faultCanonicalDelete = () => { - const storage = brain['storage'] - const orig = storage.deleteNounMetadata.bind(storage) - storage.deleteNounMetadata = async () => { throw new Error('injected: canonical undo failed') } - restore.push(() => { storage.deleteNounMetadata = orig }) - } - - it('ADOPT-FORWARD: a single-op add whose canonical undo fails commits as a durable, get-able record (no throw, no quarantine)', async () => { - const id = freshId() - const genBefore = brain.generationStore.generation() - - faultIndexAdd() // triggers rollback - faultCanonicalDelete() // the trapdoor: the metadata record can't be un-written - - // add() must NOT throw — the record it wanted is durable, so it is adopted. - const returned = await brain.add({ id, data: 'orphan adopted', type: NounType.Thing }) - expect(returned).toBe(id) - - // Un-fault so reads/subsequent writes use the real methods. - for (const r of restore) r() - restore = [] - - // The record is durable and get-able, and the generation advanced. - expect(await brain.get(id)).not.toBeNull() - expect(brain.generationStore.generation()).toBe(genBefore + 1) - - // The store is NOT quarantined — a subsequent write succeeds. - const ok = await brain.add({ id: freshId(), data: 'still writable', type: NounType.Thing }) - expect(await brain.get(ok)).not.toBeNull() - }) - - it('FAIL-LOUD: a multi-op transact whose canonical undo fails throws StoreInconsistentError and write-quarantines the brain', async () => { - // A pre-existing record to prove reads keep working under quarantine. - const anchor = await brain.add({ id: freshId(), data: 'anchor', type: NounType.Thing }) - await brain.flush() - - faultIndexAdd() - faultCanonicalDelete() - - const a = freshId() - const b = freshId() - await expect( - brain.transact([ - { op: 'add', id: a, data: 'batch A', type: NounType.Thing }, - { op: 'add', id: b, data: 'batch B', type: NounType.Thing } - ]) - ).rejects.toBeInstanceOf(StoreInconsistentError) - - // Writes are refused (quarantine); reads still work. - await expect( - brain.add({ id: freshId(), data: 'blocked', type: NounType.Thing }) - ).rejects.toThrow(/WRITE-QUARANTINED/) - expect(await brain.get(anchor)).not.toBeNull() - - // repairIndex() reconciles and lifts the quarantine → writes resume. - for (const r of restore) r() - restore = [] - await brain.repairIndex() - - const ok = await brain.add({ id: freshId(), data: 'writable again', type: NounType.Thing }) - expect(await brain.get(ok)).not.toBeNull() - }) - - it('the StoreInconsistentError names the unreconciled records with dispositions', async () => { - faultIndexAdd() - faultCanonicalDelete() - - const a = freshId() - const b = freshId() - let caught: unknown - await brain - .transact([ - { op: 'add', id: a, data: 'A', type: NounType.Thing }, - { op: 'add', id: b, data: 'B', type: NounType.Thing } - ]) - .catch((e: unknown) => (caught = e)) - - expect(caught).toBeInstanceOf(StoreInconsistentError) - const err = caught as StoreInconsistentError - expect(err.records.length).toBeGreaterThan(0) - // Every unreconciled record here is a durably-present orphan. - expect(err.records.every((r) => r.disposition === 'orphan')).toBe(true) - expect(err.message).toMatch(/WRITE-QUARANTINED/) - - // Recover so afterEach can close cleanly. - for (const r of restore) r() - restore = [] - await brain.repairIndex() - }) -}) diff --git a/tests/integration/temporal-vfs.test.ts b/tests/integration/temporal-vfs.test.ts deleted file mode 100644 index 77206ed8..00000000 --- a/tests/integration/temporal-vfs.test.ts +++ /dev/null @@ -1,200 +0,0 @@ -/** - * @module tests/integration/temporal-vfs - * @description VFS content joins the Model-B immutability model. Previously - * the temporal model had a hole exactly where files were concerned: entity - * records were versioned per write (before-images), but the content BYTES - * lived under an eager refCount GC — `rm` could physically delete bytes that - * in-window history still referenced, and overwrite never released the old - * hash at all (an unbounded silent leak that only accidentally preserved - * history). Now blob bytes are retention-protected: a content blob referenced - * by any generation inside the retention window (or live) survives, and the - * ONE reclamation point is history compaction. - * - * Pins: - * - `readFile(path, { asOf })` returns each version's exact bytes. - * - `history(path)` lists versions ascending with generation/hash/size. - * - overwrite releases the superseded live reference (leak fixed) while - * history protects the bytes; rm keeps bytes readable via asOf. - * - compaction past the last referencing generation physically reclaims - * bytes (zero live + zero history), and never reclaims in-window content — - * including the cross-file dedup case where an old file's history and a - * newer file's history share one hash. - * - overwrite refreshes the entity's `data` (embedding text) — the stale - * `.data` defect. - * - the backfill/scrub recounts exactly from the generation records. - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/brainy.js' - -describe('temporal VFS — blob immutability under Model B', () => { - let dir: string - let brain: any - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-temporal-vfs-')) - brain = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - dimensions: 384, - silent: true - }) - await brain.init() - }) - - afterEach(async () => { - await brain.close() - fs.rmSync(dir, { recursive: true, force: true }) - }) - - /** The file's blob metadata via the storage layer (test introspection). */ - async function blobMeta(hash: string) { - return brain['storage'].blobStorage.getMetadata(hash) - } - - it('readFile(path, {asOf}) returns each version‘s exact bytes; history(path) lists them', async () => { - const p = '/pages/home.json' - await brain.vfs.writeFile(p, 'v1 — original') - const g1 = brain.generationStore.generation() - await brain.vfs.writeFile(p, 'v2 — edited') - await brain.vfs.writeFile(p, 'v3 — final') - - // Live read = newest. - expect((await brain.vfs.readFile(p)).toString()).toBe('v3 — final') - - // History lists the file's write-generations ascending, distinct hashes. - const versions = await brain.vfs.history(p) - expect(versions.length).toBeGreaterThanOrEqual(3) - const gens = versions.map((v: any) => v.generation) - expect([...gens].sort((a: number, b: number) => a - b)).toEqual(gens) - - // Every listed version's exact bytes are readable. - const texts: string[] = [] - for (const v of versions) { - texts.push((await brain.vfs.readFile(p, { asOf: v.generation })).toString()) - } - expect(texts).toContain('v1 — original') - expect(texts).toContain('v2 — edited') - expect(texts).toContain('v3 — final') - - // Direct generation read too (the incident shape: recover the past bytes). - expect((await brain.vfs.readFile(p, { asOf: g1 })).toString()).toBe('v1 — original') - }) - - it('overwrite releases the superseded LIVE reference (leak fixed) while history protects the bytes', async () => { - const p = '/notes/leak.txt' - await brain.vfs.writeFile(p, 'old content A') - const oldHash = (await brain.vfs.history(p))[0].hash - const g1 = brain.generationStore.generation() - - await brain.vfs.writeFile(p, 'new content B') - - // The old hash's LIVE reference is released (previously it leaked forever)… - expect((await blobMeta(oldHash))?.refCount).toBe(0) - // …but the bytes survive (history-protected) and asOf still reads them. - expect((await brain.vfs.readFile(p, { asOf: g1 })).toString()).toBe('old content A') - }) - - it('rm keeps the bytes readable via asOf inside the window (no premature deletion)', async () => { - const p = '/docs/doomed.txt' - await brain.vfs.writeFile(p, 'still recoverable after rm') - const g = brain.generationStore.generation() - const hash = (await brain.vfs.history(p))[0].hash - - await brain.vfs.unlink(p) - - // Live path is gone… - await expect(brain.vfs.readFile(p)).rejects.toThrow() - // …the bytes are not: zero live refs, history-protected. - expect((await blobMeta(hash))?.refCount).toBe(0) - expect((await brain['storage'].blobStorage.read(hash)).toString()).toBe( - 'still recoverable after rm' - ) - // (Path-level asOf resolution of a DELETED path is future work — the - // bytes + entity history are intact; recovery reads go through the hash - // or a pre-delete generation view.) - void g - }) - - it('compaction is the ONE reclamation point: past-window bytes are reclaimed, in-window preserved', async () => { - const p = '/pages/compact.json' - await brain.vfs.writeFile(p, 'version ONE') - const v1Hash = (await brain.vfs.history(p))[0].hash - await brain.vfs.writeFile(p, 'version TWO') - await brain.vfs.writeFile(p, 'version THREE') - - // Sanity: v1's bytes exist (history-protected, live-released). - expect(await blobMeta(v1Hash)).toBeDefined() - expect((await blobMeta(v1Hash))?.refCount).toBe(0) - - // Compact everything reclaimable (retain nothing beyond the live state). - await brain.compactHistory({ maxGenerations: 0 }) - - // v1's bytes are physically GONE — zero live, zero history. - expect(await blobMeta(v1Hash)).toBeUndefined() - // The live version is untouched. - expect((await brain.vfs.readFile(p)).toString()).toBe('version THREE') - }) - - it('cross-file dedup: a shared hash survives while ANY in-window generation references it', async () => { - const shared = 'identical bytes shared across files and time' - const pA = '/a.txt' - const pB = '/b.txt' - - // A holds the shared content, then moves off it (A's history references it). - await brain.vfs.writeFile(pA, shared) - await brain.vfs.writeFile(pA, 'A moved on') - // B now holds the shared content live, then is removed (B's remove-gen - // before-image references it too). - await brain.vfs.writeFile(pB, shared) - const gBLive = brain.generationStore.generation() - const sharedHash = (await brain.vfs.history(pA))[0].hash - await brain.vfs.unlink(pB) - - // Live refs are zero (A moved off; B removed) — bytes survive on history. - expect((await blobMeta(sharedHash))?.refCount).toBe(0) - expect((await brain['storage'].blobStorage.read(sharedHash)).toString()).toBe(shared) - - // B's content at its live generation is still exactly readable… via asOf - // on A's early version too (same bytes, same blob). - const versionsA = await brain.vfs.history(pA) - const aV1 = versionsA[0] - expect((await brain.vfs.readFile(pA, { asOf: aV1.generation })).toString()).toBe(shared) - void gBLive - - // Full compaction drops the last history references → bytes reclaimed. - await brain.compactHistory({ maxGenerations: 0 }) - expect(await blobMeta(sharedHash)).toBeUndefined() - }) - - it('overwrite refreshes the entity data/embedding text (the stale-.data defect)', async () => { - const p = '/pages/data-fresh.txt' - await brain.vfs.writeFile(p, 'first words') - await brain.vfs.writeFile(p, 'second words entirely') - - const entityId = await brain.vfs['pathResolver'].resolve(p) - const entity = await brain.get(entityId) - expect(entity.data).toBe('second words entirely') - }) - - it('the scrub recounts history references exactly from the generation records', async () => { - const p = '/pages/scrubbed.md' - await brain.vfs.writeFile(p, 'scrub v1') - await brain.vfs.writeFile(p, 'scrub v2') - const v1Hash = (await brain.vfs.history(p))[0].hash - await brain.flush() - - const storage = brain['storage'] - const before = (await blobMeta(v1Hash))?.historyRefCount ?? 0 - expect(before).toBeGreaterThan(0) - - // Corrupt the counter (simulates a legacy store / crash drift), then scrub. - await storage.blobStorage.setHistoryRefCount(v1Hash, 0) - const result = await storage.scrubBlobHistoryRefCounts() - expect(result.records).toBeGreaterThan(0) - expect((await blobMeta(v1Hash))?.historyRefCount).toBe(before) - }) -}) diff --git a/tests/integration/transact-durability-barrier.test.ts b/tests/integration/transact-durability-barrier.test.ts deleted file mode 100644 index 9311ce67..00000000 --- a/tests/integration/transact-durability-barrier.test.ts +++ /dev/null @@ -1,151 +0,0 @@ -/** - * @module tests/integration/transact-durability-barrier - * @description Regression for a consumer-reported durability gap: a committed - * `transact()` reported success while its canonical entity writes were still - * only in the page cache (tmp+rename, not yet fsync'd), whereas the generation - * counter/manifest WERE fsync'd — so a hard kill could leave the counter ahead - * of the persisted entity bytes ("phantom progress" for any generation-based - * consumer resuming from the counter). - * - * Fix: `commitTransaction` opens a write barrier before running the planned - * operations and flushes it (fsync of every canonical write + the parent dir of - * every canonical delete) BEFORE advancing the counter and writing the manifest. - * These tests assert the ordering property directly by recording the storage's - * `syncRawObjects` calls: the entity writes are fsync'd in an earlier call than - * the manifest, and a precommit-rejected batch opens no barrier and advances - * nothing. - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType, VerbType } from '../../src/types/graphTypes.js' -import { MemoryStorage } from '../../src/storage/adapters/memoryStorage.js' - -const MANIFEST_REL = '_system/manifest.json' - -describe('transact durability barrier — entity writes fsync before the counter advances', () => { - let dir: string - let brain: any - let syncCalls: string[][] - let beginCount: number - let flushCount: number - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-durability-')) - brain = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - dimensions: 384, - silent: true - }) - await brain.init() - - // Instrument the real filesystem storage: record every fsync batch in order, - // and count barrier open/flush, delegating to the originals. - syncCalls = [] - beginCount = 0 - flushCount = 0 - const storage = brain['storage'] - const origSync = storage.syncRawObjects.bind(storage) - storage.syncRawObjects = async (paths: string[]) => { - syncCalls.push([...paths]) - return origSync(paths) - } - const origBegin = storage.beginWriteBarrier.bind(storage) - storage.beginWriteBarrier = () => { - beginCount++ - return origBegin() - } - const origFlush = storage.flushWriteBarrier.bind(storage) - storage.flushWriteBarrier = async () => { - flushCount++ - return origFlush() - } - }) - - afterEach(async () => { - await brain.close() - fs.rmSync(dir, { recursive: true, force: true }) - }) - - /** Index of the first sync call whose paths satisfy `pred` (or -1). */ - const findSync = (pred: (p: string) => boolean): number => - syncCalls.findIndex((paths) => paths.some(pred)) - - it('a committed transact fsyncs the entity write BEFORE the manifest', async () => { - const id = '00000000-0000-4000-8000-00000000ab01' - await brain.transact([{ op: 'add', id, data: 'durable entity', type: NounType.Thing }]) - - // The barrier opened and flushed exactly once. - expect(beginCount).toBe(1) - expect(flushCount).toBe(1) - - // The entity's canonical metadata write was fsync'd… - const entitySyncIdx = findSync((p) => p.includes(`/${id}/`) && p.includes('entities/nouns')) - expect(entitySyncIdx).toBeGreaterThanOrEqual(0) - // …and the manifest (the commit point) was fsync'd in a LATER call. - const manifestSyncIdx = findSync((p) => p === MANIFEST_REL) - expect(manifestSyncIdx).toBeGreaterThanOrEqual(0) - expect(entitySyncIdx).toBeLessThan(manifestSyncIdx) - }) - - it('a multi-op transact (add + relate) fsyncs both endpoints and the edge before the manifest', async () => { - const a = '00000000-0000-4000-8000-00000000ab02' - const b = '00000000-0000-4000-8000-00000000ab03' - await brain.transact([ - { op: 'add', id: a, data: 'A', type: NounType.Thing }, - { op: 'add', id: b, data: 'B', type: NounType.Thing }, - { op: 'relate', from: a, to: b, type: VerbType.Contains } - ]) - - expect(flushCount).toBe(1) - const aIdx = findSync((p) => p.includes(`/${a}/`)) - const bIdx = findSync((p) => p.includes(`/${b}/`)) - const manifestIdx = findSync((p) => p === MANIFEST_REL) - expect(aIdx).toBeGreaterThanOrEqual(0) - expect(bIdx).toBeGreaterThanOrEqual(0) - expect(manifestIdx).toBeGreaterThanOrEqual(0) - expect(Math.max(aIdx, bIdx)).toBeLessThan(manifestIdx) - - // Sanity: the data is actually there and traverses. - expect((await brain.related(a)).map((r: any) => r.to)).toContain(b) - }) - - it('a precommit-rejected batch opens no barrier and advances nothing', async () => { - const anchor = await brain.add({ id: '00000000-0000-4000-8000-00000000ab04', data: 'anchor', type: NounType.Thing }) - const genBefore = brain.generationStore.generation() - syncCalls.length = 0 - beginCount = 0 - flushCount = 0 - - const never = '00000000-0000-4000-8000-00000000ab05' - await expect( - brain.transact([ - { op: 'add', id: never, data: 'never', type: NounType.Thing }, - { op: 'update', id: anchor, metadata: { poke: 1 }, ifRev: 999 } - ]) - ).rejects.toMatchObject({ name: 'RevisionConflictError' }) - - // Precommit throws before the barrier opens (it lives just before execute): - // the barrier never opens, nothing is flushed, and the generation counter is - // unchanged. (A manifest sync CAN appear here from transact() flushing the - // anchor's previously-buffered single-op — that is the anchor's deferred - // durability, not this rejected batch committing; beginCount is the clean - // signal that this batch's commit path never reached execute.) - expect(beginCount).toBe(0) - expect(flushCount).toBe(0) - expect(brain.generationStore.generation()).toBe(genBefore) - expect(await brain.get(never)).toBeNull() - }) - - it('MemoryStorage does not implement the barrier (optional-chaining no-op)', () => { - const mem = new MemoryStorage() as any - // In-memory has no durability seam; the generation store treats the absent - // barrier methods as no-ops via optional chaining. - expect(mem.beginWriteBarrier).toBeUndefined() - expect(mem.flushWriteBarrier).toBeUndefined() - }) -}) diff --git a/tests/integration/transact-forward-ref-graph.test.ts b/tests/integration/transact-forward-ref-graph.test.ts deleted file mode 100644 index a9619424..00000000 --- a/tests/integration/transact-forward-ref-graph.test.ts +++ /dev/null @@ -1,140 +0,0 @@ -/** - * @module tests/integration/transact-forward-ref-graph - * @description Regression for a consumer-reported native/JS parity bug: - * `transact([{op:'add', id:X}, {op:'relate', to:X}])` — the platform's first - * atomic add+relate consumer — threw in `EntityIdMapper.getOrAssign` at PLAN - * time on the native id mapper (an entity added in the same batch does not - * exist when the planner runs, so a strict mapper rightly refuses to assign; - * the permissive JS mapper masked the bug — and silently leaked an int - * assignment whenever the batch was later rejected at precommit). - * - * Fix: graph-index operations resolve endpoint ints at EXECUTE time (a lazy - * thunk, mirroring the existing lazy `generationFn`), after the batch's add - * operations have applied. These tests pin every forward-ref shape on the JS - * path, including the JS-observable deferral proof: a REJECTED batch leaves - * the id mapper untouched (pre-fix, plan-time resolution assigned ints for - * entities that never came to exist). The native side is locked in by the - * accelerator's own gate running this same shape. - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType, VerbType } from '../../src/types/graphTypes.js' - -let seq = 0 -const freshId = (): string => - `00000000-0000-4000-8000-${(++seq).toString(16).padStart(12, '0')}` - -describe('transact() forward references into the graph index (native/JS parity)', () => { - let dir: string - let brain: any - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-fwd-ref-')) - brain = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - dimensions: 384, - silent: true - }) - await brain.init() - }) - - afterEach(async () => { - await brain.close() - fs.rmSync(dir, { recursive: true, force: true }) - }) - - it('CASE 1 (the exact repro): add + relate-to-that-add in ONE transact', async () => { - const agent = await brain.add({ id: freshId(), data: 'agent probe', type: NounType.Person }) - const thread = freshId() - - const db = await brain.transact([ - { op: 'add', id: thread, data: 'thread PROBE-ONE', type: NounType.Thing }, - { op: 'relate', from: agent, to: thread, type: VerbType.RelatedTo } - ]) - - // The batch committed as one generation and the edge traverses. - expect(db.generation).toBeGreaterThan(0) - const related = await brain.related(agent) - expect(related.map((r: any) => r.to)).toContain(thread) - }) - - it('both endpoints created in-batch: add A + add B + relate A→B', async () => { - const a = freshId() - const b = freshId() - await brain.transact([ - { op: 'add', id: a, data: 'node A', type: NounType.Thing }, - { op: 'add', id: b, data: 'node B', type: NounType.Thing }, - { op: 'relate', from: a, to: b, type: VerbType.Contains } - ]) - expect((await brain.related(a)).map((r: any) => r.to)).toContain(b) - }) - - it('bidirectional relate to an in-batch add: both edges traverse', async () => { - const hub = await brain.add({ id: freshId(), data: 'hub', type: NounType.Thing }) - const spoke = freshId() - await brain.transact([ - { op: 'add', id: spoke, data: 'spoke', type: NounType.Thing }, - { op: 'relate', from: hub, to: spoke, type: VerbType.RelatedTo, bidirectional: true } - ]) - expect((await brain.related(hub)).map((r: any) => r.to)).toContain(spoke) - expect((await brain.related(spoke)).map((r: any) => r.to)).toContain(hub) - }) - - it('add + relate + remove in ONE transact: the cascade covers the in-batch verb', async () => { - const keeper = await brain.add({ id: freshId(), data: 'keeper', type: NounType.Thing }) - const doomed = freshId() - await brain.transact([ - { op: 'add', id: doomed, data: 'doomed', type: NounType.Thing }, - { op: 'relate', from: keeper, to: doomed, type: VerbType.Contains }, - { op: 'remove', id: doomed } - ]) - expect(await brain.get(doomed)).toBeNull() - expect((await brain.related(keeper)).length).toBe(0) - }) - - it('CASE 2 control: the same two ops split across two transacts still work', async () => { - const agent = await brain.add({ id: freshId(), data: 'agent two', type: NounType.Person }) - const thread = freshId() - await brain.transact([{ op: 'add', id: thread, data: 'thread two', type: NounType.Thing }]) - await brain.transact([{ op: 'relate', from: agent, to: thread, type: VerbType.RelatedTo }]) - expect((await brain.related(agent)).map((r: any) => r.to)).toContain(thread) - }) - - it('a REJECTED forward-ref batch leaves the id mapper untouched (the deferral proof)', async () => { - const existing = await brain.add({ - id: freshId(), - data: 'cas anchor', - type: NounType.Thing - }) - const never = freshId() - - // Reject the batch via a stale per-op CAS — planning completes, the - // commit precondition throws, nothing applies. - await expect( - brain.transact([ - { op: 'add', id: never, data: 'never exists', type: NounType.Thing }, - { op: 'relate', from: existing, to: never, type: VerbType.RelatedTo }, - { op: 'update', id: existing, metadata: { poke: 1 }, ifRev: 999 } - ]) - ).rejects.toMatchObject({ name: 'RevisionConflictError' }) - - // Nothing applied… - expect(await brain.get(never)).toBeNull() - // …and the id mapper was never asked to assign for the phantom entity — - // pre-fix, plan-time resolution leaked an int here on every rejected batch. - const idMapper = brain['metadataIndex'].getIdMapper() - expect(idMapper.getInt(never)).toBeUndefined() - - // The same shape then succeeds cleanly on retry. - await brain.transact([ - { op: 'add', id: never, data: 'exists now', type: NounType.Thing }, - { op: 'relate', from: existing, to: never, type: VerbType.RelatedTo } - ]) - expect((await brain.related(existing)).map((r: any) => r.to)).toContain(never) - }) -}) diff --git a/tests/integration/vfs-rename-containment.test.ts b/tests/integration/vfs-rename-containment.test.ts deleted file mode 100644 index 0169bf01..00000000 --- a/tests/integration/vfs-rename-containment.test.ts +++ /dev/null @@ -1,157 +0,0 @@ -/** - * @module tests/integration/vfs-rename-containment - * @description A cross-directory rename must MOVE the containment edge, not - * accumulate one per parent. The pre-fix rename added the new parent's - * Contains edge but skipped removing the old one ("not critical") — leaving - * the entity a child of BOTH directories: readdir(oldDir) kept listing it, - * re-creating the old path showed the name twice (the duplicate-readdir / - * "cosmetic ghost" field report), and tree-walking consumers saw the file in - * two places. Also covers the repair sweep (repairIndex → vfs.repairContainment) - * that heals ghosts left by earlier versions, and move-to-root (whose edge - * used to be skipped entirely). - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy, VerbType } from '../../src/index.js' - -describe('VFS rename moves the containment edge (no ghost in the old directory)', () => { - let brain: any - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true, dimensions: 384 }) - await brain.init() - await brain.vfs.mkdir('/a', { recursive: true }) - await brain.vfs.mkdir('/b', { recursive: true }) - }) - afterEach(async () => { - await brain.close?.().catch(() => {}) - }) - - it('cross-directory move: old dir stops listing it, new dir lists it exactly once', async () => { - await brain.vfs.writeFile('/a/x.txt', 'v1') - await brain.vfs.rename('/a/x.txt', '/b/x.txt') - - expect(await brain.vfs.readdir('/a')).toEqual([]) - expect(await brain.vfs.readdir('/b')).toEqual(['x.txt']) - expect(String(await brain.vfs.readFile('/b/x.txt'))).toBe('v1') - }) - - it('re-creating the old path lists each name exactly once in each directory', async () => { - await brain.vfs.writeFile('/a/x.txt', 'moved away') - await brain.vfs.rename('/a/x.txt', '/b/x.txt') - await brain.vfs.writeFile('/a/x.txt', 'new file at old path') - - const a = (await brain.vfs.readdir('/a')) as string[] - const b = (await brain.vfs.readdir('/b')) as string[] - expect(a).toEqual(['x.txt']) // exactly once — the pre-fix ghost made this list the moved entity too - expect(b).toEqual(['x.txt']) - expect(String(await brain.vfs.readFile('/a/x.txt'))).toBe('new file at old path') - expect(String(await brain.vfs.readFile('/b/x.txt'))).toBe('moved away') - }) - - it('repeated moves never accumulate containment edges', async () => { - await brain.vfs.writeFile('/a/f.txt', 'wanderer') - for (let i = 0; i < 3; i++) { - await brain.vfs.rename('/a/f.txt', '/b/f.txt') - await brain.vfs.rename('/b/f.txt', '/a/f.txt') - } - expect(await brain.vfs.readdir('/a')).toEqual(['f.txt']) - expect(await brain.vfs.readdir('/b')).toEqual([]) - - // Exactly ONE containment edge exists on the entity. - const stat = await brain.vfs.stat('/a/f.txt') - const edges = await brain.related({ to: stat.entityId, type: VerbType.Contains }) - expect(edges).toHaveLength(1) - }) - - it('move to the root gets a containment edge (used to be skipped → orphan)', async () => { - await brain.vfs.writeFile('/a/up.txt', 'to the top') - await brain.vfs.rename('/a/up.txt', '/up.txt') - - const rootListing = (await brain.vfs.readdir('/')) as string[] - expect(rootListing).toContain('up.txt') - expect(await brain.vfs.readdir('/a')).toEqual([]) - expect(String(await brain.vfs.readFile('/up.txt'))).toBe('to the top') - }) -}) - -describe('repairIndex() heals pre-fix containment ghosts (vfs.repairContainment)', () => { - let brain: any - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true, dimensions: 384 }) - await brain.init() - await brain.vfs.mkdir('/a', { recursive: true }) - await brain.vfs.mkdir('/b', { recursive: true }) - }) - afterEach(async () => { - await brain.close?.().catch(() => {}) - }) - - /** Reproduce the PRE-FIX defect state: file lives at /b/g.txt but a stale - * vfs-contains edge from /a lingers (what old renames left behind). */ - const synthesizeGhost = async () => { - await brain.vfs.writeFile('/b/g.txt', 'ghost target') - const fileId = (await brain.vfs.stat('/b/g.txt')).entityId - const aId = (await brain.vfs.stat('/a')).entityId - await brain.relate({ - from: aId, - to: fileId, - type: VerbType.Contains, - subtype: 'vfs-contains', - metadata: { isVFS: true } - }) - return { fileId, aId } - } - - it('a stale old-parent edge is removed; listings become honest', async () => { - const { fileId } = await synthesizeGhost() - // The defect state is visible: /a lists a file whose path says /b. - expect((await brain.vfs.readdir('/a')) as string[]).toContain('g.txt') - - await brain.repairIndex() - - expect(await brain.vfs.readdir('/a')).toEqual([]) - expect(await brain.vfs.readdir('/b')).toEqual(['g.txt']) - const edges = await brain.related({ to: fileId, type: VerbType.Contains }) - expect(edges).toHaveLength(1) - }) - - it("a user's own knowledge Contains edge onto the file survives the repair", async () => { - const { fileId } = await synthesizeGhost() - // A knowledge-graph containment from a NON-directory entity (a collection - // curating the file) — same verb TYPE, not a vfs-contains edge. The repair - // must remove only VFS containment ghosts, never user knowledge edges. - // (Edges dedupe by (from,to,type), so the user edge needs its own source.) - const collectionId = await brain.add({ - data: 'reading list', - type: 'collection', - metadata: { kind: 'curation' } - }) - await brain.relate({ from: collectionId, to: fileId, type: VerbType.Contains, subtype: 'curates' }) - - await brain.repairIndex() - - const edges = await brain.related({ to: fileId, type: VerbType.Contains }) - const subtypes = edges.map((e: any) => e.subtype).sort() - // The stale vfs ghost edge is gone; the correct vfs edge + the user's - // curation edge remain. - expect(subtypes).toEqual(['curates', 'vfs-contains']) - expect(edges.find((e: any) => e.subtype === 'curates')?.from).toBe(collectionId) - }) - - it('a missing expected edge is restored (entity unreachable from its own directory)', async () => { - await brain.vfs.writeFile('/b/lost.txt', 'find me') - const fileId = (await brain.vfs.stat('/b/lost.txt')).entityId - // Simulate total edge loss (an older damage shape). - for (const e of await brain.related({ to: fileId, type: VerbType.Contains })) { - await brain.unrelate(e.id) - } - expect((await brain.vfs.readdir('/b')) as string[]).not.toContain('lost.txt') - - await brain.repairIndex() - - expect((await brain.vfs.readdir('/b')) as string[]).toContain('lost.txt') - }) -}) diff --git a/tests/regression/metadata-index-cleanup.unit.test.ts b/tests/regression/metadata-index-cleanup.unit.test.ts index 0984d727..266b9a4d 100644 --- a/tests/regression/metadata-index-cleanup.unit.test.ts +++ b/tests/regression/metadata-index-cleanup.unit.test.ts @@ -205,10 +205,10 @@ describe('Metadata index cleanup after remove / removeMany', () => { } }) - it('refuses an empty ids array loudly (a silent no-op is not "graceful")', async () => { - // 8.8.2: an empty selector used to resolve successfully having deleted - // NOTHING — the caller believed the delete happened. Now it throws. - await expect(brain.removeMany({ ids: [] })).rejects.toThrow(/ids: \[\]/) + it('handles empty ids array gracefully', async () => { + const result = await brain.removeMany({ ids: [] }) + expect(result.successful).toHaveLength(0) + expect(result.failed).toHaveLength(0) }) it('handles large batch (> 1 chunk) without leaving stale index entries', async () => { diff --git a/tests/transaction/TransactionManager.unit.test.ts b/tests/transaction/TransactionManager.unit.test.ts index 86b1692c..5ffee866 100644 --- a/tests/transaction/TransactionManager.unit.test.ts +++ b/tests/transaction/TransactionManager.unit.test.ts @@ -10,7 +10,6 @@ import { describe, it, expect, beforeEach } from 'vitest' import { TransactionManager } from '../../src/transaction/TransactionManager.js' -import { transactTimeoutBudget } from '../../src/transaction/Transaction.js' import { TransactionError } from '../../src/transaction/errors.js' describe('TransactionManager', () => { @@ -106,17 +105,14 @@ describe('TransactionManager', () => { const result = await manager.executeTransactionWithResult(async (tx) => { tx.addOperation({ execute: async () => { - await new Promise(resolve => setTimeout(resolve, 25)) + await new Promise(resolve => setTimeout(resolve, 10)) return async () => {} } }) return 'done' }) - // Timer coalescing can fire a setTimeout up to a few ms EARLY under - // load, so assert well below the sleep — this tests that time is - // MEASURED, not the OS timer's precision. - expect(result.executionTimeMs).toBeGreaterThanOrEqual(20) + expect(result.executionTimeMs).toBeGreaterThanOrEqual(10) }) }) @@ -329,45 +325,4 @@ describe('TransactionManager', () => { expect(stats1).toEqual(stats2) // Same values }) }) - - describe('Timeout budget + telemetry', () => { - it('transactTimeoutBudget: explicit override wins; default scales with batch size', () => { - expect(transactTimeoutBudget(1)).toBe(30_000) // small batches keep the 30s floor - expect(transactTimeoutBudget(15)).toBe(30_000) // the old flat cap's break-even point - expect(transactTimeoutBudget(100)).toBe(200_000) // 100 ops × 2s — bulk gets an honest budget - expect(transactTimeoutBudget(1000, 5_000)).toBe(5_000) // caller override is untouched - }) - - it('a tripped budget rolls back and names the operation, progress, and budget', async () => { - const rolledBack: string[] = [] - - const failing = manager.executeTransaction( - async (tx) => { - tx.addOperation({ - name: 'slow-first-op', - execute: async () => { - await new Promise((r) => setTimeout(r, 30)) - return async () => { - rolledBack.push('slow-first-op') - } - } - }) - tx.addOperation({ - name: 'never-reached', - execute: async () => undefined - }) - }, - { timeout: 5 } // the first op's 30ms sleep guarantees the pre-op-2 check trips - ) - - await expect(failing).rejects.toThrow(TransactionError) - const err = await failing.catch((e) => e) - expect(err.name).toBe('TransactionTimeoutError') - expect(err.message).toContain('operation 1/2') // which op, of how many - expect(err.message).toContain("('never-reached')") // its name - expect(err.message).toContain('budget 5ms') // the budget that tripped - expect(err.message).toContain('rolled back') // the retryability statement - expect(rolledBack).toEqual(['slow-first-op']) // the applied op was undone - }) - }) }) diff --git a/tests/unit/aggregation/AggregationIndex.test.ts b/tests/unit/aggregation/AggregationIndex.test.ts index a0c10e62..05f4a62a 100644 --- a/tests/unit/aggregation/AggregationIndex.test.ts +++ b/tests/unit/aggregation/AggregationIndex.test.ts @@ -675,104 +675,4 @@ describe('AggregationIndex', () => { await reloaded.close() }) }) - - // ============= Boot-order reconciliation ============= - // - // The production boot pattern: defineAggregate() is synchronous and always - // beats the async init() that loads persisted state. The old code flagged a - // backfill at define time and init never cleared it — so the loaded state - // was wiped and the whole store re-walked on EVERY restart. - - describe('boot-order reconciliation (define-before-init)', () => { - const DEF: AggregateDefinition = { - name: 'boot_agg', - source: { type: NounType.Event }, - groupBy: ['category'], - metrics: { count: { op: 'count' } } - } - - const entity = (id: string, category: string): Record => ({ - id, - noun: NounType.Event, - metadata: { category }, - createdAt: Date.now(), - updatedAt: Date.now() - }) - - /** Simulate the previous session: define, contribute, flush. */ - async function seedAndFlush(store: MemoryStorage): Promise { - const first = new AggregationIndex(store) - await first.init() - first.defineAggregate(DEF) - first.onEntityAdded('e1', entity('e1', 'food')) - first.onEntityAdded('e2', entity('e2', 'food')) - first.onEntityAdded('e3', entity('e3', 'transport')) - await first.flush() - } - - it('adopts persisted state when define beats init with an unchanged definition', async () => { - const store = new MemoryStorage() - await store.init() - await seedAndFlush(store) - - const second = new AggregationIndex(store) - second.defineAggregate(DEF) // synchronous define FIRST — the real boot order - await second.init() - await second.ready() - - expect(second.getPendingBackfills()).toEqual([]) - const rows = second.queryAggregate({ name: 'boot_agg' }) - const food = rows.find(r => r.groupKey.category === 'food')! - expect(food.metrics.count).toBe(2) - }) - - it('a write landing before adoption forces an exact rescan instead', async () => { - const store = new MemoryStorage() - await store.init() - await seedAndFlush(store) - - const second = new AggregationIndex(store) - second.defineAggregate(DEF) - second.onEntityAdded('e4', entity('e4', 'food')) // lands before init settles - await second.init() - await second.ready() - - // Adoption would lose e4's contribution — the engine must rescan. - expect(second.getPendingBackfills()).toEqual(['boot_agg']) - }) - - it('init never clobbers a changed app definition registered before it', async () => { - const store = new MemoryStorage() - await store.init() - await seedAndFlush(store) - - const CHANGED: AggregateDefinition = { - ...DEF, - metrics: { count: { op: 'count' }, total: { op: 'sum', field: 'amount' } } - } - const second = new AggregationIndex(store) - second.defineAggregate(CHANGED) - await second.init() - await second.ready() - - const def = second.getDefinitions().find(d => d.name === 'boot_agg')! - expect(Object.keys(def.metrics).sort()).toEqual(['count', 'total']) - expect(second.getPendingBackfills()).toEqual(['boot_agg']) - }) - - it('init alone restores persisted definitions with adopted state, no backfill', async () => { - const store = new MemoryStorage() - await store.init() - await seedAndFlush(store) - - const second = new AggregationIndex(store) - await second.init() - await second.ready() - - expect(second.hasAggregate('boot_agg')).toBe(true) - expect(second.getPendingBackfills()).toEqual([]) - const rows = second.queryAggregate({ name: 'boot_agg' }) - expect(rows.reduce((s, r) => s + (r.metrics.count as number), 0)).toBe(3) - }) - }) }) diff --git a/tests/unit/aggregation/materialize-failure-loud.test.ts b/tests/unit/aggregation/materialize-failure-loud.test.ts deleted file mode 100644 index 9aa2d1cb..00000000 --- a/tests/unit/aggregation/materialize-failure-loud.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -/** - * @module tests/unit/aggregation/materialize-failure-loud - * @description Aggregation (Pattern C): a failed materialization must not be - * swallowed. `scheduleMaterialize`'s debounced `materializeOne(key).catch(() => {})` - * silently discarded the error, leaving the materialized Measurement entity stale - * with no signal. It now emits a loud warning (the value is still rebuildable via - * backfill-on-query, so it stays non-fatal — but visible). Load-bearing assertion: - * prodLog.warn fires instead of an empty catch. - */ -import { describe, it, expect, vi, afterEach } from 'vitest' -import { AggregateMaterializer } from '../../../src/aggregation/materializer.js' -import { prodLog } from '../../../src/utils/logger.js' -import { NounType } from '../../../src/types/graphTypes.js' - -describe('Aggregation — swallowed materialize failure is surfaced', () => { - afterEach(() => { - vi.restoreAllMocks() - vi.useRealTimers() - }) - - it('logs loudly when materializeOne rejects', async () => { - vi.useFakeTimers() - const warn = vi.spyOn(prodLog, 'warn').mockImplementation(() => {}) - - // Brain access whose add()/update() always reject → doMaterialize throws. - const brain = { - add: vi.fn().mockRejectedValue(new Error('storage down')), - update: vi.fn().mockRejectedValue(new Error('storage down')) - } - const m = new AggregateMaterializer(brain as any, 10) - const def: any = { - name: 'sales', - source: { type: NounType.Document }, - groupBy: ['region'], - metrics: { total: { op: 'count' } }, - materialize: { debounceMs: 10 } - } - m.scheduleMaterialize( - 'sales', - def, - { region: 'us' }, - { groupKey: { region: 'us' }, metrics: { total: { count: 1 } } } as any - ) - - await vi.advanceTimersByTimeAsync(20) // fire the debounce timer - await Promise.resolve() - - expect(warn).toHaveBeenCalled() - expect(String(warn.mock.calls[0][0])).toMatch(/Failed to materialize aggregate group/) - }) -}) diff --git a/tests/unit/brainy/batch-operations.test.ts b/tests/unit/brainy/batch-operations.test.ts index 58b25744..b2cc7f09 100644 --- a/tests/unit/brainy/batch-operations.test.ts +++ b/tests/unit/brainy/batch-operations.test.ts @@ -533,10 +533,8 @@ describe('Brainy Batch Operations', () => { expect(result.successful).toHaveLength(0) await brain.updateMany({ items: [] }) - // removeMany is the exception (8.8.2): an empty id list is a refused - // selector, not an empty batch — deleting "nothing" silently was the - // bug class (a positional/bare-array call looked identical). - await expect(brain.removeMany({ ids: [] })).rejects.toThrow(/ids: \[\]/) + await brain.removeMany({ ids: [] }) + // Should not throw }) it('should validate batch size limits', async () => { diff --git a/tests/unit/brainy/degraded-reads-surfaced.test.ts b/tests/unit/brainy/degraded-reads-surfaced.test.ts deleted file mode 100644 index 29a8a77c..00000000 --- a/tests/unit/brainy/degraded-reads-surfaced.test.ts +++ /dev/null @@ -1,79 +0,0 @@ -/** - * @module tests/unit/brainy/degraded-reads-surfaced - * @description Finding 10: a known-degraded derived index must be SURFACED, not - * silently served as authoritative. Two degraded sources: - * (a) a non-fatal index rebuild failure at init() (`_indexRebuildFailed`), and - * (b) an adopt-forward failed-rollback recovery (`commitSingleOp`'s `degraded` - * ids, previously dropped on the floor by persistSingleOp). - * Both now fold into checkHealth()/validateIndexConsistency() and emit ONE loud - * read-path warning per degraded window; repairIndex() reconciles + clears them. - * - * The private fields are the observable contract of the fix, so the test drives - * them directly (a real failed rollback is exercised elsewhere). - */ -import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../../src/brainy.js' -import { NounType } from '../../../src/types/graphTypes.js' -import { prodLog } from '../../../src/utils/logger.js' - -const UUID = (suffix: string): string => `00000000-0000-4000-8000-0000000000${suffix}` - -describe('Finding 10 — degraded derived-index state is surfaced on reads', () => { - beforeEach(() => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - }) - afterEach(() => vi.restoreAllMocks()) - - it('checkHealth() reports adopt-forward degraded ids as unhealthy', async () => { - const brain = new Brainy({ storage: { type: 'memory' }, dimensions: 384, requireSubtype: false }) - await brain.init() - ;(brain as any)._indexDegradedIds.add(UUID('de')) - - const health = await brain.checkHealth() - expect(health.healthy).toBe(false) - expect(health.recommendation).toMatch(/repairIndex\(\)/) - }) - - it('find()/get() warn loudly while degraded, ONCE, then repairIndex() clears it', async () => { - const warn = vi.spyOn(prodLog, 'warn').mockImplementation(() => {}) - const brain = new Brainy({ storage: { type: 'memory' }, dimensions: 384, requireSubtype: false }) - await brain.init() - await brain.add({ id: UUID('a1'), data: 'x', type: NounType.Document }) - ;(brain as any)._indexRebuildFailed = new Error('rebuild boom') - - await brain.find({ type: NounType.Document }) - await brain.get(UUID('a1')) // second read: must NOT double-warn - const degradedWarns = warn.mock.calls.filter((c) => - String(c[0]).includes('derived index is INCOMPLETE') - ) - expect(degradedWarns.length).toBe(1) - - await brain.repairIndex() - expect((brain as any)._indexRebuildFailed).toBeNull() - expect((brain as any)._indexDegradedIds.size).toBe(0) - - warn.mockClear() - await brain.find({ type: NounType.Document }) - expect(warn.mock.calls.filter((c) => String(c[0]).includes('INCOMPLETE')).length).toBe(0) - }) - - it('persistSingleOp records receipt.degraded (widened return type, not dropped)', async () => { - const brain = new Brainy({ storage: { type: 'memory' }, dimensions: 384, requireSubtype: false }) - await brain.init() - // Simulate a degraded receipt by wrapping the generation store's commitSingleOp. - const gs: any = (brain as any).generationStore - const realCommit = gs.commitSingleOp.bind(gs) - vi.spyOn(gs, 'commitSingleOp').mockImplementation(async (args: any) => { - const r = await realCommit(args) - return { ...r, degraded: [...(args.touched.nouns ?? [])] } - }) - - await brain.add({ id: UUID('b2'), data: 'y', type: NounType.Document }) - expect((brain as any)._indexDegradedIds.size).toBeGreaterThan(0) - expect((brain as any)._indexDegradedIds.has(UUID('b2'))).toBe(true) - - const health = await brain.checkHealth() - expect(health.healthy).toBe(false) - expect(health.recommendation).toMatch(/repairIndex\(\)/) - }) -}) diff --git a/tests/unit/brainy/migration-gate-family-scoped.test.ts b/tests/unit/brainy/migration-gate-family-scoped.test.ts deleted file mode 100644 index b71c3899..00000000 --- a/tests/unit/brainy/migration-gate-family-scoped.test.ts +++ /dev/null @@ -1,94 +0,0 @@ -/** - * @module tests/unit/brainy/migration-gate-family-scoped - * @description The coordinated migration LOCK is FAMILY-SCOPED: a read waits only - * on the index families it actually consults. A one-time native migration of ONE - * family (its `isMigrating()` held) must NOT block a read served entirely from - * canonical storage (get / VFS content + dir) or from a HEALTHY family — only a - * read that needs the migrating family blocks. Regression for the whole-brain - * gate that hung getStats / readdir / readFile behind an unrelated family's - * migration until the wait timed out. - */ -import { describe, it, expect, beforeEach } from 'vitest' -import { Brainy } from '../../../src/brainy.js' -import { MigrationInProgressError } from '../../../src/errors/brainyError.js' - -const seed = async () => { - const brain = new Brainy({ - storage: { type: 'memory' }, - dimensions: 384, - requireSubtype: false, - silent: true, - // Short so a genuine block resolves fast; a real block would otherwise wait - // the 30 s default — this test asserts the wait does NOT happen for scoped - // reads, and DOES (bounded) for the one that needs the migrating family. - migrationWaitTimeoutMs: 300 - }) - await brain.init() - for (let i = 0; i < 5; i++) { - await brain.add({ data: `doc ${i}`, type: 'document', metadata: { i } }) - } - await brain.vfs.writeFile('/notes/hello.txt', 'canonical bytes — no index needed') - await brain.flush() - return brain -} - -/** Force a provider to report an in-flight (stuck) migration. */ -const jam = (provider: unknown) => { - ;(provider as { isMigrating: () => boolean }).isMigrating = () => true -} - -describe('migration LOCK is family-scoped', () => { - beforeEach(() => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - }) - - it('a stuck VECTOR migration does not block canonical or graph/metadata reads', async () => { - const brain = await seed() - const childId = ( - (await brain.vfs.readdir('/notes', { withFileTypes: true })) as Array<{ entityId: string }> - )[0].entityId - - jam((brain as any).index) // vector provider migrating; graph + metadata healthy - - // None of these consult the vector family → they serve immediately. - await expect(brain.getStats()).resolves.toBeDefined() - await expect(brain.vfs.readdir('/notes')).resolves.toHaveLength(1) // graph traversal - await expect(brain.vfs.readFile('/notes/hello.txt')).resolves.toBeDefined() // canonical blob - await expect(brain.get(childId)).resolves.not.toBeNull() // canonical by id - await expect(brain.find({ where: { i: 1 } })).resolves.toBeDefined() // metadata filter - }) - - it('a stuck VECTOR migration STILL blocks a read that needs the vector family', async () => { - const brain = await seed() - jam((brain as any).index) - - // A semantic query consults the vector index — it must wait, and (bounded by - // migrationWaitTimeoutMs) surface the retryable MigrationInProgressError - // rather than serve a half-built result. - await expect(brain.find({ query: 'doc' })).rejects.toBeInstanceOf(MigrationInProgressError) - }) - - it('a stuck GRAPH migration blocks traversal but not vector/canonical reads', async () => { - const brain = await seed() - const childId = ( - (await brain.vfs.readdir('/notes', { withFileTypes: true })) as Array<{ entityId: string }> - )[0].entityId - - jam((brain as any).graphIndex) // graph provider migrating; vector + metadata healthy - - // Canonical + vector + metadata are unaffected. - await expect(brain.get(childId)).resolves.not.toBeNull() - await expect(brain.find({ query: 'doc' })).resolves.toBeDefined() - await expect(brain.find({ where: { i: 1 } })).resolves.toBeDefined() - - // A graph traversal needs the migrating family → it blocks (bounded). - await expect(brain.related({ from: childId })).rejects.toBeInstanceOf(MigrationInProgressError) - }) - - it('with no migration in flight, every read serves (the fast path is a no-op)', async () => { - const brain = await seed() - await expect(brain.getStats()).resolves.toBeDefined() - await expect(brain.find({ query: 'doc' })).resolves.toBeDefined() - await expect(brain.vfs.readdir('/notes')).resolves.toHaveLength(1) - }) -}) diff --git a/tests/unit/db/fact-log.test.ts b/tests/unit/db/fact-log.test.ts deleted file mode 100644 index f1c226cc..00000000 --- a/tests/unit/db/fact-log.test.ts +++ /dev/null @@ -1,237 +0,0 @@ -/** - * @module tests/unit/db/fact-log - * @description The generation fact log in isolation: wire-format round-trip - * (positional msgpack facts, bin16 uuids, body-less tombstones), crc32c - * framing with torn-tail detection, open-time truncation to committed truth - * (the log can only ever be AHEAD after a crash; open cuts it back), rotation - * with a manifest-first flip, exactly-once scans with the frozen telemetry - * shape, and the mmap segment handoff excluding the mutable tail. - */ -import { describe, it, expect, beforeEach } from 'vitest' -import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' -import { - FactLog, - FACTS_PREFIX, - type CommitFact, - type FactLogStorage, - storageSupportsFactLog -} from '../../../src/db/factLog.js' -import { crc32c } from '../../../src/utils/crc32c.js' - -const UUID = (n: number): string => - `00000000-0000-4000-8000-${String(n).padStart(12, '0')}` - -const fact = (generation: number, overrides?: Partial): CommitFact => ({ - generation, - timestamp: 1_700_000_000_000 + generation, - ops: [ - { - kind: 'noun', - id: UUID(generation), - record: { metadata: { noun: 'document', title: `doc ${generation}` }, vector: { v: [1, 2] } } - } - ], - ...overrides -}) - -describe('crc32c known-answer vectors', () => { - it('matches the RFC 3720 test vectors', () => { - expect(crc32c(new TextEncoder().encode('123456789'))).toBe(0xe3069283) - expect(crc32c(new Uint8Array(32))).toBe(0x8a9136aa) - }) -}) - -describe('fact log — round-trip, framing, reconcile, rotation, scan', () => { - let storage: FactLogStorage - let log: FactLog - - beforeEach(async () => { - const mem: any = new MemoryStorage() - await mem.init() - expect(storageSupportsFactLog(mem)).toBe(true) - storage = mem - log = new FactLog(storage) - await log.open(0) - }) - - it('facts round-trip byte-exactly: ops, tombstones, meta, blobHashes', async () => { - await log.append(fact(1)) - await log.append( - fact(2, { - ops: [ - { kind: 'verb', id: UUID(21), record: { metadata: { verb: 'contains' }, vector: null } }, - { kind: 'noun', id: UUID(22), record: null } // TOMBSTONE - ], - meta: { source: 'test' }, - blobHashes: ['abc123', 'abc123'] // multiset — duplicates preserved - }) - ) - await log.sync() - - const scan = log.scanFacts() - expect(scan.headGeneration).toBe(2) - const all: CommitFact[] = [] - for await (const batch of scan.batches()) all.push(...batch.facts) - - expect(all).toHaveLength(2) - expect(all[0].generation).toBe(1) - expect(all[0].ops[0].id).toBe(UUID(1)) - expect(all[0].ops[0].record?.metadata).toEqual({ noun: 'document', title: 'doc 1' }) - expect(all[1].ops[0].kind).toBe('verb') - expect(all[1].ops[1].record).toBeNull() // the tombstone is body-less - expect(all[1].meta).toEqual({ source: 'test' }) - expect(all[1].blobHashes).toEqual(['abc123', 'abc123']) - expect(scan.summary().factsYielded).toBe(2) - }) - - it('appends are monotonic — a replayed/duplicate generation throws', async () => { - await log.append(fact(5)) - await expect(log.append(fact(5))).rejects.toThrow(/non-monotonic/) - await expect(log.append(fact(3))).rejects.toThrow(/non-monotonic/) - await expect(log.append(fact(6))).resolves.toBeUndefined() // gaps are fine (aborted reservations) - }) - - it('a torn tail (partial frame) is detected and ignored — intact prefix survives', async () => { - await log.append(fact(1)) - await log.append(fact(2)) - await log.sync() - - // Simulate a crash mid-append: chop bytes off the tail file. - const tailPath = `${FACTS_PREFIX}/seg-${'1'.padStart(20, '0')}.bfl` - const bytes = (await storage.readRawBytes(tailPath))! - await storage.writeRawBytes(tailPath, bytes.subarray(0, bytes.length - 7)) - - const reopened = new FactLog(storage) - await reopened.open(2) - expect(reopened.headGeneration()).toBe(1) // fact 2's frame was torn → gone - - const all: CommitFact[] = [] - for await (const b of reopened.scanFacts().batches()) all.push(...b.facts) - expect(all.map((f) => f.generation)).toEqual([1]) - }) - - it('open() truncates facts beyond committed truth (the crash-ahead shape)', async () => { - await log.append(fact(1)) - await log.append(fact(2)) - await log.append(fact(3)) - await log.sync() - - // The store's committed generation is 1 — facts 2..3 never committed. - const reopened = new FactLog(storage) - await reopened.open(1) - expect(reopened.headGeneration()).toBe(1) - - const all: CommitFact[] = [] - for await (const b of reopened.scanFacts().batches()) all.push(...b.facts) - expect(all.map((f) => f.generation)).toEqual([1]) - - // And appends continue cleanly from the truncated head. - await reopened.append(fact(2)) - expect(reopened.headGeneration()).toBe(2) - }) - - it('a cleared store (committed=0) truncates everything', async () => { - await log.append(fact(1)) - await log.append(fact(2)) - await log.sync() - const reopened = new FactLog(storage) - await reopened.open(0) - expect(reopened.headGeneration()).toBe(0) - }) - - it('scan honors fromGeneration/toGeneration inclusively and filters kinds', async () => { - for (let g = 1; g <= 6; g++) await log.append(fact(g)) - await log.sync() - - const scan = log.scanFacts({ fromGeneration: 2, toGeneration: 4 }) - const all: CommitFact[] = [] - for await (const b of scan.batches()) all.push(...b.facts) - expect(all.map((f) => f.generation)).toEqual([2, 3, 4]) - - const verbsOnly = log.scanFacts({ kinds: ['verb'] }) - for await (const b of verbsOnly.batches()) { - for (const f of b.facts) expect(f.ops.every((op) => op.kind === 'verb')).toBe(true) - } - }) - - it('batch telemetry carries the frozen shape', async () => { - for (let g = 1; g <= 5; g++) await log.append(fact(g)) - await log.sync() - - const scan = log.scanFacts({ batchSize: 2 }) - expect(scan.approxFactCount).toBe(5) - const batches = [] - for await (const b of scan.batches()) batches.push(b) - expect(batches.length).toBe(3) - expect(batches[0]).toMatchObject({ firstGeneration: 1, lastGeneration: 2, factCount: 2 }) - expect(batches[0].byteSize).toBeGreaterThan(0) - expect(typeof batches[0].segmentId).toBe('string') - expect(scan.summary()).toEqual({ factsYielded: 5, segmentsRead: 1 }) - }) - - it('survives reopen: head and content come back from disk', async () => { - for (let g = 1; g <= 3; g++) await log.append(fact(g)) - await log.sync() - - const reopened = new FactLog(storage) - await reopened.open(3) - expect(reopened.headGeneration()).toBe(3) - const all: CommitFact[] = [] - for await (const b of reopened.scanFacts().batches()) all.push(...b.facts) - expect(all.map((f) => f.generation)).toEqual([1, 2, 3]) - }) - - it('segmentPaths excludes the mutable tail (mmap handoff = sealed only)', async () => { - await log.append(fact(1)) - await log.sync() - expect(log.segmentPaths()).toEqual([]) // only a tail exists — nothing sealed - }) - - describe('scanFacts liveness contract (Stage-2 D1)', () => { - it('a wedged store fails LOUDLY within the first-batch bound — never a silent hang', async () => { - // Force a sealed segment (tiny rotateBytes) so the scan must READ from - // storage, then wedge that read: the exact production shape (a - // backlogged brain whose segment read never returned). - const mem: any = new MemoryStorage() - await mem.init() - const wedgeable = new FactLog(mem, { rotateBytes: 1 }) - await wedgeable.open(0) - await wedgeable.append(fact(1)) - await wedgeable.append(fact(2)) // second append rotates → seg 1 sealed - await wedgeable.sync() - - const realRead = mem.readRawBytes.bind(mem) - mem.readRawBytes = (p: string) => - p.includes('facts/seg-') ? new Promise(() => {}) : realRead(p) // hangs forever - - const scan = wedgeable.scanFacts({ firstBatchTimeoutMs: 200 }) - const started = Date.now() - await expect(scan.batches().next()).rejects.toThrow(/no first batch within 200ms/) - expect(Date.now() - started).toBeLessThan(5_000) // bound held, not a hang - }) - - it('a healthy scan is unaffected — first batch well inside the bound, all facts delivered', async () => { - for (let g = 1; g <= 5; g++) await log.append(fact(g)) - await log.sync() - const scan = log.scanFacts({ batchSize: 2 }) - const all: CommitFact[] = [] - for await (const b of scan.batches()) all.push(...b.facts) - expect(all.map((f) => f.generation)).toEqual([1, 2, 3, 4, 5]) - expect(scan.summary().factsYielded).toBe(5) - }) - - it('consumer think-time between pulls never counts against the producer', async () => { - for (let g = 1; g <= 4; g++) await log.append(fact(g)) - await log.sync() - // Bound tighter than the consumer's pause: only the FIRST pull is - // raced, so a slow consumer after batch 1 must not trip the deadline. - const gen = log.scanFacts({ batchSize: 2, firstBatchTimeoutMs: 150 }).batches() - const first = await gen.next() - expect(first.done).toBe(false) - await new Promise((r) => setTimeout(r, 400)) // dawdle past the bound - const second = await gen.next() - expect(second.done).toBe(false) - expect((await gen.next()).done).toBe(true) - }) - }) -}) diff --git a/tests/unit/db/generation-segments.test.ts b/tests/unit/db/generation-segments.test.ts deleted file mode 100644 index 27ab85cb..00000000 --- a/tests/unit/db/generation-segments.test.ts +++ /dev/null @@ -1,150 +0,0 @@ -/** - * @module tests/unit/db/generation-segments - * @description The generation-segment store (Stage-2 D1+D3 file format). - * Laws: (1) fold → read round-trips deltas and records byte-faithfully via - * sidecar point-reads; (2) the manifest is the ONLY discovery path — reopen - * reads one file, never a listing; (3) a lost/corrupt sidecar rebuilds from - * its segment loudly, a damaged SEGMENT fails loudly (never silent wrong - * data); (4) D3 reclaim drops whole segments only and bumps compactedBelow; - * (5) the packed digest is deterministic across reopen; (6) immutability — - * fold refuses overlap with sealed ranges. - */ -import { describe, it, expect, beforeEach } from 'vitest' -import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' -import { - GenerationSegmentStore, - SEGMENTS_PREFIX, - type FoldGeneration -} from '../../../src/db/generationSegments.js' - -const UUID = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, '0')}` - -const gen = (g: number, recordCount = 2): FoldGeneration => ({ - generation: g, - timestamp: 1_700_000_000_000 + g, - delta: { generation: g, nouns: [UUID(g)], verbs: [], bytes: 123 + g }, - records: Array.from({ length: recordCount }, (_, i) => ({ - kind: (i % 2 === 0 ? 'noun' : 'verb') as 'noun' | 'verb', - id: UUID(g * 100 + i), - record: { metadata: { noun: 'document', v: g }, vector: { v: [g, i] } } - })) -}) - -describe('db/GenerationSegmentStore — the D1+D3 packed tier', () => { - let storage: MemoryStorage - let store: GenerationSegmentStore - - beforeEach(async () => { - storage = new MemoryStorage() - await storage.init() - store = new GenerationSegmentStore(storage as any) - await store.open() - }) - - it('fold → read round-trips deltas and records via sidecar point-reads', async () => { - const meta = await store.fold([gen(1), gen(2), gen(3)]) - expect(meta).toMatchObject({ firstGeneration: 1, lastGeneration: 3, frames: 3 }) - expect(meta.checksum).toBeGreaterThan(0) - - expect(store.hasGeneration(2)).toBe(true) - expect(store.hasGeneration(4)).toBe(false) - - const d2 = await store.readDelta(2) - expect(d2?.delta).toEqual({ generation: 2, nouns: [UUID(2)], verbs: [], bytes: 125 }) - expect(d2?.timestamp).toBe(1_700_000_000_002) - - const records = await store.readRecords(3) - expect(records).toHaveLength(2) - expect(records![0]).toEqual({ - kind: 'noun', - id: UUID(300), - record: { metadata: { noun: 'document', v: 3 }, vector: { v: [3, 0] } } - }) - // Point read by id, both kinds. - expect(await store.readRecord(3, 'verb', UUID(301))).toEqual({ - metadata: { noun: 'document', v: 3 }, - vector: { v: [3, 1] } - }) - expect(await store.readRecord(3, 'noun', UUID(999))).toBeNull() - }) - - it('reopen discovers everything from the manifest alone — no listing', async () => { - await store.fold([gen(1), gen(2)]) - await store.fold([gen(3), gen(4)]) - - const reopened = new GenerationSegmentStore(storage as any) - await reopened.open() - expect(reopened.segments()).toHaveLength(2) - expect(reopened.hasGeneration(4)).toBe(true) - expect((await reopened.readDelta(1))?.timestamp).toBe(1_700_000_000_001) - }) - - it('a lost sidecar rebuilds from its segment; a damaged segment fails LOUDLY', async () => { - const meta = await store.fold([gen(1), gen(2)]) - const idxPath = `${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.idx` - await storage.deleteRawObject(idxPath) - - const reopened = new GenerationSegmentStore(storage as any) - await reopened.open() - // Rebuild path: still serves correct data. - expect((await reopened.readRecords(2))!).toHaveLength(2) - - // Now damage the SEGMENT itself: flip a payload byte → CRC mismatch, loud. - const segPath = `${SEGMENTS_PREFIX}/${meta.file}` - const bytes = (await storage.readRawBytes(segPath))! - bytes[bytes.length - 3] ^= 0xff - await storage.writeRawBytes(segPath, bytes) - const damaged = new GenerationSegmentStore(storage as any) - await damaged.open() - ;(damaged as any).sidecars.clear() - await storage.deleteRawObject(idxPath) // force the sequential rebuild over damaged bytes - await expect(damaged.readRecords(2)).rejects.toThrow(/CRC mismatch|damaged/) - }) - - it('D3 reclaim drops whole segments only and bumps compactedBelow', async () => { - await store.fold([gen(1), gen(2)]) - await store.fold([gen(3), gen(4)]) - await store.fold([gen(5), gen(6)]) - - // Horizon mid-segment-2 (below 4): only segment 1 is FULLY below → drops. - const r1 = await store.dropSegmentsBelow(4) - expect(r1).toEqual({ dropped: 1, compactedBelow: 3 }) - expect(store.hasGeneration(1)).toBe(false) - expect(store.hasGeneration(3)).toBe(true) // partial segment survives whole - - // Bytes actually gone. - expect(await storage.readRawBytes(`${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.bgs`)).toBeNull() - - // Horizon past everything: the rest drop; compactedBelow is durable. - const r2 = await store.dropSegmentsBelow(7) - expect(r2.dropped).toBe(2) - const reopened = new GenerationSegmentStore(storage as any) - await reopened.open() - expect(reopened.compactedBelow()).toBe(7) - expect(reopened.segments()).toHaveLength(0) - }) - - it('the packed digest is deterministic across reopen and changes with history', async () => { - await store.fold([gen(1), gen(2), gen(3)]) - const atSeal = await store.digestThroughPacked(3) - const midSegment = await store.digestThroughPacked(2) - expect(atSeal).not.toBeNull() - expect(midSegment).not.toBeNull() - expect(midSegment).not.toBe(atSeal) - - const reopened = new GenerationSegmentStore(storage as any) - await reopened.open() - expect(await reopened.digestThroughPacked(3)).toBe(atSeal) - expect(await reopened.digestThroughPacked(2)).toBe(midSegment) - - await reopened.fold([gen(4)]) - expect(await reopened.digestThroughPacked(4)).not.toBe(atSeal) - }) - - it('sealed segments are immutable — fold refuses overlap, requires ascending input', async () => { - await store.fold([gen(1), gen(2)]) - await expect(store.fold([gen(2), gen(3)])).rejects.toThrow(/overlaps the packed tier/) - await expect(store.fold([gen(4), gen(4)])).rejects.toThrow(/strictly ascending/) - await expect(store.fold([])).rejects.toThrow(/at least one generation/) - }) -}) diff --git a/tests/unit/db/generationStore.test.ts b/tests/unit/db/generationStore.test.ts index 5b667415..8073d7ee 100644 --- a/tests/unit/db/generationStore.test.ts +++ b/tests/unit/db/generationStore.test.ts @@ -249,11 +249,7 @@ describe('db/GenerationStore', () => { store.release(pinned) const result = await store.compact() expect(result.removedGenerations).toBeGreaterThan(0) - // History record-sets only — the fact log (at `_generations/facts/`) is - // deliberately NOT reclaimed by history compaction. - const remaining = (await storage.listRawObjects(GENERATIONS_PREFIX)).filter( - (p: string) => !p.startsWith(`${GENERATIONS_PREFIX}/facts/`) - ) + const remaining = await storage.listRawObjects(GENERATIONS_PREFIX) expect(remaining).toEqual([]) }) @@ -488,95 +484,5 @@ describe('db/GenerationStore', () => { expect(result.removedGenerations).toBe(2) store.release(2) }) - - it('timeBudgetMs bounds a pass; the next pass resumes the same prefix', async () => { - await manyGens(4) - // A spent budget (0ms) stops before reclaiming anything — an early stop - // is a consistent prefix, never a partial generation. - const bounded = await store.compact({ timeBudgetMs: 0 }) - expect(bounded.removedGenerations).toBe(0) - expect(bounded.horizon).toBe(0) - // The next (unbounded) pass picks up exactly where the bounded one - // stopped and completes the same work. - const resumed = await store.compact() - expect(resumed.removedGenerations).toBe(4) - expect(resumed.horizon).toBe(4) - }) - }) - - // ========================================================================== - describe('history-bytes running total (the O(1) retention check)', () => { - /** A fresh walk with the cache dropped — ground truth for the invariant. */ - async function groundTruthBytes(): Promise { - ;(store as any).historyBytesTotal = null - return store.historyBytes() - } - - it('is seeded once, then maintained through commits WITHOUT re-walks', async () => { - await commitWrite(ID_A, 1) - await commitWrite(ID_A, 2) - const seeded = await store.historyBytes() - expect(seeded).toBe(await groundTruthBytes()) - - // From here every read must come from the running total, not a walk: - // getDelta re-reads are the walk's cost — commits must not trigger any. - const getDeltaSpy = vi.spyOn(store as any, 'getDelta') - await commitWrite(ID_B, 1) - const afterCommit = await store.historyBytes() - expect(getDeltaSpy).not.toHaveBeenCalled() - getDeltaSpy.mockRestore() - expect(afterCommit).toBe(await groundTruthBytes()) - }) - - it('stays exact through single-op group commits and compaction', async () => { - await commitWrite(ID_A, 1) - await store.historyBytes() // seed - // Single-op path: buffered generations flushed as one group commit. - await store.commitSingleOp({ - touched: { nouns: [ID_B] }, - execute: async () => { - await storage.saveNounMetadata(ID_B, metadataFixture(1)) - } - }) - await store.flushPendingSingleOps() - expect(await store.historyBytes()).toBe(await groundTruthBytes()) - - await store.historyBytes() // re-seed after ground-truth reset - await store.compact({ maxGenerations: 1 }) - expect(await store.historyBytes()).toBe(await groundTruthBytes()) - }) - - it('historyStats reports counts, bytes, range, and horizon read-only', async () => { - await commitWrite(ID_A, 1) - await commitWrite(ID_B, 1) - const stats = await store.historyStats() - expect(stats.generations).toBe(2) - expect(stats.bytes).toBe(await store.historyBytes()) - expect(stats.oldestGeneration).toBe(1) - expect(stats.newestGeneration).toBe(2) - expect(stats.oldestTimestamp).toBeLessThanOrEqual(stats.newestTimestamp!) - expect(stats.horizon).toBe(0) - // Read-only: nothing was reclaimed by asking. - expect(store.committedGeneration()).toBe(2) - - await store.compact({ maxGenerations: 1 }) - const after = await store.historyStats() - expect(after.generations).toBe(1) - expect(after.oldestGeneration).toBe(2) - expect(after.horizon).toBe(1) - }) - - it('empty history reports null range and zero bytes', async () => { - const stats = await store.historyStats() - expect(stats).toMatchObject({ - generations: 0, - bytes: 0, - oldestGeneration: null, - newestGeneration: null, - oldestTimestamp: null, - newestTimestamp: null, - horizon: 0 - }) - }) }) }) diff --git a/tests/unit/db/pending-flush-durability.test.ts b/tests/unit/db/pending-flush-durability.test.ts deleted file mode 100644 index f8501c46..00000000 --- a/tests/unit/db/pending-flush-durability.test.ts +++ /dev/null @@ -1,106 +0,0 @@ -/** - * @module tests/unit/db/pending-flush-durability - * @description Finding 8: the async group-commit flush that persists single-op - * generation HISTORY must not swallow persist failures. Before the fix a failed - * background flush was a bare `prodLog.warn` — writes kept succeeding while their - * before-images silently piled up in memory, never durable and unbounded. - * - * The contract now: - * - a single transient flush failure does NOT trip the alarm (tolerance), and - * - after PENDING_FLUSH_FAILURE_THRESHOLD consecutive failures the store LATCHES - * and REFUSES further writes with a typed PendingFlushDurabilityError rather - * than accumulate un-durable history, and - * - it self-heals: once a flush finally succeeds the latch lifts and writes resume. - * Live canonical data is never touched — only the immutable history is stuck. - * - * Fake timers keep the background retry/coalesce timers from firing, so the exact - * number of flush attempts (and thus the latch point) is deterministic — the test - * drives every flush explicitly. - */ -import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' -import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' -import { GenerationStore } from '../../../src/db/generationStore.js' -import { PendingFlushDurabilityError } from '../../../src/db/errors.js' -import { NounType } from '../../../src/types/graphTypes.js' - -const ID = (suffix: string): string => `00000000-0000-4000-8000-0000000000${suffix}` -const meta = (v: number): Record => ({ - noun: NounType.Document, - subtype: 'note', - data: `payload-v${v}`, - version: v, - _rev: v -}) - -describe('db/GenerationStore pending-flush durability (finding 8)', () => { - let storage: MemoryStorage - let store: GenerationStore - - beforeEach(async () => { - vi.useFakeTimers() - storage = new MemoryStorage() - await storage.init() - store = new GenerationStore(storage) - await store.open() - // High size threshold so single-ops never auto-flush — the test owns every flush. - ;(store as any).pendingFlushThreshold = 1000 - }) - - afterEach(() => { - vi.useRealTimers() - }) - - /** One single-op write that buffers a pending generation. */ - const singleOp = (id: string, v: number) => - store.commitSingleOp({ - touched: { nouns: [id], verbs: [] }, - execute: async () => { - await storage.saveNounMetadata(id, meta(v)) - } - }) - - it('latches after repeated flush failures, refuses writes, then self-heals', async () => { - // Buffer a pending generation while storage is healthy. - await singleOp(ID('aa'), 1) - - // Fault the HISTORY flush (raw generation-record writes) — NOT the live - // canonical write, which uses a different path (saveNounMetadata). - const fault = Object.assign(new Error('EIO simulated'), { code: 'EIO' }) - const spy = vi.spyOn(storage as any, 'writeRawObject').mockRejectedValue(fault) - - // Each explicit flush fails and is accounted; the third trips the latch. - for (let i = 0; i < 3; i++) { - await expect(store.flushPendingSingleOps()).rejects.toThrow('EIO simulated') - } - - // Writes are now refused LOUDLY rather than piling up un-durable history. - await expect(singleOp(ID('bb'), 1)).rejects.toBeInstanceOf(PendingFlushDurabilityError) - // The live counter did not advance for the refused write (no generation consumed). - expect(store.generation()).toBe(1) - - // Storage recovers → an explicit flush drains the retained tier and lifts the latch. - spy.mockRestore() - await expect(store.flushPendingSingleOps()).resolves.toBeUndefined() - - // Writes resume; the next single-op commits normally. - const receipt = await singleOp(ID('cc'), 1) - expect(receipt.generation).toBe(2) - }) - - it('a single transient flush failure does NOT latch — writes continue', async () => { - await singleOp(ID('aa'), 1) - - const spy = vi.spyOn(storage as any, 'writeRawObject').mockRejectedValue(new Error('blip')) - await expect(store.flushPendingSingleOps()).rejects.toThrow('blip') // failure #1, below threshold - spy.mockRestore() - - // Below the latch threshold → the store still accepts writes. - const receipt = await singleOp(ID('bb'), 1) - expect(receipt.generation).toBe(2) - - // A subsequent healthy flush drains the tier and resets the failure counter. - await expect(store.flushPendingSingleOps()).resolves.toBeUndefined() - expect((store as any).pendingFlushFailures).toBe(0) - expect((store as any).pendingFlushError).toBeNull() - }) -}) diff --git a/tests/unit/get-index-status-readiness.test.ts b/tests/unit/get-index-status-readiness.test.ts deleted file mode 100644 index 7f82ec5d..00000000 --- a/tests/unit/get-index-status-readiness.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -/** - * @module tests/unit/get-index-status-readiness - * @description Pattern-A / Finding 9: getIndexStatus reported `populated: size>0` - * and only `migrating`, so a native index that loaded its count but not its - * serving structure reported populated:true — a k8s readiness probe would 200 a - * brain that serves []. It now folds in the honest isReady() signal + the - * _indexRebuildFailed / _indexDegradedIds degraded states (mirroring - * validateIndexConsistency / checkHealth). - */ -import { describe, it, expect, beforeEach } from 'vitest' -import { Brainy, NounType } from '../../src/index.js' - -describe('getIndexStatus honest readiness (Finding 9)', () => { - let brain: any - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, dimensions: 384 }) - await brain.init() - await brain.add({ data: 'x', type: NounType.Concept }) - await brain.flush() - }) - - it('a not-ready provider makes populated honest (false) and exposes ready:false', async () => { - brain.index.isReady = () => false // count present, serving structure NOT loaded - const status = await brain.getIndexStatus() - expect(status.hnswIndex.populated).toBe(false) - expect(status.hnswIndex.ready).toBe(false) - delete brain.index.isReady - }) - - it('a ready provider reports populated:true + ready:true', async () => { - brain.index.isReady = () => true - const status = await brain.getIndexStatus() - expect(status.hnswIndex.populated).toBe(true) - expect(status.hnswIndex.ready).toBe(true) - delete brain.index.isReady - }) - - it('rebuildFailed / rebuildError surface the init-degraded state', async () => { - brain._indexRebuildFailed = new Error('boom') - const status = await brain.getIndexStatus() - expect(status.rebuildFailed).toBe(true) - expect(status.rebuildError).toBe('boom') - brain._indexRebuildFailed = null - }) - - it('degradedIds surfaces the adopt-forward degraded set', async () => { - brain._indexDegradedIds.add('00000000-0000-4000-8000-0000000000de') - const status = await brain.getIndexStatus() - expect(status.degradedIds).toBe(1) - brain._indexDegradedIds.clear() - }) - - it('a JS-baseline provider (no isReady) omits ready and falls back to size>0', async () => { - const status = await brain.getIndexStatus() - expect(status.hnswIndex.ready).toBeUndefined() - expect(status.hnswIndex.populated).toBe(brain.index.size() > 0) - }) -}) diff --git a/tests/unit/graph/graph-fastpath-honest-readiness.test.ts b/tests/unit/graph/graph-fastpath-honest-readiness.test.ts deleted file mode 100644 index 46d318b4..00000000 --- a/tests/unit/graph/graph-fastpath-honest-readiness.test.ts +++ /dev/null @@ -1,95 +0,0 @@ -/** - * @module tests/unit/graph/graph-fastpath-honest-readiness - * @description Pattern-A / Finding 2: getVerbsBySource/ByTarget gated the graph - * fast path on `graphIndex.isInitialized` (a proxy that reads true once the - * manifest/count loaded even if the source→target adjacency did NOT → the fast - * path then returned a silent []). The honest gate skips the fast path when the - * provider reports isReady()===false, falling to the correct canonical shard - * scan; and a one-shot probe self-heals a no-isReady provider whose adjacency - * did not cold-load. - */ -import { describe, it, expect, beforeEach } from 'vitest' -import { Brainy, NounType, VerbType } from '../../../src/index.js' - -describe('graph fast-path honest readiness (Finding 2)', () => { - let brain: any - let storage: any - let a: string - let b: string - let c: string - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, dimensions: 384 }) - await brain.init() - a = await brain.add({ data: 'a', type: NounType.Concept }) - b = await brain.add({ data: 'b', type: NounType.Concept }) - c = await brain.add({ data: 'c', type: NounType.Concept }) - await brain.relate({ from: a, to: b, type: VerbType.Contains }) - await brain.relate({ from: a, to: c, type: VerbType.Contains }) - await brain.flush() - storage = brain.storage - // Warm + wire the storage graph index (lazy) so we can stub it. - await storage.getVerbsBySource(a) - }) - - it('not-ready provider → shard scan returns the REAL edges, not a silent []', async () => { - const gi = storage.graphIndex - // Simulate a cold native provider: count/manifest loaded (isInitialized) but - // the source→target adjacency is NOT (isReady false; fast-path lookup empty). - gi.isReady = () => false - const origBySource = gi.getVerbIdsBySource.bind(gi) - gi.getVerbIdsBySource = async () => [] // cold adjacency — old fast path trusted this - storage._graphFastPathProbed = true // isolate the GATE (probe skips isReady-capable anyway) - try { - const verbs = await storage.getVerbsBySource(a) - // Honest gate skipped the cold fast path → canonical shard scan → real edges. - expect(verbs.length).toBe(2) - } finally { - delete gi.isReady - gi.getVerbIdsBySource = origBySource - } - }) - - it('ready provider → fast path is used and an empty result is trusted', async () => { - const gi = storage.graphIndex - gi.isReady = () => true - let fastPathCalls = 0 - const origBySource = gi.getVerbIdsBySource.bind(gi) - gi.getVerbIdsBySource = async (...args: any[]) => { fastPathCalls++; return origBySource(...args) } - storage._graphFastPathProbed = true - try { - // `c` has no OUTGOING edges — a genuinely empty result via the fast path. - const verbs = await storage.getVerbsBySource(c) - expect(verbs).toEqual([]) - expect(fastPathCalls).toBeGreaterThan(0) // fast path was taken, not a shard scan - } finally { - delete gi.isReady - gi.getVerbIdsBySource = origBySource - } - }) - - it('no-isReady provider whose adjacency did not cold-load → probe rebuilds once', async () => { - const gi = storage.graphIndex - if ('isReady' in gi) delete gi.isReady // force the "unknown" (no honest signal) path - // Force the probe to see a cold adjacency: a known edge resolves to no ints. - const origBySource = gi.getVerbIdsBySource.bind(gi) - let cold = true - gi.getVerbIdsBySource = async (...args: any[]) => (cold ? [] : origBySource(...args)) - let rebuilds = 0 - const origRebuild = gi.rebuild.bind(gi) - gi.rebuild = async (...args: any[]) => { rebuilds++; cold = false; return origRebuild(...args) } - storage._graphFastPathProbed = false // re-arm the one-shot probe - try { - await storage.getVerbsBySource(a) - expect(rebuilds).toBe(1) // probe detected cold adjacency + rebuilt once - expect(storage._graphFastPathProbed).toBe(true) // latched — no second rebuild - rebuilds = 0 - await storage.getVerbsByTarget(b) - expect(rebuilds).toBe(0) // probe does not re-run - } finally { - gi.getVerbIdsBySource = origBySource - gi.rebuild = origRebuild - } - }) -}) diff --git a/tests/unit/graph/lsm-partial-load-failclosed.test.ts b/tests/unit/graph/lsm-partial-load-failclosed.test.ts deleted file mode 100644 index 7082c0c6..00000000 --- a/tests/unit/graph/lsm-partial-load-failclosed.test.ts +++ /dev/null @@ -1,76 +0,0 @@ -/** - * @module tests/unit/graph/lsm-partial-load-failclosed - * @description LSMTree partial-load fail-closed guard (Finding 3 of the honest - * index-readiness spine plan). `loadSSTables()` used to swallow a per-SSTable - * load failure with only a `prodLog.warn`, then `loadManifest()` still published - * the FULL persisted `totalRelationships` count — so on a partial cold load, - * `size()`/`isHealthy()` reported the full count and "healthy" while a subset of - * the tree's edges were silently unqueryable. Fixed by failing the whole load - * closed on ANY per-SSTable failure: `loadManifest()`'s existing catch then - * resets `sstables`/`totalRelationships`/`sstablesByLevel` to honest-empty, so - * `size()` reports 0 and the graph layer's existing `size()===0` self-heal - * rebuilds from canonical records — the dishonest partial state can no longer be - * observed. - */ -import { describe, it, expect } from 'vitest' -import { LSMTree } from '../../../src/graph/lsm/LSMTree.js' -import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' - -describe('LSMTree partial SSTable load fails closed (honest size())', () => { - it('one SSTable fails to load → size() reports 0, not the manifest count', async () => { - const storage = new MemoryStorage() - await storage.init() - const prefix = 'test-lsm-partial' - - const t1 = new LSMTree(storage, { storagePrefix: prefix, enableCompaction: false }) - await t1.init() - await t1.add('a', 'b'); await t1.flush() // SSTable #1 - await t1.add('c', 'd'); await t1.flush() // SSTable #2 - expect(t1.size()).toBeGreaterThanOrEqual(2) - await t1.close() - - // Find an SSTable id from the persisted manifest. - const manifest = await storage.getMetadata(`${prefix}-manifest`) - const sstableIds = Object.keys((manifest!.data as any).sstables) - expect(sstableIds.length).toBeGreaterThanOrEqual(2) - - // Reopen with ONE SSTable load forced to fail (deterministic). - const t2 = new LSMTree(storage, { storagePrefix: prefix, enableCompaction: false }) - const realGet = storage.getMetadata.bind(storage) - ;(storage as any).getMetadata = async (key: string) => { - if (key === `${prefix}-${sstableIds[1]}`) throw new Error('simulated SSTable load failure') - return realGet(key) - } - await t2.init() - - // BEFORE THE FIX: t2.size() === (full manifest count) ← LIE (partial load, full count) - // AFTER THE FIX: t2.size() === 0 ← fail-closed → self-heal rebuilds - expect(t2.size()).toBe(0) - expect(t2.isHealthy()).toBe(true) // honestly-empty-but-initialized, not "healthy with a lie" - - await t2.close() - }) - - it('a clean load with no failures still reports the full honest count', async () => { - const storage = new MemoryStorage() - await storage.init() - const prefix = 'test-lsm-clean' - - const t1 = new LSMTree(storage, { storagePrefix: prefix, enableCompaction: false }) - await t1.init() - await t1.add('a', 'b'); await t1.flush() - await t1.add('c', 'd'); await t1.flush() - const expectedSize = t1.size() - expect(expectedSize).toBeGreaterThanOrEqual(2) - await t1.close() - - // Reopen with no injected failures — a normal, fully-successful cold load. - const t2 = new LSMTree(storage, { storagePrefix: prefix, enableCompaction: false }) - await t2.init() - - expect(t2.size()).toBe(expectedSize) - expect(t2.isHealthy()).toBe(true) - - await t2.close() - }) -}) diff --git a/tests/unit/hnsw/flush-failure.test.ts b/tests/unit/hnsw/flush-failure.test.ts deleted file mode 100644 index 14bdff6e..00000000 --- a/tests/unit/hnsw/flush-failure.test.ts +++ /dev/null @@ -1,157 +0,0 @@ -/** - * HNSW deferred-flush durability tests (Finding 6 — spine-plan Part B, - * "blind catch" audit). - * - * `JsHnswVectorIndex.flush()` used to swallow a per-node - * `persistNodeConnections` failure (and a system-record `saveHNSWSystem` - * failure) behind `console.error`, then unconditionally clear the dirty set - * and report the pre-flush node count as "success". A transient write fault - * therefore silently dropped that node's connections from durable storage - * forever, with `flush()` having lied about it. These tests pin the fix: - * a node (or the system record) that fails to persist stays in the - * dirty/retry set, and `flush()` throws {@link HnswFlushError} instead of - * returning a count. The immediate-mode first-noun `saveHNSWSystem` swallow - * (while `addItem()` still returned the id) is covered too. - */ - -import { describe, it, expect, vi } from 'vitest' -import { v4 as uuidv4 } from 'uuid' -import { JsHnswVectorIndex, HnswFlushError } from '../../../src/hnsw/hnswIndex.js' -import { euclideanDistance } from '../../../src/utils/index.js' -import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' - -// Helper: generate a random vector of given dimension (mirrors lazy-vectors.test.ts). -function randomVector(dim: number): number[] { - return Array.from({ length: dim }, () => Math.random() * 2 - 1) -} - -describe('HNSW deferred-flush durability (Finding 6)', () => { - const dim = 8 - - it('a clean flush persists every dirty node and clears the dirty set', async () => { - const storage = new MemoryStorage() - const index = new JsHnswVectorIndex( - { M: 4, efConstruction: 50, efSearch: 20 }, - euclideanDistance, - { useParallelization: false, storage, persistMode: 'deferred' } - ) - - for (let i = 0; i < 5; i++) { - await index.addItem({ id: uuidv4(), vector: randomVector(dim) }) - } - - // Sanity: inserting several items in deferred mode does dirty something. - expect((index as any).dirtyNodes.size).toBeGreaterThan(0) - - const flushed = await index.flush() - expect(typeof flushed).toBe('number') - expect((index as any).dirtyNodes.size).toBe(0) - expect((index as any).dirtySystem).toBe(false) - }) - - it('retains a node whose connections failed to persist and surfaces HnswFlushError instead of reporting success', async () => { - const storage = new MemoryStorage() - const index = new JsHnswVectorIndex( - { M: 4, efConstruction: 50, efSearch: 20 }, - euclideanDistance, - { useParallelization: false, storage, persistMode: 'deferred' } - ) - - const ids: string[] = [] - for (let i = 0; i < 5; i++) { - const id = uuidv4() - ids.push(id) - await index.addItem({ id, vector: randomVector(dim) }) - } - - // The very first node is guaranteed to become a neighbor of the second - // insert (it is the only existing node in the graph at that point), so it - // is deterministically present in the dirty set before any fault. - const failId = ids[0] - const dirtyBefore = (index as any).dirtyNodes as Set - expect(dirtyBefore.has(failId)).toBe(true) - - const originalSave = storage.saveVectorIndexData.bind(storage) - const spy = vi - .spyOn(storage, 'saveVectorIndexData') - .mockImplementation(async (nounId, hnswData) => { - if (nounId === failId) { - throw Object.assign(new Error('simulated write fault'), { code: 'EIO' }) - } - return originalSave(nounId, hnswData) - }) - - await expect(index.flush()).rejects.toBeInstanceOf(HnswFlushError) - - const dirtyAfter = (index as any).dirtyNodes as Set - // The failed node stays dirty for the next retry ... - expect(dirtyAfter.has(failId)).toBe(true) - // ... and every node that DID persist successfully leaves the dirty set — - // the failure of one node must not re-dirty (or fail to clear) the rest. - expect(dirtyAfter.size).toBe(1) - - // Recovery: once the fault clears, the retained node persists and the - // flush reports success again (the intended retry path). - spy.mockRestore() - const flushed = await index.flush() - expect(typeof flushed).toBe('number') - expect((index as any).dirtyNodes.size).toBe(0) - }) - - it('surfaces a system-record persist failure as HnswFlushError and keeps dirtySystem set for retry', async () => { - const storage = new MemoryStorage() - const index = new JsHnswVectorIndex( - { M: 4, efConstruction: 50, efSearch: 20 }, - euclideanDistance, - { useParallelization: false, storage, persistMode: 'deferred' } - ) - - // First noun in deferred mode marks dirtySystem (entryPoint/maxLevel). - await index.addItem({ id: uuidv4(), vector: randomVector(dim) }) - expect((index as any).dirtySystem).toBe(true) - - const spy = vi - .spyOn(storage, 'saveHNSWSystem') - .mockRejectedValue( - Object.assign(new Error('simulated system write fault'), { code: 'EIO' }) - ) - - let caught: unknown - try { - await index.flush() - } catch (error) { - caught = error - } - - expect(caught).toBeInstanceOf(HnswFlushError) - expect((caught as HnswFlushError).systemFailed).toBe(true) - // The system record must stay dirty — a lost entry point/maxLevel update - // must be retried, not silently dropped. - expect((index as any).dirtySystem).toBe(true) - - spy.mockRestore() - const flushed = await index.flush() - expect(typeof flushed).toBe('number') - expect((index as any).dirtySystem).toBe(false) - }) - - it('surfaces the immediate-mode first-noun system persist failure via a rejecting addItem()', async () => { - const storage = new MemoryStorage() - const index = new JsHnswVectorIndex( - { M: 4, efConstruction: 50, efSearch: 20 }, - euclideanDistance, - { useParallelization: false, storage } // default persistMode: 'immediate' - ) - - vi.spyOn(storage, 'saveHNSWSystem').mockRejectedValue( - Object.assign(new Error('simulated system write fault'), { code: 'EIO' }) - ) - - // Previously this swallowed the error (console.error) and addItem() - // still resolved with the id — stranding a brand-new index whose root - // (entry point) was never actually persisted. Now it must reject. - await expect( - index.addItem({ id: uuidv4(), vector: randomVector(dim) }) - ).rejects.toThrow('simulated system write fault') - }) -}) diff --git a/tests/unit/indexes/columnStore/segment-load-fault.test.ts b/tests/unit/indexes/columnStore/segment-load-fault.test.ts deleted file mode 100644 index deb0868f..00000000 --- a/tests/unit/indexes/columnStore/segment-load-fault.test.ts +++ /dev/null @@ -1,128 +0,0 @@ -/** - * @module tests/unit/indexes/columnStore/segment-load-fault - * @description Pattern-B acceptance for the ColumnStore (finding 4): a segment - * the manifest LISTS but that cannot be loaded must never be silently skipped — - * doing so dropped every entity in that segment out of `filter`/`rangeQuery`/ - * `sortTopK` with no error, so a corrupt index looked like a merely short result. - * - * The three failure classes and their required behaviour: - * - a real storage IO fault (EIO) PROPAGATES verbatim — a present-but-unreadable - * segment is not "absent", so it must not read as an empty result; - * - a manifest-listed segment with undecodable bytes throws `ColumnSegmentLoadError`; - * - a manifest-listed segment with NO bytes (gone on disk) throws `ColumnSegmentLoadError`. - * Only genuine absence stays benign: querying a field that has no manifest at all - * returns empty (nothing was ever written for it) — that is not a fault. - */ -import { describe, it, expect, beforeEach } from 'vitest' -import { - ColumnStore, - ColumnSegmentLoadError -} from '../../../../src/indexes/columnStore/ColumnStore.js' -import { MemoryStorage } from '../../../../src/storage/adapters/memoryStorage.js' -import { EntityIdMapper } from '../../../../src/utils/entityIdMapper.js' - -type FaultMode = 'none' | 'io' | 'corrupt' | 'missing' - -/** - * A MemoryStorage that can fault reads of persisted column SEGMENTS only - * (keys/paths containing the `L0-` segment marker). Manifest and DELETED-bitmap - * reads pass through untouched so a fresh store still initialises normally — the - * fault is isolated to the exact seam finding 4 hardened. - */ -class FaultInjectingStorage extends MemoryStorage { - public faultMode: FaultMode = 'none' - - private eio(): Error { - const e = new Error('simulated disk read fault') as Error & { code: string } - e.code = 'EIO' - return e - } - - public async loadBinaryBlob(key: string): Promise { - if (this.faultMode !== 'none' && key.includes('/L0-')) { - if (this.faultMode === 'io') throw this.eio() - // Too small to hold even a header → readSegmentFromBuffer throws → wrapped. - if (this.faultMode === 'corrupt') return Buffer.from([1, 2, 3, 4, 5]) - if (this.faultMode === 'missing') return null - } - return super.loadBinaryBlob(key) - } -} - -describe('ColumnStore segment-load faults surface loudly, absence stays benign (finding 4)', () => { - let storage: FaultInjectingStorage - let idMapper: EntityIdMapper - - beforeEach(async () => { - storage = new FaultInjectingStorage() - await storage.init() - idMapper = new EntityIdMapper({ storage, storageKey: 'test:idMapper' }) - await idMapper.init() - - // Write one persisted L0 segment for `createdAt`, then close the writer. - const writer = new ColumnStore({ flushThreshold: 10 }) - await writer.init(storage, idMapper) - for (let i = 0; i < 5; i++) { - writer.addEntity(BigInt(idMapper.getOrAssign(`e${i}`)), { - createdAt: (i + 1) * 100 - }) - } - await writer.flush() - await writer.close() - storage.faultMode = 'none' - }) - - // Fresh reader over the same storage: empty segment cache, so every query is - // forced to actually load the persisted segment (that is the seam under test). - const reopen = async (): Promise => { - const s = new ColumnStore({ flushThreshold: 10 }) - await s.init(storage, idMapper) - return s - } - - it('propagates a storage IO fault verbatim — not [] and not a ColumnSegmentLoadError', async () => { - storage.faultMode = 'io' - const store = await reopen() - await expect(store.filter('createdAt', 300)).rejects.toMatchObject({ - code: 'EIO' - }) - await store.close() - }) - - it('throws ColumnSegmentLoadError when a manifest-listed segment is undecodable', async () => { - storage.faultMode = 'corrupt' - const store = await reopen() - await expect( - store.sortTopK('createdAt', 'desc', 10) - ).rejects.toBeInstanceOf(ColumnSegmentLoadError) - await store.close() - }) - - it('throws ColumnSegmentLoadError when a manifest-listed segment has no loadable bytes', async () => { - storage.faultMode = 'missing' - const store = await reopen() - await expect( - store.rangeQuery('createdAt', 100, 500) - ).rejects.toBeInstanceOf(ColumnSegmentLoadError) - await store.close() - }) - - it('a field with no manifest is genuine absence — returns empty, never throws', async () => { - storage.faultMode = 'none' - const store = await reopen() - const bitmap = await store.filter('no_such_field', 'x') - expect(bitmap.size).toBe(0) - const sorted = await store.sortTopK('no_such_field', 'asc', 10) - expect(sorted).toEqual([]) - await store.close() - }) - - it('with no fault, the persisted segment still loads and answers queries', async () => { - storage.faultMode = 'none' - const store = await reopen() - const sorted = await store.sortTopK('createdAt', 'desc', 10) - const uuids = sorted.map((id) => idMapper.getUuid(Number(id))) - expect(uuids).toEqual(['e4', 'e3', 'e2', 'e1', 'e0']) - await store.close() - }) -}) diff --git a/tests/unit/migration-lock.test.ts b/tests/unit/migration-lock.test.ts index f0fbbe4c..81d44a9f 100644 --- a/tests/unit/migration-lock.test.ts +++ b/tests/unit/migration-lock.test.ts @@ -2,16 +2,11 @@ * Migration LOCK (#18) — coordinated, automatic 7.x → 8.0 auto-upgrade. * * Exercises the real block-and-queue behavior of `awaitMigrationLock` at the - * `ensureInitialized` choke point. The gate is FAMILY-SCOPED: a write (which - * touches every index) WAITS while ANY provider reports `isMigrating() === true`, - * and a read WAITS only when the migrating provider is one of the index families - * that read consults (so a read served from canonical storage or a healthy family - * is never blocked by an unrelated family's migration — see - * `tests/unit/brainy/migration-gate-family-scoped.test.ts`). Observability - * (`getIndexStatus`, `health`) and the lock-clearing `stampBrainFormat` stay - * exempt; a lock that outlives the configured window surfaces a retryable - * `MigrationInProgressError`. Here the graph provider holds the lock, so the - * read cases use a graph traversal (`related`) — the family that actually waits. + * `ensureInitialized` choke point: while ANY index provider reports + * `isMigrating() === true`, data-plane reads and writes WAIT (no operation + * touches a half-built index); observability (`getIndexStatus`, `health`) and + * the lock-clearing `stampBrainFormat` stay exempt; and a lock that outlives the + * configured window surfaces a retryable `MigrationInProgressError`. * * A native (cortex) provider owns the real migration; here we simulate the lock * by feature-detect-injecting `isMigrating()` onto a live provider, exactly as @@ -89,12 +84,11 @@ describe('Migration LOCK (#18) — coordinated 7.x→8.0 auto-upgrade', () => { expect(id).toBeTruthy() }) - it('blocks a graph read while the graph provider migrates, then releases when the flag clears (poll path)', async () => { - const anchor = await brain.add({ data: 'anchor', type: NounType.Concept }) + it('blocks a read while migrating, then releases when the flag clears (poll path)', async () => { let migrating = true brain.graphIndex.isMigrating = () => migrating - const p = brain.related({ from: anchor }) // a GRAPH read → gated on the graph migration + const p = brain.find({ type: NounType.Concept }) // a read → gated let resolved = false p.then(() => { resolved = true @@ -115,12 +109,9 @@ describe('Migration LOCK (#18) — coordinated 7.x→8.0 auto-upgrade', () => { migrationWaitTimeoutMs: 120 }) await shortBrain.init() - const anchor = await shortBrain.add({ data: 'anchor', type: NounType.Concept }) - setMigrating(shortBrain, true) // graph lock never clears + setMigrating(shortBrain, true) // never clears - // A graph read needs the migrating family → it waits out the window and - // surfaces the retryable error rather than serving a half-built traversal. - await expect(shortBrain.related({ from: anchor })).rejects.toBeInstanceOf( + await expect(shortBrain.find({ type: NounType.Concept })).rejects.toBeInstanceOf( MigrationInProgressError ) try { diff --git a/tests/unit/storage/blob-save-durability.test.ts b/tests/unit/storage/blob-save-durability.test.ts deleted file mode 100644 index f775ebdf..00000000 --- a/tests/unit/storage/blob-save-durability.test.ts +++ /dev/null @@ -1,111 +0,0 @@ -/** - * @module tests/unit/storage/blob-save-durability - * @description Finding 13 (cortex sweep): saveBinaryBlob must never acknowledge a - * durable write that stored nothing. With the UNIQUE per-writer temp suffix, a - * rename ENOENT can no longer mean "a concurrent idempotent writer already - * renamed it" — nobody else has this writer's temp — so it means our temp - * vanished before the rename and the bytes did NOT land. The old code returned - * success on that ENOENT; the native provider mmaps these blobs, so a - * phantom-acked blob is the exact silent-loss class the registered-blob work - * chases. The fix retries once with a fresh temp, then fails loud. - */ -import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js' - -const enoent = (): Error => Object.assign(new Error('temp removed'), { code: 'ENOENT' }) - -describe('FileSystemStorage.saveBinaryBlob durable-write honesty (finding 13)', () => { - let dir: string - let storage: any - - beforeEach(async () => { - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-blob-save-')) - storage = new FileSystemStorage(dir) - await storage.init() - }) - - afterEach(() => { - vi.restoreAllMocks() - fs.rmSync(dir, { recursive: true, force: true }) - }) - - it('persists and reads back a blob under normal conditions', async () => { - await storage.saveBinaryBlob('graph-lsm/seg-1', Buffer.from([1, 2, 3, 4])) - expect(await storage.loadBinaryBlob('graph-lsm/seg-1')).toEqual(Buffer.from([1, 2, 3, 4])) - }) - - it('throws (never acks) when the temp vanishes on every rename attempt', async () => { - vi.spyOn(fs.promises, 'rename').mockRejectedValue(enoent()) - - await expect( - storage.saveBinaryBlob('k/silent-loss', Buffer.from([9])) - ).rejects.toThrow(/did NOT persist/) - - // Prove the write truly did not land — and was not acknowledged. (loadBinaryBlob - // uses readFile, not rename, so it reflects the real on-disk state.) - expect(await storage.loadBinaryBlob('k/silent-loss')).toBeNull() - }) - - it('self-heals: a single transient temp-vanish retries and persists', async () => { - const real = fs.promises.rename.bind(fs.promises) - let calls = 0 - vi.spyOn(fs.promises, 'rename').mockImplementation(async (from: any, to: any) => { - calls++ - if (calls === 1) throw enoent() - return real(from, to) - }) - - await expect( - storage.saveBinaryBlob('k/heals', Buffer.from([7, 7])) - ).resolves.toBeUndefined() - expect(calls).toBe(2) // first attempt vanished, retry landed - expect(await storage.loadBinaryBlob('k/heals')).toEqual(Buffer.from([7, 7])) - }) - - it('a non-ENOENT rename fault propagates verbatim (not swallowed)', async () => { - vi.spyOn(fs.promises, 'rename').mockRejectedValue( - Object.assign(new Error('disk fault'), { code: 'EIO' }) - ) - await expect( - storage.saveBinaryBlob('k/eio', Buffer.from([1])) - ).rejects.toMatchObject({ code: 'EIO' }) - }) -}) - -describe('FileSystemStorage.loadBinaryBlob fault-propagation (finding 11; cor 3.0.13 lockstep)', () => { - let dir: string - let storage: any - - beforeEach(async () => { - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-blob-load-')) - storage = new FileSystemStorage(dir) - await storage.init() - }) - - afterEach(() => { - vi.restoreAllMocks() - fs.rmSync(dir, { recursive: true, force: true }) - }) - - it('returns null for a genuinely absent blob (ENOENT)', async () => { - expect(await storage.loadBinaryBlob('never/written')).toBeNull() - }) - - it('reads back a present blob', async () => { - await storage.saveBinaryBlob('present/blob', Buffer.from([1, 2, 3])) - expect(await storage.loadBinaryBlob('present/blob')).toEqual(Buffer.from([1, 2, 3])) - }) - - it('propagates a real IO fault instead of masking it as absent', async () => { - await storage.saveBinaryBlob('present/blob', Buffer.from([1, 2, 3])) - vi.spyOn(fs.promises, 'readFile').mockRejectedValue( - Object.assign(new Error('disk fault'), { code: 'EIO' }) - ) - // A present-but-unreadable blob must NOT read as null (that drove needless - // native rebuilds / empty reads); cor 3.0.13's column-store handles the throw. - await expect(storage.loadBinaryBlob('present/blob')).rejects.toMatchObject({ code: 'EIO' }) - }) -}) diff --git a/tests/unit/storage/blobStorage.test.ts b/tests/unit/storage/blobStorage.test.ts index 55bc5276..8138c64f 100644 --- a/tests/unit/storage/blobStorage.test.ts +++ b/tests/unit/storage/blobStorage.test.ts @@ -258,31 +258,21 @@ describe('BlobStorage', () => { expect(metadata?.refCount).toBe(2) }) - it('release() drops live references; bytes are reclaimed ONLY by reclaimIfUnreferenced at zero-zero', async () => { + it('should only delete when refCount reaches 0', async () => { const data = Buffer.from('test') - // Write twice (live refCount = 2) + // Write twice (refCount = 2) const hash = await blobStorage.write(data) await blobStorage.write(data) - // Release once (refCount = 1, blob still exists) - await blobStorage.release(hash) + // Delete once (refCount = 1, blob still exists) + await blobStorage.delete(hash) + expect(await blobStorage.has(hash)).toBe(true) - // Release again (refCount = 0) — bytes STILL exist: content is - // immutable under the temporal model; only history compaction reclaims. - await blobStorage.release(hash) - expect(await blobStorage.has(hash)).toBe(true) - expect((await blobStorage.getMetadata(hash))?.refCount).toBe(0) + // Delete again (refCount = 0, blob deleted) + await blobStorage.delete(hash) - // A history reference blocks reclamation even at live-zero. - await blobStorage.recordHistoryReference(hash) - expect(await blobStorage.reclaimIfUnreferenced(hash)).toBe(false) - expect(await blobStorage.has(hash)).toBe(true) - - // Drop the history reference → zero-zero → reclaim succeeds. - await blobStorage.releaseHistoryReference(hash) - expect(await blobStorage.reclaimIfUnreferenced(hash)).toBe(true) expect(await blobStorage.has(hash)).toBe(false) }) }) diff --git a/tests/unit/storage/clear-native-footprint.test.ts b/tests/unit/storage/clear-native-footprint.test.ts deleted file mode 100644 index 313ed9b8..00000000 --- a/tests/unit/storage/clear-native-footprint.test.ts +++ /dev/null @@ -1,74 +0,0 @@ -/** - * @module tests/unit/storage/clear-native-footprint - * @description clear() must remove the COMPLETE derived/native on-disk footprint - * (finding 7). Before the fix it wiped only entities/indexes/system/_cas and left - * three top-level trees behind: - * - `_blobs` raw binary blobs (HNSW/LSM segments, the native dkann index, - * and column-store segment bytes under `_blobs/_column_index/`) - * - `_id_mapper` the native shared mmap id-mapper - * - `_column_index` the column-store MANIFEST.json files - * A cleared brain therefore re-read stale native state, and — worst — an orphaned - * column manifest whose segment bytes had been removed with `_blobs`, which the - * hardened load path now refuses with ColumnSegmentLoadError. The three must fall - * together as a set. - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js' - -const exists = (p: string): boolean => fs.existsSync(p) - -describe('FileSystemStorage.clear() wipes the full native/derived footprint (finding 7)', () => { - let dir: string - let storage: any - - beforeEach(async () => { - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-clear-footprint-')) - storage = new FileSystemStorage(dir) - await storage.init() - }) - - afterEach(() => { - fs.rmSync(dir, { recursive: true, force: true }) - }) - - it('removes _blobs, _id_mapper and _column_index (previously left stranded)', async () => { - // A raw blob (this is how HNSW/LSM/column segments persist) → creates _blobs. - await storage.saveBinaryBlob('graph-lsm/source/sstable-1', Buffer.from([1, 2, 3, 4])) - // A column segment blob lives UNDER _blobs/_column_index/... - await storage.saveBinaryBlob('_column_index/createdAt/L0-000000', Buffer.from([5, 6, 7, 8])) - - // Simulate the native id-mapper mmap dir and the column manifest object-tree. - const idMapperDir = path.join(dir, '_id_mapper') - fs.mkdirSync(idMapperDir, { recursive: true }) - fs.writeFileSync(path.join(idMapperDir, 'main.slotmap'), Buffer.from([0, 1])) - const colDir = path.join(dir, '_column_index', 'createdAt') - fs.mkdirSync(colDir, { recursive: true }) - fs.writeFileSync( - path.join(colDir, 'MANIFEST.json'), - JSON.stringify({ segments: [{ id: 0, level: 0, count: 1 }] }) - ) - - // Sanity: everything is present before the clear. - expect(exists(path.join(dir, '_blobs'))).toBe(true) - expect(exists(idMapperDir)).toBe(true) - expect(exists(path.join(dir, '_column_index'))).toBe(true) - - await storage.clear() - - // The complete derived footprint is gone — no stale native state, and no - // orphaned column manifest pointing at removed segment bytes. - expect(exists(path.join(dir, '_blobs'))).toBe(false) - expect(exists(idMapperDir)).toBe(false) - expect(exists(path.join(dir, '_column_index'))).toBe(false) - }) - - it('clear() is a no-op-safe when the native dirs never existed', async () => { - // Fresh store, no blobs written — clear must not throw on absent dirs. - await expect(storage.clear()).resolves.toBeUndefined() - expect(exists(path.join(dir, '_blobs'))).toBe(false) - expect(exists(path.join(dir, '_column_index'))).toBe(false) - }) -}) diff --git a/tests/unit/storage/pagination-parallel-hydration.test.ts b/tests/unit/storage/pagination-parallel-hydration.test.ts deleted file mode 100644 index ada324bb..00000000 --- a/tests/unit/storage/pagination-parallel-hydration.test.ts +++ /dev/null @@ -1,94 +0,0 @@ -/** - * @module tests/unit/storage/pagination-parallel-hydration - * @description The canonical enumeration walk (getNounsWithPagination) hydrated - * each item's vector + metadata ONE-AT-A-TIME — N×per-op-latency serially, the - * dominant term in an index heal (cortex heal-cost decomposition). It now - * hydrates 16-way, and a new getNounIdsWithPagination returns ids WITHOUT - * hydration (zero per-entity reads when unfiltered). Both must preserve the exact - * pagination contract: same order, cursor continuation, filters, totalCount. - */ -import { describe, it, expect, beforeEach, vi } from 'vitest' -import { Brainy, NounType } from '../../../src/index.js' - -describe('paginated enumeration — parallel hydration + id-only (cortex heal-cost)', () => { - let brain: any - let storage: any - const N = 30 - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, dimensions: 384 }) - await brain.init() - for (let i = 0; i < N; i++) { - await brain.add({ - data: `n${i}`, - type: i % 3 === 0 ? NounType.Task : NounType.Concept, - metadata: { i } - }) - } - await brain.flush() - storage = brain.storage - }) - - /** Page the whole dataset through a small limit via cursor and collect ordered ids. */ - const pageAll = async (fn: (opts: any) => Promise, key: 'items' | 'ids') => { - const out: string[] = [] - let cursor: string | undefined - for (let guard = 0; guard < 1000; guard++) { - const page = await fn({ limit: 4, cursor }) - const batch = key === 'items' ? page.items.map((n: any) => n.id) : page.ids - out.push(...batch) - if (!page.hasMore) break - cursor = page.nextCursor - } - return out - } - - it('parallel hydration yields the SAME ordered pages as one big page', async () => { - const big = await storage.getNounsWithPagination({ limit: 1000, offset: 0 }) - const bigIds = big.items.map((n: any) => n.id) - // At least the N we added (a brain also has its VFS root entity). - expect(bigIds.length).toBeGreaterThanOrEqual(N) - expect(big.totalCount).toBe(bigIds.length) - - const paged = await pageAll((o) => storage.getNounsWithPagination(o), 'items') - expect(paged).toEqual(bigIds) // identical order, no dupes, no gaps across pages - }) - - it('getNounIdsWithPagination returns exactly the same ids, in the same order', async () => { - const idsPaged = await pageAll((o) => storage.getNounIdsWithPagination(o), 'ids') - const itemsPaged = await pageAll((o) => storage.getNounsWithPagination(o), 'items') - expect(idsPaged).toEqual(itemsPaged) - expect(new Set(idsPaged).size).toBe(idsPaged.length) // every id exactly once - expect(idsPaged.length).toBeGreaterThanOrEqual(N) - }) - - it('id-only enumeration does ZERO per-entity hydration reads when unfiltered', async () => { - const readSpy = vi.spyOn(storage as any, 'readCanonicalObject') - await storage.getNounIdsWithPagination({ limit: 1000, offset: 0 }) - expect(readSpy).not.toHaveBeenCalled() - readSpy.mockRestore() - - // The hydrating walk, by contrast, DOES read each entity. - const readSpy2 = vi.spyOn(storage as any, 'readCanonicalObject') - await storage.getNounsWithPagination({ limit: 1000, offset: 0 }) - expect(readSpy2.mock.calls.length).toBeGreaterThan(0) - readSpy2.mockRestore() - }) - - it('a type filter matches between the hydrating and id-only walks', async () => { - const taskItems = await storage.getNounsWithPagination({ - limit: 1000, - offset: 0, - filter: { nounType: NounType.Task } - }) - const taskIds = await storage.getNounIdsWithPagination({ - limit: 1000, - offset: 0, - filter: { nounType: NounType.Task } - }) - const expected = Math.ceil(N / 3) // every 3rd is a Task - expect(taskItems.items.length).toBe(expected) - expect(new Set(taskIds.ids)).toEqual(new Set(taskItems.items.map((n: any) => n.id))) - }) -}) diff --git a/tests/unit/storage/registered-blob-contract.test.ts b/tests/unit/storage/registered-blob-contract.test.ts deleted file mode 100644 index 6ad51ae7..00000000 --- a/tests/unit/storage/registered-blob-contract.test.ts +++ /dev/null @@ -1,130 +0,0 @@ -/** - * @module tests/unit/storage/registered-blob-contract - * @description Declared derived-index blob FAMILIES are - * undeletable through the storage layer — an in-process GC/sweeper cannot remove - * a load-bearing index file (the lost-main.dkann class). Covers declare → - * protected-delete-throws; unregister → delete-ok; transient passthrough; - * namespace prefix protects children; removeRawPrefix refuses a protected - * intersection; persistence across reopen; missing-on-open detection. - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' -import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js' -import { ProtectedArtifactError } from '../../../src/index.js' - -describe('registered-blob family contract', () => { - let storage: any - - beforeEach(async () => { - storage = new MemoryStorage() - await storage.init() - }) - - const seedVectorFamily = async () => { - await storage.saveBinaryBlob('_system/vector-index/main.dkann', Buffer.from([1])) - await storage.saveBinaryBlob('_system/vector-index/main.slotmap', Buffer.from([2])) - await storage.saveBinaryBlob('_system/vector-index/main.slotrev', Buffer.from([3])) - await storage.registerDerivedFamily({ - name: 'vector-base', - members: [ - '_system/vector-index/main.dkann', - '_system/vector-index/main.slotmap', - '_system/vector-index/main.slotrev' - ] - }) - } - - it('a declared member is undeletable (throws ProtectedArtifactError)', async () => { - await seedVectorFamily() - await expect(storage.deleteBinaryBlob('_system/vector-index/main.dkann')).rejects.toBeInstanceOf( - ProtectedArtifactError - ) - // The blob is still there. - expect(await storage.loadBinaryBlob('_system/vector-index/main.dkann')).not.toBeNull() - }) - - it('unregistering the family makes its members deletable again', async () => { - await seedVectorFamily() - await storage.unregisterDerivedFamily('vector-base') - await expect(storage.deleteBinaryBlob('_system/vector-index/main.dkann')).resolves.toBeUndefined() - expect(await storage.loadBinaryBlob('_system/vector-index/main.dkann')).toBeNull() - }) - - it('an undeclared blob is still deletable (contract does not lock everything)', async () => { - await seedVectorFamily() - await storage.saveBinaryBlob('graph-lsm/source/sstable-1', Buffer.from([9])) - await expect(storage.deleteBinaryBlob('graph-lsm/source/sstable-1')).resolves.toBeUndefined() - }) - - it('a transient (*.tmp.*) is deletable even if it sits under a protected namespace', async () => { - await storage.registerDerivedFamily({ - name: 'vector-ns', - members: ['_system/vector-index/'], - namespace: true - }) - await storage.saveBinaryBlob('_system/vector-index/main.dkann.tmp.123', Buffer.from([1])) - await expect( - storage.deleteBinaryBlob('_system/vector-index/main.dkann.tmp.123') - ).resolves.toBeUndefined() - }) - - it('a namespace family protects a growing child key', async () => { - await storage.registerDerivedFamily({ - name: 'vector-segments', - members: ['_system/vector-index/'], - namespace: true - }) - await storage.saveBinaryBlob('_system/vector-index/seg-42', Buffer.from([7])) - await expect(storage.deleteBinaryBlob('_system/vector-index/seg-42')).rejects.toBeInstanceOf( - ProtectedArtifactError - ) - }) - - it('checkDerivedFamiliesPresent() names a member deleted outside the write path', async () => { - await seedVectorFamily() - // Simulate an EXTERNAL deleter (bypasses the in-process refusal): drop a - // member straight from the underlying store. - ;(storage as any).blobStore.delete('_system/vector-index/main.slotmap') - const incomplete = await storage.checkDerivedFamiliesPresent() - expect(incomplete).toHaveLength(1) - expect(incomplete[0].name).toBe('vector-base') - expect(incomplete[0].missing).toContain('_system/vector-index/main.slotmap') - expect(incomplete[0].rebuildable).toBe(true) - }) - - it('no families registered → deleteBinaryBlob behaves exactly as before (no enforcement)', async () => { - await storage.saveBinaryBlob('some/blob', Buffer.from([1])) - await expect(storage.deleteBinaryBlob('some/blob')).resolves.toBeUndefined() - }) -}) - -describe('registered-blob family protection survives a reopen (FileSystemStorage)', () => { - let dir: string - - beforeEach(() => { - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-regblob-')) - }) - afterEach(() => fs.rmSync(dir, { recursive: true, force: true })) - - it('a family declared in one session protects members after reopen', async () => { - const s1: any = new FileSystemStorage(dir) - await s1.init() - await s1.saveBinaryBlob('_system/vector-index/main.dkann', Buffer.from([1])) - await s1.registerDerivedFamily({ name: 'vector-base', members: ['_system/vector-index/main.dkann'] }) - - // Reopen — the registry is loaded from _system/derived-artifacts.json. - const s2: any = new FileSystemStorage(dir) - await s2.init() - expect(await s2.listDerivedFamilies()).toHaveLength(1) - await expect(s2.deleteBinaryBlob('_system/vector-index/main.dkann')).rejects.toBeInstanceOf( - ProtectedArtifactError - ) - // removeRawPrefix nuking the vector dir is also refused. - await expect(s2.removeRawPrefix('_blobs/_system/vector-index')).rejects.toBeInstanceOf( - ProtectedArtifactError - ) - }) -}) diff --git a/tests/unit/storage/sharding-new-installation-log.test.ts b/tests/unit/storage/sharding-new-installation-log.test.ts index c1af8427..ca674137 100644 --- a/tests/unit/storage/sharding-new-installation-log.test.ts +++ b/tests/unit/storage/sharding-new-installation-log.test.ts @@ -87,29 +87,6 @@ describe('FileSystemStorage boot log: an established brain is not "New installat await teardown(second) }) - it('counts RECOVER from the canonical layout when counts.json is lost (container-restart case)', async () => { - const dir = mkTmp() - - // Establish a store with 3 entities (canonical layout + counts.json). - const first: any = new FileSystemStorage(dir) - await first.init() - for (const c of ['a', 'b', 'c']) await seedOne(first, c.repeat(32)) - await teardown(first) - - // Simulate the lost/corrupted counts.json a container restart can leave. - for (const f of ['counts.json', 'counts.json.gz']) { - rmSync(join(dir, '_system', f), { force: true }) - } - - // Reopen: the recovery scan must count the CANONICAL tree. The removed - // implementation scanned the vestigial `entities/*/hnsw` dirs and - // re-initialized every counter to ZERO on real data. - const second: any = new FileSystemStorage(dir) - await second.init() - expect(await second.getNounCount()).toBe(3) - await teardown(second) - }) - it('hasCanonicalEntities() sees the canonical shard tree — independent of counts.json', async () => { const dir = mkTmp() const s: any = new FileSystemStorage(dir) diff --git a/tests/unit/storage/verb-cursor-pagination.test.ts b/tests/unit/storage/verb-cursor-pagination.test.ts index cff41d02..985a650f 100644 --- a/tests/unit/storage/verb-cursor-pagination.test.ts +++ b/tests/unit/storage/verb-cursor-pagination.test.ts @@ -95,15 +95,9 @@ describe('verb cursor pagination (graph-perf #2)', () => { expect(new Set(cursorSeen)).toEqual(new Set(offsetSeen)) }) - it('a foreign/malformed cursor FAILS LOUDLY — never a silent restart from page 1', async () => { - // The old behavior (decode-null → silent offset-0 fallback) re-served page 1 - // forever to any while(hasMore) walker: an unbounded CPU loop with no log - // line. An undecodable resume token now refuses the walk instead. - await expect( - storage.getVerbs({ pagination: { limit: 5, cursor: 'not-a-cv1-token' } }) - ).rejects.toThrow('invalid pagination cursor') - await expect( - storage.getNouns({ pagination: { limit: 5, cursor: 'not-a-cv1-token' } }) - ).rejects.toThrow('invalid pagination cursor') + it('a foreign/malformed cursor falls back gracefully (no throw, starts from the beginning)', async () => { + const page = await storage.getVerbs({ pagination: { limit: 5, cursor: 'not-a-cv1-token' } }) + expect(page.items.length).toBe(5) + expect(page.hasMore).toBe(true) }) }) diff --git a/tests/unit/transaction/graphIndexOperations-generation.test.ts b/tests/unit/transaction/graphIndexOperations-generation.test.ts index 9dc5c90c..46e7dde2 100644 --- a/tests/unit/transaction/graphIndexOperations-generation.test.ts +++ b/tests/unit/transaction/graphIndexOperations-generation.test.ts @@ -55,7 +55,7 @@ describe('Graph index operations — generation threading', () => { it('AddToGraphIndexOperation stamps addVerb (and its rollback removeVerb) at the resolved generation', async () => { const { provider, calls } = makeSpyProvider() const verb = makeVerb('verb-1') - const op = new AddToGraphIndexOperation(provider, verb, { sourceInt: 10n, targetInt: 20n }, () => 7n) + const op = new AddToGraphIndexOperation(provider, verb, 10n, 20n, () => 7n) const rollback = await op.execute() expect(calls).toEqual([{ method: 'addVerb', generation: 7n, verbId: 'verb-1' }]) @@ -67,7 +67,7 @@ describe('Graph index operations — generation threading', () => { it('RemoveFromGraphIndexOperation stamps removeVerb (and its rollback addVerb) at the resolved generation', async () => { const { provider, calls } = makeSpyProvider() const verb = makeVerb('verb-2') - const op = new RemoveFromGraphIndexOperation(provider, verb, { sourceInt: 10n, targetInt: 20n }, () => 12n) + const op = new RemoveFromGraphIndexOperation(provider, verb, 10n, 20n, () => 12n) const rollback = await op.execute() expect(calls).toEqual([{ method: 'removeVerb', generation: 12n, verbId: 'verb-2' }]) @@ -83,7 +83,7 @@ describe('Graph index operations — generation threading', () => { // at its execute-time value. const { provider, calls } = makeSpyProvider() let current = 1n - const op = new AddToGraphIndexOperation(provider, makeVerb('verb-3'), { sourceInt: 10n, targetInt: 20n }, () => current) + const op = new AddToGraphIndexOperation(provider, makeVerb('verb-3'), 10n, 20n, () => current) current = 42n // store assigns the batch generation after the op is built await op.execute() @@ -97,7 +97,8 @@ describe('Graph index operations — generation threading', () => { const op = new AddToGraphIndexOperation( provider, makeVerb('verb-4'), - { sourceInt: 10n, targetInt: 20n }, + 10n, + 20n, () => 5n, (verbInt) => { seen = verbInt diff --git a/tests/unit/transaction/timeout-rollback.test.ts b/tests/unit/transaction/timeout-rollback.test.ts deleted file mode 100644 index c09ff098..00000000 --- a/tests/unit/transaction/timeout-rollback.test.ts +++ /dev/null @@ -1,177 +0,0 @@ -/** - * @module tests/unit/transaction/timeout-rollback - * @description Regression for a consumer-reported P0 data-integrity bug: - * `Transaction.execute()` checked its time budget at the TOP of the operation - * loop and threw `TransactionTimeoutError` from OUTSIDE the per-operation - * try/catch — so the timeout bypassed rollback entirely (only per-operation - * failures rolled back). On a bulk transact that tripped the 30s budget - * mid-flight, the operations already applied to canonical storage were left in - * place while the generation was never stamped: torn, generation-less state. - * The generation-store commit path's abort cleanup explicitly ASSUMES a throw - * from `execute()` already restored the applied operations byte-identically - * (it only discards the uncommitted staging directory), so the missing - * rollback broke that invariant. - * - * Fix: `execute()` now has a SINGLE rollback point — any error escaping the - * operation loop (operation failure OR mid-flight timeout) rolls back every - * applied operation in reverse order, then surfaces the original error. - * - * These tests model canonical storage as an in-memory map and give each - * operation a real rollback action that restores the map byte-identically — - * exercising the exact fixed code path and asserting the reported requirement: - * a mid-flight timeout leaves storage byte-identical to its pre-transaction - * state. - */ -import { describe, it, expect } from 'vitest' -import { Transaction } from '../../../src/transaction/Transaction.js' -import type { Operation, RollbackAction } from '../../../src/transaction/types.js' -import { - TransactionTimeoutError, - TransactionExecutionError, - TransactionRollbackError -} from '../../../src/transaction/errors.js' - -/** A tiny stand-in for canonical storage. */ -type Store = Map - -const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) - -/** - * An operation that writes `value` at `key` in `store` and returns a rollback - * action restoring the key's PRIOR state byte-identically (delete if it was - * absent, else restore the previous value) — exactly how the real graph/metadata - * operations undo themselves. `delayMs` lets an op consume wall-clock so a small - * transaction budget trips on the NEXT loop iteration; `fail` makes it throw - * after doing its write-and-rollback capture (to exercise the shared path via an - * operation failure rather than a timeout). - */ -function writeOp( - store: Store, - key: string, - value: string, - opts: { delayMs?: number; fail?: boolean; name?: string } = {} -): Operation & { executed: boolean } { - const op = { - name: opts.name ?? `write:${key}`, - executed: false, - async execute(): Promise { - op.executed = true - if (opts.delayMs) await sleep(opts.delayMs) - // Operations are individually atomic: a failing op throws having left NO - // net change (it never gets far enough to register an undo), so only the - // PRIOR operations need rolling back. - if (opts.fail) throw new Error(`operation ${key} failed`) - const had = store.has(key) - const prev = store.get(key) - store.set(key, value) - return async () => { - if (had) store.set(key, prev as string) - else store.delete(key) - } - } - } - return op -} - -describe('Transaction — mid-flight timeout rolls back (P0 data integrity)', () => { - it('a mid-flight timeout leaves storage BYTE-IDENTICAL to its pre-transaction state', async () => { - const store: Store = new Map([['seed', 'unchanged']]) - const snapshot = new Map(store) - - // Budget 1ms; op0 sleeps 60ms so the budget is blown when the loop checks - // it before op1 — a deterministic mid-flight timeout at operation index 1. - const op0 = writeOp(store, 'A', 'applied-then-rolled-back', { delayMs: 60 }) - const op1 = writeOp(store, 'B', 'never-applied') - - const tx = new Transaction({ timeout: 1 }) - tx.addOperation(op0) - tx.addOperation(op1) - - await expect(tx.execute()).rejects.toBeInstanceOf(TransactionTimeoutError) - - // op0 ran (and was rolled back); op1 never started. - expect(op0.executed).toBe(true) - expect(op1.executed).toBe(false) - // The whole point: canonical storage is byte-identical — no torn state. - expect(store).toEqual(snapshot) - expect(store.has('A')).toBe(false) - expect(store.has('B')).toBe(false) - // And the transaction ended in the rolled-back terminal state (so the - // manager counts it correctly), not stranded in 'executing'. - expect(tx.getState()).toBe('rolled_back') - }) - - it('an operation failure mid-flight rolls back byte-identically (the shared single-rollback path)', async () => { - const store: Store = new Map([['seed', 'unchanged']]) - const snapshot = new Map(store) - - const op0 = writeOp(store, 'A', 'applied-then-rolled-back') - const op1 = writeOp(store, 'B', 'partially-applied', { fail: true }) - const op2 = writeOp(store, 'C', 'never-applied') - - const tx = new Transaction() - tx.addOperation(op0) - tx.addOperation(op1) - tx.addOperation(op2) - - await expect(tx.execute()).rejects.toBeInstanceOf(TransactionExecutionError) - - expect(op0.executed).toBe(true) - expect(op1.executed).toBe(true) - expect(op2.executed).toBe(false) - // op1's partial write is undone by its own rollback; op0's write is undone; - // op2 never wrote. Byte-identical. - expect(store).toEqual(snapshot) - expect(tx.getState()).toBe('rolled_back') - }) - - it('a rollback failure during a timeout surfaces TransactionRollbackError wrapping the timeout (loud, not silent)', async () => { - const store: Store = new Map() - - // op0 applies, but its rollback throws every time (maxRollbackRetries - // exhausted) — the manager must surface a TransactionRollbackError whose - // originalError is the timeout, never swallow it. - const op0: Operation & { executed: boolean } = { - name: 'unrollbackable', - executed: false, - async execute() { - op0.executed = true - await sleep(60) - store.set('A', 'applied') - return async () => { - throw new Error('rollback is impossible for this op') - } - } - } - const op1 = writeOp(store, 'B', 'never-applied') - - const tx = new Transaction({ timeout: 1, maxRollbackRetries: 1 }) - tx.addOperation(op0) - tx.addOperation(op1) - - let caught: unknown - await tx.execute().catch((e) => (caught = e)) - - expect(caught).toBeInstanceOf(TransactionRollbackError) - expect((caught as TransactionRollbackError).originalError).toBeInstanceOf( - TransactionTimeoutError - ) - expect(op1.executed).toBe(false) - }) - - it('a clean (non-timeout) transaction still commits and applies all writes', async () => { - const store: Store = new Map() - const op0 = writeOp(store, 'A', 'a') - const op1 = writeOp(store, 'B', 'b') - - const tx = new Transaction() - tx.addOperation(op0) - tx.addOperation(op1) - - await tx.execute() - - expect(tx.getState()).toBe('committed') - expect(store.get('A')).toBe('a') - expect(store.get('B')).toBe('b') - }) -}) diff --git a/tests/unit/utils/osLimits.test.ts b/tests/unit/utils/osLimits.test.ts deleted file mode 100644 index a56d593e..00000000 --- a/tests/unit/utils/osLimits.test.ts +++ /dev/null @@ -1,76 +0,0 @@ -/** - * @module tests/unit/utils/osLimits - * @description OS-limit detection for pool-scale use. Laws: - * (1) the /proc/self/limits parser reads soft/hard NOFILE exactly, including - * 'unlimited'; (2) assessment warns ONLY below the pool floors and NEVER - * on an unreadable (null) limit — no measurement, no claim; (3) the full - * check composes both sources and survives unreadable /proc silently. - */ -import { describe, it, expect } from 'vitest' -import { - parseProcLimits, - assessOsLimits, - checkOsLimits, - NOFILE_POOL_FLOOR, - MAX_MAP_COUNT_POOL_FLOOR -} from '../../../src/utils/osLimits.js' - -const SAMPLE_LIMITS = [ - 'Limit Soft Limit Hard Limit Units', - 'Max cpu time unlimited unlimited seconds', - 'Max open files 1024 1048576 files', - 'Max locked memory 8388608 8388608 bytes' -].join('\n') - -describe('osLimits — detect + warn at pool scale', () => { - it('parses soft/hard NOFILE from /proc/self/limits, including unlimited', () => { - expect(parseProcLimits(SAMPLE_LIMITS)).toEqual({ soft: 1024, hard: 1048576 }) - expect( - parseProcLimits('Max open files unlimited unlimited files') - ).toEqual({ soft: Infinity, hard: Infinity }) - expect(parseProcLimits('no such row here')).toEqual({ soft: null, hard: null }) - }) - - it('warns below the floors, stays quiet at or above them', () => { - const low = assessOsLimits({ nofileSoft: 1024, nofileHard: 1048576, maxMapCount: 65530 }) - expect(low).toHaveLength(2) - expect(low[0]).toContain('RLIMIT_NOFILE soft limit is 1024') - expect(low[0]).toContain(`ulimit -n ${NOFILE_POOL_FLOOR}`) - expect(low[0]).toContain('raise the soft limit only') // hard already allows it - expect(low[1]).toContain('vm.max_map_count is 65530') - expect(low[1]).toContain(`vm.max_map_count=${MAX_MAP_COUNT_POOL_FLOOR}`) - - expect( - assessOsLimits({ - nofileSoft: NOFILE_POOL_FLOOR, - nofileHard: Infinity, - maxMapCount: MAX_MAP_COUNT_POOL_FLOOR - }) - ).toEqual([]) - }) - - it('an unreadable limit makes NO claim — nulls never warn', () => { - expect(assessOsLimits({ nofileSoft: null, nofileHard: null, maxMapCount: null })).toEqual([]) - }) - - it('checkOsLimits composes both sources and survives unreadable /proc silently', async () => { - const report = await checkOsLimits(async (p) => { - if (p === '/proc/self/limits') return SAMPLE_LIMITS - if (p === '/proc/sys/vm/max_map_count') return '65530\n' - throw new Error('unexpected path') - }) - expect(report.nofileSoft).toBe(1024) - expect(report.maxMapCount).toBe(65530) - expect(report.warnings).toHaveLength(2) - - const offLinux = await checkOsLimits(async () => { - throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) - }) - expect(offLinux).toEqual({ - nofileSoft: null, - nofileHard: null, - maxMapCount: null, - warnings: [] - }) - }) -}) diff --git a/tests/unit/utils/paramValidation.test.ts b/tests/unit/utils/paramValidation.test.ts index 4dc83554..e7113506 100644 --- a/tests/unit/utils/paramValidation.test.ts +++ b/tests/unit/utils/paramValidation.test.ts @@ -270,24 +270,27 @@ describe('Zero-Config Parameter Validation', () => { expect(config.availableMemory).toBeGreaterThan(0) }) - it('never mutates the cap from query timing (telemetry only)', () => { - const initialLimit = getValidationConfig().maxLimit - - // Fast queries with large results: no silent growth. + it('should adapt limits based on query performance', () => { + const initialConfig = getValidationConfig() + const initialLimit = initialConfig.maxLimit + + // Simulate fast queries with large results for (let i = 0; i < 10; i++) { recordQueryPerformance(50, initialLimit * 0.9) } - expect(getValidationConfig().maxLimit).toBe(initialLimit) - - // A burst of catastrophically slow queries must not strangle the cap. - // The removed "learning" ratchet shrank it 20% per recorded query down - // to a floor of 1000 — below the documented MIN_AUTO_QUERY_LIMIT — and - // the error message blamed "available free memory" (a production - // incident: every find({ limit: 5000 }) failed on an idle 23GB-free box). - for (let i = 0; i < 50; i++) { - recordQueryPerformance(90_000, 100) + + const updatedConfig = getValidationConfig() + // Limit might increase if performance is good + expect(updatedConfig.maxLimit).toBeGreaterThanOrEqual(initialLimit) + + // Simulate slow queries + for (let i = 0; i < 10; i++) { + recordQueryPerformance(2000, 100) } - expect(getValidationConfig().maxLimit).toBe(initialLimit) + + const finalConfig = getValidationConfig() + // Limit should decrease if performance is poor + expect(finalConfig.maxLimit).toBeLessThanOrEqual(updatedConfig.maxLimit) }) }) }) \ No newline at end of file diff --git a/tests/unit/validate-invariants-delegation.test.ts b/tests/unit/validate-invariants-delegation.test.ts deleted file mode 100644 index 45e12ccd..00000000 --- a/tests/unit/validate-invariants-delegation.test.ts +++ /dev/null @@ -1,99 +0,0 @@ -/** - * @module tests/unit/validate-invariants-delegation - * @description validateIndexConsistency() was blind to native - * providers — it only saw the JS metadata index, so a native manifest↔segments↔count - * divergence read as "healthy". It now feature-detects + aggregates each provider's - * validateInvariants(), and repairIndex() maps a failing invariant with heal:'rebuild' - * to that provider's rebuild(). "healthy-while-broken must be impossible." - */ -import { describe, it, expect, beforeEach } from 'vitest' -import { Brainy, NounType } from '../../src/index.js' -import type { ProviderInvariantReport } from '../../src/index.js' - -const healthyReport = (provider: string): ProviderInvariantReport => ({ - provider, - healthy: true, - serving: true, - invariants: [{ name: 'manifest-residency', holds: true, detail: 'ok', heal: 'none' }], - checkedAt: 1, - durationMs: 1 -}) - -const brokenReport = (provider: string): ProviderInvariantReport => ({ - provider, - healthy: false, - serving: true, - invariants: [ - { name: 'manifest-residency', holds: true, detail: 'ok', heal: 'none' }, - { - name: 'posted-count-floor', - holds: false, - detail: 'posted 2304 < canonical 2354', - expected: 2354, - actual: 2304, - heal: 'rebuild' - } - ], - checkedAt: 1, - durationMs: 2 -}) - -describe('validateIndexConsistency delegates to provider validateInvariants() (Pass 3)', () => { - let brain: any - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, dimensions: 384 }) - await brain.init() - await brain.add({ data: 'x', type: NounType.Concept }) - await brain.flush() - }) - - it('a broken provider report makes the store unhealthy and names the failing invariant', async () => { - brain.index.validateInvariants = async () => brokenReport('vector') - const v = await brain.validateIndexConsistency() - expect(v.healthy).toBe(false) - expect(v.recommendation).toMatch(/posted-count-floor/) - expect(v.recommendation).toMatch(/posted 2304 < canonical 2354/) - expect(v.recommendation).toMatch(/repairIndex\(\)/) - expect(v.providers?.some((p: ProviderInvariantReport) => p.provider === 'vector' && !p.healthy)).toBe(true) - delete brain.index.validateInvariants - }) - - it('all-healthy provider reports do not flip the store unhealthy', async () => { - brain.index.validateInvariants = async () => healthyReport('vector') - brain.graphIndex.validateInvariants = async () => healthyReport('graph') - const v = await brain.validateIndexConsistency() - expect(v.healthy).toBe(true) - expect(v.providers?.length).toBe(2) - delete brain.index.validateInvariants - delete brain.graphIndex.validateInvariants - }) - - it('a validateInvariants() that THROWS is surfaced as unhealthy, never swallowed', async () => { - brain.index.validateInvariants = async () => { throw new Error('provider blew up') } - const v = await brain.validateIndexConsistency() - expect(v.healthy).toBe(false) - expect(v.recommendation).toMatch(/validate-invariants-threw/) - delete brain.index.validateInvariants - }) - - it('providers without validateInvariants() are omitted (JS baseline unchanged)', async () => { - const v = await brain.validateIndexConsistency() - expect(v.providers).toBeUndefined() - expect(typeof v.healthy).toBe('boolean') - }) - - it('repairIndex() rebuilds a provider whose failing invariant asks for it', async () => { - let rebuilt = false - brain.index.validateInvariants = async () => (rebuilt ? healthyReport('vector') : brokenReport('vector')) - const origRebuild = brain.index.rebuild.bind(brain.index) - brain.index.rebuild = async (...a: any[]) => { rebuilt = true; return origRebuild(...a) } - await brain.repairIndex() - expect(rebuilt).toBe(true) - // After repair, the store validates healthy again. - const v = await brain.validateIndexConsistency() - expect(v.providers?.find((p: ProviderInvariantReport) => p.provider === 'vector')?.healthy).toBe(true) - brain.index.rebuild = origRebuild - delete brain.index.validateInvariants - }) -}) diff --git a/tests/unit/vector-cold-read-guard.test.ts b/tests/unit/vector-cold-read-guard.test.ts deleted file mode 100644 index 49ca6426..00000000 --- a/tests/unit/vector-cold-read-guard.test.ts +++ /dev/null @@ -1,115 +0,0 @@ -/** - * @module tests/unit/vector-cold-read-guard - * @description Pattern-A / Finding 1: a pure semantic find({ query }) has no - * filter, so verifyMetadataLive never fires — nothing guarded the vector index. - * A cold native vector index that loaded its COUNT but not its serving structure - * returned a silent []. verifyVectorLive() closes that: honest isReady() first, - * else a known-vector self-match probe; self-heal (rebuild) or throw - * VectorIndexNotReadyError — never a silent empty result. - */ -import { describe, it, expect, beforeEach } from 'vitest' -import { Brainy, NounType, VectorIndexNotReadyError } from '../../src/index.js' - -const V = (): number[] => Array.from({ length: 384 }, (_, i) => Math.sin(i * 0.1) + 0.001) - -describe('Vector cold-read guard (verifyVectorLive) — silent-[] on cold semantic find', () => { - let brain: any - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, dimensions: 384 }) - await brain.init() - await brain.add({ vector: V(), type: NounType.Concept, metadata: { status: 'active' } }) - await brain.add({ vector: V(), type: NounType.Concept, metadata: { status: 'archived' } }) - await brain.flush() - }) - - it('warm brain: semantic find is correct and the guard does not rebuild', async () => { - const vi = brain.index - let rebuilds = 0 - const origRebuild = vi.rebuild.bind(vi) - vi.rebuild = async (...a: any[]) => { rebuilds++; return origRebuild(...a) } - await brain.find({ query: 'anything', searchMode: 'semantic', limit: 100 }) - expect(rebuilds).toBe(0) - expect(brain._vectorVerified).toBe(true) - vi.rebuild = origRebuild - }) - - it('cold index: verifyVectorLive self-heals via rebuild — semantic find is correct, NOT silent []', async () => { - const vi = brain.index - const origSearch = vi.search.bind(vi) - const origRebuild = vi.rebuild.bind(vi) - let cold = true - brain._vectorVerified = false - // size()>0 (count present) but search returns nothing until a rebuild warms it. - vi.search = async (...a: any[]) => (cold ? [] : origSearch(...a)) - vi.rebuild = async (...a: any[]) => { await origRebuild(...a); cold = false } - try { - const res = await brain.find({ query: 'x', searchMode: 'semantic', limit: 100 }) - expect(res.length).toBeGreaterThan(0) // self-healed - } finally { - vi.search = origSearch; vi.rebuild = origRebuild - } - }) - - it('unrecoverably cold index: semantic find throws VectorIndexNotReadyError', async () => { - const vi = brain.index - const origSearch = vi.search.bind(vi) - const origRebuild = vi.rebuild.bind(vi) - brain._vectorVerified = false - vi.search = async () => [] // always cold; rebuild can't fix it - vi.rebuild = async () => {} - try { - await expect( - brain.find({ query: 'x', searchMode: 'semantic', limit: 100 }) - ).rejects.toBeInstanceOf(VectorIndexNotReadyError) - } finally { - vi.search = origSearch; vi.rebuild = origRebuild - } - }) - - it('native provider reporting isReady()===false rebuilds, then serves', async () => { - const vi = brain.index - const origRebuild = vi.rebuild.bind(vi) - let ready = false - brain._vectorVerified = false - vi.isReady = () => ready - vi.rebuild = async (...a: any[]) => { await origRebuild(...a); ready = true } - try { - const res = await brain.find({ query: 'x', searchMode: 'semantic', limit: 100 }) - expect(ready).toBe(true) // rebuild ran because isReady() was false - expect(res).toBeDefined() - } finally { - delete vi.isReady; vi.rebuild = origRebuild - } - }) - - it('a text-only query does not trigger the vector guard', async () => { - brain._vectorVerified = false - await brain.find({ query: 'active', searchMode: 'text', limit: 5 }) - expect(brain._vectorVerified).toBe(false) // executeVectorSearch never called - }) - - // Regression: the probe must check "returns ANY hit", not an exact self-match — - // HNSW is approximate and get() may re-hydrate the vector, so a healthy - // many-entity index would false-positive under an exact-self check, wrongly - // rebuild, and throw VectorIndexNotReadyError on working data. - it('a healthy many-entity brain with distinct vectors serves semantic find, never throws/rebuilds', async () => { - const many = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, dimensions: 384 }) - await many.init() - for (let i = 0; i < 25; i++) { - const v = Array.from({ length: 384 }, (_, j) => Math.sin((i * 7 + j) * 0.13) + 0.001) - await many.add({ vector: v, type: NounType.Concept, metadata: { n: i } }) - } - await many.flush() - let rebuilds = 0 - const vi = (many as any).index - const origRebuild = vi.rebuild.bind(vi) - vi.rebuild = async (...a: any[]) => { rebuilds++; return origRebuild(...a) } - const res = await many.find({ query: 'x', searchMode: 'semantic', limit: 10 }) - expect(res.length).toBeGreaterThan(0) - expect(rebuilds).toBe(0) - expect((many as any)._vectorVerified).toBe(true) - vi.rebuild = origRebuild - await many.close() - }) -})