diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..cdb2ab14 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,40 @@ +name: CI + +on: + push: + pull_request: + +jobs: + node: + name: Node ${{ matrix.node-version }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + node-version: ['22', '24'] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: npm + - run: npm ci + - run: npm run test:unit + + bun: + name: Bun (latest) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + - uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + - run: npm ci + # test:bun imports the built dist/, so build first. + - run: npm run build + # Bun as a runtime is the supported Bun story (`bun add` / `bun run`). + - run: npm run test:bun diff --git a/.gitignore b/.gitignore index d7d40cbb..64e235b3 100644 --- a/.gitignore +++ b/.gitignore @@ -93,9 +93,14 @@ docs/internal/ # Rust/Cargo build artifacts src/embeddings/candle-wasm/target/ src/embeddings/candle-wasm/Cargo.lock -src/embeddings/wasm/pkg/ -# But keep the pre-built WASM (committed for users without Rust) +# Ignore the wasm-pack output dir's CONTENTS (note the `/*`, not `/`, so the +# re-includes below can take effect — git cannot re-include a file whose parent +# DIRECTORY is excluded). Keep the pre-built WASM committed: it ships in the npm +# package anyway, it lets consumers + CI build without a Rust/wasm-pack toolchain, +# and versioning it makes the shipped artifact reproducible (not "whatever the +# maintainer last built"). +src/embeddings/wasm/pkg/* !src/embeddings/wasm/pkg/*.wasm !src/embeddings/wasm/pkg/*.js !src/embeddings/wasm/pkg/*.d.ts diff --git a/.versionrc.json b/.versionrc.json deleted file mode 100644 index dac6ceca..00000000 --- a/.versionrc.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "types": [ - {"type": "feat", "section": "✨ Features"}, - {"type": "fix", "section": "🐛 Bug Fixes"}, - {"type": "docs", "section": "📚 Documentation"}, - {"type": "refactor", "section": "♻️ Code Refactoring"}, - {"type": "perf", "section": "⚡ Performance Improvements"}, - {"type": "test", "section": "✅ Tests"}, - {"type": "build", "section": "🔧 Build System"}, - {"type": "ci", "section": "🔄 CI/CD"}, - {"type": "style", "hidden": true}, - {"type": "chore", "hidden": true} - ], - "compareUrlFormat": "https://github.com/soulcraftlabs/brainy/compare/{{previousTag}}...{{currentTag}}", - "commitUrlFormat": "https://github.com/soulcraftlabs/brainy/commit/{{hash}}", - "issueUrlFormat": "https://github.com/soulcraftlabs/brainy/issues/{{id}}", - "userUrlFormat": "https://github.com/{{user}}", - "releaseCommitMessageFormat": "chore(release): {{currentTag}}", - "issuePrefixes": ["#"], - "header": "# Changelog\n\nAll notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.\n", - "scripts": { - "postbump": "echo '✅ Version bumped to' $(node -p \"require('./package.json').version\")" - } -} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 459e168b..fd0de54e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,362 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [8.9.0](https://github.com/soulcraftlabs/brainy/compare/v8.8.2...v8.9.0) (2026-07-19) + +- docs: measured performance envelopes v1 (per-op p50/p95 at 1k and 10k, pure-JS floor) (5cabd78) +- fix: release drains in-flight writer-lock heartbeat — no phantom lock after unlink (70e4bc8) +- feat: flush() never compacts — history maintenance moves to close() with bounded passes (300d9f2) + + +### [8.8.2](https://github.com/soulcraftlabs/brainy/compare/v8.8.1...v8.8.2) (2026-07-19) + +- fix: one field-resolution law across aggregation hooks, source.where, removeMany, and find() spellings (945d92d) +- chore: push public docs to the soulcraft.com ingest door on release (42037d0) + + +### [8.8.1](https://github.com/soulcraftlabs/brainy/compare/v8.8.0...v8.8.1) (2026-07-18) + +- fix: O(1) adaptive retention accounting + historyStats fleet audit (6207e48) +- fix: import dedup off-switch honesty + brain-owned lifecycle for the background pass (4fcef7b) + + +### [8.8.0](https://github.com/soulcraftlabs/brainy/compare/v8.7.1...v8.8.0) (2026-07-17) + +- feat: OS-limit detection for pool-scale deployments (16a73b8) + + +### [8.7.1](https://github.com/soulcraftlabs/brainy/compare/v8.7.0...v8.7.1) (2026-07-17) + +- fix: race-proof writer-lock acquisition + machine-readable conflict through init (01a3b46) + + +### [8.7.0](https://github.com/soulcraftlabs/brainy/compare/v8.6.0...v8.7.0) (2026-07-17) + +- feat: scaled transact budgets + labeled timeout diagnostics + envelope docs (6ef9fcb) + + +### [8.6.0](https://github.com/soulcraftlabs/brainy/compare/v8.5.2...v8.6.0) (2026-07-17) + +- feat: brain.auditGraph() — read-only graph-truth audit (2a03fae) + + +### [8.5.2](https://github.com/soulcraftlabs/brainy/compare/v8.5.1...v8.5.2) (2026-07-17) + +- fix: exception-safe aggregation backfill + generation-verified adoption + loud open-path guards (a77b064) + + +### [8.5.1](https://github.com/soulcraftlabs/brainy/compare/v8.5.0...v8.5.1) (2026-07-17) + +- fix: aggregation state adoption on reopen + single-flight backfill + query-cap ratchet removal (da55be7) +- docs: external-backups/sparse-storage guide + generation fact log concept (593bb8b) + + +### [8.5.0](https://github.com/soulcraftlabs/brainy/compare/v8.4.0...v8.5.0) (2026-07-15) + +- test: tolerant timing assertion in the execution-time measure test (4dc0a92) +- feat: committedGeneration capability + pinned durability/stability contracts (d1ecee1) +- docs: RELEASES.md entry for 8.5.0 (provider fact-log access + shared verifier) (e4f37cd) +- feat: provider access to the fact log + shared stamp verifier via internals (352e356) + + +### [8.4.0](https://github.com/soulcraftlabs/brainy/compare/v8.3.3...v8.4.0) (2026-07-15) + +- docs: RELEASES.md entry for 8.4.0 (generation fact log + family stamp) (4a60b43) +- feat: entity-tree family stamp — sourceGeneration + rollup coherence at open (2888ae6) +- feat: generation fact log — after-image commit records, dual-written at every commit point (38b0041) + + +### [8.3.3](https://github.com/soulcraftlabs/brainy/compare/v8.3.2...v8.3.3) (2026-07-15) + +- docs: RELEASES.md entry for 8.3.3 (rename containment fix + repair) (c3feafd) +- test: lens-consistency regression — combined vs subtype-only vs canonical ground truth (4fb41f9) +- fix: VFS rename moves the containment edge — no ghost in the old directory (af8c179) + + +### [8.3.2](https://github.com/soulcraftlabs/brainy/compare/v8.3.1...v8.3.2) (2026-07-14) + +- docs: RELEASES.md entry for 8.3.2 (honest counters) (0932ecd) +- fix: honest counters — removal never re-reads the removed record + repairIndex recounts and persists all rollups (2e2ba9c) + + +### [8.3.1](https://github.com/soulcraftlabs/brainy/compare/v8.3.0...v8.3.1) (2026-07-14) + +- docs: RELEASES.md entry for 8.3.1 (full-removal deletes + family-scoped gate) (c0c68ac) +- fix: full-removal canonical deletes + family-scoped migration gate (366f9a9) +- docs: cite the cross-layer integrity contract generically in comments and notes (1d26988) + + +### [8.3.0](https://github.com/soulcraftlabs/brainy/compare/v8.2.8...v8.3.0) (2026-07-13) + +- docs: RELEASES.md entry for 8.3.0 (heal-cost + cross-layer integrity contract) (7692c6f) +- perf: parallel + id-only canonical enumeration (heal-cost dominant term) (ec5b933) +- feat: registered-blob family contract — declared index blobs are undeletable (bfa1762) +- feat: validateIndexConsistency delegates to provider invariants (6bcb54f) + + +### [8.2.8](https://github.com/soulcraftlabs/brainy/compare/v8.2.7...v8.2.8) (2026-07-13) + +- fix: honest index readiness — no silently-empty queries on a cold index (d0f69c7) + + +### [8.2.7](https://github.com/soulcraftlabs/brainy/compare/v8.2.6...v8.2.7) (2026-07-13) + +- fix: restore loadBinaryBlob fault-propagation (native column-store lockstep) (b6c7039) + + +### [8.2.6](https://github.com/soulcraftlabs/brainy/compare/v8.2.5...v8.2.6) (2026-07-13) + +- docs: RELEASES.md entry for 8.2.6 (write/index-spine hardening) (a873852) +- chore: hold loadBinaryBlob fault-propagation for the cortex column-store lockstep (36c10c1) +- fix: aggregation surfaces materialize/state-load failures loudly (02eff64) +- fix: surface a degraded derived index on reads instead of serving it silently (ba958d9) +- fix: saveBinaryBlob never acks a durable write that stored nothing (7feba49) +- fix: refuse writes when single-op history cannot be made durable (54c1836) +- fix: clear() wipes the full native/derived footprint, not a subset (d8301f8) +- fix: surface segment/entity read faults loudly instead of masking as absent (af5d2f3) +- fix: spine hardening pass 1 (part) — count symmetry, honest partial-load, flush durability, read-fault propagation (119087a) +- test: pin the read-your-writes contract under the single writer (eb9c4eb) + + +### [8.2.5](https://github.com/soulcraftlabs/brainy/compare/v8.2.4...v8.2.5) (2026-07-12) + +- docs: RELEASES.md entry for 8.2.5 (honest rollback-failure response) (a7c7aa5) +- fix: honest response when a transaction rollback cannot complete (711d2f0) + + +### [8.2.4](https://github.com/soulcraftlabs/brainy/compare/v8.2.3...v8.2.4) (2026-07-12) + +- docs: RELEASES.md entry for 8.2.4 (non-destructive restore) (4574695) +- fix: non-destructive, crash-resumable restore (a2f4f6a) + + +### [8.2.3](https://github.com/soulcraftlabs/brainy/compare/v8.2.2...v8.2.3) (2026-07-12) + +- docs: RELEASES.md entry for 8.2.3 (transact durability barrier) (be5ce0b) +- fix: transact durability barrier — committed transactions are durable on return (3b8fa51) + + +### [8.2.2](https://github.com/soulcraftlabs/brainy/compare/v8.2.1...v8.2.2) (2026-07-11) + +- docs: RELEASES.md entry for 8.2.2 (transaction timeout rollback) (ed97006) +- fix: transaction timeout rolls back applied operations (no torn state) (508a8e3) + + +### [8.2.1](https://github.com/soulcraftlabs/brainy/compare/v8.2.0...v8.2.1) (2026-07-10) + +- test: update graph-index operation constructors to the VerbEndpointInts signature (62a449d) +- docs: RELEASES.md entry for 8.2.1 (transact forward-ref parity fix) (7089782) +- fix: transact forward references resolve graph endpoint ints at execute time (a175406) + + +### [8.2.0](https://github.com/soulcraftlabs/brainy/compare/v8.1.0...v8.2.0) (2026-07-10) + +- docs: RELEASES.md entry for 8.2.0 (temporal VFS) (98ceadc) +- feat: temporal VFS — file content joins the Model-B immutability model (a3467e1) +- docs: pin the write-path invariant in the plugin contract (the onChange change-feed guarantee) (4af8fb3) + + +### [8.1.0](https://github.com/soulcraftlabs/brainy/compare/v8.0.17...v8.1.0) (2026-07-10) + +- docs: RELEASES.md entry for 8.1.0 (brain.onChange change feed) (4e9be08) +- feat: brain.onChange — the in-process change feed for every committed mutation (fd5edb5) + + +### [8.0.17](https://github.com/soulcraftlabs/brainy/compare/v8.0.16...v8.0.17) (2026-07-08) + +- docs: RELEASES.md entry for 8.0.17 (canonical count recovery + dead-machinery sweep) (6b8b9cb) +- fix: count recovery scans the canonical layout; remove the dead 7.x hnsw sharding machinery (352e2da) + + +### [8.0.16](https://github.com/soulcraftlabs/brainy/compare/v8.0.15...v8.0.16) (2026-07-08) + +- docs: RELEASES.md entry for 8.0.16 (atomic ifAbsent/upsert + exact blob refCounts) (54e7c0e) +- fix: atomic ifAbsent/upsert inserts + exact blob reference counts under concurrency (867939e) + + +### [8.0.15](https://github.com/soulcraftlabs/brainy/compare/v8.0.14...v8.0.15) (2026-07-08) + +- docs: RELEASES.md entry for 8.0.15 (atomic ifRev CAS) (b1fe25a) +- fix: ifRev CAS is atomic — the revision check now runs under the commit mutex (9a3d1bd) + + +### [8.0.14](https://github.com/soulcraftlabs/brainy/compare/v8.0.13...v8.0.14) (2026-07-07) + +- docs: RELEASES.md entry for 8.0.14 (migration preserves branch-scoped non-entity state) (64188a3) +- fix: 7→8 migration preserves branch-scoped non-entity state instead of deleting it (a93bb4e) + + +### [8.0.13](https://github.com/soulcraftlabs/brainy/compare/v8.0.12...v8.0.13) (2026-07-07) + +- docs: RELEASES.md entry for 8.0.13 (accurate boot log for established stores) (38e8de5) +- fix: an established store no longer boot-logs "New installation" (3086916) + + +### [8.0.12](https://github.com/soulcraftlabs/brainy/compare/v8.0.11...v8.0.12) (2026-07-07) + +- docs: RELEASES.md entry for 8.0.12 (7→8 VFS recovery, zero-rebuild cold open, strict query operators) (d9017e7) +- fix: recover VFS content blobs stranded by a 7→8 upgrade, in place on open (c0f6ccd) +- fix: validate where-operators and align the in-memory matcher to the documented set (6821e19) +- docs: correct rc-era time-travel staleness + record the embedding-model ordering constraint (68da660) +- fix: cold-open no longer re-derives durable indexes — complete the readiness contract for all three providers (61c247c) +- docs: RELEASES.md entry for 8.0.11 (exit-hang class closed for every op shape) (4fde94b) + + +### [8.0.11](https://github.com/soulcraftlabs/brainy/compare/v8.0.10...v8.0.11) (2026-07-02) + +- fix: no script shape can hang on brainy's internals — unref every maintenance timer + one-shot beforeExit (30eacbd) +- docs: RELEASES.md entry for 8.0.10 (clean process exit after close) (2da2736) + + +### [8.0.10](https://github.com/soulcraftlabs/brainy/compare/v8.0.9...v8.0.10) (2026-07-02) + +- fix: a bare script now exits cleanly after close() — release every process keep-alive (c540d63) + + +### [8.0.9](https://github.com/soulcraftlabs/brainy/compare/v8.0.8...v8.0.9) (2026-07-02) + +- feat: guarded plugin auto-detection — installing @soulcraft/cor is the opt-in (588267b) + + +### [8.0.8](https://github.com/soulcraftlabs/brainy/compare/v8.0.7...v8.0.8) (2026-07-02) + +- docs: plugins are explicit opt-in — correct the README scale section and plugins config comment (e420369) + + +### [8.0.7](https://github.com/soulcraftlabs/brainy/compare/v8.0.1...v8.0.7) (2026-07-02) + +- docs: GA version is 8.0.7 — npm retired 8.0.0-8.0.6 (January dev-cycle unpublishes) (5db2c41) +- chore(release): 8.0.1 (48bea9e) +- docs: flagship README for the 8.0 GA; GA version is 8.0.1 (e44620e) +- docs: rename the native provider to @soulcraft/cor across public docs and JSDoc (bf4a333) +- chore(release): 8.0.0 (a3c2717) +- docs: RELEASES.md 8.0.0 GA entry (RC notes become history) (4584d0b) +- feat: promote the 8.0 u64-id line to main for the 8.0.0 GA (55d57f8) +- fix(8.0): byte-copy _id_mapper/* in the pre-upgrade backup (cor's write-new nuance) (a30ed72) +- chore(release): 7.33.5 (29b9d5f) +- fix: metadata cold-read guard — no more silent [] on cold find({where}) (7.33.5) (9dc4c5e) +- fix(8.0): metadata cold-read guard — no more silent [] on cold find({where}) (79e8709) +- docs(8.0): add module JSDoc to typeValidation.ts (the one file missing a module block) (ab53fa0) +- feat(8.0): auto pre-upgrade backup — hard-link snapshot before the 7.x→8.0 migration (1aad1f6) +- fix(ci): commit the prebuilt wasm pkg + build before test:bun (green CI on fresh clone) (ed178e2) +- chore(release): 7.33.4 (2be3d0f) +- fix: never serve a silent [] from find({connected}) on a cold-loaded graph (fd699d0) +- chore(release): 7.33.3 (d1665bb) +- fix: re-validate find() results against the predicate (index-integrity guard) (7b5db0d) +- chore(release): 7.33.2 (9593a27) +- fix: graph adjacency cold-load consistency guard — no more silent [] on connected (1694f68) +- chore(release): 7.33.1 (811c7da) +- fix: getNouns cursor pagination re-scanned the first page forever (permanent CPU loop) (6721c52) +- chore(release): 7.33.0 (526aaad) +- feat: visibility tier (public/internal/system) on nouns + verbs (3a62445) +- chore(release): 7.32.2 (c53dd61) +- refactor: rename BackupData → PortableGraph (the type is interchange, not a backup) (89036de) +- chore(release): 7.32.1 (5e7379d) +- fix: getNouns().totalCount reports true total, not page size; quiet benign mmap-vector log (edff637) +- chore(release): 7.32.0 (adec0ba) +- feat: portable graph export()/import() (BackupData v1) on brain.data() (a408d37) +- chore(release): 7.31.8 (89c6d04) +- fix: query-cap memory misread (MemAvailable + floor) + rootDirectory getter for native mmap fast-path (3f8e097) +- chore(release): 7.31.7 (4f8159c) +- fix: vfs.rename() issues a metadata-only update + rollback of fresh adds removes them (ac29b0e) +- chore(release): 7.31.6 (9b52629) +- fix: remap reserved fields from update() metadata patches to their canonical location (67e5fc8) +- chore(release): 7.31.5 (e5ec658) +- fix: feature-detect setVectorBackend before wiring the mmap-vector backend (a537b36) +- chore(release): 7.31.4 (a8cbab6) +- fix: feature-detect setConnectionsCodec before wiring the connections codec (747ab97) +- chore(release): 7.31.3 (cfb051c) +- fix: mmap-vector backend capacity NaN at the provider FFI boundary (eade6ff) + + +### [8.0.0-rc.9](https://github.com/soulcraftlabs/brainy/compare/v8.0.0-rc.8...v8.0.0-rc.9) (2026-07-01) + +- docs(8.0): RELEASES.md — rc.9 (migration LOCK + 6x cosine + ES2023/Node22 floor) (3a33987) +- chore(8.0): ES2023 target + drop DOM lib + downlevelIteration (config truth-up) (cf74c25) +- perf(8.0): allocation-free distance loops (6x cosine) — evidence-revised Fork X (b5bc73f) +- feat(8.0): #18 coordinated migration LOCK — block-and-queue the 7.x→8.0 auto-upgrade (67bbf69) +- chore(8.0): modernize toolchain + position Bun as a runtime (ca9129a) + + +### [8.0.0-rc.8](https://github.com/soulcraftlabs/brainy/compare/v8.0.0-rc.7...v8.0.0-rc.8) (2026-06-30) + +- docs(8.0): RELEASES.md — rc.8 (no-freeze online whole-brain auto-upgrade) (5af48a9) +- feat(8.0): no-freeze auto-upgrade hooks — isMigrating() deference + stampBrainFormat() + brain-format export (b6b9198) + + +### [8.0.0-rc.7](https://github.com/soulcraftlabs/brainy/compare/v8.0.0-rc.6...v8.0.0-rc.7) (2026-06-30) + +- docs(8.0): RELEASES.md — rc.7 (cold-graph self-heal + billion-scale RAM + version handshake) (1ddc786) +- perf(8.0): bound per-id generation history chains (O(W+L) resident, was O(N)) (a859d6e) +- feat(8.0): eager graphIndex.init() before the isReady() rebuild gate (8f4787b) +- feat(8.0): version-handshake marker (formatInfo + indexEpoch) for whole-brain auto-upgrade (fc7f110) +- fix(8.0): never serve a silent [] from find({connected}) on a cold-loaded graph (229b067) +- perf(8.0): represent the committed-generation ledger as an interval set (93f61db) +- perf(8.0): drop O(N)-resident id-keyed storage caches; source counts from the record (b6beb7f) + + +### [8.0.0-rc.6](https://github.com/soulcraftlabs/brainy/compare/v8.0.0-rc.5...v8.0.0-rc.6) (2026-06-29) + +- docs(8.0): RELEASES.md — rc.6 (perf + native-provider contract + test hygiene) (6daa70e) +- feat(8.0): wire the two cor-confirmed metadata-provider contract additions (8b19122) +- test(8.0): re-home orphaned test files into the gate + guard against recurrence (3f9f140) +- perf(8.0): negation/absence where-operators via roaring-bitmap difference (5f974ab) +- perf(8.0): HNSW removeItem is O(in-degree) via a reverse-adjacency index (72df557) + + +### [8.0.0-rc.5](https://github.com/soulcraftlabs/brainy/compare/v8.0.0-rc.4...v8.0.0-rc.5) (2026-06-29) + +- docs(8.0): RELEASES.md — rc.5 hardening + the breaking operator removal (6c9a438) +- refactor(8.0): remove the 4 deprecated query-operator aliases (clean break) (ddcc0c7) +- refactor(8.0): remove dead/deprecated code (legacy sweep) (b9369f2) +- refactor(8.0): API-surface + quality polish from the readiness audit (a52dba2) +- fix(8.0): close GA-blocking correctness gaps from the readiness audit (47e8031) +- docs(8.0): correct public docs to the real 8.0 API + honest perf claims (40d2cd5) +- fix(8.0): re-validate find() results against the predicate (index-integrity guard) (3d11619) + + +### [8.0.0-rc.4](https://github.com/soulcraftlabs/brainy/compare/v8.0.0-rc.3...v8.0.0-rc.4) (2026-06-24) + +- docs(8.0): drop the DeletedItemsIndex section + pseudo-code from index-architecture (e7b50cf) +- fix(8.0): gate native graph analytics on the provider readiness flag (d321cf5) +- refactor(8.0): remove dead, unreachable, and unwired modules (bf0afe8) +- build(8.0): clean dist before every build so stale artifacts never ship (03d6540) +- feat(8.0): #35 part-3 — supply at-gen candidate vectors for the native exact-rerank (c9e2169) + + +### [8.0.0-rc.3](https://github.com/soulcraftlabs/brainy/compare/v8.0.0-rc.2...v8.0.0-rc.3) (2026-06-23) + +- test(8.0): de-flake the VFS path-cache timing assertion (0e8972c) +- feat(8.0): asOf at-gen vector defer — provider-served historical semantic search (#35) (1c363e8) +- perf(8.0): bound find({ where, orderBy }) sort to the page (CTX-BR-FIND-ORDERBY) (450084b) +- feat(8.0): brain.graph.subgraph(query) query→expand fusion (#61) (82dde92) +- feat(8.0): vector allowedIds predicate-pushdown into find() (#46) (dd325f2) +- feat(8.0): graph analytics — brain.graph.rank / communities / path (632d90a) +- fix(8.0): restore() reloads a native entity-id mapper before graphIndex.rebuild() (4d0b64f) +- test(8.0): cover pending-tier range queries + setRetentionBudget adaptive reclaim (3783e61) +- feat(8.0): Model-B per-write generation-stamping + adaptive retention knob (5c3bb2c) +- test(8.0): Model-B write-perf + scalability spike harnesses (afac7f9) +- perf(8.0): per-id history chains for O(log) historical reads + bounded delta cache (ceed70d) +- refactor(8.0): graph analytics contract — intent names, not algorithm names (f3e6911) +- docs(8.0): RELEASES — native provider is @soulcraft/cor 3.0 (fix cortex 3.0 self-contradiction) (96d9c0b) +- test(8.0): cover the native graph seam + make provider resolution factory-tolerant (29410bc) + + +### [8.0.0-rc.2](https://github.com/soulcraftlabs/brainy/compare/v8.0.0-rc.1...v8.0.0-rc.2) (2026-06-21) + +- docs(8.0): RELEASES rc.2 additions — graph engine + additive wins + correctness fixes (18f27cb) +- feat(8.0): brain.graph.export() + noun-walk cursor + noun visibility hydration (c2a84c9) +- feat(8.0): brain.graph.subgraph() + native-provider routing (8c2b57a) +- feat(8.0): related({ node }) — one-call both-direction incident edges (d4de48d) +- feat(8.0): GraphAccelerationProvider contract — the native graph-engine seam (a3d6fdb) +- perf(8.0): cursor pagination for the verb walk — full edge pagination O(N²) → O(N) (682e786) +- perf(8.0): visibility-aware fast adjacency — related() stays O(degree) under default visibility (a914313) +- feat(8.0): upsert + FindParams.includeVectors + removeMany adaptive chunking (4cc2088) +- docs(8.0): note reserved-field default-throw in RELEASES rc additions (1bc709d) +- feat(8.0): reserved-field enforcement — reservedFieldPolicy defaults to throw (54c7c39) +- docs: mark 8.0.0-rc.1 published (npm tag rc) + note rc.1 additions (ae3fe82) + + ### [8.0.0-rc.1](https://github.com/soulcraftlabs/brainy/compare/v7.31.2...v8.0.0-rc.1) (2026-06-20) - feat(8.0): id-normalization (#18) + aggregation min/max delete-safety + RC-safe release (d02e522) diff --git a/README.md b/README.md index 75536c82..d1342f52 100644 --- a/README.md +++ b/README.md @@ -1,439 +1,217 @@ -# Brainy -

- Brainy Logo + Brainy

-[![npm version](https://badge.fury.io/js/%40soulcraft%2Fbrainy.svg)](https://www.npmjs.com/package/@soulcraft/brainy) -[![npm downloads](https://img.shields.io/npm/dm/@soulcraft/brainy.svg)](https://www.npmjs.com/package/@soulcraft/brainy) -[![Documentation](https://img.shields.io/badge/docs-soulcraft.com-blue.svg)](https://soulcraft.com/docs) -[![MIT License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) -[![TypeScript](https://img.shields.io/badge/%3C%2F%3E-TypeScript-%230074c1.svg)](https://www.typescriptlang.org/) +

Brainy

-**Three database paradigms. One API. Zero configuration.** +

+ Three database paradigms. One API. Zero configuration.
+ The in-process knowledge database for TypeScript — vector search, graph traversal,
+ and metadata filtering unified in a single query. +

-Built because we were tired of stitching together Pinecone + Neo4j + MongoDB and spending weeks on configuration before writing a single line of business logic. Brainy unifies vector search, graph traversal, and metadata filtering so you don't have to choose. +

+ npm version + npm downloads + CI + Documentation + MIT License + TypeScript +

-**New here?** → **[What is Brainy? — plain-language overview, no jargon](docs/eli5.md)** +

+ Quick start · + One query · + Features · + Scale with Cor · + Docs +

--- -## Install +Built because we were tired of stitching a vector store to a graph database to a document store — and spending weeks on plumbing before writing a line of business logic. Brainy indexes every fact **three ways at once** and lets one call query them together: + +| You write | Brainy indexes it as | You query it with | +|---|---|---| +| `data: 'Ada wrote the first program'` | a **384-dim vector** (local embedding — no API key) | `find({ query: 'computing pioneers' })` | +| `metadata: { field: 'CS', year: 1843 }` | **structured fields** (O(1) exact, O(log n) range) | `find({ where: { year: { lessThan: 1900 } } })` | +| `relate({ from: ada, to: babbage })` | a **typed, directed graph edge** | `find({ connected: { to: babbage, depth: 2 } })` | + +It runs **inside your process** — no server, no Docker, nothing to operate — and persists to plain files you can snapshot with a hard link. + +**New here?** → **[What is Brainy? — plain-language overview, no jargon](docs/eli5.md)** + +## Quick start ```bash -npm install @soulcraft/brainy +bun add @soulcraft/brainy # Bun ≥ 1.1 — recommended +npm install @soulcraft/brainy # Node.js ≥ 22 ``` -## Quick Start - ```javascript import { Brainy, NounType, VerbType } from '@soulcraft/brainy' -const brain = new Brainy() +const brain = new Brainy() // in-memory; one line swaps to disk await brain.init() -// Add knowledge — text auto-embeds, metadata auto-indexes -const reactId = await brain.add({ +// Text auto-embeds locally; metadata auto-indexes +const react = await brain.add({ data: 'React is a JavaScript library for building user interfaces', type: NounType.Concept, + subtype: 'library', metadata: { category: 'frontend', year: 2013 } }) -const nextId = await brain.add({ - data: 'Next.js framework for React with server-side rendering', +const next = await brain.add({ + data: 'Next.js is a React framework with server-side rendering', type: NounType.Concept, - metadata: { category: 'framework', year: 2016 } + subtype: 'framework', + metadata: { category: 'frontend', year: 2016 } }) -// Create a relationship -await brain.relate({ from: nextId, to: reactId, type: VerbType.BuiltOn }) - -// Query all three paradigms at once -const results = await brain.find({ - query: 'modern frontend frameworks', // Vector similarity - where: { year: { greaterThan: 2015 } }, // Metadata filtering - connected: { to: reactId, depth: 2 } // Graph traversal -}) +await brain.relate({ from: next, to: react, type: VerbType.DependsOn, subtype: 'runtime' }) ``` -**[Full API Reference](docs/api/README.md)** | **[soulcraft.com/docs](https://soulcraft.com/docs)** - ---- - -## Three Indexes, One Query - -Every piece of knowledge lives in three indexes simultaneously: - -- **`data`** → **Vector index** — Content for semantic search. Strings auto-embed into 384-dim vectors. Queried with `find({ query: '...' })`. -- **`metadata`** → **Metadata index** — Structured fields for filtering. O(1) lookups. Queried with `find({ where: { ... } })`. -- **`relate()`** → **Graph index** — Typed, directed relationships between entities. Traversed with `find({ connected: { ... } })`. - -```javascript -// Data → vector index (semantic search) -const articleId = await brain.add({ - data: 'A deep dive into transformer architectures', - type: NounType.Document, - metadata: { author: 'Dr. Chen', year: 2024, tags: ['AI'] } // → metadata index -}) - -// Relationships → graph index -await brain.relate({ from: authorId, to: articleId, type: VerbType.Authored }) - -// Query all three at once -brain.find({ - query: 'attention mechanisms', // Vector similarity - where: { year: { greaterThan: 2023 } }, // Metadata filter - connected: { from: authorId, depth: 1 } // Graph traversal -}) -``` - -**[Data Model Reference](docs/DATA_MODEL.md)** | **[Query Operators](docs/QUERY_OPERATORS.md)** - ---- - -## Features - -### Triple Intelligence - -Vector search + graph traversal + metadata filtering in every query. No stitching services together — one `find()` call combines all three. +## One query, three engines ```javascript const results = await brain.find({ - query: 'machine learning', - where: { department: 'engineering', level: 'senior' }, - connected: { from: teamLeadId, via: VerbType.WorksWith, depth: 2 } + query: 'modern frontend frameworks', // vector — what it means + where: { year: { greaterThan: 2015 } }, // metadata — what it is + connected: { to: react, depth: 2 } // graph — what it touches }) ``` -### Hybrid Search +Every clause is optional; any combination composes. Under the hood Brainy plans the query across an HNSW vector index, a roaring-bitmap field index, and an adjacency graph index — and re-validates every result against your predicate before returning it, so a corrupt index can never hand you a wrong answer. -Automatically combines keyword (text) and semantic (vector) search. No configuration needed. +## Feature tour + +### The database is a value + +Pin it, rewind it, fork it. Snapshot isolation without a server. ```javascript -await brain.find({ query: 'David Smith' }) // Auto: text + semantic -await brain.find({ query: 'AI concepts', searchMode: 'semantic' }) // Semantic only -await brain.find({ query: 'exact id', searchMode: 'text' }) // Text only +const db = brain.now() // pin current state — O(1) + +await brain.transact([ // atomic all-or-nothing, CAS-guarded + { op: 'update', id: order, metadata: { status: 'paid' } }, + { op: 'relate', from: invoice, to: order, type: VerbType.References, subtype: 'billing' } +], { ifAtGeneration: db.generation }) + +await db.get(order) // still 'pending' — pinned forever +await brain.get(order) // 'paid' — live + +const lastWeek = await brain.asOf(Date.now() - 7 * 86_400_000) // full query surface, past state +const whatIf = await db.with([{ op: 'remove', id: order }]) // speculative — never touches disk +await brain.now().persist('/backups/today') // instant hard-link snapshot ``` -### Query Operators +**[Consistency model](docs/concepts/consistency-model.md)** · **[Snapshots & time travel](docs/guides/snapshots-and-time-travel.md)** -Filter metadata with equality, comparison, array, existence, pattern, and logical operators: +### Local embeddings — no API keys + +Strings embed on-device with a bundled MiniLM model (WASM). Semantic search works offline, in CI, and on air-gapped machines, at zero cost per call. Hybrid keyword + semantic ranking is the default: ```javascript -await brain.find({ - where: { - status: 'active', // Exact match - score: { greaterThan: 90 }, // Comparison - tags: { contains: 'ai' }, // Array - anyOf: [{ role: 'admin' }, { role: 'owner' }] // Logical OR - } -}) +await brain.find({ query: 'David Smith' }) // auto: text + semantic +await brain.find({ query: 'AI concepts', searchMode: 'semantic' }) // semantic only ``` -**[Query Operators Reference](docs/QUERY_OPERATORS.md)** — all operators with indexed/in-memory matrix +### A typed graph, not a bag of edges -### Graph Relationships - -Typed, directed edges between entities. Traverse connections at any depth. +42 entity types × 127 relationship types form a shared vocabulary for any domain — healthcare (`Patient → diagnoses → Condition`), finance (`Account → transfers → Transaction`), yours. Your own taxonomy layers on with `subtype`, enforced at write time: ```javascript -await brain.relate({ from: personId, to: projectId, type: VerbType.WorksOn }) +await brain.add({ data: 'Avery Brooks', type: NounType.Person, subtype: 'employee' }) -const results = await brain.find({ - connected: { from: personId, via: VerbType.WorksOn, depth: 3 } -}) +brain.counts.bySubtype(NounType.Person) // O(1) — { employee: 12, customer: 847 } +brain.requireSubtype(NounType.Person, { values: ['employee', 'customer'], required: true }) ``` -### Database as a Value +**[Type system](docs/architecture/noun-verb-taxonomy.md)** · **[Subtypes & facets](docs/guides/subtypes-and-facets.md)** -The whole database, pinned as an immutable value. Snapshot isolation, time travel, atomic transactions, instant hard-link snapshots. +### Graph analytics built in ```javascript -const db = brain.now() // Pin current state — O(1) - -// Atomic multi-write transaction (all-or-nothing, with CAS) -await brain.transact([ - { op: 'update', id: orderId, metadata: { status: 'paid' } }, - { op: 'relate', from: invoiceId, to: orderId, type: VerbType.References, subtype: 'billing' } -], { meta: { author: 'billing-service' }, ifAtGeneration: db.generation }) - -await db.get(orderId) // Still 'pending' — pinned, forever -await brain.get(orderId) // 'paid' — live - -// Time travel: full query surface at any past state -const yesterday = await brain.asOf(new Date(Date.now() - 86_400_000)) -const past = await yesterday.find({ query: 'unpaid orders' }) - -// What-if: speculative writes, nothing touches disk -const whatIf = await db.with([{ op: 'remove', id: orderId }]) - -// Instant backup: hard-link snapshot, opens read-only with Brainy.load() -await brain.now().persist('/backups/today') +await brain.graph.rank() // which entities matter most (centrality) +await brain.graph.communities() // natural clusters +await brain.graph.path(a, b) // how two things connect +await brain.graph.subgraph([seed], { depth: 2 }) // bounded neighborhood → { nodes, edges } +await brain.graph.export() // whole graph, one O(N+E) streaming pass ``` -**[Consistency Model](docs/concepts/consistency-model.md)** | **[Snapshots & Time Travel](docs/guides/snapshots-and-time-travel.md)** +### Write-time aggregations -### Virtual Filesystem +`SUM` / `COUNT` / `AVG` / `MIN` / `MAX` with `GROUP BY` and time windows, maintained incrementally on every write — reads are O(1) lookups, not scans. **[Aggregation guide](docs/guides/aggregation.md)** -File operations with semantic search built in. - -```javascript -const vfs = brain.vfs - -await vfs.writeFile('/docs/readme.md', 'Project documentation') -const content = await vfs.readFile('/docs/readme.md') -const tree = await vfs.getTreeStructure('/docs', { maxDepth: 3 }) - -// Semantic file search -const matches = await vfs.search('React components with hooks') -``` - -**[VFS Quick Start](docs/vfs/QUICK_START.md)** | **[Common Patterns](docs/vfs/COMMON_PATTERNS.md)** - -### Import Anything - -CSV, Excel, PDF, URLs — auto-detected format, auto-classified entities. +### Import anything ```javascript await brain.import('customers.csv') -await brain.import('sales-data.xlsx') // all sheets processed -await brain.import('research-paper.pdf') // tables extracted automatically +await brain.import('sales.xlsx') // every sheet +await brain.import('research-paper.pdf') // tables extracted await brain.import('https://api.example.com/data.json') -await brain.import('./handbook.md', { vfsPath: '/imports/handbook' }) // preserve in VFS ``` -**[Import Guide](docs/guides/import-anything.md)** +Entities auto-classify on the way in; `brain.extractEntities(text)` exposes the same NER ensemble directly. **[Import guide](docs/guides/import-anything.md)** -### Entity Extraction - -AI-powered named entity recognition with 4-signal ensemble scoring. +### A filesystem that understands content ```javascript -const entities = await brain.extractEntities('John Smith founded Acme Corp in New York') -// [ -// { text: 'John Smith', type: NounType.Person, confidence: 0.95 }, -// { text: 'Acme Corp', type: NounType.Organization, confidence: 0.92 }, -// { text: 'New York', type: NounType.Location, confidence: 0.88 } -// ] +await brain.vfs.writeFile('/docs/readme.md', 'Project documentation') +await brain.vfs.search('React components with hooks') // semantic file search ``` -**[Neural Extraction Guide](docs/neural-extraction.md)** +**[VFS quick start](docs/vfs/QUICK_START.md)** -### Plugin System +### Operations-grade by default -Optional native acceleration via `@soulcraft/cortex` — SIMD distance calculations, CRoaring bitmaps, Candle ML embeddings. +- **Single-writer, many-reader** — an exclusive lock protects the data directory; `Brainy.openReadOnly()` and the `brainy inspect` CLI examine a live brain from another process, safely. +- **Self-upgrading data files** — a 7.x brain opens under 8.x and migrates itself behind an observable lock (`getIndexStatus().migration`), with an automatic pre-upgrade backup. No migration scripts. +- **No silent wrong answers** — cold-open guards self-heal or throw typed errors (`MetadataIndexNotReadyError`, `GraphIndexNotReadyError`); they never return `[]` for data that exists. + +**[Multi-process model](docs/concepts/multi-process.md)** · **[Inspection guide](docs/guides/inspection.md)** + +## From laptop to hundreds of millions + +Brainy's TypeScript engines take you a long way. When you outgrow them, add the native engine — **the API doesn't change**: + +```bash +npm install @soulcraft/cor +``` ```javascript -const brain = new Brainy({ plugins: ['@soulcraft/cortex'] }) -await brain.init() +const brain = new Brainy({ storage: { type: 'filesystem', path: './data' } }) +await brain.init() // @soulcraft/cor detected — same code, native engines underneath ``` -Plugins are opt-in. Brainy never auto-imports packages unless listed in `plugins`. +Installing the package is the opt-in: if `@soulcraft/cor` is present, it loads and announces itself in the init log; if it's present but broken, `init()` **throws** — an installed accelerator never silently vanishes behind the JS engines. Opt out with `plugins: []`, or pin exactly what loads with `plugins: ['@soulcraft/cor']`. [`@soulcraft/cor`](https://www.npmjs.com/package/@soulcraft/cor) (Brainy 8.x ↔ Cor 3.x, version-matched) registers Rust implementations behind every provider seam: SIMD distance kernels, memory-mapped storage, a disk-native vector index that doesn't need your dataset in RAM, durable LSM field/graph indexes that serve cold opens instantly, and native aggregation. Recall@10 measured **0.99 / 0.96 / 0.96 at 1M / 10M / 100M vectors** in Cor's release gate. -**[Plugin Documentation](docs/PLUGINS.md)** +Open core, commercial accelerator: Brainy is MIT and complete on its own; Cor is licensed and funds both. ---- +## Performance -## Type System +- JS distance kernels: **~6× faster cosine, ~1.4× euclidean** than 7.x (measured: [`tests/benchmarks/distance-microbench.mjs`](tests/benchmarks/distance-microbench.mjs), 384-dim, median of 41). +- Whole-graph reads are single **O(N + E)** cursor walks — a consumer-measured 19k-edge export dropped from ~27 s of per-node calls to one scan. +- Full numbers and capacity planning: **[docs/PERFORMANCE.md](docs/PERFORMANCE.md)** · **[docs/SCALING.md](docs/SCALING.md)** -42 noun types and 127 verb types form a universal knowledge protocol: +## Use cases -``` -42 Nouns × 127 Verbs = 5,334 base relationship combinations -``` - -Model any domain — healthcare (`Patient → diagnoses → Condition`), finance (`Account → transfers → Transaction`), education (`Student → completes → Course`), or your own. - -### Subtypes — sub-classification within a NounType *or* VerbType - -Both noun types and verb types are intentionally coarse. Use the top-level `subtype` field to sub-classify entities AND relationships within a type — flat string, no hierarchy, your choice of vocabulary: - -```javascript -// Nouns: sub-classify entities -await brain.add({ - data: 'Avery Brooks — runs the AI lab', - type: NounType.Person, - subtype: 'employee' // 'customer', 'vendor', 'contractor', … -}) - -// Verbs: sub-classify relationships -await brain.relate({ - from: ceoId, - to: vpId, - type: VerbType.ReportsTo, - subtype: 'direct' // 'dotted-line', 'matrix', … -}) - -// Filter on the fast path — column-store hit, not metadata fallback: -const employees = await brain.find({ type: NounType.Person, subtype: 'employee' }) -const directReports = await brain.related({ from: ceoId, subtype: 'direct' }) - -// O(1) counts via the persisted rollups: -brain.counts.bySubtype(NounType.Person) -// → { employee: 12, customer: 847, vendor: 34 } - -brain.counts.byRelationshipSubtype(VerbType.ReportsTo) -// → { direct: 12, 'dotted-line': 3 } -``` - -**Enforce the pairing.** Register a vocabulary per type or turn on brain-wide strict mode to ensure every entity AND relationship has both `type` AND `subtype`: - -```javascript -// Per-type rule with a closed vocabulary -brain.requireSubtype(NounType.Person, { values: ['employee', 'customer'], required: true }) - -// 8.0 default: every write requires a subtype. Exempt genuine catch-all types… -const brain = new Brainy({ requireSubtype: { except: [NounType.Thing] } }) - -// …or opt out while migrating pre-8.0 data, then audit and back-fill: -const legacy = new Brainy({ requireSubtype: false }) -await legacy.audit() // gaps, grouped by type -await legacy.fillSubtypes({ [NounType.Person]: 'unspecified' }) // close them -``` - -For other facets you want counted (`status`, `source`, `role`), register them with `brain.trackField(name)`. Renaming an existing convention to `subtype`? Use `brain.migrateField({from, to, entityKind: 'both'})` to walk nouns AND verbs in one pass. Full guide: **[Subtypes & Facets](docs/guides/subtypes-and-facets.md)**. - -**[Noun-Verb Taxonomy](docs/architecture/noun-verb-taxonomy.md)** | **[Stage 3 Canonical Reference](docs/STAGE3-CANONICAL-TAXONOMY.md)** - ---- - -## Storage: Memory and Filesystem - -The same API at every scale. Change one config line to go from prototype to production. - -### Development — Zero Config - -```javascript -const brain = new Brainy() -``` - -### Production — Filesystem (gzip compression on by default) - -```javascript -const brain = new Brainy({ - storage: { type: 'filesystem', path: './data' } -}) -``` - -### Backups and Portability — Snapshots - -```javascript -const db = brain.now() -await db.persist('/backups/2026-06-11') // instant hard-link snapshot -await db.release() - -const snapshot = await Brainy.load('/backups/2026-06-11') // read-only Db -const hits = await snapshot.find({ query: 'quarterly invoices' }) -await snapshot.release() -``` - -A snapshot directory is self-contained — copy it to another machine, open it with `Brainy.load()`, or restore it wholesale with `brain.restore(path, { confirm: true })`. - -Performance benchmarks and capacity planning in **[docs/PERFORMANCE.md](docs/PERFORMANCE.md)**. - -**[Capacity Planning](docs/operations/capacity-planning.md)** - ---- - -## Use Cases - -- **AI agents** — Persistent memory with semantic recall and relationship tracking -- **Knowledge bases** — Auto-linking, semantic search, relationship-aware navigation -- **Semantic search** — Find by meaning across codebases, documents, or media -- **Enterprise knowledge** — CRM, product catalogs, institutional memory -- **Interactive experiences** — Game worlds, NPCs, and characters that remember -- **Content platforms** — Similarity-based discovery, intelligent tagging - ---- +**AI agent memory** — persistent semantic recall with relationship tracking · **Knowledge bases** — auto-linking and meaning-aware navigation · **Semantic search** over codebases, documents, media · **Enterprise data** — CRM, catalogs, institutional memory · **Games & simulations** — worlds and characters that remember. ## Documentation -### Start Here - -- **[Brainy explained simply](docs/eli5.md)** — Plain-language overview, no jargon, no code - -### Core - -- **[API Reference](docs/api/README.md)** — Every method with parameters, returns, and examples -- **[Data Model](docs/DATA_MODEL.md)** — Entity structure, data vs metadata -- **[Query Operators](docs/QUERY_OPERATORS.md)** — All BFO operators with examples -- **[Find System](docs/FIND_SYSTEM.md)** — Natural language find() and hybrid search - -### Architecture - -- **[Architecture Overview](docs/architecture/overview.md)** — System design and components -- **[Triple Intelligence](docs/architecture/triple-intelligence.md)** — Vector + graph + metadata unified query -- **[Noun-Verb Taxonomy](docs/architecture/noun-verb-taxonomy.md)** — Universal type system -- **[Data Storage Architecture](docs/architecture/data-storage-architecture.md)** — Type-aware indexing and HNSW - -### Virtual Filesystem - -- **[VFS Quick Start](docs/vfs/QUICK_START.md)** — Build file explorers that never crash -- **[VFS Core](docs/vfs/VFS_CORE.md)** — Full VFS API reference -- **[Semantic VFS](docs/vfs/SEMANTIC_VFS.md)** — AI-powered file navigation - -### Guides - -- **[Import Anything](docs/guides/import-anything.md)** — CSV, Excel, PDF, URLs -- **[Framework Integration](docs/guides/framework-integration.md)** — React, Vue, Angular, Svelte -- **[Natural Language Queries](docs/guides/natural-language.md)** — Master the find() method - -### Operations - -- **[Capacity Planning](docs/operations/capacity-planning.md)** — Memory, storage, and scaling -- **[Performance](docs/PERFORMANCE.md)** — Benchmarks and architecture details - ---- +| Start | Core | Going deeper | +|---|---|---| +| [Brainy explained simply](docs/eli5.md) | [API reference](docs/api/README.md) | [Architecture overview](docs/architecture/overview.md) | +| [Installation](docs/guides/installation.md) | [Data model](docs/DATA_MODEL.md) | [Consistency model](docs/concepts/consistency-model.md) | +| [Natural-language queries](docs/guides/natural-language.md) | [Query operators](docs/QUERY_OPERATORS.md) | [Multi-process model](docs/concepts/multi-process.md) | +| | [Find system](docs/FIND_SYSTEM.md) | [Scaling](docs/SCALING.md) | ## Requirements -**Bun 1.0+** (recommended) or **Node.js 22 LTS** +**Bun ≥ 1.1** (recommended) or **Node.js ≥ 22**. Brainy 8.x is server-only; the 7.x line remains on npm for browser use. -```bash -bun install @soulcraft/brainy # Bun — best performance -npm install @soulcraft/brainy # Node.js — fully supported -``` +## Contributing & license -> Brainy 8.0 is server-only. Browser support (OPFS storage, Web Workers, in-browser WASM embeddings) was removed in 8.0 — the 7.x line remains available on npm if you need it. - -## Single-Writer Model - -Brainy is **single-writer, many-reader** on filesystem storage. One writer -holds an exclusive lock on the data directory; any number of readers can -inspect it concurrently. Opening a second writer throws with the PID of the -existing one. - -```typescript -// Live application — writer mode is the default -const brain = new Brainy({ storage: { type: 'filesystem', path: '/data/brain' } }) -await brain.init() - -// Out-of-band diagnostics from a separate process — safe to run while the -// writer is live -const reader = await Brainy.openReadOnly({ - storage: { type: 'filesystem', path: '/data/brain' } -}) -await reader.requestFlush({ timeoutMs: 5000 }) -const stats = await reader.stats() -``` - -For incident debugging, use the `brainy inspect` CLI: - -```bash -brainy inspect stats /data/brain -brainy inspect find /data/brain --where '{"entityType":"booking"}' -brainy inspect explain /data/brain --where '{"entityType":"booking"}' -brainy inspect health /data/brain -``` - -See [the multi-process model](docs/concepts/multi-process.md) and the -[inspection guide](docs/guides/inspection.md) for the full story, including -stale-lock detection and the cross-process flush RPC. - -## Contributing - -We welcome contributions! See **[CONTRIBUTING.md](CONTRIBUTING.md)** for guidelines. - -## License - -MIT © Brainy Contributors +Contributions welcome — see **[CONTRIBUTING.md](CONTRIBUTING.md)**. MIT © Brainy Contributors. diff --git a/RELEASES.md b/RELEASES.md index e95dc9a7..7799c6f6 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -8,16 +8,1176 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so - Debugging data, query, or storage behaviour - A new Brainy feature is available that you want to adopt +## Removed APIs — 7.x → 8.x (the complete ledger) + +Every public API removed at the 8.0 major, with its sanctioned replacement. If your code +still calls a left-column name on 8.x it throws (or the config key is rejected) — the +replacement is always a one-line change. (Standing contract from 8.9.0 forward: removals +happen only at majors, after ≥1 minor of loud runtime deprecation naming the replacement.) + +| Removed (7.x) | Replacement (8.x) | +|---|---| +| `brain.search(query, k)` | `find({ query })` — semantic; `find({ query, searchMode })` for hybrid | +| `brain.getRelations({...})` | `related(id, opts)` for adjacency; `find({ connected: {...} })` for scoped traversal | +| `brain.neural()` clustering | `find({ vector })` + aggregation `GROUP BY` | +| `Db.search()` | `db.find({ vector })` | +| Pre-8.0 storage path aliases (`directory`, `basePath`, …) | one `storage.path` key (old aliases throw) | +| Reserved keys inside `metadata` bags (silently remapped in 7.x) | top-level params (`subtype`, `visibility`, `confidence`, `weight`, …) — reserved-in-bag throws | +| 7.x COW branches layout (`branches/main/`) | generational MVCC (`asOf()`, `now()`, `db.persist(path)`) — on-disk migration is automatic at first 8.x open | + +The fork/snapshot family (`brain.snapshot()`, `createSnapshot()`, `restoreSnapshot()`) +is sometimes cited as a 7.x removal — those methods never existed on 7.x; the 8.0 Db API +(`asOf`/`persist`/`restore({confirm})`) is their first real implementation. + --- -## v8.0.0 — release candidate (unreleased, branch `feat/8.0-u64-ids`) +## v8.9.0 — 2026-07-19 (flush is durability-only: history maintenance moves to close()) + +The write path stops paying maintenance costs — the last structural piece of the +flush-storm class (a production deployment measured single writes blocked 25–191s behind +history reclaim running inline on flush under memory pressure): + +- **`flush()` never compacts history.** It persists the current window's deltas and + nothing else — its cost no longer depends on history backlog or retention mode, in any + configuration. **`close()` is the auto-compaction site** (time-bounded per pass, ~5s; + an early stop is a consistent prefix and the next pass resumes). +- **`compactHistory()` gains `timeBudgetMs`** — bound your own maintenance windows; the + same resumable-prefix guarantee applies. +- **The documented trade**: a long-lived writer that never closes accumulates history + until its next explicit `compactHistory()`. Predictable writes, explicit maintenance. + If you run bounded retention on an always-on service, schedule a periodic + `compactHistory({ ...caps, timeBudgetMs })` in your maintenance window. +- **New public doc: `docs/performance-envelopes.md`** — measured per-op envelopes + (p50/p95 at stated scales, hardware, and backend, with the measuring script cited). + Refresh rule going forward: any release touching a measured path re-runs that op's + benchmark and updates the envelope in the same release. +- **New in this file: the Removed APIs 7.x→8.x table** (top of this document) — every + removal with its sanctioned replacement, one place, per the engine-currency contract. + Standing from here: removals only at majors, after ≥1 minor of loud runtime deprecation. + +## v8.8.2 — 2026-07-19 (one field-resolution law: reserved-field aggregates stop drifting) + +Four fixes from a consumer conformance audit, all rooted in the same disease — two field-resolution +regimes where there must be one: + +- **Aggregates grouped by a RESERVED field (`subtype`, `visibility`, …) now decrement on + delete.** The delete/update hooks fed the aggregation engine a partial entity view (type, + service, data, metadata only), so a reserved-field `groupBy` resolved to a nonexistent group + on the way DOWN — counts drifted upward forever after any delete, and updates that moved an + entity between reserved-field groups double-counted it. The hooks now pass the full-fidelity + entity view (every reserved field top-level, the same shape the add path uses). If your + deployment derives stats from reserved-field aggregates, re-define those aggregates once + after upgrading (a changed definition triggers one rescan) or run them fresh — the drifted + persisted counts do not self-heal retroactively. +- **Aggregation `source.where` on reserved fields now filters** instead of silently matching + nothing: the matcher resolves fields through the same resolver `groupBy` uses (top-level + standard fields + custom metadata), so `where: { subtype: 'note' }` means what it says. +- **`removeMany()` refuses empty/invalid selectors loudly.** A bare array passed positionally + (`removeMany([id])` instead of `removeMany({ ids: [id] })`), an empty params object, or + `ids: []` used to resolve successfully having deleted nothing. All three now throw. +- **`find()` accepts both where-key spellings.** Metadata is flattened at index time + (`metadata.entry.title` indexes as `entry.title`); a `metadata.`-prefixed where key now + falls back to its flattened spelling when the prefixed one isn't indexed — the + "unindexed field(s), returning []" confusion for storage-shaped spellings is gone. (A + literal nested custom key named `metadata` still wins when indexed as spelled.) + +## v8.8.1 — 2026-07-18 (flush no longer walks the whole generation history + the import dedup off-switch is now honest) + +### The flush-storm fix (production incident, reported by a long-running deployment) + +Under the default adaptive retention, **every `flush()` re-walked the entire committed +generation history** to compute total history bytes for the budget check — O(all +generations) with disk re-reads past the 4,096-entry delta-cache bound. On a brain with +70,000+ accumulated generations that turned every write into a full-tail scan (60-100s +writes), even though the budget (free-RAM-based) never tripped and nothing was ever +reclaimed. Fixed: + +- `historyBytes()` now maintains a **running total**: seeded by one walk on first use, + then updated incrementally at every commit and reclaim — the adaptive retention check + on every flush is O(1). Invariant regression-pinned (running total ≡ fresh walk through + both commit paths and compaction). +- New **`brain.historyStats()`** (read-only, exported `HistoryStats`): generation count, + total on-disk bytes, generation/timestamp range, compaction horizon, retention mode, + and the effective adaptive budget — the one-call fleet-audit for sizing retention + exposure per brain. +- Interim guidance for keep-everything deployments already affected: `retention: 'all'` + skips the adaptive accounting entirely (and is the correct policy if you never want + history reclaimed). The accumulated files are harmless at rest; this release removes + the per-write cost of their existence. + +### The import dedup off-switch (lifecycle honesty) + +The post-import background deduplication pass (a merge-DELETE writer that runs ~5 minutes +after an import, merging entities judged duplicates by id / name / vector similarity) had +three lifecycle defects, all fixed: + +- **`enableDeduplication: false` now actually disables it.** The background pass was + scheduled unconditionally — an import that explicitly opted out could still have + entities auto-removed 5 minutes later. The flag now gates BOTH the inline merge and + the background pass (regression-pinned). +- **One deduplicator per brain, owned by the brain.** Each `import()` call constructed its + own coordinator + deduplicator, so the "debounced" timer never actually debounced across + imports (N imports = N delete timers). The brain now owns a single instance — the + debounce genuinely spans imports — and `close()` cancels pending work, so a delete pass + can never fire against a closed brain. +- **The 5-minute timer is unref'd** — a pending pass no longer holds the process open + (the exit-hang class; this timer had escaped the earlier sweep). + +Retention note for keep-everything deployments: with `enableDeduplication: false` on +import calls and `retention: 'all'` in config, no engine path removes records +automatically. + +## v8.8.0 — 2026-07-17 (OS-limit detection for pool-scale deployments) + +Small minor: brains now detect the two OS limits that bite at pool scale and warn **before** +the incident instead of during it. + +- At open (once per process, Linux-only, measurement-only), Brainy reads `RLIMIT_NOFILE` + (soft/hard, from `/proc/self/limits`) and `vm.max_map_count`, and warns loudly when either + sits below the pool-scale floors (soft NOFILE < 65 536; max_map_count < 262 144) — with the + exact raise commands (`ulimit -n` / `LimitNOFILE=` / `sysctl vm.max_map_count`). On stock + defaults the failure otherwise arrives as `EMFILE` or a failed mmap deep inside an index + open, long after the real cause stopped being visible. An unreadable limit produces **no** + warning — no measurement, no claim (non-Linux platforms stay silent). +- Exported for ops doors: `checkOsLimits()` returns the full `OsLimitsReport` + (values + warnings) programmatically, with the floors exported as constants. + +## v8.7.1 — 2026-07-17 (writer-lock acquisition is race-proof + machine-readable through init) + +Two hardenings of the multi-process writer lock (the `locks/_writer.lock` lease that makes a +second writer on the same brain directory fail loudly): + +- **Lock acquisition claims atomically.** The acquire path used read-then-write, leaving a + window where two processes racing an *absent* lock could both "succeed" — and the loser + kept running unlocked, silently. The claim is now an atomic create-exclusive write + (`O_EXCL`): exactly one racer wins; the loser re-evaluates and either fails loudly with + the winner's details or performs a verified stale-takeover. Bounded retries; contention + beyond them fails loudly rather than degrading into a lockless open. +- **`BRAINY_WRITER_LOCKED` survives `init()`.** The conflict error documents a + machine-readable contract (`err.code`, `err.lockInfo` with the holder's pid/host/ + heartbeat), but init's error wrapping silently stripped both, leaving consumers a message + to regex against. The error now passes through unwrapped. + +Measured while verifying (for operators sizing audits): `brain.auditGraph()` at a +production-consumer scale of ~2,600 relationships / 800 entities costs ~0.1 s warm and +~0.5 s cold, with exact scar counting across reopen. + +## v8.7.0 — 2026-07-17 (bulk-transact ergonomics: scaled budgets + timeout telemetry) + +The bulk-import ergonomics release, from a consumer's measured production incident (a serial +import on network-attached storage at ~2 s/op met a flat 30 s transaction budget): + +- **The transact apply budget now scales with the batch** — `max(30 s, opCount × 2 s)` — or + is exactly what you pass as the new `TransactOptions.timeoutMs`. A flat 30 s cap silently + limited honest bulk work to ~15 operations on slow disks while looking generous for small + batches. Internal batch paths (e.g. `removeMany` chunks) get the same scaling. +- **`TransactionTimeoutError` is a diagnosis, not just a failure**: it now reports the + operation it stopped at as `i/N` with the operation's name, elapsed vs budgeted time, and + states the batch rolled back atomically and is retryable. Its `context` carries the same + fields programmatically. +- **The transact envelope is documented** — batch sizing, budget math, chunking with + `ifAbsent` idempotency, and the precompute pattern (`embedBatch` + per-op `vector`) that + keeps model inference out of the commit path. Guide: `docs/guides/optimistic-concurrency.md`. +- Note: `brain.embed()` / `brain.embedBatch()` (the precompute APIs) already ship — public, + with native-provider passthrough, verified end-to-end (batch and single paths produce + bit-identical vectors; a vector-supplied `add` is fully searchable). Honest measurement: + on the default WASM engine, batch throughput ≈ sequential (~160 ms/text) — the precompute + win is keeping inference out of the budgeted commit path, not raw embedding speed. + +## v8.6.0 — 2026-07-17 (brain.auditGraph — the graph-truth verification instrument) + +A minor release adding one new public API, from the fleet's graph-trust program: a read-only +audit that **proves whether relationship reads return stored truth** on a given brain. + +- **`brain.auditGraph(options?)`** walks every canonical relationship record, queries the same + read path applications use (`related()` / VFS `readdir`) with all visibility tiers included, + and classifies every discrepancy into its failure family: `missingFromReads` (records the + read path omits — a stale adjacency index), `danglingEndpoints` (relationships whose endpoint + entity no longer exists — the historical partial-delete scar class), and `readOnlyVerbIds` + (read-path edges with no stored record — ghosts). Design-hidden internal/system edges are + counted separately so intentional hiding is never misclassified as loss. Counts are exact; + example lists cap at `maxExamples` with an explicit `truncatedExamples` flag; the result is + narrated loudly on incoherence. Mutates nothing — safe on a live brain. +- The operational pairing: audit → if incoherent, `repairIndex()` → audit again. A `coherent` + report after the repair is the verified all-clear. Run it after any engine upgrade, restore, + or migration. Guide: `docs/guides/inspection.md`. +- Also: `getNounIds` pagination now refuses an undecodable resume cursor loudly (the same + contract `getNouns`/`getVerbs` gained in 8.5.2 — the third and final walk brought under it). +- Types exported: `GraphAuditReport`, `GraphAuditDiscrepancy`. + +## v8.5.2 — 2026-07-17 (aggregation backfill: exception-safe, generation-verified, and loud) + +Hardening patch from a migration incident (a byte-copied store on new hardware; the service +entered a silent full-CPU loop at boot). Four changes, all in the aggregation engine's +backfill/adoption path: + +- **Backfill walks are exception-safe and non-destructive.** A rescan now builds into a + staging map and swaps in atomically on completion; a mid-walk failure drops the staging map, + keeps the previous live state serving, and surfaces the storage error to the failing query. + Previously the walk wiped live state *before* a scan that could throw, never cleared the + pending flag on failure, and re-ran a full walk on every subsequent query — a silent + wipe/walk/throw loop at the caller's retry rate. +- **Failed walks are latched.** After a walk fails, retries within a 30-second cooldown rethrow + the recorded error instantly instead of re-walking — a tight caller-side retry loop now costs + one loud error per query, never a full store walk per query. +- **Adoption is generation-verified.** Persisted aggregation state is stamped with the store's + committed generation at flush; reopen adoption requires the stamp to equal the current + watermark. Stale state (unclean shutdown) or over-counting state (a fact-log truncation on a + copied store pulled the watermark back) triggers exactly one loud rescan — never a silent + adopt. Pre-8.5.2 state on generation-aware stores rescans once after upgrade, then is stamped. +- **The path narrates.** Adoption decisions, no-adoptable-state outcomes, walk start/finish + (entity count + duration), and walk failures all log by default; a non-advancing storage + pagination cursor aborts the walk loudly instead of looping forever. + +Plus three guards from a full audit of every loop in the open/init path: + +- **Invalid pagination cursors fail loudly.** A supplied-but-undecodable resume token to + `getNouns`/`getVerbs` used to silently restart the walk at offset 0 — to a `while(hasMore)` + caller that re-serves page 1 forever (an unbounded silent CPU loop). It now throws with a + clear message instead. +- **The graph cold-load verb walk has a stall guard.** `hasMore=true` with a missing or + non-advancing cursor aborts loudly instead of re-reading the same page forever. +- **A derived index AHEAD of the store is named at open.** Brainy already surfaced a provider + generation *behind* the committed watermark; the *ahead* direction (the signature of a + byte-copy of a live service, or a log truncation during crash recovery) now logs a loud + warning explaining what happened and that `brain.repairIndex()` forces a heal — instead of + passing unnamed into whatever the derived index does next. + +## v8.5.1 — 2026-07-17 (aggregation state survives restarts + the query-cap ratchet removed) + +Patch release from a production incident (aggregate/count paths taking 40–90 s on an idle box +while vector search stayed fast, and every `find({ limit: 5000 })` suddenly failing against an +"auto-configured query limit of 1000"). Three fixes, one cosmetic: + +- **Aggregation state is actually adopted on reopen.** The boot pattern `defineAggregate()` → + query raced the engine's async state load: the synchronous define always won, flagged a + backfill, and the first query then wiped the just-loaded persisted state and re-walked the + entire store — every restart, forever. Reopening with an unchanged definition now adopts the + persisted state directly (zero scans); a backfill runs only on a real definition change, a + missing/failed state load, or a write that landed before adoption (exactness wins). Apps that + rely on persisted definitions without re-defining at boot also no longer race a spurious + "Aggregate not defined". +- **Backfills are single-flight and batched.** Concurrent queries on a cold aggregate used to + each wipe the others' partial state and start their own full store walk — under steady query + arrival the store never converged (the 40–90 s loop). Now all concurrent queries share one + walk, and one walk fills every aggregate pending backfill (M aggregates ≠ M scans). +- **The query-cap "learning" ratchet is removed.** `maxLimit` is a memory-protection bound, but + a hidden tuner shrank it 20 % per recorded query while the lifetime-average query time + exceeded 1 s — down to a floor of 1000, below the documented 10 000 auto floor, with no + practical recovery, and the resulting error blamed "available free memory" (stale basis + label). The cap now comes from its construction-time basis (or your explicit + `maxQueryLimit` / `reservedQueryMemory`) alone and never changes at runtime; query timing is + recorded for diagnostics only. +- **Cosmetic:** the engine's own persistence keys (`__aggregation_*`, `brainy:entityIdMapper`) + no longer log `[Storage] Unknown key format` at boot — they were always routed correctly; + they're now recognized before the warning fires. + +Operationally: if a host was bitten, upgrading and restarting is the whole fix — no repair +ritual needed. Setting `maxQueryLimit` explicitly remains the valve that bypasses auto-detection +entirely. + +## v8.5.0 — 2026-07-15 (provider access to the fact log + the shared stamp verifier) + +Small additive follow-up to 8.4.0, from the native accelerator's first consumption pass: + +- **Index providers can now reach the fact log through the storage adapter** — new optional + capability `storage.scanFacts()` / `storage.factLogHeadGeneration()` / `storage.factSegmentPaths()`, + wired by the host brain at init as a closure over its live log. Providers hold only `storage` and + must never construct their own fact-log reader (the log's open path is writer-side); `null` means + "no fact log here — use the enumeration walk." +- **The family-stamp verifier is shared** via `@soulcraft/brainy/internals` + (`readFamilyStamp` / `writeFamilyStamp` / `verifyFamilyStamp` + types) so first-party native + providers run literally the same verification function, never a synchronized copy. +- **Rollup stamp invariants accept strings** (`number | string`) — content fingerprints such as a + per-tree SHA-256 are valid invariant values; a type mismatch reads as incoherence, never a pass. +- **`storage.committedGeneration()`** — the committed watermark as a capability, so a provider + compares its stamp's `sourceGeneration` against the store's truth without parsing the private + manifest format. +- **Two durability/stability contracts pinned in the suite:** fsync-before-ack (holds for + `transact()` today; the single-op path is pinned as the documented future target — group commit + becomes latency batching, never durability skipping) and scan-stability-under-rotation (a scan + snapshot yields exactly its facts — no gaps, duplicates, or bleed-in — while segments rotate + beneath it). + +No behavior change for applications; all additions. + +## v8.4.0 — 2026-07-15 (the generation fact log — a sequential, self-verifying commit stream) + +A minor release, fully backward-compatible (all additions; no behavior change for existing APIs). +This is infrastructure: it changes nothing about how you query today, and lays the substrate that +makes index heals and incremental catch-up sequential-read problems instead of directory walks. + +- **Every committed write now also appends a "fact" — an after-image commit record.** Alongside the + existing before-image history, each committed generation (single-op and `transact()` alike) appends + what each touched entity/relationship *became* — or a body-less tombstone for a removal — to an + append-only, checksummed segment log under `_generations/facts/`. Crash-safe by construction: a + torn tail is detected and ignored; on open the log is reconciled to committed truth, so an absent + generation always means "never committed." `transact()` facts are durable when `transact()` + returns; single-op facts ride the same group-commit flush as their history. + +- **New: `brain.scanFacts()`** — stream committed facts in commit order, in batches, with heal-grade + telemetry (total scope up front; per-batch generation range, byte size, and segment; a summary + cross-check at the end; loud abort on any gap — never a silent skip). **`brain.factSegmentPaths()`** + hands zero-copy consumers the immutable sealed segment files directly. New exported types: + `CommitFact`, `FactOp`, `FactScanBatch`, `FactScanHandle`. Facts accumulate from the first write + after upgrading — pre-existing history is not retroactively converted (enumeration remains the + fallback for old data). + +- **New: the entity-tree family stamp.** At every flush/close, brainy stamps which committed + generation the canonical entity files reflect plus the rollup invariants (entity/relationship + counts) that verify the tree whole. At open, coherence is a comparison — a genuine divergence is + loud and names the failing invariant; `repairIndex()` recounts from canonical and re-stamps. New + exports: `readFamilyStamp`, `verifyFamilyStamp`, `ENTITY_TREE_STAMP_PATH`, `FamilyStamp` types. + +- **Storage adapters** gain optional binary raw-byte primitives (`appendRawBytes`, `readRawBytes`, + `writeRawBytes`, `rawByteSize`) — feature-detected; the filesystem and memory adapters implement + them; an adapter without them simply hosts no fact log. The fact-log namespace is registered as a + protected family: no sweeper or GC can delete under it. + +No API breaks. 24 new tests; the full commit-path regression suite is green. + +## v8.3.3 — 2026-07-15 (rename moves the containment edge — no ghost in the old directory) + +One production-reported fix plus a repair path, completing the delete/move hygiene arc (8.3.1 fixed +deletes, 8.3.2 fixed counters, this fixes moves). + +- **A cross-directory `vfs.rename()` now MOVES the containment edge instead of accumulating one per + parent.** The old parent's `Contains` edge was never removed on a move, leaving the entity a child + of **both** directories: `readdir(oldDir)` kept listing it after the move, re-creating the old path + showed the same name twice, and any tree-walking consumer (sync engines, file browsers) saw the + file in two places. The old edge is now removed by edge id, resolved from the graph's own adjacency + — a removal never requires reading the thing being removed. Bonus fix in the same seam: a move **to + the root** now gets its containment edge (it was previously skipped, orphaning the file out of + `readdir('/')`). + +- **`repairIndex()` now also reconciles VFS containment** (new `vfs.repairContainment()`): every VFS + entity's containment edges are checked against its canonical `metadata.path` — stale old-parent + ghosts and duplicate edges are removed, a missing expected edge is restored, and user + knowledge-graph edges are never touched (only `vfs-contains` edges are candidates). Loud per + repair. Stores that performed cross-directory renames under ≤8.3.2 should run `brain.repairIndex()` + once after upgrading — the same single ritual now heals orphan directories, counters, **and** + containment edges. + +- Also ships a permanent lens-consistency regression suite (combined type+subtype vs subtype-only vs + canonical ground truth, id-for-id, warm and after a cold reopen), ported from the field + investigation that closed the historical lens-drop report. + +No API changes beyond the new optional `vfs.repairContainment()` (also invoked by `repairIndex()`). + +## v8.3.2 — 2026-07-14 (honest counters — the recount + removal-without-re-reading) + +Completes 8.3.1's delete-hygiene story at the counter layer, from a production proof chain reported +by a downstream deployment: persisted entity totals were permanently **inflated** — deletes whose +count decrement was silently skipped — and because paginated `totalCount` serves +`Math.max(persistedTotal, scanned)`, the inflated number always won and **no disk cleanup could ever +lower it**. + +- **A removal's count decrement no longer requires re-reading the record being removed.** The + decrement was sourced from re-reading the entity's metadata inside the delete; if that read + returned `null` (a replace race, or a ghost left by a pre-8.3.1 partial delete) the decrement was + silently skipped while the paired add had counted — minting drift on every write→delete→re-create + cycle. The caller's pre-delete read now rides through the whole delete path + (`remove()`/`removeMany()` → the delete operation → `deleteNoun`/`deleteVerb` → + `deleteNounMetadata`/`deleteVerbMetadata`, both sides symmetric): a null internal read falls back + to the known prior record instead of skipping. The `StorageAdapter` signatures gain an optional + `priorMetadata` parameter (additive; existing adapters unaffected). + +- **`repairIndex()` is the sanctioned counter recount — unconditional, and it actually persists.** + `rebuildTypeCounts()` previously rebuilt only the type-statistics arrays and computed the total + *just to log it* — the persisted scalar (`counts.json`) survived every "rebuild" untouched, so an + already-inflated brain could never be corrected. One canonical walk now rebuilds **every** counter + rollup — scalar totals, per-type maps, and type statistics — and persists them, and `repairIndex()` + runs it unconditionally (not only when orphan directories are found: counters can be inflated over + perfectly clean shelves). Brains with delete history should run `brain.repairIndex()` once after + upgrading; the correction survives reopen. + +No API breaks (optional-parameter additions only). Regression tests cover the drift cycle, the +null-read decrement fallback, and the persisted recount across reopen. + +## v8.3.1 — 2026-07-14 (full-removal deletes + family-scoped migration gate) + +Two production-reported fixes in the write/index spine, plus an operator repair path. No API changes; +all behavior changes make previously-wrong states honest. + +- **Deleting an entity now removes it completely — no more "ghost" leftovers on disk.** A canonical + noun delete removed the metadata (content) leg but left the entity's `vectors.json` and its `/` + directory behind. Consequences observed in a production deployment: deleted rows were + indistinguishable on disk from damage scars, enumerated counts inflated monotonically with every + delete (the leftovers were counted forever), and locator-style reads hit unreadable ghost rows. + `remove()`/`removeMany()`/`deleteNoun`/`deleteVerb` now remove **both legs and the entity + container**, with a full two-leg before-image rollback inside the transaction. The generation log + still holds the delete's before-image, so `asOf()` time-travel reconstructs deleted entities exactly + as before — this is live-HEAD hygiene, not a history change. This also fixes **duplicate `readdir` + entries for re-created VFS paths** at the root: with no ghost state, a delete always unposts its + index rows, so a delete→recreate cycle lists the path exactly once (regression-tested across + repeated cycles). + +- **`repairIndex()` prunes ghost/scar directories left by earlier versions.** Stores that deleted + entities under ≤8.3.0 may hold orphaned entity directories (a vector-only leg, or an empty dir). + `brain.repairIndex()` now sweeps them: it removes only containers with **no metadata content leg** + (never a directory that still holds content), logs every removal, and recomputes type/subtype + counts afterward so totals stop counting ghosts. + +- **Reads no longer hang behind an unrelated index migration (family-scoped gate).** During a native + provider's one-time background migration, *every* read — including plain `get()`, VFS + `readdir`/`readFile`, and metadata-only `find({ where })` — blocked on the whole-brain migration + lock until timeout, even when the migrating index was irrelevant to the read. The gate is now + scoped to the index families a read actually consults: canonical reads (`get`, `batchGet`, VFS + content) never wait; a `find` waits only on the families its query shape needs (vector for + semantic, metadata for `where`/type, graph for `connected`); graph traversals wait only on the + graph family. Writes and unclassified operations keep the conservative whole-brain wait. A read + that *does* need the migrating family still blocks (bounded by `migrationWaitTimeoutMs`) and + surfaces the retryable `MigrationInProgressError` — never a partial result. + +No breaking API change. Each fix ships with regression tests. + +## v8.3.0 — 2026-07-13 (faster index heals + the cross-layer integrity contract) + +Three additive changes. The first is an immediate, standalone performance win; the other two are the +brainy side of the write/index-spine integrity contract, inert until a native accelerator +that implements the matching hooks is present — so this release changes nothing for a JS-only brain +beyond the speedup. + +- **Canonical enumeration is up to ~16× faster — the dominant term in an index heal.** The paginated + entity walk (`getNounsWithPagination`) hydrated each entity's vector + metadata one-at-a-time; since + every index rebuild enumerates canonical storage, that serial per-item latency dominated multi-minute + heals. Hydration is now 16-way bounded-concurrency, with the pagination contract (order, cursor + resume, filters, totalCount) byte-identical to before. New **`getNounIdsWithPagination()`** returns + ids without hydrating anything (zero per-entity reads when unfiltered) for callers that own their own + IO schedule. + +- **Cross-layer integrity — `validateIndexConsistency()` is no longer blind to native providers.** It + only ran the JS metadata index's own check, so a native provider whose manifest/segments/counts had + diverged still read as "healthy". It now feature-detects and aggregates each provider's optional + `validateInvariants()` self-report, names every failing invariant with its numbers, and exposes the + per-provider reports. `repairIndex()` now reconciles native derived state from canonical too + (rebuilding any provider whose failing invariant asks for it). New exported types + `ProviderInvariantReport` / `InvariantResult` / `InvariantHeal`. + +- **Registered-blob families — declared index files are undeletable through the storage layer.** A + provider can declare a derived-index blob *family* (a set of members that are load-bearing together); + once declared, `deleteBinaryBlob` / `removeRawPrefix` refuse to remove a member (new exported + `ProtectedArtifactError`), so a stray in-process sweeper cannot delete a load-bearing index file. The + declaration persists across reopen; `checkDerivedFamiliesPresent()` names any member missing on open. + New optional `StorageAdapter` surface (`registerDerivedFamily` / `unregisterDerivedFamily` / + `listDerivedFamilies` + `DerivedFamilyDeclaration`) and exported `DerivedArtifactMissingError`. + +No breaking API change (all additions are optional/new). Each change ships with regression tests. + +## v8.2.8 — 2026-07-13 (honest index readiness — no more silently-empty queries on a cold index) + +Closes the last of the three spine anti-patterns: the "dishonest readiness proxy," where `size() > 0` +was treated as "this index actually serves queries." On a cold open (fresh boot, restart, crash +recovery) a native index can load its **count** before its **serving structure** — so for a brief +window it has data but cannot answer, and a query returned a silent empty result indistinguishable +from "no such data." Every fix here asks the index whether it can *actually serve*, and if not, +self-heals or fails loudly instead of returning `[]`. + +- **Semantic search no longer returns a silent `[]` on a cold vector index.** A pure semantic + `find({ query })` has no filter, so nothing previously guarded the vector index. A new one-shot + guard verifies the vector index serves a known persisted vector on the first semantic/proximity + search — preferring the provider's honest `isReady()` signal, else a known-vector self-match probe. + It rebuilds from canonical records if the serving structure did not load, and throws the new + **`VectorIndexNotReadyError`** only if a rebuild still cannot serve — never a silent empty result. + +- **Relationship reads fall back to the canonical scan instead of an empty result on a cold graph + index.** `getVerbsBySource`/`getVerbsByTarget` (used by relationship queries and virtual-filesystem + traversal) skipped the fast path only on `isInitialized` — which reads true once the manifest loaded + even if the source→target adjacency did not. They now consult the honest readiness signal and, when + the adjacency is not serving, take the correct-but-slower canonical shard scan. A one-shot self-heal + probe covers providers that expose no readiness signal. + +- **`getIndexStatus()` tells the truth for readiness probes.** It reported `populated: size>0` only, so + a Kubernetes readiness check could route traffic to a brain still warming up. It now folds in the + honest per-index `ready` signal (making `populated` honest), plus the degraded states already + surfaced by `checkHealth()`/`validateIndexConsistency()` (`rebuildFailed`/`rebuildError` and a + `degradedIds` count) — so a probe never reports 200-ready over a known-degraded index. + +This completes the write/index-spine hardening end to end. New export: **`VectorIndexNotReadyError`**; +`getIndexStatus()` gains additive fields (`rebuildFailed`, `rebuildError?`, `degradedIds`, per-index +`ready?`). No breaking API change. Each fix ships with a dedicated regression test. + +## v8.2.7 — 2026-07-13 (loadBinaryBlob fault-propagation — the lockstep completion of 8.2.6) + +Completes the Pass-1 spine hardening. `loadBinaryBlob` (the raw-blob read a native accelerator mmaps +for its index files) previously returned `null` on ANY read error, so a real IO fault +(EIO/EACCES/EMFILE) on a present-but-unreadable index blob masqueraded as "the blob is absent" — +driving a needless full rebuild or an empty read. It now distinguishes genuine absence (ENOENT → +`null`, the documented contract) from a real fault (→ throw), so a transient disk fault surfaces +loudly instead of silently degrading the index. + +This one change was deliberately held out of 8.2.6 and ships now, in lockstep with the native +accelerator release that hardened its two column-store read sites to handle the throw (a faulted +segment read marks the field unavailable and throws a named error, instead of relying on +null-on-error). A consumer on new brainy + an older accelerator was never at risk: 8.2.6 kept the +prior swallow-on-fault behavior for this method until the accelerator was ready. + +No API change. Regression: `tests/unit/storage/blob-save-durability.test.ts` gains the loadBinaryBlob +leg (absent → null; present → bytes; real fault → throws). + +## v8.2.6 — 2026-07-13 (write/index-spine hardening — loud errors, never quiet losses) + +Durability + integrity hardening across the write and index paths. Every fix converts a place that +could *silently* lose data, serve a partial result, or acknowledge a write that did not land into a +loud, observable failure. No breaking API changes; one new exported error. + +- **A durable blob write that stored nothing is no longer acknowledged.** The atomic blob write + (`saveBinaryBlob`, used for vector/graph index segments and the native index files) uses a unique + per-writer temp file, so a rename `ENOENT` can only mean *our* temp vanished before the rename — + the bytes never landed. It previously returned success on that path; it now retries once with a + fresh temp and, if the temp vanishes again, throws instead of acknowledging a write that persisted + nothing. A downstream deployment that mmaps these files no longer finds a "successfully written" + blob missing on the next open. + +- **Single-op history durability is enforced, not assumed.** The asynchronous group-commit flush + that persists single-op generation history previously swallowed a persist failure as a warning + while writes kept succeeding and their history piled up, undurable, in memory. It now tolerates a + transient blip (retry with capped backoff) and, after repeated failures, **refuses further writes** + with the new **`PendingFlushDurabilityError`** rather than promise a durability it cannot deliver. + Live canonical data is untouched; the latch self-heals the moment a flush succeeds. Callers needing + hard per-write durability should keep using `transact()` (or `flush()` after a single-op). + +- **A degraded derived index is surfaced on reads, not served silently.** When a non-fatal index + rebuild fails at open, or a write commits via adopt-forward recovery with an incomplete derived + index, `find()` and `get()` now emit one loud warning per degraded window (reads still return — + canonical is the source of truth), the state folds into `checkHealth()` / + `validateIndexConsistency()`, and `repairIndex()` reconciles and clears it. + +- **`clear()` removes the full derived footprint.** It previously left raw index blobs, the native + id-mapper, and column-index manifests on disk, so a cleared brain could re-read stale native state + on reopen. It now wipes them together as a set. + +- **Counts stay honest across deletes.** A delete decremented the per-type breakdown but not the + scalar total, so the total inflated permanently (and won pagination). Delete now decrements both; + the invariant `total === Σ per-type` holds across any interleaving of add / visibility-flip / + delete and across a reopen. + +- **Index-maintenance and aggregation failures are loud.** A partial LSM/segment load no longer + publishes the manifest's full count as if healthy; an HNSW flush that can't persist a node throws + (`HnswFlushError`) instead of returning a lying count and dropping the node; a corrupt or missing + manifest-listed column segment throws (`ColumnSegmentLoadError`) instead of silently dropping its + entities from every query; and aggregation materialization / state-load failures now warn instead + of vanishing into an empty catch. + +New export: **`PendingFlushDurabilityError`** (with `.cause` and `.failedAttempts`). No other public +API change. Each fix ships with a dedicated regression test. + +## v8.2.5 — 2026-07-12 (honest response when a transaction rollback can't complete) + +Data-integrity fix. When a transaction failed and its rollback then *also* failed to undo a +canonical write (retries exhausted), the old behavior logged, continued, and threw +`TransactionRollbackError` — while the record it couldn't undo stayed durable on disk, and the +transaction even reported its state as `'rolled_back'`. The caller got an error implying the write +was undone; a read-back showed the record. A failed rollback had no truthful response. + +Rollback now tells the truth, with a two-branch contract: + +- **Adopt-forward (safe case).** A single-op write (`add`/`update`/…) whose only damage is a + durably-present record — the write the caller asked for — is **adopted**: its generation is + committed, the write returns success, and a loud warning records that the derived index may be + incomplete for that id until the next rebuild/`repairIndex()`. No error, no double-write; the + record is immediately durable and retrievable by `get()`. +- **Fail loud + quarantine (unsafe case).** A multi-operation batch, or *any* case where a record + was lost (a remove/update whose restore-undo failed), throws the new **`StoreInconsistentError`** + naming every unreconciled record and its disposition (`orphan` vs `loss`), and puts the brain into + **write-quarantine**: reads keep working, but writes are refused until `repairIndex()` reconciles + the derived indexes against canonical storage and lifts the quarantine. The generation counter is + not advanced, and a transaction whose rollback failed is now `'inconsistent'`, never the + `'rolled_back'` lie. + +The decision is made by *observation*, not guesswork: both commit paths already hold byte-identical +before-images, so after a failed rollback the store compares current canonical state to them to +classify exactly which records are orphaned or lost. + +New export: `StoreInconsistentError` (with `.records` and `.cause`) and the `UnreconciledRecord` type. +No other API change; `repairIndex()` gains the quarantine-lift behavior. Regression +(`tests/integration/rollback-trapdoor.test.ts`) injects the exact failure (index add throws → +canonical delete-undo fails) and pins adopt-forward (durable, get-able, not quarantined), fail-loud +(`StoreInconsistentError` + quarantine + reads work + `repairIndex()` lifts it), and the error's +record naming. + +## v8.2.4 — 2026-07-12 (restore can no longer destroy the store it's recovering) + +Recovery-safety fix. `restore()` removed the entire live brain directory and THEN copied the +snapshot in — so any copy failure left the store destroyed with only a partial copy. The sharpest +edge: the copy (`fs.cp`) materialized the holes of sparse mmap blob files, so a snapshot that fits +on disk could balloon and `ENOSPC` mid-copy, and the recovery tool would have just destroyed the +brain it was asked to recover. Reported from a downstream incident's recovery forensics. + +Restore is now **non-destructive and crash-resumable**: + +- The snapshot is copied into a staging area (`_restore_staging/`) **before any live data is + touched**. The copy is **sparse-aware** — all-zero regions are left as holes, so a store of + mostly-hole blobs restores at its true allocated size instead of its apparent size. +- A copy failure (including `ENOSPC`) removes only the half-written staging area and throws; the + live store is left **exactly as it was**. +- Only after the copy succeeds and a completion marker is `fsync`'d does an **atomic per-entry + swap** move the staged data into place — same-filesystem renames that cannot fail for disk space. +- A crash mid-swap is finished **forward** on the next open: startup resumes a committed-but- + incomplete swap, or discards an uncommitted staging area (live data still authoritative). + +No API change — `restore(path, { confirm: true })` is unchanged. `persist()` was already safe +(hard-link snapshot). Regression (`tests/integration/restore-nondestructive.test.ts`): a forced +copy failure leaves live data fully intact, a normal restore round-trips, an interrupted-but- +committed restore completes on reopen, an uncommitted staging area is discarded, and the sparse +copy is byte-identical with allocation far below apparent size. + +## v8.2.3 — 2026-07-12 (a committed transaction is durable on return) + +Durability fix. A `transact()` reported "committed" while its canonical entity writes were still +only in the OS page cache (written via tmp+rename, not yet `fsync`'d), even though the generation +counter and manifest WERE fsync'd. A hard kill (power loss, SIGKILL) in that window could leave the +durable generation counter **ahead of** the persisted entity bytes — so a consumer resuming from the +counter would see "phantom progress": a generation that claims writes the disk never kept. Reported +from a downstream migration's crash-lifecycle forensics. + +`commitTransaction` now runs a **durability barrier**: it records every canonical write and delete +the batch's operations make, then `fsync`s that entire footprint (file contents, the rename +directory entries, and the parent directories of any deletes) **before** advancing the generation +counter and manifest. A committed transaction is therefore durable the moment `transact()` returns — +the counter can never outrun the entity bytes. + +**Durability contract, now explicit.** `transact()` is durable-on-return (above). A **single-op** +write (`add`/`update`/`remove`/`relate`/…) is Model-B group-commit: its live bytes are written but +become durable at the next `flush()` or `close()`, not the instant the call resolves — the counter +is buffered alongside the data, so a crash loses both together (never a torn counter-ahead-of-state +store). Need per-write durability? Use `transact()` (even for one op), or `flush()` after the write. +This deliberately trades single-op fsync latency (a 3-5x write regression) for throughput. + +No API change; no accelerator involvement (the barrier is in the filesystem storage adapter). In-memory +and durable-per-call (cloud object-PUT) adapters treat the barrier as a no-op. Regression: +`tests/integration/transact-durability-barrier.test.ts` proves entity writes fsync in an earlier +batch than the manifest, for single-op and multi-op (add+relate) transactions, and that a +precommit-rejected batch opens no barrier and advances nothing. + +## v8.2.2 — 2026-07-11 (P0: a timed-out transaction now rolls back — no torn state) + +Data-integrity fix. A transaction that exceeded its time budget **mid-flight** (e.g. a bulk +`transact()` on slower hardware crossing the 30s ceiling) threw a timeout error WITHOUT rolling +back the operations it had already applied. The budget check sat outside the per-operation +rollback path, so only per-operation *failures* rolled back — a timeout stranded the partial +writes in canonical storage while the generation was never stamped, leaving torn, generation-less +state. This was caught by a downstream migration that crossed the ceiling on a large batch. + +`Transaction.execute()` now has a **single rollback point**: any error that escapes the operation +loop — an operation failure OR a mid-flight timeout — rolls back every applied operation in +reverse order, then surfaces the original error (a rollback failure supersedes it, loudly, as +before). This restores the invariant the generation-store commit path already depended on: a throw +from execute means the applied operations were undone byte-identically, and the aborted +transaction leaves `generation()` unchanged and storage byte-identical to its pre-transaction +state. + +No API change. Regression pins the reported requirement — after a mid-flight timeout, +storage is byte-identical and the transaction ends in the `rolled_back` terminal state +(`tests/unit/transaction/timeout-rollback.test.ts`), plus the operation-failure and +rollback-failure paths through the same single rollback point. + +**Note on the 30s ceiling itself** (configurable/scaled timeout, batched embedding precompute, +timeout telemetry) — that ergonomics work is tracked separately; this release fixes only the +correctness bug (a timeout must never leave partial state), independent of where the ceiling sits. + +## v8.2.1 — 2026-07-11 (transact forward references work on the native accelerator) + +Parity fix. `transact()` has always promised atomic forward references — `add` an entity and +`relate` to it in one batch — but the transaction planner resolved relationship endpoint ids at +**plan** time, before the batch's adds had applied. Asking the id mapper about an entity that +doesn't exist yet made the accelerator's (correctly strict) native mapper throw in +`EntityIdMapper.getOrAssign`, so `transact([{op:'add', id:X}, {op:'relate', to:X}])` failed on +native deployments while the permissive JS mapper masked the bug — and silently leaked an id +assignment whenever a batch was later rejected by a commit precondition. + +Endpoint resolution is now **lazy** — evaluated when the graph operation executes inside the +commit, after the batch's adds have applied (the same lazy pattern the operation's generation +stamp already used). Fixed across all transact-planned graph operations: relate (including the +bidirectional reverse edge), remove's relationship cascade, and unrelate. Single-operation writes +are unchanged. Also fixed as a byproduct: a rejected batch no longer pollutes the id mapper. + +Regression suite covers the exact reported shape, both-endpoints-in-batch, bidirectional, +add+relate+remove in one batch, and the mapper-cleanliness proof on rejected batches +(`tests/integration/transact-forward-ref-graph.test.ts`). No API changes; no accelerator version +pairing required. + +## v8.2.0 — 2026-07-10 (temporal VFS — file content joins the immutability model) + +Time travel now covers Virtual Filesystem **content**. Previously the temporal model had a hole +exactly where files were concerned: every entity write was an immutable generation, but the +content *bytes* lived under an eager reference-count GC — deleting a file could physically destroy +bytes that in-window history still referenced, while overwriting never released the old content at +all (an unbounded silent leak that only *accidentally* preserved history). A production consumer's +recovery from a bad deploy succeeded only because a stale field happened to hold the good value — +luck, not a guarantee. Both directions are now fixed by making blob reclamation a **history** +decision instead of a **liveness** decision: + +- **Content blobs are retention-protected.** Each blob tracks live references AND history + references (one per persisted generation record that carries its hash). Deleting/overwriting a + file drops only the live reference; bytes are physically reclaimed in exactly one place — + history compaction — once no live reference and no retained generation references the hash. + Pinned views are exempt automatically (compaction already respects pins). Crash ordering is + over-count-only (never under-count), so a crash can leak until the built-in scrub recounts, but + can never reclaim bytes history still needs. Existing stores get a one-time, marker-gated + backfill on open; cross-file content dedup is handled exactly. +- **`vfs.readFile(path, { asOf })`** — the file's exact bytes as of a generation or `Date`, + guaranteed present within the retention window. +- **`vfs.history(path)`** — the file's versions (`{ generation, timestamp, hash, size, + mimeType? }`, ascending). Restore = read the old bytes `asOf` and write them back (a new write; + history is never rewritten). +- **Overwrites now refresh the file entity's `data`/embedding text** — previously semantic search + and the `data` field served the FIRST version's text forever. + +Lifecycle note: deleting a file no longer frees its bytes immediately — old content lives until +compaction reclaims its generations, under the same `retention` budget as all Model-B history +(`retention: 'all'` keeps every version forever). Guide: the new "Time travel for files" section in +`docs/guides/snapshots-and-time-travel.md`. Native accelerator: unaffected (canonical write path +only) — no version pairing required. + +## v8.1.0 — 2026-07-10 (`brain.onChange` — the in-process change feed) + +New public API: subscribe to every committed mutation with +`brain.onChange(cb) → unsubscribe`. One event per affected record, for **every** +canonical write regardless of origin — direct calls, batch methods, `transact()`, +imports, and Virtual Filesystem writes all funnel through the same commit point the +feed is emitted from. This is the authoritative in-process signal for live UIs, +cache invalidation, and realtime sync layers (downstream SDKs can forward it over +their own transports to make local and remote brains uniform). + +Event shape (`BrainyChangeEvent`, exported): `kind` (`entity`/`relation`/`store`), +`op` (`add`/`update`/`remove`/`relate`/`unrelate`/`updateRelation`/`clear`/`restore`), +the post-commit `entity` or `relation` view (type + full custom metadata), the +committed `generation`, and `timestamp`. Notable properties: + +- **Post-commit only** — an aborted write (a losing `ifRev` CAS, a rejected + transaction) never emits. If you got the event, the write is durable. +- **Deletes fully described** — `remove`/`unrelate` carry the record's last + committed state (from the commit's own history record), not just an id; batch + deletes included. +- **Batches emit per item**; a `transact()` batch's events share its single + generation. Entity deletes also emit `unrelate` for each cascaded relationship. +- **Commit-ordered, asynchronous, isolated** — events arrive in commit order, + dispatched in a microtask so a slow listener never delays a write and a throwing + listener never affects the write or other listeners. Zero overhead with no + subscribers. +- **`kind: 'store'`** events for `clear()`/`restore()` mean "refetch everything". +- Fire-and-forget by design; each event's `generation` composes with + `asOf()`/`transactionLog()` for catch-up after a gap. + +Guide: `docs/guides/reacting-to-changes.md`. No behavior changes for existing code. + +## v8.0.17 — 2026-07-08 (count recovery scans the real layout · ~1,100 lines of dead 7.x machinery removed) + +A cleanup of the vestigial 7.x "hnsw sharding" machinery that turned up two real fixes. + +**1 — Count recovery now scans the layout the database actually uses.** When `counts.json` is +lost or corrupted (container restarts, partial copies), the recovery scan rebuilt the entity/ +relationship counters by counting files in `entities/*/hnsw/` — directories the 8.0 write path +never populates — so an established store recovered to **zero counts** on real data: wrong +`getNounCount()`/stats, a "New installation"-style boot log, and a mis-sized rebuild-strategy +decision at open. The scan now counts the canonical `entities////` tree. +Regression-tested: delete `counts.json` from a populated store → reopen → exact counts. + +**2 — A latent init-time deadlock removed.** The recovery scan's type-distribution sampler called +a guarded accessor (`getNounMetadata` → `ensureInitialized`) from **inside** `init()`, which +re-enters `init()` and hangs the open. It was unreachable before only because the old scan never +found anything to sample; the fix reads the sampled metadata files directly. + +**3 — The dead machinery itself is gone (−1,138 lines).** Twenty-three methods with zero callers: +the 7.x hnsw-layout entity/edge CRUD, the sharding-depth prober + depth-migration engine, and an +orphaned streaming paginator. Verified by reachability analysis before deletion; no public API +touched; the full suite is green. New stores no longer carry the always-empty legacy directories' +scan cost, and the boot log keeps the truthful new-vs-established wording from v8.0.13. + +## v8.0.16 — 2026-07-08 (concurrency fast-follow: atomic `ifAbsent`/`upsert` + exact blob reference counts) + +Closes the two remaining check-then-act races found in the sweep that followed v8.0.15's CAS fix +(disclosed in that release's notes). Same bug class — a check performed before the serialization +point that guards the apply — same cure. + +**1 — `add({ ifAbsent })` and `add({ upsert })` are now atomic.** The absence check ran before the +commit mutex, so N concurrent same-id creates could all pass it and all write — the second +silently overwriting the first, violating ifAbsent's "returns the existing id **without writing**" +contract and upsert's "merge, never clobber" contract. The insert leg now carries a must-be-absent +precondition (the same conditional-commit primitive as `ifRev`), verified under the commit mutex: +exactly one concurrent create wins; every other caller takes its documented resolution — ifAbsent +returns the existing id with zero writes, upsert merges into the now-existing entity (with a +bounded retry if a concurrent delete intervenes). Verified: 8 concurrent `ifAbsent` creates +advance the store by exactly ONE generation; 8 concurrent upserts on an absent id produce one +create + seven merges (`_rev` lands at exactly 8). Note: `transact()`'s batch-level +ifAbsent/upsert keep planning-time semantics (the batch converges to a valid entity; the window is +documented, not silent). + +**2 — Blob reference counts are exact under concurrency.** The content-addressed blob store's +`write()` (dedup: exists → add a reference, absent → create) and `delete()` (decrement → remove at +zero) were unserialized read-modify-writes over the blob's metadata. Concurrent writes of +identical content could lose references — making a later delete remove bytes **another file still +referenced** (data loss), or leak unreferenced blobs. All reference-count-bearing mutations are +now serialized per content hash (distinct content never contends): N concurrent identical writes +yield exactly N references, and a blob is physically removed only when the true last reference +drops. Verified with concurrent write/delete storms. + +Both fixes are in-process complete by construction — storage enforces single-writer-per-directory, +so the process is the whole concurrency domain. No API changes. + +## v8.0.15 — 2026-07-08 (`ifRev` CAS is now atomic — exactly one winner under concurrency) + +Correctness fix for optimistic concurrency, reported from a production cutover rehearsal. N +**concurrent** `update({ ifRev })` calls carrying the same expected revision ALL succeeded — zero +`RevisionConflictError`s, last-writer-wins, the other N−1 writes silently lost. (Sequential calls +conflicted correctly.) The revision check ran before the commit mutex, so interleaved callers all +passed it before any apply landed — breaking the exactly-one-winner semantics the +[optimistic-concurrency guide](docs/guides/optimistic-concurrency.md) promises, which advisory +locks and per-entity ledgers/counters build on. + +The fix makes the check-and-apply a **conditional commit**: the generation store's commit paths +accept a precondition that runs under the commit mutex against the just-read authoritative +before-images — the per-record analogue of `ifAtGeneration`, which always ran there. `update()` +and `transact()` per-op `ifRev` both re-verify at that point (the earlier check remains as a cheap +fast-fail). A conflict aborts atomically: nothing staged, nothing applied, the same +`RevisionConflictError` as before. Two adjacent behaviors also became honest: + +- **`_rev` is now monotonic under concurrency.** The winner's stamp derives from the + authoritative before-image, so N concurrent plain updates advance `_rev` by N (previously they + could all stamp the same stale value). +- **CAS against a concurrently-deleted entity** now throws `EntityNotFoundError` instead of + silently resurrecting it (plain updates keep their last-writer-wins re-create semantics). + +Verified with a concurrency regression suite: 8 parallel same-rev updates → exactly 1 winner + 7 +conflicts; the documented read→CAS→retry ledger loop converges exactly (8 workers, 8 decrements, +0 lost) — `tests/integration/ifrev-concurrent-cas.test.ts`. If you serialized writes in your own +adapter as a workaround, it can come out after this upgrade. + +## v8.0.14 — 2026-07-07 (7→8 migration preserves branch-scoped non-entity state instead of deleting it) + +Defense-in-depth for the one-time 7→8 layout migration. The migration rescues the head branch's +entities (`branches//entities/*` → `entities/*`) and then drained the whole `branches//` +directory. If a 7.x engine had written durable state under the head branch *outside* `entities/` +(a branch-scoped index, blob area, or field registry), that drain would have silently deleted it — +the same failure class as the VFS content blobs a 7.x store kept in `_cow/`. The migration now drains +the branch only when nothing but the moved entities remains; if any non-entity object survives, it +**preserves the branch** (no delete) and logs a loud warning naming the leftover keys, so the state is +recoverable rather than lost. No effect on a normal migration (a clean branch still drains); purely a +guard against silent data loss. + +Cosmetic boot-log fix. Every persisted 8.0 store logged `📁 New installation: using depth 1 +sharding` on **every** open — even established brains holding thousands of entities — which is +alarming to read during a restart or incident. Cause: 8.0 stores nouns in the canonical +`entities/nouns///vectors.json` layout, but the legacy sharding probe inspected the +`entities/nouns/hnsw/` directory, which the 8.0 write path never populates, so it always concluded +"new". The new-vs-existing decision now consults the layout the database actually reads and writes +(plus the known noun count), so an established store logs `📁 Using depth 1 sharding (N entities)` +and only a genuinely empty store reports a new installation. No behavior change beyond the log line — +it never triggered a rebuild or migration; entities were always read correctly. + +## v8.0.12 — 2026-07-07 (7→8 VFS-content recovery · zero-rebuild cold open · strict query operators) + +Three consumer-facing fixes. + +**1 — A 7→8 upgrade recovers Virtual Filesystem content automatically (data-integrity).** +7.x stored VFS file content as blobs in the branch system's copy-on-write area (`_cow/`). 8.0 +removed that system and stores content blobs in the content-addressed store (`_cas/`), but the +one-time layout migration only moved entities — it never adopted the `_cow/` content blobs. On a +store that used the VFS (for example, a CMS with published pages), that left every VFS-backed read +throwing `Blob metadata not found` and the pages 500ing. **8.0.12 adds an on-open recovery pass** +that adopts every orphaned `_cow/` blob into `_cas/` — copying both the bytes and the metadata, +idempotently and **non-destructively** (the `_cow/` originals are never deleted). It heals both a +fresh 7→8 upgrade and a store **already** upgraded by an earlier 8.0.x that stranded them, with no +operator action: just open the store under 8.0.12. An explicit force path is exposed as +`brain.vfs.adoptOrphanedBlobs()` (returns `{ cowBlobs, adopted, alreadyPresent, incomplete }`). The +automatic pre-upgrade backup is now **retained** if any blob can't be fully adopted +(`incomplete > 0`) instead of being removed on entity-migration "success" alone. Affects any 7.x +store that used the VFS; native-8.0 and non-VFS stores no-op on a cheap existence check. Full guide: +`docs/guides/upgrading-7-to-8.md`. + +**2 — A cold open no longer re-derives durable indexes it can just load.** +Opening a persisted store rebuilt the vector, graph, and metadata indexes from the canonical +records even when the durable index state was present and loadable — an O(N) cost paid on every +boot (measured at ~48 s on an ~11k-entity store). The rebuild gate now consults an honest +per-provider durability signal (`init()` / `isReady()`) instead of an in-memory size heuristic, so +a provider that has loaded (or can cheaply demand-load) its persisted index is not rebuilt. The +built-in JS indexes keep today's behavior (a rebuild *is* their load path); the live query-time +guards that self-heal a genuinely lost index are unchanged. **The zero-rebuild boot activates when +the accelerator exposes the durability signal (`@soulcraft/cor@3.0.5`);** on earlier accelerator +versions 8.0.12 falls back to the previous behavior safely — no regression, just no speedup yet. + +**3 — Unknown query operators now throw instead of silently returning nothing.** +`find({ where })` had two gaps: the in-memory matcher (used for egress re-validation and historical +reads) was missing several documented operators (`in`, `greaterThanOrEqual`, `lessThanOrEqual`), so +it silently disagreed with the index path; and an unknown operator key (a typo, or `notIn` written +where `not: { in }` was meant) was treated as a nested-object field and silently matched nothing. +8.0.12 aligns the matcher to the full documented operator set and **validates the `where` filter +up front**, throwing a typed `BrainyError('INVALID_QUERY')` naming the bad operator. Dotted paths +remain the supported form for nested fields. If you relied on an unknown key silently returning `[]`, +it now throws — the fix is to use the documented operator or dot-notation. + +## v8.0.11 — 2026-07-02 (no script shape can hang on brainy's internals) + +Completes v8.0.10's exit fix for every operation class. Two further mechanisms found and fixed: +the `beforeExit` auto-flush hook looped forever on any script that never reaches `close()` (Node +re-emits `beforeExit` after each event-loop drain and the async flush schedules new work — it now +self-deregisters before its single flush, which still lands your buffered data before exit), and +every background-maintenance interval (graph auto-flush, LSM compaction, metadata write-buffer, +VFS/path-cache maintenance, statistics debounce) is now unref'd at creation. Verified against the +published package: an `add + relate` script exits cleanly both with `close()` (~0.5 s) and with no +teardown at all — with the data confirmed durable on reopen. A per-operation-class sweep test now +asserts no ref'd timer survives `close()`, so this bug class stays closed. + +## v8.0.10 — 2026-07-02 (a bare script exits cleanly after `close()`) + +A minimal `init → add → close` script used to hang forever after `close()` returned — brainy held +four process keep-alives it never released: the global SIGTERM/SIGINT shutdown hooks (never +removed; now deregistered when the last live instance closes), an internal cache-fairness interval +(unclearable by construction; now unref'd), and the VFS/PathResolver maintenance intervals +(`close()` never shut the VFS down; now wired). Verified against the published package: the same +script now exits ~1 ms after `close()` resolves. No API change; servers and long-running processes +are unaffected. + +## v8.0.9 — 2026-07-02 (guarded plugin auto-detection — the "install it and it's on" contract) + +With the default config (`plugins` unset), brainy now **auto-detects the first-party accelerator**: +installing `@soulcraft/cor` is the opt-in. The detection is guarded — everything except "not +installed" fails loud: + +- Not installed → plain brainy, silently (the free path — zero noise, zero cost). +- Installed and healthy → it loads and announces itself (`[brainy] Plugin activated`). +- Installed but broken (unresolvable, invalid shape, failed activation, version mismatch) → + **`init()` throws.** An installed accelerator never silently vanishes behind the JS engines. +- `plugins: []` / `false` = explicit opt-out; `plugins: ['@soulcraft/cor']` pins the exact list + (unchanged semantics). + +Also new: if a plugin activates but registers **zero** native providers (e.g. a licensing gate +declining to engage), brainy warns loudly instead of leaving you to discover every query is running +on the JS engines. + +> This supersedes v8.0.8's "plugins are explicit opt-in" wording, which documented the pre-GA +> loader accurately but contradicted the published product contract ("add the package and it +> activates"). 8.0.9 makes the contract true — with the loud-failure guarantees intact. + +## v8.0.8 — 2026-07-02 (docs-only patch on the GA) + +Corrects the README's scale-up section: the native provider is **not** auto-detected — plugins are +**explicit opt-in** (`new Brainy({ plugins: ['@soulcraft/cor'] })`; brainy never auto-imports a +package you didn't list, and a listed plugin that fails to load throws rather than silently falling +back to the JS engines). Also fixes the stale `plugins` config comment in the public types and one +phrase in the plugin-author guide. No code change. + +## v8.0.7 — 2026-07-02 (GA, npm tag `latest`) + +> **Why 8.0.7:** npm permanently retires unpublished version numbers, and `8.0.0`–`8.0.6` were +> consumed by a January development cycle (published and immediately unpublished). `8.0.7` is +> the first stable release of the 8.x line; there are no earlier stable 8.0.x releases. + +**8.0.7 is the first stable major on the u64-id core.** It ships in lockstep with the optional +native provider's `3.0` (the billion-scale path). Everything below works standalone on the open-core +JS engine — a native provider is feature-detected and only changes the scale ceiling, never the API. + +**Upgrading from 7.x: nothing to script.** The first time 8.0 opens a 7.x brain it upgrades itself — +an automatic, observable, **coordinated migration lock** rebuilds every derived index from your +canonical records while Brainy **blocks and queues** reads and writes, so no operation ever touches a +half-built index and no write is ever lost. Budget seconds-to-minutes per brain at large sizes; small +brains rebuild inline on open. A pre-upgrade hard-link **backup** is taken automatically (removed on +success, kept on failure for rollback). Observe it via `getIndexStatus().migration`; a caller that +hits the window gets a typed, retryable **`MigrationInProgressError`** (catch → HTTP 503 + +`Retry-After`). Bounded by `migrationWaitTimeoutMs` (default 30 s — it bounds *your wait*, not the +rebuild). Opt out of the backup with `migrationBackup: false`. + +### Headline changes + +- **The cold-open silent-`[]` class is gone.** On a cold reopen, filtered finds (`find({ where })`) + and graph reads (`find({ connected })` / `neighbors()` / `related()`) never silently return `[]` + for data that is actually present. With a native provider the durable indexes cold-serve every + filter and edge with zero rebuild; the open-core engine adds belt-and-suspenders guards that + self-heal from canonical data or throw a loud, typed error — never a silent empty result. Two new + exported errors: **`MetadataIndexNotReadyError`**, **`GraphIndexNotReadyError`**. + +- **Faster vector search (open-core).** The JS distance functions are allocation-free loops instead + of `reduce`: **~6× cosine, ~1.4× euclidean** (MEASURED — `tests/benchmarks/distance-microbench.mjs`, + 384-dim, median of 41). Numerically identical, so recall is unchanged. Exact metadata-filter + pushdown and scale-aware search width keep `find()` recall-correct as data grows. + +- **Per-write immutable history.** Every write is generation-stamped; `now()` / `asOf()` / + `transact()` read a consistent point in time, and a retention knob bounds history growth. Single + writes participate in the same immutable timeline as transactions. + +- **Billion-scale resident memory.** Nothing is O(N)-resident on the write or time-travel path — + per-id caches removed, the committed-generation ledger is an interval set, per-id history chains + are bounded (hot-window + LRU). Resident memory is independent of entity count. + +- **Deterministic, round-trip-free writes.** Supply your own ids, forward-reference not-yet-written + entities inside a `transact()`, `ifAbsent` upsert, and `brain.newId()` (uuidv7) — write graphs + without a write→wait→get round-trip. + +### Breaking changes (summary — the full migration guide follows below) + +- **Runtime floor: Node ≥ 22 / Bun ≥ 1.1** (Bun recommended). Compiler target ES2023; the DOM lib is + dropped — 8.0 is a Node/Bun/Deno engine with no browser path. +- **`neural()` removed** and **`Db.search` removed** — use `find()` on the brain. +- **Storage config is one `path` key.** Removed aliases now throw with a message pointing at `path`. +- **`get()` omits the vector by default** — pass `{ includeVectors: true }` when you need it. +- **Export/import type is `PortableGraph`** (was `BackupData`; wire tag `brainy-backup` → + `brainy-portable-graph`). Re-export any snapshots you version outside Brainy. +- **Reserved `visibility` field** (`public` / `internal` / `system`) on nouns and verbs; `internal` + and `system` are excluded from normal reads by default. +- **4 deprecated query operators removed** (`is`/`isNot`/`greaterEqual`/`lessEqual`) and reserved + keys in a `metadata` bag now **throw** by default — details in the rc notes below. + +> Post-rc.9 hardening folded into GA (no on-disk or provider-contract change vs `8.0.0-rc.9`): +> the metadata cold-read guard, the auto pre-upgrade backup (with an `_id_mapper/*` mmap byte-copy +> correctness fix), and a CI fix so a fresh clone builds green. + +### RC history (rc.1 → rc.9, npm tag `rc`) + +> **rc.9 changes (2026-07-01):** +> - **Whole-brain auto-upgrade is now a coordinated, observable LOCK — supersedes rc.8's no-freeze +> approach.** When a large brain's derived-index format changes (7.x→8.0), the native provider +> rebuilds the indexes from the canonical records **in place** while Brainy **blocks and queues** +> reads and writes — so no operation ever touches a half-built index. This is a clean *blocking +> upgrade to a known-good state* (vs rc.8's online background swap): unknown/halfway states are +> more dangerous than a bounded wait. Small brains still rebuild inline on open. New surface, all +> additive: a typed, exported, retryable **`MigrationInProgressError`** (catch it → HTTP 503 + +> `Retry-After`); `getIndexStatus()` gains **`migrating` + `migration`** progress (never gated — +> the readiness-probe signal); a **`migrationWaitTimeoutMs`** config (default 30 s — it bounds the +> *caller's wait*, NOT the rebuild, which is unbounded); `health()` / `checkHealth()` report the +> upgrade without blocking. No effect without a native provider. +> - **Faster vector search (open-core).** The JS distance functions are rewritten from `reduce` to +> allocation-free loops — **~6× cosine, ~1.4× euclidean** (MEASURED, `tests/benchmarks/distance-microbench.mjs`, +> 384-dim, median of 41). Numerically identical → recall unchanged. +> - **Runtime floor + Bun.** Engines are now **Node ≥22 / Bun ≥1.1** (fixes a Node-24 `EBADENGINE`); +> compiler target ES2023 with DOM dropped from the type lib (8.0 is Node/Bun/Deno-only, no browser +> path). **Bun is recommended as a runtime** (`bun add` / `bun run`); the single-binary +> `bun build --compile` is not a supported target (native addon can't embed + a Bun 1.3.10 codegen +> regression). `.d.ts` stays TS-5.x-parseable. + +> **rc.8 additions (2026-06-30) — additive, no breaking change:** +> - **No-freeze (online) whole-brain auto-upgrade.** When a derived-index format changes, a large +> brain upgrades **without blocking** — the native provider rebuilds the new indexes in the +> background and serves correct reads from the canonical records meanwhile, then atomically swaps; +> no minutes-long freeze on the first open/query. (Small brains still auto-rebuild inline on open.) +> New surface: an optional provider `isMigrating()` deference signal, a public `brain.stampBrainFormat()` +> the provider calls once its swap verifies, and a `@soulcraft/brainy/brain-format` export so the +> provider shares the `indexEpoch` constant. No effect without a native provider. + +> **rc.7 additions (2026-06-30) — additive, no breaking change:** +> - **8.0 cold-graph self-heal.** `find({ connected })` / `neighbors()` / `related()` never +> silently return `[]` on a cold open where the graph adjacency loaded its membership but not +> its source→target edges — it self-heals (rebuild from storage) or throws a loud +> `GraphIndexNotReadyError`, never empty-for-persisted-data. (The 8.0 equivalent of the 7.33.4 +> fix 7.x consumers already have; gates on the native provider's honest `isReady()` edge-readiness +> signal.) +> - **Billion-scale RAM — nothing O(N)-resident on the write/time-travel path.** Eliminated the +> per-id storage caches (per-type counts now sourced from the record), made the +> committed-generation ledger an interval set, and bounded the per-id time-travel history chains +> (hot-window + LRU, with lock-light reconstruction so historical reads never stall writers). +> Resident memory is now independent of entity count. +> - **Whole-brain version handshake.** A `_system/brain-format.json` marker + `brain.formatInfo()` +> + a shared, lockstep `indexEpoch`: a future derived-index format change auto-rebuilds the +> indexes from the canonical records on open (non-destructively) — the foundation for +> whole-brain auto-upgrade with the native provider. No effect on an unchanged brain. + +> **rc.6 additions (2026-06-29) — additive, no breaking change:** +> - **Open-core perf**: HNSW delete is now O(in-degree) (was O(N²) for bulk delete) via a +> reverse-adjacency index; the negation/absence operators (`ne`/`exists:false`/`missing:true`) +> are served as a roaring-bitmap difference instead of materializing the whole corpus. +> - **Native provider contract** (cor lockstep, optional/feature-detected — no effect without a +> native provider): a cold-open `probeConsistency()` self-heal hook, and a `getIdsForFilter` +> page bound so the native index can early-stop the unsorted `find({ type, where, limit })` path. +> - **Test hygiene**: re-homed previously-unrun test suites into CI + a guard so a test file can +> never silently fall outside every config again. No source/API change from these. + **Affected products:** every consumer — this is a major release with removed surfaces, hard renames (no aliases, no deprecation period), and one flipped default. This entry **is** the migration guide: the find/replace pairs, the sed snippet, and the upgrade checklist below are the complete story — there is -no separate migration doc. Latest published version remains 7.31.x until this -ships. +no separate migration doc. **`8.0.7` is GA on `latest`** — install with +`npm i @soulcraft/brainy@latest`. + +> **rc.3–rc.5 additions (2026-06-24):** +> - **Showcase-quality GA hardening (rc.5).** A full readiness audit closed a cold-init +> version-coupling bug (a matched native provider could be rejected on first open), turned +> silent storage-read failures into named `BrainyError`s (no more empty-result-on-error), +> stopped the JS graph LSM from orphaning compacted SSTables, corrected the public docs to +> the real API, and documented the two headline methods. Plus a dead/deprecated-code sweep. +> - **BREAKING — 4 deprecated query operators removed.** `is`→`eq`, `isNot`→`ne`, +> `greaterEqual`→`gte`, `lessEqual`→`lte`. The canonical operators and their clean long-form +> aliases (`equals`/`notEquals`/`greaterThan`/`greaterThanOrEqual`/`lessThan`/`lessThanOrEqual`) +> are unchanged — only the four redundant spellings are gone. Find/replace if you used them. +> - **`find()` search mode** is now the single `searchMode: SearchMode` option (the redundant, +> silently-ignored `mode` alias and the unwired `explain` flag were removed from `FindParams`). +> - The phantom index-integrity guard (find() re-validates every result against its predicate) +> shipped in rc.3; rc.4 added the `accel.isInitialized` gate + #35 at-gen candidate vectors. + +> **rc.1 additions (this entry predates them — full writeup lands at GA):** entity-id +> normalization — `brain.newId()` (UUID v7) + v7 default ids, and non-UUID string ids are +> transparently normalized to a stable UUID v5 (original key preserved under `_originalId`); +> `brain.neural()` clustering and `Db.search()` removed (use `find({ vector })` / +> `find()` + aggregation `GROUP BY`); storage config collapsed to one `path` key (the +> pre-8.0 aliases now throw); and `queryAggregate` no longer hangs/staled min-max after a delete. +> **Reserved fields in a `metadata` bag now throw by default** — passing a Brainy-reserved key +> (`confidence`, `weight`, `subtype`, `visibility`, `service`, `createdBy`, `noun`/`verb`, `data`, +> `createdAt`, `updatedAt`, `_rev`) inside `metadata` on `add()`/`update()`/`relate()`/`updateRelation()` +> (untyped/JS callers — TypeScript already blocks it) is rejected with an Error naming the correct +> param, instead of the old silent remap-or-drop. Pass reserved values as their dedicated top-level +> params. Opt back into the legacy behavior with `new Brainy({ reservedFieldPolicy: 'warn' | 'remap' })`. + +> **rc.2 additions (2026-06-21):** +> - **Graph performance + the `brain.graph` namespace.** New `brain.graph.subgraph(seeds, opts)` +> (bounded multi-hop neighborhood → `{ nodes, edges, truncated }`) and `brain.graph.export(opts)` +> (stream the whole graph in one O(N+E) pass — the right primitive for visualizing all data +> instead of paging per node). `related({ node })` returns every edge incident to an entity in +> **both directions** in one O(degree) call. Under the hood: `related({ from/to })` now stays +> O(degree) under the default visibility filter (was a full scan), and the verb **and** noun +> pagination walks are cursor-based, so a full edge/node walk is **O(N), not O(N²)** (a consumer +> measured a 19k-edge whole-graph read drop from ~27s toward a single scan). These transparently +> use a native graph engine when present (the `@soulcraft/cor` 3.0 acceleration layer) and fall +> back to pure-TS adjacency otherwise. +> - **Additive ergonomics:** `add({ upsert: true })` (create-or-update in one call — merges into an +> existing id instead of overwriting), `find({ includeVectors: true })`, and `removeMany`'s batch +> size is now storage-adaptive (was a hardcoded 10). +> - **Correctness fixes (apply to all consumers, not just graph users):** `related()` results now +> carry `visibility`; whole-graph/`getNouns` streaming no longer leaks `system`/`internal` entities; +> and small-page cursor walks over nouns no longer loop. No API change — just correct behavior. + +> **rc.3 additions (2026-06-23):** +> - **Graph analytics on the `brain.graph` namespace.** Three intent-level reads that answer +> whole-graph questions in one call: +> - **`brain.graph.rank(opts?)`** → `{ id, score }[]` descending — "which entities matter most" +> (importance / centrality). `topK` to cap. +> - **`brain.graph.communities(opts?)`** → `{ groups: string[][], count }` — "which things group +> together". Weakly-connected components by default; `{ directed: true }` returns +> strongly-connected components. +> - **`brain.graph.path(from, to, opts?)`** → `{ nodes, relationships, cost } | null` — the best +> route between two entities. Fewest hops by default; `{ by: 'weight' }` minimizes summed edge +> weight (the 0–1 connection strength); `direction`, `type`, and `maxDepth` filters apply. +> These are **intent contracts, not algorithm contracts** — the question is the promise, the +> algorithm is the engine's choice. They transparently use the native `@soulcraft/cor` 3.0 graph +> engine when present, and fall back to pure-TS kernels (PageRank, connected components / Tarjan +> SCC, BFS / Dijkstra) otherwise. Both paths return identical shapes and respect the default +> visibility filter (opt in with `includeInternal` / `includeSystem`). +> - **Filtered vector search keeps its recall (`allowedIds` pushdown).** A `find({ query, where })` +> that combines semantic search with a metadata filter now restricts the vector walk to the +> matching candidates *inside* the search (walk-all, collect-allowed) instead of filtering the +> top-k afterward — so a query whose nearest vectors are all filtered out still returns the best +> matches that DO pass the filter, rather than coming back empty. No API change. With the native +> `@soulcraft/cor` 3.0 stack the matched universe is forwarded as an opaque roaring buffer with +> zero id materialization in TypeScript; the pure-JS path restricts the beam walk with a string set. +> - **Historical (`asOf`) semantic search can skip the rebuild.** A filtered semantic query at a +> past generation — `db.asOf(g).find({ query, where })` — no longer always rebuilds an ephemeral +> in-memory vector index over every at-`g` vector (O(n@G)). When a native versioned vector engine +> is registered and can serve the pinned generation, Brainy resolves the at-`g` filter universe from +> its record layer (no rebuild) and routes the vector leg to the engine with the matched ids + the +> generation; without one it falls back to the materialization (unchanged). The `VectorIndexProvider.search` +> options gained an optional `generation?: bigint` ("omitted = now") to carry this — additive, the +> built-in index ignores it. No public API change. +> - **`find({ where, orderBy })` bounds the sort to the page.** A broad filter + `orderBy` +> returning one page no longer materializes every matching sorted id (it produced the full sorted +> match set, then sliced) — the page bound (`offset + limit`) is threaded into the column store's +> top-K sort, so returning 20 rows from a million matches stays a bounded-K heap. No API change; +> ordering and pagination are unchanged. +> - **`brain.graph.subgraph()` accepts a query (query→expand).** The seed selector now takes not +> just entity id(s) but a `find()` result or a `FindParams` query — `brain.graph.subgraph({ where: +> { team: 'platform' } }, { depth: 1 })` runs the query and expands the neighborhood of every +> match in one call. With the native engine a metadata-only query's matched universe is handed to +> the traversal as an opaque set with no id materialization in TypeScript (the query→expand +> fusion); the pure-JS path materializes the matched ids and expands from them. Existing +> id-seeded calls are unchanged. ### Headline: Database as a Value @@ -79,13 +1239,29 @@ What you get: - **`brain.transactionLog({ from?, to?, limit? })`** — the commit log over an **inclusive** generation/`Date` window (contrast `since`'s exclusive lower bound); `limit` applies last. -- **`brain.compactHistory({ retainGenerations, retainMs })`** — explicit - retention. Compaction never breaks a pinned read. `diff`/`since` throw - `GenerationCompactedError` below the horizon; `history` truncates to it. +- **Every write is versioned (Model-B).** History granularity is **per-write**: + EVERY write — `transact()` AND a single-operation `add`/`update`/`remove`/ + `relate` — is its own immutable generation. A `now()` pin always freezes + against later writes, and `asOf`/`since`/`diff`/`history`/`transactionLog` + reflect single-ops exactly like transacts. `transact()` groups several + operations into ONE atomic generation. Single-op history durability is **async + group-commit** (the live write is acknowledged immediately; its before-image + is batched to disk in one fsync) — a hard crash can lose only the last + un-flushed window's *history*, never live data, and a crash mid-flush is + recovered by drop-without-restore. A freshly-initialized brain is + `generation() === 0` with an empty `transactionLog()` (init-time + infrastructure is the un-versioned baseline); the first user write is gen 1. +- **`new Brainy({ retention })`** — the retention knob governs auto-compaction on + `flush()`/`close()`: **unset → ADAPTIVE** (disk/RAM-pressure byte budget, + zero-config; a coordinator can drive it via `brain.setRetentionBudget(bytes)`) + · **`'all'`** → unbounded · **`{ maxGenerations?, maxAge?, maxBytes? }`** → + explicit caps (reclaim oldest-unpinned while ANY cap is exceeded). Live pins + are ALWAYS exempt. +- **`brain.compactHistory({ maxGenerations?, maxAge?, maxBytes? })`** — manual + reclaim on the same caps. Compaction never breaks a pinned read. `diff`/`since` + throw `GenerationCompactedError` below the horizon; `history` truncates to it. -The precise guarantees (and the honest limits — history granularity is -`transact()` commits; single-operation writes advance the clock but produce no -historical records) are documented in: +The precise guarantees are documented in: - [docs/concepts/consistency-model.md](docs/concepts/consistency-model.md) — the guarantees, each proven by a dedicated test in @@ -155,7 +1331,7 @@ write-time `sum`/`count`/`avg`/`min`/`max`. | Cloud + browser storage adapters (`s3` / `gcs` / `r2` / `opfs` storage types, Azure Blob, plus the `storage.branch` option) | `filesystem` and `memory` only. Cloud backup is now an operator concern: `db.persist()` produces a plain directory — sync it with `gsutil` / `aws s3 cp` / `rclone` / `azcopy`. Passing a removed storage type throws at construction with this exact guidance. | | Browser support | Node.js 22 LTS or Bun ≥ 1.0 (`engines` enforces `node: 22.x`); Deno works through its Node compatibility layer. | | `brain.migrateToDiskAnn()` / `brain.migrateToHnsw()` | Gone — there is one canonical `'vector'` provider slot. A registered native vector provider selects its own operating mode; there is nothing to call. | -| Distributed-clustering subsystem — `config.distributed`, the `DistributedRole` enum, the 13 `BRAINY_*` cluster env vars (`BRAINY_DISTRIBUTED`, `BRAINY_ROLE`, `BRAINY_HTTP_PORT`, `BRAINY_WS_PORT`, `BRAINY_DNS`, `BRAINY_SERVICE`, `BRAINY_NAMESPACE`, `BRAINY_CONSENSUS`, `BRAINY_COORDINATOR`, `BRAINY_NODES`, `BRAINY_REPLICAS`, `BRAINY_SHARDS`, `BRAINY_TRANSPORT`), the storage `setDistributedComponents` hook, and the `@soulcraft/brainy/config` preset/augmentation registry | Removed. Brainy 8.0 is a single-process library — there is no coordinator, peer discovery, or consensus to operate. Scale via: the optional native provider (`@soulcraft/cortex` — on-disk DiskANN to 10B+ vectors on one machine); per-tenant pools (one instance + storage dir per tenant); and horizontal read scaling (many reader processes against one shared store, single writer). The `mode: 'reader' \| 'writer'` multi-process roles are unchanged. | +| Distributed-clustering subsystem — `config.distributed`, the `DistributedRole` enum, the 13 `BRAINY_*` cluster env vars (`BRAINY_DISTRIBUTED`, `BRAINY_ROLE`, `BRAINY_HTTP_PORT`, `BRAINY_WS_PORT`, `BRAINY_DNS`, `BRAINY_SERVICE`, `BRAINY_NAMESPACE`, `BRAINY_CONSENSUS`, `BRAINY_COORDINATOR`, `BRAINY_NODES`, `BRAINY_REPLICAS`, `BRAINY_SHARDS`, `BRAINY_TRANSPORT`), the storage `setDistributedComponents` hook, and the `@soulcraft/brainy/config` preset/augmentation registry | Removed. Brainy 8.0 is a single-process library — there is no coordinator, peer discovery, or consensus to operate. Scale via: the optional native provider (`@soulcraft/cor` — on-disk DiskANN to 10B+ vectors on one machine); per-tenant pools (one instance + storage dir per tenant); and horizontal read scaling (many reader processes against one shared store, single writer). The `mode: 'reader' \| 'writer'` multi-process roles are unchanged. | | CLI `fork` / `branch` / `checkout` / `migrate` | CLI `snapshot ` / `restore ` / `history` / `generation`. | | `BrainyZeroConfig` type | Removed. 8.0 zero-config is automatic and internal — `new Brainy()` auto-detects storage, derives HNSW quality from `config.vector.recall`, picks `persistMode` from the adapter, and sizes caches to the detected container-memory limit. There is no config-generation type to import. | | `brain.isFullyInitialized()` / `brain.awaitBackgroundInit()` | `await brain.ready`. The built-in filesystem/memory adapters finish initialization synchronously inside `init()`, so a single readiness promise is the whole story — these methods were no-ops once cloud adapters were removed. | @@ -197,6 +1373,9 @@ Hard renames, no aliases. Every pair below is verified against the 8.0 source. | `brain.stats().indexHealth.hnsw` | `brain.stats().indexHealth.vector` | | Storage adapter contract `saveHNSWData()` / `getHNSWData()` | `saveVectorIndexData()` / `getVectorIndexData()` | | Cache category `'hnsw'` (`CacheProvider` union) | `'vectors'` | +| `config.history = { retainGenerations, retainMs, autoCompact }` | `config.retention` — `'all'` \| `'adaptive'` \| `{ maxGenerations?, maxAge?, maxBytes?, budgetBytes?, autoCompact? }`; **unset → adaptive** (was keep-100/7-days). The fields are now **caps**, not floors. | +| `compactHistory({ retainGenerations, retainMs })` | `compactHistory({ maxGenerations, maxAge, maxBytes })` — `retainGenerations`→`maxGenerations`, `retainMs`→`maxAge` (now upper-bound CAPS), plus new `maxBytes`. | +| `CompactHistoryOptions.retainGenerations` / `.retainMs` | `.maxGenerations` / `.maxAge` (+ `.maxBytes`) | Mechanical renames as one command (run from your repo root, review the diff): @@ -411,12 +1590,14 @@ its MVCC bookkeeping (`_system/generation.json`, `_system/manifest.json`, `_system/tx-log.jsonl`, `_generations/`) alongside the existing files on first use. -### Cortex compatibility +### Native-provider (Cor) compatibility -Brainy 8.0 pairs with `@soulcraft/cortex` 3.0 (shipping in lockstep). Cortex -2.x cannot accelerate Brainy 8.0: 8.0 never consults the `'hnsw'` / -`'diskann'` provider keys, and the graph/column provider contracts are BigInt -at the boundary. Upgrade both packages together. +Brainy 8.0 pairs with `@soulcraft/cor` 3.0 — the renamed successor to the +`@soulcraft/cortex` 2.x native provider — shipping in lockstep. The old +`@soulcraft/cortex` 2.x cannot accelerate Brainy 8.0: 8.0 never consults the +`'hnsw'` / `'diskann'` provider keys, and the graph/column provider contracts +are BigInt at the boundary. Upgrade both packages together (`@soulcraft/brainy@8` ++ `@soulcraft/cor@3`). --- diff --git a/docs/ADR-001-generational-mvcc.md b/docs/ADR-001-generational-mvcc.md index d6e32441..b4c4d8dd 100644 --- a/docs/ADR-001-generational-mvcc.md +++ b/docs/ADR-001-generational-mvcc.md @@ -149,12 +149,22 @@ overlay entities carry no embeddings (`with()` never invokes the embedder), so a "full" index query over an overlay would silently exclude the overlay's own entities. Commit with `transact()` to get the full surface. -**History granularity.** Generation *records* are written per `transact()` -batch only. Single-operation writes advance the counter (so watermarks and -CAS stay sound) but do not stage before-images: they remain visible through -earlier pins and are not reported by `db.since()`. Code that needs pinned -isolation across its writes uses `transact()` — that is the documented 8.0 -contract, stated rather than papered over. +**History granularity (Model-B).** EVERY write is its own immutable generation +— `transact()` batches AND single-operation `add`/`update`/`remove`/`relate`. +Single-ops stage a before-image and are reported by `db.since()`/`asOf()`/ +`diff()`/`history()` exactly like transacts; a pin always freezes against later +writes. `transact()` groups several operations into ONE atomic generation. + +Single-op history durability is **async group-commit**: the live write hits +canonical storage immediately (acknowledged), while its before-image is buffered +and persisted to disk in one batched fsync on a size/timer trigger (or forced by +`flush()`/`close()`/`transact()`/`compactHistory()`). The buffer participates in +point-in-time resolution exactly like on-disk generations, so the synchronous +`now()` freezes with no forced flush. A hard crash before the flush loses only +the buffered *history* of the last window — never live data — and a crash +*mid-flush* is recovered by **drop-without-restore** (the partial generation's +before-images are discarded, never replayed, because the live write was already +acknowledged; restoring them would silently revert it). ### Pinning, retention, compaction @@ -162,11 +172,17 @@ contract, stated rather than papered over. `pin(generation)` on every registered `VersionedIndexProvider`, whose explicit pin lifetime overrides any time-based snapshot retention the provider has). -- `compactHistory({ retainGenerations?, retainMs? })` reclaims record-sets. - A record-set `N` is reclaimed only when `N` is at or below **every** live +- The constructor **`retention`** knob governs auto-compaction (on `flush()`/ + `close()`): unset → ADAPTIVE (disk/RAM-pressure byte budget, zero-config; + driven by a coordinator's `budgetBytes` or a local `os.freemem` probe) · + `'all'` → unbounded · `{ maxGenerations?, maxAge?, maxBytes? }` → explicit + CAPS. `compactHistory({ maxGenerations?, maxAge?, maxBytes? })` reclaims + manually on the same caps — the oldest unpinned record-sets are reclaimed + while ANY supplied cap is exceeded. +- A record-set `N` is reclaimed only when `N` is at or below **every** live pin — deleting `N` can only break readers pinned *below* `N`, because resolution reads before-images from generations strictly greater than the - pin. With both retention options supplied, both must allow the reclaim. + pin. Live pins are ALWAYS exempt, in every retention mode. - The manifest records the **horizon** (highest reclaimed generation). Generations below the horizon are unreachable; `asOf()` on them throws `GenerationCompactedError`. The horizon itself stays reachable, resolved diff --git a/docs/BATCHING.md b/docs/BATCHING.md index 96239ba6..e70a9e18 100644 --- a/docs/BATCHING.md +++ b/docs/BATCHING.md @@ -78,10 +78,10 @@ for (const [id, metadata] of metadataMap) { - ✅ Direct O(1) path construction from ID (no type lookup needed!) - ✅ Sharding preservation (all paths include `{shard}/{id}`) - ✅ Write-cache coherent (read-after-write consistency) -- ✅ 40x faster than v5.x type-first architecture +- ✅ O(1) path construction — eliminates the per-entity type search of the old type-first layout **Performance:** -- ~1ms per 100 entities (consistent, no cache misses!) +- Constant-time path construction per ID — no type-cache misses - Filesystem: parallel reads bounded by IOPS - No type search delays — every ID maps directly to storage path @@ -121,7 +121,7 @@ for (const [sourceId, verbs] of results) { - Bulk export of relationship data **Performance:** -- Memory storage: <10ms for 1000 relationships +- Memory storage: single in-memory pass over the metadata index - Filesystem storage: parallel reads through the metadata index --- @@ -171,7 +171,7 @@ entities/verbs/{SHARD}/{ID}/metadata.json **Direct O(1) Path Construction:** ```typescript -// Every ID maps directly to exactly ONE path - 40x faster! +// Every ID maps directly to exactly ONE path - O(1), no type search const id = 'abc-123' const shard = getShardIdFromUuid(id) // → 'ab' (first 2 hex chars) const path = `entities/nouns/${shard}/${id}/metadata.json` @@ -183,9 +183,9 @@ const path = `entities/nouns/${shard}/${id}/metadata.json` ``` **Benefits:** -- **40x faster** path lookups (eliminates 42-type sequential search) +- **O(1)** path lookups (eliminates the 42-type sequential search the old type-first layout required) - **Simpler code** - removed 500+ lines of type cache complexity -- **Scalable** - works at billion-scale without type tracking overhead +- **Scalable** - works at large scale without type tracking overhead --- @@ -220,28 +220,22 @@ await db.release() --- -## Performance Benchmarks +## Why Batching Is Faster -### VFS Operations (12 Files) +Batching's advantage is structural, not a fixed multiplier (the actual speedup +depends on storage backend, IOPS, and batch size): -| Storage | Before optimization | After optimization | Improvement | -|---------|---------------------|--------------------|-------------| -| **Memory** | 150ms | 50ms | **67% faster** | -| **Filesystem** | ~500ms | ~80ms | **84% faster** | +- **N+1 elimination** — N sequential reads collapse into a single parallel pass + (`Promise.all` over the batch). +- **O(1) path construction** — every ID maps directly to one storage path, with + no per-type cache lookup. +- **One metadata round-trip** — relationship batches fetch all sources' metadata + in a single pass instead of one query per source. -### Entity Batch Retrieval (100 Entities) - -| Storage | Individual Gets | Batch Get | Improvement | -|---------|-----------------|-----------|-------------| -| **Memory** | 180ms | 15ms | **92% faster** | -| **Filesystem** | ~1.2s | ~120ms | **90% faster** | - -### Throughput (Entities/Second) - -| Storage | Individual | Batch | Improvement | -|---------|------------|-------|-------------| -| **Memory** | 556 ent/s | 6667 ent/s | **12x** | -| **Filesystem** | ~80 ent/s | ~800 ent/s | **10x** | +The integration test `tests/integration/storage-batch-operations.test.ts` +exercises batch vs. individual reads and asserts that batch retrieval is not +slower than the per-entity loop for large batches; it does not pin a specific +multiplier, since that is hardware- and IOPS-dependent. --- @@ -306,7 +300,7 @@ const results = await brain.batchGet(ids) const entities = Array.from(results.values()) ``` -**Performance Gain:** 10-20x faster on filesystem storage. +**Performance Gain:** Replaces N sequential reads with a single batched pass — no fixed multiplier, it scales with storage IOPS. --- @@ -332,7 +326,7 @@ for (const verbs of results.values()) { } ``` -**Performance Gain:** 5-10x faster due to batched metadata fetches. +**Performance Gain:** One batched metadata fetch instead of one query per source entity. --- @@ -450,8 +444,8 @@ return await Promise.all(resolvedPaths.map(path => this.read(path))) - `storage.getVerbsBySourceBatch(sourceIds, verbType?)` - Batch relationship queries **Performance Improvements:** -- VFS operations: 90%+ faster than the naive per-entity loop -- Entity retrieval: 10-20x throughput improvement +- VFS operations: single batched pass instead of N sequential reads +- Entity retrieval: N+1 reads collapsed into one batched pass - Zero N+1 query patterns **Compatibility:** diff --git a/docs/FIND_SYSTEM.md b/docs/FIND_SYSTEM.md index 5bf8ef1a..1cc38ce9 100644 --- a/docs/FIND_SYSTEM.md +++ b/docs/FIND_SYSTEM.md @@ -22,7 +22,7 @@ Brainy's `find()` method is the most advanced query system in any vector databas ### 1. Vector Intelligence (HNSW Index) - **Purpose**: Semantic similarity search using embeddings - **Algorithm**: Hierarchical Navigable Small World (HNSW) -- **Performance**: O(log n) search, ~1.8ms typical +- **Performance**: O(log n) search - **Data Structure**: Multi-layer graph with 16 connections per node - **Use Cases**: "Find similar documents", "Content like this" @@ -36,14 +36,14 @@ Brainy's `find()` method is the most advanced query system in any vector databas ### 3. Metadata Intelligence (Incremental Indices) - **Purpose**: Fast filtering on structured data - **Algorithm**: HashMap for exact matches, Sorted arrays for ranges -- **Performance**: O(1) exact, O(log n) ranges, <1ms typical +- **Performance**: O(1) exact, O(log n) ranges - **Data Structure**: `Map>` + sorted value arrays - **Use Cases**: "Documents from 2023", "Status equals active" ### 4. Graph Intelligence (Adjacency Maps) - **Purpose**: Relationship traversal and connection analysis - **Algorithm**: Pure O(1) neighbor lookups via Map operations -- **Performance**: O(1) per hop, ~0.1ms typical +- **Performance**: O(1) per hop (measured <1 ms per neighbor lookup, validated up to 1M relationships — `tests/performance/graph-scale-performance.test.ts:238`) - **Data Structure**: `Map>` - **Use Cases**: "Papers connected to MIT", "Authors who collaborated" @@ -373,36 +373,40 @@ return results.slice(offset, offset + limit) ### Query Performance by Type -| Query Type | Index Used | Performance | Example | -|------------|------------|-------------|---------| -| **Semantic Search** | HNSW Vector | O(log n), ~1.8ms | `"AI research papers"` | -| **Exact Metadata** | HashMap | O(1), ~0.8ms | `{status: "published"}` | -| **Range Metadata** | Sorted Array | O(log n), ~0.6ms | `{year: {greaterThan: 2020}}` | -| **Graph Traversal** | Adjacency Map | O(1), ~0.1ms | `{connected: {to: "mit"}}` | -| **Type Detection** | Pre-embedded Types | O(t), ~0.3ms | `"documents"` → `NounType.Document` | -| **Field Matching** | Field Embeddings | O(f), ~0.1ms | `"by"` → `"author"` | -| **Combined Query** | All Indices | O(log n), ~1.8ms | NLP + filters + graph | +| Query Type | Index Used | Complexity | Example | +|------------|------------|------------|---------| +| **Semantic Search** | HNSW Vector | O(log n) | `"AI research papers"` | +| **Exact Metadata** | HashMap | O(1) | `{status: "published"}` | +| **Range Metadata** | Sorted Array | O(log n) | `{year: {greaterThan: 2020}}` | +| **Graph Traversal** | Adjacency Map | O(1) | `{connected: {to: "mit"}}` | +| **Type Detection** | Pre-embedded Types | O(t) | `"documents"` → `NounType.Document` | +| **Field Matching** | Field Embeddings | O(f) | `"by"` → `"author"` | +| **Combined Query** | All Indices | O(log n) | NLP + filters + graph | Where: - n = number of entities in database - t = number of types (169 total: 42 noun + 127 verb) - f = number of fields for detected entity type (typically 5-15) +Absolute latencies depend on hardware, embedding model, and storage backend; the columns above describe how each stage scales. The graph stage is the one path with a committed scale benchmark (see [Scalability](#scalability)). + ### Scalability -| Database Size | Vector Search | Metadata Filter | Graph Query | Combined | -|---------------|---------------|-----------------|-------------|----------| -| **1K entities** | 0.8ms | 0.3ms | 0.05ms | 1.1ms | -| **10K entities** | 1.2ms | 0.5ms | 0.08ms | 1.5ms | -| **100K entities** | 1.8ms | 0.8ms | 0.1ms | 2.1ms | -| **1M entities** | 2.5ms | 1.2ms | 0.1ms | 2.8ms | -| **10M entities** | 3.8ms | 1.8ms | 0.1ms | 4.2ms | +The cost of each query stage is governed by its algorithmic complexity, not a fixed millisecond figure — absolute latency depends on hardware, embedding model, and storage backend. Only the graph adjacency index carries a committed scale assertion: + +| Query stage | Complexity | Scaling behavior | +|-------------|------------|------------------| +| Metadata filter (exact) | O(1) | Constant — independent of dataset size | +| Metadata filter (range) | O(log n) + O(k) | Sub-linear; k = matching results | +| Vector search (HNSW) | O(log n) | Degrades gracefully via hierarchical layers | +| Graph hop | O(1) | Measured <1 ms per neighbor lookup, validated up to 1M relationships (`tests/performance/graph-scale-performance.test.ts:238`) | +| Combined query | O(log n) | Bounded by the vector stage; metadata and graph stages stay O(1)/O(log n) | **Key Performance Notes:** - Graph queries stay O(1) regardless of scale - Metadata ranges scale as O(log n), not O(n) - Vector search degrades gracefully due to HNSW -- Type-aware NLP adds minimal overhead (~0.4ms) +- Type-aware NLP adds minimal overhead (single embedding pass, no full scan) ## Example Query Flows @@ -455,7 +459,7 @@ await brain.find({ }, limit: 10 }) -// Total performance: ~1.2ms for 100K entities +// Executes as O(1) type filter + O(1)/O(log n) metadata + O(1) graph lookup — no full scan ``` ## Filter Syntax Reference @@ -1001,7 +1005,7 @@ const results = await brain.find({ }) ``` -**Performance**: O(log n) vector search + O(log n) metadata filters + O(1) graph traversal = ~2-3ms total. +**Performance**: O(log n) vector search + O(log n) metadata filters + O(1) graph traversal — bounded by the vector stage. ### Excluding Soft-Deleted Entities diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index 0caa123d..248a2c70 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -2,11 +2,11 @@ ## Performance Characteristics -Brainy achieves industry-leading performance through carefully optimized data structures and algorithms. All performance claims are verified through actual benchmarks on production code. +Brainy achieves high performance through carefully optimized data structures and algorithms. The tables below describe each component by its **algorithmic complexity** — the durable, defensible guarantee. The example latencies are figures from a single 100-item run on one machine (see [Benchmarks](#benchmarks)); they are illustrative, not a committed benchmark, and vary with hardware, embedding model, and storage backend. The one component with a committed scale assertion is the graph adjacency index (`tests/performance/graph-scale-performance.test.ts:238`). ### Core Performance Summary -| Component | Operation | Time Complexity | Measured Performance | Data Structure | +| Component | Operation | Time Complexity | Example latency (100-item run)\* | Data Structure | |-----------|-----------|-----------------|---------------------|----------------| | **Metadata Index** | Exact match | **O(1)** | 0.8ms | `Map>` | | **Metadata Index** | Range query | **O(log n) + O(k)** | 0.6ms | Sorted array + binary search | @@ -17,6 +17,8 @@ Brainy achieves industry-leading performance through carefully optimized data st | **Type Detection** | Noun/Verb matching | **O(t)** | 0.3ms | Pre-embedded type vectors | | **Triple Intelligence** | Combined query | **O(1) to O(log n)** | 1.8ms | Parallel execution | +\* Illustrative single-run figures at 100 items on one machine — not a committed benchmark. Only the graph index carries an asserted scale bound (measured <1 ms per neighbor lookup up to 1M relationships, `tests/performance/graph-scale-performance.test.ts:238`). + Where: - `n` = number of items in index - `k` = number of results returned @@ -26,25 +28,32 @@ Where: ### brain.get() Metadata-Only Optimization -✨ **Massive Performance Improvement**: `brain.get()` is now **76-81% faster** by default! +`brain.get()` returns **metadata only by default**, skipping the 384-dimensional +embedding — the bulk of an entity's payload. Callers that need the vector opt in +with `{ includeVectors: true }`. -| Operation | Before | After | Speedup | Use Case | -|-----------|------------------|-----------------|---------|----------| -| **brain.get() (metadata-only)** | 43ms, 6KB | **10ms, 300 bytes** | **76-81%** | VFS, existence checks, metadata | -| **brain.get({ includeVectors: true })** | 43ms, 6KB | 43ms, 6KB | 0% | Similarity calculations | -| **VFS readFile()** | 53ms | **~13ms** | **75%** | File operations | -| **VFS readdir(100 files)** | 5.3s | **~1.3s** | **75%** | Directory listings | +| Operation | Default (metadata-only) | With `includeVectors: true` | Use Case | +|-----------|-------------------------|-----------------------------|----------| +| **brain.get()** | Skips vector load | Loads full vector | VFS, existence checks, metadata | +| **VFS readFile() / readdir()** | Inherits metadata-only path | n/a | File operations, directory listings | -**Key Innovation**: Lazy vector loading - only load 384-dimensional embeddings when explicitly needed. +**Key Innovation**: Lazy vector loading — only load the 384-dimensional embedding when explicitly needed. + +The integration test `tests/integration/metadata-only-comprehensive.test.ts:306` +asserts metadata-only `get()` is faster than the full-entity `get()` +(`metadataTime < fullTime`). The *magnitude* of the speedup is +environment-dependent (the percentage assertion in +`tests/integration/vfs-performance-v5.11.1.test.ts` is intentionally skipped on +CI for that reason), so no fixed percentage is quoted here. **Why this matters**: -- **94% of brain.get() calls** don't need vectors (VFS, admin tools, import utilities, data APIs) -- **Vector data is 95% of entity size** (6KB vectors vs 300 bytes metadata) -- **Zero code changes** for most applications - automatic speedup! +- Most `brain.get()` calls don't need vectors (VFS, admin tools, import utilities, data APIs) +- The embedding dominates an entity's serialized size, so skipping it is the largest win +- **Zero code changes** for most applications — automatic by default **When to use what**: ```typescript -// DEFAULT: Metadata-only (76-81% faster) - use for: +// DEFAULT: Metadata-only (skips the vector load) - use for: const entity = await brain.get(id) // - VFS operations (readFile, stat, readdir) // - Existence checks: if (await brain.get(id)) ... @@ -252,28 +261,34 @@ const results = await Promise.all(searchPromises) ## Benchmarks -### Real-world Performance Test (100 items) +### Illustrative Single Run (100 items, one machine) + +Example output from a single 100-item run — illustrative only, not a committed +benchmark; absolute numbers vary by hardware. The values feed the +[Core Performance Summary](#core-performance-summary) example-latency column. ``` -📊 Metadata exact match: 0.818ms (50 items matched) -📊 Metadata range query: 0.631ms (40 items in range) -🔗 Graph neighbor lookup: 0.092ms (2 connections) -🎯 Vector k-NN search: 1.773ms (10 nearest neighbors) -🧠 NLP query parsing: 8.906ms (full natural language) -⚡ Triple Intelligence: 1.830ms (combined query) +Metadata exact match: 0.818ms (50 items matched) +Metadata range query: 0.631ms (40 items in range) +Graph neighbor lookup: 0.092ms (2 connections) +Vector k-NN search: 1.773ms (10 nearest neighbors) +NLP query parsing: 8.906ms (full natural language) +Triple Intelligence: 1.830ms (combined query) ``` ### Scaling Characteristics -| Items | Metadata O(1) | Range O(log n) | Graph O(1) | Vector O(log n) | -|-------|---------------|----------------|------------|-----------------| -| 100 | 0.8ms | 0.6ms | 0.09ms | 1.8ms | -| 1,000 | 0.8ms | 0.9ms | 0.09ms | 2.5ms | -| 10,000 | 0.8ms | 1.2ms | 0.09ms | 3.2ms | -| 100,000 | 0.8ms | 1.5ms | 0.09ms | 4.1ms | -| 1,000,000 | 0.8ms | 1.8ms | 0.09ms | 5.0ms | +Each stage scales by its algorithmic complexity, not a fixed millisecond figure +— absolute latency depends on hardware, embedding model, and storage backend. +Only the graph adjacency index carries a committed scale assertion: -*Note: O(1) operations maintain constant time regardless of scale* +| Query stage | Complexity | Scaling behavior | +|-------------|------------|------------------| +| Metadata filter (exact) | O(1) | Constant — independent of dataset size | +| Metadata filter (range) | O(log n) + O(k) | Sub-linear; k = matching results | +| Vector search (HNSW) | O(log n) | Degrades gracefully via hierarchical layers | +| Graph hop | O(1) | Measured <1 ms per neighbor lookup, validated up to 1M relationships (`tests/performance/graph-scale-performance.test.ts:238`) | +| Combined query | O(log n) | Bounded by the vector stage; metadata and graph stages stay O(1)/O(log n) | ## Comparison with Other Systems @@ -402,7 +417,7 @@ const brain = new Brainy({ }) ``` -The default JS index is `JsHnswVectorIndex`. An optional native acceleration package (`@soulcraft/cortex`) can replace it with a higher-performing implementation; the public knobs stay the same. +The default JS index is `JsHnswVectorIndex`. An optional native acceleration package (`@soulcraft/cor`) can replace it with a higher-performing implementation; the public knobs stay the same. ### Scale Scenarios @@ -470,8 +485,8 @@ For off-site replication, snapshot `path` from your scheduler (`gsutil rsync`, ` - **Auto-configuration system** with environment detection - **Zero-config operation** with intelligent defaults - **Auto-flush and auto-optimize** in indices -- **Sub-2ms response times** for complex queries +- **Low-latency Triple Intelligence queries** (O(log n) vector + O(1) metadata/graph) ## Conclusion -Brainy delivers on its promise of **production-ready Triple Intelligence** with measured, verified performance characteristics. All listed features are fully implemented, tested, and benchmarked. No stubs, no mocks, no theoretical claims - just real, working code with measured performance. \ No newline at end of file +Brainy delivers on its promise of **production-ready Triple Intelligence** with documented algorithmic-complexity guarantees and a committed graph-scale benchmark (`tests/performance/graph-scale-performance.test.ts`). All listed features are fully implemented and tested. No stubs, no mocks — just real, working code with characterized performance. \ No newline at end of file diff --git a/docs/PLUGINS.md b/docs/PLUGINS.md index 8e82b646..d9a4d3e7 100644 --- a/docs/PLUGINS.md +++ b/docs/PLUGINS.md @@ -12,7 +12,7 @@ next: # Plugin Development Guide -Brainy has a plugin system that allows third-party packages to replace internal subsystems with custom implementations. This is how `@soulcraft/cortex` provides optional native acceleration, and it's the same system available to any developer. +Brainy has a plugin system that allows third-party packages to replace internal subsystems with custom implementations. This is how `@soulcraft/cor` provides optional native acceleration, and it's the same system available to any developer. ## Architecture Overview @@ -23,20 +23,19 @@ Brainy's plugin system uses **named providers** — string keys mapped to implem 3. The plugin calls `context.registerProvider(key, implementation)` for each subsystem it provides 4. Brainy checks each provider key and wires the implementation into its internal pipeline -Plugins are **opt-in** — brainy never auto-imports packages. You must explicitly list plugins in the config: +Installing the first-party accelerator is the opt-in: with the default config, brainy probes for `@soulcraft/cor` and loads it when present. Everything except "not installed" fails **loud** — a present-but-broken accelerator makes `init()` throw rather than silently degrading to the JS engines. ```typescript -const brain = new Brainy({ - plugins: ['@soulcraft/cortex'] // explicitly load the native acceleration package -}) +const brain = new Brainy() // @soulcraft/cor auto-detected when installed +const pinned = new Brainy({ plugins: ['@soulcraft/cor'] }) // or pin exactly what loads +const plain = new Brainy({ plugins: [] }) // or opt out of detection entirely ``` | `plugins` value | Behavior | |---|---| -| `undefined` (default) | No plugins loaded | -| `false` | No plugins loaded | -| `[]` | No plugins loaded | -| `['@soulcraft/cortex']` | Load only the listed packages | +| `undefined` (default) | Guarded auto-detection of `@soulcraft/cor`: not installed → no plugins, silently; installed → loads + announces; installed-but-broken → `init()` throws | +| `false` / `[]` | No plugins, no detection (explicit opt-out) | +| `['@soulcraft/cor']` | Load only the listed packages; a listed plugin that fails to load throws | Plugins registered programmatically via `brain.use(plugin)` are always activated regardless of the `plugins` config. @@ -70,7 +69,7 @@ export default myPlugin ### 2. Package exports -Your package must export the plugin as the default export so brainy's auto-detection works: +Your package must export the plugin as the default export so brainy's plugin loader can resolve it: ```typescript // index.ts @@ -150,6 +149,17 @@ context.registerProvider('embedBatch', async (texts: string[]) => { ### Index Providers +> **Write-path invariant (the change-feed contract).** Every canonical +> mutation flows through Brainy's generation-store commit points — index +> providers are invoked *inside* that commit and never originate canonical +> writes of their own. The `brain.onChange` change feed is emitted from those +> commit points and relies on this: **a plugin must never introduce a write +> path that bypasses the generation-store commit.** If a future provider ever +> needs a direct native ingest path, it must either route through the commit +> or emit equivalent change events — otherwise every `onChange` consumer +> (live UIs, cache invalidation, realtime sync) silently develops a blind +> spot. + #### `vector` **Type:** `(config: object, distanceFunction: Function, options: object) => VectorIndexProvider-compatible` @@ -182,6 +192,27 @@ context.registerProvider('vector', (config, distanceFn, options) => { }) ``` +#### The readiness contract (all three index providers) + +A provider that **persists its derived index** should implement the optional readiness +members so a warm reopen never pays a redundant rebuild-from-canonical: + +- **`init?(): Promise`** — eager cold-load. Brainy awaits it once during + `brain.init()`, after the metadata provider's `init()` (the id-mapper hydrates first) + and **before the rebuild gate**. +- **`isReady?(): boolean`** — honest durability signal. `true` ⇔ the persisted index is + loaded (or cheaply demand-loadable) and consistent with what was last persisted. When + exposed, the rebuild gate defers to this signal **instead of** the `size() === 0` / + `totalEntries === 0` heuristics — a disk-native index may report 0 resident entries + while fully durable. Never return `true` if the durable state failed to load: the + signal is honest in both directions, and a not-ready provider gets its rebuild even + when `size() > 0`. +- **`isMigrating?(): boolean`** — while `true`, the provider owns its index (background + migration); brainy skips its rebuild entirely. + +Providers that implement none of these keep the size/count heuristics — correct for +engines whose `rebuild()` *is* their load path (like brainy's built-in JS vector index). + #### `metadataIndex` **Type:** `(storage: StorageAdapter) => MetadataIndexManager-compatible` @@ -215,7 +246,7 @@ context.registerProvider('aggregation', (storage) => { }) ``` -When provided by an optional native acceleration plugin (such as `@soulcraft/cortex`), this enables: +When provided by an optional native acceleration plugin (such as `@soulcraft/cor`), this enables: - Compiled source filters (vs per-entity JS object traversal) - Precise MIN/MAX via sorted data structures (vs lazy recompute) - Parallel aggregate rebuild across CPU cores @@ -251,7 +282,7 @@ Native msgpack encode/decode for SSTable serialization. ### Analytics Providers (Native-Only) -These provider keys have **no JavaScript fallback** — they represent capabilities that require native code (SIMD, mmap, sub-microsecond latency). They are available when an optional native acceleration plugin (such as `@soulcraft/cortex`) is installed. +These provider keys have **no JavaScript fallback** — they represent capabilities that require native code (SIMD, mmap, sub-microsecond latency). They are available when an optional native acceleration plugin (such as `@soulcraft/cor`) is installed. Use `brain.getProvider('analytics:hyperloglog')` to check availability. Returns `undefined` if no plugin provides it. @@ -359,8 +390,8 @@ brainy diagnostics When a plugin is active, brainy automatically logs a provider summary after `init()`: ``` -[brainy] Plugin activated: @soulcraft/cortex -[brainy] Providers: 8/10 native (@soulcraft/cortex) | default: vector, cache +[brainy] Plugin activated: @soulcraft/cor +[brainy] Providers: 8/10 native (@soulcraft/cor) | default: vector, cache ``` This tells you at a glance how many subsystems are accelerated and which ones are falling back to JavaScript. The log respects `config.silent`. @@ -381,7 +412,7 @@ If a required provider is missing, the error message tells you exactly what's wr ``` [brainy] Required providers using JS fallback: graphIndex. -Active plugins: @soulcraft/cortex. +Active plugins: @soulcraft/cor. These providers must be supplied by a plugin for this deployment. Check plugin installation, license, and native module availability. ``` diff --git a/docs/PRODUCTION_SERVICE_ARCHITECTURE.md b/docs/PRODUCTION_SERVICE_ARCHITECTURE.md index a7e088f4..4568cd31 100644 --- a/docs/PRODUCTION_SERVICE_ARCHITECTURE.md +++ b/docs/PRODUCTION_SERVICE_ARCHITECTURE.md @@ -215,9 +215,9 @@ console.log('Server running on http://localhost:3000') ``` **Benefits:** -- ✅ Native Bun performance (~2x faster than Node.js) +- ✅ Native Bun runtime performance - ✅ No framework dependencies -- ✅ Works with `bun --compile` for single-binary deployment +- ✅ Pure WASM — no native binaries, bundler-friendly - ✅ Built-in TypeScript support ### Pattern 4: Express/Node.js Middleware (Legacy) diff --git a/docs/QUERY_OPERATORS.md b/docs/QUERY_OPERATORS.md index 16482de1..f4ac04e6 100644 --- a/docs/QUERY_OPERATORS.md +++ b/docs/QUERY_OPERATORS.md @@ -10,8 +10,8 @@ All operators work with `find({ where: { ... } })` and filter on **metadata fiel | Operator | Alias | Description | Example | |----------|-------|-------------|---------| -| `equals` | `eq`, `is` | Exact match | `{ status: { equals: 'active' } }` | -| `notEquals` | `ne`, `isNot` | Not equal | `{ status: { notEquals: 'deleted' } }` | +| `eq` | `equals` | Exact match | `{ status: { eq: 'active' } }` | +| `ne` | `notEquals` | Not equal | `{ status: { ne: 'deleted' } }` | **Shorthand:** A bare value is treated as `equals`: @@ -27,10 +27,10 @@ brain.find({ where: { status: { equals: 'active' } } }) | Operator | Alias | Description | Example | |----------|-------|-------------|---------| -| `greaterThan` | `gt` | Greater than | `{ age: { greaterThan: 18 } }` | -| `greaterEqual` | `gte` | Greater or equal | `{ score: { greaterEqual: 90 } }` | -| `lessThan` | `lt` | Less than | `{ price: { lessThan: 100 } }` | -| `lessEqual` | `lte` | Less or equal | `{ rating: { lessEqual: 3 } }` | +| `gt` | `greaterThan` | Greater than | `{ age: { gt: 18 } }` | +| `gte` | `greaterThanOrEqual` | Greater or equal | `{ score: { gte: 90 } }` | +| `lt` | `lessThan` | Less than | `{ price: { lt: 100 } }` | +| `lte` | `lessThanOrEqual` | Less or equal | `{ rating: { lte: 3 } }` | | `between` | — | Inclusive range `[min, max]` | `{ year: { between: [2020, 2025] } }` | ```typescript @@ -160,9 +160,9 @@ Brainy's MetadataIndex supports a subset of operators natively for O(1) field lo | `equals` / `eq` | Yes | Yes | | `notEquals` / `ne` | — | Yes | | `greaterThan` / `gt` | Yes | Yes | -| `greaterEqual` / `gte` | Yes | Yes | +| `greaterThanOrEqual` / `gte` | Yes | Yes | | `lessThan` / `lt` | Yes | Yes | -| `lessEqual` / `lte` | Yes | Yes | +| `lessThanOrEqual` / `lte` | Yes | Yes | | `between` | Yes | Yes | | `oneOf` / `in` | Yes | Yes | | `noneOf` | — | Yes | @@ -234,7 +234,7 @@ const all = await brain.related({ }) // Graph traversal — subtype filters traversal edges (depth-1 in 7.30 JS path; -// multi-hop subtype filtering lands on Cortex native) +// multi-hop subtype filtering lands on Cor native) const reports = await brain.find({ connected: { from: ceoId, diff --git a/docs/RELEASE-GUIDE.md b/docs/RELEASE-GUIDE.md index ce7d32c5..94c6a6cb 100644 --- a/docs/RELEASE-GUIDE.md +++ b/docs/RELEASE-GUIDE.md @@ -119,4 +119,13 @@ gh release create vY.Y.Y --generate-notes - **Most releases should be MINOR or PATCH** - **Major versions should be RARE** - **When in doubt, it's probably MINOR** -- **NEVER use "BREAKING CHANGE" for internal changes** \ No newline at end of file +- **NEVER use "BREAKING CHANGE" for internal changes** +## Hard Ordering Constraints (check before EVERY release) + +- **Embedding model changes are SEQUENCED, not free.** No release may change the + embedding model (or its quantization/dimensions) before **vector model-version + stamping + hard-error-on-mismatch** ships. Stored vectors carry no model version + today; mixing vectors from two models silently corrupts every similarity + comparison. If a model bump is ever proposed, the stamping work moves ahead of + it in the schedule — coordinate with the native provider so both engines stamp + and enforce identically. (Registered with the native-provider team 2026-07-07.) diff --git a/docs/SCALING.md b/docs/SCALING.md index c1e00987..e9ae1136 100644 --- a/docs/SCALING.md +++ b/docs/SCALING.md @@ -37,7 +37,7 @@ The three knobs that matter most: 1. **`config.vector.recall`** — `'fast'`, `'balanced'`, or `'accurate'` (default `'balanced'`) 2. **`config.vector.persistMode`** — `'immediate'` for durability, `'deferred'` for throughput -The native vector provider (via the optional `@soulcraft/cortex` package) extends this with a higher-performing index — and its own at-scale acceleration such as on-disk compressed indexing — when installed. +The native vector provider (via the optional `@soulcraft/cor` package) extends this with a higher-performing index — and its own at-scale acceleration such as on-disk compressed indexing — when installed. ## Measured Performance @@ -77,7 +77,7 @@ graph, metadata index, and 100k edges). **Scale ceiling (open-core).** The pure-JS HNSW *build* cost (~100 inserts/s at 384-dim on one core) makes the in-process open-core path most appropriate up to ~10⁵–10⁶ entities. *Query* latency stays low well beyond that, but for the 10⁸–10¹⁰ regime install the native -provider (`@soulcraft/cortex`, on-disk DiskANN) — same API, no code change. _Projected from +provider (`@soulcraft/cor`, on-disk DiskANN) — same API, no code change. _Projected from the two measured points, vector p50 at 1M is ~2 ms; metadata-heavy composition grows with match-set size and is the path to move onto the native provider first._ @@ -213,12 +213,12 @@ const stats = await brain.stats() ### Issue: Slow queries 1. Switch to `vector.recall: 'fast'` 2. Increase the read cache (`storage.cache.maxSize`) -3. Consider the optional native vector provider via `@soulcraft/cortex` +3. Consider the optional native vector provider via `@soulcraft/cor` ### Issue: Memory pressure 1. Reduce `storage.cache.maxSize` 2. Move to `vector.persistMode: 'deferred'` to batch writes -3. Consider the optional native vector provider via `@soulcraft/cortex` for at-scale index acceleration +3. Consider the optional native vector provider via `@soulcraft/cor` for at-scale index acceleration ### Issue: Slow startup after a crash 1. Use `vector.persistMode: 'immediate'` so the index file stays in sync with storage diff --git a/docs/api/README.md b/docs/api/README.md index fa35ff29..4ca84364 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -413,9 +413,9 @@ Brainy uses clean, readable operators (BFO — Brainy Field Operators): | `equals` / `eq` | Exact match | `{age: {equals: 25}}` | | `notEquals` / `ne` | Not equal | `{status: {notEquals: 'deleted'}}` | | `greaterThan` / `gt` | Greater than | `{age: {greaterThan: 18}}` | -| `greaterEqual` / `gte` | Greater or equal | `{score: {greaterEqual: 90}}` | +| `gte` / `greaterThanOrEqual` | Greater or equal | `{score: {gte: 90}}` | | `lessThan` / `lt` | Less than | `{price: {lessThan: 100}}` | -| `lessEqual` / `lte` | Less or equal | `{rating: {lessEqual: 3}}` | +| `lte` / `lessThanOrEqual` | Less or equal | `{rating: {lte: 3}}` | | `between` | Inclusive range | `{year: {between: [2020, 2025]}}` | | `oneOf` / `in` | In array | `{color: {oneOf: ['red', 'blue']}}` | | `noneOf` | Not in array | `{status: {noneOf: ['deleted']}}` | @@ -901,16 +901,18 @@ that generation), once per `Db`, freed on `release()`. **Throws:** `GenerationCompactedError` when the generation's records were reclaimed by `compactHistory()`. -**History granularity:** only `transact()` batches produce historical -records; single-operation writes advance the clock but stay visible through -earlier pins. See the [consistency model](../concepts/consistency-model.md). +**History granularity:** every write is its own immutable generation — +`transact()` batches AND single-operation writes — so a pin always freezes and +every write is addressable via `asOf()`. See the +[consistency model](../concepts/consistency-model.md). --- ### `transactionLog(options?)` → `Promise` -Read the reified transaction log — one entry per committed `transact()` -batch, newest first: `{ generation, timestamp, meta? }`. +Read the reified transaction log — one entry per committed generation (every +`transact()` AND single-op write), newest first: `{ generation, timestamp, +meta? }`. Single-op generations carry no `meta` (it is a `transact()`-only field). ```typescript const [latest] = await brain.transactionLog({ limit: 1 }) @@ -921,13 +923,15 @@ latest.meta // { author: 'order-service', requestId: 'req-9f2' } ### `compactHistory(options?)` → `Promise` -Reclaim historical record-sets that no retention rule and no live `Db` pin -protects. Pinned reads stay correct across compaction, always. +Reclaim historical record-sets that no retention cap and no live `Db` pin +protects. Pinned reads stay correct across compaction, always. (Auto-compaction +on `flush()`/`close()` is governed by the constructor `retention` knob — unset → +adaptive, `'all'` → unbounded, `{ … }` → explicit caps.) ```typescript await brain.compactHistory({ - retainGenerations: 100, // keep the 100 most recent commits - retainMs: 7 * 24 * 60 * 60 * 1000 // and everything from the last 7 days + maxGenerations: 100, // keep at most the 100 most recent generations + maxAge: 7 * 24 * 60 * 60 * 1000 // and only those from the last 7 days }) ``` diff --git a/docs/architecture/aggregation.md b/docs/architecture/aggregation.md index 10b55689..443ee727 100644 --- a/docs/architecture/aggregation.md +++ b/docs/architecture/aggregation.md @@ -33,7 +33,7 @@ │ ▼ │ │ ┌──────────────────────┐ │ │ │ AggregationProvider │ (optional, registered by │ -│ │ ├─ incrementalUpdate│ plugin like @soulcraft/cortex)│ +│ │ ├─ incrementalUpdate│ plugin like @soulcraft/cor) │ │ │ ├─ rebuildAggregate │ │ │ │ ├─ queryAggregate │ │ │ │ └─ serialize/restore│ │ @@ -135,7 +135,7 @@ M2 is clamped to zero on remove to prevent floating-point drift from producing n The TypeScript engine uses simple comparison for add operations and marks MIN/MAX as potentially stale on delete (since removing the current min/max value requires a rescan). Stale values are lazily recomputed on the next query. -The Cortex native engine uses a `BTreeMap, u64>` that tracks the exact frequency of every value, providing precise MIN/MAX after any sequence of operations without rescanning. +The Cor native engine uses a `BTreeMap, u64>` that tracks the exact frequency of every value, providing precise MIN/MAX after any sequence of operations without rescanning. ### Time Window Bucketing diff --git a/docs/architecture/data-storage-architecture.md b/docs/architecture/data-storage-architecture.md index a51ebf05..48064757 100644 --- a/docs/architecture/data-storage-architecture.md +++ b/docs/architecture/data-storage-architecture.md @@ -369,7 +369,10 @@ await brain.restore('/backups/today', { confirm: true }) // replace store stat through the metadata index, not the filesystem. - **`_system/`** holds singletons plus 256 hash buckets of index state. - **`_generations/` + `manifest.json` + `tx-log.jsonl`** implement - generational MVCC; `transact()` is the unit of history. + generational MVCC. History is per-write: every `add()`/`update()`/`remove()`/ + `relate()` gets its own generation, and `transact()` groups several ops into + one atomic generation. (Single-op retention has been the model since 8.0; + you never need to route a write through `transact()` just to keep its history.) - **`_column_index/` + `_blobs/`** hold the columnar metadata runs and binary blobs (VFS content included). - **`locks/`** coordinates the single writer and reader flush requests, and diff --git a/docs/architecture/index-architecture.md b/docs/architecture/index-architecture.md index 993072e9..8b3dc540 100644 --- a/docs/architecture/index-architecture.md +++ b/docs/architecture/index-architecture.md @@ -183,7 +183,7 @@ async getIdsForMultipleFields(pairs: [{field, value}, ...]): Promise { - Memory usage: **90% reduction** (17.17 MB → 2.01 MB for 100K entities) - Hardware acceleration: SIMD instructions make bitmap operations nearly free -**Benchmark Results** (1,000 queries on various dataset sizes): +**Benchmark Results** — example output from a single run of `tests/performance/roaring-bitmap-benchmark.ts` (1,000 queries per size, one machine; absolute times vary by hardware, the relative speedup and memory savings are the durable signal): | Dataset Size | Operation | Set Time | Roaring Time | Speedup | Memory Savings | |--------------|-----------|----------|--------------|---------|----------------| | 10,000 entities | 3-field intersection | 3.74ms | 1.14ms | **3.3x faster** | 90% | @@ -402,7 +402,7 @@ const DEFAULT_EXCLUDE_FIELDS = [ **Purpose**: O(log n) semantic similarity search using vector embeddings. -The default JS implementation is `JsHnswVectorIndex`; an optional native acceleration package (`@soulcraft/cortex`) can register a higher-performing `VectorIndexProvider` through the plugin system. The public API stays the same either way. +The default JS implementation is `JsHnswVectorIndex`; an optional native acceleration package (`@soulcraft/cor`) can register a higher-performing `VectorIndexProvider` through the plugin system. The public API stays the same either way. ### Internal Architecture @@ -578,16 +578,6 @@ const reachable = await this.graphIndex.traverse({ // Complexity: O(V + E) breadth-first search, but each neighbor lookup is O(1) ``` -## Notes on Other Indexes - -### DeletedItemsIndex (Not Currently Used) - -**Status**: Utility class exists in `src/utils/deletedItemsIndex.ts` but is **not instantiated** in the Brainy class. - -**Purpose** (if used): O(1) tracking of soft-deleted items without removing data. - -**Current Behavior**: Brainy uses hard deletes via `storage.deleteNoun()` and `storage.deleteVerb()`. Soft-delete functionality is not currently integrated. - ## Shared Memory Management: UnifiedCache All three main indexes share a single **UnifiedCache** instance for coordinated memory management. @@ -684,9 +674,6 @@ async find(query: FindQuery): Promise { results = results.filter(r => connectedIds.includes(r.id)) } - // Step 4: Filter deleted items - results = results.filter(r => !this.deletedItemsIndex.isDeleted(r.id)) - return results } ``` @@ -728,9 +715,6 @@ async stats(): Promise { relationships: this.graphIndex.getTotalRelationshipCount(), relationshipTypes: this.graphIndex.getRelationshipCountsByType(), - // From deleted items index - deletedItems: this.deletedItemsIndex.getDeletedCount(), - // From vector index vectorIndexSize: this.index.getSize() } @@ -858,14 +842,15 @@ Where: ### Scalability -All indexes scale gracefully: +All indexes scale gracefully. The cost of each stage is governed by its algorithmic complexity, not a fixed millisecond figure — absolute latency depends on hardware, embedding model, and storage backend. Only the graph adjacency index carries a committed scale assertion: -| Database Size | Metadata Filter | Vector Search | Graph Hop | Combined Query | -|---------------|----------------|---------------|-----------|----------------| -| **1K entities** | 0.3ms | 0.8ms | 0.05ms | 1.1ms | -| **10K entities** | 0.5ms | 1.2ms | 0.08ms | 1.5ms | -| **100K entities** | 0.8ms | 1.8ms | 0.1ms | 2.1ms | -| **1M entities** | 1.2ms | 2.5ms | 0.1ms | 2.8ms | +| Query stage | Complexity | Scaling behavior | +|-------------|------------|------------------| +| Metadata filter (exact) | O(1) | Constant — independent of dataset size | +| Metadata filter (range) | O(log n) + O(k) | Sub-linear; k = matching results | +| Vector search (HNSW) | O(log n) | Degrades gracefully via hierarchical layers | +| Graph hop | O(1) | Measured <1 ms per neighbor lookup, validated up to 1M relationships (`tests/performance/graph-scale-performance.test.ts:238`) | +| Combined query | O(log n) | Bounded by the vector stage; metadata and graph stages stay O(1)/O(log n) | **Key observations**: - Graph queries stay O(1) regardless of scale diff --git a/docs/architecture/multiprocess-storage-mixin.md b/docs/architecture/multiprocess-storage-mixin.md index 7b53fa33..46f98398 100644 --- a/docs/architecture/multiprocess-storage-mixin.md +++ b/docs/architecture/multiprocess-storage-mixin.md @@ -22,7 +22,7 @@ requestFlushOverFilesystem(timeoutMs) They live on `BaseStorage` as no-op defaults and are overridden on `FileSystemStorage` with real implementations. Any adapter extending -`FileSystemStorage` (e.g. Cortex's `MmapFileSystemStorage`) inherits the +`FileSystemStorage` (e.g. Cor's `MmapFileSystemStorage`) inherits the real ones for free. This works correctly today. The question is whether the methods *belong* @@ -83,7 +83,7 @@ Benefits: ## The case against doing it now - Breaking change for any adapter that already overrides these methods. - `FileSystemStorage` is the only one in-tree, but Cortex's + `FileSystemStorage` is the only one in-tree, but Cor's `MmapFileSystemStorage` inherits from it — interface relocation would ripple through the plugin ecosystem. - The current state works. The real failure modes seen in the field diff --git a/docs/architecture/noun-verb-taxonomy.md b/docs/architecture/noun-verb-taxonomy.md index d896016d..286464be 100644 --- a/docs/architecture/noun-verb-taxonomy.md +++ b/docs/architecture/noun-verb-taxonomy.md @@ -5,7 +5,7 @@ public: true category: concepts template: concept order: 2 -description: 42 NounTypes and 127 VerbTypes cover 96-97% of all human knowledge. The universal vocabulary for structuring anything from people and documents to events and relationships. +description: 42 NounTypes and 127 VerbTypes cover ~95% of all domains. The universal vocabulary for structuring anything from people and documents to events and relationships. next: - concepts/triple-intelligence - api/reference @@ -14,114 +14,141 @@ next: # The Universal Knowledge Protocol: Noun-Verb Taxonomy > **Brainy is the Universal Knowledge Protocol™ powered by Triple Intelligence™** -> -> We're the world's first to unify vector, graph, and document search in one magical API. This breakthrough—Triple Intelligence—enables us to create a universal language for knowledge that all tools, augmentations, and AI models can speak. +> +> Brainy unifies vector, graph, and document search behind one API. That unification — Triple Intelligence — rests on a shared, standardized vocabulary for knowledge: a fixed set of entity types (nouns) and relationship types (verbs) that every tool, integration, and model can speak. + +Every example on this page is written against the real Brainy 8.0 API. The setup is always the same: + +```typescript +import { Brainy, NounType, VerbType } from '@soulcraft/brainy' + +const brain = new Brainy() +await brain.init() +``` + +- `brain.add({ data, type, subtype?, metadata? })` creates a noun and returns its `string` id. +- `brain.relate({ from, to, type, metadata? })` creates a verb (relationship) between two nouns. +- `brain.find({ query?, type?, where?, connected? })` runs Triple Intelligence search. +- `brain.related({ from })` / `brain.neighbors(id)` read a noun's relationships. ## Universal & Infinite Expressiveness -Brainy's **Noun-Verb Taxonomy** achieves **universal coverage** of all human knowledge through **infinite expressiveness**: +Brainy's **Noun-Verb Taxonomy** achieves broad coverage of human knowledge through composable expressiveness: - **42 Noun Types × 127 Verb Types = 5,334 Base Combinations** -- **Unlimited Metadata Fields = ∞ Domain Specificity** -- **Multi-hop Graph Traversals = ∞ Relationship Complexity** -- **Result: Can Model ANY Data in ANY Industry** +- **Unlimited Metadata Fields = Domain Specificity** +- **Multi-hop Graph Traversals = Relationship Complexity** +- **Result: Model data across virtually any industry** -This isn't marketing—it's mathematically provable. Every piece of information that exists can be represented as entities (nouns) connected by relationships (verbs) with properties (metadata). +Every piece of information can be represented as entities (nouns) connected by relationships (verbs) carrying properties (metadata). The standardized type system from `@soulcraft/brainy` (`NounType`, `VerbType`) gives those nouns and verbs a stable, shared name. ## The Power of Standardization: Universal Interoperability ### Why Standardized Types = Seamless Integration -The standardized noun-verb taxonomy creates a **universal language** that enables: +Because every entity is classified with a `NounType` and every relationship with a `VerbType`, the same data is legible to any code that imports the same enums. + +#### 1. Tool Interoperability -#### 1. **Tool Interoperability** ```typescript -// Any tool that understands Brainy types can work with any other -const analyticsAugmentation = await brain.augment('analytics') -const visualizationAugmentation = await brain.augment('visualization') +// Any tool that understands Brainy's NounType/VerbType can read the same graph. +// One module writes; another reads — no schema translation in between. +const authors = await brain.find({ type: NounType.Person }) -// Both understand "person", "document", "creates" without translation -const authors = await analyticsAugmentation.findTopAuthors() -await visualizationAugmentation.graphRelationships(authors) +for (const author of authors) { + const authored = await brain.related({ + from: author.id, + type: VerbType.Creates + }) + console.log(`${author.data} → ${authored.length} document(s)`) +} ``` -#### 2. **Data Portability** +#### 2. Data Portability + ```typescript -// Snapshot from one Brainy instance, restore into another — -// types are universally understood +// Snapshot one brain and open it as another — the noun/verb vocabulary +// travels with the data, so the types line up exactly. const pin = brain1.now() -await pin.persist('/snapshots/brain1') -await pin.release() +try { + await pin.persist('/snapshots/brain-1') +} finally { + await pin.release() +} -const brain2 = await Brainy.load('/snapshots/brain1') // Types match perfectly +const brain2 = await Brainy.load('/snapshots/brain-1') // same NounType/VerbType vocabulary ``` -#### 3. **AI Model Compatibility** +#### 3. Model & Agent Compatibility + ```typescript -// Different AI models can share the same knowledge graph -const gptBrain = await brain.connectModel('gpt-4') -const claudeBrain = await brain.connectModel('claude-3') -const llamaBrain = await brain.connectModel('llama-2') - -// All models understand the same noun-verb structure -const knowledge = await brain.add("Quantum Computer", { type: "thing" }) -// Any model can now reason about this knowledge -``` - -#### 4. **Augmentation Ecosystem** -```typescript -// Augmentations build on standard types, ensuring compatibility -await brain.augment.install('medical-records') // Extends "person" type -await brain.augment.install('financial-analysis') // Extends "transaction" events -await brain.augment.install('social-graph') // Uses "follows", "likes" verbs - -// All augmentations work together seamlessly -const patient = await brain.find("patient with financial transactions who follows Dr. Smith") -``` - -#### 5. **Cross-Platform Integration** -```typescript -// Standard types enable integration with external systems -// CRM understands "person" and "organization" -await brain.sync.salesforce({ - mapping: { - Contact: "person", - Account: "organization", - Opportunity: "event" - } +// Different models and agents reason over the SAME typed structure. +// Whatever produced the entity, every consumer reads the same NounType. +const conceptId = await brain.add({ + data: 'Quantum Computer', + type: NounType.Thing, + subtype: 'computing-hardware' }) -// Project management understands "task" and "project" -await brain.sync.jira({ - mapping: { - Issue: "task", - Epic: "project", - Sprint: "event" - } -}) +// Any downstream consumer can now retrieve and reason about this entity. +const concept = await brain.get(conceptId) +console.log(concept?.type) // 'thing' +``` + +#### 4. Extensibility Without Forking the Schema + +```typescript +// Subtypes extend a standard NounType for a specific domain — no schema +// migration, no new noun type. The base type stays universally understood. +await brain.add({ data: 'Patient #12345', type: NounType.Person, subtype: 'patient' }) +await brain.add({ data: 'Invoice #4471', type: NounType.Document, subtype: 'invoice' }) +await brain.add({ data: 'Follower edge', type: NounType.Relationship, subtype: 'social-graph' }) + +// Domains that share the base types interoperate even with custom subtypes. +const patients = await brain.find({ type: NounType.Person, subtype: 'patient' }) +``` + +#### 5. Cross-Platform Integration + +```typescript +// Map external systems onto the standard taxonomy. A CRM's Contact/Account/ +// Opportunity become Person/Organization/Event — the same vocabulary everywhere. +const externalRecords = [ + { kind: 'Contact', name: 'Dana Lee', type: NounType.Person }, + { kind: 'Account', name: 'Acme Corp', type: NounType.Organization }, + { kind: 'Opportunity', name: 'Q3 Renewal', type: NounType.Event } +] + +for (const record of externalRecords) { + await brain.add({ + data: record.name, + type: record.type, + metadata: { source: 'crm', externalKind: record.kind } + }) +} ``` ### The Network Effect: Brainy as the Universal Knowledge Protocol -Like **HTTP** became the protocol for the web and **TCP/IP** for the internet, Brainy's noun-verb taxonomy is becoming the **Universal Knowledge Protocol**: +Like **HTTP** became the protocol for the web and **TCP/IP** for the internet, Brainy's noun-verb taxonomy aims to be a **Universal Knowledge Protocol**: -- **Learn Once**: Developers learn 42 nouns + 127 verbs, not 1000s of schemas +- **Learn Once**: Developers learn 42 nouns + 127 verbs, not thousands of bespoke schemas - **Build Anywhere**: Tools built for one domain work in others - **Share Everything**: Knowledge graphs are universally shareable -- **Compose Freely**: Augmentations compose without conflicts +- **Compose Freely**: Subtypes and metadata extend types without schema migrations -This isn't just a database—it's a **protocol for how humanity represents knowledge**. +This isn't just a database — it's a **shared model for how knowledge is represented**. ## Overview -Brainy 2.0 introduces a revolutionary **Noun-Verb Taxonomy** that models data as entities (nouns) and relationships (verbs), creating a semantic knowledge graph that mirrors how humans naturally think about information. +Brainy's **Noun-Verb Taxonomy** models data as entities (nouns) and relationships (verbs), creating a semantic knowledge graph that mirrors how humans naturally think about information. ## Why Noun-Verb? Traditional databases force you to think in tables, documents, or nodes. Brainy lets you think naturally: - **Nouns**: Things that exist (people, documents, products, concepts) -- **Verbs**: How things relate (creates, owns, references, similar-to) +- **Verbs**: How things relate (creates, owns, references, related-to) This simple mental model scales from basic storage to complex knowledge graphs while remaining intuitive. @@ -129,70 +156,78 @@ This simple mental model scales from basic storage to complex knowledge graphs w ### Nouns (Entities) -Nouns represent any entity in your system: +Nouns represent any entity in your system. `add()` takes a single object and returns the new entity's id: ```typescript // Add any entity as a noun -const personId = await brain.add("John Smith, Senior Engineer", { - type: "person", - department: "engineering", - skills: ["TypeScript", "React", "Node.js"] +const personId = await brain.add({ + data: 'John Smith, Senior Engineer', + type: NounType.Person, + subtype: 'employee', + metadata: { department: 'engineering', skills: ['TypeScript', 'React', 'Node.js'] } }) -const documentId = await brain.add("Q3 2024 Financial Report", { - type: "document", - category: "financial", - confidential: true, - created: "2024-10-01" +const documentId = await brain.add({ + data: 'Q3 2024 Financial Report', + type: NounType.Document, + subtype: 'report', + metadata: { category: 'financial', confidential: true, created: '2024-10-01' } }) -const conceptId = await brain.add("Machine Learning", { - type: "concept", - domain: "technology", - complexity: "advanced" +const conceptId = await brain.add({ + data: 'Machine Learning', + type: NounType.Concept, + metadata: { domain: 'technology', complexity: 'advanced' } }) ``` #### Noun Properties Every noun automatically gets: -- **Unique ID**: System-generated or custom -- **Vector Embedding**: For semantic similarity -- **Metadata**: Flexible JSON properties -- **Timestamps**: Created/updated tracking -- **Indexing**: Automatic field indexing +- **Unique ID**: System-generated UUID, or supply your own via `id` +- **Vector Embedding**: `data` is embedded for semantic similarity +- **Metadata**: Flexible, queryable JSON properties +- **Timestamps**: `createdAt` / `updatedAt` tracking +- **Indexing**: Automatic field indexing for `where` filters ### Verbs (Relationships) -Verbs define how nouns relate to each other: +Verbs define how nouns relate to each other. `relate()` also takes a single object: ```typescript // Create relationships between entities -await brain.relate(personId, documentId, "authored", { - role: "primary_author", - contribution: "80%" +await brain.relate({ + from: personId, + to: documentId, + type: VerbType.Creates, + metadata: { role: 'primary_author', contribution: '80%' } }) -await brain.relate(documentId, conceptId, "discusses", { - sections: ["methodology", "results"], - depth: "detailed" +await brain.relate({ + from: documentId, + to: conceptId, + type: VerbType.Describes, + metadata: { sections: ['methodology', 'results'], depth: 'detailed' } }) -await brain.relate(personId, conceptId, "expert_in", { - years_experience: 5, - certification: "Advanced ML Certification" +await brain.relate({ + from: personId, + to: conceptId, + type: VerbType.RelatedTo, + subtype: 'expertise', + metadata: { yearsExperience: 5, certification: 'Advanced ML Certification' } }) ``` #### Verb Properties Every verb includes: -- **Source**: The noun initiating the relationship -- **Target**: The noun receiving the relationship -- **Type**: The relationship type/name -- **Direction**: Directional or bidirectional -- **Metadata**: Relationship-specific data -- **Strength**: Optional relationship weight +- **Source** (`from`): The noun initiating the relationship +- **Target** (`to`): The noun receiving the relationship +- **Type**: The `VerbType` classification +- **Subtype**: Optional per-product sub-classification (fast-path indexed) +- **Metadata**: Relationship-specific queryable data +- **Weight**: Optional relationship strength (0–1) ## Benefits @@ -200,92 +235,104 @@ Every verb includes: ```typescript // Think naturally about your data -const taskId = await brain.add("Implement payment system") -const userId = await brain.add("Alice Johnson") -const projectId = await brain.add("E-commerce Platform") +const taskId = await brain.add({ data: 'Implement payment system', type: NounType.Task }) +const userId = await brain.add({ data: 'Alice Johnson', type: NounType.Person }) +const projectId = await brain.add({ data: 'E-commerce Platform', type: NounType.Project }) // Express relationships clearly -await brain.relate(userId, taskId, "assigned_to") -await brain.relate(taskId, projectId, "part_of") -await brain.relate(userId, projectId, "manages") +await brain.relate({ from: userId, to: taskId, type: VerbType.ParticipatesIn, subtype: 'assignee' }) +await brain.relate({ from: taskId, to: projectId, type: VerbType.PartOf }) +await brain.relate({ from: userId, to: projectId, type: VerbType.ParticipatesIn, subtype: 'manager' }) ``` ### 2. Semantic Understanding -The noun-verb model preserves meaning: +The noun-verb model preserves meaning. `find()` accepts a natural-language string or a structured query: ```typescript -// The system understands semantic relationships -const results = await brain.find("tasks assigned to Alice") -// Automatically understands: assigned_to verb + Alice noun +// Natural language — embedded and matched semantically +const results = await brain.find({ query: 'tasks for the payment system' }) -const related = await brain.find("people who manage projects with payment tasks") -// Traverses: person -> manages -> project -> contains -> task +// Structured — type + graph traversal in one call +const aliceTasks = await brain.find({ + type: NounType.Task, + connected: { from: userId, via: VerbType.ParticipatesIn } +}) ``` ### 3. Flexible Schema -No rigid schema requirements: +No rigid schema requirements — add any type, extend with a `subtype`: ```typescript // Add any noun type without schema changes -await brain.add("New IoT Sensor", { - type: "device", - protocol: "MQTT", - location: "Building A" +const sensorId = await brain.add({ + data: 'New IoT Sensor', + type: NounType.Thing, + subtype: 'iot-device', + metadata: { protocol: 'MQTT', location: 'Building A' } }) -// Create new relationship types on the fly -await brain.relate(sensorId, buildingId, "monitors", { - metrics: ["temperature", "humidity"], - interval: "5 minutes" +const buildingId = await brain.add({ data: 'Building A', type: NounType.Location }) + +// Relationships carry their own structured metadata +await brain.relate({ + from: sensorId, + to: buildingId, + type: VerbType.Measures, + metadata: { metrics: ['temperature', 'humidity'], interval: '5 minutes' } }) ``` ### 4. Graph Traversal -Navigate relationships naturally: +Navigate relationships naturally with `connected`: ```typescript -// Find all documents authored by team members +// Find documents reachable from a team via two relationship hops const teamDocs = await brain.find({ + type: NounType.Document, connected: { from: teamId, - through: ["member_of", "authored"], + via: [VerbType.MemberOf, VerbType.Creates], depth: 2 } }) -// Find similar products purchased by similar users +// Find products two hops out from a user const recommendations = await brain.find({ + type: NounType.Product, connected: { from: userId, - through: ["similar_to", "purchased"], - depth: 2, - type: "product" + via: VerbType.Owns, + depth: 2 } }) ``` ### 5. Temporal Relationships -Track how relationships change over time: +Track how relationships change over time by storing dates in edge metadata: ```typescript -// Relationships with temporal data -await brain.relate(employeeId, companyId, "worked_at", { - from: "2020-01-01", - to: "2023-12-31", - position: "Senior Developer" +await brain.relate({ + from: employeeId, + to: companyId, + type: VerbType.MemberOf, + subtype: 'past-employment', + metadata: { from: '2020-01-01', to: '2023-12-31', position: 'Senior Developer' } }) -await brain.relate(employeeId, newCompanyId, "works_at", { - from: "2024-01-01", - position: "Tech Lead" +await brain.relate({ + from: employeeId, + to: newCompanyId, + type: VerbType.MemberOf, + subtype: 'current-employment', + metadata: { from: '2024-01-01', position: 'Tech Lead' } }) -// Query historical relationships -const employment = await brain.find("where did John work in 2022") +// Query with natural language +const employment = await brain.find({ query: 'where did this person work in 2022' }) ``` ## Real-World Use Cases @@ -294,97 +341,108 @@ const employment = await brain.find("where did John work in 2022") ```typescript // Documents and their relationships -const paperId = await brain.add("Neural Networks Paper", { - type: "research_paper", - year: 2024 +const paperId = await brain.add({ + data: 'Neural Networks Paper', + type: NounType.Document, + subtype: 'research-paper', + metadata: { year: 2024 } }) -const authorId = await brain.add("Dr. Sarah Chen", { - type: "researcher" +const authorId = await brain.add({ + data: 'Dr. Sarah Chen', + type: NounType.Person, + subtype: 'researcher' }) -const topicId = await brain.add("Deep Learning", { - type: "topic" -}) +const topicId = await brain.add({ data: 'Deep Learning', type: NounType.Concept }) +const otherPaperId = await brain.add({ data: 'Backpropagation Survey', type: NounType.Document }) // Rich relationship network -await brain.relate(authorId, paperId, "authored") -await brain.relate(paperId, topicId, "covers") -await brain.relate(paperId, otherPaperId, "cites") -await brain.relate(authorId, topicId, "researches") +await brain.relate({ from: authorId, to: paperId, type: VerbType.Creates }) +await brain.relate({ from: paperId, to: topicId, type: VerbType.Describes }) +await brain.relate({ from: paperId, to: otherPaperId, type: VerbType.References }) +await brain.relate({ from: authorId, to: topicId, type: VerbType.RelatedTo, subtype: 'research-focus' }) // Query the knowledge graph -const related = await brain.find("papers about deep learning by Sarah Chen") +const related = await brain.find({ query: 'papers about deep learning by Sarah Chen' }) ``` ### Social Networks ```typescript // Users and connections -const user1 = await brain.add("Alice", { type: "user" }) -const user2 = await brain.add("Bob", { type: "user" }) -const post = await brain.add("Great article on AI!", { type: "post" }) +const user1 = await brain.add({ data: 'Alice', type: NounType.Person }) +const user2 = await brain.add({ data: 'Bob', type: NounType.Person }) +const post = await brain.add({ data: 'Great article on AI!', type: NounType.Message, subtype: 'post' }) // Social interactions -await brain.relate(user1, user2, "follows") -await brain.relate(user2, user1, "follows") // Mutual -await brain.relate(user1, post, "created") -await brain.relate(user2, post, "liked") -await brain.relate(user2, post, "shared") +await brain.relate({ from: user1, to: user2, type: VerbType.Follows }) +await brain.relate({ from: user2, to: user1, type: VerbType.Follows }) // mutual +await brain.relate({ from: user1, to: post, type: VerbType.Creates }) +await brain.relate({ from: user2, to: post, type: VerbType.Likes }) +await brain.relate({ from: user2, to: post, type: VerbType.Communicates, subtype: 'share' }) // Find social patterns -const influencers = await brain.find("users with most followers who post about AI") +const influencers = await brain.find({ query: 'people who post about AI with many followers' }) ``` ### E-commerce ```typescript // Products and purchases -const product = await brain.add("Wireless Headphones", { - type: "product", - price: 99.99, - category: "electronics" +const product = await brain.add({ + data: 'Wireless Headphones', + type: NounType.Product, + metadata: { price: 99.99, category: 'electronics' } }) -const customer = await brain.add("Customer #12345", { - type: "customer", - tier: "premium" +const customer = await brain.add({ + data: 'Customer #12345', + type: NounType.Person, + subtype: 'customer', + metadata: { tier: 'premium' } }) -// Purchase relationships -await brain.relate(customer, product, "purchased", { - date: "2024-01-15", - quantity: 1, - price: 99.99 +// Purchase and review relationships +await brain.relate({ + from: customer, + to: product, + type: VerbType.Owns, + subtype: 'purchase', + metadata: { date: '2024-01-15', quantity: 1, price: 99.99 } }) -await brain.relate(customer, product, "reviewed", { - rating: 5, - text: "Excellent sound quality!" +await brain.relate({ + from: customer, + to: product, + type: VerbType.Evaluates, + subtype: 'review', + metadata: { rating: 5, text: 'Excellent sound quality!' } }) // Recommendation queries -const recs = await brain.find("products purchased by customers who bought headphones") +const recs = await brain.find({ query: 'products bought by customers who bought headphones' }) ``` ### Project Management ```typescript // Projects, tasks, and teams -const project = await brain.add("Website Redesign", { type: "project" }) -const task = await brain.add("Update homepage", { type: "task" }) -const developer = await brain.add("Jane Developer", { type: "person" }) -const designer = await brain.add("John Designer", { type: "person" }) +const project = await brain.add({ data: 'Website Redesign', type: NounType.Project }) +const task = await brain.add({ data: 'Update homepage', type: NounType.Task }) +const otherTask = await brain.add({ data: 'Design system audit', type: NounType.Task }) +const developer = await brain.add({ data: 'Jane Developer', type: NounType.Person, subtype: 'employee' }) +const designer = await brain.add({ data: 'John Designer', type: NounType.Person, subtype: 'employee' }) // Work relationships -await brain.relate(task, project, "belongs_to") -await brain.relate(developer, task, "assigned_to") -await brain.relate(designer, developer, "collaborates_with") -await brain.relate(task, otherTask, "depends_on") +await brain.relate({ from: task, to: project, type: VerbType.PartOf }) +await brain.relate({ from: developer, to: task, type: VerbType.ParticipatesIn, subtype: 'assignee' }) +await brain.relate({ from: designer, to: developer, type: VerbType.WorksWith }) +await brain.relate({ from: task, to: otherTask, type: VerbType.DependsOn }) // Project queries -const blockers = await brain.find("tasks that depend on incomplete tasks") -const workload = await brain.find("people assigned to multiple active projects") +const blockers = await brain.find({ query: 'tasks blocked by incomplete work' }) +const workload = await brain.find({ query: 'people assigned to multiple active projects' }) ``` ## Advanced Patterns @@ -392,54 +450,56 @@ const workload = await brain.find("people assigned to multiple active projects") ### Bidirectional Relationships ```typescript -// Some relationships are naturally bidirectional -await brain.relate(user1, user2, "friend_of", { bidirectional: true }) -// Automatically creates inverse relationship +// Symmetric relationships create the inverse edge automatically +await brain.relate({ from: user1, to: user2, type: VerbType.FriendOf, bidirectional: true }) ``` ### Weighted Relationships ```typescript -// Add strength/weight to relationships -await brain.relate(doc1, doc2, "similar_to", { - similarity_score: 0.95, - algorithm: "cosine" +// Add strength/weight to relationships (top-level weight, 0–1) +await brain.relate({ + from: doc1, + to: doc2, + type: VerbType.SimilarityDegree, + weight: 0.95, + metadata: { algorithm: 'cosine' } }) -// Use weights in queries -const stronglyRelated = await brain.find({ - connected: { - type: "similar_to", - minWeight: 0.8 - } -}) +// Weights come back on the Relation, so you can filter on them +const edges = await brain.related({ from: doc1, type: VerbType.SimilarityDegree }) +const stronglyRelated = edges.filter((edge) => (edge.weight ?? 0) >= 0.8) ``` -### Relationship Chains +### Relationship Chains (Multi-hop) ```typescript -// Follow relationship chains +// Follow a chain of relationship types out to a fixed depth. +// `via` accepts an array of VerbTypes; `depth` bounds the traversal. const results = await brain.find({ + type: NounType.Thing, connected: { from: userId, - chain: [ - { type: "owns", to: "company" }, - { type: "produces", to: "product" }, - { type: "uses", to: "technology" } - ] + via: [VerbType.Owns, VerbType.Creates, VerbType.Uses], + depth: 3 } }) -// Finds: technologies used by products made by companies owned by user +// Finds: things used by products made by companies owned by the user ``` ### Meta-Relationships +Relationships can themselves be reasoned about. The `Relationship` NounType reifies an edge as a first-class entity, and the meta-level verbs (`Endorses`, `Supports`, `Contradicts`, `Supersedes`) express second-order claims between entities: + ```typescript -// Relationships about relationships -const verbId = await brain.relate(user1, user2, "recommends") -await brain.relate(user3, verbId, "endorses", { - reason: "Accurate recommendation", - trust_score: 0.9 +// A second person endorses a claim, and a third supports it with evidence. +const claim = await brain.add({ data: 'X improves retention', type: NounType.Proposition }) +await brain.relate({ from: user2, to: claim, type: VerbType.Endorses }) +await brain.relate({ + from: user3, + to: claim, + type: VerbType.Supports, + metadata: { reason: 'Matches the A/B test', trustScore: 0.9 } }) ``` @@ -449,1121 +509,1048 @@ await brain.relate(user3, verbId, "endorses", { ```typescript // By type -const people = await brain.find({ where: { type: "person" } }) +const people = await brain.find({ type: NounType.Person }) -// By properties +// By type + metadata filters (bare operators — no `$` prefixes) const documents = await brain.find({ + type: NounType.Document, where: { - type: "document", confidential: false, - created: { $gte: "2024-01-01" } + created: { gte: '2024-01-01' } } }) -// By similarity +// By semantic similarity — use `query`, optionally narrowed by type const similar = await brain.find({ - like: "machine learning research", - where: { type: "document" } + query: 'machine learning research', + type: NounType.Document }) ``` -### Finding Verbs +> **`where` operators** are bare (never dollar-prefixed): `eq`/`equals`/`is`, `ne`/`notEquals`, `in`/`oneOf`, `gt`/`greaterThan`, `gte`/`greaterThanOrEqual`, `lt`/`lessThan`, `lte`/`lessThanOrEqual`, `between`, `contains`, `exists`, `missing`, plus the logical combinators `allOf`/`anyOf`/`not`. + +### Finding Verbs (Relationships) ```typescript -// Get all relationships for a noun -const relationships = await brain.getVerbs(nounId) +// All relationships originating from a noun +const outgoing = await brain.related({ from: nounId }) -// Find specific relationship types -const authorships = await brain.find({ - verb: { - type: "authored", - from: authorId - } -}) +// Every edge touching a noun, in either direction +const incident = await brain.related({ node: nounId }) -// Find by relationship properties -const recentPurchases = await brain.find({ - verb: { - type: "purchased", - where: { - date: { $gte: "2024-01-01" } - } - } -}) +// Filter by relationship type +const authorships = await brain.related({ from: authorId, type: VerbType.Creates }) + +// Filter returned relationships by their metadata (Relation carries `.metadata`) +const purchases = await brain.related({ from: customerId, type: VerbType.Owns, subtype: 'purchase' }) +const recentPurchases = purchases.filter((edge) => edge.metadata?.date >= '2024-01-01') + +// Just the count of relationships in the graph +const totalEdges = await brain.getVerbCount() ``` -### Combined Queries +### Combined Queries (Query → Expand) ```typescript -// Find nouns through relationships +// Start from a semantic query, then expand along the graph. +// Vector + graph in a single find() call. const results = await brain.find({ - // Start with similar documents - like: "AI research", - // That are authored by + query: 'AI research', connected: { - through: "authored", - // People who work at - where: { - connected: { - to: "Stanford", - type: "works_at" - } - } + via: VerbType.Creates, + depth: 2 } }) ``` -## Performance Optimizations +## The Complete Noun Taxonomy (42 Types) -### Noun Indexing -- Automatic vector indexing for similarity -- Field indexing for metadata queries -- Full-text indexing for content search +`NounType` is the stable, exported vocabulary for classifying entities. Every value is a plain string, so you can write `NounType.Person` or the literal `'person'`. Pick the closest standard type and refine with `subtype` and `metadata`. -### Verb Indexing -- Relationship type indexing -- Source/target indexing -- Temporal indexing for time-based queries - -### Query Optimization -- Automatic query plan optimization -- Parallel execution of independent operations -- Result caching for repeated queries - -## Best Practices - -1. **Use Descriptive Types**: Make noun and verb types self-documenting -2. **Rich Metadata**: Include relevant metadata for better querying -3. **Consistent Naming**: Use consistent verb names across your application -4. **Temporal Data**: Include timestamps for time-based analysis -5. **Bidirectional When Appropriate**: Mark symmetric relationships as bidirectional - -## Migration from Traditional Models - -### From Relational (SQL) ```typescript -// Instead of JOIN queries -// SELECT * FROM users JOIN orders ON users.id = orders.user_id - -// Use noun-verb relationships -const userId = await brain.add("User", userData) -const orderId = await brain.add("Order", orderData) -await brain.relate(userId, orderId, "placed") - -// Query naturally -const userOrders = await brain.find({ - connected: { from: userId, type: "placed" } +const physicistId = await brain.add({ + data: 'Albert Einstein', + type: NounType.Person, + metadata: { role: 'physicist', born: '1879-03-14' } }) ``` -### From Document (NoSQL) -```typescript -// Instead of embedded documents -// { user: { orders: [...] } } - -// Use explicit relationships -const userId = await brain.add("User", userData) -for (const order of orders) { - const orderId = await brain.add("Order", order) - await brain.relate(userId, orderId, "has_order") -} -``` +### Core Entity Types (7) -### From Graph Databases -```typescript -// Similar to graph databases but with added benefits: -// 1. Automatic vector embeddings for similarity -// 2. Natural language querying -// 3. Unified with metadata filtering +| NounType | Value | Use for | +|----------|-------|---------| +| `Person` | `'person'` | Individual human entities | +| `Organization` | `'organization'` | Companies, institutions, collectives | +| `Location` | `'location'` | Geographic and named spatial entities | +| `Thing` | `'thing'` | Discrete physical objects and artifacts | +| `Concept` | `'concept'` | Abstract ideas, principles, intangibles | +| `Event` | `'event'` | Temporal occurrences and happenings | +| `Agent` | `'agent'` | Non-human autonomous actors (AI agents, bots) | -// Enhanced graph queries -const results = await brain.find("similar users who purchased similar products") -``` +### Biological & Material Types (2) -## Universal Knowledge Coverage +| NounType | Value | Use for | +|----------|-------|---------| +| `Organism` | `'organism'` | Living biological entities (animals, plants, bacteria) | +| `Substance` | `'substance'` | Physical materials and matter (water, iron, DNA) | -The Noun-Verb taxonomy is designed to represent **all human knowledge** through a comprehensive set of types that can be combined infinitely. +### Property, Temporal & Functional Types (3) -### Complete Noun Types (31 Types) +| NounType | Value | Use for | +|----------|-------|---------| +| `Quality` | `'quality'` | Properties and attributes that inhere in entities | +| `TimeInterval` | `'timeInterval'` | Temporal regions, periods, durations | +| `Function` | `'function'` | Purposes, capabilities, functional roles | -#### Core Entity Types (6) +### Informational Type (1) -##### 1. **Person** - Individual human entities -```typescript -await brain.add("Albert Einstein", { - type: "person", - role: "physicist", - born: "1879-03-14" -}) -``` +| NounType | Value | Use for | +|----------|-------|---------| +| `Proposition` | `'proposition'` | Statements, claims, assertions, declarative content | -##### 2. **Organization** - Collective entities -```typescript -await brain.add("OpenAI", { - type: "organization", - industry: "AI research", - founded: 2015 -}) -``` +### Digital/Content Types (4) -##### 3. **Location** - Geographic and spatial entities -```typescript -await brain.add("San Francisco", { - type: "location", - category: "city", - coordinates: [37.7749, -122.4194] -}) -``` +| NounType | Value | Use for | +|----------|-------|---------| +| `Document` | `'document'` | Text-based files and written content | +| `Media` | `'media'` | Non-text media (audio, video, images) | +| `File` | `'file'` | Generic digital files and data blobs | +| `Message` | `'message'` | Communication content and correspondence | -##### 4. **Thing** - Physical objects -```typescript -await brain.add("Tesla Model 3", { - type: "thing", - category: "vehicle", - manufacturer: "Tesla" -}) -``` +### Collection Types (2) -##### 5. **Concept** - Abstract ideas and intangibles -```typescript -await brain.add("Machine Learning", { - type: "concept", - domain: "technology", - complexity: "advanced" -}) -``` +| NounType | Value | Use for | +|----------|-------|---------| +| `Collection` | `'collection'` | Groups and sets of items | +| `Dataset` | `'dataset'` | Structured data collections and databases | -##### 6. **Event** - Temporal occurrences -```typescript -await brain.add("Product Launch 2024", { - type: "event", - date: "2024-09-15", - attendees: 500 -}) -``` +### Business/Application Types (4) -#### Digital/Content Types (5) +| NounType | Value | Use for | +|----------|-------|---------| +| `Product` | `'product'` | Commercial products and offerings | +| `Service` | `'service'` | Service offerings and intangible products | +| `Task` | `'task'` | Actions, todos, work items | +| `Project` | `'project'` | Organized initiatives and programs | -##### 7. **Document** - Text-based files -```typescript -await brain.add("Quarterly Report", { - type: "document", - format: "PDF", - pages: 47 -}) -``` +### Descriptive Types (6) -##### 8. **Media** - Non-text media files -```typescript -await brain.add("Product Demo Video", { - type: "media", - format: "MP4", - duration: "5:30" -}) -``` +| NounType | Value | Use for | +|----------|-------|---------| +| `Process` | `'process'` | Workflows, procedures, ongoing activities | +| `State` | `'state'` | Conditions, status, situational contexts | +| `Role` | `'role'` | Positions, responsibilities, classifications | +| `Language` | `'language'` | Natural and formal languages | +| `Currency` | `'currency'` | Monetary units and exchange mediums | +| `Measurement` | `'measurement'` | Metrics, quantities, measured values | -##### 9. **File** - Generic digital files -```typescript -await brain.add("config.json", { - type: "file", - size: "2KB", - modified: Date.now() -}) -``` +### Scientific & Legal Types (4) -##### 10. **Message** - Communication content -```typescript -await brain.add("Support ticket #1234", { - type: "message", - priority: "high", - channel: "email" -}) -``` +| NounType | Value | Use for | +|----------|-------|---------| +| `Hypothesis` | `'hypothesis'` | Scientific theories, research hypotheses | +| `Experiment` | `'experiment'` | Controlled studies, trials, methodologies | +| `Contract` | `'contract'` | Legal agreements, terms, binding documents | +| `Regulation` | `'regulation'` | Laws, rules, compliance requirements | -##### 11. **Content** - Generic content -```typescript -await brain.add("Landing page copy", { - type: "content", - category: "marketing", - language: "en" -}) -``` - -#### Collection Types (2) +### Technical Infrastructure Types (2) -##### 12. **Collection** - Groups of items -```typescript -await brain.add("Premium Features", { - type: "collection", - items: 25, - category: "features" -}) -``` +| NounType | Value | Use for | +|----------|-------|---------| +| `Interface` | `'interface'` | APIs, protocols, specifications, endpoints | +| `Resource` | `'resource'` | Compute, bandwidth, storage, infrastructure assets | -##### 13. **Dataset** - Structured data collections -```typescript -await brain.add("Customer Analytics", { - type: "dataset", - records: 10000, - schema: "v2" -}) -``` +### Social Structure Types (3) -#### Business/Application Types (5) +| NounType | Value | Use for | +|----------|-------|---------| +| `SocialGroup` | `'socialGroup'` | Informal social groups and collectives | +| `Institution` | `'institution'` | Formal social structures and practices | +| `Norm` | `'norm'` | Social norms, conventions, expectations | -##### 14. **Product** - Commercial offerings -```typescript -await brain.add("Pro Subscription", { - type: "product", - price: 99.99, - tier: "premium" -}) -``` +### Information Theory Types (2) -##### 15. **Service** - Service offerings -```typescript -await brain.add("Cloud Hosting", { - type: "service", - sla: "99.9%", - region: "us-west" -}) -``` +| NounType | Value | Use for | +|----------|-------|---------| +| `InformationContent` | `'informationContent'` | Abstract information (stories, ideas, schemas) | +| `InformationBearer` | `'informationBearer'` | Physical or digital carrier of information | -##### 16. **User** - User accounts -```typescript -await brain.add("user@example.com", { - type: "user", - tier: "enterprise", - created: Date.now() -}) -``` +### Meta-Level & Extensible Types (2) -##### 17. **Task** - Actions and todos -```typescript -await brain.add("Deploy v2.0", { - type: "task", - priority: "high", - assignee: "devops" -}) -``` +| NounType | Value | Use for | +|----------|-------|---------| +| `Relationship` | `'relationship'` | Relationships reified as first-class entities | +| `Custom` | `'custom'` | Domain-specific entities outside the standard set | -##### 18. **Project** - Organized initiatives -```typescript -await brain.add("Website Redesign", { - type: "project", - deadline: "2024-12-31", - status: "active" -}) -``` +## The Complete Verb Taxonomy (127 Types) -#### Descriptive Types (7) +`VerbType` is the exported vocabulary for classifying relationships. As with nouns, every value is a plain string — write `VerbType.Creates` or `'creates'`. Where no verb is an exact fit, choose the closest base verb and refine it with `subtype` and `metadata` (see [Coverage Completeness](#coverage-completeness-analysis)). -##### 19. **Process** - Workflows and procedures ```typescript -await brain.add("CI/CD Pipeline", { - type: "process", - steps: 7, - automated: true -}) +await brain.relate({ from: authorId, to: documentId, type: VerbType.Creates }) ``` -##### 20. **State** - Conditions or status -```typescript -await brain.add("System Health", { - type: "state", - status: "operational", - uptime: "99.99%" -}) -``` +### Foundational Ontological (3) -##### 21. **Role** - Positions or responsibilities -```typescript -await brain.add("Admin Role", { - type: "role", - permissions: ["read", "write", "delete"], - level: "superuser" -}) -``` +| VerbType | Value | Meaning | +|----------|-------|---------| +| `InstanceOf` | `'instanceOf'` | Individual to class (Fido instanceOf Dog) | +| `SubclassOf` | `'subclassOf'` | Taxonomic hierarchy (Dog subclassOf Mammal) | +| `ParticipatesIn` | `'participatesIn'` | Entity participation in events/processes | -##### 22. **Topic** - Subjects or themes -```typescript -await brain.add("Machine Learning", { - type: "topic", - field: "AI", - popularity: "high" -}) -``` +### Core Relationships (4) -##### 23. **Language** - Languages or linguistic entities -```typescript -await brain.add("English", { - type: "language", - iso_code: "en", - speakers_millions: 1500 -}) -``` +| VerbType | Value | Meaning | +|----------|-------|---------| +| `RelatedTo` | `'relatedTo'` | Generic relationship (fallback) | +| `Contains` | `'contains'` | Containment relationship | +| `PartOf` | `'partOf'` | Part-whole (mereological) relationship | +| `References` | `'references'` | Citation and referential relationship | -##### 24. **Currency** - Monetary units -```typescript -await brain.add("US Dollar", { - type: "currency", - symbol: "$", - code: "USD" -}) -``` - -##### 25. **Measurement** - Metrics or quantities -```typescript -await brain.add("Temperature Reading", { - type: "measurement", - value: 23.5, - unit: "celsius" -}) -``` - -#### Scientific/Research Types (2) - -##### 26. **Hypothesis** - Scientific theories and propositions -```typescript -await brain.add("String Theory", { - type: "hypothesis", - field: "physics", - status: "unproven" -}) -``` - -##### 27. **Experiment** - Studies and research trials -```typescript -await brain.add("Clinical Trial XYZ", { - type: "experiment", - phase: 3, - participants: 1000 -}) -``` - -#### Legal/Regulatory Types (2) - -##### 28. **Contract** - Legal agreements and terms -```typescript -await brain.add("Service Agreement", { - type: "contract", - duration: "2 years", - value: 100000 -}) -``` - -##### 29. **Regulation** - Laws and compliance requirements -```typescript -await brain.add("GDPR", { - type: "regulation", - jurisdiction: "EU", - category: "data protection" -}) -``` - -#### Technical Infrastructure Types (2) - -##### 30. **Interface** - APIs and protocols -```typescript -await brain.add("REST API", { - type: "interface", - version: "v2", - endpoints: 45 -}) -``` - -##### 31. **Resource** - Infrastructure and compute assets -```typescript -await brain.add("Database Server", { - type: "resource", - capacity: "1TB", - availability: "99.9%" -}) -``` - -### Complete Verb Types (40 Types) - -#### Core Relationship Types (5) - -##### 1. **RelatedTo** - Generic relationship (default) -```typescript -await brain.relate(entityA, entityB, "relatedTo") -``` +### Spatial (2) -##### 2. **Contains** - Containment relationship -```typescript -await brain.relate(folderId, fileId, "contains") -``` +| VerbType | Value | Meaning | +|----------|-------|---------| +| `LocatedAt` | `'locatedAt'` | Spatial location relationship | +| `AdjacentTo` | `'adjacentTo'` | Spatial proximity relationship | -##### 3. **PartOf** - Part-whole relationship -```typescript -await brain.relate(componentId, systemId, "partOf") -``` +### Temporal (3) -##### 4. **LocatedAt** - Spatial relationship -```typescript -await brain.relate(deviceId, locationId, "locatedAt") -``` +| VerbType | Value | Meaning | +|----------|-------|---------| +| `Precedes` | `'precedes'` | Temporal sequence (before) | +| `During` | `'during'` | Temporal containment | +| `OccursAt` | `'occursAt'` | Temporal location | -##### 5. **References** - Citation relationship -```typescript -await brain.relate(paperId, sourceId, "references") -``` +### Causal & Dependency (5) -#### Temporal/Causal Types (5) +| VerbType | Value | Meaning | +|----------|-------|---------| +| `Causes` | `'causes'` | Direct causal relationship | +| `Enables` | `'enables'` | Enablement without direct causation | +| `Prevents` | `'prevents'` | Prevention relationship | +| `DependsOn` | `'dependsOn'` | Dependency relationship | +| `Requires` | `'requires'` | Necessity relationship | + +### Creation & Transformation (5) + +| VerbType | Value | Meaning | +|----------|-------|---------| +| `Creates` | `'creates'` | Creation relationship | +| `Transforms` | `'transforms'` | Transformation relationship | +| `Becomes` | `'becomes'` | State change relationship | +| `Modifies` | `'modifies'` | Modification relationship | +| `Consumes` | `'consumes'` | Consumption relationship | -##### 6. **Precedes** - Temporal sequence (before) -```typescript -await brain.relate(event1Id, event2Id, "precedes") -``` +### Lifecycle (1) -##### 7. **Succeeds** - Temporal sequence (after) -```typescript -await brain.relate(event2Id, event1Id, "succeeds") -``` +| VerbType | Value | Meaning | +|----------|-------|---------| +| `Destroys` | `'destroys'` | Termination and destruction relationship | + +### Ownership & Attribution (2) + +| VerbType | Value | Meaning | +|----------|-------|---------| +| `Owns` | `'owns'` | Ownership relationship | +| `AttributedTo` | `'attributedTo'` | Attribution relationship | -##### 8. **Causes** - Causal relationship -```typescript -await brain.relate(actionId, effectId, "causes") -``` +### Property & Quality (2) + +| VerbType | Value | Meaning | +|----------|-------|---------| +| `HasQuality` | `'hasQuality'` | Entity to quality attribution | +| `Realizes` | `'realizes'` | Function realization relationship | + +### Effects & Experience (1) + +| VerbType | Value | Meaning | +|----------|-------|---------| +| `Affects` | `'affects'` | Patient/experiencer relationship | -##### 9. **DependsOn** - Dependency relationship -```typescript -await brain.relate(moduleId, libraryId, "dependsOn") -``` +### Composition (2) -##### 10. **Requires** - Necessity relationship -```typescript -await brain.relate(taskId, resourceId, "requires") -``` - -#### Creation/Transformation Types (5) - -##### 11. **Creates** - Creation relationship -```typescript -await brain.relate(authorId, documentId, "creates") -``` - -##### 12. **Transforms** - Transformation relationship -```typescript -await brain.relate(processId, dataId, "transforms") -``` - -##### 13. **Becomes** - State change relationship -```typescript -await brain.relate(caterpillarId, butterflyId, "becomes") -``` - -##### 14. **Modifies** - Modification relationship -```typescript -await brain.relate(editorId, fileId, "modifies") -``` - -##### 15. **Consumes** - Consumption relationship -```typescript -await brain.relate(processId, resourceId, "consumes") -``` - -#### Ownership/Attribution Types (4) - -##### 16. **Owns** - Ownership relationship -```typescript -await brain.relate(userId, assetId, "owns") -``` - -##### 17. **AttributedTo** - Attribution relationship -```typescript -await brain.relate(quoteId, authorId, "attributedTo") -``` - -##### 18. **CreatedBy** - Creation attribution -```typescript -await brain.relate(productId, teamId, "createdBy") -``` - -##### 19. **BelongsTo** - Belonging relationship -```typescript -await brain.relate(itemId, collectionId, "belongsTo") -``` - -#### Social/Organizational Types (9) - -##### 20. **MemberOf** - Membership relationship -```typescript -await brain.relate(userId, organizationId, "memberOf") -``` - -##### 21. **WorksWith** - Professional relationship -```typescript -await brain.relate(employee1Id, employee2Id, "worksWith") -``` - -##### 22. **FriendOf** - Friendship relationship -```typescript -await brain.relate(user1Id, user2Id, "friendOf") -``` - -##### 23. **Follows** - Following relationship -```typescript -await brain.relate(followerId, influencerId, "follows") -``` - -##### 24. **Likes** - Liking relationship -```typescript -await brain.relate(userId, postId, "likes") -``` - -##### 25. **ReportsTo** - Reporting relationship -```typescript -await brain.relate(employeeId, managerId, "reportsTo") -``` - -##### 26. **Supervises** - Supervisory relationship -```typescript -await brain.relate(managerId, employeeId, "supervises") -``` - -##### 27. **Mentors** - Mentorship relationship -```typescript -await brain.relate(seniorId, juniorId, "mentors") -``` - -##### 28. **Communicates** - Communication relationship -```typescript -await brain.relate(sender, receiver, "communicates") -``` - -#### Descriptive/Functional Types (8) - -##### 29. **Describes** - Descriptive relationship -```typescript -await brain.relate(documentId, conceptId, "describes") -``` - -##### 30. **Defines** - Definition relationship -```typescript -await brain.relate(glossaryId, termId, "defines") -``` - -##### 31. **Categorizes** - Categorization relationship -```typescript -await brain.relate(taxonomyId, itemId, "categorizes") -``` - -##### 32. **Measures** - Measurement relationship -```typescript -await brain.relate(sensorId, metricId, "measures") -``` - -##### 33. **Evaluates** - Evaluation relationship -```typescript -await brain.relate(reviewerId, proposalId, "evaluates") -``` - -##### 34. **Uses** - Utilization relationship -```typescript -await brain.relate(applicationId, libraryId, "uses") -``` - -##### 35. **Implements** - Implementation relationship -```typescript -await brain.relate(classId, interfaceId, "implements") -``` - -##### 36. **Extends** - Extension relationship -```typescript -await brain.relate(childClassId, parentClassId, "extends") -``` - -#### Enhanced Relationships (4) - -##### 37. **Inherits** - Inheritance relationship -```typescript -await brain.relate(childId, parentId, "inherits") -``` - -##### 38. **Conflicts** - Conflict relationship -```typescript -await brain.relate(policy1Id, policy2Id, "conflicts") -``` - -##### 39. **Synchronizes** - Synchronization relationship -```typescript -await brain.relate(service1Id, service2Id, "synchronizes") -``` - -##### 40. **Competes** - Competition relationship -```typescript -await brain.relate(company1Id, company2Id, "competes") -``` +| VerbType | Value | Meaning | +|----------|-------|---------| +| `ComposedOf` | `'composedOf'` | Material composition (distinct from partOf) | +| `Inherits` | `'inherits'` | Inheritance relationship | + +### Social & Organizational (8) + +| VerbType | Value | Meaning | +|----------|-------|---------| +| `MemberOf` | `'memberOf'` | Membership relationship | +| `WorksWith` | `'worksWith'` | Professional collaboration | +| `FriendOf` | `'friendOf'` | Friendship relationship | +| `Follows` | `'follows'` | Following/subscription relationship | +| `Likes` | `'likes'` | Liking/favoriting relationship | +| `ReportsTo` | `'reportsTo'` | Hierarchical reporting relationship | +| `Mentors` | `'mentors'` | Mentorship relationship | +| `Communicates` | `'communicates'` | Communication relationship | + +### Descriptive & Functional (8) + +| VerbType | Value | Meaning | +|----------|-------|---------| +| `Describes` | `'describes'` | Descriptive relationship | +| `Defines` | `'defines'` | Definition relationship | +| `Categorizes` | `'categorizes'` | Categorization relationship | +| `Measures` | `'measures'` | Measurement relationship | +| `Evaluates` | `'evaluates'` | Evaluation relationship | +| `Uses` | `'uses'` | Utilization relationship | +| `Implements` | `'implements'` | Implementation relationship | +| `Extends` | `'extends'` | Extension relationship | + +### Advanced Relationships (5) + +| VerbType | Value | Meaning | +|----------|-------|---------| +| `EquivalentTo` | `'equivalentTo'` | Equivalence/identity relationship | +| `Believes` | `'believes'` | Epistemic relationship (cognitive state) | +| `Conflicts` | `'conflicts'` | Conflict relationship | +| `Synchronizes` | `'synchronizes'` | Synchronization relationship | +| `Competes` | `'competes'` | Competition relationship | + +### Modal (6) + +| VerbType | Value | Meaning | +|----------|-------|---------| +| `CanCause` | `'canCause'` | Potential causation (possibility) | +| `MustCause` | `'mustCause'` | Necessary causation (necessity) | +| `WouldCauseIf` | `'wouldCauseIf'` | Counterfactual causation | +| `CouldBe` | `'couldBe'` | Possible states | +| `MustBe` | `'mustBe'` | Necessary identity | +| `Counterfactual` | `'counterfactual'` | General counterfactual relationship | + +### Epistemic States (9) + +| VerbType | Value | Meaning | +|----------|-------|---------| +| `Knows` | `'knows'` | Knowledge (justified true belief) | +| `Doubts` | `'doubts'` | Uncertainty/skepticism | +| `Desires` | `'desires'` | Want/preference | +| `Intends` | `'intends'` | Intentionality | +| `Fears` | `'fears'` | Fear/anxiety | +| `Loves` | `'loves'` | Strong positive emotional attitude | +| `Hates` | `'hates'` | Strong negative emotional attitude | +| `Hopes` | `'hopes'` | Hopeful expectation | +| `Perceives` | `'perceives'` | Sensory perception | + +### Learning & Cognition (1) + +| VerbType | Value | Meaning | +|----------|-------|---------| +| `Learns` | `'learns'` | Cognitive acquisition and learning | + +### Uncertainty & Probability (4) + +| VerbType | Value | Meaning | +|----------|-------|---------| +| `ProbablyCauses` | `'probablyCauses'` | Probabilistic causation | +| `UncertainRelation` | `'uncertainRelation'` | Unknown relationship with confidence bounds | +| `CorrelatesWith` | `'correlatesWith'` | Statistical correlation (not causation) | +| `ApproximatelyEquals` | `'approximatelyEquals'` | Fuzzy equivalence | + +### Scalar Properties (5) + +| VerbType | Value | Meaning | +|----------|-------|---------| +| `GreaterThan` | `'greaterThan'` | Scalar comparison | +| `SimilarityDegree` | `'similarityDegree'` | Graded similarity | +| `MoreXThan` | `'moreXThan'` | Comparative property | +| `HasDegree` | `'hasDegree'` | Scalar property assignment | +| `PartiallyHas` | `'partiallyHas'` | Graded possession | + +### Information Theory (2) + +| VerbType | Value | Meaning | +|----------|-------|---------| +| `Carries` | `'carries'` | Bearer carries content | +| `Encodes` | `'encodes'` | Encoding relationship | + +### Deontic (5) + +| VerbType | Value | Meaning | +|----------|-------|---------| +| `ObligatedTo` | `'obligatedTo'` | Moral/legal obligation | +| `PermittedTo` | `'permittedTo'` | Permission/authorization | +| `ProhibitedFrom` | `'prohibitedFrom'` | Prohibition/forbidden | +| `ShouldDo` | `'shouldDo'` | Normative expectation | +| `MustNotDo` | `'mustNotDo'` | Strong prohibition | + +### Context & Perspective (5) + +| VerbType | Value | Meaning | +|----------|-------|---------| +| `TrueInContext` | `'trueInContext'` | Context-dependent truth | +| `PerceivedAs` | `'perceivedAs'` | Subjective perception | +| `InterpretedAs` | `'interpretedAs'` | Interpretation relationship | +| `ValidInFrame` | `'validInFrame'` | Frame-dependent validity | +| `TrueFrom` | `'trueFrom'` | Perspective-dependent truth | + +### Advanced Temporal (6) + +| VerbType | Value | Meaning | +|----------|-------|---------| +| `Overlaps` | `'overlaps'` | Partial temporal overlap | +| `ImmediatelyAfter` | `'immediatelyAfter'` | Direct temporal succession | +| `EventuallyLeadsTo` | `'eventuallyLeadsTo'` | Long-term consequence | +| `SimultaneousWith` | `'simultaneousWith'` | Exact temporal alignment | +| `HasDuration` | `'hasDuration'` | Temporal extent | +| `RecurringWith` | `'recurringWith'` | Cyclic temporal relationship | + +### Advanced Spatial (9) + +| VerbType | Value | Meaning | +|----------|-------|---------| +| `ContainsSpatially` | `'containsSpatially'` | Spatial containment | +| `OverlapsSpatially` | `'overlapsSpatially'` | Spatial overlap | +| `Surrounds` | `'surrounds'` | Encirclement | +| `ConnectedTo` | `'connectedTo'` | Topological connection | +| `Above` | `'above'` | Vertical (superior position) | +| `Below` | `'below'` | Vertical (inferior position) | +| `Inside` | `'inside'` | Within containment boundaries | +| `Outside` | `'outside'` | Beyond containment boundaries | +| `Facing` | `'facing'` | Directional orientation | + +### Social Structures (5) + +| VerbType | Value | Meaning | +|----------|-------|---------| +| `Represents` | `'represents'` | Representative relationship | +| `Embodies` | `'embodies'` | Exemplification or personification | +| `Opposes` | `'opposes'` | Opposition relationship | +| `AlliesWith` | `'alliesWith'` | Alliance relationship | +| `ConformsTo` | `'conformsTo'` | Norm conformity | + +### Measurement (4) + +| VerbType | Value | Meaning | +|----------|-------|---------| +| `MeasuredIn` | `'measuredIn'` | Unit relationship | +| `ConvertsTo` | `'convertsTo'` | Unit conversion | +| `HasMagnitude` | `'hasMagnitude'` | Quantitative value | +| `DimensionallyEquals` | `'dimensionallyEquals'` | Dimensional analysis | + +### Change & Persistence (4) + +| VerbType | Value | Meaning | +|----------|-------|---------| +| `PersistsThrough` | `'persistsThrough'` | Persistence through change | +| `GainsProperty` | `'gainsProperty'` | Property acquisition | +| `LosesProperty` | `'losesProperty'` | Property loss | +| `RemainsSame` | `'remainsSame'` | Identity through time | + +### Parthood Variations (4) + +| VerbType | Value | Meaning | +|----------|-------|---------| +| `FunctionalPartOf` | `'functionalPartOf'` | Functional component | +| `TopologicalPartOf` | `'topologicalPartOf'` | Spatial part | +| `TemporalPartOf` | `'temporalPartOf'` | Temporal slice | +| `ConceptualPartOf` | `'conceptualPartOf'` | Abstract decomposition | + +### Dependency Variations (3) + +| VerbType | Value | Meaning | +|----------|-------|---------| +| `RigidlyDependsOn` | `'rigidlyDependsOn'` | Necessary dependency | +| `FunctionallyDependsOn` | `'functionallyDependsOn'` | Operational dependency | +| `HistoricallyDependsOn` | `'historicallyDependsOn'` | Causal history dependency | + +### Meta-Level (4) + +| VerbType | Value | Meaning | +|----------|-------|---------| +| `Endorses` | `'endorses'` | Second-order validation | +| `Contradicts` | `'contradicts'` | Logical contradiction | +| `Supports` | `'supports'` | Evidential support | +| `Supersedes` | `'supersedes'` | Replacement relationship | ## Coverage Completeness Analysis -### Is Anything Missing? +### Is Anything Missing? -While we could add more specific verb types (like "approves", "delegates", "shares"), our current taxonomy is **mathematically complete** for several reasons: +The taxonomy is intentionally bounded. When no type is an exact fit, three escape hatches keep it complete: #### 1. Generic Fallbacks -- **`Custom` noun type**: For any entity that doesn't fit standard types -- **`RelatedTo` verb type**: For any relationship not explicitly defined -- **Unlimited metadata**: Any additional semantics via properties -#### 2. Semantic Flexibility Through Metadata +- **`Custom` noun type**: For any entity that doesn't fit a standard type +- **`RelatedTo` verb type**: For any relationship not explicitly named +- **`subtype` + metadata**: Refine a base type with domain-specific semantics -Instead of adding dozens more verb types, we use metadata for specificity: +#### 2. Semantic Flexibility Through Subtype & Metadata + +Instead of adding ever more verb types, refine a base verb with a `subtype` and structured metadata: ```typescript -// Instead of adding "approves" verb: -await brain.relate(managerId, requestId, "evaluates", { - result: "approved", - timestamp: Date.now() +// "approves" → Evaluates with a result +await brain.relate({ + from: managerId, + to: requestId, + type: VerbType.Evaluates, + subtype: 'approval', + metadata: { result: 'approved', timestamp: Date.now() } }) -// Instead of adding "shares" verb: -await brain.relate(userId, documentId, "communicates", { - action: "shared", - permissions: "read-only" +// "shares" → Communicates with an action +await brain.relate({ + from: userId, + to: documentId, + type: VerbType.Communicates, + subtype: 'share', + metadata: { permissions: 'read-only' } }) -// Instead of adding "delegates" verb: -await brain.relate(managerId, taskId, "creates", { - delegatedTo: employeeId, - authority: "full" +// "delegates" → ParticipatesIn with a role + delegation metadata +await brain.relate({ + from: employeeId, + to: taskId, + type: VerbType.ParticipatesIn, + subtype: 'delegate', + metadata: { delegatedBy: managerId, authority: 'full' } }) ``` #### 3. Edge Cases Are Covered -Even exotic scenarios work with our current types: +Even exotic scenarios fit the standard types: ```typescript // Quantum computing -const qubitId = await brain.add("Qubit-1", { - type: "thing", - subtype: "quantum_bit", - superposition: [0.707, 0.707] +const qubitId = await brain.add({ + data: 'Qubit-1', + type: NounType.Thing, + subtype: 'quantum-bit', + metadata: { superposition: [0.707, 0.707] } }) // Cryptocurrency transactions -const txId = await brain.add("Bitcoin Transfer", { - type: "event", - subtype: "blockchain_transaction", - hash: "1A2B3C..." +const txId = await brain.add({ + data: 'Bitcoin Transfer', + type: NounType.Event, + subtype: 'blockchain-transaction', + metadata: { hash: '1A2B3C...' } }) // AI model training -const modelId = await brain.add("Neural Network", { - type: "process", - subtype: "ml_model", - architecture: "transformer" +const modelId = await brain.add({ + data: 'Neural Network', + type: NounType.Process, + subtype: 'ml-model', + metadata: { architecture: 'transformer' } }) ``` ### The Philosophy: Simplicity Over Specificity -We intentionally keep the type system minimal because: -1. **Fewer types = easier to learn** -2. **Metadata provides infinite extensibility** -3. **Consistent patterns across domains** -4. **Avoids taxonomy explosion** +A bounded type system stays learnable: +1. **A fixed vocabulary is easier to learn** than thousands of bespoke schemas +2. **Subtype + metadata provide open-ended extensibility** +3. **Consistent patterns** carry across domains +4. **No taxonomy explosion** as new use cases appear ## Industry-Specific Coverage Analysis -### Why 42 Nouns + 127 Verbs = Universal Coverage +### Why 42 Nouns + 127 Verbs = Broad Coverage -The combination of **31 noun types** and **40 verb types** creates **1,240 basic combinations**, but with metadata and multi-hop relationships, this expands to **infinite expressiveness**. Here's how it covers every industry: +The combination of **42 noun types** and **127 verb types** yields **5,334 base combinations**, and with subtypes, metadata, and multi-hop relationships that expands to cover essentially any domain. Here's how it maps onto common industries. ### Healthcare & Medical + ```typescript -// Patient records with medical history -const patientId = await brain.add("John Doe", { - type: "person", - subtype: "patient", - mrn: "12345" +const patientId = await brain.add({ + data: 'John Doe', + type: NounType.Person, + subtype: 'patient', + metadata: { mrn: '12345' } }) -const diagnosisId = await brain.add("Type 2 Diabetes", { - type: "state", - subtype: "diagnosis", - icd10: "E11.9" +const diagnosisId = await brain.add({ + data: 'Type 2 Diabetes', + type: NounType.State, + subtype: 'diagnosis', + metadata: { icd10: 'E11.9' } }) -const medicationId = await brain.add("Metformin", { - type: "thing", - subtype: "medication", - dosage: "500mg" +const medicationId = await brain.add({ + data: 'Metformin', + type: NounType.Substance, + subtype: 'medication', + metadata: { dosage: '500mg' } }) +const doctorId = await brain.add({ data: 'Dr. Reyes', type: NounType.Person, subtype: 'physician' }) + // Medical relationships -await brain.relate(patientId, diagnosisId, "diagnoses") -await brain.relate(medicationId, diagnosisId, "treats") -await brain.relate(doctorId, patientId, "treats") +await brain.relate({ from: patientId, to: diagnosisId, type: VerbType.HasQuality, subtype: 'diagnosis' }) +await brain.relate({ from: medicationId, to: diagnosisId, type: VerbType.Affects, subtype: 'treats' }) +await brain.relate({ from: doctorId, to: patientId, type: VerbType.RelatedTo, subtype: 'treats' }) ``` ### Finance & Banking + ```typescript -// Financial instruments and transactions -const accountId = await brain.add("Checking Account", { - type: "thing", - subtype: "account", - balance: 10000 +const accountId = await brain.add({ + data: 'Checking Account', + type: NounType.Thing, + subtype: 'account', + metadata: { balance: 10000 } }) -const transactionId = await brain.add("Wire Transfer", { - type: "event", - subtype: "transaction", - amount: 5000 +const transactionId = await brain.add({ + data: 'Wire Transfer', + type: NounType.Event, + subtype: 'transaction', + metadata: { amount: 5000 } }) -const regulationId = await brain.add("GDPR Compliance", { - type: "concept", - subtype: "regulation" +const regulationId = await brain.add({ + data: 'KYC Requirement', + type: NounType.Regulation, + subtype: 'compliance' }) +const customerId = await brain.add({ data: 'Account Holder', type: NounType.Person, subtype: 'customer' }) + // Financial relationships -await brain.relate(customerId, accountId, "owns") -await brain.relate(transactionId, accountId, "modifies") -await brain.relate(accountId, regulationId, "compliesWith") +await brain.relate({ from: customerId, to: accountId, type: VerbType.Owns }) +await brain.relate({ from: transactionId, to: accountId, type: VerbType.Modifies }) +await brain.relate({ from: accountId, to: regulationId, type: VerbType.ConformsTo }) ``` ### Manufacturing & Supply Chain + ```typescript -// Production and logistics -const factoryId = await brain.add("Plant #3", { - type: "location", - subtype: "facility" +const factoryId = await brain.add({ + data: 'Plant #3', + type: NounType.Location, + subtype: 'facility' }) -const assemblyLineId = await brain.add("Assembly Line A", { - type: "process", - subtype: "production" +const assemblyLineId = await brain.add({ + data: 'Assembly Line A', + type: NounType.Process, + subtype: 'production' }) -const componentId = await brain.add("Circuit Board v2", { - type: "thing", - subtype: "component" +const componentId = await brain.add({ + data: 'Circuit Board v2', + type: NounType.Thing, + subtype: 'component' }) +const productId = await brain.add({ data: 'Controller Unit', type: NounType.Product }) +const supplierId = await brain.add({ data: 'Acme Components', type: NounType.Organization, subtype: 'supplier' }) + // Manufacturing relationships -await brain.relate(assemblyLineId, componentId, "produces") -await brain.relate(componentId, productId, "partOf") -await brain.relate(supplierId, componentId, "supplies") +await brain.relate({ from: assemblyLineId, to: componentId, type: VerbType.Creates }) +await brain.relate({ from: componentId, to: productId, type: VerbType.PartOf }) +await brain.relate({ from: supplierId, to: componentId, type: VerbType.RelatedTo, subtype: 'supplies' }) ``` ### Education & Learning + ```typescript -// Educational content and progress -const courseId = await brain.add("Machine Learning 101", { - type: "collection", - subtype: "course" +const courseId = await brain.add({ + data: 'Machine Learning 101', + type: NounType.Collection, + subtype: 'course' }) -const lessonId = await brain.add("Neural Networks", { - type: "content", - subtype: "lesson" +const lessonId = await brain.add({ + data: 'Neural Networks', + type: NounType.Document, + subtype: 'lesson' }) -const assessmentId = await brain.add("Final Exam", { - type: "event", - subtype: "assessment" +const assessmentId = await brain.add({ + data: 'Final Exam', + type: NounType.Event, + subtype: 'assessment' }) +const studentId = await brain.add({ data: 'Student #88', type: NounType.Person, subtype: 'student' }) + // Educational relationships -await brain.relate(studentId, courseId, "enrolledIn") -await brain.relate(courseId, lessonId, "contains") -await brain.relate(studentId, assessmentId, "completed") +await brain.relate({ from: studentId, to: courseId, type: VerbType.MemberOf, subtype: 'enrolled' }) +await brain.relate({ from: courseId, to: lessonId, type: VerbType.Contains }) +await brain.relate({ from: studentId, to: assessmentId, type: VerbType.ParticipatesIn, subtype: 'completed' }) ``` ### Legal & Compliance + ```typescript -// Legal documents and cases -const contractId = await brain.add("Service Agreement", { - type: "document", - subtype: "contract" +const contractId = await brain.add({ + data: 'Service Agreement', + type: NounType.Contract, + subtype: 'service-agreement' }) -const clauseId = await brain.add("Liability Clause", { - type: "content", - subtype: "clause" +const clauseId = await brain.add({ + data: 'Liability Clause', + type: NounType.Document, + subtype: 'clause' }) -const caseId = await brain.add("Case #2024-1234", { - type: "event", - subtype: "legal_case" +const caseId = await brain.add({ + data: 'Case #2024-1234', + type: NounType.Event, + subtype: 'legal-case' }) +const partyId = await brain.add({ data: 'Counterparty LLC', type: NounType.Organization }) + // Legal relationships -await brain.relate(contractId, clauseId, "contains") -await brain.relate(party1Id, contractId, "signedBy") -await brain.relate(caseId, contractId, "references") +await brain.relate({ from: contractId, to: clauseId, type: VerbType.Contains }) +await brain.relate({ from: partyId, to: contractId, type: VerbType.RelatedTo, subtype: 'signatory' }) +await brain.relate({ from: caseId, to: contractId, type: VerbType.References }) ``` ### Retail & E-commerce + ```typescript -// Products and customer behavior -const productId = await brain.add("iPhone 15", { - type: "product", - sku: "IP15-128-BLK" +const productId = await brain.add({ + data: 'Wireless Earbuds', + type: NounType.Product, + metadata: { sku: 'WE-128-BLK' } }) -const cartId = await brain.add("Shopping Cart", { - type: "collection", - subtype: "cart" +const cartId = await brain.add({ + data: 'Shopping Cart', + type: NounType.Collection, + subtype: 'cart' }) -const promotionId = await brain.add("Black Friday Sale", { - type: "event", - subtype: "promotion" +const promotionId = await brain.add({ + data: 'Holiday Sale', + type: NounType.Event, + subtype: 'promotion' }) +const customerId = await brain.add({ data: 'Customer #5521', type: NounType.Person, subtype: 'customer' }) + // Retail relationships -await brain.relate(customerId, productId, "views") -await brain.relate(cartId, productId, "contains") -await brain.relate(promotionId, productId, "applies") +await brain.relate({ from: customerId, to: productId, type: VerbType.RelatedTo, subtype: 'view' }) +await brain.relate({ from: cartId, to: productId, type: VerbType.Contains }) +await brain.relate({ from: promotionId, to: productId, type: VerbType.Affects, subtype: 'applies' }) ``` ### Real Estate + ```typescript -// Properties and transactions -const propertyId = await brain.add("123 Main St", { - type: "location", - subtype: "property" +const propertyId = await brain.add({ + data: '123 Main St', + type: NounType.Location, + subtype: 'property' }) -const listingId = await brain.add("MLS #789", { - type: "document", - subtype: "listing" +const listingId = await brain.add({ + data: 'MLS #789', + type: NounType.Document, + subtype: 'listing' }) -const inspectionId = await brain.add("Home Inspection", { - type: "event", - subtype: "inspection" +const inspectionId = await brain.add({ + data: 'Home Inspection', + type: NounType.Event, + subtype: 'inspection' }) +const ownerId = await brain.add({ data: 'Property Owner', type: NounType.Person }) + // Real estate relationships -await brain.relate(ownerId, propertyId, "owns") -await brain.relate(listingId, propertyId, "describes") -await brain.relate(inspectionId, propertyId, "evaluates") +await brain.relate({ from: ownerId, to: propertyId, type: VerbType.Owns }) +await brain.relate({ from: listingId, to: propertyId, type: VerbType.Describes }) +await brain.relate({ from: inspectionId, to: propertyId, type: VerbType.Evaluates }) ``` ### Government & Public Sector + ```typescript -// Civic data and services -const citizenId = await brain.add("Citizen #123", { - type: "person", - subtype: "citizen" +const citizenId = await brain.add({ + data: 'Citizen #123', + type: NounType.Person, + subtype: 'citizen' }) -const permitId = await brain.add("Building Permit", { - type: "document", - subtype: "permit" +const permitId = await brain.add({ + data: 'Building Permit', + type: NounType.Document, + subtype: 'permit' }) -const departmentId = await brain.add("Planning Dept", { - type: "organization", - subtype: "government" +const departmentId = await brain.add({ + data: 'Planning Dept', + type: NounType.Organization, + subtype: 'government' }) +const propertyId = await brain.add({ data: '500 Oak Ave', type: NounType.Location, subtype: 'property' }) + // Government relationships -await brain.relate(citizenId, permitId, "requests") -await brain.relate(departmentId, permitId, "issues") -await brain.relate(permitId, propertyId, "authorizes") +await brain.relate({ from: citizenId, to: permitId, type: VerbType.RelatedTo, subtype: 'request' }) +await brain.relate({ from: departmentId, to: permitId, type: VerbType.Creates, subtype: 'issues' }) +await brain.relate({ from: permitId, to: propertyId, type: VerbType.PermittedTo, subtype: 'authorizes' }) ``` -### Why This Covers All Knowledge +### Why This Covers Most Knowledge -#### 1. **Mathematical Completeness** -The noun-verb model forms a **complete graph structure** where: +#### 1. Structural Completeness + +The noun-verb model forms a **graph structure** where: - Any entity can be represented as a noun - Any relationship can be represented as a verb - Complex knowledge emerges from simple combinations -#### 2. **Semantic Completeness** -Every piece of human knowledge falls into one of these categories: +#### 2. Semantic Coverage + +Most information falls into one of these categories: - **Entities** (who, what, where) → Nouns -- **Actions** (how, when, why) → Verbs +- **Actions/relations** (how, when, why) → Verbs - **Attributes** (properties) → Metadata - **Context** (conditions) → Graph structure -#### 3. **Compositional Power** +#### 3. Compositional Power + Simple types combine to represent complex knowledge: + ```typescript -// Complex knowledge from simple building blocks -const researchPaper = await brain.add("AI Ethics Study", { - type: "document" -}) +const researchPaper = await brain.add({ data: 'AI Ethics Study', type: NounType.Document }) +const researcher = await brain.add({ data: 'Dr. Smith', type: NounType.Person }) +const institution = await brain.add({ data: 'MIT', type: NounType.Organization }) +const concept = await brain.add({ data: 'AI Ethics', type: NounType.Concept }) -const researcher = await brain.add("Dr. Smith", { - type: "person" -}) - -const institution = await brain.add("MIT", { - type: "organization" -}) - -const concept = await brain.add("AI Ethics", { - type: "concept" -}) - -// Rich knowledge graph emerges -await brain.relate(researcher, researchPaper, "authors") -await brain.relate(researcher, institution, "affiliated") -await brain.relate(researchPaper, concept, "explores") -await brain.relate(institution, researchPaper, "publishes") +// A rich knowledge graph emerges from a handful of typed edges +await brain.relate({ from: researcher, to: researchPaper, type: VerbType.Creates }) +await brain.relate({ from: researcher, to: institution, type: VerbType.MemberOf }) +await brain.relate({ from: researchPaper, to: concept, type: VerbType.Describes }) +await brain.relate({ from: institution, to: researchPaper, type: VerbType.Creates, subtype: 'publishes' }) ``` -#### 4. **Domain Independence** -The same types work across all domains: +#### 4. Domain Independence + +The same types work across domains: **Science:** ```typescript -await brain.add("H2O", { type: "thing", category: "molecule" }) -await brain.add("Photosynthesis", { type: "process" }) -await brain.relate(moleculeId, processId, "participates") +const moleculeId = await brain.add({ data: 'H2O', type: NounType.Substance, metadata: { category: 'molecule' } }) +const processId = await brain.add({ data: 'Photosynthesis', type: NounType.Process }) +await brain.relate({ from: moleculeId, to: processId, type: VerbType.ParticipatesIn }) ``` **Business:** ```typescript -await brain.add("Q3 Revenue", { type: "metric", value: 10000000 }) -await brain.add("Sales Team", { type: "organization" }) -await brain.relate(teamId, metricId, "achieves") +const metricId = await brain.add({ data: 'Q3 Revenue', type: NounType.Measurement, metadata: { value: 10_000_000 } }) +const teamId = await brain.add({ data: 'Sales Team', type: NounType.Organization }) +await brain.relate({ from: teamId, to: metricId, type: VerbType.RelatedTo, subtype: 'achieves' }) ``` **Social:** ```typescript -await brain.add("John", { type: "person" }) -await brain.add("Community Group", { type: "organization" }) -await brain.relate(personId, groupId, "joins") +const personId = await brain.add({ data: 'John', type: NounType.Person }) +const groupId = await brain.add({ data: 'Community Group', type: NounType.SocialGroup }) +await brain.relate({ from: personId, to: groupId, type: VerbType.MemberOf }) ``` -#### 5. **Temporal Coverage** -Handles all temporal aspects: +#### 5. Temporal Coverage + +Time lives in edge metadata, so past, present, and future all fit: + ```typescript // Past -await brain.relate(personId, companyId, "worked", { - from: "2010", to: "2020" +await brain.relate({ + from: personId, + to: companyId, + type: VerbType.MemberOf, + subtype: 'past-employment', + metadata: { from: '2010', to: '2020' } }) // Present -await brain.relate(personId, projectId, "manages", { - since: "2024-01-01" +await brain.relate({ + from: personId, + to: projectId, + type: VerbType.ParticipatesIn, + subtype: 'manager', + metadata: { since: '2024-01-01' } }) // Future -await brain.relate(eventId, venueId, "scheduled", { - date: "2025-06-15" +await brain.relate({ + from: eventId, + to: venueId, + type: VerbType.LocatedAt, + metadata: { scheduledFor: '2025-06-15' } }) ``` -#### 6. **Hierarchical Representation** -Supports all levels of abstraction: +#### 6. Hierarchical Representation + +Every level of abstraction fits: + ```typescript // Micro level -await brain.add("Electron", { type: "thing", scale: "quantum" }) +await brain.add({ data: 'Electron', type: NounType.Thing, metadata: { scale: 'quantum' } }) // Macro level -await brain.add("Solar System", { type: "place", scale: "astronomical" }) +await brain.add({ data: 'Solar System', type: NounType.Location, metadata: { scale: 'astronomical' } }) // Abstract level -await brain.add("Justice", { type: "concept", domain: "philosophy" }) +await brain.add({ data: 'Justice', type: NounType.Concept, metadata: { domain: 'philosophy' } }) ``` ### Extensibility -While the core types cover all knowledge, you can extend with domain-specific subtypes: +While the core types cover most domains, you extend with `subtype` (and metadata) — never a schema migration: ```typescript -// Extend person for medical domain -await brain.add("Patient #12345", { - type: "person", - subtype: "patient", - medicalRecord: "MR-12345" +// Extend Person for the medical domain +await brain.add({ + data: 'Patient #12345', + type: NounType.Person, + subtype: 'patient', + metadata: { medicalRecord: 'MR-12345' } }) -// Extend document for legal domain -await brain.add("Contract ABC", { - type: "document", - subtype: "contract", - jurisdiction: "California" +// Extend Document for the legal domain +await brain.add({ + data: 'Contract ABC', + type: NounType.Document, + subtype: 'contract', + metadata: { jurisdiction: 'California' } }) -// Custom verb for specific domain -await brain.relate(lawyerId, contractId, "negotiates", { - verbSubtype: "legal-action", - billableHours: 10 +// Extend a verb with a domain-specific subtype + billing metadata +await brain.relate({ + from: lawyerId, + to: contractId, + type: VerbType.ParticipatesIn, + subtype: 'negotiator', + metadata: { billableHours: 10 } }) ``` -### Mathematical Proof of Universal Coverage +### How the Taxonomy Stays Complete -The noun-verb taxonomy achieves **Turing completeness** for knowledge representation: +The noun-verb model is designed to represent any knowledge that can be expressed as entities and relations: -1. **Storage Completeness**: Any data can be stored as nouns -2. **Relational Completeness**: Any relationship can be expressed as verbs -3. **Property Completeness**: Unlimited metadata captures all attributes -4. **Graph Completeness**: Multi-hop traversals express any complexity -5. **Temporal Completeness**: Time metadata handles all temporal aspects -6. **Semantic Completeness**: Vector embeddings capture meaning and similarity +1. **Storage**: Any data can be stored as nouns +2. **Relational**: Any relationship can be expressed as verbs +3. **Property**: Open-ended metadata captures all attributes +4. **Graph**: Multi-hop traversals express arbitrary complexity +5. **Temporal**: Date metadata handles all temporal aspects +6. **Semantic**: Vector embeddings capture meaning and similarity -#### The Infinity Formula +#### The Composition Formula ``` -Expressiveness = (31 nouns × 40 verbs) × ∞ metadata × ∞ graph depth - = 1,240 × ∞ × ∞ - = ∞ (Infinite Expressiveness) +Expressiveness = (42 nouns × 127 verbs) × metadata × graph depth + = 5,334 base combinations × open-ended refinement ``` -This mathematical infinity means Brainy can represent: -- **All Scientific Knowledge**: From quantum physics to molecular biology -- **All Business Data**: From transactions to supply chains -- **All Social Graphs**: From friendships to organizational hierarchies -- **All Historical Records**: From events to archaeological findings -- **All Creative Works**: From art metadata to story relationships -- **All Technical Systems**: From software architecture to network topology -- **All Personal Information**: From memories to preferences -- **Literally ANY Information That Can Exist** +That composition lets Brainy represent: +- **Scientific Knowledge**: From quantum physics to molecular biology +- **Business Data**: From transactions to supply chains +- **Social Graphs**: From friendships to organizational hierarchies +- **Historical Records**: From events to archaeological findings +- **Creative Works**: From media metadata to story relationships +- **Technical Systems**: From software architecture to network topology +- **Personal Information**: From memories to preferences ### Real-World Proof: Unmappable Becomes Mappable Even the most complex scenarios map naturally: ```typescript -// String Theory - 11-dimensional physics -const braneId = await brain.add("D3-Brane", { - type: "concept", - dimensions: 11, - vibrational_modes: ["0,1", "1,0", "2,1"] +// String Theory — high-dimensional physics +const braneId = await brain.add({ + data: 'D3-Brane', + type: NounType.Concept, + metadata: { dimensions: 11, vibrationalModes: ['0,1', '1,0', '2,1'] } }) -// Consciousness - The "hard problem" of philosophy -const qualiaId = await brain.add("Red Qualia", { - type: "concept", - subtype: "phenomenal_experience", - ineffable: true +// Consciousness — the "hard problem" of philosophy +const qualiaId = await brain.add({ + data: 'Red Qualia', + type: NounType.Concept, + subtype: 'phenomenal-experience', + metadata: { ineffable: true } }) -// Time Travel Paradoxes -const futureEvent = await brain.add("Future Effect", { - type: "event", - temporal_position: "future" +// Causal paradoxes +const futureEvent = await brain.add({ + data: 'Future Effect', + type: NounType.Event, + metadata: { temporalPosition: 'future' } }) -const pastCause = await brain.add("Past Cause", { - type: "event", - temporal_position: "past" +const pastCause = await brain.add({ + data: 'Past Cause', + type: NounType.Event, + metadata: { temporalPosition: 'past' } }) -await brain.relate(futureEvent, pastCause, "causes", { - paradox_type: "bootstrap" +await brain.relate({ + from: futureEvent, + to: pastCause, + type: VerbType.Causes, + metadata: { paradoxType: 'bootstrap' } }) ``` -If it exists, thinks, happens, or can be imagined—Brainy can model it. +If it exists, thinks, happens, or can be imagined — Brainy can model it. + +## Migration from Traditional Models + +### From Relational (SQL) + +```typescript +// Instead of JOIN queries: +// SELECT * FROM users JOIN orders ON users.id = orders.user_id + +// Use noun-verb relationships +const userId = await brain.add({ data: 'User', type: NounType.Person, metadata: { email: 'u@example.com' } }) +const orderId = await brain.add({ data: 'Order #1', type: NounType.Event, subtype: 'order' }) +await brain.relate({ from: userId, to: orderId, type: VerbType.Creates, subtype: 'placed' }) + +// Query naturally via the graph +const userOrders = await brain.find({ + type: NounType.Event, + connected: { from: userId, via: VerbType.Creates } +}) +``` + +### From Document (NoSQL) + +```typescript +// Instead of embedded documents: { user: { orders: [...] } } + +// Use explicit relationships +const userId = await brain.add({ data: 'User', type: NounType.Person }) +for (const order of orders) { + const orderId = await brain.add({ data: order.summary, type: NounType.Event, subtype: 'order' }) + await brain.relate({ from: userId, to: orderId, type: VerbType.Creates, subtype: 'placed' }) +} +``` + +### From Graph Databases + +```typescript +// Similar to a graph database, with added benefits: +// 1. Automatic vector embeddings for similarity +// 2. Natural language querying +// 3. Unified with metadata filtering + +// Vector + graph in one query +const results = await brain.find({ query: 'users who bought similar products' }) +``` ## Conclusion -The Noun-Verb taxonomy in Brainy 2.0 provides a natural, flexible, and powerful way to model any domain. By thinking in terms of entities and their relationships, you can build everything from simple data stores to complex knowledge graphs while maintaining code clarity and query simplicity. +The Noun-Verb Taxonomy gives Brainy a natural, flexible, and powerful way to model any domain. By thinking in terms of entities (42 `NounType`s) and their relationships (127 `VerbType`s) — refined with `subtype` and metadata — you can build everything from simple data stores to complex knowledge graphs while keeping code clear and queries simple. ## See Also -- [Triple Intelligence](./triple-intelligence.md) -- [Natural Language Queries](../guides/natural-language.md) -- [API Reference](../api/README.md) \ No newline at end of file +- [Triple Intelligence](/docs/concepts/triple-intelligence) +- [Subtypes & Facets](/docs/guides/subtypes-and-facets) +- [The Find System](/docs/guides/find-system) +- [API Reference](/docs/api/reference) diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 3f77e133..1bc6e66c 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -48,7 +48,7 @@ Pluggable vector index (`VectorIndexProvider`) for efficient nearest-neighbor se - **Configurable recall**: `fast` / `balanced` / `accurate` presets trade recall for latency - **Scalable**: Handles millions of vectors per process - **Persistent**: Serializable to storage -- **Swappable**: Replace with a native implementation (such as `@soulcraft/cortex`) via the plugin system without changing application code +- **Swappable**: Replace with a native implementation (such as `@soulcraft/cor`) via the plugin system without changing application code ### Metadata Index Manager High-performance field indexing system: diff --git a/docs/architecture/triple-intelligence.md b/docs/architecture/triple-intelligence.md index 0751444a..fbec56a9 100644 --- a/docs/architecture/triple-intelligence.md +++ b/docs/architecture/triple-intelligence.md @@ -23,29 +23,37 @@ Traditional databases force you to choose between vector search, graph traversal ### Unified Query Structure +`find()` accepts a single `FindParams` object (or a natural-language string). One +object combines all three intelligences: + ```typescript -interface TripleQuery { - // Vector/Semantic search - like?: string | Vector | any - similar?: string | Vector | any - - // Graph/Relationship search +interface FindParams { + // Vector intelligence — semantic similarity + query?: string // Natural-language / semantic query (embedded, matched via HNSW + text index) + vector?: number[] // Pre-computed embedding for direct vector search + + // Metadata intelligence — structured field filters + type?: NounType | NounType[] // Filter by entity type + subtype?: string | string[] // Filter by per-product subtype + where?: Record // Field predicates with bare operators (gte, lt, in, contains, exists…) + + // Graph intelligence — relationship traversal connected?: { - to?: string | string[] - from?: string | string[] - type?: string | string[] - depth?: number + to?: string // Reachable to this entity + from?: string // Reachable from this entity + via?: VerbType | VerbType[] // Relationship type(s) to traverse (alias: type) + depth?: number // Max traversal depth (default: 1) direction?: 'in' | 'out' | 'both' } - - // Field/Attribute search - where?: Record - - // Advanced options - limit?: number - boost?: 'recent' | 'popular' | 'verified' | string - explain?: boolean - threshold?: number + + // Proximity — nearest neighbours of a known entity + near?: { id: string; threshold?: number } + + // Control + limit?: number // Max results (default: 10) + offset?: number // Skip N results + orderBy?: string // Field to sort by (e.g. 'createdAt') + order?: 'asc' | 'desc' // Sort direction } ``` @@ -74,10 +82,10 @@ const results = await brain.find("machine learning concepts") #### Combined Intelligence Query ```typescript const results = await brain.find({ - like: "neural networks", + query: "neural networks", where: { category: "research", - year: { $gte: 2023 } + year: { gte: 2023 } }, connected: { to: "deep-learning-team", @@ -109,8 +117,8 @@ All three search types execute simultaneously: ```typescript // Parallel execution for balanced query const results = await brain.find({ - like: "AI research", // ~1000 potential matches - where: { type: "paper" }, // ~500 potential matches + query: "AI research", // ~1000 potential matches + where: { kind: "paper" }, // ~500 potential matches connected: { to: "stanford" } // ~200 potential matches }) // All three execute in parallel, results fused @@ -126,7 +134,7 @@ Operations chain for maximum efficiency: // Progressive execution for selective query const results = await brain.find({ where: { userId: "user123" }, // Very selective (1-10 matches) - like: "recent posts", // Applied to filtered set + query: "recent posts", // Applied to filtered set limit: 5 }) // Metadata filter first, then vector search on results @@ -166,10 +174,10 @@ const results = await brain.find( ) // Automatically converts to: // { -// like: "AI papers", -// where: { +// query: "AI papers", +// where: { // institution: "Stanford", -// published: { $gte: "2024-01-01" } +// published: { gte: "2024-01-01" } // } // } ``` @@ -188,10 +196,10 @@ The NLP processor identifies query intent: Successful execution plans are cached: ```typescript -// First query: 50ms (plan generation + execution) +// First call parses the natural-language query and builds an execution plan await brain.find("machine learning papers") -// Subsequent similar queries: 10ms (cached plan) +// A structurally similar query reuses that plan, skipping plan generation await brain.find("deep learning papers") ``` @@ -213,50 +221,45 @@ Triple Intelligence leverages all available indexes: ### Explain Mode -Understand how your query was executed: +Diagnose how a query's `where` fields map to the index. Run `brain.explain()` +first whenever `find()` returns surprising or empty results: ```typescript -const results = await brain.find({ - like: "quantum computing", - where: { category: "research" }, - explain: true +const plan = await brain.explain({ + query: "quantum computing", + where: { category: "research" } }) -console.log(results[0].explanation) -// { -// plan: "field-first-progressive", -// timing: { -// fieldFilter: 2, -// vectorSearch: 8, -// fusion: 1 -// }, -// selectivity: { -// field: 0.1, -// vector: 0.3 -// } -// } +console.log(plan.fieldPlan) +// [ +// { field: 'category', path: 'column-store', notes: '...' } +// ] + +console.log(plan.warnings) +// e.g. ['Field "category" has no index entries. find() will return [] silently...'] ``` -### Boosting +### Result Ordering -Apply custom ranking boosts: +Sort results by any stored field with `orderBy` / `order`: ```typescript const results = await brain.find({ - like: "news articles", - boost: 'recent', // Boost recent items - where: { verified: true } + query: "news articles", + where: { verified: true }, + orderBy: 'createdAt', // Newest first + order: 'desc' }) ``` -### Threshold Control +### Similarity Threshold -Set minimum similarity thresholds: +Find the nearest neighbours of a known entity and keep only close matches with +`near`: ```typescript const results = await brain.find({ - like: "exact match needed", - threshold: 0.9, // Only very similar results + near: { id: anchorId, threshold: 0.9 }, // Only results >= 0.9 similarity limit: 10 }) ``` @@ -283,8 +286,8 @@ const results = await brain.find({ ```typescript // Find similar content with constraints const results = await brain.find({ - like: query, - where: { + query: searchText, + where: { status: 'published', language: 'en' } @@ -295,10 +298,10 @@ const results = await brain.find({ ```typescript // Find items related to a specific item const results = await brain.find({ - connected: { + connected: { to: itemId, depth: 2, - type: 'similar' + via: VerbType.RelatedTo }, limit: 20 }) @@ -309,10 +312,11 @@ const results = await brain.find({ // Recent items matching criteria const results = await brain.find({ where: { - timestamp: { $gte: Date.now() - 86400000 } + timestamp: { gte: Date.now() - 86400000 } }, - like: "trending topics", - boost: 'recent' + query: "trending topics", + orderBy: 'timestamp', + order: 'desc' }) ``` diff --git a/docs/architecture/zero-config.md b/docs/architecture/zero-config.md index 64a10461..d42d6784 100644 --- a/docs/architecture/zero-config.md +++ b/docs/architecture/zero-config.md @@ -84,7 +84,7 @@ const brain = new Brainy({ ``` The default JS index is `JsHnswVectorIndex`. An optional native acceleration -provider (the `@soulcraft/cortex` package) can replace it with a +provider (the `@soulcraft/cor` package) can replace it with a higher-performing implementation; the public knobs stay the same. Quantization and other index-internal acceleration are the native provider's concern, not a Brainy configuration option. diff --git a/docs/concepts/consistency-model.md b/docs/concepts/consistency-model.md index db8efd91..bad996c5 100644 --- a/docs/concepts/consistency-model.md +++ b/docs/concepts/consistency-model.md @@ -212,10 +212,12 @@ Historical records cost disk space, so retention is explicit: - Every live `Db` holds a refcounted **pin**; a record-set is never reclaimed while any pin could need it — pinned reads stay correct across compaction, always. -- `brain.compactHistory({ retainGenerations?, retainMs? })` reclaims - everything no retention rule and no pin protects, and records the - **horizon** — `asOf()` below it throws `GenerationCompactedError`, - explicitly, never partial data. +- The **`retention`** knob governs auto-compaction (on every `flush()`/ + `close()`): unset → ADAPTIVE (disk/RAM-pressure, zero-config) · `'all'` → + unbounded · `{ maxGenerations?, maxAge?, maxBytes? }` → explicit caps. + `brain.compactHistory({ maxGenerations?, maxAge?, maxBytes? })` reclaims + manually on the same caps, and records the **horizon** — `asOf()` below it + throws `GenerationCompactedError`, explicitly, never partial data. - To keep a state readable forever, `persist()` it first: snapshots are self-contained and unaffected by compaction of the source store. @@ -357,9 +359,13 @@ Stated plainly, so nothing surprises you in production: ([multi-process model](./multi-process.md)). Transactions are atomic within one writer process — there is no distributed or cross-process transaction coordination. -- **History granularity.** Only `transact()` batches produce historical - records; single-operation writes between commits stay visible through - earlier pins (see above). +- **History granularity.** Every write is its own immutable generation — + `transact()` batches AND single-operation `add`/`update`/`remove`/`relate`. + A pin always freezes against later writes, and every write is addressable + via `asOf()`. (`transact()` groups several operations into ONE atomic + generation; durability of single-op history is batched via async + group-commit — a hard crash can lose only the last un-flushed window's + *history*, never live data.) - **Compacted history is gone.** `asOf()` below the compaction horizon fails explicitly; persist what you must keep. - **Counter persistence is coalesced for single-operation writes.** Durable diff --git a/docs/concepts/generation-fact-log.md b/docs/concepts/generation-fact-log.md new file mode 100644 index 00000000..f9b3e974 --- /dev/null +++ b/docs/concepts/generation-fact-log.md @@ -0,0 +1,117 @@ +--- +title: The Generation Fact Log +slug: concepts/generation-fact-log +public: true +category: concepts +template: concept +order: 6 +description: Every committed write also appends a self-verifying "fact" — an after-image commit record — to an append-only log. What facts are, the crash-safety model, the scanFacts() streaming surface, family stamps, and how index providers consume the log for sequential heals. +next: + - concepts/consistency-model + - guides/snapshots-and-time-travel +--- + +# The Generation Fact Log + +Since 8.4.0, every committed generation also appends a **fact** — a compact record of what each +touched entity or relationship *became* — to an append-only, checksummed log under +`_generations/facts/`. Where the generational history answers *"what did things look like +before?"* (before-images, powering `asOf()` and rollback), the fact log answers *"what happened, +in order?"* — one sequential, self-verifying stream of the store's present being written. + +Nothing about querying changes. The fact log exists for three consumers: + +1. **Index heals and rebuilds** — one sequential read in commit order replaces a per-entity + directory walk over millions of files. +2. **Incremental catch-up** — a derived index that knows which generation it reflects reads *just + the gap*, instead of rebuilding from scratch. +3. **Replay and audit tooling** — anything that wants the store's committed timeline as a stream. + +## What a fact is + +One fact per committed generation: + +- **`generation`** and **`timestamp`** — which commit, when. +- **`ops`** — every write in that commit: `{ kind: 'noun' | 'verb', id, record }` where `record` + holds the entity's full after-image (both stored legs), or **`null` for a tombstone** — a + removal carries no body, by design. +- **`meta`** — the transaction metadata `transact()` was submitted with, when present. +- **`blobHashes`** — content-blob references, for exact reclamation accounting. + +Facts accumulate **from the first write after upgrading** — pre-existing history is not +retroactively converted, and consumers fall back to the enumeration walk when no log exists. + +## Crash safety, in one paragraph + +Facts are appended and fsynced **inside the same durability window as the commit itself**, before +the commit point — so after a crash, the log can only ever be *ahead* of committed truth, never +behind it with a hole. On open, the store reconciles the log back to the committed watermark: +torn tails are detected by per-record checksums and cut; whole records beyond the watermark are +truncated. The invariant every reader can rely on: **an absent generation was never committed; a +present fact was.** `transact()` facts are durable the moment `transact()` returns; single-op +facts share the same group-commit flush as the rest of their generation, so a hard kill loses the +fact and the generation *together* — never a torn state. + +## Reading the log + +```typescript +const scan = brain.scanFacts({ fromGeneration: 1 }) +if (scan) { + // Telemetry up front — progress bars get a denominator from second zero. + console.log(scan.headGeneration, scan.segmentCount, scan.approxFactCount) + + for await (const batch of scan.batches()) { + // Each batch: { facts, firstGeneration, lastGeneration, factCount, byteSize, segmentId } + for (const fact of batch.facts) { + for (const op of fact.ops) { + if (op.record === null) { + // a tombstone: op.id was removed in this generation + } + } + } + } + + console.log(scan.summary()) // { factsYielded, segmentsRead } — the cross-check +} +``` + +- `scanFacts()` returns `null` when the store hosts no fact log (older store, or a storage adapter + without binary append support) — fall back to enumerating entities. +- Scans run against a **snapshot**: facts appended after the scan opens never bleed in, each fact + is yielded exactly once, and a detected gap aborts loudly — never a silent skip. +- `brain.factSegmentPaths()` returns the immutable, *sealed* segment files for zero-copy consumers + (the append-mutable tail is excluded — read it through `scanFacts()`). + +## Family stamps: how a projection proves it's current + +Anything derived from the store — an index, the entity file tree itself — carries a **family +stamp**: a small JSON record of *which committed generation the projection reflects* +(`sourceGeneration`) plus the invariants that verify it whole (exact per-file byte sizes for +bounded families; rollup invariants like entity counts for unbounded ones). At open, coherence is +a **comparison**, not a walk: + +- stamp equals the committed watermark and invariants hold → serve; +- stamp is behind → the projection reads just the gap from the fact log; +- invariants fail → loud, named divergence — `brain.repairIndex()` rebuilds from canonical and + re-stamps. + +The verifier is exported (`verifyFamilyStamp`) so every projection — TypeScript or native — runs +literally the same check. + +## For plugin authors: the storage capability + +Index providers receive the storage adapter, not the brain — so the host wires the log onto it. +Feature-detect and prefer the stream; fall back to enumeration: + +```typescript +const scan = storage.scanFacts?.({ fromGeneration: stamp.sourceGeneration + 1 }) +if (scan) { + // sequential catch-up from the log +} else { + // enumeration walk (older store or adapter) +} +const committed = storage.committedGeneration?.() // the watermark stamps compare against +``` + +Providers must never construct their own reader over the log's files — the open path belongs to +the single writer (it reconciles the log at open); the capability is the sanctioned seam. diff --git a/docs/concepts/multi-process.md b/docs/concepts/multi-process.md index cd65594b..8fda315f 100644 --- a/docs/concepts/multi-process.md +++ b/docs/concepts/multi-process.md @@ -55,20 +55,20 @@ process opening the same directory: The fix is the lock: refuse to open a second writer. SQLite has done the same since the late 1990s (`SQLITE_BUSY`). -## What about Cortex? +## What about Cor? -Brainy + Cortex compose cleanly under this model: +Brainy + Cor compose cleanly under this model: -- Cortex stores its column-index segments inside the same `rootDir` (under +- Cor stores its column-index segments inside the same `rootDir` (under `indexes/_column_index/{field}/`). -- Segments (`*.cidx` files) are **immutable** once written. Cortex mmaps them +- Segments (`*.cidx` files) are **immutable** once written. Cor mmaps them read-only. - The `MANIFEST.json` per field is updated via atomic rename — readers see either the old or new manifest, never a torn file. -A reader process can safely mmap Cortex segments alongside a live writer +A reader process can safely mmap Cor segments alongside a live writer without coordination. The single Brainy writer lock at -`/locks/_writer.lock` covers Cortex too, because Cortex segment +`/locks/_writer.lock` covers Cor too, because Cor segment writes happen on the writer's side. ## Stale-lock detection @@ -141,7 +141,7 @@ The CLI `brainy inspect` subcommands all do this for you by default processes can both succeed at `init()` in writer mode and clobber each other's writes. A best-effort warning is logged in writer mode against a non-filesystem backend. -- **Long-running readers** do not automatically pick up new Cortex segments +- **Long-running readers** do not automatically pick up new Cor segments the writer publishes. One-shot inspector calls re-open the store and see fresh segments; a reader that stays open for hours sees its column store as-of the time it opened. @@ -150,4 +150,4 @@ The CLI `brainy inspect` subcommands all do this for you by default - `Brainy.openReadOnly()` — [API reference](../api/brainy.md) - `brainy inspect` — [inspection guide](../guides/inspection.md) -- Cortex columnar storage — see `node_modules/@soulcraft/cortex/README.md` +- Cor columnar storage — see `node_modules/@soulcraft/cor/README.md` diff --git a/docs/concepts/storage-adapters.md b/docs/concepts/storage-adapters.md index e0f9b70d..af6d068f 100644 --- a/docs/concepts/storage-adapters.md +++ b/docs/concepts/storage-adapters.md @@ -33,7 +33,7 @@ BaseStorage (type-statistics, lifecycle helpers, generation FileSystemStorage (real filesystem I/O, writer-lock implementation, flush-request watcher, atomic writes) ↑ - (Cortex's MmapFileSystemStorage, etc.) + (Cor's MmapFileSystemStorage, etc.) ``` When you `extend FileSystemStorage`, your adapter inherits every method on diff --git a/docs/eli5.md b/docs/eli5.md index 303de040..e0bb9a19 100644 --- a/docs/eli5.md +++ b/docs/eli5.md @@ -11,7 +11,7 @@ next: - getting-started/quick-start --- -# Brainy and Cortex — Explained Simply +# Brainy and Cor — Explained Simply *A plain-language guide for anyone who wants to understand what this thing actually does.* @@ -65,13 +65,13 @@ Brainy can narrow any result set down by exact labels or ranges in the same brea --- -## What is Cortex? +## What is Cor? -Cortex is a turbocharger for Brainy. +Cor is a turbocharger for Brainy. Same car. Same controls. Same fuel. You just swap in a faster engine under the hood, and everything that used to take a moment now happens instantly. -Technically, Cortex is an optional plugin written in Rust — a lower-level language that runs much closer to the raw metal of your processor. It plugs into Brainy and takes over the most compute-intensive work: the distance calculations that power meaning search, the number-crunching behind live aggregates, and the set operations that drive label filtering. +Technically, Cor is an optional plugin written in Rust — a lower-level language that runs much closer to the raw metal of your processor. It plugs into Brainy and takes over the most compute-intensive work: the distance calculations that power meaning search, the number-crunching behind live aggregates, and the set operations that drive label filtering. You install it with one line, register it with one call, and Brainy automatically uses it everywhere it can help. @@ -83,9 +83,9 @@ Plain language: - **Searches** go from "the blink of an eye" to "faster than a blink." The overall speedup is **5.2× on average** across all operations. - **Live aggregates** are rebuilt using all CPU cores in parallel, so re-indexing large datasets takes a fraction of the time. -- **Analytics** that aren't even possible in pure JavaScript — real-time anomaly detection, streaming percentile estimates, approximate unique counts — become available because Cortex brings the native capabilities required to run them efficiently. +- **Analytics** that aren't even possible in pure JavaScript — real-time anomaly detection, streaming percentile estimates, approximate unique counts — become available because Cor brings the native capabilities required to run them efficiently. -If Brainy is what makes knowledge fast, Cortex is what makes Brainy feel instant. +If Brainy is what makes knowledge fast, Cor is what makes Brainy feel instant. --- @@ -135,7 +135,7 @@ Brainy is the only row with every box checked. And it runs all of them in a sing Brainy scales from a quick experiment to serious production datasets without changing a line of code. Small datasets live entirely in memory. Larger ones spill to disk, where Brainy shards and compresses everything automatically. Need a backup or a copy? Snapshots are instant — the same API the whole way. -Add Cortex and you also unlock memory-mapped storage — aggregate state lives directly in the operating system's memory with zero serialization overhead, as fast as the hardware allows. +Add Cor and you also unlock memory-mapped storage — aggregate state lives directly in the operating system's memory with zero serialization overhead, as fast as the hardware allows. --- diff --git a/docs/guides/aggregation.md b/docs/guides/aggregation.md index 1ae2c98e..11d86ec8 100644 --- a/docs/guides/aggregation.md +++ b/docs/guides/aggregation.md @@ -11,7 +11,13 @@ No batch jobs. No scheduled recalculations. Aggregates stay current with every w **Defining over existing data:** if you define an aggregate on a store that already holds matching entities, Brainy backfills it from those entities on the first query (a one-time scan, then purely incremental). So `defineAggregate()` behaves the same whether you define it before -or after the data exists — including when a persisted brain reopens already populated. +or after the data exists. + +**Reopening a persisted brain:** aggregate state persists across restarts. Re-defining the +same aggregate at boot (the normal declarative pattern) adopts the persisted state directly — +no rescan. A backfill scan runs only when the definition actually changed, when no persisted +state exists, or when the state failed to load; and however many aggregates need backfilling, +they share a single scan. ## Quick Start @@ -487,7 +493,7 @@ Aggregate definitions and running state are automatically persisted: ## Native Acceleration -When [Cortex](https://github.com/soulcraftlabs/cortex) is installed as a plugin, the aggregation engine automatically uses Rust-accelerated computation: +When [Cor](https://www.npmjs.com/package/@soulcraft/cor) is installed as a plugin, the aggregation engine automatically uses Rust-accelerated computation: - Incremental updates run in Rust with BTreeMap-backed precise MIN/MAX - Welford's online stddev/variance computed natively @@ -496,7 +502,7 @@ When [Cortex](https://github.com/soulcraftlabs/cortex) is installed as a plugin, ```typescript const brain = new Brainy({ - plugins: ['@soulcraft/cortex'] + plugins: ['@soulcraft/cor'] }) await brain.init() @@ -577,7 +583,7 @@ brain.defineAggregate({ Aggregation complexity per write is O(A x G x M) where A = matching aggregates, G = groupBy dimensions, M = metrics. For typical configurations (2-5 aggregates, 1-3 dimensions, 3-5 metrics), this is effectively O(1). -With Cortex native acceleration: +With Cor native acceleration: | Operation | Throughput | Latency | |-----------|-----------|---------| diff --git a/docs/guides/enterprise-for-everyone.md b/docs/guides/enterprise-for-everyone.md index de099d04..b79844e9 100644 --- a/docs/guides/enterprise-for-everyone.md +++ b/docs/guides/enterprise-for-everyone.md @@ -174,13 +174,13 @@ await brain.syncWith({ const brain = new Brainy() // 1 → ~1M vectors: pure-JS HNSW, zero extra setup -// 1M → 10B+ vectors: install @soulcraft/cortex for the native DiskANN provider +// 1M → 10B+ vectors: install @soulcraft/cor for the native DiskANN provider ``` **Scaling model:** - **Single process, no cluster**: Brainy runs in one process — no coordinator, no peer discovery, no consensus to operate -- **Optional native provider**: install `@soulcraft/cortex` to back the index with +- **Optional native provider**: install `@soulcraft/cor` to back the index with on-disk DiskANN that scales to 10B+ vectors on one machine - **Per-tenant pools**: isolate tenants by giving each its own Brainy instance and storage directory diff --git a/docs/guides/external-backups-and-sparse-storage.md b/docs/guides/external-backups-and-sparse-storage.md new file mode 100644 index 00000000..f28dcfd7 --- /dev/null +++ b/docs/guides/external-backups-and-sparse-storage.md @@ -0,0 +1,99 @@ +--- +title: External Backups & Sparse Storage +slug: guides/external-backups +public: true +category: guides +template: guide +order: 10 +description: How to back up a brain directory with external tools (tar, rsync, cp) without exploding sparse files — why a store can show 100+ GB "apparent" size on a small disk, which files are sparse, and how persist()/restore() handle it for you. +next: + - guides/snapshots-and-time-travel + - concepts/storage-adapters +--- + +# External Backups & Sparse Storage + +The built-in snapshot path — [`db.persist()` and `brain.restore()`](/docs/guides/snapshots-and-time-travel) — +already handles everything on this page for you. Read this when you back up a brain directory with +**external tools**: `tar`, `rsync`, `cp`, `scp`, or a filesystem-level backup agent. + +## The one-sentence rule + +> **Always use the sparse-aware flag**: `tar czSf` (capital `S`), `rsync --sparse`, +> `cp --sparse=always`. A naive copy can turn a 2 GB store into a 100+ GB one — or fail +> the disk entirely. + +## Why: some files are sparse + +When a native accelerator plugin is active, parts of the index live in **memory-mapped files** +created at a large fixed virtual size — the file's *apparent* size — while the filesystem only +allocates blocks that were actually written. A brand-new id-mapper file can report tens of +gigabytes in `ls -l` while occupying a few megabytes on disk. + +Check the difference yourself: + +```bash +ls -lh brain-data/_id_mapper/ # APPARENT size (can be huge) +du -sh brain-data/ # ALLOCATED size (the real footprint) +``` + +The sparse candidates in a brain directory: + +| Path | What it is | +|---|---| +| `_id_mapper/` | The native id-mapper's mmap files (large fixed virtual size) | +| `_blobs/` | Native index files (vector base, segments) — may be mmap-backed | + +Everything else (entities, `_system`, `_generations`, `_cas` content blobs) is ordinary dense data. + +## Doing it right + +**tar** — the `S` flag detects holes and stores only real data: + +```bash +tar czSf brain-backup.tgz /data/brain +# restore preserves the holes: +tar xzSf brain-backup.tgz -C /data/ +``` + +**rsync**: + +```bash +rsync -a --sparse /data/brain/ backup-host:/backups/brain/ +``` + +**cp**: + +```bash +cp -a --sparse=always /data/brain /backups/brain +``` + +**What goes wrong without the flag:** the copy *materializes* every hole as real zero bytes. +A store whose apparent size exceeds the target disk fails with `ENOSPC` partway through — and a +copy that *does* fit silently costs the full apparent size in storage and transfer time. + +## What the built-in paths do (so you don't have to) + +- **`db.persist(path)`** snapshots via **hard links** — instant and space-shared, since every data + file is immutable-by-rename. The handful of append-in-place files (the transaction log, the + commit fact log's tail segment) and mmap-mutated directories (`_id_mapper/`) are **byte-copied** + instead, so a post-snapshot write can never reach through a shared inode into your backup. +- **`brain.restore(path, { confirm: true })`** is **non-destructive and sparse-aware**: the snapshot + is copied into a staging area *before* any live data is touched (all-zero blocks stay holes), and + only after the copy fully succeeds does an atomic swap move it into place. A failed copy — + including `ENOSPC` — leaves the live store exactly as it was. A crash mid-swap completes forward + on the next open. + +## Live-store caveats for external tools + +1. **Prefer snapshotting a `persist()` output, not the live directory.** `persist()` produces a + crash-consistent, immutable snapshot; running `tar` against a live, actively-written directory + can capture a torn mid-write state. If you must archive live, stop writes first (or accept that + the archive is only as consistent as the moment's flush state). +2. **Never prune or "clean up" files inside a brain directory.** Index files that look stale or + redundant are load-bearing; the store protects its declared index families from in-process + deletion, but an external `rm` bypasses that fence. If space is the concern, `du -sh` first — + the allocated size is usually far smaller than it looks. +3. **Verify restores by opening them.** `Brainy.load(path)` opens any snapshot or restored + directory read-only — the store verifies its own coherence at open and reports loudly if + anything is missing or torn. diff --git a/docs/guides/find-limits.md b/docs/guides/find-limits.md index 66418b80..4c7fd252 100644 --- a/docs/guides/find-limits.md +++ b/docs/guides/find-limits.md @@ -40,6 +40,10 @@ Brainy picks `maxLimit` from the first of these that's available: Worked example: a 4 GB Cloud Run container picks priority 3 → `floor(4 GB × 0.25 / 25 KB) = floor(40 960) = 40 000` results. A 900 MB free-memory box on priority 4 gets `floor(900 MB / 25 KB) = ~36 000`. +The cap is fixed at construction and never changes at runtime. Query timing is recorded +for diagnostics only — a burst of slow queries cannot silently shrink the cap, and the +auto-detected tiers (3 and 4) never go below a floor of 10 000. + > **Calibration note.** Pre-7.30.2 used 100 KB per result instead of 25 KB, which produced caps that were 4× too tight for typical workloads (an 8 KB / result reality). 7.30.2 recalibrated to match observed entity sizes; existing `limit: 10_000` safety patterns now pass silently on any reasonably-sized box. ## What happens when you exceed the cap diff --git a/docs/guides/import-anything.md b/docs/guides/import-anything.md index e0cd94ed..b1bb15ef 100644 --- a/docs/guides/import-anything.md +++ b/docs/guides/import-anything.md @@ -300,7 +300,10 @@ await brain.import(data, { // Deduplication enableDeduplication: true, // Check for duplicate entities (default: true) deduplicationThreshold: 0.85, // Similarity threshold for duplicates (0-1, default: 0.85) - // Note: Auto-disabled for imports >100 entities + // Notes: false disables BOTH the inline merge and the background pass that + // runs ~5 min after the last import (merged duplicates are deleted). + // The inline pass auto-disables for imports >100 entities (O(n²) cost); + // the background pass still covers those unless the flag is false. // Performance chunkSize: 100, // Batch size for processing (default: varies by operation) diff --git a/docs/guides/import-quick-reference.md b/docs/guides/import-quick-reference.md index 5a850a86..7837d49e 100644 --- a/docs/guides/import-quick-reference.md +++ b/docs/guides/import-quick-reference.md @@ -86,11 +86,22 @@ await brain.import(file, { ```typescript await brain.import(file, { - enableDeduplication: true, // Check for duplicates (default: false) + enableDeduplication: true, // Check for duplicates (default: true) deduplicationThreshold: 0.85 // Similarity threshold (default: 0.85) }) ``` +Deduplication merges entities judged duplicates — the non-primary records are +**deleted**. Set `enableDeduplication: false` to disable it entirely: the flag +gates both the inline merge during import and the background pass that runs +about 5 minutes after the last import. + +```typescript +await brain.import(file, { + enableDeduplication: false // No merging, inline or background +}) +``` + ### Import Tracking Track and organize imports by project: diff --git a/docs/guides/inspection.md b/docs/guides/inspection.md index 015c5118..240e81ae 100644 --- a/docs/guides/inspection.md +++ b/docs/guides/inspection.md @@ -166,6 +166,34 @@ brainy inspect diff /data/brain-prod /data/brain-staging Sample-based — for a full diff, dump both with `inspect dump` and compare the JSONL. +## Auditing graph-read truth + +`brain.auditGraph()` (8.6.0+) proves — or disproves — that relationship reads +return canonical truth on a given brain, without mutating anything. It walks +every stored relationship record, asks the same read path your application +uses (`related()`, VFS `readdir`) with every visibility tier included, and +classifies every discrepancy: + +```typescript +const report = await brain.auditGraph() + +report.coherent // true = related()/readdir can be trusted on this brain +report.missingFromReadsCount // records the read path omits — stale index +report.danglingEndpointsCount // relationships whose endpoint entity is gone +report.readOnlyCount // read-path edges with NO stored record — ghosts +report.visibilityHiddenCount // internal/system edges hidden by design (not a fault) +``` + +Counts are always exact; the example lists (`missingFromReads`, +`danglingEndpoints`, `readOnlyVerbIds`) are capped at `maxExamples` +(default 100) and `truncatedExamples` says so when they are. + +Run it after any engine upgrade, restore, or migration. If it reports +discrepancies, run `brain.repairIndex()` and audit again — a `coherent` +report after the repair is the verified statement that the heal worked. +Cost: one relationship-record walk plus one indexed read per distinct +source entity — safe on a live brain. + ## Repairing a corrupted store If invariants fail and you suspect index corruption, `inspect repair` diff --git a/docs/guides/installation.md b/docs/guides/installation.md index e1bfb2b6..0a36f632 100644 --- a/docs/guides/installation.md +++ b/docs/guides/installation.md @@ -45,20 +45,20 @@ console.log('Brainy ready.') ## Native Acceleration (Optional) -For production workloads, add Cortex for Rust-accelerated SIMD distance calculations and native embeddings: +For production workloads, add Cor for Rust-accelerated SIMD distance calculations and native embeddings: ```bash -npm install @soulcraft/cortex +npm install @soulcraft/cor ``` ```typescript import { Brainy } from '@soulcraft/brainy' -const brain = new Brainy({ plugins: ['@soulcraft/cortex'] }) +const brain = new Brainy({ plugins: ['@soulcraft/cor'] }) await brain.init() // native providers registered during init ``` -Cortex registers native (Rust/SIMD) vector, metadata, and graph engines behind the same Brainy `find()` API — no code changes, an optional dependency for production-scale workloads. +Cor registers native (Rust/SIMD) vector, metadata, and graph engines behind the same Brainy `find()` API — no code changes, an optional dependency for production-scale workloads. ## Server-only since 8.0 diff --git a/docs/guides/model-loading.md b/docs/guides/model-loading.md index 415fe2e3..e5b7b1d6 100644 --- a/docs/guides/model-loading.md +++ b/docs/guides/model-loading.md @@ -35,15 +35,18 @@ This single WASM file contains everything needed for sentence embeddings. ### Bun (Recommended) -```typescript -// Works with Bun runtime +```bash +# Bun as a runtime — supported and recommended +bun add @soulcraft/brainy bun run server.ts - -// Works with bun --compile (single binary deployment!) -bun build --compile --target=bun server.ts -./server // Self-contained binary with embedded model ``` +Brainy is pure WebAssembly with no native binaries, so the module graph stays +bundler-friendly. Single-binary `bun build --compile` is **not a supported +target** at present: Bun 1.3.10 has a `--compile` codegen regression +(`__promiseAll is not defined`) triggered by top-level `await` in the bundled +graph. Run Brainy under the Bun runtime (above) instead. + ### Node.js ```typescript @@ -164,7 +167,7 @@ await brain.init() **What's new:** - Faster initialization -- Works with `bun --compile` +- Bundler-friendly (pure WASM, no native binaries) - No network requirements ### From Custom Embedding Functions @@ -216,9 +219,8 @@ export { brain } ### Deployment ```bash -# Option 1: Bun compile (single binary) -bun build --compile server.ts -./server # Contains everything +# Option 1: Bun runtime +bun run server.ts # Option 2: Docker docker build -t my-app . diff --git a/docs/guides/optimistic-concurrency.md b/docs/guides/optimistic-concurrency.md index a21a8af0..268bc5fa 100644 --- a/docs/guides/optimistic-concurrency.md +++ b/docs/guides/optimistic-concurrency.md @@ -180,3 +180,35 @@ Brainy 8.0 has exactly two write-coordination counters, at two granularities: They compose: a `transact()` batch can carry per-entity `ifRev` checks *and* a whole-store `ifAtGeneration`; any failed check rejects the entire batch before anything is staged. Generations also power snapshots and time travel (`brain.now()`, `brain.asOf()`, `db.persist()`) — see the [consistency model](../concepts/consistency-model.md) and [Snapshots & Time Travel](./snapshots-and-time-travel.md). A snapshot or historical view captures each entity *including* its `_rev` at that moment, so reading the past and writing back with `ifRev` against the live state works exactly as you'd hope: the write fails if the entity moved since the state you copied from. + +## The transact envelope: batch size, budget, and bulk imports + +`transact()` applies its batch atomically under one commit — which means the whole batch +shares one **apply budget**. Since 8.7.0 the budget scales with the batch: +`max(30 s, opCount × 2 s)`, or exactly what you pass as `timeoutMs`. A tripped budget rolls +the entire batch back (nothing partial survives) and throws a retryable +`TransactionTimeoutError` that names the operation it stopped at, the batch size, and the +elapsed vs budgeted time — a diagnosis, not just a failure: + +``` +Transaction timed out at operation 41/120 ('add') — 246012ms elapsed, budget 240000ms. +The batch rolled back atomically; retry with a higher timeoutMs or a smaller batch. +``` + +Practical envelope guidance for bulk work: + +1. **Precompute embeddings outside the commit path.** Embedding inside `transact()` spends + the budget on model inference. Use `brain.embedBatch(texts)` and pass each vector via + the op's `vector` field — the commit then pays only storage costs, and a retried batch + never re-pays inference. (The win is *where* the inference happens, not raw embedding + throughput: on the default WASM engine, batch and sequential embedding measure + comparably, ~160 ms/text; native embedding providers may batch faster.) +2. **Chunk very large imports** into batches of a few hundred ops with one `transact()` + each. You lose whole-import atomicity but keep per-chunk atomicity, bounded memory, and + resumability — pair with `ifAbsent` upserts so a retried chunk is idempotent. +3. **Slow disks change the math, not the contract.** On network-attached storage a single + op can cost ~2 s (canonical write + fsync + index maintenance). The scaled default + absorbs that; pass an explicit `timeoutMs` only when you know better than the scale. +4. **`addMany`/`relateMany` are the convenience tier** — they chunk and batch-embed for + you, with per-item error reporting instead of batch atomicity. Choose by what you need: + atomic-all-or-nothing → `transact()`; resilient bulk load → `addMany`. diff --git a/docs/guides/quick-start.md b/docs/guides/quick-start.md index 2a689bd3..097c55fe 100644 --- a/docs/guides/quick-start.md +++ b/docs/guides/quick-start.md @@ -60,17 +60,17 @@ const nextId: string = await brain.add({ await brain.relate({ from: nextId, to: reactId, - type: VerbType.BuiltOn + type: VerbType.DependsOn }) ``` ## 5. Query with Triple Intelligence ```typescript -import type { FindResult } from '@soulcraft/brainy' +import type { Result } from '@soulcraft/brainy' // All three search paradigms in one call -const results: FindResult[] = await brain.find({ +const results: Result[] = await brain.find({ query: 'modern frontend frameworks', // Vector similarity search where: { year: { greaterThan: 2015 } }, // Metadata filtering connected: { to: reactId, depth: 2 } // Graph traversal diff --git a/docs/guides/reacting-to-changes.md b/docs/guides/reacting-to-changes.md new file mode 100644 index 00000000..7367a5e3 --- /dev/null +++ b/docs/guides/reacting-to-changes.md @@ -0,0 +1,117 @@ +--- +title: Reacting to Changes +slug: guides/reacting-to-changes +public: true +category: guides +template: guide +order: 11 +description: Subscribe to every committed mutation with `brain.onChange` — the in-process change feed behind live UIs, cache invalidation, and realtime sync. Covers the event shape, delivery guarantees, and catch-up patterns. +next: + - guides/optimistic-concurrency + - guides/snapshots-and-time-travel +--- + +# Reacting to Changes + +`brain.onChange(cb)` is Brainy's in-process change feed: subscribe once and +receive one event per committed mutation — **every** mutation, regardless of +how it happened. Direct calls, batch methods, `transact()`, imports, and +Virtual Filesystem writes all funnel through the same commit point the feed is +emitted from, so nothing slips past it. + +```ts +const off = brain.onChange((e) => { + if (e.kind === 'entity') { + console.log(`${e.op} ${e.entity?.type} ${e.id} @ generation ${e.generation}`) + } +}) + +await brain.add({ data: 'Ada Lovelace', type: 'person' }) +// → "add person 0198... @ generation 42" + +off() // unsubscribe when done +``` + +## The event + +```ts +interface BrainyChangeEvent { + kind: 'entity' | 'relation' | 'store' + op: 'add' | 'update' | 'remove' | 'relate' | 'unrelate' | 'updateRelation' + | 'clear' | 'restore' + id?: string + entity?: { id: string; type: string; subtype?: string; + metadata: Record; service?: string } + relation?: { id: string; from: string; to: string; type: string; + metadata?: Record } + generation?: number + timestamp: number +} +``` + +- **Entity events** (`add` / `update` / `remove`) carry the post-commit indexed + view — `type`, `subtype`, and the full custom `metadata`, so you can match + your own `where`-style filters against events without a read. +- **Deletes are fully described.** A `remove` or `unrelate` event carries the + record's *last committed state* (sourced from the commit's own history + record), not just an id. +- **Batches emit per item.** `addMany` / `updateMany` / `relateMany` / + `removeMany` emit one event per affected record; a `transact()` batch emits + one event per item, all sharing the batch's single `generation`. +- **Cascades are visible.** Removing an entity also emits `unrelate` for each + relationship the delete cascaded to. +- **Store-level events** (`kind: 'store'`) fire for the two wholesale + operations — `clear()` and `restore()` — and mean *"everything may have + changed; refetch what you care about."* + +## Delivery guarantees + +- **Post-commit only.** An aborted write — a losing + [`ifRev` compare-and-swap](optimistic-concurrency.md), a rejected + transaction — never emits. If you received the event, the write is durable. +- **Commit-ordered.** Events arrive in the order writes committed; + `generation` is monotonic. +- **Asynchronous, never blocking.** Delivery happens in a microtask after the + write completes. A slow listener cannot delay a write; a throwing listener + is logged and isolated from other listeners. +- **Zero overhead when unused.** With no subscribers, the write path does no + event work at all. +- **Fire-and-forget.** There is no replay or backpressure. For catch-up after + a disconnect, use the `generation` on each event together with + [`asOf()` / the transaction log](snapshots-and-time-travel.md): record the + last generation you processed, and on reconnect diff from there. For file + content specifically, `vfs.readFile(path, { asOf })` and + `vfs.history(path)` are the temporal read — see + [Snapshots & Time Travel](snapshots-and-time-travel.md). + +## Patterns + +**Cache invalidation** — drop cached reads for whatever changed: + +```ts +brain.onChange((e) => { + if (e.kind === 'store') return cache.clear() + if (e.id) cache.delete(e.id) +}) +``` + +**Live queries (notify-and-refetch)** — re-run a query when a relevant change +lands, rather than diffing incrementally: + +```ts +brain.onChange((e) => { + if (e.kind === 'entity' && e.entity?.type === 'order') { + refreshOpenOrdersView() // debounce as needed + } +}) +``` + +**Forwarding to other processes** — the feed is in-process by design. To push +changes to browsers or other services, forward events through your own +transport (WebSocket, SSE) from the process that owns the brain. + +## Lifecycle + +`onChange` returns an unsubscribe function — call it when tearing down a +subscriber (for example, when evicting a pooled instance). `brain.close()` +drops all listeners; no events are delivered for or after `close()`. diff --git a/docs/guides/snapshots-and-time-travel.md b/docs/guides/snapshots-and-time-travel.md index 41e2284a..490aecab 100644 --- a/docs/guides/snapshots-and-time-travel.md +++ b/docs/guides/snapshots-and-time-travel.md @@ -9,6 +9,7 @@ description: Recipes for the Db API — instant backups with persist(), restore, next: - concepts/consistency-model - guides/optimistic-concurrency + - guides/external-backups --- # Snapshots & Time Travel @@ -41,6 +42,10 @@ bytes. Cross-device targets fall back to per-file byte copies, and persisting an in-memory brain serializes it to the same directory layout — a real, durable store. +> Archiving a brain directory with **external tools** (`tar`, `rsync`, `cp`)? +> Some index files are sparse and can explode to their apparent size under a +> naive copy — see [External Backups & Sparse Storage](/docs/guides/external-backups). + Two things to know: - `persist()` requires the view to still be the store's **latest** @@ -135,10 +140,12 @@ await after.release() Three things to remember: -- History granularity is `transact()` commits — single-operation writes - advance the clock but do not produce historical records (see the - [consistency model](../concepts/consistency-model.md)). Use `transact()` - for writes you want to travel back through. +- History granularity is per-write: EVERY write — `transact()` AND a + single-operation `add`/`update`/`remove`/`relate` — is its own immutable + generation, so a pin always freezes against later writes and every write is + individually addressable via `asOf()` (see the + [consistency model](../concepts/consistency-model.md)). Use `transact()` when + you want several operations to share ONE atomic generation. - The first index-accelerated query (semantic search, traversal, cursors, aggregation) at a historical generation builds an in-memory index materialization — O(n at that generation), once per `Db`, freed on @@ -336,20 +343,95 @@ For per-entity write coordination (rather than whole-store history), the ## Keeping history bounded -Historical records cost disk space. Reclaim what no live pin protects: +Under Model-B every write is a generation, so history can grow quickly — +Brainy auto-compacts at `close()` (time-bounded per pass) under the +**`retention`** knob (configured on the constructor). Since 8.9.0, `flush()` +never compacts: flushing is durability work and costs only what the current +window's writes cost, regardless of history backlog. A long-lived writer that +never closes keeps its history until its next explicit `compactHistory()` — +schedule one in your maintenance window if you run bounded retention: ```typescript -await brain.compactHistory({ - retainGenerations: 100, // keep the 100 most recent commits - retainMs: 7 * 24 * 60 * 60 * 1000 // and everything from the last 7 days -}) +// Zero-config: ADAPTIVE — keep as much history as free disk/RAM allows, +// reclaiming oldest-first under pressure. (This is the default.) +new Brainy({ /* retention unset */ }) + +// Unbounded — never reclaim history (opt in explicitly): +new Brainy({ retention: 'all' }) + +// Explicit CAPS — reclaim oldest-unpinned generations while ANY cap is exceeded: +new Brainy({ retention: { maxGenerations: 1000, maxAge: 7 * 86_400_000, maxBytes: 512 * 1024 ** 2 } }) +``` + +Reclaim manually at any time (the same caps, plus an optional per-pass time +budget for maintenance windows — an early stop is a consistent prefix and the +next pass resumes): + +```typescript +await brain.compactHistory({ maxGenerations: 100, maxAge: 7 * 24 * 60 * 60 * 1000 }) +await brain.compactHistory({ maxBytes: 512 * 1024 ** 2, timeBudgetMs: 10_000 }) ``` Compaction never breaks a pinned read — record-sets are reclaimed only when -no live `Db` could need them. Release views you are done with (including the -ones `transact()` returns), and `persist()` any generation you want to keep -beyond the retention window: snapshots are self-contained and unaffected by -compaction. +no live `Db` could need them (live pins are ALWAYS exempt). Release views you +are done with (including the ones `transact()` returns), and `persist()` any +generation you want to keep beyond the retention window: snapshots are +self-contained and unaffected by compaction. + +## Time travel for files (the VFS) + +Since 8.2.0, time travel covers Virtual Filesystem **content**, not just +entity records. File bytes are retention-protected: a content blob referenced +by any generation inside the retention window is never reclaimed, so reading +the past always returns the exact bytes — never a stale field or a +dangling hash. + +**`vfs.readFile(path, { asOf })`** takes a generation number or a `Date` and +returns the file's exact bytes as they stood then. It resolves the path's +current entity, then materializes its state at the target generation — so it +answers *"what did the file at this path hold at that point?"* It bypasses +the content cache; the `encoding` option still applies. Asking about a +generation before the file existed throws the usual not-found error, and +asking past the retention window's compaction horizon throws a +compacted-generation error. + +**`vfs.history(path)`** returns the file's versions inside the retention +window, oldest first — one `FileVersion` per generation that wrote the file, +the newest entry being the current state: + +```typescript +// A CMS page evolves… +await brain.vfs.writeFile('/pages/home.json', '{"title":"Launch"}') +await brain.vfs.writeFile('/pages/home.json', '{"title":"Launch v2"}') +await brain.vfs.writeFile('/pages/home.json', '{"title":""}') // bad deploy! + +// Every version is listed and readable: +const versions = await brain.vfs.history('/pages/home.json') +// → [{ generation, timestamp, hash, size, mimeType? }, …] ascending + +const good = versions[versions.length - 2] +const bytes = await brain.vfs.readFile('/pages/home.json', { + asOf: good.generation +}) + +// Restore = write the old bytes back. This is a NEW write (a new +// generation) — history is never rewritten, so the bad version stays +// visible in the audit trail. +await brain.vfs.writeFile('/pages/home.json', bytes) +``` + +Two lifecycle consequences worth stating plainly: + +- **Deleting or overwriting a file no longer frees its bytes immediately.** + Old content lives until history compaction reclaims the generations that + reference it — the same `retention` budget that bounds all Model-B history + (and pinned views are exempt, exactly as above). Size your `retention` for + the file-version depth you want; `retention: 'all'` keeps every version of + every file forever. +- **After `compactHistory()` reclaims a generation, its file versions are + gone** and their bytes are physically reclaimed. (This also fixed a + pre-8.2.0 defect where overwritten content was never reclaimed at all — an + unbounded silent leak.) ## From branches to values diff --git a/docs/guides/subtypes-and-facets.md b/docs/guides/subtypes-and-facets.md index 0b38b146..ff5de320 100644 --- a/docs/guides/subtypes-and-facets.md +++ b/docs/guides/subtypes-and-facets.md @@ -332,7 +332,7 @@ await brain.updateRelation({ id: relationId, weight: 0.5, confidence: 0.9 }) `find({ connected, subtype })` filters traversal edges by their subtype. Composes with `via` (verb-type filter): ```typescript -// All direct reports two hops deep (will be supported in Cortex; for now depth-1) +// All direct reports two hops deep (will be supported in Cor; for now depth-1) const directChain = await brain.find({ connected: { from: ceoId, @@ -343,7 +343,7 @@ const directChain = await brain.find({ }) ``` -Multi-hop subtype filtering (`depth > 1`) lights up on the Cortex native path; the JS path throws today rather than return incorrect partial results. +Multi-hop subtype filtering (`depth > 1`) lights up on the Cor native path; the JS path throws today rather than return incorrect partial results. ### Counts diff --git a/docs/guides/upgrading-7-to-8.md b/docs/guides/upgrading-7-to-8.md new file mode 100644 index 00000000..a3c64fb9 --- /dev/null +++ b/docs/guides/upgrading-7-to-8.md @@ -0,0 +1,186 @@ +--- +title: Upgrading from 7.x to 8.0 +slug: guides/upgrading-7-to-8 +public: true +category: guides +template: guide +order: 10 +description: What the one-time 7→8 on-disk migration does, how 8.0 automatically recovers Virtual Filesystem content that older layouts stored in the removed copy-on-write area, and how to verify (or force) that recovery. +next: + - guides/storage-adapters + - guides/inspection +--- + +# Upgrading from 7.x to 8.0 + +Opening a 7.x on-disk store with Brainy 8.0 runs a **one-time, in-place layout +migration** the first time the store is opened. It is automatic (`autoMigrate` +defaults to `true`), it runs once, and it stamps a marker so every later open is +a no-op. + +Almost everything about the upgrade is transparent: your entities, relationships, +metadata, and indexes migrate and rebuild without any action on your part. This +guide covers the **one case that needs attention** — Virtual Filesystem (VFS) +content — and how 8.0 recovers it for you. + +## TL;DR + +- **Just upgrade to `@soulcraft/brainy@8.0.12` (or later) and open the store.** + If a previous upgrade left VFS content stranded, 8.0.12 **heals it on open**, + with no operator action. +- Want to force or script it? Call **`await brain.vfs.adoptOrphanedBlobs()`**. +- The recovery is **non-destructive and idempotent** — it copies, never moves, + and running it twice is a no-op. + +## What the migration does + +The 7.x on-disk layout stored entities under a per-branch path +(`branches//entities/...`). 8.0 uses a flat layout (`entities/...`). On +first open, Brainy: + +1. Collapses `branches//entities/*` into the flat `entities/*` layout. +2. Rebuilds the derived indexes (vector, graph, metadata) and count rollups from + the canonical entities. +3. Stamps `_system/migration-layout.json` so re-opening is a no-op. +4. Takes an automatic **pre-upgrade backup** of the directory before it starts, + and removes it once the upgrade is verified complete (see + [The safety net](#the-safety-net-pre-upgrade-backup) below). + +The log line to expect (once per store): + +``` +[brainy] Migrating a 7.x branch layout (branches/main) to the 8.0 flat layout + in place — 13175 entity files. This runs once; back up the directory first if + you need a rollback (8.0 does not keep the old layout). +``` + +## The one thing that needs recovery: VFS content + +If your application uses the Virtual Filesystem (VFS) to store file content (for +example, a CMS that keeps page documents at paths like +`/pages/homepage/page.json`), that content is held as **content blobs**. + +7.x kept those blobs in the branch system's **copy-on-write area** (`_cow/`). +8.0 removed the branch/copy-on-write system, and its content blobs live in the +content-addressed store (`_cas/`). The layout migration moves entities — it does +**not** move the VFS content blobs. Left alone, a 7.x store's VFS blobs would +remain in `_cow/`, and a read of one would throw: + +``` +VFS: Cannot read blob for /pages/homepage/page.json: + Blob metadata not found: 6b87cfb71ad2f04602a5c157214dc42000... +``` + +### 8.0.12 recovers them automatically + +Brainy 8.0.12 adds an **on-open recovery pass** that runs right after the layout +migration. It scans `_cow/`, and for every content blob whose `blob:` + +`blob-meta:` pair is not already in `_cas/`, it copies **both** across into the +8.0 store. It: + +- **Heals a fresh 7→8 upgrade** (where `_cow/` still holds the blobs), **and** +- **Heals a store already upgraded by an earlier 8.0.x** that stranded them — + the recovery is gated on the presence of `_cow/` and its own marker, not on + the layout-migration marker, so an already-migrated store is still healed. +- Is a **cheap no-op** on a native-8.0 or fresh store (there is no `_cow/`), and + on a store already healed (a marker records completion so later opens skip the + scan). + +So the operator action for a stranded store is simply: **upgrade to 8.0.12 and +open it.** + +```ts +import { Brainy } from '@soulcraft/brainy' + +// Opening the store is all that is required — recovery runs during init(). +const brain = new Brainy({ storage: { type: 'filesystem', path: '/data/my-store' } }) +await brain.init() + +// The previously-failing read now succeeds. +const page = await brain.vfs.readFile('/pages/homepage/page.json') +``` + +On a store that needed recovery you will see: + +``` +[brainy] Recovered 337 VFS content blob(s) stranded by a 7→8 upgrade + (adopted _cow/ → _cas/ in place). +``` + +### Forcing recovery explicitly + +If you would rather run the recovery deliberately (for example, in an upgrade +script that asserts a clean result before flipping traffic), call it directly: + +```ts +const result = await brain.vfs.adoptOrphanedBlobs() +// → { cowBlobs, adopted, alreadyPresent, incomplete } +console.log(`adopted ${result.adopted}, already present ${result.alreadyPresent}`) +if (result.incomplete > 0) { + // One or more _cow/ blobs are missing their bytes or metadata — investigate + // _cow/ before discarding your own backup. See "The safety net" below. +} +``` + +`adoptOrphanedBlobs()` self-initializes the brain, so it is safe to call +immediately after construction. It returns: + +| Field | Meaning | +| ---------------- | ---------------------------------------------------------------- | +| `cowBlobs` | Distinct content-blob hashes found in `_cow/`. | +| `adopted` | Newly copied into `_cas/` on this call. | +| `alreadyPresent` | Already in `_cas/` (a prior open or run adopted them). | +| `incomplete` | A `_cow/` blob missing its bytes *or* its metadata — **skipped** rather than half-adopted. | + +## Verifying recovery + +After opening under 8.0.12, confirm a previously-failing path reads: + +```ts +const content = await brain.vfs.readFile('/pages/homepage/page.json') +console.log(content.toString().slice(0, 80)) +``` + +If you scripted it with `adoptOrphanedBlobs()`, a clean result is +`incomplete === 0` and `adopted + alreadyPresent === cowBlobs`. + +## The safety net: pre-upgrade backup + +Brainy takes an automatic pre-upgrade backup and removes it once the upgrade is +**verified complete**. In 8.0.12 that verification includes VFS content: if the +blob recovery reports `incomplete > 0`, the completion marker is **not** stamped +(the next open retries) and the **pre-upgrade backup is retained** so you still +have a rollback while blobs remain unaccounted for. You will see: + +``` +[brainy] VFS blob recovery adopted N blob(s) but M orphaned _cow/ blob(s) are + missing their bytes or metadata and were left in place. The pre-upgrade backup + is being retained; inspect _cow/ before discarding it. +``` + +The recovery never deletes anything from `_cow/`, so the original blobs stay in +place for inspection or a manual rollback regardless. + +## Who is affected + +**Any 7.x store that used the VFS to store file content** (so it has a `_cow/` +area) is a candidate for stranded blobs on a 7→8 upgrade. Stores that never used +the VFS have no `_cow/` content blobs and are unaffected. + +Because the recovery is **gated on the presence of `_cow/`**, it is +self-selecting and safe to roll out everywhere: + +- On a store with stranded blobs → it adopts them. +- On a native-8.0, fresh, or non-VFS store → it is a no-op on the existence + check. + +There is no configuration to set and nothing to opt into. Upgrading to 8.0.12 +and opening each store is sufficient. + +## Rollback + +The recovery is copy-only, so no rollback of the recovery itself is ever needed. +If you need to roll back the **whole** 7→8 upgrade, restore the directory from +your pre-upgrade backup (retained automatically while recovery is incomplete, or +your own snapshot) and pin `@soulcraft/brainy@7.x`. 8.0 does not keep the old +branch layout in place, so a directory-level restore is the rollback path. diff --git a/docs/performance-envelopes.md b/docs/performance-envelopes.md new file mode 100644 index 00000000..d29677e3 --- /dev/null +++ b/docs/performance-envelopes.md @@ -0,0 +1,83 @@ +--- +title: Performance Envelopes +slug: guides/performance-envelopes +public: true +category: guides +template: guide +order: 40 +description: Measured per-operation latency envelopes at stated scales — what to expect, on what hardware, and exactly how each number was produced. +next: + - guides/find-limits +--- + +# Performance Envelopes + +Every number on this page is **measured, never projected** — produced by the script +cited at the bottom, against the built package (the artifact you install), on the stated +hardware. Each entry says what was measured, at what scale, on which storage backend. +When a release touches a measured path, that operation is re-measured and this page +updates in the same release. + +Two scopes to keep straight: + +- **These envelopes are the pure-JS engine** (no native accelerator registered) on + filesystem storage. This is the floor every deployment gets from `npm install` alone. +- **Accelerated deployments** (the optional native provider) publish their own numbers — + this page never claims them. + +## Read operations + +Reads are where the architecture pays off: after the write path has done its indexing +work, queries answer from purpose-built indexes without scanning. + +| Operation | 1,000 entities | 10,000 entities | Notes | +|---|---|---|---| +| `get(id)` (warm) | p50 < 0.1ms | p50 < 0.1ms | served from cache/metadata index | +| `find` (metadata: indexed equality + range, limit 100) | p50 1.0ms · p95 1.8ms | p50 7.0ms · p95 8.9ms | column-store bitmap paths | +| `related(id)` (per-node adjacency) | p50 < 0.1ms · p95 0.2ms | p50 < 0.1ms | LSM adjacency index — O(degree), scale-independent | +| `find` (semantic: embed + HNSW, 1k docs) | p50 178ms · p95 393ms | — | dominated by WASM query embedding (measured on a machine under concurrent load — treat the p95 as an upper bound); the vector search itself is single-digit ms | + +## Write operations + +Under Model-B **every write is its own durable generation** — a single-op `add` pays +serialization, before-image staging, and fsync before it acks. That durability is priced +into the write path visibly, by design: + +| Operation | 1,000 entities | 10,000 entities | Notes | +|---|---|---|---| +| `add` (single-op) | p50 167ms · p95 171ms | p50 165ms · p95 172ms | full durable generation per write — flat across scale | +| `addMany` (bulk) | ~163ms/entity | ~187ms/entity | **currently per-item commits** — see the honest note below | +| `relateMany` | ~0.8ms/edge | ~0.9ms/edge | edges batch efficiently today | +| `flush` (steady-state, 1 pending write) | p50 8ms · p95 10ms | p50 45ms · p95 52ms | durability-only since 8.9.0 — cost no longer depends on history backlog or retention mode | + +**The honest note on bulk writes:** `addMany` today commits each item as its own +generation (the same durability as single-op `add`, serialized by the single-writer +lock), so bulk-load cost is N × single-op cost. Batched chunk commits (one generation +and one fsync window per chunk, as `removeMany` already does) are designed into the +unified-commit work on the current roadmap. Until that ships, size bulk imports +accordingly — 10k entities is minutes, not seconds, on filesystem storage. + +## Open / close + +| Operation | 1,000 entities | 10,000 entities | Notes | +|---|---|---|---| +| `open` (empty store) | ~560ms | ~190ms | includes embedder initialization | +| `open` (warm, populated, clean shutdown) | 763ms | 4.9s | pure-JS vector index load dominates and grows with entity count; the native accelerator exists precisely to remove this | +| `close` | bounded | bounded | auto-compaction pass is time-bounded (~5s max) since 8.9.0 | + +A store that was NOT cleanly closed pays index rebuilds on top of the warm-open +number (tens of seconds at 10k) — clean shutdown is worth engineering for. + +## How these were produced + +- **Hardware**: Intel Core i9-14900HX (32 threads), 62GB RAM, NVMe, Linux, Node v22. +- **Backend**: `storage: { type: 'filesystem' }`, pure JS (no native providers). +- **Embeddings**: deterministic stub for non-semantic ops (isolates engine cost); + the real WASM embedder for the semantic row (that's what you'll run). +- **Method**: p50/p95 over 50–200 samples per op against the built `dist/`; + the measuring script ships in the repo history and re-runs per release. + +Numbers on different hardware will differ; the *shape* (sub-2ms indexed reads, +~160ms embedding-bound semantic queries, durability-priced writes) is the envelope +you should hold your deployment against. If your measurements diverge from these +shapes by an order of magnitude, something is wrong — file it. diff --git a/docs/vfs/PROJECTION_STRATEGY_API.md b/docs/vfs/PROJECTION_STRATEGY_API.md index ab66e7c7..380862e1 100644 --- a/docs/vfs/PROJECTION_STRATEGY_API.md +++ b/docs/vfs/PROJECTION_STRATEGY_API.md @@ -289,7 +289,7 @@ export class SizeProjection extends BaseProjectionStrategy { where: { vfsType: 'file', size: { - greaterEqual: min, + gte: min, lessThan: max } }, @@ -305,7 +305,7 @@ export class SizeProjection extends BaseProjectionStrategy { where: { vfsType: 'file', size: { - greaterEqual: min, + gte: min, lessThan: max } }, @@ -359,7 +359,7 @@ export class StatusProjection extends BaseProjectionStrategy { const results = await brain.find({ where: { vfsType: 'file', - modified: { greaterEqual: oneDayAgo }, + modified: { gte: oneDayAgo }, reviewStatus: { missing: true } // No review status set }, limit: 1000 @@ -387,7 +387,7 @@ export class StatusProjection extends BaseProjectionStrategy { vfsType: 'file', anyOf: [ { reviewStatus: { exists: true } }, - { modified: { greaterEqual: Date.now() - 86400000 } } + { modified: { gte: Date.now() - 86400000 } } ] }, limit @@ -415,7 +415,7 @@ Projection strategies use **Brainy Field Operators** (BFO), not MongoDB-style op { size: { $gte: 1000, $lte: 5000 } } // ✅ BFO style (CORRECT) -{ size: { greaterEqual: 1000, lessEqual: 5000 } } +{ size: { gte: 1000, lte: 5000 } } ``` ### Logical Operators @@ -451,9 +451,9 @@ Projection strategies use **Brainy Field Operators** (BFO), not MongoDB-style op // Comparison { field: value } // Exact match { field: { greaterThan: 10 } } // > -{ field: { greaterEqual: 10 } } // >= +{ field: { gte: 10 } } // >= { field: { lessThan: 10 } } // < -{ field: { lessEqual: 10 } } // <= +{ field: { lte: 10 } } // <= { field: { not: value } } // != // Logical @@ -485,7 +485,7 @@ All metadata fields are automatically indexed. Use direct equality or range quer ```typescript // ✅ Fast: Direct index lookup (O(log n)) { priority: 'high' } -{ size: { greaterEqual: 1000 } } +{ size: { gte: 1000 } } // ⚠️ Slower: Must scan results { path: { matches: /complex-regex/ } } @@ -668,7 +668,7 @@ async resolve(brain, vfs, period: string) { const results = await brain.find({ where: { - modified: { greaterEqual: since } + modified: { gte: since } } }) return this.extractIds(results) diff --git a/docs/vfs/SEMANTIC_VFS.md b/docs/vfs/SEMANTIC_VFS.md index 40807d3c..9298c822 100644 --- a/docs/vfs/SEMANTIC_VFS.md +++ b/docs/vfs/SEMANTIC_VFS.md @@ -127,7 +127,7 @@ await vfs.readFile('/as-of/2024-03-15/auth.ts') // the path only resolves if auth.ts was modified that day ``` -**How it works:** Tracks the `modified` timestamp on every file and runs a range query (`greaterEqual`/`lessEqual`) over one 24-hour window for O(log n) performance. The VFS does not store historical file contents — `/as-of/` filters by *when a file last changed*; reads return the current bytes. For point-in-time state, use the Db API (`brain.asOf(generation)`). +**How it works:** Tracks the `modified` timestamp on every file and runs a range query (`gte`/`lte`) over one 24-hour window for O(log n) performance. The VFS does not store historical file contents — `/as-of/` filters by *when a file last changed*; reads return the current bytes. For point-in-time state, use the Db API (`brain.asOf(generation)`). **Status:** ✅ Fully implemented and tested at 10K file scale diff --git a/examples/directory-import-with-caching.ts b/examples/directory-import-with-caching.ts deleted file mode 100644 index d3395b5a..00000000 --- a/examples/directory-import-with-caching.ts +++ /dev/null @@ -1,237 +0,0 @@ -/** - * Directory Import with Entity Extraction Caching Example - * - * Demonstrates: - * - Importing directories with progress tracking - * - Entity extraction caching for performance - * - Relationship detection with confidence scores - * - Cache statistics monitoring - */ - -import { Brainy, NounType, VerbType } from '../src/brainy.js' -import { DirectoryImporter } from '../src/vfs/importers/DirectoryImporter.js' -import { ProgressTracker, formatProgress } from '../src/types/progress.types.js' -import { detectRelationshipsWithConfidence } from '../src/neural/relationshipConfidence.js' -import { NeuralEntityExtractor } from '../src/neural/entityExtractor.js' - -async function main() { - console.log('🧠 Brainy 3.21.0 - Directory Import with Caching Example\n') - - // Initialize Brainy - const brain = new Brainy({ verbose: false }) - await brain.init() - - console.log('✅ Brainy initialized\n') - - // The entity extractor (and its extraction cache) is constructed directly. - const extractor = new NeuralEntityExtractor(brain) - - // Example 1: Import directory with entity extraction caching - console.log('📁 Example 1: Import Directory with Caching\n') - - const vfs = brain.vfs - const importer = new DirectoryImporter(vfs, brain) - - // Progress tracking - const tracker = ProgressTracker.create(100) - tracker.start() - - try { - // Import with progress (using async generator) - console.log('Importing directory...') - - let filesProcessed = 0 - for await (const progress of importer.importStream('./examples', { - batchSize: 10, - recursive: true, - generateEmbeddings: true, - extractMetadata: true - })) { - if (progress.type === 'progress') { - filesProcessed = progress.processed - const trackedProgress = tracker.update(progress.processed, progress.current) - console.log(` ${formatProgress(trackedProgress)}`) - } else if (progress.type === 'complete') { - console.log(`\n✅ Import complete! Processed ${progress.processed} files\n`) - } else if (progress.type === 'error') { - console.error(`❌ Error: ${progress.error?.message}`) - } - } - - tracker.complete({ filesProcessed }) - - } catch (error) { - console.error('Import failed:', error) - } - - // Example 2: Entity extraction with caching - console.log('\n📝 Example 2: Entity Extraction with Caching\n') - - const sampleText = ` - John Smith created the user authentication system for the application. - The authentication system uses JWT tokens and bcrypt for password hashing. - Mary Johnson manages the backend team that maintains the system. - The system was built using Node.js and PostgreSQL database. - ` - - console.log('First extraction (cache miss):') - const startTime1 = Date.now() - const entities1 = await extractor.extract(sampleText, { - types: [NounType.Person, NounType.Service, NounType.Technology], - confidence: 0.7, - cache: { - enabled: true, - ttl: 7 * 24 * 60 * 60 * 1000, // 7 days - invalidateOn: 'hash' - } - }) - const time1 = Date.now() - startTime1 - console.log(` Extracted ${entities1.length} entities in ${time1}ms`) - console.log(` Entities: ${entities1.map(e => e.text).join(', ')}\n`) - - console.log('Second extraction (cache hit):') - const startTime2 = Date.now() - const entities2 = await extractor.extract(sampleText, { - types: [NounType.Person, NounType.Service, NounType.Technology], - confidence: 0.7, - cache: { - enabled: true, - invalidateOn: 'hash' - } - }) - const time2 = Date.now() - startTime2 - console.log(` Extracted ${entities2.length} entities in ${time2}ms`) - console.log(` Speedup: ${Math.round(time1 / time2)}x faster!\n`) - - // Show cache statistics - const cacheStats = extractor.getCacheStats() - console.log('📊 Cache Statistics:') - console.log(` Hits: ${cacheStats.hits}`) - console.log(` Misses: ${cacheStats.misses}`) - console.log(` Hit Rate: ${(cacheStats.hitRate * 100).toFixed(1)}%`) - console.log(` Total Entries: ${cacheStats.totalEntries}`) - console.log(` Avg Entities per Entry: ${cacheStats.averageEntitiesPerEntry}\n`) - - // Example 3: Relationship detection with confidence - console.log('🔗 Example 3: Relationship Detection with Confidence\n') - - const relationships = detectRelationshipsWithConfidence( - entities1, - sampleText, - { - minConfidence: 0.6, - maxDistance: 100, - useProximityBoost: true, - usePatternMatching: true, - useStructuralAnalysis: true - } - ) - - console.log(`Detected ${relationships.length} relationships:\n`) - for (const rel of relationships.slice(0, 5)) { // Show top 5 - console.log(` ${rel.sourceEntity.text} --[${rel.verbType}]--> ${rel.targetEntity.text}`) - console.log(` Confidence: ${(rel.confidence * 100).toFixed(1)}%`) - console.log(` Evidence: ${rel.evidence.reasoning}`) - console.log(` Method: ${rel.evidence.method}`) - console.log(` Source: "${rel.evidence.sourceText?.substring(0, 60)}..."\n`) - } - - // Example 4: Create relationships in graph with confidence - console.log('📊 Example 4: Creating Relationships in Graph\n') - - const createdRelations = [] - for (const rel of relationships.slice(0, 3)) { // Create top 3 - try { - // Add entities to brain - const sourceId = await brain.add({ - data: rel.sourceEntity.text, - type: rel.sourceEntity.type, - metadata: { - confidence: rel.sourceEntity.confidence, - extractedFrom: 'sample text' - } - }) - - const targetId = await brain.add({ - data: rel.targetEntity.text, - type: rel.targetEntity.type, - metadata: { - confidence: rel.targetEntity.confidence, - extractedFrom: 'sample text' - } - }) - - // Create relationship with confidence - const relationId = await brain.relate({ - from: sourceId, - to: targetId, - type: rel.verbType, - confidence: rel.confidence, - evidence: rel.evidence, - metadata: { - autoDetected: true, - detectedAt: new Date().toISOString() - } - }) - - createdRelations.push(relationId) - console.log(` ✅ Created: ${rel.sourceEntity.text} → ${rel.targetEntity.text}`) - } catch (error) { - console.error(` ❌ Failed to create relationship:`, error) - } - } - - console.log(`\n✅ Created ${createdRelations.length} relationships in knowledge graph`) - - // Example 5: Query relationships by confidence - console.log('\n🔍 Example 5: Query High-Confidence Relationships\n') - - const allRelations = await brain.getRelations({ - limit: 100 - }) - - const highConfidence = allRelations.filter(r => (r.confidence || 0) >= 0.7) - console.log(`Found ${highConfidence.length} high-confidence relationships (≥70%):\n`) - - for (const rel of highConfidence.slice(0, 5)) { - console.log(` ${rel.from} → ${rel.to} (${rel.type})`) - console.log(` Confidence: ${((rel.confidence || 0) * 100).toFixed(1)}%`) - if (rel.evidence) { - console.log(` Method: ${rel.evidence.method}`) - console.log(` Reasoning: ${rel.evidence.reasoning}\n`) - } - } - - // Example 6: Cache management - console.log('🧹 Example 6: Cache Management\n') - - console.log('Cache operations:') - - // Cleanup expired entries - const cleaned = extractor.cleanupCache() - console.log(` Cleaned ${cleaned} expired entries`) - - // Invalidate specific cache entry - const invalidated = extractor.invalidateCache('hash:abc123') - console.log(` Invalidated entry: ${invalidated}`) - - // Get final stats - const finalStats = extractor.getCacheStats() - console.log(` Final cache size: ${finalStats.totalEntries} entries`) - console.log(` Memory used: ~${Math.round(finalStats.cacheSize / 1024)}KB`) - - // Clear all cache (optional) - // extractor.clearCache() - // console.log(' Cleared entire cache') - - console.log('\n✨ Example complete!') - console.log('\n📚 Key Takeaways:') - console.log(' • Entity extraction caching provides 10-100x speedup on repeated content') - console.log(' • Progress tracking gives real-time feedback for long operations') - console.log(' • Relationship confidence helps filter low-quality connections') - console.log(' • Evidence tracking makes relationships explainable and debuggable') - console.log(' • All features are opt-in and backward compatible') -} - -// Run example -main().catch(console.error) diff --git a/package-lock.json b/package-lock.json index fb259f1d..fb9262e9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "8.0.0-rc.1", + "version": "8.9.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "8.0.0-rc.1", + "version": "8.9.0", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", @@ -31,14 +31,10 @@ "brainy": "bin/brainy.js" }, "devDependencies": { - "@rollup/plugin-commonjs": "^28.0.6", - "@rollup/plugin-node-resolve": "^16.0.1", - "@rollup/plugin-replace": "^6.0.2", - "@rollup/plugin-terser": "^0.4.4", "@testcontainers/redis": "^11.5.1", "@types/js-yaml": "^4.0.9", "@types/mime": "^3.0.4", - "@types/node": "^20.11.30", + "@types/node": "^22", "@types/uuid": "^10.0.0", "@types/ws": "^8.18.1", "@typescript-eslint/eslint-plugin": "^8.0.0", @@ -46,7 +42,7 @@ "@vitest/coverage-v8": "^3.2.4", "jspdf": "^3.0.3", "minio": "^8.0.5", - "standard-version": "^9.5.0", + "prettier": "^3.9.4", "testcontainers": "^11.5.1", "tsx": "^4.19.2", "typescript": "^5.4.5", @@ -54,8 +50,8 @@ "vitest": "^3.2.4" }, "engines": { - "bun": ">=1.0.0", - "node": "22.x" + "bun": ">=1.1.0", + "node": ">=22" } }, "node_modules/@ampproject/remapping": { @@ -72,28 +68,6 @@ "node": ">=6.0.0" } }, - "node_modules/@babel/code-frame": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", - "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-validator-identifier": "^7.27.1", - "js-tokens": "^4.0.0", - "picocolors": "^1.1.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/code-frame/node_modules/js-tokens": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dev": true, - "license": "MIT" - }, "node_modules/@babel/helper-string-parser": { "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", @@ -1178,16 +1152,6 @@ "url": "https://github.com/sponsors/nzakas" } }, - "node_modules/@hutson/parse-repository-url": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@hutson/parse-repository-url/-/parse-repository-url-3.0.2.tgz", - "integrity": "sha512-H9XAx3hc0BQHY6l+IFSWHDySypcXsvsuLhgYLUGywmJ5pswRVQJUHpOsobnLYp2ZUaUlKiKDrgWWhosOwAEM8Q==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=6.9.0" - } - }, "node_modules/@inquirer/ansi": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@inquirer/ansi/-/ansi-1.0.2.tgz", @@ -1690,6 +1654,8 @@ "integrity": "sha512-ZMp1V8ZFcPG5dIWnQLr3NSI1MiCU7UETdS/A0G8V/XWHvJv3ZsFqutJn1Y5RPmAPX6F3BiE397OqveU/9NCuIA==", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.25" @@ -2003,126 +1969,6 @@ "dev": true, "license": "BSD-3-Clause" }, - "node_modules/@rollup/plugin-commonjs": { - "version": "28.0.9", - "resolved": "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-28.0.9.tgz", - "integrity": "sha512-PIR4/OHZ79romx0BVVll/PkwWpJ7e5lsqFa3gFfcrFPWwLXLV39JVUzQV9RKjWerE7B845Hqjj9VYlQeieZ2dA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@rollup/pluginutils": "^5.0.1", - "commondir": "^1.0.1", - "estree-walker": "^2.0.2", - "fdir": "^6.2.0", - "is-reference": "1.2.1", - "magic-string": "^0.30.3", - "picomatch": "^4.0.2" - }, - "engines": { - "node": ">=16.0.0 || 14 >= 14.17" - }, - "peerDependencies": { - "rollup": "^2.68.0||^3.0.0||^4.0.0" - }, - "peerDependenciesMeta": { - "rollup": { - "optional": true - } - } - }, - "node_modules/@rollup/plugin-node-resolve": { - "version": "16.0.3", - "resolved": "https://registry.npmjs.org/@rollup/plugin-node-resolve/-/plugin-node-resolve-16.0.3.tgz", - "integrity": "sha512-lUYM3UBGuM93CnMPG1YocWu7X802BrNF3jW2zny5gQyLQgRFJhV1Sq0Zi74+dh/6NBx1DxFC4b4GXg9wUCG5Qg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@rollup/pluginutils": "^5.0.1", - "@types/resolve": "1.20.2", - "deepmerge": "^4.2.2", - "is-module": "^1.0.0", - "resolve": "^1.22.1" - }, - "engines": { - "node": ">=14.0.0" - }, - "peerDependencies": { - "rollup": "^2.78.0||^3.0.0||^4.0.0" - }, - "peerDependenciesMeta": { - "rollup": { - "optional": true - } - } - }, - "node_modules/@rollup/plugin-replace": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/@rollup/plugin-replace/-/plugin-replace-6.0.3.tgz", - "integrity": "sha512-J4RZarRvQAm5IF0/LwUUg+obsm+xZhYnbMXmXROyoSE1ATJe3oXSb9L5MMppdxP2ylNSjv6zFBwKYjcKMucVfA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@rollup/pluginutils": "^5.0.1", - "magic-string": "^0.30.3" - }, - "engines": { - "node": ">=14.0.0" - }, - "peerDependencies": { - "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" - }, - "peerDependenciesMeta": { - "rollup": { - "optional": true - } - } - }, - "node_modules/@rollup/plugin-terser": { - "version": "0.4.4", - "resolved": "https://registry.npmjs.org/@rollup/plugin-terser/-/plugin-terser-0.4.4.tgz", - "integrity": "sha512-XHeJC5Bgvs8LfukDwWZp7yeqin6ns8RTl2B9avbejt6tZqsqvVoWI7ZTQrcNsfKEDWBTnTxM8nMDkO2IFFbd0A==", - "dev": true, - "license": "MIT", - "dependencies": { - "serialize-javascript": "^6.0.1", - "smob": "^1.0.0", - "terser": "^5.17.4" - }, - "engines": { - "node": ">=14.0.0" - }, - "peerDependencies": { - "rollup": "^2.0.0||^3.0.0||^4.0.0" - }, - "peerDependenciesMeta": { - "rollup": { - "optional": true - } - } - }, - "node_modules/@rollup/pluginutils": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.3.0.tgz", - "integrity": "sha512-5EdhGZtnu3V88ces7s53hhfK5KSASnJZv8Lulpc04cWO3REESroJXg73DFsOmgbU2BhwV0E20bu2IDZb3VKW4Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/estree": "^1.0.0", - "estree-walker": "^2.0.2", - "picomatch": "^4.0.2" - }, - "engines": { - "node": ">=14.0.0" - }, - "peerDependencies": { - "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" - }, - "peerDependenciesMeta": { - "rollup": { - "optional": true - } - } - }, "node_modules/@rollup/rollup-android-arm-eabi": { "version": "4.53.5", "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.53.5.tgz", @@ -2511,30 +2357,16 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/node": { - "version": "20.19.27", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.27.tgz", - "integrity": "sha512-N2clP5pJhB2YnZJ3PIHFk5RkygRX5WO/5f0WC08tp0wd+sv0rsJk3MqWn3CbNmT2J505a5336jaQj4ph1AdMug==", + "version": "22.20.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.0.tgz", + "integrity": "sha512-QWlFW2wf3nTjC13/DqRnBpR4ZO36VJH/JVBkA/vcnmbTBNQIlnObqyqZE1tUR7+Ni23Lda8R1BxMfbXRpCUx5g==", "devOptional": true, "license": "MIT", "dependencies": { "undici-types": "~6.21.0" } }, - "node_modules/@types/normalize-package-data": { - "version": "2.4.4", - "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz", - "integrity": "sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/pako": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/@types/pako/-/pako-2.0.4.tgz", @@ -2550,13 +2382,6 @@ "license": "MIT", "optional": true }, - "node_modules/@types/resolve": { - "version": "1.20.2", - "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.20.2.tgz", - "integrity": "sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/ssh2": { "version": "1.15.5", "resolved": "https://registry.npmjs.org/@types/ssh2/-/ssh2-1.15.5.tgz", @@ -3047,6 +2872,7 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -3065,13 +2891,6 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, - "node_modules/add-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/add-stream/-/add-stream-1.0.0.tgz", - "integrity": "sha512-qQLMr+8o0WC4FZGQTcJiKBVC59JylcPSrTtk6usvmIDFUOCKegapy1VHQwRbFMOFyb/inzUVqHs+eMYKDM1YeQ==", - "dev": true, - "license": "MIT" - }, "node_modules/adler-32": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz", @@ -3251,13 +3070,6 @@ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "license": "Python-2.0" }, - "node_modules/array-ify": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/array-ify/-/array-ify-1.0.0.tgz", - "integrity": "sha512-c5AMf34bKdvPhQ7tBGhqkgKNUzMr4WUs+WDtC2ZUGOUncbxKMTvqxYctiseW3+L4bA8ec+GcZ6/A/FW4m8ukng==", - "dev": true, - "license": "MIT" - }, "node_modules/asn1": { "version": "0.2.6", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", @@ -3622,7 +3434,9 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "node_modules/buildcheck": { "version": "0.0.7", @@ -3727,34 +3541,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/camelcase-keys": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-6.2.2.tgz", - "integrity": "sha512-YrwaA0vEKazPBkn0ipTiMpSajYDSe+KjQfrjhcBMxJt/znbvlHd8Pw/Vamaz5EB4Wfhs3SUR3Z9mwRu/P3s3Yg==", - "dev": true, - "license": "MIT", - "dependencies": { - "camelcase": "^5.3.1", - "map-obj": "^4.0.0", - "quick-lru": "^4.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/camelcase-keys/node_modules/camelcase": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/canvg": { "version": "3.0.11", "resolved": "https://registry.npmjs.org/canvg/-/canvg-3.0.11.tgz", @@ -3945,97 +3731,6 @@ "node": ">= 12" } }, - "node_modules/cliui": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", - "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^7.0.0" - } - }, - "node_modules/cliui/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/cliui/node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/cliui/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, - "license": "MIT" - }, - "node_modules/cliui/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/cliui/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/cliui/node_modules/wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, "node_modules/codepage": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz", @@ -4072,24 +3767,6 @@ "node": ">=16" } }, - "node_modules/commondir": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", - "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==", - "dev": true, - "license": "MIT" - }, - "node_modules/compare-func": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/compare-func/-/compare-func-2.0.0.tgz", - "integrity": "sha512-zHig5N+tPWARooBnb0Zx1MFcdfpyJrfTJ3Y5L+IFvUm8rM74hHz66z0gw0x4tijh5CorKkKUCnW82R2vmpeCRA==", - "dev": true, - "license": "MIT", - "dependencies": { - "array-ify": "^1.0.0", - "dot-prop": "^5.1.0" - } - }, "node_modules/compress-commons": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-6.0.2.tgz", @@ -4129,302 +3806,8 @@ "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "dev": true, - "license": "MIT" - }, - "node_modules/concat-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", - "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", - "dev": true, - "engines": [ - "node >= 6.0" - ], "license": "MIT", - "dependencies": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.0.2", - "typedarray": "^0.0.6" - } - }, - "node_modules/conventional-changelog": { - "version": "3.1.25", - "resolved": "https://registry.npmjs.org/conventional-changelog/-/conventional-changelog-3.1.25.tgz", - "integrity": "sha512-ryhi3fd1mKf3fSjbLXOfK2D06YwKNic1nC9mWqybBHdObPd8KJ2vjaXZfYj1U23t+V8T8n0d7gwnc9XbIdFbyQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "conventional-changelog-angular": "^5.0.12", - "conventional-changelog-atom": "^2.0.8", - "conventional-changelog-codemirror": "^2.0.8", - "conventional-changelog-conventionalcommits": "^4.5.0", - "conventional-changelog-core": "^4.2.1", - "conventional-changelog-ember": "^2.0.9", - "conventional-changelog-eslint": "^3.0.9", - "conventional-changelog-express": "^2.0.6", - "conventional-changelog-jquery": "^3.0.11", - "conventional-changelog-jshint": "^2.0.9", - "conventional-changelog-preset-loader": "^2.3.4" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/conventional-changelog-angular": { - "version": "5.0.13", - "resolved": "https://registry.npmjs.org/conventional-changelog-angular/-/conventional-changelog-angular-5.0.13.tgz", - "integrity": "sha512-i/gipMxs7s8L/QeuavPF2hLnJgH6pEZAttySB6aiQLWcX3puWDL3ACVmvBhJGxnAy52Qc15ua26BufY6KpmrVA==", - "dev": true, - "license": "ISC", - "dependencies": { - "compare-func": "^2.0.0", - "q": "^1.5.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/conventional-changelog-atom": { - "version": "2.0.8", - "resolved": "https://registry.npmjs.org/conventional-changelog-atom/-/conventional-changelog-atom-2.0.8.tgz", - "integrity": "sha512-xo6v46icsFTK3bb7dY/8m2qvc8sZemRgdqLb/bjpBsH2UyOS8rKNTgcb5025Hri6IpANPApbXMg15QLb1LJpBw==", - "dev": true, - "license": "ISC", - "dependencies": { - "q": "^1.5.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/conventional-changelog-codemirror": { - "version": "2.0.8", - "resolved": "https://registry.npmjs.org/conventional-changelog-codemirror/-/conventional-changelog-codemirror-2.0.8.tgz", - "integrity": "sha512-z5DAsn3uj1Vfp7po3gpt2Boc+Bdwmw2++ZHa5Ak9k0UKsYAO5mH1UBTN0qSCuJZREIhX6WU4E1p3IW2oRCNzQw==", - "dev": true, - "license": "ISC", - "dependencies": { - "q": "^1.5.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/conventional-changelog-config-spec": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/conventional-changelog-config-spec/-/conventional-changelog-config-spec-2.1.0.tgz", - "integrity": "sha512-IpVePh16EbbB02V+UA+HQnnPIohgXvJRxHcS5+Uwk4AT5LjzCZJm5sp/yqs5C6KZJ1jMsV4paEV13BN1pvDuxQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/conventional-changelog-conventionalcommits": { - "version": "4.6.3", - "resolved": "https://registry.npmjs.org/conventional-changelog-conventionalcommits/-/conventional-changelog-conventionalcommits-4.6.3.tgz", - "integrity": "sha512-LTTQV4fwOM4oLPad317V/QNQ1FY4Hju5qeBIM1uTHbrnCE+Eg4CdRZ3gO2pUeR+tzWdp80M2j3qFFEDWVqOV4g==", - "dev": true, - "license": "ISC", - "dependencies": { - "compare-func": "^2.0.0", - "lodash": "^4.17.15", - "q": "^1.5.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/conventional-changelog-core": { - "version": "4.2.4", - "resolved": "https://registry.npmjs.org/conventional-changelog-core/-/conventional-changelog-core-4.2.4.tgz", - "integrity": "sha512-gDVS+zVJHE2v4SLc6B0sLsPiloR0ygU7HaDW14aNJE1v4SlqJPILPl/aJC7YdtRE4CybBf8gDwObBvKha8Xlyg==", - "dev": true, - "license": "MIT", - "dependencies": { - "add-stream": "^1.0.0", - "conventional-changelog-writer": "^5.0.0", - "conventional-commits-parser": "^3.2.0", - "dateformat": "^3.0.0", - "get-pkg-repo": "^4.0.0", - "git-raw-commits": "^2.0.8", - "git-remote-origin-url": "^2.0.0", - "git-semver-tags": "^4.1.1", - "lodash": "^4.17.15", - "normalize-package-data": "^3.0.0", - "q": "^1.5.1", - "read-pkg": "^3.0.0", - "read-pkg-up": "^3.0.0", - "through2": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/conventional-changelog-ember": { - "version": "2.0.9", - "resolved": "https://registry.npmjs.org/conventional-changelog-ember/-/conventional-changelog-ember-2.0.9.tgz", - "integrity": "sha512-ulzIReoZEvZCBDhcNYfDIsLTHzYHc7awh+eI44ZtV5cx6LVxLlVtEmcO+2/kGIHGtw+qVabJYjdI5cJOQgXh1A==", - "dev": true, - "license": "ISC", - "dependencies": { - "q": "^1.5.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/conventional-changelog-eslint": { - "version": "3.0.9", - "resolved": "https://registry.npmjs.org/conventional-changelog-eslint/-/conventional-changelog-eslint-3.0.9.tgz", - "integrity": "sha512-6NpUCMgU8qmWmyAMSZO5NrRd7rTgErjrm4VASam2u5jrZS0n38V7Y9CzTtLT2qwz5xEChDR4BduoWIr8TfwvXA==", - "dev": true, - "license": "ISC", - "dependencies": { - "q": "^1.5.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/conventional-changelog-express": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/conventional-changelog-express/-/conventional-changelog-express-2.0.6.tgz", - "integrity": "sha512-SDez2f3iVJw6V563O3pRtNwXtQaSmEfTCaTBPCqn0oG0mfkq0rX4hHBq5P7De2MncoRixrALj3u3oQsNK+Q0pQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "q": "^1.5.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/conventional-changelog-jquery": { - "version": "3.0.11", - "resolved": "https://registry.npmjs.org/conventional-changelog-jquery/-/conventional-changelog-jquery-3.0.11.tgz", - "integrity": "sha512-x8AWz5/Td55F7+o/9LQ6cQIPwrCjfJQ5Zmfqi8thwUEKHstEn4kTIofXub7plf1xvFA2TqhZlq7fy5OmV6BOMw==", - "dev": true, - "license": "ISC", - "dependencies": { - "q": "^1.5.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/conventional-changelog-jshint": { - "version": "2.0.9", - "resolved": "https://registry.npmjs.org/conventional-changelog-jshint/-/conventional-changelog-jshint-2.0.9.tgz", - "integrity": "sha512-wMLdaIzq6TNnMHMy31hql02OEQ8nCQfExw1SE0hYL5KvU+JCTuPaDO+7JiogGT2gJAxiUGATdtYYfh+nT+6riA==", - "dev": true, - "license": "ISC", - "dependencies": { - "compare-func": "^2.0.0", - "q": "^1.5.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/conventional-changelog-preset-loader": { - "version": "2.3.4", - "resolved": "https://registry.npmjs.org/conventional-changelog-preset-loader/-/conventional-changelog-preset-loader-2.3.4.tgz", - "integrity": "sha512-GEKRWkrSAZeTq5+YjUZOYxdHq+ci4dNwHvpaBC3+ENalzFWuCWa9EZXSuZBpkr72sMdKB+1fyDV4takK1Lf58g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - } - }, - "node_modules/conventional-changelog-writer": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/conventional-changelog-writer/-/conventional-changelog-writer-5.0.1.tgz", - "integrity": "sha512-5WsuKUfxW7suLblAbFnxAcrvf6r+0b7GvNaWUwUIk0bXMnENP/PEieGKVUQrjPqwPT4o3EPAASBXiY6iHooLOQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "conventional-commits-filter": "^2.0.7", - "dateformat": "^3.0.0", - "handlebars": "^4.7.7", - "json-stringify-safe": "^5.0.1", - "lodash": "^4.17.15", - "meow": "^8.0.0", - "semver": "^6.0.0", - "split": "^1.0.0", - "through2": "^4.0.0" - }, - "bin": { - "conventional-changelog-writer": "cli.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/conventional-changelog-writer/node_modules/semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - } - }, - "node_modules/conventional-commits-filter": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/conventional-commits-filter/-/conventional-commits-filter-2.0.7.tgz", - "integrity": "sha512-ASS9SamOP4TbCClsRHxIHXRfcGCnIoQqkvAzCSbZzTFLfcTqJVugB0agRgsEELsqaeWgsXv513eS116wnlSSPA==", - "dev": true, - "license": "MIT", - "dependencies": { - "lodash.ismatch": "^4.4.0", - "modify-values": "^1.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/conventional-commits-parser": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/conventional-commits-parser/-/conventional-commits-parser-3.2.4.tgz", - "integrity": "sha512-nK7sAtfi+QXbxHCYfhpZsfRtaitZLIA6889kFIouLvz6repszQDgxBu7wf2WbU+Dco7sAnNCJYERCwt54WPC2Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-text-path": "^1.0.1", - "JSONStream": "^1.0.4", - "lodash": "^4.17.15", - "meow": "^8.0.0", - "split2": "^3.0.0", - "through2": "^4.0.0" - }, - "bin": { - "conventional-commits-parser": "cli.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/conventional-recommended-bump": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/conventional-recommended-bump/-/conventional-recommended-bump-6.1.0.tgz", - "integrity": "sha512-uiApbSiNGM/kkdL9GTOLAqC4hbptObFo4wW2QRyHsKciGAfQuLU1ShZ1BIVI/+K2BE/W1AWYQMCXAsv4dyKPaw==", - "dev": true, - "license": "MIT", - "dependencies": { - "concat-stream": "^2.0.0", - "conventional-changelog-preset-loader": "^2.3.4", - "conventional-commits-filter": "^2.0.7", - "conventional-commits-parser": "^3.2.0", - "git-raw-commits": "^2.0.8", - "git-semver-tags": "^4.1.1", - "meow": "^8.0.0", - "q": "^1.5.1" - }, - "bin": { - "conventional-recommended-bump": "cli.js" - }, - "engines": { - "node": ">=10" - } + "peer": true }, "node_modules/core-js": { "version": "3.47.0", @@ -4535,26 +3918,6 @@ "integrity": "sha512-CEE+jwpgLn+MmtCpVcPtiCZpVtB6Z2OKPTr34pycYYoL7sxdOkXDdQ4lRiw6ioC0q6BLqhc6cKweCVvral8yhw==", "license": "MIT" }, - "node_modules/dargs": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/dargs/-/dargs-7.0.0.tgz", - "integrity": "sha512-2iy1EkLdlBzQGvbweYRFxmFath8+K7+AKB0TlhHWkNuH+TmovaMH/Wp7V7R4u7f4SnX3OgLsU9t1NI9ioDnUpg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/dateformat": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-3.0.3.tgz", - "integrity": "sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": "*" - } - }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -4573,43 +3936,6 @@ } } }, - "node_modules/decamelize": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", - "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/decamelize-keys": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/decamelize-keys/-/decamelize-keys-1.1.1.tgz", - "integrity": "sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==", - "dev": true, - "license": "MIT", - "dependencies": { - "decamelize": "^1.1.0", - "map-obj": "^1.0.0" - }, - "engines": { - "node": ">=0.10.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/decamelize-keys/node_modules/map-obj": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-1.0.1.tgz", - "integrity": "sha512-7N/q3lyZ+LVCp7PzuxrJr4KMbBE2hW7BT7YNia330OFxIf4d3r5zVpicP2650l7CPN6RM9zOJRl3NGpqSiw3Eg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/decode-uri-component": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz", @@ -4638,16 +3964,6 @@ "license": "MIT", "peer": true }, - "node_modules/deepmerge": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", - "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/define-data-property": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", @@ -4666,26 +3982,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/detect-indent": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/detect-indent/-/detect-indent-6.1.0.tgz", - "integrity": "sha512-reYkTUJAZb9gUuZ2RvVCNhVHdg62RHnJ7WJl8ftMi4diZ6NWlciOzQN88pUhSELEwflJht4oQDv0F0BMlwaYtA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/detect-newline": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", - "integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/dingbat-to-unicode": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dingbat-to-unicode/-/dingbat-to-unicode-1.0.1.tgz", @@ -4795,123 +4091,6 @@ "@types/trusted-types": "^2.0.7" } }, - "node_modules/dot-prop": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.3.0.tgz", - "integrity": "sha512-QM8q3zDe58hqUqjraQOmzZ1LIH9SWQJTlEKCH4kJ2oQvLZk7RbQXvtDM2XEq3fwkV9CCvvH4LA0AV+ogFsBM2Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-obj": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/dotgitignore": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/dotgitignore/-/dotgitignore-2.1.0.tgz", - "integrity": "sha512-sCm11ak2oY6DglEPpCB8TixLjWAxd3kJTs6UIcSasNYxXdFPV+YKlye92c8H4kKFqV5qYMIh7d+cYecEg0dIkA==", - "dev": true, - "license": "ISC", - "dependencies": { - "find-up": "^3.0.0", - "minimatch": "^3.0.4" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/dotgitignore/node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/dotgitignore/node_modules/find-up": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", - "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", - "dev": true, - "license": "MIT", - "dependencies": { - "locate-path": "^3.0.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/dotgitignore/node_modules/locate-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", - "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^3.0.0", - "path-exists": "^3.0.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/dotgitignore/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/dotgitignore/node_modules/p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/dotgitignore/node_modules/p-locate": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", - "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-limit": "^2.0.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/dotgitignore/node_modules/path-exists": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", - "integrity": "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, "node_modules/duck": { "version": "0.1.12", "resolved": "https://registry.npmjs.org/duck/-/duck-0.1.12.tgz", @@ -4959,16 +4138,6 @@ "once": "^1.4.0" } }, - "node_modules/error-ex": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", - "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-arrayish": "^0.2.1" - } - }, "node_modules/es-define-property": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", @@ -5325,13 +4494,6 @@ "node": ">=4.0" } }, - "node_modules/estree-walker": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", - "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", - "dev": true, - "license": "MIT" - }, "node_modules/esutils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", @@ -5458,32 +4620,6 @@ "dev": true, "license": "MIT" }, - "node_modules/figures": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", - "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", - "dev": true, - "license": "MIT", - "dependencies": { - "escape-string-regexp": "^1.0.5" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/figures/node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.8.0" - } - }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", @@ -5514,6 +4650,7 @@ "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "locate-path": "^6.0.0", "path-exists": "^4.0.0" @@ -5679,69 +4816,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/get-pkg-repo": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/get-pkg-repo/-/get-pkg-repo-4.2.1.tgz", - "integrity": "sha512-2+QbHjFRfGB74v/pYWjd5OhU3TDIC2Gv/YKUTk/tCvAz0pkn/Mz6P3uByuBimLOcPvN2jYdScl3xGFSrx0jEcA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@hutson/parse-repository-url": "^3.0.0", - "hosted-git-info": "^4.0.0", - "through2": "^2.0.0", - "yargs": "^16.2.0" - }, - "bin": { - "get-pkg-repo": "src/cli.js" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/get-pkg-repo/node_modules/readable-stream": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", - "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", - "dev": true, - "license": "MIT", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/get-pkg-repo/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true, - "license": "MIT" - }, - "node_modules/get-pkg-repo/node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dev": true, - "license": "MIT", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, - "node_modules/get-pkg-repo/node_modules/through2": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", - "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "readable-stream": "~2.3.6", - "xtend": "~4.0.1" - } - }, "node_modules/get-port": { "version": "7.1.0", "resolved": "https://registry.npmjs.org/get-port/-/get-port-7.1.0.tgz", @@ -5782,77 +4856,6 @@ "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" } }, - "node_modules/git-raw-commits": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/git-raw-commits/-/git-raw-commits-2.0.11.tgz", - "integrity": "sha512-VnctFhw+xfj8Va1xtfEqCUD2XDrbAPSJx+hSrE5K7fGdjZruW7XV+QOrN7LF/RJyvspRiD2I0asWsxFp0ya26A==", - "dev": true, - "license": "MIT", - "dependencies": { - "dargs": "^7.0.0", - "lodash": "^4.17.15", - "meow": "^8.0.0", - "split2": "^3.0.0", - "through2": "^4.0.0" - }, - "bin": { - "git-raw-commits": "cli.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/git-remote-origin-url": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/git-remote-origin-url/-/git-remote-origin-url-2.0.0.tgz", - "integrity": "sha512-eU+GGrZgccNJcsDH5LkXR3PB9M958hxc7sbA8DFJjrv9j4L2P/eZfKhM+QD6wyzpiv+b1BpK0XrYCxkovtjSLw==", - "dev": true, - "license": "MIT", - "dependencies": { - "gitconfiglocal": "^1.0.0", - "pify": "^2.3.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/git-semver-tags": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/git-semver-tags/-/git-semver-tags-4.1.1.tgz", - "integrity": "sha512-OWyMt5zBe7xFs8vglMmhM9lRQzCWL3WjHtxNNfJTMngGym7pC1kh8sP6jevfydJ6LP3ZvGxfb6ABYgPUM0mtsA==", - "dev": true, - "license": "MIT", - "dependencies": { - "meow": "^8.0.0", - "semver": "^6.0.0" - }, - "bin": { - "git-semver-tags": "cli.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/git-semver-tags/node_modules/semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - } - }, - "node_modules/gitconfiglocal": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/gitconfiglocal/-/gitconfiglocal-1.0.0.tgz", - "integrity": "sha512-spLUXeTAVHxDtKsJc8FkFVgFtMdEN9qPGpL23VfSHx4fP4+Ds097IXLvymbnDH8FnmxX5Nr9bPw3A+AQ6mWEaQ==", - "dev": true, - "license": "BSD", - "dependencies": { - "ini": "^1.3.2" - } - }, "node_modules/glob": { "version": "10.5.0", "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", @@ -5922,38 +4925,6 @@ "dev": true, "license": "ISC" }, - "node_modules/handlebars": { - "version": "4.7.8", - "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", - "integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "minimist": "^1.2.5", - "neo-async": "^2.6.2", - "source-map": "^0.6.1", - "wordwrap": "^1.0.0" - }, - "bin": { - "handlebars": "bin/handlebars" - }, - "engines": { - "node": ">=0.4.7" - }, - "optionalDependencies": { - "uglify-js": "^3.1.4" - } - }, - "node_modules/hard-rejection": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/hard-rejection/-/hard-rejection-2.1.0.tgz", - "integrity": "sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -6019,19 +4990,6 @@ "node": ">= 0.4" } }, - "node_modules/hosted-git-info": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-4.1.0.tgz", - "integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==", - "dev": true, - "license": "ISC", - "dependencies": { - "lru-cache": "^6.0.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", @@ -6136,29 +5094,12 @@ "node": ">=0.8.19" } }, - "node_modules/indent-string": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", - "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "license": "ISC" }, - "node_modules/ini": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", - "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "dev": true, - "license": "ISC" - }, "node_modules/inquirer": { "version": "12.11.1", "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-12.11.1.tgz", @@ -6219,13 +5160,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-arrayish": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", - "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", - "dev": true, - "license": "MIT" - }, "node_modules/is-callable": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", @@ -6239,22 +5173,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-core-module": { - "version": "2.16.1", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", - "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", - "dev": true, - "license": "MIT", - "dependencies": { - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -6321,43 +5239,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/is-module": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-module/-/is-module-1.0.0.tgz", - "integrity": "sha512-51ypPSPCoTEIN9dy5Oy+h4pShgJmPCygKfyRCISBI+JoWT/2oJvK8QPxmwv7b/p239jXrm9M1mlQbyKJ5A152g==", - "dev": true, - "license": "MIT" - }, - "node_modules/is-obj": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", - "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/is-plain-obj": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", - "integrity": "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-reference": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-1.2.1.tgz", - "integrity": "sha512-U82MsXXiFIrjCK4otLT+o2NA2Cd2g5MLoOVXUZjIOhLurrRxpEXzI8O0KZHr3IjLvlAH1kTPYSuqer5T9ZVBKQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/estree": "*" - } - }, "node_modules/is-regex": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", @@ -6390,19 +5271,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/is-text-path": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-text-path/-/is-text-path-1.0.1.tgz", - "integrity": "sha512-xFuJpne9oFz5qDaodwmmG08e3CawH/2ZV8Qqza1Ko7Sk8POWbkRdwIoAWVhqvq0XeUzANEhKo2n0IXUGBm7A/w==", - "dev": true, - "license": "MIT", - "dependencies": { - "text-extensions": "^1.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/is-typed-array": { "version": "1.1.15", "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.15.tgz", @@ -6541,20 +5409,6 @@ "license": "MIT", "peer": true }, - "node_modules/json-parse-better-errors": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", - "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", - "dev": true, - "license": "MIT" - }, - "node_modules/json-parse-even-better-errors": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", - "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", - "dev": true, - "license": "MIT" - }, "node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", @@ -6571,40 +5425,6 @@ "license": "MIT", "peer": true }, - "node_modules/json-stringify-safe": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", - "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", - "dev": true, - "license": "ISC" - }, - "node_modules/jsonparse": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", - "integrity": "sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg==", - "dev": true, - "engines": [ - "node >= 0.2.0" - ], - "license": "MIT" - }, - "node_modules/JSONStream": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz", - "integrity": "sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==", - "dev": true, - "license": "(MIT OR Apache-2.0)", - "dependencies": { - "jsonparse": "^1.2.0", - "through": ">=2.2.7 <3" - }, - "bin": { - "JSONStream": "bin.js" - }, - "engines": { - "node": "*" - } - }, "node_modules/jspdf": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/jspdf/-/jspdf-3.0.4.tgz", @@ -6682,16 +5502,6 @@ "json-buffer": "3.0.1" } }, - "node_modules/kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/kleur": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", @@ -6771,45 +5581,13 @@ "immediate": "~3.0.5" } }, - "node_modules/lines-and-columns": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", - "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", - "dev": true, - "license": "MIT" - }, - "node_modules/load-json-file": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz", - "integrity": "sha512-Kx8hMakjX03tiGTLAIdJ+lL0htKnXjEZN6hk/tozf/WOuYGdZBJrZ+rCJRbVCugsjB3jMLn9746NsQIf5VjBMw==", - "dev": true, - "license": "MIT", - "dependencies": { - "graceful-fs": "^4.1.2", - "parse-json": "^4.0.0", - "pify": "^3.0.0", - "strip-bom": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/load-json-file/node_modules/pify": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", - "integrity": "sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "p-locate": "^5.0.0" }, @@ -6834,13 +5612,6 @@ "dev": true, "license": "MIT" }, - "node_modules/lodash.ismatch": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/lodash.ismatch/-/lodash.ismatch-4.4.0.tgz", - "integrity": "sha512-fPMfXjGQEV9Xsq/8MTSgUf255gawYRbjwMyDbcvDhXgV7enSZA0hynz6vMPnpAb5iONEzBHBPsT+0zes5Z301g==", - "dev": true, - "license": "MIT" - }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -6902,19 +5673,6 @@ "dev": true, "license": "MIT" }, - "node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, - "license": "ISC", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", @@ -6986,19 +5744,6 @@ "sprintf-js": "~1.0.2" } }, - "node_modules/map-obj": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-4.3.0.tgz", - "integrity": "sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -7009,204 +5754,6 @@ "node": ">= 0.4" } }, - "node_modules/meow": { - "version": "8.1.2", - "resolved": "https://registry.npmjs.org/meow/-/meow-8.1.2.tgz", - "integrity": "sha512-r85E3NdZ+mpYk1C6RjPFEMSE+s1iZMuHtsHAqY0DT3jZczl0diWUZ8g6oU7h0M9cD2EL+PzaYghhCLzR0ZNn5Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/minimist": "^1.2.0", - "camelcase-keys": "^6.2.2", - "decamelize-keys": "^1.1.0", - "hard-rejection": "^2.1.0", - "minimist-options": "4.1.0", - "normalize-package-data": "^3.0.0", - "read-pkg-up": "^7.0.1", - "redent": "^3.0.0", - "trim-newlines": "^3.0.0", - "type-fest": "^0.18.0", - "yargs-parser": "^20.2.3" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/meow/node_modules/find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "dev": true, - "license": "MIT", - "dependencies": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/meow/node_modules/hosted-git-info": { - "version": "2.8.9", - "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz", - "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==", - "dev": true, - "license": "ISC" - }, - "node_modules/meow/node_modules/locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^4.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/meow/node_modules/p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/meow/node_modules/p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/meow/node_modules/parse-json": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", - "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.0.0", - "error-ex": "^1.3.1", - "json-parse-even-better-errors": "^2.3.0", - "lines-and-columns": "^1.1.6" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/meow/node_modules/read-pkg": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-5.2.0.tgz", - "integrity": "sha512-Ug69mNOpfvKDAc2Q8DRpMjjzdtrnv9HcSMX+4VsZxD1aZ6ZzrIE7rlzXBtWTyhULSMKg076AW6WR5iZpD0JiOg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/normalize-package-data": "^2.4.0", - "normalize-package-data": "^2.5.0", - "parse-json": "^5.0.0", - "type-fest": "^0.6.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/meow/node_modules/read-pkg-up": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-7.0.1.tgz", - "integrity": "sha512-zK0TB7Xd6JpCLmlLmufqykGE+/TlOePD6qKClNW7hHDKFh/J7/7gCWGR7joEQEW1bKq3a3yUZSObOoWLFQ4ohg==", - "dev": true, - "license": "MIT", - "dependencies": { - "find-up": "^4.1.0", - "read-pkg": "^5.2.0", - "type-fest": "^0.8.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/meow/node_modules/read-pkg-up/node_modules/type-fest": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", - "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", - "dev": true, - "license": "(MIT OR CC0-1.0)", - "engines": { - "node": ">=8" - } - }, - "node_modules/meow/node_modules/read-pkg/node_modules/normalize-package-data": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", - "integrity": "sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "hosted-git-info": "^2.1.4", - "resolve": "^1.10.0", - "semver": "2 || 3 || 4 || 5", - "validate-npm-package-license": "^3.0.1" - } - }, - "node_modules/meow/node_modules/read-pkg/node_modules/type-fest": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.6.0.tgz", - "integrity": "sha512-q+MB8nYR1KDLrgr4G5yemftpMC7/QLqVndBmEEdqzmNj5dcFOO4Oo8qlwZE3ULT3+Zim1F8Kq4cBnikNhlCMlg==", - "dev": true, - "license": "(MIT OR CC0-1.0)", - "engines": { - "node": ">=8" - } - }, - "node_modules/meow/node_modules/semver": { - "version": "5.7.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", - "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver" - } - }, - "node_modules/meow/node_modules/type-fest": { - "version": "0.18.1", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.18.1.tgz", - "integrity": "sha512-OIAYXk8+ISY+qTOwkHtKqzAuxchoMiD9Udx+FSGQDuiRR+PJKJHc2NJAXlbhkGwTt/4/nKZxELY1w3ReWOL8mw==", - "dev": true, - "license": "(MIT OR CC0-1.0)", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/mime": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/mime/-/mime-4.1.0.tgz", @@ -7257,16 +5804,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/min-indent": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/min-indent/-/min-indent-1.0.1.tgz", - "integrity": "sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, "node_modules/minimatch": { "version": "9.0.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", @@ -7283,41 +5820,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/minimist": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/minimist-options": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/minimist-options/-/minimist-options-4.1.0.tgz", - "integrity": "sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==", - "dev": true, - "license": "MIT", - "dependencies": { - "arrify": "^1.0.1", - "is-plain-obj": "^1.1.0", - "kind-of": "^6.0.3" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/minimist-options/node_modules/arrify": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", - "integrity": "sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/minio": { "version": "8.0.6", "resolved": "https://registry.npmjs.org/minio/-/minio-8.0.6.tgz", @@ -7406,16 +5908,6 @@ "dev": true, "license": "MIT" }, - "node_modules/modify-values": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/modify-values/-/modify-values-1.0.1.tgz", - "integrity": "sha512-xV2bxeN6F7oYjZWTe/YPAy6MN2M+sL4u/Rlm2AHCIVGfo2p1yGmBHQ6vHehl4bRTZBdHu3TSkWdYgkwpYzAGSw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -7466,29 +5958,6 @@ "dev": true, "license": "MIT" }, - "node_modules/neo-async": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", - "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", - "dev": true, - "license": "MIT" - }, - "node_modules/normalize-package-data": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-3.0.3.tgz", - "integrity": "sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "hosted-git-info": "^4.0.1", - "is-core-module": "^2.5.0", - "semver": "^7.3.4", - "validate-npm-package-license": "^3.0.1" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -7578,6 +6047,7 @@ "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "yocto-queue": "^0.1.0" }, @@ -7594,6 +6064,7 @@ "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "p-limit": "^3.0.2" }, @@ -7604,16 +6075,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/p-try": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/package-json-from-dist": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", @@ -7642,26 +6103,13 @@ "node": ">=6" } }, - "node_modules/parse-json": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", - "integrity": "sha512-aOIos8bujGN93/8Ox/jPLh7RwVnPEysynVFE+fQZyg6jKELEHwzgKdLRFHUgXJL6kylijVSBC4BvN9OmsB48Rw==", - "dev": true, - "license": "MIT", - "dependencies": { - "error-ex": "^1.3.1", - "json-parse-better-errors": "^1.0.1" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=8" } @@ -7685,13 +6133,6 @@ "node": ">=8" } }, - "node_modules/path-parse": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", - "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "dev": true, - "license": "MIT" - }, "node_modules/path-scurry": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", @@ -7716,29 +6157,6 @@ "dev": true, "license": "ISC" }, - "node_modules/path-type": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz", - "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==", - "dev": true, - "license": "MIT", - "dependencies": { - "pify": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/path-type/node_modules/pify": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", - "integrity": "sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, "node_modules/pathe": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", @@ -7796,16 +6214,6 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, - "node_modules/pify": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", - "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/possible-typed-array-names": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", @@ -7856,6 +6264,22 @@ "node": ">= 0.8.0" } }, + "node_modules/prettier": { + "version": "3.9.4", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.9.4.tgz", + "integrity": "sha512-yWG/o/4oJfo036EKAfK6ACAoDOfHeRHx4tuxkfBZiauURiaSmYwlpOr5LQqKtIkRD2z1PLteme2WoxEnj4tHTg==", + "dev": true, + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, "node_modules/process": { "version": "0.11.10", "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", @@ -7978,18 +6402,6 @@ "node": ">=6" } }, - "node_modules/q": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", - "integrity": "sha512-kV/CThkXo6xyFEZUugw/+pIOywXcDbFYgSct5cT3gqlbkBE1SJdwy6UQoZvodiWF/ckQLZyDE/Bu1M6gVu5lVw==", - "deprecated": "You or someone you depend on is using Q, the JavaScript Promise library that gave JavaScript developers strong feelings about promises. They can almost certainly migrate to the native JavaScript promise now. Thank you literally everyone for joining me in this bet against the odds. Be excellent to each other.\n\n(For a CapTP with native promises, see @endo/eventual-send and @endo/captp)", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.6.0", - "teleport": ">=0.2.0" - } - }, "node_modules/query-string": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/query-string/-/query-string-7.1.3.tgz", @@ -8009,16 +6421,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/quick-lru": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-4.0.1.tgz", - "integrity": "sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/raf": { "version": "3.4.1", "resolved": "https://registry.npmjs.org/raf/-/raf-3.4.1.tgz", @@ -8030,148 +6432,6 @@ "performance-now": "^2.1.0" } }, - "node_modules/randombytes": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", - "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "safe-buffer": "^5.1.0" - } - }, - "node_modules/read-pkg": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", - "integrity": "sha512-BLq/cCO9two+lBgiTYNqD6GdtK8s4NpaWrl6/rCO9w0TUS8oJl7cmToOZfRYllKTISY6nt1U7jQ53brmKqY6BA==", - "dev": true, - "license": "MIT", - "dependencies": { - "load-json-file": "^4.0.0", - "normalize-package-data": "^2.3.2", - "path-type": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/read-pkg-up": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-3.0.0.tgz", - "integrity": "sha512-YFzFrVvpC6frF1sz8psoHDBGF7fLPc+llq/8NB43oagqWkx8ar5zYtsTORtOjw9W2RHLpWP+zTWwBvf1bCmcSw==", - "dev": true, - "license": "MIT", - "dependencies": { - "find-up": "^2.0.0", - "read-pkg": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/read-pkg-up/node_modules/find-up": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", - "integrity": "sha512-NWzkk0jSJtTt08+FBFMvXoeZnOJD+jTtsRmBYbAIzJdX6l7dLgR7CTubCM5/eDdPUBvLCeVasP1brfVR/9/EZQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "locate-path": "^2.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/read-pkg-up/node_modules/locate-path": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", - "integrity": "sha512-NCI2kiDkyR7VeEKm27Kda/iQHyKJe1Bu0FlTbYp3CqJu+9IFe9bLyAjMxf5ZDDbEg+iMPzB5zYyUTSm8wVTKmA==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^2.0.0", - "path-exists": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/read-pkg-up/node_modules/p-limit": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", - "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-try": "^1.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/read-pkg-up/node_modules/p-locate": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", - "integrity": "sha512-nQja7m7gSKuewoVRen45CtVfODR3crN3goVQ0DDZ9N3yHxgpkuBhZqsaiotSQRrADUrne346peY7kT3TSACykg==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-limit": "^1.1.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/read-pkg-up/node_modules/p-try": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", - "integrity": "sha512-U1etNYuMJoIz3ZXSrrySFjsXQTWOx2/jdi86L+2pRvph/qMKL6sbcCYdH23fqsbm8TH2Gn0OybpT4eSFlCVHww==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/read-pkg-up/node_modules/path-exists": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", - "integrity": "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/read-pkg/node_modules/hosted-git-info": { - "version": "2.8.9", - "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz", - "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==", - "dev": true, - "license": "ISC" - }, - "node_modules/read-pkg/node_modules/normalize-package-data": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", - "integrity": "sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "hosted-git-info": "^2.1.4", - "resolve": "^1.10.0", - "semver": "2 || 3 || 4 || 5", - "validate-npm-package-license": "^3.0.1" - } - }, - "node_modules/read-pkg/node_modules/semver": { - "version": "5.7.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", - "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver" - } - }, "node_modules/readable-stream": { "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", @@ -8210,20 +6470,6 @@ "node": ">=10" } }, - "node_modules/redent": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz", - "integrity": "sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==", - "dev": true, - "license": "MIT", - "dependencies": { - "indent-string": "^4.0.0", - "strip-indent": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/regenerator-runtime": { "version": "0.13.11", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", @@ -8242,27 +6488,6 @@ "node": ">=0.10.0" } }, - "node_modules/resolve": { - "version": "1.22.11", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.11.tgz", - "integrity": "sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-core-module": "^2.16.1", - "path-parse": "^1.0.7", - "supports-preserve-symlinks-flag": "^1.0.0" - }, - "bin": { - "resolve": "bin/resolve" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", @@ -8445,16 +6670,6 @@ "node": ">=10" } }, - "node_modules/serialize-javascript": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.2.tgz", - "integrity": "sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "randombytes": "^2.1.0" - } - }, "node_modules/set-function-length": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", @@ -8527,19 +6742,14 @@ "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", "license": "MIT" }, - "node_modules/smob": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/smob/-/smob-1.5.0.tgz", - "integrity": "sha512-g6T+p7QO8npa+/hNx9ohv1E5pVCmWrVCUzUXJyLdMmftX6ER0oiWY/w9knEonLpnOp6b6FenKnMfR8gqwWdwig==", - "dev": true, - "license": "MIT" - }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", "dev": true, "license": "BSD-3-Clause", + "optional": true, + "peer": true, "engines": { "node": ">=0.10.0" } @@ -8560,60 +6770,13 @@ "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "buffer-from": "^1.0.0", "source-map": "^0.6.0" } }, - "node_modules/spdx-correct": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.2.0.tgz", - "integrity": "sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "spdx-expression-parse": "^3.0.0", - "spdx-license-ids": "^3.0.0" - } - }, - "node_modules/spdx-exceptions": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.5.0.tgz", - "integrity": "sha512-PiU42r+xO4UbUS1buo3LPJkjlO7430Xn5SVAhdpzzsPHsjbYVflnnFdATgabnLude+Cqu25p6N+g2lw/PFsa4w==", - "dev": true, - "license": "CC-BY-3.0" - }, - "node_modules/spdx-expression-parse": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz", - "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "spdx-exceptions": "^2.1.0", - "spdx-license-ids": "^3.0.0" - } - }, - "node_modules/spdx-license-ids": { - "version": "3.0.22", - "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.22.tgz", - "integrity": "sha512-4PRT4nh1EImPbt2jASOKHX7PB7I+e4IWNLvkKFDxNhJlfjbYlleYQh285Z/3mPTHSAK/AvdMmw5BNNuYH8ShgQ==", - "dev": true, - "license": "CC0-1.0" - }, - "node_modules/split": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", - "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", - "dev": true, - "license": "MIT", - "dependencies": { - "through": "2" - }, - "engines": { - "node": "*" - } - }, "node_modules/split-ca": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", @@ -8631,16 +6794,6 @@ "node": ">=6" } }, - "node_modules/split2": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/split2/-/split2-3.2.2.tgz", - "integrity": "sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==", - "dev": true, - "license": "ISC", - "dependencies": { - "readable-stream": "^3.0.0" - } - }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -8717,113 +6870,6 @@ "node": ">=0.1.14" } }, - "node_modules/standard-version": { - "version": "9.5.0", - "resolved": "https://registry.npmjs.org/standard-version/-/standard-version-9.5.0.tgz", - "integrity": "sha512-3zWJ/mmZQsOaO+fOlsa0+QK90pwhNd042qEcw6hKFNoLFs7peGyvPffpEBbK/DSGPbyOvli0mUIFv5A4qTjh2Q==", - "dev": true, - "license": "ISC", - "dependencies": { - "chalk": "^2.4.2", - "conventional-changelog": "3.1.25", - "conventional-changelog-config-spec": "2.1.0", - "conventional-changelog-conventionalcommits": "4.6.3", - "conventional-recommended-bump": "6.1.0", - "detect-indent": "^6.0.0", - "detect-newline": "^3.1.0", - "dotgitignore": "^2.1.0", - "figures": "^3.1.0", - "find-up": "^5.0.0", - "git-semver-tags": "^4.0.0", - "semver": "^7.1.1", - "stringify-package": "^1.0.1", - "yargs": "^16.0.0" - }, - "bin": { - "standard-version": "bin/cli.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/standard-version/node_modules/ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-convert": "^1.9.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/standard-version/node_modules/chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/standard-version/node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/standard-version/node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", - "dev": true, - "license": "MIT" - }, - "node_modules/standard-version/node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/standard-version/node_modules/has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/standard-version/node_modules/supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/std-env": { "version": "3.10.0", "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", @@ -8955,14 +7001,6 @@ "node": ">=8" } }, - "node_modules/stringify-package": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/stringify-package/-/stringify-package-1.0.1.tgz", - "integrity": "sha512-sa4DUQsYciMP1xhKWGuFM04fB0LG/9DlluZoSVywUMRNvzid6XucHK0/90xGxRoHrAaROrcHK1aPKaijCtSrhg==", - "deprecated": "This module is not used anymore, and has been replaced by @npmcli/package-json", - "dev": true, - "license": "ISC" - }, "node_modules/strip-ansi": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.2.tgz", @@ -9002,29 +7040,6 @@ "node": ">=8" } }, - "node_modules/strip-bom": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", - "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/strip-indent": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-3.0.0.tgz", - "integrity": "sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "min-indent": "^1.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", @@ -9065,19 +7080,6 @@ "node": ">=8" } }, - "node_modules/supports-preserve-symlinks-flag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", - "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/svg-pathdata": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/svg-pathdata/-/svg-pathdata-6.0.3.tgz", @@ -9122,6 +7124,8 @@ "integrity": "sha512-t/R3R/n0MSwnnazuPpPNVO60LX0SKL45pyl9YlvxIdkH0Of7D5qM2EVe+yASRIlY5pZ73nclYJfNANGWPwFDZw==", "dev": true, "license": "BSD-2-Clause", + "optional": true, + "peer": true, "dependencies": { "@jridgewell/source-map": "^0.3.3", "acorn": "^8.15.0", @@ -9140,7 +7144,9 @@ "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "node_modules/test-exclude": { "version": "7.0.1", @@ -9191,16 +7197,6 @@ "b4a": "^1.6.4" } }, - "node_modules/text-extensions": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/text-extensions/-/text-extensions-1.9.0.tgz", - "integrity": "sha512-wiBrwC1EhBelW12Zy26JeOUkQ5mRu+5o8rpsJk5+2t+Y5vE7e842qtZDQ2g1NpX/29HdyFeJ4nSIhI47ENSxlQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10" - } - }, "node_modules/text-segmentation": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/text-segmentation/-/text-segmentation-1.0.3.tgz", @@ -9212,13 +7208,6 @@ "utrie": "^1.0.2" } }, - "node_modules/through": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", - "dev": true, - "license": "MIT" - }, "node_modules/through2": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/through2/-/through2-4.0.2.tgz", @@ -9300,16 +7289,6 @@ "node": ">=14.14" } }, - "node_modules/trim-newlines": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-3.0.1.tgz", - "integrity": "sha512-c1PTsA3tYrIsLGkJkzHF+w9F2EyxfXGo4UyJc4pFL++FMjnq0HJS69T3M7d//gKrFKwy429bouPescbjecU+Zw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/ts-api-utils": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.1.0.tgz", @@ -9382,13 +7361,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/typedarray": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==", - "dev": true, - "license": "MIT" - }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -9403,20 +7375,6 @@ "node": ">=14.17" } }, - "node_modules/uglify-js": { - "version": "3.19.3", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.19.3.tgz", - "integrity": "sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ==", - "dev": true, - "license": "BSD-2-Clause", - "optional": true, - "bin": { - "uglifyjs": "bin/uglifyjs" - }, - "engines": { - "node": ">=0.8.0" - } - }, "node_modules/underscore": { "version": "1.13.7", "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.7.tgz", @@ -9496,17 +7454,6 @@ "uuid": "dist/bin/uuid" } }, - "node_modules/validate-npm-package-license": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", - "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "spdx-correct": "^3.0.0", - "spdx-expression-parse": "^3.0.0" - } - }, "node_modules/vite": { "version": "7.3.0", "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.0.tgz", @@ -9790,13 +7737,6 @@ "node": ">=0.10.0" } }, - "node_modules/wordwrap": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", - "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==", - "dev": true, - "license": "MIT" - }, "node_modules/wrap-ansi": { "version": "9.0.2", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", @@ -9976,16 +7916,6 @@ "node": ">=4.0" } }, - "node_modules/xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.4" - } - }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", @@ -9996,13 +7926,6 @@ "node": ">=10" } }, - "node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true, - "license": "ISC" - }, "node_modules/yaml": { "version": "2.8.2", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.2.tgz", @@ -10019,86 +7942,13 @@ "url": "https://github.com/sponsors/eemeli" } }, - "node_modules/yargs": { - "version": "16.2.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", - "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", - "dev": true, - "license": "MIT", - "dependencies": { - "cliui": "^7.0.2", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.0", - "y18n": "^5.0.5", - "yargs-parser": "^20.2.2" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/yargs-parser": { - "version": "20.2.9", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", - "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=10" - } - }, - "node_modules/yargs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/yargs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, - "license": "MIT" - }, - "node_modules/yargs/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/yargs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=10" }, diff --git a/package.json b/package.json index 2cea5b22..7366ce98 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "8.0.0-rc.1", + "version": "8.9.0", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", @@ -56,16 +56,22 @@ "import": "./dist/internals.js", "types": "./dist/internals.d.ts" }, + "./brain-format": { + "import": "./dist/storage/brainFormat.js", + "types": "./dist/storage/brainFormat.d.ts" + }, "./embeddings/wasm": { "import": "./dist/embeddings/wasm/index.js", "types": "./dist/embeddings/wasm/index.d.ts" } }, "engines": { - "node": "22.x", - "bun": ">=1.0.0" + "node": ">=22", + "bun": ">=1.1.0" }, "scripts": { + "clean": "node -e \"require('fs').rmSync('dist',{recursive:true,force:true})\"", + "prebuild": "npm run clean", "build": "npm run build:types:if-needed && npm run build:patterns:if-needed && tsc && tsc -p tsconfig.cli.json && npm run build:copy-wasm", "build:copy-wasm": "node -e \"const fs=require('fs');const src='src/embeddings/wasm/pkg';const dst='dist/embeddings/wasm/pkg';const skip=new Set(['package.json','.gitignore']);if(fs.existsSync(src)){fs.mkdirSync(dst,{recursive:true});fs.readdirSync(src).filter(f=>!skip.has(f)).forEach(f=>fs.copyFileSync(src+'/'+f,dst+'/'+f));console.log('Copied WASM pkg to dist')}\"", "build:types": "tsx scripts/buildTypeEmbeddings.ts", @@ -88,8 +94,7 @@ "test:ci-unit": "CI=true vitest run --config tests/configs/vitest.unit.config.ts", "test:ci-integration": "NODE_OPTIONS='--max-old-space-size=16384' CI=true vitest run --config tests/configs/vitest.integration.config.ts", "test:ci": "npm run test:ci-unit && npm run test:ci-integration", - "test:bun": "bun tests/integration/bun-compile-test.ts", - "test:bun:compile": "bun build tests/integration/bun-compile-test.ts --compile --outfile /tmp/brainy-bun-test && /tmp/brainy-bun-test", + "test:bun": "bun tests/integration/bun-runtime-test.ts", "test:wasm": "npx vitest run tests/integration/wasm-embeddings.test.ts", "typecheck": "tsc --noEmit", "lint": "eslint --ext .ts,.js src/", @@ -100,11 +105,7 @@ "release:patch": "./scripts/release.sh patch", "release:minor": "./scripts/release.sh minor", "release:major": "./scripts/release.sh major", - "release:dry": "./scripts/release.sh patch --dry-run", - "release:standard-version": "standard-version", - "release:standard-version:patch": "standard-version --release-as patch", - "release:standard-version:minor": "standard-version --release-as minor", - "release:standard-version:major": "standard-version --release-as major" + "release:dry": "./scripts/release.sh patch --dry-run" }, "keywords": [ "ai-database", @@ -151,14 +152,10 @@ "boolean": "3.2.0" }, "devDependencies": { - "@rollup/plugin-commonjs": "^28.0.6", - "@rollup/plugin-node-resolve": "^16.0.1", - "@rollup/plugin-replace": "^6.0.2", - "@rollup/plugin-terser": "^0.4.4", "@testcontainers/redis": "^11.5.1", "@types/js-yaml": "^4.0.9", "@types/mime": "^3.0.4", - "@types/node": "^20.11.30", + "@types/node": "^22", "@types/uuid": "^10.0.0", "@types/ws": "^8.18.1", "@typescript-eslint/eslint-plugin": "^8.0.0", @@ -166,7 +163,7 @@ "@vitest/coverage-v8": "^3.2.4", "jspdf": "^3.0.3", "minio": "^8.0.5", - "standard-version": "^9.5.0", + "prettier": "^3.9.4", "testcontainers": "^11.5.1", "tsx": "^4.19.2", "typescript": "^5.4.5", @@ -204,33 +201,5 @@ "tabWidth": 2, "trailingComma": "none", "useTabs": false - }, - "eslintConfig": { - "root": true, - "extends": [ - "eslint:recommended", - "plugin:@typescript-eslint/recommended" - ], - "parser": "@typescript-eslint/parser", - "plugins": [ - "@typescript-eslint" - ], - "rules": { - "@typescript-eslint/no-explicit-any": "off", - "no-unused-vars": "off", - "@typescript-eslint/no-unused-vars": [ - "warn", - { - "args": "after-used", - "argsIgnorePattern": "^_" - } - ], - "semi": "off", - "@typescript-eslint/semi": [ - "error", - "never" - ], - "no-extra-semi": "off" - } } } diff --git a/scripts/push-docs.js b/scripts/push-docs.js new file mode 100644 index 00000000..699d332b --- /dev/null +++ b/scripts/push-docs.js @@ -0,0 +1,116 @@ +#!/usr/bin/env node +/** + * @module scripts/push-docs + * @description Push this repo's PUBLIC docs to the soulcraft.com docs ingest + * door after an npm publish (VENUE-DOCS-RELEASE-PUSH — retires the old + * build-time docs sync). + * + * Contract (mirrors the reference implementation on the serving side): + * POST {base}/api/docs/ingest + * headers: x-service-secret: $DOCS_INGEST_SECRET, Content-Type: application/json + * body: { docs: [{ slug, title, markdown, nav: { order, section } }] } + * batches of 10, idempotent per slug. + * + * A doc is public iff its frontmatter has `public: true` AND a `slug`. The + * frontmatter is stripped; `category` → nav.section, `order` → nav.order. + * + * Deliberately NOT pushed: the combined /docs landing index. It spans BOTH + * engine corpora (this repo's and the native accelerator's), so a per-repo + * push would clobber the union — the index is authored on the serving side. + * + * Env: DOCS_INGEST_SECRET (required), DOCS_INGEST_BASE (default + * https://soulcraft.com). Exits 0 with a LOUD warning when the secret is + * absent (the npm publish has already happened; the serving side runs its + * interim sync on request) and exits 1 when a push actually fails — the docs + * site would silently trail npm otherwise, and that must be visible. + */ +import * as fs from 'node:fs' +import * as path from 'node:path' + +const BASE = (process.env.DOCS_INGEST_BASE || 'https://soulcraft.com').replace(/\/+$/, '') +const SECRET = process.env.DOCS_INGEST_SECRET +const DOCS_DIR = path.join(path.dirname(new URL(import.meta.url).pathname), '..', 'docs') +const BATCH = 10 + +if (!SECRET) { + console.warn( + '⚠️ DOCS PUSH SKIPPED: DOCS_INGEST_SECRET is not set.\n' + + ' soulcraft.com/docs now TRAILS this npm release until docs are pushed.\n' + + ' Either export DOCS_INGEST_SECRET and re-run `node scripts/push-docs.js`,\n' + + ' or ping venue on VENUE-DOCS-RELEASE-PUSH for the interim sync.' + ) + process.exit(0) +} + +/** Minimal frontmatter split — returns [meta, body] or [null, raw]. */ +function parseFrontmatter(raw) { + const m = raw.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/) + if (!m) return [null, raw] + const meta = {} + for (const line of m[1].split('\n')) { + const kv = line.match(/^(\w[\w-]*):\s*(.*)$/) + if (kv) meta[kv[1]] = kv[2].trim().replace(/^["']|["']$/g, '') + } + return [meta, m[2]] +} + +const docs = [] +;(function walk(dir) { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name) + if (entry.isDirectory()) walk(full) + else if (entry.name.endsWith('.md')) { + const [meta, body] = parseFrontmatter(fs.readFileSync(full, 'utf-8')) + if (!meta || meta.public !== 'true' || !meta.slug) continue + docs.push({ + slug: meta.slug, + title: meta.title || meta.slug, + markdown: body.trim(), + nav: { + order: Number.parseInt(meta.order || '99', 10) || 99, + section: meta.category || 'guides' + } + }) + } + } +})(DOCS_DIR) + +if (docs.length === 0) { + console.error('❌ DOCS PUSH FAILED: zero public docs collected — refusing to push an empty corpus.') + process.exit(1) +} +docs.sort((a, b) => a.slug.localeCompare(b.slug)) +console.log(`Pushing ${docs.length} public docs to ${BASE}/api/docs/ingest …`) + +let failed = false +for (let i = 0; i < docs.length; i += BATCH) { + const batch = docs.slice(i, i + BATCH) + try { + const res = await fetch(`${BASE}/api/docs/ingest`, { + method: 'POST', + headers: { + 'x-service-secret': SECRET, + 'Content-Type': 'application/json', + 'User-Agent': 'brainy-docs-push/1.0' + }, + body: JSON.stringify({ docs: batch }), + signal: AbortSignal.timeout(120_000) + }) + if (!res.ok) { + throw new Error(`HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`) + } + console.log(` batch ${i / BATCH + 1}: ${batch.map((d) => d.slug).join(', ')} → ok`) + } catch (err) { + failed = true + console.error(` batch ${i / BATCH + 1} FAILED: ${err instanceof Error ? err.message : err}`) + } +} + +if (failed) { + console.error( + '❌ DOCS PUSH INCOMPLETE — soulcraft.com/docs may trail npm. ' + + 'Re-run `node scripts/push-docs.js` or ping venue on VENUE-DOCS-RELEASE-PUSH.' + ) + process.exit(1) +} +console.log('✅ Docs pushed.') diff --git a/scripts/release.sh b/scripts/release.sh index 0e6a9c43..7d860564 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -196,6 +196,18 @@ else fi echo -e "${GREEN}✅ GitHub release created${NC}\n" +# Step 12: Push public docs to the soulcraft.com docs ingest door +# (VENUE-DOCS-RELEASE-PUSH). Skips with a loud warning when +# DOCS_INGEST_SECRET is unset; fails loudly (without undoing the publish — +# that already happened) when a push errors, so the docs site never +# silently trails npm. +echo -e "${BLUE}1️⃣2️⃣ Pushing public docs to soulcraft.com/docs...${NC}" +if node scripts/push-docs.js; then + echo -e "${GREEN}✅ Docs push step done${NC}\n" +else + echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n" +fi + echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}" echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" diff --git a/src/aggregation/AggregationIndex.ts b/src/aggregation/AggregationIndex.ts index bf9dc5de..f9382218 100644 --- a/src/aggregation/AggregationIndex.ts +++ b/src/aggregation/AggregationIndex.ts @@ -29,6 +29,7 @@ import { matchesMetadataFilter } from '../utils/metadataFilter.js' import { compareCodePoints } from '../utils/collation.js' import { bucketTimestamp } from './timeWindows.js' import { NounType } from '../types/graphTypes.js' +import { prodLog } from '../utils/logger.js' /** Persistence key for aggregate definitions */ const DEFINITIONS_KEY = '__aggregation_definitions__' @@ -87,10 +88,18 @@ function matchesSource(entity: Record, source: AggregateDefinit if (entity.service !== source.service) return false } - // Metadata where filter — match against the entity's metadata sub-object + // Where filter — resolve each filtered field through resolveEntityField, + // the SAME single source of truth groupBy uses (top-level standard fields + // + custom metadata). Matching only the metadata sub-object made + // where:{subtype}/{visibility}/… a silent no-op: reserved fields never + // live in the custom bag, so those filters could never match anything. if (source.where && Object.keys(source.where).length > 0) { - const metadata = (entity.metadata ?? entity) as Record - if (!matchesMetadataFilter(metadata, source.where)) return false + const e = entity as unknown as HNSWNounWithMetadata + const resolved: Record = {} + for (const key of Object.keys(source.where)) { + resolved[key] = resolveEntityField(e, key) + } + if (!matchesMetadataFilter(resolved, source.where)) return false } return true @@ -269,7 +278,7 @@ function recomputeMinMaxFromCounts(state: MetricState): void { /** * Exact percentile over a value multiset, using linear interpolation between closest ranks * (numpy 'linear' / type-7): `rank = p·(n−1)`, interpolating between the floor and ceil - * positions of the sorted expansion. Mirrors Cortex's `MetricState::percentile` bit-for-bit. + * positions of the sorted expansion. Mirrors Cor's `MetricState::percentile` bit-for-bit. */ function computePercentile(valueCounts: Record, count: number, p: number): number { if (count === 0) return 0 @@ -327,6 +336,30 @@ export class AggregationIndex { /** Track aggregates with stale MIN/MAX (need lazy recompute) */ private staleMinMax = new Map>() + /** Resolves when init() has finished loading persisted definitions/state. */ + private initPromise: Promise | null = null + + /** True once init() has settled (success or failure). */ + private initDone = false + + /** + * Aggregates registered by the app before init() finished loading persisted + * state, awaiting reconciliation: init() adopts the persisted state when the + * definition hash matches; anything left unadopted when init settles resolves + * to a backfill. Deciding backfill eagerly at define time was the boot-order + * bug that wiped valid persisted state on every restart — the synchronous + * defineAggregate() always beats the async init(). + */ + private pendingAdopt = new Set() + + /** + * In-flight rescan targets. While a name has a staging map, ALL + * contributions (the walk's and concurrent write hooks') land there instead + * of the live map; the live map keeps serving until {@link finishBackfill} + * swaps the staging map in atomically. + */ + private backfillStaging = new Map>() + constructor(storage: StorageAdapter, nativeProvider?: AggregationProvider) { this.storage = storage this.nativeProvider = nativeProvider @@ -336,21 +369,128 @@ export class AggregationIndex { /** * Initialize: load persisted definitions and state, detect changes, rebuild stale. + * + * Idempotent — repeated calls return the same promise. Definitions registered + * *before* this completes (the normal boot order: `defineAggregate()` is + * synchronous and always beats this async load) are reconciled rather than + * clobbered: the app's definition wins, and its persisted state is adopted + * when the definition hash matches — backfill happens only on a real change. */ - async init(): Promise { + init(): Promise { + if (!this.initPromise) { + this.initPromise = this.loadPersisted().finally(() => { + this.resolvePendingAdoptToBackfill() + this.initDone = true + }) + } + return this.initPromise + } + + /** + * Await the persisted-state load (if one was started) and settle every + * pending adoption decision. After this resolves, `getPendingBackfills()` + * is authoritative: a name is listed iff it genuinely needs a rescan. + * Query paths must await this before consulting backfill state. + */ + async ready(): Promise { + if (this.initPromise) { + try { + await this.initPromise + } catch { + // The owner already surfaced the load failure loudly; backfill covers. + } + } + this.resolvePendingAdoptToBackfill() + } + + /** + * Any definition still awaiting state adoption has no persisted state to + * adopt (or init never ran / failed) — it must backfill. + */ + private resolvePendingAdoptToBackfill(): void { + if (this.pendingAdopt.size > 0) { + prodLog.info( + `[Aggregation] no adoptable persisted state for: ${Array.from(this.pendingAdopt).join(', ')} — flagged for backfill` + ) + } + for (const name of this.pendingAdopt) this.needsBackfill.add(name) + this.pendingAdopt.clear() + } + + /** + * May this persisted state be ADOPTED? When the store exposes its committed + * watermark, the state's `sourceGeneration` must EQUAL it: behind means + * later writes are missing from the state (unclean shutdown); ahead means + * it counts writes that no longer exist (e.g. a fact-log truncation on a + * copied store pulled the watermark back). Either way: one exact rescan, + * said out loud — never a silent adopt. Stores without the capability (and + * pre-stamp state on them) fall back to hash-only adoption. + */ + private stateGenerationAdoptable(name: string, stateData: unknown): boolean { + const committed = this.storage.committedGeneration?.() ?? null + if (committed === null) return true + const raw = (stateData as Record).sourceGeneration + const stamped = typeof raw === 'number' ? raw : null + if (stamped === committed) return true + prodLog.warn( + `[Aggregation] '${name}': persisted state is at generation ${stamped ?? 'unstamped'} ` + + `but the store's committed generation is ${committed} — rescanning instead of adopting` + ) + return false + } + + private async loadPersisted(): Promise { // Load persisted definitions const savedDefs = await this.storage.getMetadata(DEFINITIONS_KEY) if (savedDefs && typeof savedDefs === 'object' && savedDefs.definitions) { const defs = savedDefs.definitions as Array for (const def of defs) { - this.definitions.set(def.name, def) - const currentHash = hashDefinition(def) const savedHash = def._hash || '' - // Load persisted state + if (this.definitions.has(def.name)) { + // The app re-registered this aggregate before the load finished. + // The app's definition wins — never clobber it with the persisted + // copy. Adopt the persisted state when the definition is unchanged + // AND no write has landed for it yet (a landed write would be lost + // by adoption; the hook flips such names to backfill). + const appHash = this.definitionHashes.get(def.name) || '' + if (appHash === savedHash && this.pendingAdopt.has(def.name)) { + const stateData = await this.storage.getMetadata(`${STATE_KEY_PREFIX}${def.name}__`) + if ( + stateData && + stateData.groups && + this.stateGenerationAdoptable(def.name, stateData) + ) { + const groupMap = new Map() + for (const group of stateData.groups as AggregateGroupState[]) { + groupMap.set(serializeGroupKey(group.groupKey), group) + } + this.states.set(def.name, groupMap) + this.pendingAdopt.delete(def.name) + this.needsBackfill.delete(def.name) + prodLog.info( + `[Aggregation] '${def.name}': adopted persisted state (${groupMap.size} groups) — no rescan` + ) + } + // No/invalid persisted state: stays in pendingAdopt and resolves + // to backfill when init settles. + } + continue + } + + // Not registered this session — restore definition + state from + // persistence. + this.definitions.set(def.name, def) + const currentHash = hashDefinition(def) + const stateData = await this.storage.getMetadata(`${STATE_KEY_PREFIX}${def.name}__`) - if (stateData && stateData.groups && savedHash === currentHash) { + if ( + stateData && + stateData.groups && + savedHash === currentHash && + this.stateGenerationAdoptable(def.name, stateData) + ) { // Definition unchanged — load state const groupMap = new Map() for (const group of stateData.groups as AggregateGroupState[]) { @@ -358,6 +498,10 @@ export class AggregationIndex { groupMap.set(serialized, group) } this.states.set(def.name, groupMap) + this.needsBackfill.delete(def.name) + prodLog.info( + `[Aggregation] '${def.name}': restored definition + adopted persisted state (${groupMap.size} groups)` + ) } else { // Definition changed or no saved state — start fresh and backfill from // existing entities (the owner drains needsBackfill on first query). @@ -398,14 +542,22 @@ export class AggregationIndex { })) await this.storage.saveMetadata(DEFINITIONS_KEY, { definitions: defsToSave }) - // Persist dirty states + // Persist dirty states, stamped with the committed generation they + // reflect. The stamp is what makes reopen-adoption verifiable: state at a + // different generation than the store's committed watermark is stale (an + // unclean shutdown after later writes) or over-counts (a fact-log + // truncation on a copied store pulled the watermark BACK below the + // stamp) — either way the answer is one exact rescan, never a silent + // adopt. Read the generation after collecting groups so any racing + // commit resolves toward rescan, not wrong-adopt. for (const name of this.dirty) { const stateMap = this.states.get(name) if (stateMap) { const groups = Array.from(stateMap.values()) + const sourceGeneration = this.storage.committedGeneration?.() ?? null await this.storage.saveMetadata( `${STATE_KEY_PREFIX}${name}__`, - { groups } + sourceGeneration === null ? { groups } : { groups, sourceGeneration } ) } } @@ -452,10 +604,19 @@ export class AggregationIndex { this.definitions.set(def.name, def) this.definitionHashes.set(def.name, newHash) + // First sight this session, before init() settled: defer the backfill + // decision — init() adopts the persisted state on hash match, and anything + // left unadopted resolves to backfill. Deciding eagerly here wiped valid + // persisted state on every restart. + if (!this.states.has(def.name) && !this.initDone) { + this.states.set(def.name, new Map()) + this.pendingAdopt.add(def.name) + } // Reset state if definition changed or doesn't exist yet, and flag it for // backfill so already-stored entities are counted (write-time hooks only see // future writes). The owner drains this on the next query via getPendingBackfills(). - if (!this.states.has(def.name) || (oldHash && oldHash !== newHash)) { + else if (!this.states.has(def.name) || (oldHash && oldHash !== newHash)) { + this.pendingAdopt.delete(def.name) this.states.set(def.name, new Map()) this.needsBackfill.add(def.name) } @@ -476,6 +637,8 @@ export class AggregationIndex { this.definitionHashes.delete(name) this.states.delete(name) this.staleMinMax.delete(name) + this.pendingAdopt.delete(name) + this.needsBackfill.delete(name) // Notify native provider if (this.nativeProvider?.removeAggregate) { @@ -513,9 +676,17 @@ export class AggregationIndex { return Array.from(this.needsBackfill) } - /** Clear an aggregate's state so a full rescan cannot double-count. */ + /** + * Begin a rescan into a STAGING map. The live state is not touched — it + * keeps serving (possibly stale, but flagged pending) until the rescan + * completes and swaps in atomically. A mid-walk failure drops the staging + * map via {@link abortBackfill} and loses nothing: wiping live state before + * a scan that could throw was the destructive-before-durable defect. + * Contributions (walk + concurrent write hooks) land in staging while it + * exists, so the swapped-in result reflects writes that raced the walk. + */ beginBackfill(name: string): void { - this.states.set(name, new Map()) + this.backfillStaging.set(name, new Map()) // Reset native provider state for this aggregate too, if present. const def = this.definitions.get(name) if (def && this.nativeProvider?.removeAggregate && this.nativeProvider?.defineAggregate) { @@ -524,6 +695,15 @@ export class AggregationIndex { } } + /** + * Abandon an in-flight rescan after a failure: drop the staging map, keep + * the live state serving, leave the aggregate flagged as pending so a later + * attempt rescans. The failure itself must be surfaced loudly by the owner. + */ + abortBackfill(name: string): void { + this.backfillStaging.delete(name) + } + /** Feed one already-stored entity into a single aggregate during backfill. */ backfillEntity(name: string, entity: Record): void { if (isAggregateEntity(entity)) return @@ -537,14 +717,33 @@ export class AggregationIndex { } } - /** Mark an aggregate's backfill complete; rebuilt state persists on next flush(). */ + /** Swap the rebuilt staging state in atomically; persists on next flush(). */ finishBackfill(name: string): void { + const staged = this.backfillStaging.get(name) + if (staged) { + this.states.set(name, staged) + this.backfillStaging.delete(name) + } this.needsBackfill.delete(name) this.dirty.add(name) } // ============= Write-Time Hooks ============= + /** + * A write is landing for an aggregate whose persisted-state adoption is still + * pending — adopting after this write would lose its contribution. Settle the + * decision now: an exact rescan instead of adoption. The window is the few + * milliseconds between a boot-time defineAggregate() and init() completing, + * so this rarely fires; when it does, correctness wins over the walk. + */ + private resolveAdoptOnWrite(name: string): void { + if (this.pendingAdopt.has(name)) { + this.pendingAdopt.delete(name) + this.needsBackfill.add(name) + } + } + /** * Called when an entity is added. Updates all matching aggregates. */ @@ -553,6 +752,7 @@ export class AggregationIndex { for (const [name, def] of this.definitions) { if (!matchesSource(entity, def.source)) continue + this.resolveAdoptOnWrite(name) if (this.nativeProvider) { const results = this.nativeProvider.incrementalUpdate(name, def, entity, 'add') @@ -579,6 +779,10 @@ export class AggregationIndex { const oldMatches = matchesSource(oldEntity, def.source) const newMatches = matchesSource(newEntity, def.source) + if (oldMatches || newMatches) { + this.resolveAdoptOnWrite(name) + } + if (this.nativeProvider && (oldMatches || newMatches)) { const results = this.nativeProvider.incrementalUpdate(name, def, newEntity, 'update', oldEntity) this.applyNativeResults(name, results) @@ -605,6 +809,7 @@ export class AggregationIndex { for (const [name, def] of this.definitions) { if (!matchesSource(entity, def.source)) continue + this.resolveAdoptOnWrite(name) if (this.nativeProvider) { const results = this.nativeProvider.incrementalUpdate(name, def, entity, 'delete') @@ -757,7 +962,7 @@ export class AggregationIndex { def: AggregateDefinition, entity: Record ): void { - const stateMap = this.states.get(aggName)! + const stateMap = (this.backfillStaging.get(aggName) ?? this.states.get(aggName))! // Fan out: an unnest dimension makes one entity contribute to several groups. for (const groupKey of computeGroupKeys(entity, def.groupBy)) { @@ -815,7 +1020,7 @@ export class AggregationIndex { def: AggregateDefinition, entity: Record ): void { - const stateMap = this.states.get(aggName)! + const stateMap = (this.backfillStaging.get(aggName) ?? this.states.get(aggName))! // Fan out: reverse the entity's contribution from every group it joined. for (const groupKey of computeGroupKeys(entity, def.groupBy)) { @@ -871,7 +1076,7 @@ export class AggregationIndex { * Apply results from native provider back into the state maps. */ private applyNativeResults(aggName: string, results: AggregateGroupState[]): void { - const stateMap = this.states.get(aggName)! + const stateMap = (this.backfillStaging.get(aggName) ?? this.states.get(aggName))! for (const group of results) { const serialized = serializeGroupKey(group.groupKey) stateMap.set(serialized, group) diff --git a/src/aggregation/materializer.ts b/src/aggregation/materializer.ts index 3a8dcb3f..f752648a 100644 --- a/src/aggregation/materializer.ts +++ b/src/aggregation/materializer.ts @@ -14,6 +14,7 @@ import type { AggregateMetricDef } from '../types/brainy.types.js' import { serializeGroupKey } from './AggregationIndex.js' +import { prodLog } from '../utils/logger.js' /** * Callback interface for the materializer to create/update Brainy entities. @@ -86,8 +87,15 @@ export class AggregateMaterializer { if (existing) clearTimeout(existing) this.debounceTimers.set(key, setTimeout(() => { - this.materializeOne(key).catch(() => { - // Non-fatal — materialization is derived data + this.materializeOne(key).catch((err) => { + // Materialization is derived data (rebuildable via backfill-on-query), so + // a failure is non-fatal — but it leaves the materialized Measurement + // entity STALE. Surface it loudly rather than swallow it silently. + prodLog.warn( + `[Aggregation] Failed to materialize aggregate group '${key}': ` + + `${(err as Error).message}. The materialized value is stale until the ` + + `next successful materialization or a backfill-on-query.` + ) }) }, debounceMs)) } diff --git a/src/brainy.ts b/src/brainy.ts index 2f6a5293..9ab3acee 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -1,8 +1,10 @@ /** - * 🧠 Brainy 3.0 - The Future of Neural Databases - * - * Beautiful, Professional, Planet-Scale, Fun to Use - * NO STUBS, NO MOCKS, REAL IMPLEMENTATION + * @module brainy + * @description The `Brainy` class — the Triple-Intelligence database that unifies + * vector similarity, graph traversal, and metadata filtering behind a single API + * (`add`/`find`/`relate`/`related`/`get`/`similar`/`graph.*`/`now`/`asOf`/`transact`/ + * `export`/`import`). Native acceleration is feature-detected through the provider + * boundary; when no provider is registered, the built-in JS engines serve everything. */ import { v4 as uuidv4, v7 as uuidv7 } from './universal/uuid.js' @@ -15,6 +17,13 @@ import type { StorageOptions } from './storage/storageFactory.js' import { rebuildCounts } from './utils/rebuildCounts.js' import type { MetadataWriteBuffer } from './utils/metadataWriteBuffer.js' import { BaseStorage } from './storage/baseStorage.js' +import { + CURRENT_DATA_FORMAT, + EXPECTED_INDEX_EPOCH, + readBrainFormat, + writeBrainFormat +} from './storage/brainFormat.js' +import type { BrainFormat } from './storage/brainFormat.js' import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb, STANDARD_ENTITY_FIELDS } from './coreTypes.js' import type { HNSWNounWithMetadata, HNSWVerbWithMetadata, EntityVisibility } from './coreTypes.js' import { @@ -23,7 +32,7 @@ import { getBrainyVersion } from './utils/index.js' import { embeddingManager } from './embeddings/EmbeddingManager.js' -import { matchesMetadataFilter } from './utils/metadataFilter.js' +import { matchesMetadataFilter, validateWhereFilter } from './utils/metadataFilter.js' import { NaturalLanguageProcessor } from './neural/naturalLanguageProcessor.js' import { NeuralEntityExtractor, ExtractedEntity } from './neural/entityExtractor.js' import { TripleIntelligenceSystem } from './triple/TripleIntelligenceSystem.js' @@ -31,12 +40,31 @@ import { VirtualFileSystem } from './vfs/VirtualFileSystem.js' import { MetadataIndexManager } from './utils/metadataIndex.js' import { detectContentType, extractForHighlighting } from './utils/contentExtractor.js' import { GraphAdjacencyIndex } from './graph/graphAdjacencyIndex.js' +import { + connectedComponents, + stronglyConnectedComponents, + pageRank, + MinHeap +} from './graph/analyticsFallback.js' +import { runGraphAudit, type GraphAuditReport } from './graph/graphAudit.js' import { createPipeline } from './streaming/pipeline.js' import { configureLogger, LogLevel, prodLog } from './utils/logger.js' +import { warnOnLowOsLimits } from './utils/osLimits.js' import { setGlobalCache } from './utils/unifiedCache.js' import type { UnifiedCache } from './utils/unifiedCache.js' import { rankIndicesByScore, reorderByIndices } from './utils/resultRanking.js' -import { PluginRegistry, isVersionedIndexProvider } from './plugin.js' +import { PluginRegistry, isVersionedIndexProvider, isGraphAccelerationProvider } from './plugin.js' +import type { + GraphAccelerationProvider, + TraverseOptions, + Subgraph, + RankOptions, + CommunitiesOptions, + PathOptions, + MetadataIndexProvider, + OpaqueIdSet, + AtGenerationVectors +} from './plugin.js' import type { BrainyPlugin, BrainyPluginContext, @@ -44,6 +72,7 @@ import type { } from './plugin.js' import { ConnectionsCodec } from './hnsw/connectionsCodec.js' import { TransactionManager } from './transaction/TransactionManager.js' +import { transactTimeoutBudget } from './transaction/Transaction.js' import { RevisionConflictError } from './transaction/RevisionConflictError.js' import { EntityNotFoundError, RelationNotFoundError } from './errors/notFound.js' import { @@ -88,6 +117,18 @@ import { FindParams, SimilarParams, RelatedParams, + GraphApi, + GraphView, + GraphNode, + SubgraphOptions, + SubgraphSelector, + GraphExportOptions, + GraphRankOptions, + GraphRankEntry, + GraphCommunitiesOptions, + GraphCommunitiesResult, + GraphPathOptions, + GraphPathResult, GetOptions, AddManyParams, RemoveManyParams, @@ -113,9 +154,12 @@ import type { MigrationPreview, MigrationResult, MigrateOptions } from './migrat import { AggregationIndex } from './aggregation/AggregationIndex.js' import { AggregateMaterializer } from './aggregation/materializer.js' import type { AggregateDefinition, AggregateQueryParams, AggregateResult } from './types/brainy.types.js' +import type { MigrationProgress } from './types/brainy.types.js' import { resolveJsHnswConfig, DEFAULT_RECALL } from './utils/recallPreset.js' import * as fs from 'node:fs' +import * as os from 'node:os' import { Db, type DbHost, type HistoricalQueryHandle } from './db/db.js' +import { entityMatchesFind } from './db/whereMatcher.js' import { importGraph, isPortableGraph, @@ -125,13 +169,30 @@ import { type ImportOptions, type ImportResult } from './db/portableGraph.js' -import { GenerationStore } from './db/generationStore.js' +import { GenerationStore, type CommitBeforeImages } from './db/generationStore.js' +import type { FactScanHandle } from './db/factLog.js' +import { + ENTITY_TREE_STAMP_PATH, + readFamilyStamp, + verifyFamilyStamp, + writeFamilyStamp, + type FamilyStamp +} from './db/familyStamp.js' +import { + ChangeFeed, + type BrainyChangeEvent, + type ChangeListener, + type PendingChangeEvent +} from './events/changeFeed.js' import { isDeterministicEmbedMode } from './embeddings/deterministicEmbedMode.js' -import { GenerationConflictError } from './db/errors.js' +import { GenerationConflictError, StoreInconsistentError } from './db/errors.js' +import { BrainyError, GraphIndexNotReadyError, MetadataIndexNotReadyError, MigrationInProgressError, VectorIndexNotReadyError } from './errors/brainyError.js' +import { assessIndexReadiness } from './utils/indexReadiness.js' import { MemoryStorage } from './storage/adapters/memoryStorage.js' import type { CompactHistoryOptions, CompactHistoryResult, + HistoryStats, TransactOptions, TransactReceipt, TxLogEntry, @@ -141,8 +202,8 @@ import type { HistoryVersion } from './db/types.js' import { stableDeepEqual } from './db/stableEqual.js' -import type { VersionedIndexProvider } from './plugin.js' -import type { Operation } from './transaction/types.js' +import type { VersionedIndexProvider, ProviderInvariantReport } from './plugin.js' +import type { Operation, TransactionFunction } from './transaction/types.js' /** * Stopwords for semantic highlighting @@ -202,8 +263,9 @@ type ResolvedBrainyConfig = Required< | 'vector' | 'plugins' | 'integrations' - | 'history' + | 'retention' | 'eagerEmbeddings' + | 'migrationWaitTimeoutMs' > > & Pick< @@ -213,8 +275,9 @@ type ResolvedBrainyConfig = Required< | 'vector' | 'plugins' | 'integrations' - | 'history' + | 'retention' | 'eagerEmbeddings' + | 'migrationWaitTimeoutMs' > /** @@ -270,8 +333,78 @@ interface PlannedTransact { touchedVerbs: string[] /** Aggregation-index hooks to run after the commit point. */ postCommit: Array<() => void> + /** + * One entry per staged `{ op: 'update' }`, re-verified UNDER the commit + * mutex (the generation store's `precommit`) so per-op `ifRev` CAS is + * atomic with the apply — planning-time checks can interleave with + * concurrent writers. `updatedMetadata` is the staged metadata object (held + * by reference by the staged operation), re-stamped there with the + * authoritative `_rev`. A conflict rejects the WHOLE batch. + */ + casUpdates: Array<{ + id: string + ifRev?: number + updatedMetadata: { _rev?: number } + }> + /** + * Ids whose revision baseline THIS batch resets via an `add` op (a fresh + * create, or add's overwrite semantics which restamp `_rev: 1`). Updates to + * these ids sequence against in-batch state no concurrent writer can touch, + * so their plan-time CAS checks and rev stamps are already exact — the + * commit precondition leaves them untouched. + */ + createdNouns: Set + /** + * Change-feed events, one per affected record in op order, populated by the + * planners ONLY when a listener is subscribed. Stamped with the batch's + * committed generation and emitted after `commitTransaction` returns — a + * rejected batch (CAS conflict, failed apply) emits nothing. + */ + changeEvents: PendingChangeEvent[] } +/** + * Internal control-flow signal: an insert guarded by a must-be-absent + * precondition (`add({ ifAbsent })` / `add({ upsert })`) found the entity + * already present in the authoritative before-image under the commit mutex — + * a concurrent writer created it after the planning-time absence check. + * Never escapes `add()`: the caller converts it into the documented + * resolution (ifAbsent → return the existing id without writing; upsert → + * merge into the now-existing entity via `update()`). + */ +class InsertPreconditionExistsSignal extends Error { + constructor(readonly id: string) { + super(`insert precondition: entity ${id} already exists`) + this.name = 'InsertPreconditionExistsSignal' + } +} + +/** + * @description The derived-index families a read may depend on. A read that + * consults none of them (a canonical-storage read: `get`, an entity + * enumeration, a VFS content/dir read) is index-independent and must never + * block on another family's one-time migration. Used by the family-scoped + * migration gate ({@link Brainy.awaitMigrationLock}). + */ +export type IndexFamily = 'vector' | 'metadata' | 'graph' + +/** + * How long a failed aggregation-backfill walk suppresses fresh walk attempts. + * Within the window, queries rethrow the recorded failure instantly (loud, + * cheap); after it, one new attempt is allowed. Bounds the damage of a + * caller-side tight retry loop against a deterministically-failing store. + */ +const AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS = 30_000 + +/** + * Time budget for the auto-compaction pass at close() (8.9.0). Bounds how long + * a clean shutdown spends reclaiming history backlog — an early stop is a + * consistent prefix and the next close/explicit pass resumes. Explicit + * `compactHistory()` calls are unbounded unless the caller passes their own + * `timeBudgetMs` (maintenance windows choose their own budgets). + */ +const CLOSE_COMPACTION_BUDGET_MS = 5_000 + /** * The main Brainy class - Clean, Beautiful, Powerful * REAL IMPLEMENTATION - No stubs, no mocks @@ -283,6 +416,23 @@ export class Brainy implements BrainyInterface { private static shutdownHooksRegisteredGlobally = false private static instances: Brainy[] = [] + /** The globally-registered shutdown listeners, kept so the LAST close() can + * deregister them. A process.on('SIGINT'/'SIGTERM') listener holds a ref'd + * signal handle that keeps the Node event loop alive — a library that never + * removes its listeners makes every bare script hang after close(). + * See {@link registerShutdownHooks} / {@link deregisterShutdownHooksIfIdle}. */ + private static sigtermListener?: () => void + private static sigintListener?: () => void + private static beforeExitListener?: () => void + + /** Poll cadence (ms) for the migration LOCK when a provider exposes no + * event-driven `whenMigrationComplete()` signal. See {@link awaitMigrationLock}. */ + private static readonly MIGRATION_POLL_INTERVAL_MS = 250 + + /** First-party accelerator packages probed by guarded auto-detection when + * `plugins` is undefined (installing one IS the opt-in). See {@link loadPlugins}. */ + private static readonly AUTO_DETECT_PLUGIN_PACKAGES = ['@soulcraft/cor'] + // Core components private index!: JsHnswVectorIndex private storage!: BaseStorage @@ -336,6 +486,13 @@ export class Brainy implements BrainyInterface { /** Cap for {@link Brainy.verbIntWarmCache} (~100k entries ≈ a few MB). */ private static readonly VERB_INT_WARM_CACHE_MAX = 100_000 + /** + * Default cap on how many `find()` matches seed a `graph.subgraph(query)` when the + * caller pins no explicit `limit` — bounds the materialized seed set on the JS / + * non-opaque path (the native opaque path forwards the whole universe, uncapped). + */ + private static readonly QUERY_SEED_CAP = 10_000 + // Silent mode state private originalConsole?: { log: typeof console.log @@ -346,6 +503,95 @@ export class Brainy implements BrainyInterface { // Plugin system private pluginRegistry = new PluginRegistry() + /** + * Resolved native graph-acceleration provider, cached on first `brain.graph.*` + * access. `undefined` = not yet resolved; `null` = resolved, none registered + * (use the TS fallback); otherwise the provider instance. + */ + private _graphAccelProvider: GraphAccelerationProvider | null | undefined = undefined + /** + * Set when a non-fatal index rebuild fails at init() (the brain is allowed to + * start, but in a degraded state where queries may be incomplete). Surfaced + * through {@link checkHealth} so consumers can observe the failure + * programmatically rather than only via the console. A storage *read* failure + * during rebuild is NOT recorded here — it re-throws and aborts init() loudly. + */ + private _indexRebuildFailed: Error | null = null + /** + * Ids of records that committed via an adopt-forward failed-rollback recovery + * ({@link GenerationStore.commitSingleOp} `degraded`): the canonical record is + * durable but its derived index entry may be incomplete until + * {@link repairIndex}. Non-empty = a queryable degraded state, folded into + * {@link checkHealth}/{@link validateIndexConsistency} and warned on the read + * paths. Cleared by {@link repairIndex}. + */ + private _indexDegradedIds: Set = new Set() + /** One-shot guard so the degraded-reads warning fires once per degraded window + * (reset when the degraded state clears). See {@link warnIfReadsDegraded}. */ + private _degradedReadWarned = false + /** One-shot guard so the metadata cold-open consistency probe runs once per brain. */ + private _metadataConsistencyProbed = false + /** Graph-adjacency cold-load consistency: verified-live this session (one-shot). */ + private _graphAdjacencyVerified = false + /** Re-entrancy guard: a verify (rebuild → reads) is in flight. */ + private _graphAdjacencyVerifying = false + /** Metadata field-index cold-read guard: verified-serving this session (one-shot). */ + private _metadataVerified = false + /** Re-entrancy guard for {@link verifyMetadataLive}. */ + private _metadataVerifying = false + /** Vector-index cold-read guard: verified-serving this session (one-shot). */ + private _vectorVerified = false + /** Re-entrancy guard for {@link verifyVectorLive}. */ + private _vectorVerifying = false + /** + * Coordinated migration LOCK (#18): dedup guards so the "upgrading, blocking" + * and "upgrade complete, resumed" lines each log once per migration window, + * not once per blocked operation. See {@link awaitMigrationLock}. + */ + private _migrationBlockLogged = false + private _migrationReleaseLogged = false + /** Epoch ms when brainy first observed the migration LOCK held, or `null` when + * not migrating — the fallback `elapsedMs` a provider that omits `migrationStatus()` + * timing still gets in `getIndexStatus().migration`. See {@link migrationSnapshot}. */ + private _migrationObservedAt: number | null = null + /** Path of the pre-upgrade backup taken this open (default-on, opt-out via + * `migrationBackup: false`), or `null` if none. Removed once the 7.x→8.0 + * upgrade verifies + stamps; retained on failure. See {@link createMigrationBackupIfNeeded}. */ + private _migrationBackupPath: string | null = null + + /** + * The in-process change feed behind {@link onChange}. Emitted from the + * commit seam ({@link persistSingleOp} / {@link transact}), so every + * canonical mutation — any origin — produces exactly one post-commit event + * per affected record. See src/events/changeFeed.ts for the delivery + * contract. + */ + private readonly _changeFeed = new ChangeFeed() + + /** Set when the on-open VFS-blob adoption left one or more `_cow/` blobs it + * could not fully adopt (bytes or metadata missing). While true, the + * pre-upgrade backup is NOT auto-removed — the upgrade is not verifiably + * complete for the VFS. See {@link autoAdoptLegacyVfsBlobsIfNeeded}. */ + private _vfsBlobAdoptionIncomplete = false + + /** + * 8.0 ⇄ native-provider version handshake — the on-disk {@link BrainFormat} + * marker (`_system/brain-format.json`) as read during the store-open phase, + * or `null` for a pre-handshake / brand-new brain. Populated BEFORE any + * derived index or native provider is constructed, so {@link formatInfo} + * answers synchronously when the provider reads it at its own init(). + * Refreshed to the current marker once it is (re)stamped. + */ + private _brainFormat: BrainFormat | null = null + /** + * True when the on-disk {@link _brainFormat} is absent OR carries an + * `indexEpoch` different from {@link EXPECTED_INDEX_EPOCH} — i.e. the derived + * JS indexes on disk predate this build and must be rebuilt from the + * canonical records (the epoch-drift rebuild trigger that closes the gap + * where JS indexes rebuilt only on `size()===0`, never on a format change). + * Cleared once the rebuild verifies and the marker is re-stamped. + */ + private _indexEpochStale = false // Sub-APIs (lazy-loaded) private _nlp?: NaturalLanguageProcessor @@ -353,9 +599,31 @@ export class Brainy implements BrainyInterface { private _tripleIntelligence?: TripleIntelligenceSystem private _vfs?: VirtualFileSystem private _vfsInitialized = false // Track VFS init completion separately + /** + * 8.0 MVCC: single-op writes create generations (Model-B) only AFTER init + * completes. Init-time infrastructure writes (the VFS root directory, etc.) + * form the generation-0 baseline of a freshly-materialized brain and are NOT + * versioned — so a brand-new brain reports `generation() === 0` / empty + * `transactionLog()`, and the first USER write is generation 1. Enabled at + * the tail of `init()`; see {@link persistSingleOp}. + */ + private _generationStampingActive = false + /** + * 8.0 MVCC: the coordinator-driven adaptive history byte budget for this + * brain (set via {@link setRetentionBudget}; e.g. cor's `ResourceManager` + * fair-shares one box-wide budget across co-located instances). Overrides the + * local `os.freemem` probe while `retention` is adaptive. `undefined` = use + * the probe. See {@link adaptiveHistoryBudgetBytes}. + */ + private _retentionBudgetBytes?: number private _hub?: IntegrationHub // Integration Hub for external tools private _pendingMigrationRunner?: MigrationRunner // Deferred migration runner for large datasets private _aggregationIndex?: AggregationIndex // Incremental aggregation engine + private _aggregationBackfillFlight: Promise | null = null // Single-flight backfill walk + // A failed walk latches its error: retries within the cooldown rethrow it + // instantly instead of re-walking, so a tight caller-side retry loop costs + // one loud error per query, never a full store walk per query. + private _aggregationBackfillFailure: { at: number; error: Error } | null = null private _materializer?: AggregateMaterializer // Debounced materialization of aggregate results /** * Fields registered via `brain.trackField()` — drives optional value validation on @@ -385,6 +653,13 @@ export class Brainy implements BrainyInterface { // set this to ReaderMode so historical instances are protected the same way. private operationalMode: BaseOperationalMode + // Write-quarantine after a failed transaction rollback left the store + // inconsistent (see StoreInconsistentError). Set at the moment the failure is + // surfaced; every subsequent mutation is refused via assertWritable until + // repairIndex() reconciles canonical vs derived state and clears it. Reads are + // never affected. null = healthy. + private storeInconsistency: StoreInconsistentError | null = null + // Ready Promise state (Unified readiness API) // Allows consumers to await brain.ready for initialization completion private _readyPromise: Promise | null = null @@ -506,13 +781,13 @@ export class Brainy implements BrainyInterface { * Whether the active storage adapter (anywhere in its prototype chain) * implements a given optional method. * - * Storage-adapter plugins (e.g. `@soulcraft/cortex`'s `MmapFileSystemStorage + * Storage-adapter plugins (e.g. `@soulcraft/cor`'s `MmapFileSystemStorage * extends FileSystemStorage`) inherit new methods Brainy adds to * `FileSystemStorage` / `BaseStorage` automatically — `typeof` walks the * prototype chain, so there's no in-package version skew to worry about as * long as the plugin's own dist resolves `@soulcraft/brainy` dynamically * (which Cortex 2.2.x onward does — see - * `node_modules/@soulcraft/cortex/dist/storage/mmapFileSystemStorage.js`). + * `node_modules/@soulcraft/cor/dist/storage/mmapFileSystemStorage.js`). * * This helper exists for the **build/install** failure modes the import * resolution can't catch: @@ -550,6 +825,17 @@ export class Brainy implements BrainyInterface { `Open in writer mode to modify data.` ) } + // Write-quarantine: a prior transaction's rollback failed and left the store + // inconsistent. Refuse further writes (which would compound the damage) until + // repairIndex() reconciles and lifts the quarantine. Reads still work. + if (this.storeInconsistency) { + throw new Error( + `Cannot call ${method}() — the store is WRITE-QUARANTINED after a failed ` + + `transaction rollback left it inconsistent. Reads still work; run repairIndex() ` + + `to reconcile the derived indexes against canonical storage and lift the ` + + `quarantine. Original inconsistency: ${this.storeInconsistency.message}` + ) + } } /** @@ -647,7 +933,7 @@ export class Brainy implements BrainyInterface { try { // Auto-detect and activate plugins BEFORE storage setup - // so plugin-provided storage factories (e.g., filesystem override from cortex) are available + // so plugin-provided storage factories (e.g., filesystem override from cor) are available await this.loadPlugins() // 7.x → 8.0 layout migration, BEFORE storage setup: collapse a legacy @@ -660,11 +946,18 @@ export class Brainy implements BrainyInterface { this.storage = await this.setupStorage() await this.storage.init() + // OS-limit detection (once per process, Linux-only, measurement-only): + // warn NOW about RLIMIT_NOFILE / vm.max_map_count values that will bite + // at pool scale, instead of letting the operator meet them as EMFILE or + // a failed mmap deep inside an index open. Fire-and-forget — the check + // never affects open. + void warnOnLowOsLimits() + // Acquire the writer lock for filesystem (and other locking-capable) backends. // Skipped in reader mode and on backends that don't support multi-process locking. // Throws if another live writer holds the directory (unless force: true). // - // Defensive call: older storage adapters (e.g. `@soulcraft/cortex@2.2.0` + // Defensive call: older storage adapters (e.g. `@soulcraft/cor@2.2.0` // and earlier) bundle a pre-7.21 `BaseStorage` that doesn't define these // methods. We feature-detect each one rather than fail boot. if (this.config.mode !== 'reader') { @@ -721,6 +1014,68 @@ export class Brainy implements BrainyInterface { readOnly: this.config.mode === 'reader' }) + // The generation fact log is CANONICAL state, not a derived index — no + // sweeper, GC, or blob-lifecycle path may ever delete under it. Declare + // its namespace as a protected family (rebuildable: false — a lost fact + // segment is NOT reconstructable) so the storage layer REFUSES such + // deletes; refusal beats trust. Feature-detected + idempotent per name. + if ( + this.config.mode !== 'reader' && + this.generationStore.getFactLog() && + typeof this.storage.registerDerivedFamily === 'function' + ) { + await this.storage.registerDerivedFamily({ + name: 'generation-facts', + members: ['_generations/facts/'], + namespace: true, + rebuildable: false + }) + } + + // Fact-scan capability: wire the storage seam through which index + // providers (which hold only `storage`) reach the fact log. A closure + // over the LIVE log — restore/reopen swaps the instance transparently — + // so a provider's heal can switch from the enumeration walk to one + // sequential fact scan whenever the log exists. + if (typeof (this.storage as BaseStorage).setFactScanSource === 'function') { + ;(this.storage as BaseStorage).setFactScanSource({ + factLog: () => this.generationStore?.getFactLog() ?? null, + // The committed watermark, exposed as a capability so providers + // never parse the store's private manifest format. + committedGeneration: () => this.generationStore?.committedGeneration() ?? 0 + }) + } + + // Entity-tree stamp coherence: compare the stamped sourceGeneration + + // rollup invariants against the log head + live counters. Loud on + // genuine incoherence (repairIndex heals), silent on absent/coherent, + // benign-behind refreshes at the next flush. Never blocks open. + await this.verifyEntityTreeStamp() + + // 8.0 ⇄ native-provider version handshake: load the on-disk brain-format + // marker (`_system/brain-format.json`) into an in-memory field NOW — + // after the store-open phase, but BEFORE any derived index or native + // provider is constructed below — so `formatInfo()` is populated when the + // provider reads it synchronously at its own init(). A drifted or absent + // `indexEpoch` means the on-disk derived-index format predates this build + // (the derived JS indexes are stale); `rebuildIndexesIfNeeded()` rebuilds + // them from the canonical records and then re-stamps the marker AFTER the + // rebuild verifies (non-destructive: a crash mid-rebuild leaves the old / + // absent marker, so the next open idempotently re-rebuilds). + this._brainFormat = await readBrainFormat(this.storage) + this._indexEpochStale = + this._brainFormat === null || this._brainFormat.indexEpoch !== EXPECTED_INDEX_EPOCH + + // Pre-upgrade backup (default-on, opt-out via `migrationBackup: false`): the + // on-disk format is stale, so a one-time 7.x → 8.0 rebuild will run below. + // Snapshot the brain dir NOW — before any provider (native or JS) rebuilds a + // derived index — so a migration bug can be rolled back. Removed once the + // upgrade verifies + stamps; retained on failure. No-op for a reader, for + // non-filesystem storage, or for a brain with no persisted data. + if (this._indexEpochStale && this.config.migrationBackup && !this.isReadOnly) { + await this.createMigrationBackupIfNeeded() + } + // Provider: embeddings (reassign embedder if plugin provides one) const embeddingProvider = this.pluginRegistry.getProvider('embeddings') if (embeddingProvider) { @@ -747,7 +1102,7 @@ export class Brainy implements BrainyInterface { setMsgpackImplementation(msgpackProvider) } - // Provider: sort:topK (e.g. cortex's native partial-sort / heap-select) — swaps the + // Provider: sort:topK (e.g. cor's native partial-sort / heap-select) — swaps the // JS result-ranking used by find() to pick the top `offset + limit` rows. The provider // returns indices into a scores array, ordered descending with stable ties, identical // to the JS sortTopKIndicesJs. rankIndicesByScore validates the provider's output and @@ -774,7 +1129,7 @@ export class Brainy implements BrainyInterface { if (metadataFactory) { this.metadataIndex = metadataFactory(this.storage) } else { - // JS fallback — inject native EntityIdMapper if cortex provides one + // JS fallback — inject native EntityIdMapper if cor provides one const entityIdMapperFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('entityIdMapper') this.metadataIndex = new MetadataIndexManager(this.storage, {}, { entityIdMapper: entityIdMapperFactory ? entityIdMapperFactory(this.storage) : undefined, @@ -795,6 +1150,26 @@ export class Brainy implements BrainyInterface { this.graphIndex = graphIndex } + // Eager cold-load (readiness contract). A provider that persists its + // derived state exposes init?(): trigger the load NOW — AFTER + // metadataIndex.init() above (the id-mapper is hydrated first, so a + // native int-keyed index resolves endpoints/slots through it — the + // CTX-BR-RESTORE-REBUILD order) and BEFORE the rebuild gate — so a + // durable index reports its real size()/isReady() at the gate instead + // of eating a spurious rebuild-from-canonical on every open (§7.1 + // rebuild()==0). Vector first, then graph. JS engines: the JS vector + // index has no init() (rebuild() IS its load path); the JS graph's + // init() already cold-loaded its LSM inside storage.getGraphIndex() + // (idempotent here). + const vectorWithInit = this.index as { init?: () => Promise } + if (typeof vectorWithInit.init === 'function') { + await vectorWithInit.init() + } + const graphWithInit = this.graphIndex as { init?: () => Promise } + if (typeof graphWithInit.init === 'function') { + await graphWithInit.init() + } + // 8.0 u64 contract: thread the shared UUID ↔ int resolver into the JS // graph index and the storage layer's verb read paths. this.wireGraphIdResolver() @@ -839,9 +1214,46 @@ export class Brainy implements BrainyInterface { `(storage committed: ${committed}) — provider replays the gap per ` + `the post-commit applier contract` ) + } else if (providerGen > committed) { + // The AHEAD direction is incoherence, not a replay gap: the provider's + // persisted index claims writes the store no longer has — the signature + // of a torn copy or a log truncation that pulled the committed + // watermark back (crash recovery, byte-copy of a live store). A replay + // can never converge on it and index answers may reference vanished + // writes. Name it loudly at open so it is never diagnosed from a + // silent journal; the provider's own coherence check / heal walk (or + // brain.repairIndex()) is the cure. + prodLog.warn( + `[Brainy] Versioned index provider is AHEAD of the store: provider ` + + `generation ${providerGen} vs committed ${committed}. This store was ` + + `likely copied from a live service or truncated during crash recovery. ` + + `Derived-index answers may reference rolled-back writes until the ` + + `provider heals from canonical (brain.repairIndex() forces it).` + ) } } + // Recover VFS content blobs a 7→8 upgrade left stranded in the removed + // branch system's copy-on-write area (`_cow/`). Runs BEFORE the rebuild + // (whose success stamp removes the pre-upgrade backup) so the backup still + // covers this step, and works on the raw object primitives (no blob store + // needed yet). A cheap no-op on native-8.0 / fresh brains and on a brain + // already healed. See adoptLegacyCowBlobs. + await this.autoAdoptLegacyVfsBlobsIfNeeded() + + // Temporal-blob contract: one-time (marker-gated) backfill of blob + // history reference counts for stores whose generation history predates + // the contract — after it, compaction reclaims blob bytes exactly (zero + // live AND zero history references). On a scrub failure the store runs + // leak-safe (no blob reclamation) rather than risk a premature delete. + if (typeof (this.storage as unknown as { + backfillBlobHistoryRefCountsIfNeeded?: () => Promise + }).backfillBlobHistoryRefCountsIfNeeded === 'function') { + await (this.storage as unknown as { + backfillBlobHistoryRefCountsIfNeeded: () => Promise + }).backfillBlobHistoryRefCountsIfNeeded() + } + // Rebuild indexes if needed for existing data await this.rebuildIndexesIfNeeded() @@ -875,18 +1287,47 @@ export class Brainy implements BrainyInterface { } else { console.log(`[brainy] Providers: ${native.length}/${wellKnownKeys.length} native (${plugins}) | default: ${fallback.join(', ')}`) } + // An activated accelerator that registered NOTHING is running as pure + // decoration — every query silently serves from the JS engines. Most + // often a licensing gate declining to engage. Say so, loudly, so an + // evaluator never concludes the accelerator "does nothing". + if (native.length === 0) { + console.warn( + `[brainy] ⚠ ${plugins} activated but registered 0 native providers — all queries are ` + + `running on the default JS engines. Check the plugin's requirements (e.g. a license ` + + `key) and its logs; see docs/PLUGINS.md.` + ) + } } // Mark as initialized BEFORE VFS init // VFS.init() needs brain to be marked initialized to call brain methods this.initialized = true + // Migration LOCK (#18): if a native provider is running the one-time + // 7.x → 8.0 rebuild, WAIT for it to finish HERE — before VFS bootstrap and + // before the brain serves — so init-internal reads/writes (the VFS root + // get/add/find below) run against the rebuilt indexes, never a half-built + // one. This is the "not ready until upgraded" contract: a blocking upgrade + // to a known-good state. Bounded by `migrationWaitTimeoutMs`: a brain whose + // upgrade outlives the budget surfaces MigrationInProgressError here (raise + // the budget, or run the offline migrator) instead of bootstrapping the VFS + // against a partial index. A non-migrating brain returns instantly. + await this.awaitMigrationLock() + // Initialize VFS: Ensure VFS is ready when accessed as property // This eliminates need for separate vfs.init() calls - zero additional complexity this._vfs = new VirtualFileSystem(this) await this._vfs.init() this._vfsInitialized = true // Mark VFS as fully initialized + // 8.0 MVCC: infrastructure bootstrap (VFS root, etc.) is now the + // generation-0 baseline. From here, every single-op write is a Model-B + // generation. (Writers only — readers never stamp; a no-op for them.) + if (!this.isReadOnly) { + this._generationStampingActive = true + } + // Eager embedding initialization. // // Adaptive default (8.0): the WASM embedding engine eagerly initializes @@ -950,6 +1391,13 @@ export class Brainy implements BrainyInterface { if (this._readyReject) { this._readyReject(error instanceof Error ? error : new Error(String(error))) } + // Machine-readable init failures pass through UNWRAPPED — the writer-lock + // conflict documents an err.code/err.lockInfo contract ("callers detect + // this case via err.code"), and wrapping in a fresh Error silently + // stripped both, leaving consumers only a message to regex against. + if (error instanceof Error && (error as Error & { code?: string }).code === 'BRAINY_WRITER_LOCKED') { + throw error + } throw new Error(`Failed to initialize Brainy: ${error}`) } } @@ -1041,20 +1489,52 @@ export class Brainy implements BrainyInterface { } } - // Graceful shutdown signals (registered once globally) - process.on('SIGTERM', async () => { + // Graceful shutdown signals (registered once globally). The listeners are + // kept as statics so the last live instance's close() can deregister them + // — the signal handles they hold are ref'd and would otherwise keep the + // process alive forever after every brain is closed. + Brainy.sigtermListener = async () => { await flushOnShutdown() process.exit(0) - }) - - process.on('SIGINT', async () => { + } + Brainy.sigintListener = async () => { await flushOnShutdown() process.exit(0) - }) - - process.on('beforeExit', async () => { + } + Brainy.beforeExitListener = async () => { + // Self-deregister FIRST: Node re-emits 'beforeExit' after every event- + // loop drain, and this flush schedules new async work — with the + // listener still attached, a script that never calls close() would spin + // flush → drain → flush forever and never exit. One flush, then the + // next drain finds no listener and the process exits. + if (Brainy.beforeExitListener) { + process.off('beforeExit', Brainy.beforeExitListener) + Brainy.beforeExitListener = undefined + } await flushOnShutdown() - }) + } + process.on('SIGTERM', Brainy.sigtermListener) + process.on('SIGINT', Brainy.sigintListener) + process.on('beforeExit', Brainy.beforeExitListener) + } + + /** + * Deregister the global shutdown hooks once NO live instance remains, so a + * script that closed every brain exits on its own — a library must never + * keep its host process alive. Re-initializing later re-registers them + * (the `shutdownHooksRegisteredGlobally` flag resets here). + */ + private static deregisterShutdownHooksIfIdle(): void { + if (Brainy.instances.length > 0 || !Brainy.shutdownHooksRegisteredGlobally) { + return + } + if (Brainy.sigtermListener) process.off('SIGTERM', Brainy.sigtermListener) + if (Brainy.sigintListener) process.off('SIGINT', Brainy.sigintListener) + if (Brainy.beforeExitListener) process.off('beforeExit', Brainy.beforeExitListener) + Brainy.sigtermListener = undefined + Brainy.sigintListener = undefined + Brainy.beforeExitListener = undefined + Brainy.shutdownHooksRegisteredGlobally = false } /** @@ -1070,13 +1550,31 @@ export class Brainy implements BrainyInterface { * re-initializing — using a closed Brainy is a consumer bug, not a lazy-init * opportunity. */ - private async ensureInitialized(): Promise { + private async ensureInitialized(opts?: { + bypassMigrationLock?: boolean + needs?: IndexFamily[] + }): Promise { if (this.closed) { throw new Error('Brainy instance is not initialized: it was closed via close(). Create a new instance.') } if (!this.initialized) { await this.init() } + // Coordinated migration LOCK (#18): every data-plane read and write funnels + // through here, so this is the single choke point that holds operations while + // a native provider runs its one-time 7.x → 8.0 rebuild-from-canonical — no + // op touches a half-built index. The gate is FAMILY-SCOPED: `needs` names the + // derived-index families this operation actually consults, so a read served + // entirely from canonical storage (`needs: []`) or from a healthy family + // never blocks on an UNRELATED family's migration. `needs` omitted = the + // conservative whole-brain wait (writes, and any read not yet classified). + // Observability (`health`/`checkHealth`) and the lock-clearing path + // (`stampBrainFormat`, which does not route through here) opt out entirely so + // an operator can always watch progress and a native provider can stamp. + // A brain that never migrates pays one boolean check (see awaitMigrationLock). + if (!opts?.bypassMigrationLock) { + await this.awaitMigrationLock(opts?.needs) + } } /** @@ -1221,6 +1719,222 @@ export class Brainy implements BrainyInterface { return uuidv7() } + /** + * @description Persist one single-operation write (`add`/`update`/`remove`/ + * `relate`/`updateRelation`/`unrelate` and the per-item calls of `addMany`/ + * `updateMany`/`relateMany`) as its OWN immutable generation — the Model-B + * "every write versioned" contract. (`removeMany` is the one exception: it + * commits each delete *chunk* as a single generation for bulk-delete efficiency, + * not one generation per removed id.) Wraps the write's existing `executeTransaction` batch in + * {@link GenerationStore.commitSingleOp}, which captures byte-identical + * before-images of `touched`, runs the batch with the generation watermark + * pinned to the reserved generation (so this write's graph-index ops stamp cor + * with that one distinct per-op generation, via the unchanged + * {@link graphWriteGeneration}), and buffers the history for async + * group-commit. A crash before the flush loses only that buffered history, not + * the acknowledged live write (drop-without-restore recovery). + * + * @param touched - The entity/relationship ids the write creates, updates, or + * deletes — the before-image + per-id-chain set. + * @param run - The single-op's existing operation batch builder (the + * `tx => {…}` body previously passed straight to `executeTransaction`). + */ + private async persistSingleOp( + touched: { nouns?: string[]; verbs?: string[] }, + run: TransactionFunction, + precommit?: (before: CommitBeforeImages) => void, + pendingEvents?: PendingChangeEvent[] + ): Promise<{ generation?: number; timestamp: number; degraded?: string[] }> { + // Change-feed capture: when this write will emit, hold a reference to the + // commit's before-images so `remove` events can carry the record's last + // committed state (free — the commit reads them anyway for Model B). + let capturedBefore: CommitBeforeImages | undefined + const captureAndCheck = + pendingEvents && pendingEvents.length > 0 + ? (before: CommitBeforeImages): void => { + capturedBefore = before + precommit?.(before) + } + : precommit + + if (!this._generationStampingActive) { + // Init-time / infrastructure baseline write (e.g. the VFS root): apply + // WITHOUT creating a generation. Generation 0 is the freshly-materialized + // brain (bootstrap included); the first USER write is generation 1. + // Bootstrap writes are single-threaded, so a supplied precondition is + // honored against directly-read before-images (no commit mutex exists + // on this path — nothing to race). + if (captureAndCheck) { + const nouns = new Map() + for (const id of touched.nouns ?? []) { + const prev = await this.storage.readNounRaw(id) + nouns.set(id, { kind: 'noun', metadata: prev.metadata, vector: prev.vector }) + } + const verbs = new Map() + for (const id of touched.verbs ?? []) { + const prev = await this.storage.readVerbRaw(id) + verbs.set(id, { kind: 'verb', metadata: prev.metadata, vector: prev.vector }) + } + captureAndCheck({ nouns, verbs } as CommitBeforeImages) + } + await this.generationStore.runWithoutGeneration(() => + this.transactionManager.executeTransaction(run, { + timeout: transactTimeoutBudget( + (touched.nouns?.length ?? 0) + (touched.verbs?.length ?? 0) + ) + }) + ) + const timestamp = Date.now() + // Bootstrap writes are not generation-stamped; emit without one. + this.emitCommitted(pendingEvents, capturedBefore, undefined, timestamp) + return { timestamp } + } + let receipt + try { + receipt = await this.generationStore.commitSingleOp({ + touched, + precommit: captureAndCheck, + execute: () => + this.transactionManager.executeTransaction(run, { + timeout: transactTimeoutBudget( + (touched.nouns?.length ?? 0) + (touched.verbs?.length ?? 0) + ) + }) + }) + } catch (err) { + // A failed rollback that left the store inconsistent (a remove/update + // whose restore-undo failed) quarantines writes until repairIndex(). + if (err instanceof StoreInconsistentError) this.storeInconsistency = err + throw err + } + // POST-COMMIT ONLY: an aborted commit (CAS conflict, failed apply) throws + // above and never reaches this line — the feed cannot announce a write + // that did not become durable. (A degraded adopt-forward write DID commit — + // it carries a generation and emits normally.) + this.emitCommitted(pendingEvents, capturedBefore, receipt.generation, receipt.timestamp) + // An adopt-forward failed-rollback recovery committed the record but may have + // left its derived index incomplete (generationStore already warned once). + // Record the ids so reads and checkHealth() surface the incompleteness rather + // than silently returning partial data; repairIndex() clears them. + if (receipt.degraded && receipt.degraded.length > 0) { + for (const degradedId of receipt.degraded) this._indexDegradedIds.add(degradedId) + this._degradedReadWarned = false // re-arm the read-path warning + if (!this.config.silent) { + prodLog.warn( + `[Brainy] A single-op write committed in a DEGRADED state — the derived ` + + `index may be incomplete for id(s) ${receipt.degraded.join(', ')}. ` + + `Reads may return partial results until repairIndex() reconciles them.` + ) + } + } + return receipt + } + + /** + * @description Stamp pending change events with their commit receipt, + * enrich entity `remove` events with the record's last committed state + * (from the commit's before-images), and hand them to the change feed for + * post-mutex dispatch. No-op when the write produced no events. + * @param pending - Events the mutation method constructed pre-commit + * (only when a listener is subscribed — see {@link ChangeFeed.hasListeners}). + * @param before - The commit's before-images (delete-payload source). + * @param generation - The committed generation (absent for bootstrap writes). + * @param timestamp - The commit timestamp. + */ + private emitCommitted( + pending: PendingChangeEvent[] | undefined, + before: CommitBeforeImages | undefined, + generation: number | undefined, + timestamp: number + ): void { + if (!pending || pending.length === 0) return + const events: BrainyChangeEvent[] = pending.map((p) => { + let entity = p.entity + if (!entity && p.kind === 'entity' && p.op === 'remove' && p.id && before) { + const record = before.nouns.get(p.id)?.metadata as + | Record + | null + | undefined + if (record) { + entity = this.entityViewFromRawRecord(p.id, record) + } + } + return { + ...p, + ...(entity && { entity }), + ...(generation !== undefined && { generation }), + timestamp + } + }) + this._changeFeed.emit(events) + } + + /** + * @description Build the change-event entity view from a stored flat + * metadata record (the shape before-images and pre-delete reads carry), + * using THE canonical reserved/custom split so the view can never drift + * from the live read paths. + * @param id - The entity's canonical UUID. + * @param record - The stored flat metadata record. + * @returns The `ChangeEventEntity` view (type, subtype, service, custom metadata). + */ + private entityViewFromRawRecord( + id: string, + record: Record + ): NonNullable { + const { reserved, custom } = splitNounMetadataRecord(record) + return { + id, + type: String(reserved.noun ?? 'unknown'), + ...(reserved.subtype !== undefined && { subtype: String(reserved.subtype) }), + metadata: custom, + ...(reserved.service !== undefined && { service: String(reserved.service) }) + } + } + + /** + * @description Build the AGGREGATION view of an entity from a stored flat + * metadata record — EVERY reserved field mapped to its top-level entity + * name (stored `noun` → `type`), custom metadata in `metadata`. This must + * mirror the add-path `entityForIndexing` shape exactly: the aggregation + * engine resolves groupBy/where fields via `resolveEntityField` + * (top-level standard fields + custom metadata), so a view that drops a + * reserved field makes every aggregate grouped by that field decrement a + * group that does not exist — counts then drift upward forever after + * deletes (SELF-AGGREGATE-DELETE-DRIFT). Do not hand-roll subsets of this. + * @param record - The stored flat metadata record (before-image or pre-delete read). + * @returns The full-fidelity entity view for aggregation hooks. + */ + private entityForAggFromRawRecord(record: Record): Record { + const { reserved, custom } = splitNounMetadataRecord(record) + const { noun, ...rest } = reserved + return { + type: noun, + ...rest, + metadata: custom + } + } + + /** + * @description Add an entity (noun) to the brain. Embeds `data` into a vector and + * indexes the entity across all three intelligences — vector similarity, graph + * adjacency, and metadata — then returns its id. When no `id` is supplied a fresh + * time-ordered **UUID v7** is generated; a supplied natural-key string is + * normalized to a stable **UUID v5**. Under Model-B every add is its own immutable + * generation, so it is time-travelable via {@link asOf}. + * @param params - The entity to add. `data` (embedded for similarity) and `type` + * (a {@link NounType}) are the essentials; `subtype`, `metadata`, an explicit + * `id`, `visibility`, `confidence`, and `weight` are optional. + * @returns The entity's id — the supplied/normalized id, or a freshly generated UUID v7. + * @throws If the brain is read-only (`mode: 'reader'`), or brain-wide `requireSubtype` + * is on and the entity's type has no subtype. + * @example + * const id = await brain.add({ + * data: 'Ada Lovelace', + * type: NounType.Person, + * metadata: { role: 'mathematician' } + * }) + */ async add(params: AddParams): Promise { this.assertWritable('add') await this.ensureInitialized() @@ -1257,15 +1971,42 @@ export class Brainy implements BrainyInterface { // ifAbsent check so a natural-key upsert is idempotent against the same key. const { id, originalId } = coerceNewEntityId(params.id) + // ifAbsent and upsert are opposite resolutions of the same id collision — + // ifAbsent SKIPS the existing entity, upsert MERGES into it. Allowing both + // would be ambiguous, so it is a hard error. + if (params.ifAbsent && params.upsert) { + throw new Error( + 'add(): ifAbsent and upsert are mutually exclusive — ifAbsent skips an existing entity, upsert merges into it.' + ) + } + // ifAbsent (7.31.0) — by-ID idempotent insert. Only meaningful when a custom id // is supplied; a freshly generated UUID can never collide. Returns the existing // (canonical) id without writing if the entity is already present (no throw, - // no overwrite). + // no overwrite). This check is a FAST-FAIL (skips the embedding cost); the + // authoritative absence check runs in the insert's commit precondition + // below, atomic with the apply, so a concurrent same-id create cannot make + // both callers write. if (params.id && params.ifAbsent) { const existing = await this.storage.getNounMetadata(id) if (existing) return id } + // upsert — by-ID create-or-update. Only meaningful when a custom id is supplied + // (a freshly generated UUID can never collide). When the id already exists, + // delegate to update() so the supplied fields MERGE into the existing entity + // (metadata merge, re-embed on changed data, _rev bump, createdAt preserved) + // instead of the default destructive overwrite. When the id is absent, fall + // through to the guarded insert below (whose commit precondition converts a + // concurrent same-id create into this same merge — never an overwrite). + if (params.id && params.upsert) { + const existing = await this.storage.getNounMetadata(id) + if (existing) { + await this.update(this.upsertMergeParams(id, params)) + return id + } + } + // Get or compute vector const vector = params.vector || (await this.embed(params.data)) @@ -1330,9 +2071,25 @@ export class Brainy implements BrainyInterface { } } - // Execute atomically with transaction system + // Execute atomically with transaction system, generation-stamped as one + // immutable Model-B generation (before-image = the absent/create sentinel). // All operations succeed or all rollback - prevents partial failures - await this.transactionManager.executeTransaction(async (tx) => { + // ifAbsent/upsert insert leg: must-be-absent commit precondition, run + // under the commit mutex against the authoritative before-image — the + // planning-time absence checks above are fast-fails that can interleave + // with a concurrent same-id create. Exactly one concurrent create wins; + // every other caller takes its documented resolution instead of + // silently overwriting the winner. + const requireAbsent = Boolean(params.id && (params.ifAbsent || params.upsert)) + const insertPrecommit = requireAbsent + ? (before: CommitBeforeImages): void => { + if (before.nouns.get(id)?.metadata) { + throw new InsertPreconditionExistsSignal(id) + } + } + : undefined + + const runInsert: TransactionFunction = async (tx) => { // Operation 1: Save metadata FIRST (TypeAwareStorage caching) // isNew=true: skip pre-read for rollback (entity doesn't exist yet) tx.addOperation( @@ -1359,7 +2116,62 @@ export class Brainy implements BrainyInterface { tx.addOperation( new AddToMetadataIndexOperation(this.metadataIndex, id, entityForIndexing) ) - }) + } + + // Bounded retry closes the remaining interleavings without ever holding + // a lock across the loop: a lost insert race resolves to skip (ifAbsent) + // or merge (upsert); a merge that then hits a concurrent delete retries + // the insert. Each persistSingleOp call constructs fresh operations, so + // re-running the callback is safe. + // Change feed: built only when someone is listening (zero-cost gate). + const addEvents: PendingChangeEvent[] | undefined = this._changeFeed.hasListeners + ? [ + { + kind: 'entity', + op: 'add', + id, + entity: { + id, + type: String(params.type), + ...(params.subtype !== undefined && { subtype: String(params.subtype) }), + metadata: (params.metadata as Record) ?? {}, + ...(params.service !== undefined && { service: String(params.service) }) + } + } + ] + : undefined + + const MAX_UPSERT_ATTEMPTS = 10 + for (let attempt = 0; ; attempt++) { + try { + await this.persistSingleOp({ nouns: [id] }, runInsert, insertPrecommit, addEvents) + break + } catch (err) { + if (!(err instanceof InsertPreconditionExistsSignal)) { + throw err + } + // A concurrent writer created the entity between our absence check + // and the commit. + if (params.ifAbsent) { + // Documented contract: return the existing id without writing. + return id + } + // upsert: merge into the now-existing entity. A concurrent delete + // between this conflict and the merge throws EntityNotFoundError — + // loop back and retry the insert. + try { + await this.update(this.upsertMergeParams(id, params)) + return id + } catch (mergeErr) { + if ( + !(mergeErr instanceof EntityNotFoundError) || + attempt >= MAX_UPSERT_ATTEMPTS + ) { + throw mergeErr + } + } + } + } // Aggregation hook (outside transaction — derived data, can be reconstructed) if (this._aggregationIndex) { @@ -1369,6 +2181,31 @@ export class Brainy implements BrainyInterface { return id } + /** + * @description The `update()` param mapping `add({ upsert })` delegates to + * when the target id already exists: every supplied field merges into the + * existing entity (metadata merge, re-embed on changed data, `_rev` bump, + * `createdAt` preserved); absent fields are left untouched. Shared by the + * planning-time upsert branch and the commit-conflict retry so the two + * paths can never drift. + * @param id - The canonical entity id the upsert resolved to. + * @param params - The original `add()` params carrying the fields to merge. + * @returns The `UpdateParams` for the merging `update()` call. + */ + private upsertMergeParams(id: string, params: AddParams): UpdateParams { + return { + id, + ...(params.data !== undefined && { data: params.data }), + ...(params.type !== undefined && { type: params.type }), + ...(params.subtype !== undefined && { subtype: params.subtype }), + ...(params.visibility !== undefined && { visibility: params.visibility }), + ...(params.metadata !== undefined && { metadata: params.metadata }), + ...(params.vector !== undefined && { vector: params.vector }), + ...(params.confidence !== undefined && { confidence: params.confidence }), + ...(params.weight !== undefined && { weight: params.weight }) + } as UpdateParams + } + /** * Get an entity by ID * @@ -1502,7 +2339,10 @@ export class Brainy implements BrainyInterface { * */ async get(id: string, options?: GetOptions): Promise | null> { - await this.ensureInitialized() + // Canonical read: a get resolves an entity by id straight from storage and + // consults no derived index — it must not wait on any family's migration. + await this.ensureInitialized({ needs: [] }) + this.warnIfReadsDegraded('get') // Id normalization (8.0): a caller may read by their natural key — resolve // it to the same canonical UUID add() stored. A real UUID passes through. @@ -1560,7 +2400,8 @@ export class Brainy implements BrainyInterface { * ``` */ async batchGet(ids: string[], options?: GetOptions): Promise>> { - await this.ensureInitialized() + // Canonical read (see get): resolves by id from storage, no derived index. + await this.ensureInitialized({ needs: [] }) // Id normalization (8.0): resolve each id to its canonical UUID so callers // can batch-read by natural key. resolveEntityId is idempotent on real @@ -1708,95 +2549,162 @@ export class Brainy implements BrainyInterface { return entity } - /** One-shot registry for reserved-field drop warnings (per process, per method+field). */ + /** One-shot registry for reserved-field warnings (per process, per method+field). */ private static warnedReservedFields = new Set() /** - * @description Warn once per process (per method+field) that a reserved - * field arrived inside a metadata bag and was dropped, naming the correct - * write path. Only dropped (system-managed) fields warn — user-settable - * fields are remapped to their dedicated param and honored silently. + * @description Resolve the human-readable "correct write path" guidance for a + * reserved field on a given write method. Single source of truth shared by the + * `'throw'` (Error message) and `'warn'` (one-shot warning) paths so the two + * never drift. The trio `confidence` / `weight` / `subtype` and the + * add()/relate()-time fields `service` / `createdBy` / `visibility` map to a + * dedicated param; everything else is system-managed. * @param method - The public write method the bag arrived through. - * @param field - The reserved field name that was dropped. - * @param rightPath - Human guidance naming the correct write path. + * @param field - The reserved field name found in the metadata bag. + * @returns Guidance naming the correct way to set the field. */ - private warnDroppedReservedField(method: string, field: string, rightPath: string): void { - const key = `${method}:${field}` - if (Brainy.warnedReservedFields.has(key)) return - Brainy.warnedReservedFields.add(key) - prodLog.warn( - `[brainy] ${method}(): '${field}' is a reserved field and cannot be set ` + - `through the metadata bag — use ${rightPath}. The value was ignored. ` + - `(This warning is shown once per field per process.)` - ) + private reservedWritePath( + method: 'add' | 'update' | 'relate' | 'updateRelation', + field: string + ): string { + const typeParam = "the top-level 'type' param" + switch (field) { + case 'noun': + case 'verb': + return typeParam + case 'data': + return "the top-level 'data' param" + case 'confidence': + return "the 'confidence' param" + case 'weight': + return "the 'weight' param" + case 'subtype': + return "the 'subtype' param" + case 'visibility': + return "the 'visibility' param ('public' | 'internal')" + case 'service': + return method === 'add' + ? "the 'service' param of add()" + : method === 'relate' + ? "the 'service' param of relate()" + : 'nothing — service is fixed at create time' + case 'createdBy': + return method === 'add' + ? "the 'createdBy' param of add()" + : 'nothing — createdBy is system-managed' + case 'createdAt': + return 'nothing — creation time is set automatically' + case 'updatedAt': + return 'nothing — set automatically on every write' + case '_rev': + return method === 'update' + ? "the 'ifRev' param for optimistic concurrency" + : 'nothing — revisions are system-managed' + default: + return 'a dedicated top-level param' + } } /** - * @description Emit the one-shot drop warnings for system-managed entity - * fields found in a metadata bag. Shared by the `add` and `update` remaps - * (live calls and their `transact()` mirrors). - * @param method - `'add'` or `'update'` (the contract is identical for the - * matching `transact()` operation). - * @param reserved - The reserved half of the split metadata bag. + * @description Enforce {@link BrainyConfig.reservedFieldPolicy} for reserved + * fields found inside a metadata bag. Called by every write-path remap once + * the bag has been split and at least one reserved key is present. + * + * - `'throw'` (default): throw a clear Error naming every offending key and + * its correct write path. The caller never reaches the remap. + * - `'warn'`: emit a ONE-SHOT (per method+field, per process) warning for + * EVERY reserved key found — both the user-mutable fields that are about to + * be remapped and the system-managed fields that are about to be dropped — + * then fall through to the legacy remap. + * - `'remap'`: silent legacy remap, no warning. + * + * @param method - The public write method the bag arrived through. + * @param reserved - The reserved half of the split metadata bag (non-empty). + * @param reservedListName - `'RESERVED_ENTITY_FIELDS'` or + * `'RESERVED_RELATION_FIELDS'` — named in the thrown Error for discoverability. + * @returns `true` when the caller should proceed with the legacy remap + * (`'warn'` / `'remap'`); `'throw'` never returns (it throws first). + * @throws {Error} When the policy is `'throw'` and any reserved key is present. */ - private warnDroppedReservedEntityFields( - method: 'add' | 'update', - reserved: Partial> - ): void { - if (reserved.noun !== undefined) { - this.warnDroppedReservedField(method, 'noun', "the top-level 'type' param") - } - if (reserved.data !== undefined) { - this.warnDroppedReservedField(method, 'data', "the top-level 'data' param") - } - if (reserved.createdAt !== undefined) { - this.warnDroppedReservedField( - method, - 'createdAt', - method === 'add' - ? 'nothing — creation time is set automatically' - : 'nothing — creation time is immutable' + private enforceReservedPolicy( + method: 'add' | 'update' | 'relate' | 'updateRelation', + reserved: Partial>, + reservedListName: 'RESERVED_ENTITY_FIELDS' | 'RESERVED_RELATION_FIELDS' + ): boolean { + const policy = this.config.reservedFieldPolicy ?? 'throw' + const keys = Object.keys(reserved) + if (keys.length === 0) return true + + if (policy === 'throw') { + const detail = keys + .map((k) => { + const path = this.reservedWritePath(method, k) + // System-managed fields resolve to a "nothing — …" sentinel; phrase + // those as "is system-managed" rather than "pass it as the nothing". + return path.startsWith('nothing') + ? `metadata.${k} is a reserved field (${path.replace(/^nothing\s*—\s*/, '')}) and cannot be set through ${method}()` + : `metadata.${k} is a reserved field — pass it as ${path} to ${method}()` + }) + .join('; ') + throw new Error( + `${detail} (reserved: see ${reservedListName}). ` + + `Set reservedFieldPolicy:'remap' to opt into legacy remapping, ` + + `or reservedFieldPolicy:'warn' to remap with a warning.` ) } - if (reserved.updatedAt !== undefined) { - this.warnDroppedReservedField(method, 'updatedAt', 'nothing — set automatically on every write') - } - if (reserved._rev !== undefined) { - this.warnDroppedReservedField( - method, - '_rev', - method === 'update' - ? "the 'ifRev' param for optimistic concurrency" - : 'nothing — revisions are system-managed' - ) - } - // 'system' visibility is Brainy-only — a user bag cannot set it. (A 'public'/'internal' - // value IS user-settable and is silently remapped to the top-level param below.) - if (reserved.visibility === 'system') { - this.warnDroppedReservedField(method, 'visibility', "the 'visibility' param ('public' | 'internal')") - } - if (method === 'update') { - if (reserved.service !== undefined) { - this.warnDroppedReservedField(method, 'service', 'nothing — fixed at add() time') - } - if (reserved.createdBy !== undefined) { - this.warnDroppedReservedField(method, 'createdBy', 'nothing — fixed at add() time') + + if (policy === 'warn') { + // One-shot warning for EVERY reserved key (today only system-managed ones + // warn — this closes that gap so user-mutable remaps are visible too). + for (const k of keys) { + this.warnReservedRemapped(method, k, this.reservedWritePath(method, k)) } } + + // 'warn' and 'remap' both fall through to the legacy remap. + return true + } + + /** + * @description One-shot (per method+field, per process) warning that a + * reserved field arrived inside a metadata bag under the `'warn'` policy. The + * wording is neutral on "remapped vs dropped" — `reservedWritePath()` already + * tells the caller where the value goes (a dedicated param, or "nothing"). + * @param method - The public write method the bag arrived through. + * @param field - The reserved field name found in the bag. + * @param rightPath - Guidance naming the correct write path. + */ + private warnReservedRemapped(method: string, field: string, rightPath: string): void { + const key = `${method}:${field}` + if (Brainy.warnedReservedFields.has(key)) return + Brainy.warnedReservedFields.add(key) + // System-managed fields resolve to a "nothing — …" sentinel; phrase the + // guidance so it reads cleanly in both the remapped and dropped cases. + const guidance = rightPath.startsWith('nothing') + ? `it is ${rightPath.replace(/^nothing\s*—\s*/, '')} and was dropped` + : `set it via ${rightPath} instead` + prodLog.warn( + `[brainy] ${method}(): '${field}' is a reserved field and was found inside the ` + + `metadata bag — ${guidance}. (Legacy remap applied because ` + + `reservedFieldPolicy is 'warn'. This warning is shown once per field per process.)` + ) } /** * @description Normalize an `add()` params object with respect to * Brainy-reserved fields arriving inside `metadata` (untyped callers only — * the compile-time guard on `AddParams.metadata` stops TypeScript callers). - * Fields with a dedicated `add()` param (`confidence`, `weight`, `subtype`, - * `service`, `createdBy`) are remapped to that param unless the caller also - * passed it explicitly (top-level wins); system-managed fields (`noun`, - * `data`, `createdAt`, `updatedAt`, `_rev`) are dropped with a one-shot - * warning naming the correct write path. The remapped `subtype` flows + * Governed by {@link BrainyConfig.reservedFieldPolicy} (default `'throw'`): + * `'throw'` rejects the write naming the offending key(s); `'warn'`/`'remap'` + * fall through to the legacy remap, where fields with a dedicated `add()` + * param (`confidence`, `weight`, `subtype`, `visibility`, `service`, + * `createdBy`) are remapped to that param unless the caller also passed it + * explicitly (top-level wins) and system-managed fields (`noun`, `data`, + * `createdAt`, `updatedAt`, `_rev`) are dropped. A remapped `subtype` flows * through subtype-pairing enforcement exactly like a top-level one. * @param params - The caller's add params (not mutated). * @returns Params with reserved fields normalized out of `metadata`. + * @throws {Error} When `reservedFieldPolicy` is `'throw'` and the bag carries a reserved key. */ private remapReservedAddMetadata(params: AddParams): AddParams { const bag = params.metadata as Record | undefined @@ -1804,7 +2712,9 @@ export class Brainy implements BrainyInterface { const { reserved, custom } = splitNounMetadataRecord(bag) if (Object.keys(reserved).length === 0) return params - this.warnDroppedReservedEntityFields('add', reserved) + // Policy gate: 'throw' (default) throws here; 'warn' warns once per key then + // remaps; 'remap' silently remaps. (Throw never returns.) + this.enforceReservedPolicy('add', reserved, 'RESERVED_ENTITY_FIELDS') const createdBy = reserved.createdBy as { augmentation?: unknown; version?: unknown } | undefined const createdByValid = @@ -1841,13 +2751,15 @@ export class Brainy implements BrainyInterface { * `update({metadata:{confidence}})` silently dropped it (the patch value * survived the merge and was then clobbered by the preserve-existing * spread; a production consumer's confidence-evolution writes no-oped until - * read back). User-mutable fields (`confidence`, `weight`, `subtype`) - * remap to their dedicated param unless the caller also passed it - * (top-level wins); everything else (`noun`, `data`, `createdAt`, - * `updatedAt`, `service`, `createdBy`, `_rev`) is system-managed or fixed - * at `add()` time and is dropped with a one-shot warning. + * read back). Governed by {@link BrainyConfig.reservedFieldPolicy} (default + * `'throw'`): `'throw'` rejects the write; `'warn'`/`'remap'` remap + * user-mutable fields (`confidence`, `weight`, `subtype`) to their dedicated + * param unless the caller also passed it (top-level wins) and drop everything + * else (`noun`, `data`, `createdAt`, `updatedAt`, `service`, `createdBy`, + * `_rev`) as system-managed or fixed at `add()` time. * @param params - The caller's update params (not mutated). * @returns Params with reserved fields normalized out of `metadata`. + * @throws {Error} When `reservedFieldPolicy` is `'throw'` and the bag carries a reserved key. */ private remapReservedUpdateMetadata(params: UpdateParams): UpdateParams { const bag = params.metadata as Record | undefined @@ -1855,7 +2767,9 @@ export class Brainy implements BrainyInterface { const { reserved, custom } = splitNounMetadataRecord(bag) if (Object.keys(reserved).length === 0) return params - this.warnDroppedReservedEntityFields('update', reserved) + // Policy gate: 'throw' (default) throws; 'warn' warns once per key then + // remaps; 'remap' silently remaps. + this.enforceReservedPolicy('update', reserved, 'RESERVED_ENTITY_FIELDS') return { ...params, @@ -1869,61 +2783,18 @@ export class Brainy implements BrainyInterface { } } - /** - * @description Emit the one-shot drop warnings for system-managed - * relationship fields found in a metadata bag. Shared by the `relate` and - * `updateRelation` remaps (live calls and the `transact()` relate mirror). - * @param method - `'relate'` or `'updateRelation'`. - * @param reserved - The reserved half of the split metadata bag. - */ - private warnDroppedReservedRelationFields( - method: 'relate' | 'updateRelation', - reserved: Partial> - ): void { - if (reserved.verb !== undefined) { - this.warnDroppedReservedField(method, 'verb', "the top-level 'type' param") - } - if (reserved.data !== undefined) { - this.warnDroppedReservedField(method, 'data', "the top-level 'data' param") - } - if (reserved.createdAt !== undefined) { - this.warnDroppedReservedField( - method, - 'createdAt', - method === 'relate' - ? 'nothing — creation time is set automatically' - : 'nothing — creation time is immutable' - ) - } - if (reserved.updatedAt !== undefined) { - this.warnDroppedReservedField(method, 'updatedAt', 'nothing — set automatically on every write') - } - if (reserved.createdBy !== undefined) { - this.warnDroppedReservedField(method, 'createdBy', 'nothing — system-managed') - } - if (reserved._rev !== undefined) { - this.warnDroppedReservedField(method, '_rev', 'nothing — system-managed') - } - // 'system' visibility is Brainy-only — a user bag cannot set it. (A 'public'/'internal' - // value IS user-settable and is silently remapped to the top-level param below.) - if (reserved.visibility === 'system') { - this.warnDroppedReservedField(method, 'visibility', "the 'visibility' param ('public' | 'internal')") - } - if (method === 'updateRelation' && reserved.service !== undefined) { - this.warnDroppedReservedField(method, 'service', 'nothing — fixed at relate() time') - } - } - /** * @description Normalize a `relate()` params object with respect to * Brainy-reserved fields arriving inside `metadata` — the relationship - * mirror of {@link remapReservedAddMetadata}. Fields with a dedicated - * `relate()` param (`confidence`, `weight`, `subtype`, `service`) remap to - * that param (top-level wins); system-managed fields (`verb`, `data`, - * `createdAt`, `updatedAt`, `createdBy`, `_rev`) are dropped with a - * one-shot warning. + * mirror of {@link remapReservedAddMetadata}. Governed by + * {@link BrainyConfig.reservedFieldPolicy} (default `'throw'`): `'throw'` + * rejects the write; `'warn'`/`'remap'` remap fields with a dedicated + * `relate()` param (`confidence`, `weight`, `subtype`, `visibility`, + * `service`) to that param (top-level wins) and drop system-managed fields + * (`verb`, `data`, `createdAt`, `updatedAt`, `createdBy`, `_rev`). * @param params - The caller's relate params (not mutated). * @returns Params with reserved fields normalized out of `metadata`. + * @throws {Error} When `reservedFieldPolicy` is `'throw'` and the bag carries a reserved key. */ private remapReservedRelateMetadata(params: RelateParams): RelateParams { const bag = params.metadata as Record | undefined @@ -1931,7 +2802,9 @@ export class Brainy implements BrainyInterface { const { reserved, custom } = splitVerbMetadataRecord(bag) if (Object.keys(reserved).length === 0) return params - this.warnDroppedReservedRelationFields('relate', reserved) + // Policy gate: 'throw' (default) throws; 'warn' warns once per key then + // remaps; 'remap' silently remaps. + this.enforceReservedPolicy('relate', reserved, 'RESERVED_RELATION_FIELDS') return { ...params, @@ -1954,12 +2827,14 @@ export class Brainy implements BrainyInterface { /** * @description Normalize an `updateRelation()` params object with respect * to Brainy-reserved fields arriving inside the metadata patch — the - * relationship mirror of {@link remapReservedUpdateMetadata}. User-mutable - * fields (`confidence`, `weight`, `subtype`) remap to their dedicated - * param (top-level wins); everything else is dropped with a one-shot - * warning. + * relationship mirror of {@link remapReservedUpdateMetadata}. Governed by + * {@link BrainyConfig.reservedFieldPolicy} (default `'throw'`): `'throw'` + * rejects the write; `'warn'`/`'remap'` remap user-mutable fields + * (`confidence`, `weight`, `subtype`, `visibility`) to their dedicated param + * (top-level wins) and drop everything else. * @param params - The caller's update-relation params (not mutated). * @returns Params with reserved fields normalized out of `metadata`. + * @throws {Error} When `reservedFieldPolicy` is `'throw'` and the bag carries a reserved key. */ private remapReservedUpdateRelationMetadata( params: UpdateRelationParams @@ -1969,7 +2844,9 @@ export class Brainy implements BrainyInterface { const { reserved, custom } = splitVerbMetadataRecord(bag) if (Object.keys(reserved).length === 0) return params - this.warnDroppedReservedRelationFields('updateRelation', reserved) + // Policy gate: 'throw' (default) throws; 'warn' warns once per key then + // remaps; 'remap' silently remaps. + this.enforceReservedPolicy('updateRelation', reserved, 'RESERVED_RELATION_FIELDS') return { ...params, @@ -2091,7 +2968,11 @@ export class Brainy implements BrainyInterface { // ifRev (7.31.0) — optimistic concurrency. If caller supplied ifRev, the persisted // _rev must match exactly. `_rev` is reserved: every read path surfaces it ONLY // top-level (entities without one are read as rev 1), so that is the one place - // to look. + // to look. This check is purely a FAST-FAIL (avoids paying the embedding + // cost on an obviously stale expectation) — the authoritative check runs + // in the commit precondition below, under the commit mutex, where it is + // atomic with the apply. Concurrent same-rev updates all pass HERE but + // exactly one survives THERE. const currentRev = typeof existing._rev === 'number' ? existing._rev : 1 if (typeof params.ifRev === 'number' && params.ifRev !== currentRev) { throw new RevisionConflictError(params.id, params.ifRev, currentRev) @@ -2167,8 +3048,38 @@ export class Brainy implements BrainyInterface { metadata: newMetadata } - // Execute atomically with transaction system - await this.transactionManager.executeTransaction(async (tx) => { + // Authoritative CAS + honest rev stamp, run by the generation store + // UNDER the commit mutex against the just-read before-image — the one + // point where check-and-apply is atomic (the fast-fail above can + // interleave with concurrent writers; this cannot). Also re-stamps + // `_rev` from the authoritative base so the counter stays monotonic + // even for concurrent non-CAS updates. The staged operations below + // capture `updatedMetadata` by reference, so the re-stamp lands. + const casPrecommit = (before: CommitBeforeImages): void => { + const beforeMeta = before.nouns.get(params.id)?.metadata as + | { _rev?: unknown } + | null + | undefined + if (!beforeMeta) { + // A concurrent remove() deleted the entity after our read. A CAS + // caller gets the truth; a plain update keeps its long-standing + // last-writer-wins semantics (the staged write re-creates it). + if (typeof params.ifRev === 'number') { + throw new EntityNotFoundError(params.id) + } + return + } + const authoritativeRev = + typeof beforeMeta._rev === 'number' ? beforeMeta._rev : 1 + if (typeof params.ifRev === 'number' && params.ifRev !== authoritativeRev) { + throw new RevisionConflictError(params.id, params.ifRev, authoritativeRev) + } + updatedMetadata._rev = authoritativeRev + 1 + } + + // Execute atomically with transaction system, generation-stamped as one + // immutable Model-B generation (before-image = the entity's prior state). + await this.persistSingleOp({ nouns: [params.id] }, async (tx) => { // Operation 1: Update metadata FIRST (updates type cache) tx.addOperation( new UpdateNounMetadataOperation(this.storage, params.id, updatedMetadata) @@ -2225,17 +3136,37 @@ export class Brainy implements BrainyInterface { tx.addOperation( new AddToMetadataIndexOperation(this.metadataIndex, params.id, entityForIndexing) ) - }) + }, casPrecommit, this._changeFeed.hasListeners + ? [ + { + kind: 'entity', + op: 'update', + id: params.id, + entity: { + id: params.id, + type: String(entityForIndexing.type), + ...(entityForIndexing.subtype !== undefined && { + subtype: String(entityForIndexing.subtype) + }), + metadata: (newMetadata as Record) ?? {}, + ...(entityForIndexing.service !== undefined && { + service: String(entityForIndexing.service) + }) + } + } + ] + : undefined) - // Aggregation hook (outside transaction — derived data) + // Aggregation hook (outside transaction — derived data). `existing` is + // the full get() view — every reserved field top-level — and must be + // passed whole: a subset view makes the old-side decrement miss any + // reserved-field group (update would then double-count it). if (this._aggregationIndex) { - const oldEntityForAgg = { - type: existing.type, - service: existing.service, - data: existing.data, - metadata: existing.metadata - } - this._aggregationIndex.onEntityUpdated(params.id, entityForIndexing, oldEntityForAgg) + this._aggregationIndex.onEntityUpdated( + params.id, + entityForIndexing, + existing as unknown as Record + ) } } @@ -2277,8 +3208,12 @@ export class Brainy implements BrainyInterface { const targetVerbs = await this.storage.getVerbsByTarget(id) const allVerbs = [...verbs, ...targetVerbs] - // Execute atomically with transaction system - await this.transactionManager.executeTransaction(async (tx) => { + // Execute atomically with transaction system, generation-stamped as one + // immutable Model-B generation covering the entity AND its cascade-deleted + // relationships (each id's before-image = its prior state). + await this.persistSingleOp( + { nouns: [id], verbs: allVerbs.map((v) => v.id) }, + async (tx) => { // Operation 1: Remove from vector index if (noun) { tx.addOperation( @@ -2293,9 +3228,11 @@ export class Brainy implements BrainyInterface { ) } - // Operation 3: Delete noun metadata + // Operation 3: Delete noun (full removal). The pre-read metadata rides + // along so the count decrement never depends on re-reading the record + // being removed (a null re-read must not silently skip it). tx.addOperation( - new DeleteNounMetadataOperation(this.storage, id) + new DeleteNounMetadataOperation(this.storage, id, metadata) ) // Operations 4+: Delete all related verbs atomically @@ -2304,28 +3241,52 @@ export class Brainy implements BrainyInterface { // rollback can re-add through the BigInt addVerb contract) const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb) tx.addOperation( - new RemoveFromGraphIndexOperation(this.graphIndex, verb, sourceInt, targetInt, this.graphWriteGeneration) + new RemoveFromGraphIndexOperation(this.graphIndex, verb, { sourceInt, targetInt }, this.graphWriteGeneration) ) // Delete verb metadata tx.addOperation( new DeleteVerbMetadataOperation(this.storage, verb.id) ) } - }) + }, + undefined, + this._changeFeed.hasListeners + ? [ + // The entity delete (payload = last state, from the pre-delete read) + // plus one unrelate per cascade-deleted relationship. + { + kind: 'entity', + op: 'remove', + id, + ...(metadata && { + entity: this.entityViewFromRawRecord(id, metadata as Record) + }) + }, + ...allVerbs.map( + (v): PendingChangeEvent => ({ + kind: 'relation', + op: 'unrelate', + id: v.id, + relation: { + id: v.id, + from: v.sourceId, + to: v.targetId, + type: String(v.verb) + } + }) + ) + ] + : undefined) - // Aggregation hook (outside transaction — derived data) + // Aggregation hook (outside transaction — derived data). The view must + // carry EVERY reserved field top-level (not a subset): a groupBy on + // subtype/visibility/etc. otherwise decrements a nonexistent group and + // the real count never comes down. if (this._aggregationIndex && metadata) { - // Reconstruct entity-like object from stored metadata via the - // canonical reserved/custom split (the hand-rolled destructure here - // missed subtype/_rev, leaking them into the aggregation view). - const { reserved, custom } = splitNounMetadataRecord(metadata) - const entityForAgg = { - type: reserved.noun, - service: reserved.service, - data: reserved.data, - metadata: custom - } - this._aggregationIndex.onEntityDeleted(id, entityForAgg) + this._aggregationIndex.onEntityDeleted( + id, + this.entityForAggFromRawRecord(metadata as Record) + ) } } @@ -2449,6 +3410,413 @@ export class Brainy implements BrainyInterface { return this.entityIntsToUuids(neighborInts) } + /** + * @description Hydrate the entity id-mapper from persisted state BEFORE a graph + * rebuild. A native int-keyed adjacency resolves every verb endpoint + * (`sourceId`/`targetId` → stable int) through this mapper, so it must reflect + * the persisted int assignments before `graphIndex.rebuild()` runs — otherwise + * edges resolve to stale/missing ints and are silently dropped + * (CTX-BR-RESTORE-REBUILD). The JS mapper has no `rebuild()` and needs no + * reload (it is re-derived by `MetadataIndex.rebuild()` via append-only + * `getOrAssign`), so this is a no-op for it. Mirrors the ordering in + * {@link restore}. + */ + private async hydrateIdMapperForGraphRebuild(): Promise { + const idMapper = this.metadataIndex.getIdMapper() as { rebuild?: () => Promise } + if (typeof idMapper.rebuild === 'function') { + await idMapper.rebuild() + } + } + + /** + * @description Verify that the graph adjacency is actually LIVE before a graph read trusts + * its result. A native graph index can load its relationship COUNT (manifest) on a cold open + * of a LARGE brain (≥10k nouns, which skips the eager index rebuild) but NOT its + * source→target adjacency, so `getNeighbors()` returns `[]` for EVERY source even though + * edges are persisted — and `find({ connected })` / `neighbors()` / `related()` would serve + * that `[]` as if it were truth. + * + * Two detection strategies, in order of honesty: + * - **Preferred (8.0 contract):** the provider exposes a sync `isReady()` that is true ONLY + * when the edges are loaded. `false` → hydrate the id-mapper (a native int adjacency + * resolves endpoints through it), rebuild from storage, and re-check `isReady()`; if it is + * still `false`, throw {@link GraphIndexNotReadyError} rather than returning `[]`. + * - **Fallback (providers without `isReady()`):** a GLOBAL known-edge sample (a real + * persisted verb's `sourceId`, which by definition HAS an outgoing edge) — NOT any queried + * anchor, because brainy cannot cheaply tell "adjacency unloaded" from "this node is + * genuinely edgeless" per-anchor. If that known-edge source resolves to no neighbors, the + * adjacency did not load: rebuild and re-probe; if even that fails, throw. + * + * @returns `'live'` when the adjacency is already trustworthy (or there is genuinely nothing + * to verify), or `'rebuilt'` when a cold-unloaded adjacency was just healed from storage — + * in which case callers that observed an empty result must RE-RUN their collection. + * @throws {GraphIndexNotReadyError} when the index claims edges but cannot serve a known + * persisted edge (or stays not-ready) even after a rebuild. + */ + private async verifyGraphAdjacencyLive(): Promise<'live' | 'rebuilt'> { + if (this._graphAdjacencyVerified) return 'live' + // Coordinated migration LOCK (#18): while the graph provider owns a locked + // rebuild-from-canonical, brainy must NOT fire its own graphIndex.rebuild() + // on a read — that would race the provider's in-place rebuild. The data-plane + // lock (awaitMigrationLock in ensureInitialized) already makes callers wait, + // so this is normally unreachable mid-migration; the guard is defensive. It + // deliberately does NOT set `_graphAdjacencyVerified`, so the real verify runs + // once the migration clears. + if (this.providerIsMigrating(this.graphIndex)) return 'live' + // Re-entrancy: rebuild() can trigger reads (neighbors/related) that call back into this + // guard. While a verify is in flight, short-circuit so we cannot recurse into rebuild(). + if (this._graphAdjacencyVerifying) return 'live' + this._graphAdjacencyVerifying = true + try { + const gi = this.graphIndex as GraphAdjacencyIndex & { isReady?: () => boolean } + + // ── Strategy 1: honest isReady() signal (cortex >= 2.7.8 / 3.0) ────────── + if (typeof gi.isReady === 'function') { + if (gi.isReady()) { + this._graphAdjacencyVerified = true + return 'live' + } + // Not ready: the edges did not load on open. Hydrate the id-mapper, then rebuild. + if (!this.config.silent) { + console.warn( + `[Brainy] Graph adjacency reports not-ready (isReady() === false) — the persisted ` + + `adjacency did not load on open. Rebuilding from storage…` + ) + } + await this.hydrateIdMapperForGraphRebuild() + await this.graphIndex.rebuild() + if (gi.isReady()) { + this._graphAdjacencyVerified = true + return 'rebuilt' + } + throw new GraphIndexNotReadyError( + `Graph adjacency index reports not-ready even after a rebuild — the persisted ` + + `adjacency could not be loaded. find({ connected }), neighbors() and related() ` + + `cannot be served reliably for this brain.` + ) + } + + // ── Strategy 2: known-edge-sample probe (providers without isReady()) ──── + const claimed = await this.graphIndex.size() + if (!claimed || claimed <= 0) return 'live' // no edges claimed — nothing to verify + + const sample = await this.storage.getVerbs({ pagination: { limit: 1 } }) + const verb = sample.items?.[0] + if (!verb || !verb.sourceId) { + // no edges in storage — stale count, harmless; don't re-probe on every read. + this._graphAdjacencyVerified = true + return 'live' + } + + // Resolve the known edge's source through THIS brain's id-mapper. An unmapped source means + // the sample is not one of this brain's own edges — e.g. a shared on-disk store reused + // across instances surfaces a foreign verb whose UUID this brain's resident mapper never + // interned. We cannot prove a cold-unloaded adjacency from such a sample, so treat it as + // INCONCLUSIVE: mark verified and return 'live' rather than rebuilding/throwing. (The honest + // cold-load signal for native providers is isReady(), checked above; the JS baseline keeps + // its mapper resident, so its OWN edges always resolve — the targeted 7.x failure mode, + // "mapper loaded but adjacency empty", still resolves the source and is detected below.) + const sourceInt = this.graphEntityInt(verb.sourceId) + if (sourceInt === undefined) { + this._graphAdjacencyVerified = true + return 'live' + } + + // Ask the adjacency for ONE neighbor of the (mapped) known-edge source. + const probeKnownSource = async (): Promise => + (await this.graphIndex.getNeighbors(sourceInt, { limit: 1 })).length > 0 + + if (await probeKnownSource()) { + this._graphAdjacencyVerified = true + return 'live' // adjacency is live — the common case + } + + // INCONSISTENT: the index reports edges but a KNOWN-mapped persisted edge's source has none → + // the adjacency did not load on open. Hydrate the mapper and rebuild from storage. + if (!this.config.silent) { + console.warn( + `[Brainy] Graph adjacency reports ${claimed} relationship(s) but a persisted edge ` + + `resolves to none — the persisted adjacency did not load on open. Rebuilding from storage…` + ) + } + await this.hydrateIdMapperForGraphRebuild() + await this.graphIndex.rebuild() + + if (await probeKnownSource()) { + this._graphAdjacencyVerified = true + return 'rebuilt' + } + throw new GraphIndexNotReadyError( + `Graph adjacency index reports ${claimed} relationship(s) but returns no edges even ` + + `after a rebuild — the persisted adjacency could not be loaded. find({ connected }), ` + + `neighbors() and related() cannot be served reliably for this brain.` + ) + } catch (err) { + if (err instanceof GraphIndexNotReadyError) throw err + // A transient probe/rebuild failure must not break the actual query NOR be + // masked as "no data". Allow a re-check on the next graph read and fall through. + this._graphAdjacencyVerified = false + if (!this.config.silent) { + console.warn(`[Brainy] Graph adjacency consistency check skipped (transient): ${err}`) + } + return 'live' + } finally { + this._graphAdjacencyVerifying = false + } + } + + /** + * @description The metadata field-index counterpart of {@link verifyGraphAdjacencyLive}. + * On a cold open a native metadata provider can report data yet not serve its + * `where` postings, so `find({ where })` silently returns `[]` — the exact + * failure a downstream deployment reported (cold reads blanking filtered pages + * after every restart). This one-shot guard, run on the first FILTERED `find()`, + * closes that: it takes a KNOWN persisted entity + one of its plain field values + * and asks the index to resolve it. If the index returns the known id the field + * postings are live (the common case, and the ONLY cost on a warm brain — one + * O(1) probe). If it does not, the postings did not load: brainy rebuilds the + * index from the canonical records and re-probes; if it STILL cannot serve the + * known value it throws a loud {@link MetadataIndexNotReadyError} rather than + * let a silent `[]` stand. Inconclusive cases (empty store, no plain field to + * probe, a shared store surfacing a foreign entity) are treated as live — never + * a false rebuild. A migrating provider is skipped (it owns its locked rebuild). + * @returns `'live'` when the index serves, `'rebuilt'` when a rebuild restored it. + */ + private async verifyMetadataLive(): Promise<'live' | 'rebuilt'> { + if (this._metadataVerified) return 'live' + // Migration LOCK (#18): a migrating provider owns its in-place rebuild — do + // not race it. Defensive; the data-plane lock already gates callers upstream. + if (this.providerIsMigrating(this.metadataIndex)) return 'live' + // Re-entrancy: rebuild() can trigger reads that call back into this guard. + if (this._metadataVerifying) return 'live' + this._metadataVerifying = true + try { + // A KNOWN persisted entity + one plain field to probe. Sample a few so a + // system-only entity (e.g. the VFS root) doesn't make every open inconclusive. + const sample = await this.storage.getNouns({ pagination: { limit: 5, offset: 0 } }) + let probe: { field: string; value: string | number | boolean; id: string } | null = null + for (const noun of sample.items ?? []) { + probe = this.pickMetadataProbe(noun as { id?: string; metadata?: Record }) + if (probe) break + } + if (!probe) { + // Empty store, or nothing with a plain user field to probe — inconclusive. + this._metadataVerified = true + return 'live' + } + const p = probe + + const probeServes = async (): Promise => { + try { + const ids = await this.metadataIndex.getIdsForFilter({ [p.field]: p.value }) + return ids.includes(p.id) + } catch { + // FIELD_NOT_INDEXED for a field a persisted entity actually holds is + // itself the cold/broken signal — treat as not-serving (→ rebuild). + return false + } + } + + if (await probeServes()) { + this._metadataVerified = true + return 'live' // field postings are live — the common case + } + + if (!this.config.silent) { + console.warn( + `[Brainy] Metadata field index returns no match for a known persisted value of ` + + `'${p.field}' — the field postings did not load on open. Rebuilding from storage…` + ) + } + await this.metadataIndex.rebuild() + + if (await probeServes()) { + this._metadataVerified = true + return 'rebuilt' + } + throw new MetadataIndexNotReadyError( + `Metadata field index cannot serve a known persisted value of '${p.field}' even after ` + + `a rebuild — find({ where }) and other filtered reads cannot be served reliably for ` + + `this brain (a silent empty result would misrepresent existing data).` + ) + } catch (err) { + if (err instanceof MetadataIndexNotReadyError) throw err + // A transient probe/rebuild failure must not break the query NOR mask as + // "no data". Allow a re-check on the next filtered read and fall through. + this._metadataVerified = false + if (!this.config.silent) { + console.warn(`[Brainy] Metadata consistency check skipped (transient): ${err}`) + } + return 'live' + } finally { + this._metadataVerifying = false + } + } + + /** + * @description Choose one plain (scalar, user-written) field from an entity's + * metadata to probe the field index with — skipping internal / system fields + * (`__words__` text hash, VFS markers, reserved `visibility`/`subtype`/`service`, + * the `noun` type alias, timestamps) that use different index paths, and any + * non-scalar value. Returns `null` when the entity has no probeable field. + */ + private pickMetadataProbe( + noun: { id?: string; metadata?: Record } + ): { field: string; value: string | number | boolean; id: string } | null { + const id = noun?.id + const metadata = noun?.metadata + if (!id || !metadata || typeof metadata !== 'object') return null + const skip = new Set([ + 'noun', 'type', 'subtype', 'service', 'visibility', 'id', 'vector', + 'isVFSEntity', 'vfsType', 'vfsPath', 'vfsName', 'createdAt', 'updatedAt' + ]) + for (const [field, value] of Object.entries(metadata)) { + if (field.startsWith('_') || skip.has(field)) continue + if (value === null || value === undefined) continue + const t = typeof value + if (t === 'string' || t === 'number' || t === 'boolean') { + return { field, value: value as string | number | boolean, id } + } + } + return null + } + + /** + * @description The vector-index counterpart of {@link verifyGraphAdjacencyLive} + * / {@link verifyMetadataLive}. On a cold open a native vector provider can + * report a non-zero `size()` (its persisted COUNT loaded) yet not have loaded + * its serving structure (the mmap/DiskANN graph) — so a pure semantic + * `find({ query })` silently returns `[]`. A pure semantic query has + * `hasFilterCriteria === false`, so the metadata guard never fires; this guard + * closes that gap. Run one-shot on the first vector/proximity search: + * - **Preferred (honest signal):** the provider exposes `isReady()`. `false` + * → rebuild from storage, re-check; if still `false`, throw + * {@link VectorIndexNotReadyError} rather than serving `[]`. + * - **Fallback (no `isReady()`):** a KNOWN persisted vector (sampled + + * hydrated) is searched against the index; if it does not self-match, the + * serving structure did not load — rebuild + re-probe, else throw. + * Inconclusive cases (empty store, no probeable vector, `size()===0` — where + * the JS baseline's cold load is `ensureIndexesLoaded`'s job) are treated as + * live: never a false rebuild. A migrating provider is skipped (it owns its + * locked rebuild). + * @returns `'live'` when the index serves, `'rebuilt'` when a rebuild restored it. + */ + private async verifyVectorLive(): Promise<'live' | 'rebuilt'> { + if (this._vectorVerified) return 'live' + // Migration LOCK (#18): a migrating provider owns its in-place rebuild. + if (this.providerIsMigrating(this.index)) return 'live' + // Re-entrancy: rebuild() can trigger reads that call back into this guard. + if (this._vectorVerifying) return 'live' + this._vectorVerifying = true + try { + // ── Strategy 1: honest isReady() signal (native provider) ────────────── + const readiness = assessIndexReadiness(this.index) + if (readiness !== 'unknown') { + if (readiness === 'ready') { + this._vectorVerified = true + return 'live' + } + // Not ready: the serving structure did not load on open. Rebuild. + if (!this.config.silent) { + console.warn( + `[Brainy] Vector index reports not-ready (isReady() === false) — the persisted ` + + `vector index did not load on open. Rebuilding from storage…` + ) + } + await this.index.rebuild() + if (assessIndexReadiness(this.index) === 'ready') { + this._vectorVerified = true + return 'rebuilt' + } + throw new VectorIndexNotReadyError( + `Vector index reports not-ready even after a rebuild — semantic find({ query }) and ` + + `proximity search cannot be served reliably for this brain (a silent empty result ` + + `would misrepresent existing data).` + ) + } + + // ── Strategy 2: known-vector probe (providers without isReady()) ─────── + const claimed = this.index.size() + if (!claimed || claimed <= 0) return 'live' // JS cold path is ensureIndexesLoaded's job + + const probe = await this.pickVectorProbe() + if (!probe) { + // Empty store, or nothing with a probeable vector — inconclusive. + this._vectorVerified = true + return 'live' + } + const p = probe + + const probeServes = async (): Promise => { + // The failure mode we guard is the SILENT EMPTY result: a cold index that + // loaded its COUNT but not its serving structure returns `[]` for a + // known-present vector, while a warm index returns at least one hit. We + // check for a NON-EMPTY result, NOT an exact self-match — HNSW is + // approximate and `get()` may return a re-hydrated/normalized vector, so + // demanding the exact self as top-1 would false-positive on a perfectly + // healthy index (and wrongly rebuild → throw). + const hits = await this.index.search(p.vector, 1) + return hits.length > 0 + } + void p.id // probe keyed on the vector; id retained for diagnostics only + + if (await probeServes()) { + this._vectorVerified = true + return 'live' // serving structure is live — the common case + } + + if (!this.config.silent) { + console.warn( + `[Brainy] Vector index reports ${claimed} vector(s) but a known persisted vector ` + + `returns no results — the serving structure did not load on open. Rebuilding…` + ) + } + await this.index.rebuild() + + if (await probeServes()) { + this._vectorVerified = true + return 'rebuilt' + } + throw new VectorIndexNotReadyError( + `Vector index reports ${claimed} vector(s) but a known persisted vector returns no ` + + `results even after a rebuild — semantic find({ query }) cannot be served reliably ` + + `for this brain (a silent empty result would misrepresent existing data).` + ) + } catch (err) { + if (err instanceof VectorIndexNotReadyError) throw err + // A transient probe/rebuild failure must not break the query NOR mask as + // "no data". Allow a re-check on the next vector read and fall through. + this._vectorVerified = false + if (!this.config.silent) { + console.warn(`[Brainy] Vector consistency check skipped (transient): ${err}`) + } + return 'live' + } finally { + this._vectorVerifying = false + } + } + + /** + * @description Sample a KNOWN persisted noun and hydrate its vector, to probe + * the vector index with. `get()` omits vectors by default, so this passes + * `{ includeVectors: true }`. Samples a few (a system-only / vectorless entity + * must not make every open inconclusive). Returns `null` when nothing has a + * probeable vector. + */ + private async pickVectorProbe(): Promise<{ id: string; vector: number[] } | null> { + const sample = await this.storage.getNouns({ pagination: { limit: 5, offset: 0 } }) + for (const noun of sample.items ?? []) { + const id = (noun as { id?: string }).id + if (!id) continue + const full = await this.get(id, { includeVectors: true }) + const vector = (full as { vector?: number[] } | null)?.vector + if (Array.isArray(vector) && vector.length > 0) { + return { id, vector } + } + } + return null + } + // ------------------------------------------------------------------------- /** @@ -2688,8 +4056,13 @@ export class Brainy implements BrainyInterface { // entities were existence-checked above) and mirror them onto the verb. const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb) - // Execute atomically with transaction system - await this.transactionManager.executeTransaction(async (tx) => { + // Reverse-edge id reserved up front (when bidirectional) so the Model-B + // generation's touched-verb set covers both edges this write creates. + const reverseId = params.bidirectional ? uuidv4() : undefined + + // Execute atomically with transaction system, generation-stamped as one + // immutable Model-B generation (before-image of each new edge = absent). + await this.persistSingleOp({ verbs: reverseId ? [id, reverseId] : [id] }, async (tx) => { // Operation 1: Save verb vector data tx.addOperation( new SaveVerbOperation(this.storage, { @@ -2710,15 +4083,14 @@ export class Brainy implements BrainyInterface { // Operation 3: Add to graph index for O(1) lookups tx.addOperation( new AddToGraphIndexOperation( - this.graphIndex, verb, sourceInt, targetInt, + this.graphIndex, verb, { sourceInt, targetInt }, this.graphWriteGeneration, (verbInt) => this.cacheVerbInt(verbInt, id) ) ) // Create bidirectional if requested - if (params.bidirectional) { - const reverseId = uuidv4() + if (params.bidirectional && reverseId) { const reverseVerb: GraphVerb = { ...verb, id: reverseId, @@ -2749,13 +4121,50 @@ export class Brainy implements BrainyInterface { // Operation 6: Add reverse relationship to graph index tx.addOperation( new AddToGraphIndexOperation( - this.graphIndex, reverseVerb, targetInt, sourceInt, + this.graphIndex, reverseVerb, { sourceInt: targetInt, targetInt: sourceInt }, this.graphWriteGeneration, (verbInt) => this.cacheVerbInt(verbInt, reverseId) ) ) } - }) + }, + undefined, + this._changeFeed.hasListeners + ? [ + { + kind: 'relation', + op: 'relate', + id, + relation: { + id, + from: params.from, + to: params.to, + type: String(params.type), + ...(params.metadata && { + metadata: params.metadata as Record + }) + } + }, + ...(reverseId + ? [ + { + kind: 'relation', + op: 'relate', + id: reverseId, + relation: { + id: reverseId, + from: params.to, + to: params.from, + type: String(params.type), + ...(params.metadata && { + metadata: params.metadata as Record + }) + } + } as PendingChangeEvent + ] + : []) + ] + : undefined) return id } @@ -2787,13 +4196,14 @@ export class Brainy implements BrainyInterface { // a rollback can re-add through the BigInt addVerb contract. const endpointInts = verb ? this.resolveVerbEndpointInts(verb) : undefined - // Execute atomically with transaction system - await this.transactionManager.executeTransaction(async (tx) => { + // Execute atomically with transaction system, generation-stamped as one + // immutable Model-B generation (before-image = the relationship's state). + await this.persistSingleOp({ verbs: [id] }, async (tx) => { // Operation 1: Remove from graph index if (verb && endpointInts) { tx.addOperation( new RemoveFromGraphIndexOperation( - this.graphIndex, verb, endpointInts.sourceInt, endpointInts.targetInt, this.graphWriteGeneration + this.graphIndex, verb, endpointInts, this.graphWriteGeneration ) ) } @@ -2802,7 +4212,26 @@ export class Brainy implements BrainyInterface { tx.addOperation( new DeleteVerbMetadataOperation(this.storage, id) ) - }) + }, + undefined, + this._changeFeed.hasListeners && verb + ? [ + { + kind: 'relation', + op: 'unrelate', + id, + relation: { + id, + from: verb.sourceId, + to: verb.targetId, + type: String(verb.verb), + ...(verb.metadata && { + metadata: verb.metadata as Record + }) + } + } + ] + : undefined) } /** @@ -2916,7 +4345,7 @@ export class Brainy implements BrainyInterface { // resolution serves both the remove (rollback re-add) and the re-add. const reindexInts = typeChanged ? this.resolveVerbEndpointInts(verbForIndex) : undefined - await this.transactionManager.executeTransaction(async (tx) => { + await this.persistSingleOp({ verbs: [params.id] }, async (tx) => { tx.addOperation( new UpdateVerbMetadataOperation(this.storage, params.id, updatedMetadata) ) @@ -2926,18 +4355,37 @@ export class Brainy implements BrainyInterface { if (typeChanged && reindexInts) { tx.addOperation( new RemoveFromGraphIndexOperation( - this.graphIndex, existing, reindexInts.sourceInt, reindexInts.targetInt, this.graphWriteGeneration + this.graphIndex, existing, reindexInts, this.graphWriteGeneration ) ) tx.addOperation( new AddToGraphIndexOperation( - this.graphIndex, verbForIndex, reindexInts.sourceInt, reindexInts.targetInt, + this.graphIndex, verbForIndex, reindexInts, this.graphWriteGeneration, (verbInt) => this.cacheVerbInt(verbInt, params.id) ) ) } - }) + }, + undefined, + this._changeFeed.hasListeners + ? [ + { + kind: 'relation', + op: 'updateRelation', + id: params.id, + relation: { + id: params.id, + from: verbForIndex.sourceId, + to: verbForIndex.targetId, + type: String(verbForIndex.verb ?? verbForIndex.type), + ...(verbForIndex.metadata && { + metadata: verbForIndex.metadata as Record + }) + } + } + ] + : undefined) } /** @@ -2976,7 +4424,10 @@ export class Brainy implements BrainyInterface { async related( paramsOrId?: string | RelatedParams ): Promise[]> { - await this.ensureInitialized() + // Graph read: relationship traversal consults only the graph adjacency + // family, so it waits on a graph migration but not on vector/metadata. + await this.ensureInitialized({ needs: ['graph'] }) + await this.verifyGraphAdjacencyLive() // Handle string ID shorthand: related(id) -> related({ from: id }) const rawParams = typeof paramsOrId === 'string' @@ -2989,31 +4440,32 @@ export class Brainy implements BrainyInterface { const params: RelatedParams = { ...rawParams, ...(rawParams.from !== undefined && { from: resolveEntityId(rawParams.from) }), - ...(rawParams.to !== undefined && { to: resolveEntityId(rawParams.to) }) + ...(rawParams.to !== undefined && { to: resolveEntityId(rawParams.to) }), + ...(rawParams.node !== undefined && { node: resolveEntityId(rawParams.node) }) } const limit = params.limit || 100 const offset = params.offset || 0 - // Production safety: warn for large unfiltered queries - if (!params.from && !params.to && !params.type && limit > 10000) { - console.warn( - `[Brainy] related(): Fetching ${limit} relationships without filters. ` + - `Consider adding 'from', 'to', or 'type' filter for better performance.` + // `node` is the both-direction shorthand; it cannot also constrain one direction. + if (params.node !== undefined && (params.from !== undefined || params.to !== undefined)) { + throw new Error( + "related(): 'node' is mutually exclusive with 'from'/'to' — pass 'node' for both-direction incident edges, or 'from'/'to' for one direction." ) } - // Build filter for storage query + // Production safety: warn for large unfiltered queries + if (!params.from && !params.to && !params.node && !params.type && limit > 10000) { + console.warn( + `[Brainy] related(): Fetching ${limit} relationships without filters. ` + + `Consider adding 'from', 'to', 'node', or 'type' filter for better performance.` + ) + } + + // Shared filter (type / subtype / service / visibility). Endpoint ids + // (`sourceId` / `targetId`) are added per-direction below. const filter: any = {} - if (params.from) { - filter.sourceId = params.from - } - - if (params.to) { - filter.targetId = params.to - } - if (params.type) { filter.verbType = Array.isArray(params.type) ? params.type : [params.type] } @@ -3026,9 +4478,8 @@ export class Brainy implements BrainyInterface { filter.service = params.service } - // Visibility (8.0): exclude hidden tiers by default. The exclusion is applied in the - // storage scan AFTER metadata load (where verb.visibility is known), so the storage - // limit stays exact. Like subtype, it disqualifies the metadata-less fast paths. + // Visibility (8.0): exclude hidden tiers by default. Applied on the O(degree) + // candidate set in the graph-index fast path (see baseStorage.applyVerbMetadataFilters). const excludedTiers = this.excludedVisibilityTiers(params) if (excludedTiers) { filter.excludeVisibility = excludedTiers @@ -3037,6 +4488,42 @@ export class Brainy implements BrainyInterface { // VFS relationships are no longer filtered // VFS is part of the knowledge graph - users can filter explicitly if needed + // Both-direction incident-edge query: union the node's out-edges (it is the + // source) and in-edges (it is the target) — each an O(degree) indexed lookup — + // then dedupe a self-loop that appears in both. Sorted by id for a stable page, + // sliced from the merged set. Fetching the node's full incidence is the intended + // O(degree) cost; for a very-high-degree node, deep `offset` paging is best-effort + // (use the native edgesForNode / a graph cursor for exhaustive paging). + if (params.node !== undefined) { + const endLimit = offset + limit + const [outEdges, inEdges] = await Promise.all([ + this.storage.getVerbs({ + pagination: { limit: endLimit }, + filter: { ...filter, sourceId: params.node } + }), + this.storage.getVerbs({ + pagination: { limit: endLimit }, + filter: { ...filter, targetId: params.node } + }) + ]) + const byId = new Map() + for (const verb of outEdges.items) byId.set(verb.id, verb) + for (const verb of inEdges.items) byId.set(verb.id, verb) + const merged = [...byId.values()].sort((a, b) => + a.id < b.id ? -1 : a.id > b.id ? 1 : 0 + ) + return this.verbsToRelations(merged.slice(offset, offset + limit)) + } + + // Single-direction query (from / to). + if (params.from) { + filter.sourceId = params.from + } + + if (params.to) { + filter.targetId = params.to + } + // Fetch from storage with pagination at storage layer (efficient!) const result = await this.storage.getVerbs({ pagination: { @@ -3051,6 +4538,726 @@ export class Brainy implements BrainyInterface { return this.verbsToRelations(result.items) } + // ============= GRAPH VIEWS (brain.graph.*) ============= + + /** + * @description The `brain.graph` namespace — graph-shaped reads. `subgraph` + * extracts the multi-hop neighborhood around seed entities; the surface grows + * with the engine (streaming `export`, `rank`, `path`, `communities`). Routes + * to a registered native {@link GraphAccelerationProvider} when present, else + * serves the same results from Brainy's pure-TS adjacency. Reads are live + * ("now"); historical graph views come from `db.asOf(g).graph.*` (a later slice). + * @returns The {@link GraphApi} bound to this brain. + * @example + * const view = await brain.graph.subgraph(personId, { depth: 2 }) + * // view.nodes = the 2-hop neighborhood; view.edges = the relations among them + */ + get graph(): GraphApi { + return { + subgraph: (selector: SubgraphSelector, options?: SubgraphOptions) => + this.graphSubgraph(selector, options), + export: (options?: GraphExportOptions) => this.graphExport(options), + rank: (options?: GraphRankOptions) => this.graphRank(options), + communities: (options?: GraphCommunitiesOptions) => this.graphCommunities(options), + path: (from: string, to: string, options?: GraphPathOptions) => + this.graphPath(from, to, options) + } + } + + /** Resolve the optional native graph-acceleration provider (feature-detected). */ + private graphAccelerationProvider(): GraphAccelerationProvider | undefined { + // Resolve + cache the provider INSTANCE once. Accept EITHER a ready instance OR + // a `(storage) => provider` factory (the convention the graphIndex/metadataIndex/ + // vector providers use) — a registered factory would otherwise fail the + // isGraphAccelerationProvider duck-test and the native path would silently never engage. + if (this._graphAccelProvider === undefined) { + const raw = this.pluginRegistry.getProvider('graphAcceleration') + let resolved: unknown = raw + if (typeof raw === 'function' && !isGraphAccelerationProvider(raw)) { + try { + resolved = (raw as (storage: StorageAdapter) => unknown)(this.storage) + } catch { + resolved = undefined + } + } + this._graphAccelProvider = isGraphAccelerationProvider(resolved) ? resolved : null + } + const provider = this._graphAccelProvider ?? undefined + // Readiness gate — checked LIVE each call, never cached: the native engine sets + // `isInitialized` false until its engine + cursor have loaded (the cold-start / + // rebuild window). Until then, route to the pure-TS path rather than calling a + // not-ready provider (which throws) — and re-engage the native path automatically + // the moment `isInitialized` flips true. Gates EVERY `brain.graph.*` route + // (subgraph/export/rank/communities/path + the query→expand fusion) at once. + return provider && provider.isInitialized ? provider : undefined + } + + /** + * @description {@link GraphApi.subgraph} dispatch: native engine when present, + * else the TS BFS fallback. The selector is an id set, a `find()` result, or a + * query (query→expand fusion). Explicit ids are id-normalized so a caller may + * seed by natural key. + */ + private async graphSubgraph( + selector: SubgraphSelector, + options?: SubgraphOptions + ): Promise> { + await this.ensureInitialized() + const depth = Math.max(0, options?.depth ?? 1) + const direction = options?.direction ?? 'both' + const accel = this.graphAccelerationProvider() + + // Query selector (a FindParams object — not a string / array) → query→expand. + if (selector && typeof selector === 'object' && !Array.isArray(selector)) { + return this.graphSubgraphFromQuery(accel, selector, depth, direction, options) + } + + // Id set: a string, an array of ids, or a `find()` result (Result[]). The + // FindParams (query) case returned above, so the remaining union is the id forms. + const seedIds = this.selectorToSeedIds(selector as string | string[] | Result[]) + if (seedIds.length === 0) return { nodes: [], edges: [], truncated: false } + return accel + ? this.graphSubgraphNative(accel, this.seedIdsToInts(seedIds), depth, direction, options) + : this.graphSubgraphFallback(seedIds, depth, direction, options) + } + + /** + * @description Query→expand fusion (graph #61): run the query, then expand the + * subgraph from every match. When the native engine is present AND the query is a + * pure metadata filter (no vector/text/proximity criteria) AND the metadata + * provider exposes the opaque-set producer, the matched universe is forwarded to + * `traverse` as an {@link OpaqueIdSet} with NO id materialization in TS — the + * O(1)-crossing path. Otherwise the query is materialized via `find()` and its + * result ids seed the traversal (native or TS fallback). + */ + private async graphSubgraphFromQuery( + accel: GraphAccelerationProvider | undefined, + selector: FindParams, + depth: number, + direction: 'in' | 'out' | 'both', + options?: SubgraphOptions + ): Promise> { + // Native opaque fast path: a metadata-only universe never leaves the engine. + if (accel && !this.hasVectorOrTextCriteria(selector)) { + const filter = this.buildMetadataFilter(selector) + const mip = this.metadataIndex as unknown as MetadataIndexProvider + if (filter && typeof mip.getIdSetForFilter === 'function') { + const universe = await mip.getIdSetForFilter(filter) + return this.graphSubgraphNative(accel, universe, depth, direction, options) + } + } + + // General path: materialize the matched ids, then expand from them. The find + // default limit (10) is too small to seed a neighborhood, so seed from ALL + // matches up to a bounded cap unless the caller pinned an explicit limit. + const matches = await this.find({ + ...selector, + limit: selector.limit ?? Brainy.QUERY_SEED_CAP + }) + const seedIds = matches.map((m) => m.id) + if (seedIds.length === 0) return { nodes: [], edges: [], truncated: false } + return accel + ? this.graphSubgraphNative(accel, this.seedIdsToInts(seedIds), depth, direction, options) + : this.graphSubgraphFallback(seedIds, depth, direction, options) + } + + /** True when a query needs vector/text/proximity search (not a pure metadata filter). */ + private hasVectorOrTextCriteria(params: FindParams): boolean { + return Boolean( + (params.query && params.query.trim() !== '') || params.vector || params.near + ) + } + + /** Normalize an id-set selector (id / id[] / `find()` result) to canonical seed ids. */ + private selectorToSeedIds(selector: string | string[] | Result[]): string[] { + const arr = Array.isArray(selector) ? selector : [selector] + return arr.map((s) => resolveEntityId(typeof s === 'string' ? s : s.id)) + } + + /** Resolve seed ids to graph entity ints, dropping any that were never mapped. */ + private seedIdsToInts(seedIds: string[]): bigint[] { + const ints: bigint[] = [] + for (const id of seedIds) { + const int = this.graphEntityInt(id) + if (int !== undefined) ints.push(int) + } + return ints + } + + /** + * @description Pure-TS subgraph BFS — expands each frontier through the + * O(degree) `related()` adjacency (visibility / type / subtype filters applied + * there), dedupes edges, tracks hop depth, and honors `maxNodes` / `maxEdges` + * caps. Correct at small/medium scale; the native provider is the at-scale path. + */ + private async graphSubgraphFallback( + seedIds: string[], + depth: number, + direction: 'in' | 'out' | 'both', + options?: SubgraphOptions + ): Promise> { + const maxNodes = options?.maxNodes + const maxEdges = options?.maxEdges + const nodeDepth = new Map() + for (const id of seedIds) nodeDepth.set(id, 0) + const edgesById = new Map>() + let truncated = false + + let frontier = [...seedIds] + for (let d = 1; d <= depth && frontier.length > 0 && !truncated; d++) { + const next: string[] = [] + for (const nodeId of frontier) { + const incident = await this.incidentEdges(nodeId, direction, options) + for (const edge of incident) { + if (!edgesById.has(edge.id)) { + if (maxEdges !== undefined && edgesById.size >= maxEdges) { + truncated = true + break + } + edgesById.set(edge.id, edge) + } + const neighbor = edge.from === nodeId ? edge.to : edge.from + if (!nodeDepth.has(neighbor)) { + if (maxNodes !== undefined && nodeDepth.size >= maxNodes) { + truncated = true + continue + } + nodeDepth.set(neighbor, d) + next.push(neighbor) + } + } + if (truncated) break + } + frontier = next + } + + return this.buildGraphView( + nodeDepth, + [...edgesById.values()], + options?.hydrateNodes !== false, + truncated + ) + } + + /** One frontier hop's incident edges in the requested direction (filters applied by `related()`). */ + private incidentEdges( + nodeId: string, + direction: 'in' | 'out' | 'both', + options?: SubgraphOptions + ): Promise[]> { + const shared: RelatedParams = { + type: options?.type, + subtype: options?.subtype, + includeInternal: options?.includeInternal, + includeSystem: options?.includeSystem, + limit: options?.maxEdges ?? 100000 + } + if (direction === 'out') return this.related({ ...shared, from: nodeId }) + if (direction === 'in') return this.related({ ...shared, to: nodeId }) + return this.related({ ...shared, node: nodeId }) + } + + /** + * @description Native subgraph: resolve seeds to ints, call the provider's + * `traverse`, then hydrate the columnar `Subgraph` (node ints → ids paired with + * their depth, edge verb ints → `Relation`s) into a {@link GraphView}. The TS + * fallback produces the same view, so Brainy CI exercises this shape via the + * fallback; the native path itself is cross-layer-tested against the provider. + */ + private async graphSubgraphNative( + accel: GraphAccelerationProvider, + seeds: bigint[] | OpaqueIdSet, + depth: number, + direction: 'in' | 'out' | 'both', + options?: SubgraphOptions + ): Promise> { + // Resolved int seeds may be empty (no mapped entities); an OpaqueIdSet (Buffer) + // is passed straight through — the provider owns its membership. + if (Array.isArray(seeds) && seeds.length === 0) return { nodes: [], edges: [], truncated: false } + + const verbTypes = options?.type + ? (Array.isArray(options.type) ? options.type : [options.type]).map((t) => + TypeUtils.getVerbIndex(t) + ) + : undefined + const excluded = this.excludedVisibilityTiers(options ?? {}) + const traverseOptions: TraverseOptions = { + depth, + direction, + ...(verbTypes && { verbTypes }), + ...(options?.subtype !== undefined && { + subtypes: Array.isArray(options.subtype) ? options.subtype : [options.subtype] + }), + ...(excluded && { excludeVisibility: excluded as unknown as string[] }), + ...(options?.maxNodes !== undefined && { maxNodes: options.maxNodes }), + ...(options?.maxEdges !== undefined && { maxEdges: options.maxEdges }), + includeEdges: true, + includeDepth: true + } + + const sub = await accel.traverse(seeds, traverseOptions) + return this.hydrateNativeSubgraph(sub, options?.hydrateNodes !== false) + } + + /** + * @description Hydrate a columnar native `Subgraph` into a {@link GraphView}: + * edge verb-ints → `Relation`s (via `verbIntsToIds` + `getVerbsBatchCached`), + * and node ints paired with their depth (position-preserving — `entityIntsToUuids` + * drops deleted ints, which would desync the depth column). Shared by the native + * `subgraph` and `export` paths. + * @param sub - The provider's columnar subgraph / cursor chunk. + * @param hydrateNodes - Whether to batch-load each node's `type` / `subtype`. + * @returns The hydrated {@link GraphView}. + */ + private async hydrateNativeSubgraph(sub: Subgraph, hydrateNodes: boolean): Promise> { + const verbIds = (await this.graphIndex.verbIntsToIds(Array.from(sub.edgeVerbInts))).filter( + (id): id is string => id !== null + ) + const verbsMap = await this.graphIndex.getVerbsBatchCached(verbIds) + const edges = this.verbsToRelations([...verbsMap.values()]) + + const idMapper = this.metadataIndex.getIdMapper() + const nodeDepth = new Map() + for (let i = 0; i < sub.nodes.length; i++) { + const uuid = idMapper.getUuid(Number(sub.nodes[i])) + if (uuid !== undefined) nodeDepth.set(uuid, sub.nodeDepth ? sub.nodeDepth[i] : 0) + } + + return this.buildGraphView(nodeDepth, edges, hydrateNodes, sub.truncated ?? false) + } + + /** + * @description Assemble a {@link GraphView} from discovered node depths + edges, + * optionally hydrating each node's `type` / `subtype` in one batch read + * (`hydrateNodes`, default `true`). + */ + private async buildGraphView( + nodeDepth: Map, + edges: Relation[], + hydrateNodes: boolean, + truncated: boolean + ): Promise> { + const ids = [...nodeDepth.keys()] + let entities: Map> | undefined + if (hydrateNodes && ids.length > 0) { + entities = await this.batchGet(ids) + } + const nodes: GraphNode[] = ids.map((id) => { + const entity = entities?.get(id) + return { + id, + ...(entity && { + type: entity.type, + ...(entity.subtype !== undefined && { subtype: entity.subtype }) + }), + depth: nodeDepth.get(id) + } + }) + return { nodes, edges, truncated } + } + + /** + * @description {@link GraphApi.export} dispatch: native snapshot-consistent + * graph cursor when present, else the TS cursor walk over nouns + verbs. Both + * are O(N+E) single passes (cursor pagination — no re-scan). + */ + private async *graphExport(options?: GraphExportOptions): AsyncGenerator> { + await this.ensureInitialized() + const accel = this.graphAccelerationProvider() + if (accel) { + yield* this.graphExportNative(accel, options) + } else { + yield* this.graphExportFallback(options) + } + } + + /** + * @description TS export: stream all nouns (node chunks) then all verbs (edge + * chunks) via cursor pagination — O(N+E), no re-scan. Node refs carry + * `type`/`subtype` straight from the noun records (no extra read). Hidden tiers + * are excluded by default. + */ + private async *graphExportFallback(options?: GraphExportOptions): AsyncGenerator> { + const chunkSize = options?.chunkSize ?? 1000 + const excludedTiers = this.excludedVisibilityTiers(options ?? {}) + const excludedSet = excludedTiers ? new Set(excludedTiers) : null + + if (options?.includeNodes !== false) { + let cursor: string | undefined + let offset = 0 + for (;;) { + const page = await this.storage.getNouns({ + pagination: cursor ? { limit: chunkSize, cursor } : { limit: chunkSize, offset } + }) + const nodes: GraphNode[] = [] + for (const noun of page.items) { + if (excludedSet && noun.visibility && excludedSet.has(noun.visibility)) continue + nodes.push({ + id: noun.id, + type: noun.type, + ...(noun.subtype !== undefined && { subtype: noun.subtype }) + }) + } + if (nodes.length > 0) yield { nodes, edges: [], truncated: false } + if (!page.hasMore || page.items.length === 0) break + if (page.nextCursor) cursor = page.nextCursor + else offset += page.items.length + } + } + + if (options?.includeEdges !== false) { + const filter = excludedTiers + ? { excludeVisibility: excludedTiers as unknown as string[] } + : undefined + let cursor: string | undefined + let offset = 0 + for (;;) { + const page = await this.storage.getVerbs({ + pagination: cursor ? { limit: chunkSize, cursor } : { limit: chunkSize, offset }, + filter + }) + if (page.items.length > 0) { + yield { nodes: [], edges: this.verbsToRelations(page.items), truncated: false } + } + if (!page.hasMore || page.items.length === 0) break + if (page.nextCursor) cursor = page.nextCursor + else offset += page.items.length + } + } + } + + /** + * @description Native export: open a snapshot-consistent graph cursor (pins a + * generation — no dup/skip under concurrent writes), pull light chunks, and + * hydrate each into a {@link GraphView}. The cursor is always closed (even on + * early break / error). Cross-layer tested against the native provider; the TS + * fallback exercises the same chunk shape in CI. + */ + private async *graphExportNative( + accel: GraphAccelerationProvider, + options?: GraphExportOptions + ): AsyncGenerator> { + const excludedTiers = this.excludedVisibilityTiers(options ?? {}) + const handle = await accel.graphCursorOpen({ + direction: 'both', + ...(excludedTiers && { excludeVisibility: excludedTiers as unknown as string[] }), + projection: 'light' + }) + try { + for (;;) { + const chunk = await accel.graphCursorNext(handle, options?.chunkSize ?? 1000) + const view = await this.hydrateNativeSubgraph(chunk.subgraph, true) + if (view.nodes.length > 0 || view.edges.length > 0) yield view + if (chunk.done) break + } + } finally { + await accel.graphCursorClose(handle) + } + } + + // ============= GRAPH ANALYTICS (rank / communities / path) ============= + + /** + * @description Load the whole visible graph into a dense integer adjacency for + * the rank / communities fallbacks: a node-id list, an id→index map, and + * `outAdj[i]` = the dense indices of node `i`'s out-edge targets (multiplicity + * preserved). Streamed via {@link graphExport} (cursor pagination — O(N+E), no + * re-scan), so hidden tiers are excluded the same way every other read excludes + * them. Isolated nodes (no edges) are retained so they form singleton groups. + */ + private async loadAnalyticsGraph(opts: { + includeInternal?: boolean + includeSystem?: boolean + }): Promise<{ ids: string[]; outAdj: number[][] }> { + const ids: string[] = [] + const index = new Map() + const outAdj: number[][] = [] + const idxOf = (id: string): number => { + let i = index.get(id) + if (i === undefined) { + i = ids.length + ids.push(id) + index.set(id, i) + outAdj.push([]) + } + return i + } + + for await (const chunk of this.graphExport({ + includeInternal: opts.includeInternal, + includeSystem: opts.includeSystem + })) { + for (const node of chunk.nodes) idxOf(node.id) + for (const edge of chunk.edges) { + const s = idxOf(edge.from) + const t = idxOf(edge.to) + outAdj[s].push(t) + } + } + + return { ids, outAdj } + } + + /** + * @description {@link GraphApi.rank} dispatch: native ranking when a provider is + * present, else the TS PageRank fallback. + */ + private async graphRank(options?: GraphRankOptions): Promise { + await this.ensureInitialized() + const accel = this.graphAccelerationProvider() + return accel ? this.graphRankNative(accel, options) : this.graphRankFallback(options) + } + + /** TS PageRank over the dense visible adjacency; sorted descending, `topK`-capped. */ + private async graphRankFallback(options?: GraphRankOptions): Promise { + const { ids, outAdj } = await this.loadAnalyticsGraph(options ?? {}) + if (ids.length === 0) return [] + const scores = pageRank(outAdj) + const entries: GraphRankEntry[] = ids.map((id, i) => ({ id, score: scores[i] })) + entries.sort((a, b) => b.score - a.score) + return options?.topK !== undefined ? entries.slice(0, options.topK) : entries + } + + /** Native ranking → hydrate node-ints to ids (descending order preserved). */ + private async graphRankNative( + accel: GraphAccelerationProvider, + options?: GraphRankOptions + ): Promise { + const excluded = this.excludedVisibilityTiers(options ?? {}) + const opts: RankOptions = { + ...(options?.topK !== undefined && { topK: options.topK }), + ...(excluded && { excludeVisibility: excluded as unknown as string[] }) + } + const result = await accel.rank(opts) + const idMapper = this.metadataIndex.getIdMapper() + const entries: GraphRankEntry[] = [] + for (let i = 0; i < result.nodeInts.length; i++) { + const uuid = idMapper.getUuid(Number(result.nodeInts[i])) + if (uuid !== undefined) entries.push({ id: uuid, score: result.scores[i] }) + } + return options?.topK !== undefined ? entries.slice(0, options.topK) : entries + } + + /** + * @description {@link GraphApi.communities} dispatch: native grouping when a + * provider is present, else the TS connected-components fallback. + */ + private async graphCommunities( + options?: GraphCommunitiesOptions + ): Promise { + await this.ensureInitialized() + const accel = this.graphAccelerationProvider() + return accel + ? this.graphCommunitiesNative(accel, options) + : this.graphCommunitiesFallback(options) + } + + /** + * @description TS grouping: weakly-connected components by default (union-find + * over the undirected projection), or strongly-connected components when + * `directed: true` (Tarjan). Largest group first. + */ + private async graphCommunitiesFallback( + options?: GraphCommunitiesOptions + ): Promise { + const { ids, outAdj } = await this.loadAnalyticsGraph(options ?? {}) + if (ids.length === 0) return { groups: [], count: 0 } + const labels = options?.directed + ? stronglyConnectedComponents(outAdj) + : connectedComponents(outAdj) + return this.groupByLabel(ids, labels) + } + + /** Collapse parallel `(id, label)` arrays into id-groups, largest first. */ + private groupByLabel(ids: string[], labels: number[]): GraphCommunitiesResult { + const byLabel = new Map() + for (let i = 0; i < ids.length; i++) { + const label = labels[i] + const existing = byLabel.get(label) + if (existing) existing.push(ids[i]) + else byLabel.set(label, [ids[i]]) + } + const groups = [...byLabel.values()].sort((a, b) => b.length - a.length) + return { groups, count: groups.length } + } + + /** Native grouping → bucket node-ints by community label, hydrate to ids. */ + private async graphCommunitiesNative( + accel: GraphAccelerationProvider, + options?: GraphCommunitiesOptions + ): Promise { + const excluded = this.excludedVisibilityTiers(options ?? {}) + const opts: CommunitiesOptions = { + ...(options?.directed !== undefined && { directed: options.directed }), + ...(excluded && { excludeVisibility: excluded as unknown as string[] }) + } + const result = await accel.communities(opts) + const idMapper = this.metadataIndex.getIdMapper() + const byCommunity = new Map() + for (let i = 0; i < result.nodeInts.length; i++) { + const uuid = idMapper.getUuid(Number(result.nodeInts[i])) + if (uuid === undefined) continue + const community = result.communityIds[i] + const existing = byCommunity.get(community) + if (existing) existing.push(uuid) + else byCommunity.set(community, [uuid]) + } + const groups = [...byCommunity.values()].sort((a, b) => b.length - a.length) + return { groups, count: groups.length } + } + + /** + * @description {@link GraphApi.path} dispatch: native pathfinding when a provider + * is present, else the TS BFS (hops) / Dijkstra (weight) fallback. Endpoints are + * id-normalized so callers may pass natural keys. + */ + private async graphPath( + from: string, + to: string, + options?: GraphPathOptions + ): Promise { + await this.ensureInitialized() + const fromId = resolveEntityId(from) + const toId = resolveEntityId(to) + const accel = this.graphAccelerationProvider() + return accel + ? this.graphPathNative(accel, fromId, toId, options) + : this.graphPathFallback(fromId, toId, options) + } + + /** + * @description On-demand TS pathfinding — expands frontiers through the O(degree) + * `related()` adjacency (visibility / type filters applied there) rather than + * loading the whole graph, so a short path terminates early. BFS for fewest hops + * (default); Dijkstra (min-heap) for least summed edge weight (`by: 'weight'` — + * each edge costs its stored `weight`, the 0–1 connection strength, default 1.0). + */ + private async graphPathFallback( + fromId: string, + toId: string, + options?: GraphPathOptions + ): Promise { + if (fromId === toId) return { nodes: [fromId], relationships: [], cost: 0 } + + const direction = options?.direction ?? 'both' + const maxDepth = options?.maxDepth ?? Number.POSITIVE_INFINITY + const subOptions: SubgraphOptions = { + ...(options?.type !== undefined && { type: options.type }), + ...(options?.includeInternal !== undefined && { includeInternal: options.includeInternal }), + ...(options?.includeSystem !== undefined && { includeSystem: options.includeSystem }) + } + const pred = new Map() + + if (options?.by === 'weight') { + const dist = new Map([[fromId, 0]]) + const hops = new Map([[fromId, 0]]) + const settled = new Set() + const heap = new MinHeap() + heap.push(fromId, 0) + while (heap.size > 0) { + const node = heap.pop() as string + if (settled.has(node)) continue + settled.add(node) + if (node === toId) break + const depth = hops.get(node) as number + if (depth >= maxDepth) continue + const baseCost = dist.get(node) as number + const edges = await this.incidentEdges(node, direction, subOptions) + for (const edge of edges) { + const neighbor = edge.from === node ? edge.to : edge.from + if (settled.has(neighbor)) continue + // Cost = the edge's stored weight (0–1 connection strength, default 1.0). + // Non-negative by construction, so Dijkstra stays valid. + const weight = edge.weight ?? 1 + const candidate = baseCost + weight + if (!dist.has(neighbor) || candidate < (dist.get(neighbor) as number)) { + dist.set(neighbor, candidate) + hops.set(neighbor, depth + 1) + pred.set(neighbor, { prev: node, edgeId: edge.id }) + heap.push(neighbor, candidate) + } + } + } + if (!settled.has(toId)) return null + return this.reconstructPath(fromId, toId, pred, dist.get(toId) as number) + } + + // Fewest hops — breadth-first, each edge cost 1. + const visited = new Set([fromId]) + const queue: Array<{ id: string; depth: number }> = [{ id: fromId, depth: 0 }] + let head = 0 + while (head < queue.length) { + const { id, depth } = queue[head++] + if (depth >= maxDepth) continue + const edges = await this.incidentEdges(id, direction, subOptions) + for (const edge of edges) { + const neighbor = edge.from === id ? edge.to : edge.from + if (visited.has(neighbor)) continue + visited.add(neighbor) + pred.set(neighbor, { prev: id, edgeId: edge.id }) + if (neighbor === toId) return this.reconstructPath(fromId, toId, pred, depth + 1) + queue.push({ id: neighbor, depth: depth + 1 }) + } + } + return null + } + + /** Walk predecessor links from `toId` back to `fromId` into a forward route. */ + private reconstructPath( + fromId: string, + toId: string, + pred: Map, + cost: number + ): GraphPathResult { + const nodes: string[] = [] + const relationships: string[] = [] + let cursor = toId + while (cursor !== fromId) { + nodes.push(cursor) + const step = pred.get(cursor) as { prev: string; edgeId: string } + relationships.push(step.edgeId) + cursor = step.prev + } + nodes.push(fromId) + nodes.reverse() + relationships.reverse() + return { nodes, relationships, cost } + } + + /** Native pathfinding → resolve endpoints to ints, hydrate the returned route. */ + private async graphPathNative( + accel: GraphAccelerationProvider, + fromId: string, + toId: string, + options?: GraphPathOptions + ): Promise { + const fromInt = this.graphEntityInt(fromId) + const toInt = this.graphEntityInt(toId) + if (fromInt === undefined || toInt === undefined) return null + + const verbTypes = options?.type + ? (Array.isArray(options.type) ? options.type : [options.type]).map((t) => + TypeUtils.getVerbIndex(t) + ) + : undefined + const excluded = this.excludedVisibilityTiers(options ?? {}) + const pathOptions: PathOptions = { + ...(options?.direction !== undefined && { direction: options.direction }), + ...(options?.by !== undefined && { by: options.by }), + ...(verbTypes && { verbTypes }), + ...(excluded && { excludeVisibility: excluded as unknown as string[] }), + ...(options?.maxDepth !== undefined && { maxDepth: options.maxDepth }) + } + + const result = await accel.path(fromInt, toInt, pathOptions) + if (!result) return null + const nodes = this.entityIntsToUuids(Array.from(result.nodeInts)) + const relationships = ( + await this.graphIndex.verbIntsToIds(Array.from(result.edgeVerbInts)) + ).filter((id): id is string => id !== null) + return { nodes, relationships, cost: result.cost } + } + // ============= SEARCH & DISCOVERY ============= /** @@ -3661,6 +5868,10 @@ export class Brainy implements BrainyInterface { ): Promise { await this.ensureInitialized() this.ensureAggregationIndex() + // Persisted definitions load asynchronously — wait for them before deciding + // the aggregate doesn't exist (an app that relies on persisted definitions + // without re-defining at boot would otherwise race a spurious throw here). + await this._aggregationIndex!.ready() if (!this._aggregationIndex!.hasAggregate(name)) { throw new Error(`Aggregate '${name}' is not defined. Call defineAggregate() first.`) } @@ -3679,8 +5890,17 @@ export class Brainy implements BrainyInterface { this._aggregationIndex = new AggregationIndex(this.storage, nativeProvider) // Note: init() is async but definitions can be registered synchronously. // State loading happens lazily on first query. - this._aggregationIndex.init().catch(() => { - // Non-fatal — aggregation state will be empty but definitions still work + this._aggregationIndex.init().catch((err) => { + // Non-fatal — definitions still work and state backfills on first query — + // but a failed state load means aggregates read empty/stale until then, so + // surface it loudly rather than swallow it. + if (!this.config.silent) { + prodLog.warn( + `[Brainy] Aggregation index state failed to load at init ` + + `(${(err as Error).message}). Aggregates may read empty until a ` + + `backfill-on-query repopulates them.` + ) + } }) } @@ -3724,13 +5944,52 @@ export class Brainy implements BrainyInterface { return new Set(ids) } + /** + * @description The unified Triple-Intelligence query. Composes vector similarity + * (`query` / `vector` / `near`), metadata filtering (`type` / `subtype` / `where` / + * `service`), and graph traversal (`connected`) into one ranked, paginated result + * set — pass a bare string for a quick semantic search, or a {@link FindParams} + * object to combine dimensions. Internal/system entities are hidden by default + * (opt in with `includeInternal` / `includeSystem`); `orderBy`/`order` and + * `limit`/`offset` apply across the composed result. Every returned row is + * re-validated against its own predicate, so a stale or cross-bucket index entry + * can never surface an entity that does not actually match. + * @param query - A semantic search string, or a {@link FindParams} object combining + * any of `query`, `vector`, `near`, `type`, `subtype`, `where`, `connected`, + * `orderBy`/`order`, `limit`, `offset`. + * @returns The matching {@link Result}s — flattened (`id`, `score`, `type`, + * `metadata`, `data`, …) and ranked; an empty array when nothing matches. + * @example + * // Semantic + metadata + graph, composed in one call: + * const rows = await brain.find({ + * query: 'climate policy', + * where: { year: { gte: 2020 } }, + * connected: { from: orgId, via: VerbType.Authored }, + * limit: 10 + * }) + */ async find(query: string | FindParams): Promise[]> { - await this.ensureInitialized() + // Init only here — the migration gate is applied family-scoped below, once + // the query params reveal which index families this read actually consults + // (a string query may parse to a where / connected / vector shape). The lazy + // loader and cold-read probes below already defer to a migrating provider. + await this.ensureInitialized({ needs: [] }) // Ensure indexes are loaded (lazy loading when disableAutoRebuild: true) // This is a production-safe, concurrency-controlled lazy load await this.ensureIndexesLoaded() + // One-shot cold-open self-heal: an O(1) probe of the metadata index (when the + // provider offers one) repairs an already-poisoned index on first read — the + // metadata counterpart of the graph cold-load guard. No-op for the JS index. + await this.ensureMetadataConsistencyProbed() + + // Loudly flag a degraded derived index (failed init rebuild, or an + // adopt-forward degraded commit) so a partial result is never mistaken for + // authoritative. The streaming search() generator delegates to find(), so it + // is covered here. + this.warnIfReadsDegraded('find') + // Parse natural language queries let params: FindParams = typeof query === 'string' ? await this.parseNaturalQuery(query) : query @@ -3753,6 +6012,13 @@ export class Brainy implements BrainyInterface { // Zero-config validation (static import for performance) validateFindParams(params) + // Family-scoped migration gate: wait only on the index families THIS query + // consults, so a filter or graph query served from a healthy family never + // blocks on an unrelated family's one-time 7.x → 8.0 migration. Placed once + // params are final (after natural-language parse + connected-id resolution) + // and before the aggregate path, which carries no gate of its own. + await this.awaitMigrationLock(this.queryIndexFamilies(params)) + // Aggregate query path — early return when params.aggregate is set if (params.aggregate) { return this.findAggregate(params) @@ -3766,7 +6032,7 @@ export class Brainy implements BrainyInterface { const isHidden = (id: string): boolean => hiddenIds.size > 0 && hiddenIds.has(id) const startTime = Date.now() - const result = await (async () => { + let result = await (async () => { let results: Result[] = [] // Distinguish between search criteria (need vector search) and filter criteria (metadata only) @@ -3775,6 +6041,24 @@ export class Brainy implements BrainyInterface { const hasFilterCriteria = params.where || params.type || params.subtype || params.service const hasGraphCriteria = params.connected + // Validate the raw where clause up front: an unrecognized operator (a typo + // like `notIn`, or any non-operator key on a field's object value) throws a + // typed BrainyError('INVALID_QUERY') instead of silently matching nothing. + // Done before the index/vector/graph paths so it fires even on an empty + // result set, and before brainy injects its own internal filter fields. + if (params.where) { + validateWhereFilter(params.where) + } + + // Metadata cold-read guard: before trusting a filter result, verify the + // field index actually serves a known persisted value (one-shot per brain). + // A cold native index that has not loaded its `where` postings self-heals + // here (rebuild) or throws MetadataIndexNotReadyError — never a silent []. + // The graph counterpart (verifyGraphAdjacencyLive) covers `connected`. + if (hasFilterCriteria) { + await this.verifyMetadataLive() + } + // Handle metadata-only queries (no vector search needed) if (!hasVectorSearchCriteria && !hasGraphCriteria && hasFilterCriteria) { // Build filter for metadata index @@ -3829,15 +6113,24 @@ export class Brainy implements BrainyInterface { // Apply sorting if requested, otherwise just filter let filteredIds: string[] if (params.orderBy) { - // Get sorted IDs using production-scale sorted filtering + // Get sorted IDs using production-scale sorted filtering. Bound the sort to + // the page (offset+limit) plus the hidden over-fetch — so a broad filter + + // orderBy returning 20 rows doesn't materialize every matching sorted id. + const sortTopK = (params.offset || 0) + (params.limit || 10) + hiddenIds.size filteredIds = await this.metadataIndex.getSortedIdsForFilter( filter, params.orderBy, - params.order || 'asc' + params.order || 'asc', + sortTopK ) } else { - // Just filter without sorting - filteredIds = await this.metadataIndex.getIdsForFilter(filter) + // Just filter without sorting. Pass a page bound so a native provider can + // early-stop and return only the page-prefix (killing the O(N) FFI marshal + // at billion scale). Over-fetch by the hidden count, then re-window below; + // offset stays 0 because the visibility filter + slice happen here. The JS + // index ignores the bound and returns all matches (behaviour unchanged). + const pageEnd = (params.offset || 0) + (params.limit || 10) + hiddenIds.size + filteredIds = await this.metadataIndex.getIdsForFilter(filter, { limit: pageEnd, offset: 0 }) } // Visibility hard filter — drop hidden ids BEFORE pagination so limit is exact. @@ -3952,42 +6245,14 @@ export class Brainy implements BrainyInterface { // bitmap), JS HNSW converts to a Set-based filter function. let preResolvedMetadataIds: string[] | null = null let preResolvedFilter: any = null + // 8.0 #46 (CTX-PUSHDOWN-ALLOWEDIDS): the matched universe as an opaque roaring + // Buffer, forwarded straight into the vector beam walk with NO id materialization + // when the active metadata provider can produce one (native/cor). Absent on the + // JS path — there the materialized `candidateIds` restricts the walk instead. + let preResolvedAllowedIds: OpaqueIdSet | undefined if (params.where || params.type || params.subtype || params.service || params.excludeVFS) { - preResolvedFilter = {} - if (params.where) { - Object.assign(preResolvedFilter, params.where) - // Alias: where.type → where.noun (storage field name for entity type) - if ('type' in preResolvedFilter && !('noun' in preResolvedFilter)) { - preResolvedFilter.noun = preResolvedFilter.type - delete preResolvedFilter.type - } - } - if (params.service) preResolvedFilter.service = params.service - if (params.excludeVFS === true) { - preResolvedFilter.vfsType = { exists: false } - preResolvedFilter.isVFSEntity = { ne: true } - } - // Subtype (top-level standard field — fast path). - // Must be assigned BEFORE the type-array expansion below. - if (params.subtype !== undefined) { - preResolvedFilter.subtype = Array.isArray(params.subtype) - ? { oneOf: params.subtype } - : params.subtype - } - if (params.type) { - const types = Array.isArray(params.type) ? params.type : [params.type] - if (types.length === 1) { - preResolvedFilter.noun = types[0] - } else { - preResolvedFilter = { - anyOf: types.map(type => ({ - noun: type, - ...preResolvedFilter - })) - } - } - } + preResolvedFilter = this.buildMetadataFilter(params) preResolvedMetadataIds = await this.metadataIndex.getIdsForFilter(preResolvedFilter) // Visibility hard filter — restrict the HNSW candidate set to non-hidden ids. @@ -3999,6 +6264,19 @@ export class Brainy implements BrainyInterface { if (preResolvedMetadataIds.length === 0) { return [] } + + // 8.0 #46: when the active metadata provider exposes the opaque-set producer + // (native/cor — the JS index does not), forward the matched universe to the + // vector beam walk as an OpaqueIdSet (zero id materialization). The opaque set + // is the RAW filter universe (the set is opaque, so the visibility exclusion + // cannot be AND-ed into it in TS) — hidden tiers are instead dropped by the + // post-search visibility hard filter below, and the JS path's materialized + // `candidateIds` stays visibility-precise. Both forms are passed; the native + // provider consumes the opaque one, the JS index the string one. + const mip = this.metadataIndex as unknown as MetadataIndexProvider + if (typeof mip.getIdSetForFilter === 'function') { + preResolvedAllowedIds = await mip.getIdSetForFilter(preResolvedFilter) + } } // Zero-Config Hybrid Search @@ -4012,14 +6290,14 @@ export class Brainy implements BrainyInterface { } // Handle semantic-only query (user explicitly wants vector search) else if ((searchMode === 'semantic' || searchMode === 'vector') && (params.query || params.vector)) { - results = await this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined) + results = await this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds) } // Handle explicit hybrid or auto mode with query else if ((searchMode === 'auto' || searchMode === 'hybrid') && params.query && params.query.trim() !== '' && !params.vector) { // Zero-config hybrid: combine text + semantic search with RRF fusion const [textResults, semanticResults] = await Promise.all([ this.executeTextSearch(params.query, limit * 2), - this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined) + this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds) ]) // Use user-specified alpha or auto-detect based on query length @@ -4033,7 +6311,7 @@ export class Brainy implements BrainyInterface { } // Handle direct vector search (no query text) - no hybrid needed else if (params.vector && !params.query) { - results = await this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined) + results = await this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds) } // Handle proximity search else if (params.near) { @@ -4130,15 +6408,18 @@ export class Brainy implements BrainyInterface { if (!params.query && !params.connected) { // Apply sorting if requested for metadata-only queries if (params.orderBy) { + // Page-bounded sort (offset+limit + hidden over-fetch) so we don't + // materialize every matching sorted id to return one page. + const limit = params.limit || 10 + const offset = params.offset || 0 const sortedIds = await this.metadataIndex.getSortedIdsForFilter( preResolvedFilter, params.orderBy, - params.order || 'asc' + params.order || 'asc', + offset + limit + hiddenIds.size ) // Paginate sorted IDs BEFORE loading entities (production-scale!) - const limit = params.limit || 10 - const offset = params.offset || 0 const pageIds = sortedIds.slice(offset, offset + limit) // Batch-load entities for paginated results (10x faster on GCS) @@ -4208,10 +6489,52 @@ export class Brainy implements BrainyInterface { return results.slice(finalOffset, finalOffset + limit) })() + // Index-integrity guard — applied ONCE here so every find() path (metadata, + // vector, text, proximity, graph) is covered uniformly. The indexes are + // acceleration structures; the loaded entity is ground truth. Re-validate + // each result against the query predicate so a stale or cross-bucket index + // entry — e.g. an id left in a field-value posting by a delete or an + // `update({ field: undefined })` — can never surface an entity that does not + // actually match `type`/`subtype`/`where`/`service`/`excludeVFS`. This is + // the live-path mirror of the historical path's per-candidate + // `entityMatchesFind` (db.ts). On a healthy index it is a no-op; on a + // corrupted one it drops the bad row instead of returning a phantom. + if (result.length > 0) { + result = result.filter((r) => { + if (r.entity == null) return false + try { + return entityMatchesFind(r.entity as unknown as Entity, params as unknown as FindParams) + } catch { + // The JS matcher doesn't implement a where-operator the provider already + // matched on (e.g. a future native-only operator). The guard cannot + // DISPROVE a match the provider made, so keep the row rather than + // hard-failing a correct provider result. (The db.ts historical path + // keeps its throw, which legitimately reroutes to materialization.) + return true + } + }) + } + + // includeVectors — opt-in vector hydration. Default (false) keeps the perf + // contract: every result path above builds entities via the metadata-only + // fast path, so `entity.vector` is the empty stub. When requested, fetch the + // stored vectors for the already-paginated page in ONE batch and attach them, + // mirroring get({ includeVectors: true }). Done once here so every find() + // path (metadata-only, vector/text/proximity, graph) honors it uniformly. + if (params.includeVectors && result.length > 0) { + const nounsMap = await this.storage.getNounBatch(result.map((r) => r.id)) + for (const r of result) { + const noun = nounsMap.get(r.id) + if (noun?.vector && r.entity) { + r.entity.vector = noun.vector + } + } + } + // Record performance for auto-tuning const duration = Date.now() - startTime recordQueryPerformance(duration, result.length) - + return result } @@ -4387,10 +6710,13 @@ export class Brainy implements BrainyInterface { /** * Add multiple entities in a single batch operation * - * Uses batch embedding (embedBatch) to pre-compute all vectors in a single - * WASM forward pass instead of N individual embed() calls, providing 5-10x - * speedup on bulk inserts. Automatically adapts batch size and parallelism - * to the storage adapter (e.g., smaller batches for cloud storage). + * Uses batch embedding (embedBatch) to pre-compute all vectors before any + * storage write, keeping model inference out of the per-item write path. + * (On the default WASM engine, batch throughput is comparable to sequential + * embed() calls — measured ~160 ms/text either way; a native embedding + * provider may batch faster.) Automatically adapts batch size and + * parallelism to the storage adapter (e.g., smaller batches for cloud + * storage). * * @param params - Batch add parameters * @param params.items - Array of AddParams (same shape as brain.add()) @@ -4506,12 +6832,22 @@ export class Brainy implements BrainyInterface { const promises = chunk.map(async (item) => { try { - // ifAbsent (7.31.0) — propagate batch-level flag to each item, but let - // per-item flag take precedence so callers can override individual rows. - const itemWithIfAbsent = params.ifAbsent && item.ifAbsent === undefined - ? { ...item, ifAbsent: true } - : item - const id = await this.add(itemWithIfAbsent) + // ifAbsent (7.31.0) / upsert — propagate each batch-level flag to every + // item, but let a per-item flag take precedence so callers can override + // individual rows. The two flags stay independent: per-item upsert handling + // (create-or-merge) and per-item ifAbsent handling (create-or-skip) both + // fire inside the delegated add() call, which also enforces their mutual + // exclusion when a single resolved item carries both. + const resolvedItem = + (params.ifAbsent && item.ifAbsent === undefined) || + (params.upsert && item.upsert === undefined) + ? { + ...item, + ...(params.ifAbsent && item.ifAbsent === undefined && { ifAbsent: true }), + ...(params.upsert && item.upsert === undefined && { upsert: true }) + } + : item + const id = await this.add(resolvedItem) result.successful.push(id) } catch (error) { result.failed.push({ @@ -4578,6 +6914,30 @@ export class Brainy implements BrainyInterface { this.assertWritable('removeMany') await this.ensureInitialized() + // Loud selector validation: a call with no usable selector used to + // resolve successfully having deleted NOTHING (total: 0) — the classic + // silent no-op being a bare array passed positionally + // (removeMany([id]) instead of removeMany({ ids: [id] })). The caller + // believes the delete happened; every count derived afterwards is "wrong" + // while the engine was never even asked. Refuse instead. + if (Array.isArray(params)) { + throw new Error( + `removeMany() takes a params object, not a bare array — use removeMany({ ids: [...] })` + ) + } + if (!params || (!params.ids && !params.type && !params.where)) { + throw new Error( + `removeMany() requires a selector: { ids } and/or { type, where }. ` + + `An empty selector would silently delete nothing — refusing.` + ) + } + if (params.ids && params.ids.length === 0) { + throw new Error( + `removeMany() received ids: [] — an empty id list deletes nothing. ` + + `Pass the ids to delete, or omit ids and select by { type, where }.` + ) + } + // Determine what to delete let idsToDelete: string[] = [] @@ -4605,9 +6965,12 @@ export class Brainy implements BrainyInterface { const startTime = Date.now() - // Batch deletes into chunks for 10x faster performance with proper error handling - // Single transaction per chunk (10 entities) = atomic within chunk, graceful failure across chunks - const chunkSize = 10 + // Batch deletes into chunks for 10x faster performance with proper error handling. + // Single transaction per chunk = atomic within chunk, graceful failure across chunks. + // Chunk size is storage-adaptive (matching addMany/relateMany): the adapter's + // maxBatchSize (memory: 1000, S3/R2: 100, GCS: 50) unless the caller overrides it. + const storageConfig = this.storage.getBatchConfig() + const chunkSize = params.chunkSize ?? storageConfig.maxBatchSize for (let i = 0; i < idsToDelete.length; i += chunkSize) { const chunk = idsToDelete.slice(i, i + chunkSize) @@ -4619,9 +6982,37 @@ export class Brainy implements BrainyInterface { const chunkQueued: string[] = [] const chunkBuilderFailed: Array<{ item: string; error: string }> = [] + // Model-B pre-pass: collect the chunk's touched-id set (entities + their + // cascade-deleted relationships) so the generation captures before-images + // for all of them. Tolerant by design — the builder below still does the + // authoritative per-id load + error handling, and an over-included id whose + // delete is skipped simply gets a before-image equal to its live state (a + // harmless no-op history entry). + const touchedNouns: string[] = [...chunk] + const touchedVerbs: string[] = [] + for (const id of chunk) { + try { + const srcVerbs = await this.storage.getVerbsBySource(id) + const tgtVerbs = await this.storage.getVerbsByTarget(id) + for (const v of [...srcVerbs, ...tgtVerbs]) touchedVerbs.push(v.id) + } catch { + // Ignore — the builder's per-id try/catch is authoritative. + } + } + try { - // Process chunk in single transaction for atomic deletion - await this.transactionManager.executeTransaction(async (tx) => { + // Change feed: the BUILDER populates this (per id actually staged), so + // an id whose builder failed never emits — the array is read only + // after the chunk's commit succeeds. Verb events dedupe within the + // chunk (two related chunk-members see the same cascade verb twice). + const chunkEvents: PendingChangeEvent[] | undefined = this._changeFeed + .hasListeners + ? [] + : undefined + const seenVerbEvents = new Set() + + // Process chunk as ONE atomic Model-B generation (entities + cascade verbs). + await this.persistSingleOp({ nouns: touchedNouns, verbs: touchedVerbs }, async (tx) => { for (const id of chunk) { try { // Load entity data @@ -4644,20 +7035,51 @@ export class Brainy implements BrainyInterface { ) } + // Pre-read metadata rides along: the count decrement must not + // depend on re-reading the record being removed (see remove()). tx.addOperation( - new DeleteNounMetadataOperation(this.storage, id) + new DeleteNounMetadataOperation(this.storage, id, metadata) ) for (const verb of allVerbs) { const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb) tx.addOperation( - new RemoveFromGraphIndexOperation(this.graphIndex, verb, sourceInt, targetInt, this.graphWriteGeneration) + new RemoveFromGraphIndexOperation(this.graphIndex, verb, { sourceInt, targetInt }, this.graphWriteGeneration) ) tx.addOperation( new DeleteVerbMetadataOperation(this.storage, verb.id) ) } + if (chunkEvents) { + chunkEvents.push({ + kind: 'entity', + op: 'remove', + id, + ...(metadata && { + entity: this.entityViewFromRawRecord( + id, + metadata as Record + ) + }) + }) + for (const verb of allVerbs) { + if (seenVerbEvents.has(verb.id)) continue + seenVerbEvents.add(verb.id) + chunkEvents.push({ + kind: 'relation', + op: 'unrelate', + id: verb.id, + relation: { + id: verb.id, + from: verb.sourceId, + to: verb.targetId, + type: String(verb.verb) + } + }) + } + } + chunkQueued.push(id) } catch (error) { chunkBuilderFailed.push({ @@ -4669,7 +7091,7 @@ export class Brainy implements BrainyInterface { } } } - }) + }, undefined, chunkEvents) // Transaction committed — queued IDs were actually deleted result.successful.push(...chunkQueued) @@ -4975,6 +7397,12 @@ export class Brainy implements BrainyInterface { // VFS was never used, reset flag for clean state this._vfsInitialized = false } + + // Change feed: clear() wipes the store wholesale (raw, not per-record) — + // one store-level event tells subscribers everything they held is gone. + this._changeFeed.emit([ + { kind: 'store', op: 'clear', timestamp: Date.now() } + ]) } // ─── Migration API ─────────────────────────────────────────────── @@ -5112,15 +7540,124 @@ export class Brainy implements BrainyInterface { return this.generationStore.generation() } + /** + * @description The data-layer + derived-index format this brain instance runs + * as — the SYNCHRONOUS half of the 7.x → 8.0 version handshake. Returns the + * compiled {@link CURRENT_DATA_FORMAT} / {@link EXPECTED_INDEX_EPOCH} + * constants (the single source of truth shared with the native provider, in + * `src/storage/brainFormat.ts`): after `init()` the brain has reconciled any + * on-disk drift and IS running the current format, so this reports what it + * runs as, not necessarily the (possibly older) marker it opened. + * + * This is a pure in-memory constant read — no `await`, no storage I/O — by + * design: a native metadata/index provider calls it synchronously at its own + * `init()` (which runs during this brain's provider-construction phase, after + * the marker has been loaded) to confirm "running data-format X, index epoch + * N" before binding its native readers. + * + * The on-disk marker (`_system/brain-format.json`) is reconciled separately: + * on open Brainy compares the on-disk `indexEpoch` against + * {@link EXPECTED_INDEX_EPOCH}; a mismatch or an absent marker rebuilds the + * derived indexes and re-stamps the marker (see `rebuildIndexesIfNeeded`). + * + * @returns `{ dataFormat, indexEpoch }` for the running build. + * @example + * const { dataFormat, indexEpoch } = brain.formatInfo() + * // dataFormat === '8.0', indexEpoch === 1 + */ + formatInfo(): BrainFormat { + return { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH } + } + + /** + * @description Stamp `_system/brain-format.json` with this build's + * `{ dataFormat, indexEpoch }` ({@link CURRENT_DATA_FORMAT} / + * {@link EXPECTED_INDEX_EPOCH}). A native provider calls this once its + * background index migration (the no-freeze path, where brainy DEFERRED the + * rebuild because the provider's `isMigrating()` was true) has + * verified-and-swapped all derived indexes — brainy authors `dataFormat`, the + * provider never does. The marker is the single switch that declares the + * on-disk derived indexes current for this build's epoch, so it is advanced + * only after the indexes it certifies are in place. Unconditional and + * idempotent (writes the same two compiled constants every call); a no-op for + * read-only brains, which never author markers. + * @returns Resolves once the marker is durable on disk. + * @example + * // cor, after its build-new→verify→swap completes: + * await brain.stampBrainFormat() + */ + async stampBrainFormat(): Promise { + if (this.isReadOnly) return + await writeBrainFormat(this.storage) + this._brainFormat = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH } + this._indexEpochStale = false + // The upgrade has verified + stamped — drop the pre-upgrade backup (no-op if + // none was taken, e.g. a non-migrating stamp of a fresh brain's initial marker). + await this.removeMigrationBackupSafe() + } + + /** + * @description Open a sequential scan over the generation FACT LOG — the + * append-only record of every committed generation as an AFTER-IMAGE fact + * (what each touched entity/relationship became, or a body-less tombstone + * for a removal). The scan is the streaming substrate for index heals and + * incremental replays: one sequential read in commit order replaces a + * per-entity directory walk. The handle carries heal-narration telemetry + * (`headGeneration` / `segmentCount` / `approxFactCount` up front; ordered + * batches each stamped with their generation range, byte size, and segment; + * a `summary()` cross-check after iteration). A detected gap or damaged + * segment ABORTS the scan loudly — never a silent skip. + * + * Returns `null` when this store hosts no fact log: the storage adapter + * lacks the binary append primitives, or the brain predates the fact log + * (its history began before dual-write — facts exist only from the first + * write after upgrade; callers fall back to the canonical enumeration walk). + * + * @param options - `fromGeneration`/`toGeneration` bound the scan (inclusive + * both ends); `kinds` filters ops to `'noun'`/`'verb'`; `batchSize` caps + * facts per yielded batch (default 256). + * @returns The scan handle, or `null` when no fact log exists. + * @example + * const scan = brain.scanFacts({ fromGeneration: 1 }) + * if (scan) { + * for await (const batch of scan.batches()) { + * for (const fact of batch.facts) { + * // fact.ops: [{ kind, id, record | null (tombstone) }, ...] + * } + * } + * } + */ + scanFacts(options?: { + fromGeneration?: number + toGeneration?: number + kinds?: Array<'noun' | 'verb'> + batchSize?: number + }): FactScanHandle | null { + const factLog = this.generationStore?.getFactLog() + return factLog ? factLog.scanFacts(options) : null + } + + /** + * @description The immutable, sealed fact-log segment files covering + * `fromGeneration` — the zero-copy handoff for consumers that map segment + * files directly instead of streaming {@link scanFacts} batches. The + * append-mutable TAIL segment is deliberately excluded (read it via + * `scanFacts`). Paths are storage-root-relative. Empty when no fact log + * exists or nothing is sealed yet. + */ + factSegmentPaths(options?: { fromGeneration?: number }): string[] { + return this.generationStore?.getFactLog()?.segmentPaths(options) ?? [] + } + /** * @description Read the reified transaction log — one entry per committed - * `transact()` batch, carrying the committed generation, the commit - * timestamp, and the `meta` the transaction was submitted with. Entries - * are returned newest first. Single-operation writes - * (`add`/`update`/`remove`/`relate`/…) advance the generation counter but - * do not append log entries (the documented 8.0 history granularity), and - * `compactHistory()` never rewrites the log — entries may reference - * generations whose record-sets were already reclaimed. + * generation, carrying the committed generation, the commit timestamp, and + * the `meta` the transaction was submitted with. Entries are returned newest + * first. Under Model-B EVERY write is its own generation, so single-operation + * writes (`add`/`update`/`remove`/`relate`/…) appear here too (without `meta` + * — transaction metadata is a `transact()`-only concept). `compactHistory()` + * never rewrites the log — entries may reference generations whose + * record-sets were already reclaimed. * * @param options - `from`/`to` (generation number or `Date`) bound the window * to commits in `[from, to]` INCLUSIVE on both ends — a log window names @@ -5269,6 +7806,11 @@ export class Brainy implements BrainyInterface { throw new Error('transact(): provide a non-empty array of transaction operations') } + // Commit any buffered single-op generations FIRST so this transaction's + // generation lands strictly after them and the committed-generation + // sequence stays gap-free (single-ops reserve generations eagerly). + await this.generationStore.flushPendingSingleOps() + // Early CAS check — avoids expensive planning (embedding) on a stale // expectation. The generation store re-checks authoritatively under the // commit mutex; this one is purely a fast-fail. @@ -5284,18 +7826,78 @@ export class Brainy implements BrainyInterface { const plan = await this.planTransact(ops) - const { generation, timestamp } = await this.generationStore.commitTransaction({ - touched: { nouns: plan.touchedNouns, verbs: plan.touchedVerbs }, - meta: options?.meta, - ifAtGeneration: options?.ifAtGeneration, - execute: async () => { - await this.transactionManager.executeTransaction(async (tx) => { - for (const operation of plan.operations) { - tx.addOperation(operation) + // Authoritative per-op ifRev CAS, run by the generation store UNDER the + // commit mutex against the just-read before-images — atomic with the + // apply (the planning-time checks above are fast-fails that can + // interleave with concurrent writers). Any conflict rejects the WHOLE + // batch before anything is staged or applied. Sequencing rules: + // - runningRev tracks multiple updates to the SAME entity in one batch + // (each op CASes against — and bumps — its predecessor's result). + // - An entity the batch itself creates (null before-image, forward-ref + // add+update) keeps its plan-sequenced stamps: no concurrent writer + // can have moved a rev that did not exist, so plan-time was exact. + const casPrecommit = (before: CommitBeforeImages): void => { + const runningRev = new Map() + for (const cas of plan.casUpdates) { + // An id this batch adds owns its revision baseline (the add restamps + // `_rev: 1` even on overwrite) — plan-time sequencing is exact, no + // concurrent writer can move a baseline the batch itself sets. + if (plan.createdNouns.has(cas.id)) { + continue + } + const beforeMeta = before.nouns.get(cas.id)?.metadata as + | { _rev?: unknown } + | null + | undefined + if (!beforeMeta && !runningRev.has(cas.id)) { + // A pre-existing entity a concurrent remove() deleted after + // planning: a CAS caller gets the truth; a plain update keeps its + // last-writer-wins re-create semantics (plan-stamped rev). + if (typeof cas.ifRev === 'number') { + throw new EntityNotFoundError(cas.id) } - }) + continue + } + const base = + runningRev.get(cas.id) ?? + (typeof beforeMeta?._rev === 'number' ? beforeMeta._rev : 1) + if (typeof cas.ifRev === 'number' && cas.ifRev !== base) { + throw new RevisionConflictError(cas.id, cas.ifRev, base) + } + const next = base + 1 + cas.updatedMetadata._rev = next + runningRev.set(cas.id, next) } - }) + } + + let generation: number + let timestamp: number + try { + ;({ generation, timestamp } = await this.generationStore.commitTransaction({ + touched: { nouns: plan.touchedNouns, verbs: plan.touchedVerbs }, + meta: options?.meta, + ifAtGeneration: options?.ifAtGeneration, + precommit: casPrecommit, + execute: async () => { + await this.transactionManager.executeTransaction( + async (tx) => { + for (const operation of plan.operations) { + tx.addOperation(operation) + } + }, + // Budget scales with the batch (or the caller's explicit + // timeoutMs): a flat 30s cap silently limited honest bulk work + // to ~15 ops on network disks (~2s/op measured in the field). + { timeout: transactTimeoutBudget(plan.operations.length, options?.timeoutMs) } + ) + } + })) + } catch (err) { + // A batch whose rollback failed and left canonical records unreconciled + // quarantines writes until repairIndex() reconciles and lifts it. + if (err instanceof StoreInconsistentError) this.storeInconsistency = err + throw err + } // Aggregation-index maintenance: derived data, applied after the commit // point — exactly where the single-operation methods apply it. @@ -5303,6 +7905,10 @@ export class Brainy implements BrainyInterface { hook() } + // Change feed: the batch's events share its single committed generation. + // A rejected batch throws at commitTransaction and never reaches here. + this.emitCommitted(plan.changeEvents, undefined, generation, timestamp) + const receipt: TransactReceipt = { generation, timestamp, ids: plan.ids } return this.createPinnedDb({ generation, timestamp, receipt }) } @@ -5311,9 +7917,9 @@ export class Brainy implements BrainyInterface { * @description Open an immutable `Db` view of PAST state: * * - **`number`** — a generation: pins it on this store; reads resolve - * through the generational record layer. History granularity is - * `transact()` commits — single-operation writes between commits do not - * produce records and remain visible through earlier pins. + * through the generational record layer. Under Model-B every write is its + * own generation, so single-operation writes are individually addressable + * too (a pin always freezes against later single-op writes). * - **`Date`** — a wall-clock instant: resolved via the transaction log to * the newest generation committed at or before it, then pinned as above. * - **`string`** — a snapshot directory previously produced by @@ -5643,57 +8249,185 @@ export class Brainy implements BrainyInterface { * keep (the GC backstop eventually releases leaked ones, but until then * compaction retains the history they protect). * - * @param options - `retainGenerations` (keep the N most recent committed - * generations) and/or `retainMs` (keep everything committed within the - * window). Both supplied = both must allow reclaim. Neither = reclaim - * everything unpinned. + * @param options - Explicit retention CAPS (`maxGenerations` / `maxAge` / + * `maxBytes`). Reclaim the oldest unpinned generations while ANY supplied + * cap is exceeded. Neither = reclaim everything unpinned. * @returns Count of reclaimed record-sets and the new horizon. * @example - * await brain.compactHistory({ retainGenerations: 100, retainMs: 7 * 86_400_000 }) + * await brain.compactHistory({ maxGenerations: 100, maxAge: 7 * 86_400_000 }) */ async compactHistory(options?: CompactHistoryOptions): Promise { this.assertWritable('compactHistory') await this.ensureInitialized() + // Persist buffered single-op generations first so compaction reasons over a + // complete on-disk history (the pending tier is never itself reclaimable). + await this.generationStore.flushPendingSingleOps() return this.generationStore.compact(options) } /** - * @description Resolve the effective generational-history retention policy - * from `config.history`, filling per-field defaults so a partial object - * inherits the rest. This is the single read site for the policy referenced - * by {@link autoCompactHistory}. - * - * Defaults (zero-config): keep the 100 most recent committed generations - * AND everything committed within the last 7 days; auto-compact on - * `flush()` / `close()`. A generation is reclaimed only when it is older - * than BOTH bounds and no live `Db` pin protects it. - * - * @returns The resolved policy: retention bounds plus the `autoCompact` flag. + * @description Repack cold generation history into sealed segments — + * re-representation, never deletion: every record and delta stays readable + * (`asOf()` unchanged); the physical file count drops by orders of + * magnitude. Runs automatically (time-bounded) at `close()`; call this for + * explicit maintenance windows on long-lived writers. The ONLY history + * transform permitted under the archival profile (`retention: 'all'`). + * @param options - `timeBudgetMs` bounds the pass (early stop = consistent + * prefix, next pass resumes); `batchGenerations` sizes each fold. + * @returns Folded generation count and segments created. */ - private resolveHistoryPolicy(): { - retainGenerations: number - retainMs: number - autoCompact: boolean - } { - const history = this.config.history + async repackHistory(options?: { + timeBudgetMs?: number + batchGenerations?: number + }): Promise<{ foldedGenerations: number; segmentsCreated: number }> { + this.assertWritable('repackHistory') + await this.ensureInitialized() + await this.generationStore.flushPendingSingleOps() + return this.generationStore.repackHistory(options) + } + + /** + * @description Read-only generational-history footprint for fleet audits: + * generation count, total on-disk bytes, generation/timestamp range, the + * compaction horizon, and the retention policy in force. Touches no data and + * changes nothing. First call pays one walk over committed deltas to seed + * the running byte total (subsequent calls — and every adaptive retention + * check — are then O(1)). + * + * A pool operator's exposure check is one call per brain: + * @example + * const stats = await brain.historyStats() + * console.log(`${stats.generations} generations, ${stats.bytes} bytes, mode=${stats.retentionMode}`) + */ + async historyStats(): Promise { + await this.ensureInitialized() + const stats = await this.generationStore.historyStats() + const policy = this.resolveRetentionPolicy() return { - retainGenerations: history?.retainGenerations ?? 100, - retainMs: history?.retainMs ?? 604_800_000, // 7 days - autoCompact: history?.autoCompact ?? true + ...stats, + retentionMode: policy.mode, + effectiveBudgetBytes: + policy.mode === 'adaptive' + ? this.adaptiveHistoryBudgetBytes(policy.budgetBytes) + : null } } /** - * @description Run history compaction with the resolved retention policy if - * `config.history.autoCompact` is on (the default). Invoked from `flush()` - * and `close()` so generational record-sets cannot accumulate unbounded - * across a long-lived writer's lifetime. + * @description A deterministic content digest of the generation log through + * `g` (D8 — gate-to-generation provenance): identical history produces the + * identical digest on any machine; divergence produces a different one. + * Release gates and suite verdicts pin `{generation, digest}` and verify + * both at execution time instead of pinning a git commit. O(segments + + * live-tier window), never O(all generations). Throws `RangeError` out of + * range and `GenerationCompactedError` below the horizon — a gate can + * never silently pin reclaimed history. + * @example + * const gate = { generation: brain.generation(), digest: await brain.generationDigest(brain.generation()) } + */ + async generationDigest(g: number): Promise { + await this.ensureInitialized() + await this.generationStore.flushPendingSingleOps() + return this.generationStore.generationDigest(g) + } + + /** + * @description Drive the adaptive retention byte budget at runtime — the + * settable input a machine-level coordinator (e.g. cor's `ResourceManager`, + * which fair-shares one box-wide history budget across co-located instances) + * pushes whenever the box's free-resource picture changes. Takes effect at the + * next auto-compaction (flush/close). Only meaningful while `retention` is + * adaptive (unset or `'adaptive'`); ignored under `'all'` or explicit caps. + * @param bytes - The new per-brain history byte budget (non-negative). + * @example brain.setRetentionBudget(2 * 1024 ** 3) // 2 GiB for this brain + */ + setRetentionBudget(bytes: number): void { + if (!Number.isFinite(bytes) || bytes < 0) { + throw new RangeError(`setRetentionBudget(): bytes must be a non-negative finite number (got ${bytes})`) + } + this._retentionBudgetBytes = bytes + } + + /** + * @description Resolve the effective `retention` policy from `config.retention`. + * The single read site for the policy {@link autoCompactHistory} applies. * + * - `'all'` → `{ mode: 'all' }` (never reclaim history). + * - unset / `'adaptive'` → `{ mode: 'adaptive' }` (budget-driven; the budget + * is `setRetentionBudget()`/`budgetBytes` when set, else a local probe). + * - `{ … }` → `{ mode: 'explicit', maxGenerations?, maxAge?, maxBytes? }`. + * + * `autoCompact` defaults to `true` in every mode. + * @returns The normalized retention policy. + */ + private resolveRetentionPolicy(): { + mode: 'all' | 'adaptive' | 'explicit' + maxGenerations?: number + maxAge?: number + maxBytes?: number + budgetBytes?: number + autoCompact: boolean + } { + const retention = this.config.retention + if (retention === 'all') { + return { mode: 'all', autoCompact: true } + } + if (retention === undefined || retention === 'adaptive') { + return { mode: 'adaptive', autoCompact: true } + } + return { + mode: 'explicit', + maxGenerations: retention.maxGenerations, + maxAge: retention.maxAge, + maxBytes: retention.maxBytes, + budgetBytes: retention.budgetBytes, + autoCompact: retention.autoCompact ?? true + } + } + + /** + * @description Compute the adaptive history byte budget for this brain: the + * coordinator-driven value when present (`setRetentionBudget()` / + * `config.retention.budgetBytes` — e.g. cor's fair-shared box budget), else a + * conservative LOCAL probe of free resources. Standalone brainy must not be + * cor-dependent, so the fallback uses `os.freemem()` (and is intentionally + * generous — a single instance owning the box can keep a lot of history). + * @returns The byte budget; `Infinity` only if no signal is available. + */ + private adaptiveHistoryBudgetBytes(driven?: number): number { + if (driven !== undefined) return driven + if (this._retentionBudgetBytes !== undefined) return this._retentionBudgetBytes + // Local single-instance probe: allot a quarter of currently-free RAM to + // history. Bounded, zero-config, and never reclaims pinned generations. + try { + const free = os.freemem() + if (Number.isFinite(free) && free > 0) return Math.floor(free / 4) + } catch { + // os unavailable (exotic runtime) — fall through to unbounded. + } + return Infinity + } + + /** + * @description Run history compaction under the resolved `retention` policy + * when `autoCompact` is on (the default). Invoked from `close()` ONLY + * (8.9.0) — flush() is durability work and never pays maintenance costs; a + * production deployment measured reclaim-on-flush blocking single writes + * for 25-191s under memory pressure. Long-lived writers that never close + * accumulate history until their next explicit `compactHistory()` — the + * documented trade: predictable writes, explicit maintenance. + * + * - `'all'` → returns without reclaiming (index compaction for speed still + * runs elsewhere; history is decoupled and kept). + * - `'adaptive'` → reclaim oldest-unpinned history down to the byte budget. + * - `'explicit'` → apply the supplied `maxGenerations`/`maxAge`/`maxBytes` caps. + * + * Every auto pass is TIME-BOUNDED ({@link CLOSE_COMPACTION_BUDGET_MS}) so a + * large backlog can never stall a clean shutdown; the next pass resumes. * Read-only instances and an explicit `autoCompact: false` skip silently. - * Pinned generations (live `Db` values) are never reclaimed — the safety - * invariant lives in {@link GenerationStore.compact}. Failures are logged - * and swallowed: compaction is housekeeping and must never fail a flush or - * a clean shutdown. + * Pinned generations are never reclaimed ({@link GenerationStore.compact}). + * Failures are logged and swallowed — compaction is housekeeping and must + * never fail a clean shutdown. */ private async autoCompactHistory(): Promise { // Nothing to compact on a read-only instance or before init wired up the @@ -5701,15 +8435,26 @@ export class Brainy implements BrainyInterface { if (this.isReadOnly || !this.generationStore) { return } - const policy = this.resolveHistoryPolicy() - if (!policy.autoCompact) { + const policy = this.resolveRetentionPolicy() + if (!policy.autoCompact || policy.mode === 'all') { return } try { - await this.generationStore.compact({ - retainGenerations: policy.retainGenerations, - retainMs: policy.retainMs - }) + if (policy.mode === 'adaptive') { + const budget = this.adaptiveHistoryBudgetBytes(policy.budgetBytes) + if (budget === Infinity) return // no pressure signal → keep everything this pass + await this.generationStore.compact({ + maxBytes: budget, + timeBudgetMs: CLOSE_COMPACTION_BUDGET_MS + }) + } else { + await this.generationStore.compact({ + maxGenerations: policy.maxGenerations, + maxAge: policy.maxAge, + maxBytes: policy.maxBytes, + timeBudgetMs: CLOSE_COMPACTION_BUDGET_MS + }) + } } catch (error) { console.warn( `Auto-compaction of generational history failed (non-fatal): ${ @@ -5732,6 +8477,14 @@ export class Brainy implements BrainyInterface { * replacement. Release them first; a warning is logged when live pins * exist. * + * NON-DESTRUCTIVE STAGING + SPARSE-AWARE: the snapshot is copied into a + * staging area BEFORE any live data is touched (all-zero blocks stay + * holes, so a sparse mmap store restores at its true allocated size — not + * its apparent size). Any copy failure, ENOSPC included, removes only the + * staging and throws; the live store is untouched. Only after the copy + * succeeds does an atomic per-entry swap move it into place; a crash + * mid-swap completes FORWARD on the next open. + * * @param path - Snapshot directory to restore from. * @param options - Must be `{ confirm: true }` — an explicit acknowledgment * that current state is destroyed. @@ -5763,13 +8516,31 @@ export class Brainy implements BrainyInterface { await this.storage.restoreFromDirectory(path) await this.generationStore.reopenAfterRestore(floorGeneration) - // Every in-memory index is now stale — rebuild all three from the - // restored canonical records (each rebuild clears its own state first). + // If the entity-id mapper is a NATIVE provider with a `rebuild()`, reload it + // from the restored snapshot FIRST. The graph index resolves every verb + // endpoint (sourceId/targetId → stable int) through this mapper, so a native + // adjacency (keyed on those ints) needs the mapper to reflect the snapshot's + // assignments BEFORE graphIndex.rebuild() — otherwise edges resolve to + // stale/missing ints and are silently dropped (CTX-BR-RESTORE-REBUILD). The + // JS mapper has no `rebuild()` and needs no reload: MetadataIndex.rebuild() + // re-derives it from the restored entities via append-only getOrAssign, + // consistently with the bitmaps it builds. + await this.hydrateIdMapperForGraphRebuild() + + // Every in-memory index is now stale — rebuild all three from the restored + // canonical records (each rebuild clears its own state first). The graph + // rebuild resolves endpoints via the (now-reloaded, for native) mapper. await Promise.all([ this.metadataIndex.rebuild(), this.index.rebuild(), this.graphIndex.rebuild() ]) + + // Change feed: a restore is a wholesale raw-state replacement, not a + // per-record commit — one store-level event tells subscribers to refetch. + this._changeFeed.emit([ + { kind: 'store', op: 'restore', timestamp: Date.now() } + ]) } /** @@ -5872,6 +8643,9 @@ export class Brainy implements BrainyInterface { persistPinned: (targetPath, generation) => this.persistPinnedGeneration(targetPath, generation), materializeAt: (generation) => this.materializeAtGeneration(generation), + canServeVectorAtGeneration: (generation) => this.canServeVectorAtGeneration(generation), + vectorSearchAtGeneration: (params, allowedIds, k, generation) => + this.vectorSearchAtGeneration(params, allowedIds, k, generation), pinGeneration: (generation) => this.pinGeneration(generation), releaseGeneration: (generation) => this.releaseGeneration(generation), registerDbForFinalization: (db, generation, closeOnRelease) => { @@ -6083,23 +8857,42 @@ export class Brainy implements BrainyInterface { const snapshotStorage = new MemoryStorage() await snapshotStorage.init() - /** Copy one id's at-`generation` state into the snapshot (or ensure absence). */ - const copyAt = async (kind: 'noun' | 'verb', id: string): Promise => { - const resolved = await this.generationStore.resolveAt(kind, id, generation) + // Copy one id's at-`generation` state into the snapshot (or ensure absence) + // as a PURE LOOKUP against a precomputed `firstAfter` map (id → the first + // generation after the pin that touched it). An id absent from the map was + // untouched since the pin → its live storage state IS its at-`generation` + // state; an id present resolves from that generation's immutable + // before-image. Both `resolveManyAt` (builds the map) and + // `readGenerationRecord` (reads the before-image) are MUTEX-FREE, so this is + // safe to call inside the reconciliation pass below (which holds the commit + // mutex via `snapshotWith`) — a per-id `resolveAt` there would re-enter the + // mutex and DEADLOCK, and would regress the O(R) bulk copy to O(N·R). + const copyAt = async ( + kind: 'noun' | 'verb', + id: string, + firstAfter: Map + ): Promise => { let record: { metadata: any; vector: any | null } | null = null - if (resolved.source === 'record') { - if (resolved.metadata !== null) { - record = { metadata: resolved.metadata, vector: resolved.vector } - } - } else if (resolved.source === 'current') { + const candidate = firstAfter.get(id) + if (candidate === undefined) { + // Untouched since the pin → live storage IS the at-`generation` state. const raw = kind === 'noun' ? await this.storage.readNounRaw(id) : await this.storage.readVerbRaw(id) - if (raw.metadata !== null) { - record = raw + if (raw.metadata !== null) record = raw + } else { + const before = await this.generationStore.readGenerationRecord(kind, candidate, id) + if (before === null) { + throw new Error( + `Generation record missing: _generations/${candidate}/prev/${id}.json ` + + `(store corrupted or records removed outside compactHistory())` + ) } + // A non-null metadata before-image is the at-`generation` live state; a + // null-metadata before-image (the create sentinel, or a metadata-less + // stored state) is absent at this generation. + if (before.metadata !== null) record = { metadata: before.metadata, vector: before.vector } } - // `record === null` — absent at this generation (or a metadata-less - // stored state, which is not a live entity): writing nulls ensures the + // `record === null` — absent at this generation: writing nulls ensures the // id is absent in the snapshot, which also un-copies ids created by // transactions that commit mid-build. if (kind === 'noun') { @@ -6124,25 +8917,42 @@ export class Brainy implements BrainyInterface { const id = entityIdFromCanonicalPath(path) if (id) verbIds.add(id) } - for (const id of nounIds) await copyAt('noun', id) - for (const id of verbIds) await copyAt('verb', id) + // ONE ascending pass per kind resolves the whole universe's first-after + // generations (O(R) getDelta reads, NOT O(N·R)), then each copy is a lookup. + const nounFirstAfter = await this.generationStore.resolveManyAt('noun', nounIds, generation) + const verbFirstAfter = await this.generationStore.resolveManyAt('verb', verbIds, generation) + for (const id of nounIds) await copyAt('noun', id, nounFirstAfter) + for (const id of verbIds) await copyAt('verb', id, verbFirstAfter) // Reconciliation pass under the commit mutex: nothing can commit during // this section, so afterwards the snapshot is exactly the at-`generation` // record set even when transactions committed while the bulk copy ran. // Reconciled ids join the enumeration sets so the vector-index build - // below sees them too. + // below sees them too. `resolveManyAt`/`readGenerationRecord` are mutex-free, + // so this section never re-enters the commit mutex (no deadlock). await this.generationStore.snapshotWith(async () => { const committedNow = this.generationStore.committedGeneration() if (committedNow === watermark) return const delta = await this.generationStore.changedBetween(watermark, committedNow) - for (const id of delta.nouns) { + const reconNouns = new Set(delta.nouns) + const reconVerbs = new Set(delta.verbs) + const reconNounFirstAfter = await this.generationStore.resolveManyAt( + 'noun', + reconNouns, + generation + ) + const reconVerbFirstAfter = await this.generationStore.resolveManyAt( + 'verb', + reconVerbs, + generation + ) + for (const id of reconNouns) { nounIds.add(id) - await copyAt('noun', id) + await copyAt('noun', id, reconNounFirstAfter) } - for (const id of delta.verbs) { + for (const id of reconVerbs) { verbIds.add(id) - await copyAt('verb', id) + await copyAt('verb', id, reconVerbFirstAfter) } watermark = committedNow }) @@ -6194,6 +9004,102 @@ export class Brainy implements BrainyInterface { } } + /** + * @description 8.0 #35 — true when the vector index is a + * {@link VersionedIndexProvider} that advertises `isGenerationVisible(generation)`: + * it can serve the at-`generation` vector leg from retained segments, so a + * historical filtered semantic read needs no O(n@G) JS-HNSW rebuild. The built-in + * `JsHnswVectorIndex` is not versioned (only ever "now"), and a native provider + * that cannot honor `generation` MUST return `false` here (refuse, never fabricate + * now-vectors-as-at-gen), so the caller falls back to materialization. + */ + private canServeVectorAtGeneration(generation: number): boolean { + return ( + isVersionedIndexProvider(this.index) && + this.index.isGenerationVisible(BigInt(generation)) + ) + } + + /** + * @description 8.0 #35 — run the vector kNN AS OF `generation`, restricted to + * `allowedIds` (the at-gen metadata∩graph universe the `Db` resolved from the + * record layer). The versioned provider serves the at-gen walk; Brainy composes + * the metadata half. Only reached when {@link canServeVectorAtGeneration} is true. + * @param params - The semantic (`query`) or explicit-`vector` find params. + * @param allowedIds - The at-gen candidate universe (membership-correct at `generation`). + * @param k - Over-fetch (page + headroom). + * @param generation - The as-of generation (Brainy's u64 commit counter). + * @returns `[id, distance]` pairs, ascending distance (descending relevance). + */ + private async vectorSearchAtGeneration( + params: FindParams, + allowedIds: ReadonlySet, + k: number, + generation: number + ): Promise> { + const vector = params.vector || (await this.embed(params.query!)) + // #35 part-3: supply the at-gen candidate VECTORS (resolved from Brainy's + // per-generation record before-images) so the provider reranks the historically + // correct vectors with zero per-vector crossing. `atGenerationVectors.ids` is the + // candidate set; `allowedIds` rides along (the provider may AND it). + const atGenerationVectors = await this.buildAtGenerationVectors(allowedIds, generation) + return this.index.search(vector, k, undefined, { + allowedIds, + generation: BigInt(generation), + ...(atGenerationVectors && { atGenerationVectors }) + }) + } + + /** + * @description 8.0 #35 part-3 — assemble the {@link AtGenerationVectors} columnar + * payload for a filtered at-gen exact-rerank: resolve each universe id's vector AS + * OF `generation` (the canonical record before-image, or live storage when the id + * was untouched since the pin), intern the id via the shared mapper, and pack the + * vectors flat row-major (`vectors[i*dim .. (i+1)*dim]` ↔ `ids[i]`). Ids absent at + * the generation, vectorless, or of the wrong dimension are dropped (they cannot be + * vector-ranked). Bounded by the filtered universe — far cheaper than the full-corpus + * rebuild it replaces. Returns `undefined` when no dimension is known or nothing maps. + * @param allowedIds - The at-gen filtered universe (candidate ids). + * @param generation - The as-of generation. + * @returns The columnar payload, or `undefined` when there is nothing to ship. + */ + private async buildAtGenerationVectors( + allowedIds: ReadonlySet, + generation: number + ): Promise { + const dim = this.dimensions + if (!dim || allowedIds.size === 0) return undefined + + const idMapper = this.metadataIndex.getIdMapper() + const ints: bigint[] = [] + const rows: number[][] = [] + for (const id of allowedIds) { + const resolved = await this.generationStore.resolveAt('noun', id, generation) + let vec: number[] | null = null + if (resolved.source === 'record') { + // The generation record carries the at-gen vector before-image. + vec = resolved.vector + } else if (resolved.source === 'current') { + // Untouched since the pin → the live vector IS the at-gen vector. Use + // getNoun (full hydration, lazy-load aware) — readNounRaw's canonical path + // is empty under lazy-vector eviction. + const noun = await this.storage.getNoun(id) + vec = noun?.vector ?? null + } + // 'absent' / vectorless / wrong-dim → skip (not vector-rankable at this gen). + if (Array.isArray(vec) && vec.length === dim) { + ints.push(BigInt(idMapper.getInt(id) ?? idMapper.getOrAssign(id))) + rows.push(vec) + } + } + if (ints.length === 0) return undefined + + // Row-major pack: INVARIANT vectors.length === ids.length * dim. + const vectors = new Float32Array(ints.length * dim) + for (let i = 0; i < rows.length; i++) vectors.set(rows[i], i * dim) + return { ids: BigInt64Array.from(ints), vectors, dim } + } + // --- Transact planner ------------------------------------------------------ /** @@ -6217,7 +9123,10 @@ export class Brainy implements BrainyInterface { ids: [], touchedNouns: [], touchedVerbs: [], - postCommit: [] + postCommit: [], + casUpdates: [], + createdNouns: new Set(), + changeEvents: [] } for (const op of ops) { @@ -6294,6 +9203,14 @@ export class Brainy implements BrainyInterface { // natural-key upsert is idempotent against the same key. const { id, originalId } = coerceNewEntityId(params.id) + // ifAbsent and upsert resolve the same id collision in opposite ways + // (skip vs. merge) — mirror of add()'s hard error. + if (params.ifAbsent && params.upsert) { + throw new Error( + 'transact add: ifAbsent and upsert are mutually exclusive — ifAbsent skips an existing entity, upsert merges into it.' + ) + } + // ifAbsent — idempotent by-id insert, resolved against batch + store. if (params.id && params.ifAbsent) { const existing = await this.planGetEntity(state, id) @@ -6302,6 +9219,32 @@ export class Brainy implements BrainyInterface { } } + // upsert — by-id create-or-update, resolved against batch + store. When the id + // already exists, delegate to planTxUpdate with a synthesized update op so the + // supplied fields MERGE (mirror of add()'s live upsert path) instead of + // overwriting. The id is already canonical here (coerceNewEntityId above). + if (params.id && params.upsert) { + const existing = await this.planGetEntity(state, id) + if (existing) { + return this.planTxUpdate( + { + op: 'update', + id, + ...(params.data !== undefined && { data: params.data }), + ...(params.type !== undefined && { type: params.type }), + ...(params.subtype !== undefined && { subtype: params.subtype }), + ...(params.visibility !== undefined && { visibility: params.visibility }), + ...(params.metadata !== undefined && { metadata: params.metadata }), + ...(params.vector !== undefined && { vector: params.vector }), + ...(params.confidence !== undefined && { confidence: params.confidence }), + ...(params.weight !== undefined && { weight: params.weight }) + } as Extract, { op: 'update' }>, + state, + plan + ) + } + } + const vector = params.vector || (await this.embed(params.data)) if (!this.dimensions) { this.dimensions = vector.length @@ -6318,6 +9261,11 @@ export class Brainy implements BrainyInterface { ? !state.nouns.has(id) && (await this.storage.getNounMetadata(id)) === null : true + // Either way (fresh create OR overwrite) this add resets the id's revision + // baseline (`_rev: 1` below), so later in-batch updates to it sequence + // against batch-local state — see PlannedTransact.createdNouns. + plan.createdNouns.add(id) + const now = Date.now() const storageMetadata = { ...params.metadata, @@ -6372,6 +9320,20 @@ export class Brainy implements BrainyInterface { this._aggregationIndex.onEntityAdded(id, entityForIndexing) } }) + if (this._changeFeed.hasListeners) { + plan.changeEvents.push({ + kind: 'entity', + op: 'add', + id, + entity: { + id, + type: String(params.type), + ...(params.subtype !== undefined && { subtype: String(params.subtype) }), + metadata: (params.metadata as Record) ?? {}, + ...(params.service !== undefined && { service: String(params.service) }) + } + }) + } state.nouns.set(id, { metadata: storageMetadata, vector }) state.removedNouns.delete(id) @@ -6411,8 +9373,10 @@ export class Brainy implements BrainyInterface { } // ifRev CAS — identical resolution to update(); a conflict rejects the - // WHOLE batch before anything is staged or applied. `_rev` is reserved: - // every read path surfaces it ONLY top-level. + // WHOLE batch. This planning-time check is purely a FAST-FAIL (it can + // interleave with concurrent writers); the authoritative re-verify runs in + // transact()'s commit precondition under the commit mutex, via the + // plan.casUpdates entry registered below. const currentRev = typeof existing._rev === 'number' ? existing._rev : 1 if (typeof params.ifRev === 'number' && params.ifRev !== currentRev) { throw new RevisionConflictError(params.id, params.ifRev, currentRev) @@ -6462,6 +9426,17 @@ export class Brainy implements BrainyInterface { visibility: params.visibility ?? existing.visibility }) } + + // Register for the authoritative under-mutex CAS re-verify + rev re-stamp + // (see PlannedTransact.casUpdates). The staged UpdateNounMetadataOperation + // holds `updatedMetadata` by reference, so the precommit re-stamp lands in + // what execute() writes. + plan.casUpdates.push({ + id: params.id, + ...(typeof params.ifRev === 'number' && { ifRev: params.ifRev }), + updatedMetadata + }) + const entityForIndexing = { id: params.id, vector, @@ -6514,17 +9489,32 @@ export class Brainy implements BrainyInterface { ) plan.touchedNouns.push(params.id) - const oldEntityForAgg = { - type: existing.type, - service: existing.service, - data: existing.data, - metadata: existing.metadata - } + // The full planGetEntity view, passed whole — a subset view makes the + // old-side decrement miss reserved-field groups (double-count on update). + const oldEntityForAgg = existing as unknown as Record plan.postCommit.push(() => { if (this._aggregationIndex) { this._aggregationIndex.onEntityUpdated(params.id, entityForIndexing, oldEntityForAgg) } }) + if (this._changeFeed.hasListeners) { + plan.changeEvents.push({ + kind: 'entity', + op: 'update', + id: params.id, + entity: { + id: params.id, + type: String(entityForIndexing.type), + ...(entityForIndexing.subtype !== undefined && { + subtype: String(entityForIndexing.subtype) + }), + metadata: (newMetadata as Record) ?? {}, + ...(entityForIndexing.service !== undefined && { + service: String(entityForIndexing.service) + }) + } + }) + } state.nouns.set(params.id, { metadata: updatedMetadata, vector }) return params.id @@ -6580,28 +9570,51 @@ export class Brainy implements BrainyInterface { if (metadata) { plan.operations.push(new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata)) } - plan.operations.push(new DeleteNounMetadataOperation(this.storage, id)) + // Pre-read metadata rides along: the count decrement must not depend on + // re-reading the record being removed (see remove()). + plan.operations.push(new DeleteNounMetadataOperation(this.storage, id, metadata)) for (const verb of cascade.values()) { - const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb) plan.operations.push( - new RemoveFromGraphIndexOperation(this.graphIndex, verb, sourceInt, targetInt, this.graphWriteGeneration), + // Endpoint ints resolve at EXECUTE time — a cascade verb (or its + // endpoints) may have been created earlier in this same batch, so a + // plan-time resolution would ask the id mapper about entities that do + // not exist yet (the native mapper rightly refuses). + new RemoveFromGraphIndexOperation(this.graphIndex, verb, () => this.resolveVerbEndpointInts(verb), this.graphWriteGeneration), new DeleteVerbMetadataOperation(this.storage, verb.id) ) plan.touchedVerbs.push(verb.id) state.verbs.delete(verb.id) state.removedVerbs.add(verb.id) + if (this._changeFeed.hasListeners) { + plan.changeEvents.push({ + kind: 'relation', + op: 'unrelate', + id: verb.id, + relation: { + id: verb.id, + from: verb.sourceId, + to: verb.targetId, + type: String(verb.verb) + } + }) + } } plan.touchedNouns.push(id) + if (this._changeFeed.hasListeners) { + plan.changeEvents.push({ + kind: 'entity', + op: 'remove', + id, + ...(metadata && { + entity: this.entityViewFromRawRecord(id, metadata as Record) + }) + }) + } if (metadata) { - // Canonical reserved/custom split — mirror of remove()'s aggregation hook. - const { reserved, custom } = splitNounMetadataRecord(metadata) - const entityForAgg = { - type: reserved.noun, - service: reserved.service, - data: reserved.data, - metadata: custom - } + // Mirror of remove()'s aggregation hook — the FULL reserved view, so + // reserved-field groupBy decrements find their group. + const entityForAgg = this.entityForAggFromRawRecord(metadata as Record) plan.postCommit.push(() => { if (this._aggregationIndex) { this._aggregationIndex.onEntityDeleted(id, entityForAgg) @@ -6708,8 +9721,6 @@ export class Brainy implements BrainyInterface { data: params.data, createdAt: now } - const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb) - plan.operations.push( new SaveVerbOperation(this.storage, { id, @@ -6720,13 +9731,31 @@ export class Brainy implements BrainyInterface { targetId: params.to }), new SaveVerbMetadataOperation(this.storage, id, verbMetadata), - new AddToGraphIndexOperation(this.graphIndex, verb, sourceInt, targetInt, this.graphWriteGeneration, (verbInt) => + // Endpoint ints resolve at EXECUTE time — after any same-batch add of + // an endpoint has applied. Plan-time resolution was a consumer-reported + // native/JS parity bug: transact([add X, relate →X]) asked the native + // id mapper to assign an int for an entity that did not exist yet. + new AddToGraphIndexOperation(this.graphIndex, verb, () => this.resolveVerbEndpointInts(verb), this.graphWriteGeneration, (verbInt) => this.cacheVerbInt(verbInt, id) ) ) plan.touchedVerbs.push(id) state.verbs.set(id, verb) state.removedVerbs.delete(id) + if (this._changeFeed.hasListeners) { + plan.changeEvents.push({ + kind: 'relation', + op: 'relate', + id, + relation: { + id, + from: params.from, + to: params.to, + type: String(params.type), + ...(params.metadata && { metadata: params.metadata as Record }) + } + }) + } if (params.bidirectional) { const reverseId = uuidv4() @@ -6734,9 +9763,9 @@ export class Brainy implements BrainyInterface { ...verb, id: reverseId, sourceId: params.to, - targetId: params.from, - sourceInt: targetInt, - targetInt: sourceInt + targetId: params.from + // sourceInt/targetInt are mirrored onto this object when the graph + // operation resolves endpoints at execute time. } plan.operations.push( new SaveVerbOperation(this.storage, { @@ -6748,13 +9777,27 @@ export class Brainy implements BrainyInterface { targetId: params.from }), new SaveVerbMetadataOperation(this.storage, reverseId, verbMetadata), - new AddToGraphIndexOperation(this.graphIndex, reverseVerb, targetInt, sourceInt, this.graphWriteGeneration, (verbInt) => + new AddToGraphIndexOperation(this.graphIndex, reverseVerb, () => this.resolveVerbEndpointInts(reverseVerb), this.graphWriteGeneration, (verbInt) => this.cacheVerbInt(verbInt, reverseId) ) ) plan.touchedVerbs.push(reverseId) state.verbs.set(reverseId, reverseVerb) state.removedVerbs.delete(reverseId) + if (this._changeFeed.hasListeners) { + plan.changeEvents.push({ + kind: 'relation', + op: 'relate', + id: reverseId, + relation: { + id: reverseId, + from: params.to, + to: params.from, + type: String(params.type), + ...(params.metadata && { metadata: params.metadata as Record }) + } + }) + } } return id @@ -6776,13 +9819,29 @@ export class Brainy implements BrainyInterface { : (state.verbs.get(id) ?? (await this.storage.getVerb(id))) if (verb) { - const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb) plan.operations.push( - new RemoveFromGraphIndexOperation(this.graphIndex, verb, sourceInt, targetInt, this.graphWriteGeneration) + // Endpoint ints resolve at EXECUTE time — the verb (or its endpoints) + // may have been created earlier in this same batch (forward refs). + new RemoveFromGraphIndexOperation(this.graphIndex, verb, () => this.resolveVerbEndpointInts(verb), this.graphWriteGeneration) ) } plan.operations.push(new DeleteVerbMetadataOperation(this.storage, id)) plan.touchedVerbs.push(id) + if (this._changeFeed.hasListeners) { + plan.changeEvents.push({ + kind: 'relation', + op: 'unrelate', + id, + ...(verb && { + relation: { + id, + from: verb.sourceId, + to: verb.targetId, + type: String(verb.verb) + } + }) + }) + } state.verbs.delete(id) state.removedVerbs.add(id) @@ -7190,6 +10249,30 @@ export class Brainy implements BrainyInterface { return await coordinator.import(source as Buffer | string | object, options) } + /** Brain-owned background deduplicator (lazy; see getBackgroundDeduplicator). */ + private _backgroundDedup?: import('./import/BackgroundDeduplicator.js').BackgroundDeduplicator + + /** + * The single brain-owned BackgroundDeduplicator, lazily constructed. + * + * Ownership matters here: the post-import dedup timer must outlive the + * per-call ImportCoordinator but never the brain. One instance per brain + * restores the intended cross-import debounce (per-coordinator instances + * each armed their own timer, so the "debounce" never spanned imports) and + * gives close() a handle to cancel pending work — a delete pass must never + * fire against a closed brain. + * @internal + */ + async getBackgroundDeduplicator(): Promise< + import('./import/BackgroundDeduplicator.js').BackgroundDeduplicator + > { + if (!this._backgroundDedup) { + const { BackgroundDeduplicator } = await import('./import/BackgroundDeduplicator.js') + this._backgroundDedup = new BackgroundDeduplicator(this) + } + return this._backgroundDedup + } + /** * Virtual File System API - Knowledge Operating System * @@ -7272,6 +10355,43 @@ export class Brainy implements BrainyInterface { return this._hub } + /** + * @description Subscribe to this brain's committed mutations — the + * authoritative in-process change feed. Fires exactly once per affected + * record for EVERY canonical write, regardless of origin: direct API calls, + * batch methods, `transact()`, imports, the Virtual Filesystem, and + * native-accelerated deployments all funnel through the same commit point + * this feed is emitted from. + * + * Delivery contract (see {@link BrainyChangeEvent}): + * - **Post-commit only** — an aborted write (e.g. a losing `ifRev` + * compare-and-swap) never emits. + * - **Commit-ordered, asynchronous** — events arrive in the order writes + * became durable, dispatched in a microtask so a slow listener never + * delays a write. A throwing listener is isolated (logged, others + * unaffected). + * - **Fully described** — `remove`/`unrelate` events carry the record's + * LAST committed state, and batch operations emit one event per item. + * - **Fire-and-forget** — no replay or backpressure. Each event carries its + * committed `generation`, so catch-up after a gap can be built on + * {@link transactionLog} / {@link asOf}. + * - **Zero overhead when unused** — with no subscribers the write path does + * no event work at all. + * + * @param listener - Called once per committed mutation. + * @returns An unsubscribe function — call it when done (e.g. on teardown of + * a pooled instance) to stop delivery. + * @example + * const off = brain.onChange((e) => { + * if (e.kind === 'entity') console.log(e.op, e.entity?.type, e.id) + * }) + * await brain.add({ data: 'hello', type: 'document' }) // → "add document " + * off() + */ + onChange(listener: ChangeListener): () => void { + return this._changeFeed.subscribe(listener) + } + /** * Get Triple Intelligence System * Advanced pattern recognition and relationship analysis @@ -7445,6 +10565,11 @@ export class Brainy implements BrainyInterface { console.log('Flushing Brainy indexes and caches to disk...') const startTime = Date.now() + // 8.0 MVCC: persist the in-memory pending single-op generation history to + // disk first (async group-commit), so a flush makes every single-op write's + // history durable, not just the live data. + await this.generationStore.flushPendingSingleOps() + // Flush all components in parallel for performance await Promise.all([ // 1. Flush storage adapter counts (entity/verb counts by type) @@ -7472,16 +10597,101 @@ export class Brainy implements BrainyInterface { this.generationStore.persistCounterNow() ]) - // 6. Auto-compact generational history per config.history (default on). - // Runs after the flush so durable state is in place; respects live - // Db pins and an explicit autoCompact: false. - await this.autoCompactHistory() + // NOTE (8.9.0): flush() no longer compacts history. Flush is DURABILITY + // work — it must cost what this window's deltas cost, never what the + // history backlog costs. Auto-compaction (a MAINTENANCE concern) runs at + // close() and via explicit compactHistory(); a production deployment + // measured reclaim-on-flush stalling single writes for 25-191s under + // memory pressure, which is exactly the class this separation ends. + + // 6. Stamp the entity tree: which source generation the canonical tree + // reflects + the rollup invariants that verify it whole (the counters + // persisted in step 1). Written at flush boundaries — the tree tracks + // every commit by construction, so the stamp is a durable checkpoint, + // not a per-commit cost. Open compares stamp vs log head + rollups. + await this.stampEntityTree() const elapsed = Date.now() - startTime console.log(`All indexes flushed to disk in ${elapsed}ms`) } + /** + * @description Write the entity tree's FAMILY STAMP: `sourceGeneration` (the + * committed generation the canonical tree reflects — equal by construction, + * since the tree is written by the commit itself) plus the rollup invariants + * (entity/relationship counts) that verify the tree whole where per-file + * checks cannot scale. Verified at open by {@link verifyEntityTreeStamp}; + * healed by `repairIndex()`, whose unconditional recount rebuilds the + * rollups from a canonical walk and re-stamps. Best-effort: a stamp-write + * fault warns loudly but never fails the flush that carried real data. + */ + private async stampEntityTree(): Promise { + if (this.isReadOnly) return + try { + const [nounCount, verbCount] = await Promise.all([ + this.storage.getNounCount(), + this.storage.getVerbCount() + ]) + await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, { + family: 'entity-tree', + sourceGeneration: this.generationStore.generation(), + members: { mode: 'rollup', invariants: { nounCount, verbCount } } + }) + } catch (error) { + prodLog.warn( + `[Brainy] entity-tree stamp write failed (coherence checking degrades until the ` + + `next successful flush): ${(error as Error).message}` + ) + } + } + + /** + * @description Open-time coherence check for the entity tree's family stamp: + * compare `sourceGeneration` against the log head and the stamped rollup + * invariants against the live counters. Verdicts: + * - `coherent` / `absent` (legacy store; first flush stamps) → silent. + * - `behind` → benign for the tree (it is written BY the commit; only the + * stamp is stale — a crash landed between commit and flush). Refreshed at + * the next flush. + * - `incoherent` → LOUD: the tree or its counters diverged from what was + * stamped — `repairIndex()` recounts from canonical and re-stamps. + * Never blocks open; a fault reading the stamp is surfaced as unverifiable, + * never conflated with absence. + */ + private async verifyEntityTreeStamp(): Promise { + let stamp: FamilyStamp | null + try { + stamp = await readFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH) + } catch (error) { + prodLog.warn( + `[Brainy] entity-tree stamp is UNVERIFIABLE (read fault, not absence): ` + + `${(error as Error).message}` + ) + return + } + const [nounCount, verbCount] = await Promise.all([ + this.storage.getNounCount(), + this.storage.getVerbCount() + ]) + const verdict = verifyFamilyStamp(stamp, this.generationStore.generation(), { + nounCount, + verbCount + }) + if (verdict.state === 'incoherent') { + prodLog.warn( + `[Brainy] entity-tree stamp INCOHERENT at open: ${verdict.failures.join('; ')}. ` + + `The canonical tree or its counters diverged from the stamped state — run ` + + `brain.repairIndex() to recount from canonical and re-stamp.` + ) + } else if (verdict.state === 'behind') { + prodLog.debug( + `[Brainy] entity-tree stamp is behind the log head (${verdict.stampSource} < ` + + `${verdict.head}) — benign (stamped at last flush); refreshes at the next flush.` + ) + } + } + /** * Ask the writer process serving this data directory to flush its in-memory * indexes to disk, so a read-only inspector can observe fresh state. @@ -7544,26 +10754,80 @@ export class Brainy implements BrainyInterface { initialized: boolean lazyRebuildCompleted: boolean disableAutoRebuild: boolean + /** `true` while a native provider runs the one-time 7.x → 8.0 rebuild LOCK. + * A readiness probe should map this to HTTP 503 + Retry-After (transiently + * not-ready), NOT 200-ready and NOT 500-broken. Never gated by the lock. */ + migrating: boolean + /** Structured progress while {@link migrating}; absent otherwise. */ + migration?: MigrationProgress + /** `true` when a non-fatal index rebuild failed at init() (degraded — queries + * may be incomplete). Folds in the same `_indexRebuildFailed` signal that + * {@link validateIndexConsistency} / {@link checkHealth} already expose. */ + rebuildFailed: boolean + /** The rebuild failure message when {@link rebuildFailed}; absent otherwise. */ + rebuildError?: string + /** Count of records committed via adopt-forward failed-rollback recovery whose + * derived index may be incomplete until repairIndex() (the 8.2.6 degraded + * set). Non-zero = degraded — a readiness probe should not report 200-ready. */ + degradedIds: number hnswIndex: { size: number + /** Honest "serving" — the provider's `isReady()` when exposed, else `size>0`. */ populated: boolean + /** The provider's honest `isReady()` signal; absent when unexposed. */ + ready?: boolean } metadataIndex: { entries: number populated: boolean + ready?: boolean } graphIndex: { relationships: number populated: boolean + ready?: boolean } storage: { totalEntities: number } }> { + // Callable before init() — a readiness/liveness probe may fire during + // construction or a fired-not-awaited init(). Report a safe not-initialized + // snapshot rather than dereferencing providers that init() has not assigned. + if (!this.initialized || this.metadataIndex == null || this.index == null || this.graphIndex == null) { + return { + initialized: false, + lazyRebuildCompleted: this.lazyRebuildCompleted, + disableAutoRebuild: this.config.disableAutoRebuild || false, + migrating: false, + rebuildFailed: this._indexRebuildFailed != null, + ...(this._indexRebuildFailed ? { rebuildError: this._indexRebuildFailed.message } : {}), + degradedIds: this._indexDegradedIds.size, + hnswIndex: { size: 0, populated: false }, + metadataIndex: { entries: 0, populated: false }, + graphIndex: { relationships: 0, populated: false }, + storage: { totalEntities: 0 } + } + } + const metadataStats = await this.metadataIndex.getStats() const hnswSize = this.index.size() const graphSize = await this.graphIndex.size() + // Honest readiness: when a provider exposes isReady(), it is the truth of + // whether the index actually SERVES (a native index can report a non-zero + // size/count yet not have loaded its serving structure). `populated` reflects + // that honest signal when present, falling back to size>0 only for providers + // that do not expose isReady(). Mirrors validateIndexConsistency/checkHealth, + // which already fold in the same signals. + const readyOf = (p: unknown): boolean | undefined => { + const r = assessIndexReadiness(p) + return r === 'unknown' ? undefined : r === 'ready' + } + const hnswReady = readyOf(this.index) + const metadataReady = readyOf(this.metadataIndex) + const graphReady = readyOf(this.graphIndex) + // Check storage entity count let storageEntityCount = 0 try { @@ -7577,17 +10841,29 @@ export class Brainy implements BrainyInterface { initialized: this.initialized, lazyRebuildCompleted: this.lazyRebuildCompleted, disableAutoRebuild: this.config.disableAutoRebuild || false, + // A non-fatal index-rebuild failure recorded at init(), or adopt-forward + // degraded ids, are degraded states (queries may be incomplete) — surface + // them here alongside the same signals validateIndexConsistency()/ + // checkHealth() already expose, so a readiness probe never reports 200-ready + // over a known-degraded index. + rebuildFailed: this._indexRebuildFailed != null, + ...(this._indexRebuildFailed ? { rebuildError: this._indexRebuildFailed.message } : {}), + degradedIds: this._indexDegradedIds.size, + ...this.migrationSnapshot(), hnswIndex: { size: hnswSize, - populated: hnswSize > 0 + populated: hnswReady ?? hnswSize > 0, + ...(hnswReady !== undefined ? { ready: hnswReady } : {}) }, metadataIndex: { entries: metadataStats.totalEntries, - populated: metadataStats.totalEntries > 0 + populated: metadataReady ?? metadataStats.totalEntries > 0, + ...(metadataReady !== undefined ? { ready: metadataReady } : {}) }, graphIndex: { relationships: graphSize, - populated: graphSize > 0 + populated: graphReady ?? graphSize > 0, + ...(graphReady !== undefined ? { ready: graphReady } : {}) }, storage: { totalEntities: storageEntityCount @@ -7619,7 +10895,9 @@ export class Brainy implements BrainyInterface { details?: Record }> }> { - await this.ensureInitialized() + // Lock-exempt: an operator must be able to probe health WHILE the brain + // upgrades — that is the whole point of an observable migration. + await this.ensureInitialized({ bypassMigrationLock: true }) const checks: Array<{ name: string status: 'pass' | 'warn' | 'fail' @@ -7627,6 +10905,28 @@ export class Brainy implements BrainyInterface { details?: Record }> = [] + // Migration LOCK (#18): while a native provider rebuilds the derived indexes, + // the parity/field checks below would report transient mismatches that read + // as failures but are not — surface the upgrade as the single honest signal. + // `warn` (not `fail`) so a readiness probe treats it as "not ready yet, retry". + const migSnap = this.migrationSnapshot() + if (migSnap.migrating) { + const pct = migSnap.migration?.percent + return { + overall: 'warn', + checks: [ + { + name: 'migration', + status: 'warn', + message: + `Brain is upgrading (7.x → 8.0)${pct !== undefined ? `, ${Math.round(pct)}% complete` : ''}. ` + + 'Reads and writes are blocked until it completes; retry shortly.', + details: { ...migSnap.migration } + } + ] + } + } + const hnswSize = this.index.size() const metadataStats = await this.metadataIndex.getStats() const graphSize = await this.graphIndex.size() @@ -7929,7 +11229,7 @@ export class Brainy implements BrainyInterface { /** * Assert that specific providers are supplied by a plugin (not using JS fallback). * - * Call after init() in production to fail fast if a paid plugin (e.g. cortex) + * Call after init() in production to fail fast if a paid plugin (e.g. cor) * isn't providing the expected acceleration. Throws if any listed key is using * the default JavaScript implementation. * @@ -7941,7 +11241,7 @@ export class Brainy implements BrainyInterface { * const brain = new Brainy() * await brain.init() * - * // Fail fast if cortex isn't providing these + * // Fail fast if cor isn't providing these * brain.requireProviders(['distance', 'embeddings', 'metadataIndex', 'graphIndex']) * ``` */ @@ -8147,15 +11447,17 @@ export class Brainy implements BrainyInterface { } }.bind(this), - // Stream relationships efficiently + // Stream relationships efficiently. Cursor-based when the adapter supports it + // (resumes after the last verb — O(N) for a full walk) and falls back to offset + // otherwise. Offset paging here re-scanned from the start every page (O(N²)). relationships: async function* (this: Brainy, filter?: { type?: string, sourceId?: string, targetId?: string }) { let offset = 0 + let cursor: string | undefined const batchSize = 100 - let hasMore = true - while (hasMore) { + for (;;) { const result = await this.storage.getVerbs({ - pagination: { offset, limit: batchSize }, + pagination: cursor ? { limit: batchSize, cursor } : { offset, limit: batchSize }, filter }) @@ -8163,8 +11465,12 @@ export class Brainy implements BrainyInterface { yield verb } - hasMore = result.hasMore - offset += batchSize + if (!result.hasMore || result.items.length === 0) break + if (result.nextCursor) { + cursor = result.nextCursor + } else { + offset += result.items.length + } } }.bind(this), @@ -9668,9 +12974,96 @@ export class Brainy implements BrainyInterface { entityCount: number indexEntryCount: number recommendation: string | null + /** Cross-layer: each provider's own invariant self-report, when + * it exposes validateInvariants(). Absent providers are simply not listed. */ + providers?: ProviderInvariantReport[] }> { await this.ensureInitialized() - return this.metadataIndex.validateConsistency() + const result = await this.metadataIndex.validateConsistency() + + // Cross-layer integrity: validateConsistency() above only sees the + // JS metadata index — it is BLIND to a native provider whose manifest ↔ + // segments ↔ counts have diverged. Delegate to each provider's own + // validateInvariants() and aggregate, so "healthy-while-broken" is impossible. + const providers = await this.collectProviderInvariants() + const brokenProviders = providers.filter((p) => !p.healthy) + + let healthy = result.healthy + const notes: string[] = [] + if (result.recommendation) notes.push(result.recommendation) + + if (this._indexRebuildFailed) { + healthy = false + notes.unshift( + `Index rebuild failed at init() (degraded — queries may be incomplete): ` + + `${this._indexRebuildFailed.message}. Rebuild the indexes and re-open.` + ) + } + if (this._indexDegradedIds.size > 0) { + healthy = false + notes.unshift( + `${this._indexDegradedIds.size} record(s) committed via adopt-forward ` + + `failed-rollback recovery — the derived index may be incomplete for them. ` + + `Run repairIndex() to reconcile.` + ) + } + for (const p of brokenProviders) { + healthy = false + const failing = p.invariants.filter((i) => !i.holds) + notes.unshift( + `Provider '${p.provider}' reports ${failing.length} failing invariant(s): ` + + failing.map((i) => `${i.name} (${i.detail}; heal=${i.heal})`).join('; ') + + `. Run repairIndex() to reconcile from canonical.` + ) + } + + return { + ...result, + healthy, + recommendation: notes.length > 0 ? notes.join(' ') : null, + ...(providers.length > 0 ? { providers } : {}) + } + } + + /** + * @description Feature-detect + call each index provider's OPTIONAL + * `validateInvariants()` and collect the reports. The contract is + * that `validateInvariants()` NEVER throws and is bounded <50ms — but if a + * provider violates that, the throw is turned into an UNHEALTHY synthetic + * report (loud), never swallowed into "healthy". Providers that do not expose + * the hook are simply omitted (the JS baseline validates via + * `metadataIndex.validateConsistency()`). + * @returns One report per provider that exposes `validateInvariants()`. + */ + private async collectProviderInvariants(): Promise { + const reports: ProviderInvariantReport[] = [] + const providers: unknown[] = [this.metadataIndex, this.index, this.graphIndex] + for (const provider of providers) { + const fn = (provider as { validateInvariants?: () => Promise } | null) + ?.validateInvariants + if (typeof fn !== 'function') continue + try { + const report = await fn.call(provider) + if (report && Array.isArray(report.invariants)) reports.push(report) + } catch (err) { + reports.push({ + provider: 'unknown', + healthy: false, + serving: false, + invariants: [ + { + name: 'validate-invariants-threw', + holds: false, + detail: `validateInvariants() threw (contract violation — it must never throw): ${(err as Error).message}`, + heal: 'rebuild' + } + ], + checkedAt: Date.now(), + durationMs: 0 + }) + } + } + return reports } /** @@ -9724,7 +13117,9 @@ export class Brainy implements BrainyInterface { limit?: number } ): Promise { - await this.ensureInitialized() + // Graph read (see related): adjacency traversal only. + await this.ensureInitialized({ needs: ['graph'] }) + await this.verifyGraphAdjacencyLive() const direction = options?.direction || 'both' const limit = options?.limit @@ -10094,6 +13489,54 @@ export class Brainy implements BrainyInterface { return params } + /** + * @description Build the MetadataIndex filter object from a query's structured + * criteria (`where` / `type` / `subtype` / `service` / `excludeVFS`) — the shared + * filter shape consumed by `getIdsForFilter` / `getIdSetForFilter`. Applies the + * `where.type → noun` alias and expands a multi-type filter into an `anyOf`. + * @param params - The structured query criteria. + * @returns The filter object, or `null` when no structured criteria are present. + */ + private buildMetadataFilter(params: { + where?: any + type?: NounType | NounType[] + subtype?: string | string[] + service?: string + excludeVFS?: boolean + }): any | null { + if (!(params.where || params.type || params.subtype || params.service || params.excludeVFS)) { + return null + } + let filter: any = {} + if (params.where) { + Object.assign(filter, params.where) + // Alias: where.type → where.noun (storage field name for entity type) + if ('type' in filter && !('noun' in filter)) { + filter.noun = filter.type + delete filter.type + } + } + if (params.service) filter.service = params.service + if (params.excludeVFS === true) { + filter.vfsType = { exists: false } + filter.isVFSEntity = { ne: true } + } + // Subtype (top-level standard field — fast path). Assigned BEFORE the type-array + // expansion below so the spread into each anyOf branch carries it through. + if (params.subtype !== undefined) { + filter.subtype = Array.isArray(params.subtype) ? { oneOf: params.subtype } : params.subtype + } + if (params.type) { + const types = Array.isArray(params.type) ? params.type : [params.type] + if (types.length === 1) { + filter.noun = types[0] + } else { + filter = { anyOf: types.map((type) => ({ noun: type, ...filter })) } + } + } + return filter + } + /** * Execute vector search component * @@ -10102,13 +13545,37 @@ export class Brainy implements BrainyInterface { * When provided, HNSW search is restricted to these candidates: * - NativeHNSWWrapper: uses native searchWithCandidates (Rust bitmap filtering) * - JS JsHnswVectorIndex: converts to filter function with O(1) Set lookups + * @param allowedIds Optional 8.0 #46 predicate-pushdown universe as an + * {@link OpaqueIdSet} (native/cor roaring Buffer) — forwarded straight into the + * beam walk with no id materialization. The native vector provider consumes it; + * the JS index ignores the opaque form and relies on `candidateIds`. */ - private async executeVectorSearch(params: FindParams, candidateIds?: string[]): Promise[]> { + private async executeVectorSearch( + params: FindParams, + candidateIds?: string[], + allowedIds?: OpaqueIdSet + ): Promise[]> { + // Vector cold-read guard: before trusting a semantic/vector result, verify the + // vector index actually SERVES a known persisted vector (one-shot per brain). + // A pure semantic find({ query }) has no filter, so verifyMetadataLive never + // fires — a cold native index that loaded its COUNT but not its serving + // structure would return a silent []. This self-heals (rebuild) or throws + // VectorIndexNotReadyError instead. + await this.verifyVectorLive() + const vector = params.vector || (await this.embed(params.query!)) const limit = params.limit || 10 - // Build search options for metadata-first candidate filtering - const searchOptions = candidateIds ? { candidateIds } : undefined + // Build search options for metadata-first candidate filtering. Pass both forms + // when available: the native provider prefers the opaque `allowedIds` (zero + // crossing), the JS index uses the materialized `candidateIds`. + const searchOptions = + candidateIds || allowedIds + ? { + ...(candidateIds && { candidateIds }), + ...(allowedIds && { allowedIds }) + } + : undefined // HNSW search with optional metadata-first candidate filtering const searchResults: [string, number][] = await this.index.search(vector, limit * 2, undefined, searchOptions) @@ -10136,6 +13603,10 @@ export class Brainy implements BrainyInterface { private async executeProximitySearch(params: FindParams): Promise[]> { if (!params.near) return [] + // Vector cold-read guard (see executeVectorSearch): proximity search also + // hits this.index.search — verify it serves before trusting an empty result. + await this.verifyVectorLive() + // Teaching error: without an anchor id the constraint is meaningless, and // letting it fall through produces an opaque storage-layer sharding error. if (!params.near.id) { @@ -10197,111 +13668,133 @@ export class Brainy implements BrainyInterface { const connectedIds = new Set() - if (subtypeFilter !== undefined) { - // Multi-hop BFS with per-hop (verbType, subtype) predicate. Works on the - // open-core JS path at any depth. When a graph-index provider exposes a - // faster `findConnectedSubtype` (native BFS with the same semantics), - // `nativeSubtypeBfs` routes through it transparently — same pattern as - // every other provider hook. - const subtypeArr = Array.isArray(subtypeFilter) ? subtypeFilter : [subtypeFilter] - const subtypeSet = new Set(subtypeArr) + // Population is extracted into a closure so it can be re-run VERBATIM after a + // cold-load heal (verdict 'rebuilt') — re-executing the SAME traversal once the + // adjacency is actually loaded, without changing any collection semantics. + const populate = async (): Promise => { + connectedIds.clear() + if (subtypeFilter !== undefined) { + // Multi-hop BFS with per-hop (verbType, subtype) predicate. Works on the + // open-core JS path at any depth. When a graph-index provider exposes a + // faster `findConnectedSubtype` (native BFS with the same semantics), + // `nativeSubtypeBfs` routes through it transparently — same pattern as + // every other provider hook. + const subtypeArr = Array.isArray(subtypeFilter) ? subtypeFilter : [subtypeFilter] + const subtypeSet = new Set(subtypeArr) - // Native fast path (D.3): single verb type + single subtype + outgoing - // walk match the native BFS semantics exactly (out-edges only, source - // excluded, visited-set cycle guard). Entity ints in, entity ints out — - // UUID conversion stays at this boundary. Returns null when the query - // shape (or provider) can't take the native route. - const nativeSubtypeBfs = async ( - anchor: string, - walk: 'in' | 'out' | 'both' - ): Promise | null> => { - if (walk !== 'out') return null - if (via === undefined || Array.isArray(via)) return null - if (subtypeArr.length !== 1) return null - const provider = this.graphIndex as Partial<{ - findConnectedSubtype( - sourceInt: bigint, - verbTypeIndex: number, - subtype: string | null, - depth: number, - limit?: number | null - ): Promise - }> - if (typeof provider.findConnectedSubtype !== 'function') return null + // Native fast path (D.3): single verb type + single subtype + outgoing + // walk match the native BFS semantics exactly (out-edges only, source + // excluded, visited-set cycle guard). Entity ints in, entity ints out — + // UUID conversion stays at this boundary. Returns null when the query + // shape (or provider) can't take the native route. + const nativeSubtypeBfs = async ( + anchor: string, + walk: 'in' | 'out' | 'both' + ): Promise | null> => { + if (walk !== 'out') return null + if (via === undefined || Array.isArray(via)) return null + if (subtypeArr.length !== 1) return null + const provider = this.graphIndex as Partial<{ + findConnectedSubtype( + sourceInt: bigint, + verbTypeIndex: number, + subtype: string | null, + depth: number, + limit?: number | null + ): Promise + }> + if (typeof provider.findConnectedSubtype !== 'function') return null - const anchorInt = this.graphEntityInt(anchor) - if (anchorInt === undefined) return new Set() // unmapped → no relations + const anchorInt = this.graphEntityInt(anchor) + if (anchorInt === undefined) return new Set() // unmapped → no relations - const verbTypeIndex = TypeUtils.getVerbIndex(via as VerbType) - // No limit: match the JS BFS exactly — overall result limiting happens - // downstream against existingResults. - const reachedInts = await provider.findConnectedSubtype( - anchorInt, verbTypeIndex, subtypeArr[0], effectiveDepth, null - ) - return new Set(this.entityIntsToUuids(reachedInts)) - } + const verbTypeIndex = TypeUtils.getVerbIndex(via as VerbType) + // No limit: match the JS BFS exactly — overall result limiting happens + // downstream against existingResults. + const reachedInts = await provider.findConnectedSubtype( + anchorInt, verbTypeIndex, subtypeArr[0], effectiveDepth, null + ) + return new Set(this.entityIntsToUuids(reachedInts)) + } - const bfsWithSubtype = async (anchor: string, walk: 'in' | 'out' | 'both'): Promise> => { - const nativeResult = await nativeSubtypeBfs(anchor, walk) - if (nativeResult !== null) return nativeResult + const bfsWithSubtype = async (anchor: string, walk: 'in' | 'out' | 'both'): Promise> => { + const nativeResult = await nativeSubtypeBfs(anchor, walk) + if (nativeResult !== null) return nativeResult - const visited = new Set([anchor]) - const reached = new Set() - let frontier = new Set([anchor]) + const visited = new Set([anchor]) + const reached = new Set() + let frontier = new Set([anchor]) - for (let hop = 0; hop < effectiveDepth && frontier.size > 0; hop++) { - const nextFrontier = new Set() - for (const node of frontier) { - // Walk outgoing edges (when direction includes outbound) and incoming - // edges (when direction includes inbound). Both = union. - const dirs: Array<'from' | 'to'> = [] - if (walk === 'out' || walk === 'both') dirs.push('from') - if (walk === 'in' || walk === 'both') dirs.push('to') + for (let hop = 0; hop < effectiveDepth && frontier.size > 0; hop++) { + const nextFrontier = new Set() + for (const node of frontier) { + // Walk outgoing edges (when direction includes outbound) and incoming + // edges (when direction includes inbound). Both = union. + const dirs: Array<'from' | 'to'> = [] + if (walk === 'out' || walk === 'both') dirs.push('from') + if (walk === 'in' || walk === 'both') dirs.push('to') - for (const dir of dirs) { - const edges = await this.related({ - ...(dir === 'from' ? { from: node } : { to: node }), - ...(via && { type: via as VerbType | VerbType[] }), - subtype: subtypeArr, - limit: 10000 - }) - for (const edge of edges) { - // Defensive: related() honors the subtype filter, but check - // again in case a future impl widens it. - if (edge.subtype !== undefined && !subtypeSet.has(edge.subtype)) continue - const neighbor = dir === 'from' ? edge.to : edge.from - if (visited.has(neighbor)) continue - visited.add(neighbor) - reached.add(neighbor) - nextFrontier.add(neighbor) + for (const dir of dirs) { + const edges = await this.related({ + ...(dir === 'from' ? { from: node } : { to: node }), + ...(via && { type: via as VerbType | VerbType[] }), + subtype: subtypeArr, + limit: 10000 + }) + for (const edge of edges) { + // Defensive: related() honors the subtype filter, but check + // again in case a future impl widens it. + if (edge.subtype !== undefined && !subtypeSet.has(edge.subtype)) continue + const neighbor = dir === 'from' ? edge.to : edge.from + if (visited.has(neighbor)) continue + visited.add(neighbor) + reached.add(neighbor) + nextFrontier.add(neighbor) + } } } + frontier = nextFrontier } - frontier = nextFrontier + return reached } - return reached - } - if (from) { - for (const id of await bfsWithSubtype(from, direction)) connectedIds.add(id) - } - if (to) { - const reverse: 'in' | 'out' | 'both' = direction === 'in' ? 'out' : direction === 'out' ? 'in' : 'both' - for (const id of await bfsWithSubtype(to, reverse)) connectedIds.add(id) - } - } else { - // No subtype filter — fast path via neighbors(), which does the same - // depth-aware BFS but only filters by verbType. - const collect = (id: string, dir: 'incoming' | 'outgoing' | 'both'): Promise => - this.neighbors(id, { direction: dir, depth: effectiveDepth, verbType: via }) + if (from) { + for (const id of await bfsWithSubtype(from, direction)) connectedIds.add(id) + } + if (to) { + const reverse: 'in' | 'out' | 'both' = direction === 'in' ? 'out' : direction === 'out' ? 'in' : 'both' + for (const id of await bfsWithSubtype(to, reverse)) connectedIds.add(id) + } + } else { + // No subtype filter — fast path via neighbors(), which does the same + // depth-aware BFS but only filters by verbType. + const collect = (id: string, dir: 'incoming' | 'outgoing' | 'both'): Promise => + this.neighbors(id, { direction: dir, depth: effectiveDepth, verbType: via }) - if (from) { - for (const id of await collect(from, toNeighborDir(direction))) connectedIds.add(id) - } + if (from) { + for (const id of await collect(from, toNeighborDir(direction))) connectedIds.add(id) + } - if (to) { - const reverse: 'in' | 'out' | 'both' = direction === 'in' ? 'out' : direction === 'out' ? 'in' : 'both' - for (const id of await collect(to, toNeighborDir(reverse))) connectedIds.add(id) + if (to) { + const reverse: 'in' | 'out' | 'both' = direction === 'in' ? 'out' : direction === 'out' ? 'in' : 'both' + for (const id of await collect(to, toNeighborDir(reverse))) connectedIds.add(id) + } + } + } + + await populate() + + // Cold-load guard: an empty connected set is suspicious. The native adjacency can report + // size()>0 (or isReady()===false) on a cold open yet have loaded NO source→target edges — so + // traversal silently returns []. Re-verify against the honest isReady() signal (or, for older + // providers, a GLOBAL known-edge sample — NOT the queried anchor, which may be genuinely + // edgeless). If the adjacency was dead and a rebuild healed it, re-collect; if it stays dead, + // verifyGraphAdjacencyLive() throws GraphIndexNotReadyError. A genuinely edgeless anchor + // verifies 'live' and the empty result stands — no spurious rebuild/throw. + if (connectedIds.size === 0) { + const verdict = await this.verifyGraphAdjacencyLive() + if (verdict === 'rebuilt') { + await populate() } } @@ -10994,16 +14487,38 @@ export class Brainy implements BrainyInterface { } // FINALIZE: mark migrated (makes re-open a no-op), then drain the head - // branch (its entities moved; the rest is stale 7.x derived state). - // Non-head branches under branches/ are 7.x version history 8.0's MVCC does - // not import — they are left in place. + // branch. Non-head branches under branches/ are 7.x version history 8.0's + // MVCC does not import — they are left in place. await probe.writeRawObject('_system/migration-layout.json', { layout: 'flat-v8', version: 8, fromBranch: head, entitiesMoved: moved }) - await probe.removeRawPrefix(`branches/${head}`) + + // Parity guard (defense-in-depth; the VFS `_cow/` stranding lesson). The + // entity move deleted every `branches//entities/*` key, so anything + // STILL under `branches//` is non-entity durable state a 7.x engine + // wrote outside `entities/` (branch-scoped blobs, an index dir, a field + // registry). Draining it unconditionally would silently DELETE it — the + // same failure class as the VFS content blobs stranded in `_cow/`. So + // drain only a clean branch (nothing but the moved entities); if any + // non-entity object survives, PRESERVE the branch (skip the drain) so the + // state stays recoverable, and surface it loudly. + const residual = ( + await probe.listRawObjects(`branches/${head}`) + ).filter((k: string) => !k.startsWith(`branches/${head}/entities/`)) + if (residual.length === 0) { + await probe.removeRawPrefix(`branches/${head}`) + } else if (!this.config.silent) { + const sample = residual.slice(0, 8).join(', ') + console.warn( + `[brainy] 7→8 migration preserved ${residual.length} non-entity object(s) under ` + + `branches/${head}/ — branch-scoped durable state written outside entities/. The ` + + `branch was NOT drained, so this state is recoverable; inspect it before removing ` + + `branches/ manually. Keys: ${sample}${residual.length > 8 ? ', …' : ''}` + ) + } } finally { if (canLock && typeof lockable.releaseWriterLock === 'function') { await lockable.releaseWriterLock() @@ -11016,6 +14531,85 @@ export class Brainy implements BrainyInterface { } } + /** + * @description On-open recovery for VFS content blobs a 7→8 upgrade left in the + * removed branch system's copy-on-write area (`_cow/`). If a `_cow/` area is + * present and a completed adoption has not already been recorded, adopt every + * orphaned blob into the 8.0 content-addressed store (`_cas/`) and stamp a + * marker so later opens skip the scan. Because it runs on every open right + * after {@link legacyLayoutMigrationPhase}, it heals BOTH a fresh 7→8 migration + * (where `_cow/` still holds the blobs) AND a brain already migrated by an + * earlier 8.0.x that stranded them — no operator action needed. + * + * Filesystem-only; a native-8.0 / fresh brain has no `_cow/` and returns on the + * cheap existence check. Non-destructive and idempotent (see + * {@link BaseStorage.adoptLegacyCowBlobs}). If any blob cannot be fully adopted + * (`incomplete > 0`), the marker is NOT stamped (the next open retries) and + * {@link _vfsBlobAdoptionIncomplete} is set so the pre-upgrade backup is + * retained — the upgrade is not verifiably complete for the VFS. + */ + private async autoAdoptLegacyVfsBlobsIfNeeded(): Promise { + if (this.getStorageType() !== 'filesystem') return + // Cheapest gate: no copy-on-write area → a native-8.0 / fresh brain, nothing + // a 7.x brain could have stranded here. + try { + const root = resolveFilesystemRoot( + (this.config.storage || {}) as StorageOptions & Record + ) + if (!fs.existsSync(`${root}/_cow`)) return + } catch { + return + } + + const storage = this.storage as unknown as { + adoptLegacyCowBlobs?: () => Promise<{ + cowBlobs: number + adopted: number + alreadyPresent: number + incomplete: number + }> + readRawObject?: (p: string) => Promise + writeRawObject?: (p: string, o: unknown) => Promise + } + if (typeof storage.adoptLegacyCowBlobs !== 'function') return + + const MARKER = '_system/vfs-blob-adoption.json' + try { + const done = (await storage.readRawObject?.(MARKER)) as { complete?: boolean } | null + if (done && done.complete) return // already fully adopted on a prior open + } catch { + // no marker yet — proceed + } + + const result = await storage.adoptLegacyCowBlobs() + + if (result.incomplete > 0) { + // Partial: retry on future opens, and hold the pre-upgrade backup. + this._vfsBlobAdoptionIncomplete = true + if (!this.config.silent) { + console.error( + `[brainy] VFS blob recovery adopted ${result.adopted} blob(s) but ${result.incomplete} ` + + `orphaned _cow/ blob(s) are missing their bytes or metadata and were left in place. ` + + `The pre-upgrade backup is being retained; inspect _cow/ before discarding it.` + ) + } + return + } + + // Clean sweep — record it so later opens skip the _cow/ scan entirely. + try { + await storage.writeRawObject?.(MARKER, { complete: true, adopted: result.adopted }) + } catch { + // marker write is best-effort; adoption already succeeded + } + if (result.adopted > 0 && !this.config.silent) { + console.warn( + `[brainy] Recovered ${result.adopted} VFS content blob(s) stranded by a 7→8 upgrade ` + + `(adopted _cow/ → _cas/ in place).` + ) + } + } + private async setupStorage(): Promise { // If the caller passed a pre-constructed storage adapter (e.g. // `storage: new MemoryStorage()`, or the historical materializer's @@ -11201,13 +14795,19 @@ export class Brainy implements BrainyInterface { // Memory management escape hatches over the RAM-derived auto limits maxQueryLimit: config?.maxQueryLimit ?? undefined, reservedQueryMemory: config?.reservedQueryMemory ?? undefined, + // Migration LOCK wait budget — left undefined when omitted so + // awaitMigrationLock() applies its 30 s default at the read site. + migrationWaitTimeoutMs: config?.migrationWaitTimeoutMs ?? undefined, + // Pre-upgrade backup — default-on; opt out with `migrationBackup: false`. + migrationBackup: config?.migrationBackup ?? true, // Vector index configuration (8.0) — algorithm-neutral surface with // `recall` preset + `persistMode` (folded in from 7.x's hnswPersistMode). // See BRAINY-8.0-RENAME-COORDINATION § A.2 + § G.2. vector: config?.vector ?? undefined, - // Generational-history retention — defaults applied at the read site - // (resolveHistoryPolicy) so a partial object inherits per-field defaults. - history: config?.history ?? undefined, + // Generational-history retention — the `retention` knob; defaults applied + // at the read site (resolveRetentionPolicy) so a partial object inherits + // per-field defaults and `undefined` resolves to ADAPTIVE. + retention: config?.retention ?? undefined, // Embedding initialization — left undefined when omitted so the adaptive // default resolves at the init() read site (eager when the WASM embedder // is the active one on a non-reader writer outside unit tests). Explicit @@ -11229,7 +14829,12 @@ export class Brainy implements BrainyInterface { requireSubtype: config?.requireSubtype ?? true, // Multi-process safety mode: config?.mode ?? 'writer', - force: config?.force ?? false + force: config?.force ?? false, + // Reserved-field-in-metadata-bag policy (8.0 — no silent failures). + // Default 'throw': an untyped caller that smuggles a reserved key past + // the compile guard gets a loud Error naming the correct write path. + // 'warn' = remap + one-shot warning per key; 'remap' = legacy silent remap. + reservedFieldPolicy: config?.reservedFieldPolicy ?? 'throw' } } @@ -11256,12 +14861,30 @@ export class Brainy implements BrainyInterface { return } - // If indexes already populated, mark as complete and skip - if (this.index.size() > 0) { + // If indexes already populated AND honestly serving, mark complete and skip. + // Honest gate: when the provider exposes isReady(), that REPLACES the size()>0 + // proxy (a native index can report a non-zero size while its serving structure + // is not loaded — the silent-empty cold-load class). A not-ready provider falls + // through so the rebuild path can load it; verifyVectorLive() is the query-time + // backstop either way. Providers without isReady() keep the size() heuristic + // (the JS index's size()>0 genuinely means loaded). + const vectorReadiness = assessIndexReadiness(this.index) + if (vectorReadiness === 'ready' || (vectorReadiness === 'unknown' && this.index.size() > 0)) { this.lazyRebuildCompleted = true return } + // Migration LOCK (#18) deference: while the vector provider runs its one-time + // 7.x → 8.0 rebuild-from-canonical, a first query must NOT trigger brainy's + // force-rebuild — the provider owns that index. Normally unreachable here: the + // data-plane lock (awaitMigrationLock) makes the caller wait upstream, so a + // query only reaches this point once the migration has cleared. Defensive + // (no `lazyRebuildCompleted` latch) so the check re-runs: once the provider + // clears the lock, `index.size() > 0` above ends the lazy path normally. + if (this.providerIsMigrating(this.index)) { + return + } + // Concurrency control: If rebuild is in progress, wait for it if (this.lazyRebuildInProgress && this.lazyRebuildPromise) { await this.lazyRebuildPromise @@ -11319,7 +14942,7 @@ export class Brainy implements BrainyInterface { /** * @description Wire the HNSW connections codec (2.4.0 #3). Activates when - * BOTH (a) the `graph:compression` provider is registered (cortex registers + * BOTH (a) the `graph:compression` provider is registered (cor registers * `{ encode: encodeConnections, decode: decodeConnections }`), and (b) the * metadata index exposes a stable idMapper. Failures are non-fatal: HNSW * keeps working via the legacy JSON-array path. @@ -11387,16 +15010,13 @@ export class Brainy implements BrainyInterface { return } - // OPTIMIZATION: Instant check - if index already has data, skip immediately - // This gives 0s startup for warm restarts (vs 50-100ms of async checks) - if (this.index.size() > 0 && !force) { - if (!this.config.silent) { - console.log( - `✅ Index already populated (${this.index.size().toLocaleString()} entities) - 0s startup!` - ) - } - return - } + // No instant fast-path here: the honest per-leg readiness checks below + // are all O(1) (one bounded storage sample + each provider's size()/ + // isReady()), and this method runs exactly once per open (init calls it; + // the lazy path passes force=true). The removed shortcut keyed off + // `this.index.size() > 0`, a dishonest proxy — it skipped the metadata + // and graph checks whenever the vector happened to be warm, and it never + // fired on a real cold process (the JS vector size is 0 until it loads). // BUG #2 FIX: Don't trust counts - check actual storage instead // Counts can be lost/corrupted in container restarts @@ -11408,6 +15028,10 @@ export class Brainy implements BrainyInterface { if (force && !this.config.silent) { console.log('✅ Storage empty - no rebuild needed') } + // A fresh / empty brain's (empty) derived indexes are trivially current + // for this build's epoch — stamp the version marker so the next open + // recognises it as current and skips the drift rebuild. + await this.stampBrainFormatIfNeeded() return } @@ -11418,15 +15042,73 @@ export class Brainy implements BrainyInterface { // Check if indexes need rebuilding const metadataStats = await this.metadataIndex.getStats() const hnswIndexSize = this.index.size() - const graphIndexSize = await this.graphIndex.size() - const needsRebuild = - metadataStats.totalEntries === 0 || - hnswIndexSize === 0 || - graphIndexSize === 0 + // Readiness contract: when a provider exposes isReady(), that honest + // signal REPLACES the size/count heuristic below — an mmap/disk-native + // index legitimately reports 0 resident entries while fully durable on + // disk, and rebuilding it from canonical re-reads every entity file on + // every boot (the 48-seconds-per-restart class a production deployment + // hit). The signal is honest in BOTH directions: a provider whose + // durable state failed to load returns false and gets its rebuild even + // when size() > 0 (the silent-empty cold-load failure). Providers + // without isReady() keep the exact prior empty-heuristics. + const providerReady = (leg: unknown): boolean | undefined => { + const candidate = leg as { isReady?: () => boolean } + return typeof candidate.isReady === 'function' ? candidate.isReady() : undefined + } + const metadataReady = providerReady(this.metadataIndex) + const vectorReady = providerReady(this.index) + const graphReady = providerReady(this.graphIndex) + + // Epoch-drift trigger: a format-version change makes EVERY derived index + // suspect even when each is non-empty, so it forces a rebuild of all + // three from the canonical records (not just the empty ones below). + const epochStale = this._indexEpochStale + + // Migration LOCK (#18) deference: when a native provider holds the lock for + // its one-time 7.x → 8.0 rebuild-from-canonical, brainy must NOT rebuild + // that index — the provider owns it in place and clears the lock only after + // it verifies and calls brain.stampBrainFormat(). Reads and writes are held + // by awaitMigrationLock meanwhile (nothing serves from a half-built index). + // Gated per-index, so a non-migrating sibling still rebuilds when it needs + // to; a migrating provider is skipped even under epoch-drift or size()===0. + const metadataMigrating = this.providerIsMigrating(this.metadataIndex) + const vectorMigrating = this.providerIsMigrating(this.index) + const graphMigrating = this.providerIsMigrating(this.graphIndex) + const anyMigrating = metadataMigrating || vectorMigrating || graphMigrating + + // Per-leg decision, in precedence order: a migrating provider owns its + // index (skip) → epoch drift forces a rebuild → an exposed isReady() + // decides → otherwise a per-leg fallback. The fallbacks differ by leg + // because "empty" means different things: + // - METADATA: past the empty-store early-return, entities exist, so the + // id-mapper SHOULD have loaded entries — totalEntries===0 is a real + // load-failure signal, so rebuild (self-heal from canonical). + // - VECTOR: the JS vector has no passive cold-load — rebuild() IS its + // load path — so size()===0 correctly triggers the load. + // - GRAPH: entities do NOT imply edges, so size()===0 is a VALID empty + // state, not a load failure. The JS graph cold-loads (and self-heals + // against canonical) inside storage.getGraphIndex() BEFORE this gate, + // so it is already authoritative here; re-deriving would be spurious + // (a full O(E) verb scan on every open of an edgeless brain). It + // therefore rebuilds only on epoch drift or a native !isReady(). + // (verifyGraphAdjacencyLive is the query-time backstop.) + const shouldRebuildMetadata = + !metadataMigrating && + (epochStale || + (metadataReady !== undefined ? !metadataReady : metadataStats.totalEntries === 0)) + const shouldRebuildVector = + !vectorMigrating && + (epochStale || (vectorReady !== undefined ? !vectorReady : hnswIndexSize === 0)) + const shouldRebuildGraph = + !graphMigrating && + (epochStale || (graphReady !== undefined ? !graphReady : false)) + + const needsRebuild = shouldRebuildMetadata || shouldRebuildVector || shouldRebuildGraph if (!needsRebuild && !force) { - // All indexes already populated, no rebuild needed + // All indexes report current — durably loaded (isReady/size), or owned + // by a background migration. No rebuild needed. return } @@ -11452,16 +15134,41 @@ export class Brainy implements BrainyInterface { : '🔄 Auto-rebuild explicitly enabled' if (!this.config.silent) { - console.log(`${rebuildReason} - rebuilding all indexes from persisted data...`) + // Name exactly which legs rebuild — "all indexes" was a lie whenever + // the durable legs were skipped (e.g. only the JS vector index loads + // here on a warm reopen), and it misread as a whole-brain rebuild in + // consumer boot logs. + const rebuildingLegs = [ + shouldRebuildMetadata && 'metadata', + shouldRebuildVector && 'vector', + shouldRebuildGraph && 'graph' + ] + .filter(Boolean) + .join(' + ') + console.log(`${rebuildReason} - loading/rebuilding ${rebuildingLegs || 'no'} index(es) from persisted data...`) } - // Rebuild all 3 indexes in parallel for performance - // Indexes load their data from storage (no recomputation) + // Before the graph rebuild, hydrate the entity id-mapper from the persisted + // snapshot. A native int-keyed adjacency resolves every verb endpoint through + // this mapper, so it must reflect the persisted int assignments BEFORE + // graphIndex.rebuild() runs — otherwise edges resolve to stale/missing ints + // and are silently dropped (CTX-BR-RESTORE-REBUILD). Mirrors restore()'s + // ordering; a no-op for the JS mapper (re-derived by metadataIndex.rebuild()). + if (shouldRebuildGraph) { + await this.hydrateIdMapperForGraphRebuild() + } + + // Rebuild the needed indexes in parallel for performance. Indexes load + // their data from storage (no recomputation). An epoch drift (`epochStale`) + // rebuilds every NON-migrating index regardless of its current size — the + // on-disk format changed, so each is rebuilt from the canonical records. A + // provider running its own background migration is skipped here (it owns + // its index until it verifies-and-swaps). const rebuildStartTime = Date.now() await Promise.all([ - metadataStats.totalEntries === 0 ? this.metadataIndex.rebuild() : Promise.resolve(), - hnswIndexSize === 0 ? this.index.rebuild() : Promise.resolve(), - graphIndexSize === 0 ? this.graphIndex.rebuild() : Promise.resolve() + shouldRebuildMetadata ? this.metadataIndex.rebuild() : Promise.resolve(), + shouldRebuildVector ? this.index.rebuild() : Promise.resolve(), + shouldRebuildGraph ? this.graphIndex.rebuild() : Promise.resolve() ]) const rebuildDuration = Date.now() - rebuildStartTime @@ -11477,8 +15184,11 @@ export class Brainy implements BrainyInterface { } // Consistency verification: metadata index must match storage entity count. - // If mismatch, the rebuild missed entities — force a second attempt. - if (metadataCountAfter === 0 && totalCount > 0) { + // If mismatch, the rebuild missed entities — force a second attempt. Skipped + // when the metadata provider holds the migration lock: a 0 count there + // reflects its in-place rebuild in progress, not a missed rebuild, so + // forcing a second rebuild would collide with the provider's own. + if (metadataCountAfter === 0 && totalCount > 0 && !metadataMigrating) { console.error( `[Brainy] CRITICAL: Metadata index has 0 entries but storage has ${totalCount} entities. ` + `Forcing second rebuild.` @@ -11488,9 +15198,344 @@ export class Brainy implements BrainyInterface { console.log(`[Brainy] Second rebuild result: ${secondAttempt} entries`) } + // 8.0 ⇄ native-provider handshake (NON-DESTRUCTIVE): the derived indexes + // have now rebuilt and verified, so they match this build's epoch — + // re-stamp the marker LAST, only here. A crash anywhere above leaves the + // old / absent marker, so the next open re-detects the drift and re-runs + // the (idempotent) rebuild; the marker is never advanced ahead of the + // indexes it certifies. A no-op when the epoch was already current. + // + // EXCEPT while a provider holds the migration lock: brainy deferred that + // index's rebuild, so it is NOT yet at this epoch. The marker (which + // certifies ALL derived indexes) must not advance ahead of the index the + // provider is still rebuilding in place — the provider calls + // brain.stampBrainFormat() itself once it has verified parity. + if (!anyMigrating) { + await this.stampBrainFormatIfNeeded() + } + } catch (error) { - console.warn('Warning: Could not rebuild indexes:', error) - // Don't throw - allow system to start even if rebuild fails + // A storage READ failure here is surfaced by getNouns/getVerbs as a named + // BrainyError — it means we could not even read the store to decide whether + // a rebuild is needed (or to perform it). That is a hard init failure, not a + // "start anyway" condition: booting a brain whose data we couldn't read would + // serve empty/incomplete results with no signal (the silent-failure class the + // 8.0 contract forbids). Re-throw it loud. + if (error instanceof BrainyError) throw error + // Any other rebuild hiccup is non-fatal — the brain may start on partially + // rebuilt indexes — but it is recorded as a queryable degraded state + // (surfaced via checkHealth) and escalated to error-level logging, rather + // than only emitting a console.warn that is trivially lost. + this._indexRebuildFailed = error instanceof Error ? error : new Error(String(error)) + console.error('[Brainy] Index rebuild failed; starting in a DEGRADED state (queries may be incomplete):', error) + } + } + + /** + * @description Re-stamp the 8.0 ⇄ native-provider version-handshake marker + * (`_system/brain-format.json`) to this build's {@link CURRENT_DATA_FORMAT} / + * {@link EXPECTED_INDEX_EPOCH} — but ONLY when the on-disk marker was absent + * or carried a drifted epoch ({@link _indexEpochStale}). Called at the + * verified-completion points of {@link rebuildIndexesIfNeeded} (after the + * derived-index rebuild, or on the empty-storage fast path), so the marker is + * never advanced ahead of the indexes it certifies — a crash before this + * point re-triggers the idempotent rebuild on the next open. Readers never + * write, so this is a no-op in reader mode. + */ + private async stampBrainFormatIfNeeded(): Promise { + if (this.isReadOnly) return + if (!this._indexEpochStale) return + await writeBrainFormat(this.storage) + this._brainFormat = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH } + this._indexEpochStale = false + // The JS inline rebuild verified + stamped — drop the pre-upgrade backup. + await this.removeMigrationBackupSafe() + } + + /** + * @description Whether an index provider holds the coordinated migration LOCK + * (#18). A native provider's `init()` flips an OPTIONAL `isMigrating(): boolean` + * to `true` the moment it detects a large 7.x epoch-drift and keeps it `true` + * while it rebuilds all derived indexes IN PLACE from the canonical records, + * clearing it only after it has verified parity and stamped the marker. While + * `true`, brainy BLOCKS/QUEUES data-plane reads AND writes on the lock (see + * {@link awaitMigrationLock}) so no operation touches a half-built index — + * the "unknown/halfway state" is closed by waiting for a known-good one. + * Feature-detected: a provider that omits `isMigrating` (every JS index) is + * treated as not migrating, so behavior is unchanged. + * @param provider - A registered index provider (metadata / vector / graph). + * @returns `true` only when the provider exposes `isMigrating()` and it reports + * an in-flight migration. + */ + private providerIsMigrating(provider: unknown): boolean { + return ( + provider != null && + typeof (provider as { isMigrating?: () => boolean }).isMigrating === 'function' && + (provider as { isMigrating: () => boolean }).isMigrating() === true + ) + } + + /** + * @description `true` when ANY index provider (metadata / vector / graph) holds + * the migration LOCK. The single predicate the data-plane gate and the status + * surface consult; a brain with no migrating provider evaluates three cheap + * feature-detects and returns `false`. + */ + private anyProviderMigrating(): boolean { + return ( + this.providerIsMigrating(this.metadataIndex) || + this.providerIsMigrating(this.index) || + this.providerIsMigrating(this.graphIndex) + ) + } + + /** + * @description The provider instance backing a given index {@link IndexFamily}. + * The single place the family label maps to the concrete provider, so the + * scoped migration gate and any future family-aware routing agree. + */ + private providerForFamily(family: IndexFamily): unknown { + switch (family) { + case 'vector': + return this.index + case 'metadata': + return this.metadataIndex + case 'graph': + return this.graphIndex + } + } + + /** + * @description Whether the migration LOCK should hold for an operation that + * depends on the given index families: + * - `needs === undefined` → WHOLE-BRAIN: any migrating provider counts (the + * conservative default for writes and unclassified reads — a write touches + * every index). + * - `needs === []` → NEVER: a canonical-storage read consults no derived index, + * so a migration elsewhere is irrelevant; it serves immediately. + * - `needs = [families]` → SCOPED: only those families' providers count, so a + * read served from a healthy family is not blocked by an unrelated family's + * one-time migration. + */ + private neededFamiliesMigrating(needs?: IndexFamily[]): boolean { + if (needs === undefined) return this.anyProviderMigrating() + for (const family of needs) { + if (this.providerIsMigrating(this.providerForFamily(family))) return true + } + return false + } + + /** + * @description The index families a {@link find} query consults, derived from + * its params — the read-side input to the family-scoped migration gate. A + * vector / near / semantic query needs `vector`; a `where` / type / subtype / + * service filter or an aggregate needs `metadata`; a `connected` traversal + * needs `graph`. A query combining shapes needs the union. Mirrors find()'s + * own criteria split so the gate and the executor never disagree. + */ + private queryIndexFamilies(params: FindParams): IndexFamily[] { + const needs: IndexFamily[] = [] + if ((params.query && params.query.trim() !== '') || params.vector || params.near) needs.push('vector') + if (params.where || params.type || params.subtype || params.service || params.aggregate) needs.push('metadata') + if (params.connected) needs.push('graph') + return needs + } + + /** + * @description Rich migration progress relayed verbatim from whichever provider + * exposes the OPTIONAL `migrationStatus(): { phase?, index?, percent?, … } | null`. + * The provider owns the rebuild, so it owns the truth of the percentage; brainy + * surfaces it through {@link getIndexStatus} and {@link health}. Returns `null` + * when no provider reports structured progress (brainy then shows only + * `migrating: true` + `elapsedMs`). Feature-detected and best-effort. + */ + private providerMigrationStatus(): MigrationProgress | null { + for (const provider of [this.metadataIndex, this.index, this.graphIndex]) { + const fn = (provider as { migrationStatus?: () => unknown }).migrationStatus + if (typeof fn === 'function') { + try { + const status = fn.call(provider) + if (status && typeof status === 'object') { + return status as MigrationProgress + } + } catch { + // best-effort: a provider status hiccup never breaks the gate/probe + } + } + } + return null + } + + /** + * @description Block-and-queue on the coordinated migration LOCK (#18). Called + * at the {@link ensureInitialized} choke point (and once inside `init()` before + * VFS bootstrap), so a data-plane operation waits here while a native provider + * runs its one-time 7.x → 8.0 rebuild-from-canonical. The caller gets the + * correct answer, never a partial read of a half-built index and never a lost + * write. A brain that is not migrating pays a single boolean check and returns + * immediately. + * + * FAMILY-SCOPED: `needs` names the index families the caller depends on, so the + * wait holds ONLY for a migration of one of those families. A read served + * entirely from canonical storage (`needs: []`) or from a healthy family never + * blocks on an unrelated family's migration. `needs` omitted = the conservative + * whole-brain wait (writes touch every index; startup wants all of them ready). + * @param needs - Index families this operation consults, or `undefined` for the + * whole-brain wait. + * + * IMPORTANT: this timeout bounds THE CALLER'S WAIT, not the migration. The + * migration itself is unbounded — the native provider rebuilds a billion-scale + * brain for as long as it needs; brainy cannot and does not abort it. The lock + * is polled every {@link Brainy.MIGRATION_POLL_INTERVAL_MS}; a small/medium + * upgrade clears well within `migrationWaitTimeoutMs` (default 30 s) and the + * caller resumes transparently. If the wait outlives the budget, a retryable + * {@link MigrationInProgressError} is thrown — the upgrade keeps running in the + * background, so the caller retries, watches `getIndexStatus().migration` + * (never gated — the primary large-upgrade signal for a readiness probe), or, + * for a very large brain, runs the offline migrator. The block/release lines + * log once per window; the flags reset on release so a later upgrade re-logs. + */ + private async awaitMigrationLock(needs?: IndexFamily[]): Promise { + if (!this.neededFamiliesMigrating(needs)) return // fast path — the overwhelming common case + + const startedAt = Date.now() + const timeoutMs = this.config.migrationWaitTimeoutMs ?? 30_000 + + if (!this._migrationBlockLogged) { + this._migrationBlockLogged = true + prodLog.info( + '[Brainy] Auto-upgrade in progress (7.x → 8.0): reads and writes are blocked ' + + 'until the rebuild completes. This is automatic and one-time; data is safe.' + ) + } + + while (this.neededFamiliesMigrating(needs)) { + const remaining = timeoutMs - (Date.now() - startedAt) + if (remaining <= 0) { + const status = this.providerMigrationStatus() + throw new MigrationInProgressError( + `Brain is still upgrading (7.x → 8.0)` + + (status?.percent !== undefined ? `, ${Math.round(status.percent)}% complete` : '') + + ` after ${Math.round((Date.now() - startedAt) / 1000)}s. The wait timed out; the upgrade ` + + `continues in the background. Retry shortly, watch brain.getIndexStatus().migration for ` + + `progress, raise config.migrationWaitTimeoutMs to wait longer, or for a very large brain ` + + `run the offline migrator. Nothing is lost.`, + Date.now() - startedAt, + status?.percent + ) + } + await new Promise((r) => setTimeout(r, Math.min(Brainy.MIGRATION_POLL_INTERVAL_MS, remaining))) + } + + if (this._migrationBlockLogged && !this._migrationReleaseLogged) { + this._migrationReleaseLogged = true + prodLog.info( + `[Brainy] Auto-upgrade complete in ${Math.round((Date.now() - startedAt) / 1000)}s — ` + + 'reads and writes resumed.' + ) + } + // Reset the once-per-window guards so a subsequent upgrade in this same + // instance re-logs cleanly and re-clocks its elapsed (a fresh observed-at). + this._migrationBlockLogged = false + this._migrationReleaseLogged = false + this._migrationObservedAt = null + } + + /** + * @description The lock-exempt observability snapshot consumed by + * {@link getIndexStatus} and {@link health}: whether the brain is upgrading and, + * if so, the provider's structured {@link MigrationProgress} enriched with a + * `startedAt` / `elapsedMs` that brainy tracks itself (so a provider that omits + * timing still yields a live elapsed). Reads the migration flag without waiting, + * so a readiness probe can always report `migrating` → HTTP 503 + Retry-After + * rather than routing traffic into a half-built index. Lazily stamps the + * observed-at clock on first sight and clears it once the lock releases. + */ + private migrationSnapshot(): { migrating: boolean; migration?: MigrationProgress } { + if (!this.anyProviderMigrating()) { + this._migrationObservedAt = null + return { migrating: false } + } + if (this._migrationObservedAt === null) { + this._migrationObservedAt = Date.now() + } + const provided = this.providerMigrationStatus() + return { + migrating: true, + migration: { + ...provided, + startedAt: provided?.startedAt ?? this._migrationObservedAt, + elapsedMs: provided?.elapsedMs ?? Date.now() - this._migrationObservedAt + } + } + } + + /** + * @description Take the pre-upgrade backup (default-on) before a native provider + * rebuilds anything for a 7.x → 8.0 migration. Called at open, once the on-disk + * format is known stale, BEFORE the providers init. Feature-detected + best-effort: + * a backend without `createMigrationBackup` (memory) or a store with no data is a + * no-op, and a backup failure NEVER blocks the upgrade (the migration is itself + * safe — it only reads canonical, and self-heals on re-open). The snapshot is + * removed once the upgrade verifies ({@link removeMigrationBackupSafe}) and + * retained on failure for rollback. + */ + private async createMigrationBackupIfNeeded(): Promise { + const storage = this.storage as StorageAdapter & { + createMigrationBackup?: () => Promise + } + if (typeof storage.createMigrationBackup !== 'function') return + try { + const backupPath = await storage.createMigrationBackup() + if (backupPath) { + this._migrationBackupPath = backupPath + prodLog.info( + `[Brainy] Pre-upgrade backup created at ${backupPath} (hard-link snapshot; ~zero ` + + `cost/space). Removed automatically once the 7.x→8.0 upgrade verifies; retained ` + + `for rollback if it fails. Opt out with { migrationBackup: false }.` + ) + } + } catch (error) { + prodLog.warn( + `[Brainy] Pre-upgrade backup could not be created (continuing — the upgrade is safe ` + + `and self-heals): ${error instanceof Error ? error.message : String(error)}` + ) + } + } + + /** + * @description Remove the pre-upgrade backup once the migration has verified and + * stamped the format marker. Called from both stamp paths (the JS inline rebuild + * and the native provider's `stampBrainFormat()`), gated on a backup having been + * taken this open. Best-effort: a removal failure leaves a harmless backup dir. + */ + private async removeMigrationBackupSafe(): Promise { + if (!this._migrationBackupPath) return + // Backup-parity gate: the index rebuild "success" that calls this does NOT + // certify the VFS. If the on-open blob adoption could not fully bridge the + // 7.x `_cow/` blobs, the upgrade is not verifiably complete for VFS content + // — retain the backup so the operator can recover from it, and log why. + if (this._vfsBlobAdoptionIncomplete) { + if (!this.config.silent) { + console.warn( + `[brainy] Retaining the pre-upgrade backup at ${this._migrationBackupPath}: ` + + `some VFS content blobs could not be adopted from _cow/ (see the earlier warning). ` + + `Resolve those before discarding the backup.` + ) + } + return + } + const backupPath = this._migrationBackupPath + this._migrationBackupPath = null + const storage = this.storage as StorageAdapter & { + removeMigrationBackup?: (location: string) => Promise + } + if (typeof storage.removeMigrationBackup !== 'function') return + try { + await storage.removeMigrationBackup(backupPath) + prodLog.info(`[Brainy] Upgrade verified — pre-upgrade backup at ${backupPath} removed.`) + } catch { + // best-effort — a leftover backup dir is harmless } } @@ -11511,20 +15556,306 @@ export class Brainy implements BrainyInterface { indexEntryCount: number recommendation: string | null }> { - await this.ensureInitialized() - return this.metadataIndex.validateConsistency() + // Lock-exempt: diagnostics must answer while the brain upgrades. + await this.ensureInitialized({ bypassMigrationLock: true }) + const result = await this.metadataIndex.validateConsistency() + // Fold in a non-fatal index-rebuild failure recorded at init() so the degraded + // state is observable through the same health surface (not just the console). + if (this._indexRebuildFailed) { + return { + ...result, + healthy: false, + recommendation: + `Index rebuild failed at init() (degraded — queries may be incomplete): ` + + `${this._indexRebuildFailed.message}. Rebuild the indexes and re-open.` + + (result.recommendation ? ` Also: ${result.recommendation}` : '') + } + } + if (this._indexDegradedIds.size > 0) { + return { + ...result, + healthy: false, + recommendation: + `${this._indexDegradedIds.size} record(s) committed via adopt-forward ` + + `failed-rollback recovery — the derived index may be incomplete for them. ` + + `Run repairIndex() to reconcile.` + + (result.recommendation ? ` Also: ${result.recommendation}` : '') + } + } + return result } /** - * Detect and repair corrupted metadata indexes - * - * Runs corruption detection and auto-rebuilds if corruption is found. - * This is the equivalent of the old init()-time corruption check, - * now available as an explicit operation. + * Run the optional metadata cold-open consistency probe at most once per brain. + * When the active provider exposes `probeConsistency()` (the native cross-bucket + * O(1) sampler), a `false` result triggers `detectAndRepairCorruption()` so an + * already-poisoned index self-heals on first read — the metadata counterpart of + * the 7.33.2 graph cold-load guard. Best-effort: a probe failure never breaks the + * read (the guard is reset so a transient failure retries). No-op for the JS index + * (it exposes no probe), and the full-scan `validateConsistency` stays the explicit + * deep diagnostic via `validateIndexConsistency()`. */ + private async ensureMetadataConsistencyProbed(): Promise { + if (this._metadataConsistencyProbed) return + // Defer while the metadata provider runs its one-time in-place migration: + // probing (and self-healing via rebuild) an index the provider is mid-rebuild + // would collide with the provider that owns it. Mirrors the vector deference + // in ensureIndexesLoaded. Do NOT latch — once the migration clears, the next + // read runs the probe. (The family-scoped find() gate waits on the metadata + // family separately before any actual filter read.) + if (this.providerIsMigrating(this.metadataIndex)) return + this._metadataConsistencyProbed = true + const provider = this.metadataIndex as { + probeConsistency?: () => Promise + detectAndRepairCorruption?: () => Promise + } + if (typeof provider.probeConsistency !== 'function') return + try { + const healthy = await provider.probeConsistency() + if (!healthy && typeof provider.detectAndRepairCorruption === 'function') { + if (!this.config.silent) { + console.warn('[Brainy] metadata index failed the cold-open consistency probe — self-healing via rebuild.') + } + await provider.detectAndRepairCorruption() + } + } catch (error) { + // The self-heal is best-effort and must never break a read. Reset the guard + // so a transient probe failure is retried on the next read. + this._metadataConsistencyProbed = false + if (!this.config.silent) { + console.warn('[Brainy] metadata cold-open consistency probe failed (continuing):', error) + } + } + } + + /** + * Detect and repair corrupted metadata indexes. + * + * Runs corruption detection and auto-rebuilds if corruption is found. This is + * the equivalent of the old init()-time corruption check, now available as an + * explicit operation. + * + * It is ALSO the recovery path for a write-quarantine: when a transaction's + * rollback fails and leaves the store inconsistent ({@link StoreInconsistentError}), + * writes are refused until this method reconciles the derived indexes against + * canonical storage (a forced rebuild — orphaned/lost records reflected + * consistently) and lifts the quarantine. Canonical is the source of truth: a + * genuinely lost record cannot be resurrected here (restore from a snapshot for + * that), but the store is made internally consistent and writes re-enabled. + */ + /** + * Emit ONE loud warning per degraded window when a read is served while the + * derived index is known-incomplete — either a non-fatal init rebuild failure + * ({@link _indexRebuildFailed}) or an adopt-forward degraded commit + * ({@link _indexDegradedIds}). Reads still return (canonical is the source of + * truth), but the caller is told the result may be partial and how to heal it. + * No-op when healthy or when the brain is configured `silent`. + * @param op - The read method name for the message (e.g. `'find'`, `'get'`). + */ + private warnIfReadsDegraded(op: string): void { + const degraded = + this._indexRebuildFailed !== null || this._indexDegradedIds.size > 0 + if (!degraded) { + this._degradedReadWarned = false + return + } + if (this._degradedReadWarned || this.config.silent) return + this._degradedReadWarned = true + const reason = this._indexRebuildFailed + ? `index rebuild failed at init() (${this._indexRebuildFailed.message})` + : `${this._indexDegradedIds.size} record(s) committed in a degraded state` + prodLog.warn( + `[Brainy] ${op}() is serving reads while the derived index is INCOMPLETE ` + + `(${reason}). Results may be missing entries — run repairIndex() to ` + + `reconcile the derived indexes against canonical storage.` + ) + } + + /** + * Read-only graph-truth audit — proves (or disproves) that relationship + * reads return canonical truth on THIS brain, and classifies every + * discrepancy into its failure family: + * + * - `missingFromReads` — canonical verb records the read path omits + * (PRESENT BUT INVISIBLE: adjacency/membership staleness) + * - `danglingEndpoints` — verbs whose endpoint entity is gone (SCAR class) + * - `readOnlyVerbIds` — read-path edges with no canonical record (GHOSTS) + * + * Design-hidden edges (internal/system visibility) are counted separately — + * the audit reads with every visibility tier included, so intentional + * hiding is never misclassified as index loss. Mutates nothing; safe on a + * live brain (cost: one canonical walk + one indexed read per source). + * Run it after any engine upgrade, restore, or migration; a `coherent` + * report is the verified statement that `related()`/`readdir` can be + * trusted. If it reports discrepancies, `repairIndex()` is the sanctioned + * heal — re-run the audit afterwards to prove the repair. + * + * @param options.maxExamples - Cap per example list (counts stay exact). Default 100. + * @returns The full audit report; also narrated via logs (loud on incoherence). + * @since 8.6.0 + */ + async auditGraph(options: { maxExamples?: number } = {}): Promise { + await this.ensureInitialized({ needs: ['graph'] }) + + const PAGE = 1000 + return runGraphAudit( + { + eachNounId: async (consume) => { + let cursor: string | undefined + let offset = 0 + for (;;) { + const page = await (this.storage as unknown as { + getNounIdsWithPagination(o: { + limit: number + offset?: number + cursor?: string + }): Promise<{ ids: string[]; hasMore: boolean; nextCursor?: string }> + }).getNounIdsWithPagination( + cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset } + ) + for (const id of page.ids) consume(id) + if (!page.hasMore || page.ids.length === 0) break + if (page.nextCursor) cursor = page.nextCursor + else offset += page.ids.length + } + }, + eachVerb: async (consume) => { + let cursor: string | undefined + let offset = 0 + for (;;) { + const page = await this.storage.getVerbs({ + pagination: cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset } + }) + for (const verb of page.items) { + const v = verb as unknown as Record + consume({ + id: String(v.id), + type: String(v.verb ?? 'unknown'), + sourceId: String(v.sourceId), + targetId: String(v.targetId), + visibility: typeof v.visibility === 'string' ? v.visibility : undefined + }) + } + if (!page.hasMore || page.items.length === 0) break + if (page.nextCursor) cursor = page.nextCursor + else offset += page.items.length + } + }, + readRelationsFrom: async (sourceId) => + this.related({ + from: sourceId, + includeInternal: true, + includeSystem: true, + limit: 100000 + }) + }, + options + ) + } + async repairIndex(): Promise { await this.ensureInitialized() + + // Prune orphaned canonical containers left by the pre-8.3.1 partial-delete + // defect: a delete that removed the metadata (content) leg but left the + // vector leg + the entity directory (a "ghost"), or left an empty directory + // (a "scar"). These are not live entities (getNoun needs the content leg) + // yet inflate enumerated counts and confuse locator resolution. Feature- + // detected (filesystem-only — key/prefix stores have no orphan containers); + // recompute counts afterward so the totals stop counting the ghosts. + const pruner = this.storage as { + pruneOrphanedEntities?: () => Promise<{ nouns: string[]; verbs: string[] }> + rebuildTypeCounts?: () => Promise + rebuildSubtypeCounts?: () => Promise + } + if (typeof pruner.pruneOrphanedEntities === 'function') { + const orphans = await pruner.pruneOrphanedEntities() + if (orphans.nouns.length + orphans.verbs.length > 0) { + prodLog.warn( + `[Brainy] repairIndex() pruned ${orphans.nouns.length} orphaned noun + ` + + `${orphans.verbs.length} orphaned verb container(s) left by a pre-8.3.1 ` + + `partial delete.` + ) + } + } + // SANCTIONED RECOUNT — unconditional, not gated on orphans found: the + // persisted counters can be inflated over perfectly clean shelves (deletes + // whose decrement was skipped by the removed-record re-read), and + // Math.max(totalNounCount, scanned) means an inflated scalar can never + // correct itself. rebuildTypeCounts() recomputes EVERY counter rollup + // (scalar totals + per-type maps + type-statistics arrays) from one + // canonical walk and persists them. + await pruner.rebuildTypeCounts?.() + await pruner.rebuildSubtypeCounts?.() + + // The recount changed the rollup truth — re-stamp the entity tree so the + // stamp's invariants match the healed counters (repair leaves a coherent + // stamp, not a stale one that warns on the next open). + await this.stampEntityTree() + + // VFS containment reconciliation: heal "cosmetic ghost" edges left by + // pre-fix renames (an entity Contains-linked from BOTH its old and new + // directory — readdir listed it in two places) and duplicate edges from + // concurrent writers. Canonical metadata.path is the truth; only VFS + // containment edges are touched. Loud per repair. + if (this._vfsInitialized && this._vfs) { + const containment = await this._vfs.repairContainment() + if (containment.removed + containment.restored > 0) { + prodLog.warn( + `[Brainy] repairIndex() reconciled VFS containment: removed ${containment.removed} ` + + `stale/duplicate edge(s), restored ${containment.restored} missing edge(s).` + ) + } + } + await this.metadataIndex.detectAndRepairCorruption() + // Lift a failed-rollback write-quarantine: force a full rebuild so the + // derived indexes are provably reconciled with canonical, then clear the + // flag so writes resume. + if (this.storeInconsistency) { + await this.rebuildIndexesIfNeeded(true) + const cleared = this.storeInconsistency + this.storeInconsistency = null + prodLog.warn( + `[Brainy] repairIndex() reconciled the store and LIFTED the write-quarantine ` + + `set by a failed transaction rollback (${cleared.records.length} record(s) affected). ` + + `Writes are re-enabled.` + ) + } + // Cross-layer repair: repairIndex must reconcile NATIVE derived + // state from canonical, not just the JS metadata index. Consult each provider's + // own validateInvariants() and rebuild any whose failing invariant asks for it + // (heal: 'rebuild') — the native counterpart of detectAndRepairCorruption(). + for (const provider of [this.metadataIndex, this.index, this.graphIndex]) { + const p = provider as { + validateInvariants?: () => Promise + rebuild?: () => Promise + } | null + if (!p || typeof p.validateInvariants !== 'function' || typeof p.rebuild !== 'function') continue + let report: ProviderInvariantReport + try { + report = await p.validateInvariants() + } catch { + continue // a throwing validateInvariants is surfaced by validateIndexConsistency; skip repair here + } + if (report.healthy) continue + if (report.invariants.some((i) => !i.holds && i.heal === 'rebuild')) { + prodLog.warn( + `[Brainy] repairIndex(): provider '${report.provider}' has a failing invariant ` + + `requiring a rebuild — reconciling its derived state from canonical.` + ) + await p.rebuild() + } + } + // detectAndRepairCorruption() above rebuilt the derived indexes from + // canonical, so any adopt-forward degraded ids and a non-fatal init + // rebuild failure are now reconciled — clear the queryable degraded state + // and re-arm the read-path warning. + if (this._indexDegradedIds.size > 0 || this._indexRebuildFailed) { + this._indexDegradedIds.clear() + this._indexRebuildFailed = null + this._degradedReadWarned = false + } } /** @@ -11549,23 +15880,59 @@ export class Brainy implements BrainyInterface { * Auto-detect and activate plugins. * Called internally during init(). */ + /** + * Import a plugin package by name. Isolated as a seam so tests can simulate + * the three auto-detect outcomes (not installed / broken install / valid) + * without the package being present. The specifier is a variable, so + * bundlers cannot statically resolve — and cannot force-include — the + * optional accelerator. + */ + private async importPluginPackage(pkg: string): Promise { + return import(pkg) + } + + /** + * True when a dynamic-import failure means "the package itself is not + * installed" (the silent free path), as opposed to "the package is present + * but broken" (which must fail loud). Node and Bun both name the missing + * package in the resolution error; a failure naming anything else — an + * internal file, a dependency of the plugin, a syntax error — is a broken + * install, never a not-installed. + */ + private static isPackageNotInstalledError(error: unknown, pkg: string): boolean { + const code = (error as { code?: string })?.code + const message = error instanceof Error ? error.message : String(error) + const namesPackage = + message.includes(`'${pkg}'`) || message.includes(`"${pkg}"`) || message.includes(` ${pkg}`) + const isResolutionFailure = + code === 'ERR_MODULE_NOT_FOUND' || + code === 'MODULE_NOT_FOUND' || + /cannot find (package|module)/i.test(message) || + /failed to resolve/i.test(message) // Bun's resolver phrasing + return isResolutionFailure && namesPackage + } + private async loadPlugins(): Promise { // plugins config: - // undefined (default) → no auto-detection (safe default) - // false → no auto-detection - // [] → no auto-detection - // ['@soulcraft/cor'] → load only these explicitly listed packages + // undefined (default) → guarded auto-detection of the first-party + // accelerator: installing @soulcraft/cor IS the + // opt-in. Not installed → plain brainy, silently. + // Installed → it loads and announces itself; if it + // is installed but broken, init() THROWS — an + // installed accelerator never silently vanishes. + // false / [] → no plugins, no detection (explicit opt-out) + // ['@soulcraft/cor'] → load only these explicitly listed packages // Note: plugins registered via brain.use() are always activated regardless of config const pluginConfig = this.config.plugins if (Array.isArray(pluginConfig) && pluginConfig.length > 0) { // Explicit list: import and register the specified packages. A package - // listed here is REQUIRED — brainy does no auto-detection, so a missing or - // invalid plugin must fail LOUD, never silently fall back to the default - // engine (the cross-repo drift this whole guard exists to prevent). + // listed here is REQUIRED — a missing or invalid plugin must fail LOUD, + // never silently fall back to the default engine (the cross-repo drift + // this whole guard exists to prevent). for (const pkg of pluginConfig) { let mod: any try { - mod = await import(pkg) + mod = await this.importPluginPackage(pkg) } catch (error) { throw new Error( `[brainy] Plugin "${pkg}" is listed in config.plugins but could not be loaded: ` + @@ -11583,6 +15950,38 @@ export class Brainy implements BrainyInterface { ) } } + } else if (pluginConfig === undefined) { + // Guarded auto-detection (default). Installing the first-party + // accelerator is the opt-in — probe for it, and apply the SAME loud + // posture as the explicit branch to everything except "not installed": + // a present-but-broken accelerator must never silently degrade to JS. + for (const pkg of Brainy.AUTO_DETECT_PLUGIN_PACKAGES) { + let mod: any + try { + mod = await this.importPluginPackage(pkg) + } catch (error) { + if (Brainy.isPackageNotInstalledError(error, pkg)) { + continue // the free path: not installed, nothing to load, no noise + } + throw new Error( + `[brainy] The accelerator "${pkg}" is installed but failed to load: ` + + `${error instanceof Error ? error.message : String(error)}. ` + + `brainy will NOT silently run the default JS engines in its place — ` + + `fix the install (npm i ${pkg}) or disable detection with plugins: [].` + ) + } + const plugin: BrainyPlugin = mod.default || mod + if (plugin && typeof plugin.activate === 'function' && plugin.name) { + this.pluginRegistry.register(plugin) + } else { + throw new Error( + `[brainy] The installed accelerator "${pkg}" is not a valid Brainy plugin ` + + `(missing { name, activate }) — a broken or incompatible install. ` + + `brainy will NOT silently run the default JS engines in its place — ` + + `fix the install (npm i ${pkg}) or disable detection with plugins: [].` + ) + } + } } // Create plugin context @@ -11703,34 +16102,112 @@ export class Brainy implements BrainyInterface { * case under durable storage, where a brain reopens pre-populated — would otherwise * return `[]`. On first query we clear the aggregate's state and stream every stored * noun back through it. Storage-agnostic: `getNouns()` works for in-memory, the - * filesystem adapter, and native (Cortex) storage alike. One-time per definition — + * filesystem adapter, and native (Cor) storage alike. One-time per definition — * the rebuilt state is persisted on flush() and reloaded on the next session. */ private async backfillAggregateIfNeeded(name: string): Promise { const index = this._aggregationIndex - if (!index || !index.getPendingBackfills().includes(name)) return + if (!index) return - index.beginBackfill(name) + // Persisted-state adoption happens inside ready(); after it resolves the + // pending-backfill set is authoritative (an unchanged definition with valid + // persisted state is NOT listed — no walk at all on a clean reopen). + await index.ready() - const PAGE = 500 - let offset = 0 - let cursor: string | undefined - for (;;) { - const page = await this.storage.getNouns({ - pagination: cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset } - }) - for (const noun of page.items) { - index.backfillEntity(name, noun as unknown as Record) + // Single-flight: concurrent queries share ONE walk instead of each wiping + // the others' partial state and starting their own (the stampede that kept + // a busy store from ever converging). The loop covers the rare case where + // the in-flight walk snapshotted its batch before `name` became pending — + // the next iteration starts a fresh walk that includes it. + while (index.getPendingBackfills().includes(name)) { + // Failure latch: a deterministically-failing walk must not be re-run at + // the caller's retry rate — that is a silent CPU loop wearing a retry + // loop's clothes. Within the cooldown, rethrow the recorded failure + // immediately; after it, one fresh attempt is allowed. + const failure = this._aggregationBackfillFailure + if (failure && Date.now() - failure.at < AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS) { + throw new Error( + `Aggregation backfill for '${name}' is in failure cooldown (retry in ` + + `${Math.ceil((AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS - (Date.now() - failure.at)) / 1000)}s). ` + + `Last failure: ${failure.error.message}` + ) } - if (!page.hasMore || page.items.length === 0) break - if (page.nextCursor) { - cursor = page.nextCursor - } else { - offset += page.items.length + if (!this._aggregationBackfillFlight) { + this._aggregationBackfillFlight = this.runAggregationBackfillWalk() + .finally(() => { + this._aggregationBackfillFlight = null + }) } + await this._aggregationBackfillFlight + } + } + + /** + * One store walk fills EVERY aggregate currently pending backfill — M pending + * aggregates cost one enumeration, not M. Only reached when an aggregate + * genuinely needs a rescan (new definition over a populated store, changed + * definition, or failed state load); a clean reopen adopts persisted state + * and never walks. + */ + private async runAggregationBackfillWalk(): Promise { + const index = this._aggregationIndex! + const names = index.getPendingBackfills() + if (names.length === 0) return + + prodLog.info(`[Aggregation] backfill walk starting for: ${names.join(', ')}`) + const startedAt = Date.now() + for (const n of names) index.beginBackfill(n) + + let scanned = 0 + try { + const PAGE = 500 + let offset = 0 + let cursor: string | undefined + for (;;) { + const page = await this.storage.getNouns({ + pagination: cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset } + }) + for (const noun of page.items) { + const record = noun as unknown as Record + for (const n of names) { + index.backfillEntity(n, record) + } + } + scanned += page.items.length + if (!page.hasMore || page.items.length === 0) break + if (page.nextCursor) { + if (page.nextCursor === cursor) { + // A non-advancing cursor with hasMore=true would loop this walk at + // CPU speed forever, silently. That is a storage pagination defect — + // fail the waiting queries loudly instead of spinning. + throw new Error( + `Aggregation backfill aborted: storage pagination returned a non-advancing cursor ` + + `after ${scanned} entities with hasMore=true — the storage adapter's getNouns cursor is broken.` + ) + } + cursor = page.nextCursor + } else { + offset += page.items.length + } + } + } catch (err) { + // Non-destructive failure: drop the staging maps (live state keeps + // serving), keep the aggregates flagged pending, latch the error so + // retries within the cooldown fail fast, and say all of it out loud. + for (const n of names) index.abortBackfill(n) + this._aggregationBackfillFailure = { at: Date.now(), error: err as Error } + prodLog.warn( + `[Aggregation] backfill walk FAILED after ${scanned} entities: ${(err as Error).message} — ` + + `prior aggregate state preserved; retries suppressed for ${AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS / 1000}s` + ) + throw err } - index.finishBackfill(name) + for (const n of names) index.finishBackfill(n) + this._aggregationBackfillFailure = null + prodLog.info( + `[Aggregation] backfill walk finished: ${scanned} entities → ${names.length} aggregate(s) in ${Date.now() - startedAt}ms` + ) } /** @@ -11740,13 +16217,39 @@ export class Brainy implements BrainyInterface { * This ensures deferred persistence mode data is saved */ async close(): Promise { - // Phase 0: Auto-compact generational history per config.history (default - // on) BEFORE the generation store closes below. Respects live Db pins and - // an explicit autoCompact: false; no-op on read-only instances. + // Cancel any pending post-import background deduplication FIRST — it is a + // writer (merge-deletes), and no delete pass may start mid- or post-close. + this._backgroundDedup?.cancelPending() + + // Change-feed teardown: no events are delivered for or after close(). + this._changeFeed.close() + + // Phase 0a: Persist buffered single-op generation history (async + // group-commit) before anything else, so a clean close never drops history + // the caller already observed. No-op when nothing is pending or read-only. + if (this.generationStore && !this.isReadOnly) { + await this.generationStore.flushPendingSingleOps() + } + + // Phase 0b: REPACK cold history into sealed segments (D1+D3 — + // re-representation, never deletion; the only history transform under the + // archival profile), then auto-compact per config.retention. Repack runs + // FIRST so bounded-retention reclaim can drop whole segments. Both are + // time-bounded maintenance passes (8.9.0 law: flush() never pays these); + // both are housekeeping — failures warn, never fail a clean shutdown. + if (!this.isReadOnly && this.generationStore) { + try { + await this.generationStore.repackHistory({ timeBudgetMs: 5_000 }) + } catch (error) { + console.warn( + `History repacking failed (non-fatal): ${error instanceof Error ? error.message : String(error)}` + ) + } + } await this.autoCompactHistory() // Phase 1: Flush ALL components in parallel to persist buffered data - // This is critical when cortex native providers buffer data in Rust memory + // This is critical when cor native providers buffer data in Rust memory await Promise.all([ // Flush HNSW dirty nodes (deferred persistence mode) (async () => { @@ -11786,6 +16289,13 @@ export class Brainy implements BrainyInterface { })() ]) + // Stamp the entity tree at the close boundary (counters + counter now + // durable from Phase 1/0a), so a cleanly-closed store reopens COHERENT + // instead of benign-behind. Best-effort, never blocks the close. + if (this.generationStore && !this.isReadOnly) { + await this.stampEntityTree() + } + // Phase 2: Close components to release resources (timers, file handles) // Data is already safe on disk from Phase 1 await Promise.all([ @@ -11851,10 +16361,26 @@ export class Brainy implements BrainyInterface { await this.storage.releaseWriterLock() } + // Shut down the VFS: stops its background maintenance interval and the + // PathResolver's — both are ref'd timers that would keep the process + // alive after the last brain closes (consumer-reported hang). + if (this._vfs) { + await this._vfs.close() + } + this.initialized = false // close() is terminal: block lazy re-initialization on any subsequent // operation (ensureInitialized() throws once this is set). this.closed = true + + // Drop this instance from the global registry, and when it was the last + // one, deregister the global shutdown hooks — their ref'd signal handles + // would otherwise keep the process alive after every brain is closed. + const instanceIndex = Brainy.instances.indexOf(this) + if (instanceIndex !== -1) { + Brainy.instances.splice(instanceIndex, 1) + } + Brainy.deregisterShutdownHooksIfIdle() } } diff --git a/src/cli/catalog.ts b/src/cli/catalog.ts deleted file mode 100644 index 31ed324f..00000000 --- a/src/cli/catalog.ts +++ /dev/null @@ -1,440 +0,0 @@ -/** - * Augmentation Catalog for CLI - * - * Displays available augmentations catalog - * Local catalog with caching support - */ - -import chalk from 'chalk' -import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs' -import { join } from 'node:path' -import { homedir } from 'node:os' - -const CATALOG_API = process.env.BRAINY_CATALOG_URL || null -const CACHE_PATH = join(homedir(), '.brainy', 'catalog-cache.json') -const CACHE_TTL = 24 * 60 * 60 * 1000 // 24 hours - -interface Augmentation { - id: string - name: string - description: string - category: string - status: 'available' | 'coming_soon' | 'deprecated' - popular?: boolean - eta?: string -} - -interface Category { - id: string - name: string - icon: string - description: string -} - -interface Catalog { - version: string - categories: Category[] - augmentations: Augmentation[] -} - -/** - * Fetch catalog from API with caching - */ -export async function fetchCatalog(): Promise { - try { - // Check cache first - const cached = loadCache() - if (cached) return cached - - // If external catalog API is configured, try to fetch - if (CATALOG_API) { - const response = await fetch(`${CATALOG_API}/api/catalog/cli`) - if (!response.ok) throw new Error('API unavailable') - - const catalog = await response.json() - - // Save to cache - saveCache(catalog) - - return catalog - } - - // Fall back to local catalog - return getDefaultCatalog() - } catch (error) { - // Try loading from cache even if expired - const cached = loadCache(true) - if (cached) { - console.log(chalk.yellow('📡 Using cached catalog')) - return cached - } - - // Fall back to hardcoded catalog - return getDefaultCatalog() - } -} - -/** - * Display catalog in CLI - */ -export async function showCatalog(options: { - category?: string - search?: string - detailed?: boolean -}) { - const catalog = await fetchCatalog() - if (!catalog) { - console.log(chalk.red('❌ Could not load augmentation catalog')) - return - } - - console.log(chalk.cyan.bold('🧠 Brainy Augmentation Catalog')) - console.log(chalk.gray(`Version ${catalog.version}`)) - console.log('') - - // Filter augmentations - let augmentations = catalog.augmentations - - if (options.category) { - augmentations = augmentations.filter(a => a.category === options.category) - } - - if (options.search) { - const query = options.search.toLowerCase() - augmentations = augmentations.filter(a => - a.name.toLowerCase().includes(query) || - a.description.toLowerCase().includes(query) - ) - } - - // Group by category - const grouped = groupByCategory(augmentations, catalog.categories) - - // Display - for (const [category, augs] of Object.entries(grouped)) { - if (augs.length === 0) continue - - const cat = catalog.categories.find(c => c.id === category) - console.log(chalk.bold(`${cat?.icon || '📦'} ${cat?.name || category}`)) - - for (const aug of augs) { - const status = getStatusIcon(aug.status) - const popular = aug.popular ? chalk.yellow(' ⭐') : '' - const eta = aug.eta ? chalk.gray(` (${aug.eta})`) : '' - - console.log(` ${status} ${aug.name}${popular}${eta}`) - if (options.detailed) { - console.log(chalk.gray(` ${aug.description}`)) - } - } - console.log('') - } - - // Show summary - const available = augmentations.filter(a => a.status === 'available').length - const coming = augmentations.filter(a => a.status === 'coming_soon').length - - console.log(chalk.gray('─'.repeat(50))) - console.log(chalk.green(`✅ ${available} available`) + chalk.gray(` • `) + - chalk.yellow(`🔜 ${coming} coming soon`)) - console.log('') - console.log(chalk.dim('Configure augmentations with "brainy augment"')) - console.log(chalk.dim('Run "brainy augment info " for details')) -} - -/** - * Show detailed info about an augmentation - */ -export async function showAugmentationInfo(id: string) { - const catalog = await fetchCatalog() - if (!catalog) { - console.log(chalk.red('❌ Could not load augmentation catalog')) - return - } - - const aug = catalog.augmentations.find(a => a.id === id) - if (!aug) { - console.log(chalk.red(`❌ Augmentation not found: ${id}`)) - console.log('') - console.log('Available augmentations:') - catalog.augmentations.forEach(a => { - console.log(` • ${a.id}`) - }) - return - } - - // Fetch full details from API if available - try { - if (!CATALOG_API) throw new Error('No external catalog configured') - - const response = await fetch(`${CATALOG_API}/api/catalog/augmentation/${id}`) - const details = await response.json() - - console.log(chalk.cyan.bold(`📦 ${details.name}`)) - if (details.popular) console.log(chalk.yellow('⭐ Popular')) - console.log('') - - console.log(chalk.bold('Category:'), getCategoryName(details.category, catalog.categories)) - console.log(chalk.bold('Status:'), getStatusText(details.status)) - if (details.eta) console.log(chalk.bold('Expected:'), details.eta) - console.log('') - - console.log(chalk.bold('Description:')) - console.log(details.longDescription || details.description) - console.log('') - - if (details.features) { - console.log(chalk.bold('Features:')) - details.features.forEach((f: string) => console.log(` ✓ ${f}`)) - console.log('') - } - - if (details.example) { - console.log(chalk.bold('Example:')) - console.log(chalk.gray('─'.repeat(50))) - console.log(details.example.code) - console.log(chalk.gray('─'.repeat(50))) - console.log('') - } - - if (details.requirements?.config) { - console.log(chalk.bold('Required Configuration:')) - details.requirements.config.forEach((c: string) => console.log(` • ${c}`)) - console.log('') - } - - if (details.pricing) { - console.log(chalk.bold('Available in:')) - details.pricing.tiers.forEach((t: string) => console.log(` • ${t}`)) - console.log('') - } - - console.log(chalk.dim('To activate: brainy augment activate')) - } catch (error) { - // Show basic info if API fails - console.log(chalk.cyan.bold(`📦 ${aug.name}`)) - console.log(aug.description) - console.log('') - console.log(chalk.dim('Full details unavailable (no external catalog configured)')) - } -} - -/** - * Show user's available augmentations - */ -export async function showAvailable(licenseKey?: string) { - // Show local catalog as default - const catalog = await fetchCatalog() - if (!catalog) { - console.log(chalk.red('❌ Could not load augmentation catalog')) - return - } - - console.log(chalk.cyan.bold('🧠 Available Augmentations')) - console.log('') - - const available = catalog.augmentations.filter(a => a.status === 'available') - const grouped = groupByCategory(available, catalog.categories) - - for (const [category, augs] of Object.entries(grouped)) { - if (augs.length === 0) continue - - const cat = catalog.categories.find(c => c.id === category) - console.log(chalk.bold(`${cat?.icon || '📦'} ${cat?.name || category}`)) - augs.forEach(aug => { - console.log(` ✅ ${aug.name}`) - console.log(chalk.gray(` ${aug.description}`)) - }) - console.log('') - } - - console.log(chalk.green(`✅ ${available.length} augmentations available`)) - - // If external API is configured and license key provided, try to fetch personalized data - if (CATALOG_API && licenseKey) { - try { - const response = await fetch(`${CATALOG_API}/api/catalog/available`, { - headers: { 'x-license-key': licenseKey } - }) - - if (response.ok) { - const data = await response.json() - console.log(chalk.gray(`Plan: ${data.plan || 'Standard'}`)) - - if (data.operations) { - const used = data.operations.used || 0 - const limit = data.operations.limit - const percent = limit === 'unlimited' ? 0 : Math.round((used / limit) * 100) - - console.log(chalk.bold('Usage:')) - if (limit === 'unlimited') { - console.log(` Unlimited operations`) - } else { - console.log(` ${used.toLocaleString()} / ${limit.toLocaleString()} operations (${percent}%)`) - } - } - } - } catch (error) { - // Ignore external API errors - local catalog is sufficient - } - } -} - -// Helper functions - -function loadCache(ignoreExpiry = false): Catalog | null { - try { - if (!existsSync(CACHE_PATH)) return null - - const data = JSON.parse(readFileSync(CACHE_PATH, 'utf8')) - - if (!ignoreExpiry && Date.now() - data.timestamp > CACHE_TTL) { - return null - } - - return data.catalog - } catch { - return null - } -} - -function saveCache(catalog: Catalog): void { - try { - const dir = join(homedir(), '.brainy') - if (!existsSync(dir)) { - mkdirSync(dir, { recursive: true }) - } - - writeFileSync(CACHE_PATH, JSON.stringify({ - catalog, - timestamp: Date.now() - })) - } catch { - // Ignore cache save errors - } -} - -function groupByCategory(augmentations: Augmentation[], categories: Category[]) { - const grouped: Record = {} - - for (const aug of augmentations) { - if (!grouped[aug.category]) { - grouped[aug.category] = [] - } - grouped[aug.category].push(aug) - } - - // Sort by category order - const ordered: Record = {} - const categoryOrder = ['memory', 'coordination', 'enterprise', 'perception', 'dialog', 'activation', 'cognition', 'websocket'] - - for (const cat of categoryOrder) { - if (grouped[cat]) { - ordered[cat] = grouped[cat] - } - } - - return ordered -} - -function getStatusIcon(status: string): string { - switch (status) { - case 'available': return chalk.green('✅') - case 'coming_soon': return chalk.yellow('🔜') - case 'deprecated': return chalk.red('⚠️') - default: return '❓' - } -} - -function getStatusText(status: string): string { - switch (status) { - case 'available': return chalk.green('Available') - case 'coming_soon': return chalk.yellow('Coming Soon') - case 'deprecated': return chalk.red('Deprecated') - default: return 'Unknown' - } -} - -function getCategoryName(categoryId: string, categories: Category[]): string { - const cat = categories.find(c => c.id === categoryId) - return cat ? `${cat.icon} ${cat.name}` : categoryId -} - -function readLicenseFile(): string | null { - try { - const licensePath = join(homedir(), '.brainy', 'license') - if (existsSync(licensePath)) { - return readFileSync(licensePath, 'utf8').trim() - } - } catch (error) { - // License file read failed, return null - console.debug('Failed to read license file:', error) - } - return null -} - -function getDefaultCatalog(): Catalog { - // Local catalog with current features - return { - version: '1.5.0', - categories: [ - { id: 'core', name: 'Core Features', icon: '🧠', description: 'Essential brainy functionality' }, - { id: 'neural', name: 'Neural API', icon: '🔗', description: 'Semantic similarity and clustering' }, - { id: 'enterprise', name: 'Enterprise', icon: '🏢', description: 'Business integrations' }, - { id: 'storage', name: 'Storage', icon: '💾', description: 'Data persistence and caching' } - ], - augmentations: [ - { - id: 'vector-search', - name: 'Vector Search', - category: 'core', - description: 'High-performance semantic search with HNSW indexing', - status: 'available', - popular: true - }, - { - id: 'neural-similarity', - name: 'Neural Similarity API', - category: 'neural', - description: 'Advanced semantic similarity, clustering, and hierarchy detection', - status: 'available', - popular: true - }, - { - id: 'intelligent-verb-scoring', - name: 'Intelligent Verb Scoring', - category: 'neural', - description: 'Smart relationship scoring with taxonomy understanding', - status: 'available' - }, - { - id: 'connection-pooling', - name: 'Connection Pooling', - category: 'enterprise', - description: 'Efficient database connection management', - status: 'available' - }, - { - id: 'batch-processing', - name: 'Batch Processing', - category: 'enterprise', - description: 'High-throughput batch operations with deduplication', - status: 'available' - }, - { - id: 's3-storage', - name: 'S3 Compatible Storage', - category: 'storage', - description: 'Cloud storage with optimized batch operations', - status: 'available' - }, - { - id: 'opfs-storage', - name: 'OPFS Storage', - category: 'storage', - description: 'Browser-based persistent storage', - status: 'available' - } - ] - } -} \ No newline at end of file diff --git a/src/coreTypes.ts b/src/coreTypes.ts index 69133b1d..e0248d17 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -754,6 +754,39 @@ export interface Change { data?: HNSWNounWithMetadata | HNSWVerbWithMetadata } +/** + * @description A declared derived-index blob FAMILY (the + * registered-blob contract). A family names the set of on-disk blobs that a + * derived index needs AS A SET (e.g. the vector base = `main.dkann` + + * `main.slotmap` + `main.slotrev`): losing ANY member corrupts the index. Once a + * family is declared: + * - its members are UNDELETABLE through the storage layer — `deleteBinaryBlob` / + * `removeRawPrefix` refuse with a `ProtectedArtifactError`, so an in-process + * GC/sweeper cannot remove a load-bearing file (intentional retirement = + * `unregisterDerivedFamily` first); + * - a member missing on open is a loud `DerivedArtifactMissingError` → rebuild. + * `COLD ≠ DEAD`: a write-once segment or a recovery-critical archive is + * load-bearing even when it has not been touched in a long time. + */ +export interface DerivedFamilyDeclaration { + /** Stable family id, e.g. `'vector-base'` / `'metadata-sstables'`. */ + name: string + /** + * The logical blob keys that make up the family. When {@link namespace} is + * set, each entry is a PREFIX protecting every key beneath it (for growing + * sets like `seg-*`); otherwise each entry is an exact member key. + */ + members: string[] + /** When true, {@link members} are prefixes (protect all keys beneath each). */ + namespace?: boolean + /** + * Whether a missing family can be rebuilt from the canonical records (default + * `true`). `false` marks an irreplaceable family (a missing member is data + * loss, not a rebuild). + */ + rebuildable?: boolean +} + export interface StorageAdapter { init(): Promise @@ -815,7 +848,17 @@ export interface StorageAdapter { */ getNounsByNounType(nounType: string): Promise - deleteNoun(id: string): Promise + /** + * Delete a noun — FULL canonical removal (both legs + the entity container). + * + * @param id The entity id. + * @param priorMetadata OPTIONAL already-known metadata of the entity being + * removed (the caller's pre-delete read). The count decrement must never + * REQUIRE re-reading the record being removed: when the internal read + * returns `null` (replace race, or a ghost left by an earlier version) the + * decrement falls back to this record instead of being silently skipped. + */ + deleteNoun(id: string, priorMetadata?: NounMetadata | null): Promise /** * Save verb - Pure HNSW verb with core fields only @@ -872,7 +915,15 @@ export interface StorageAdapter { */ getVerbsByType(type: string): Promise - deleteVerb(id: string): Promise + /** + * Delete a verb — FULL canonical removal (both legs + the container). + * + * @param id The relationship id. + * @param priorMetadata OPTIONAL already-known metadata of the edge being + * removed (the caller's pre-delete read); keeps the count decrement honest + * when the internal read returns `null` (see `deleteNoun`). + */ + deleteVerb(id: string, priorMetadata?: VerbMetadata | null): Promise /** * Save metadata @@ -888,6 +939,16 @@ export interface StorageAdapter { */ getMetadata(id: string): Promise + /** + * Delete a system/metadata object previously written with {@link saveMetadata}. + * Routes by the same key analysis as save/get (system channel), so callers that + * persist their own keyed payloads — e.g. the LSM graph store reclaiming its + * compacted-away SSTables — can free the storage instead of orphaning it. + * Idempotent: a missing key is a no-op, not an error. + * @param id The same key passed to {@link saveMetadata}. + */ + deleteMetadata(id: string): Promise + /** * Get multiple metadata objects in batches (prevents socket exhaustion) * @param ids Array of IDs to get metadata for @@ -1039,6 +1100,104 @@ export interface StorageAdapter { */ getBinaryBlobPath(key: string): string | null + /** + * @description OPTIONAL (registered-blob contract). Declare a + * derived-index blob {@link DerivedFamilyDeclaration | family} whose members + * become UNDELETABLE through this adapter — a subsequent `deleteBinaryBlob` / + * `removeRawPrefix` that would remove a declared member throws a + * `ProtectedArtifactError`. Providers declare their families on create; the + * declaration is persisted so protection survives a reopen. Idempotent per + * `name` (re-declaring replaces). + * @param family - The family to protect. + */ + registerDerivedFamily?(family: DerivedFamilyDeclaration): Promise + + /** + * @description OPTIONAL. Remove a family's protection so its members can be + * deleted again — the explicit, auditable step for intentional retirement of a + * derived index (the ONLY way a declared member becomes deletable). + * @param name - The {@link DerivedFamilyDeclaration.name} to unregister. + */ + unregisterDerivedFamily?(name: string): Promise + + /** + * @description OPTIONAL. List the currently-declared derived-index families — + * the source of truth for what `clear()` must wipe and what a + * missing-on-open check verifies. + */ + listDerivedFamilies?(): Promise + + /** + * @description OPTIONAL binary raw-byte primitives — the substrate for + * append-only log-structured files (the generation fact log's CRC-framed + * segments). Feature-detected: an adapter that omits them simply hosts no + * fact log (readers fall back to canonical enumeration). Paths are + * storage-root-relative and used VERBATIM (no `.gz`/`.bin` suffixing — + * unlike the JSON object and blob primitives). + * + * Append to a raw binary file, creating it (and parent directories) when + * absent. Append durability is the CALLER's job via `syncRawObjects` — + * matching the commit protocol, which batches fsyncs at its barrier. + */ + appendRawBytes?(path: string, bytes: Uint8Array): Promise + + /** + * Read a raw binary file whole. Absent → `null`; a real IO fault throws + * (never masked as absence). + */ + readRawBytes?(path: string): Promise + + /** + * Replace a raw binary file atomically (write-new → fsync → rename) — the + * reconcile primitive (e.g. truncating a fact-log tail back to committed + * truth after a crash). + */ + writeRawBytes?(path: string, bytes: Uint8Array): Promise + + /** + * Byte size of a raw binary file, or `null` when absent. + */ + rawByteSize?(path: string): Promise + + /** + * @description OPTIONAL fact-scan capability — how an index provider that + * holds only `storage` reaches the generation fact log (the host brain + * wires it at init; providers must never construct their own fact-log + * reader — the log's open path is writer-side). Present ⟺ this store hosts + * a fact log AND the host wired the capability. Returns a scan handle over + * committed facts (heal-grade telemetry included), or `null` when no fact + * log exists — callers fall back to the canonical enumeration walk. + */ + scanFacts?(options?: { + fromGeneration?: number + toGeneration?: number + kinds?: Array<'noun' | 'verb'> + batchSize?: number + }): import('./db/factLog.js').FactScanHandle | null + + /** + * @description OPTIONAL (rides the fact-scan capability): the fact log's + * head generation — the replay target for `stamp.sourceGeneration + 1` + * catch-ups. `null` when no fact log exists. + */ + factLogHeadGeneration?(): number | null + + /** + * @description OPTIONAL (rides the fact-scan capability): the COMMITTED + * generation watermark — the manifest truth a projection's + * `sourceGeneration` compares against. Exposed as a capability so a + * provider never parses the store's private manifest format. `null` when + * the capability is unwired. + */ + committedGeneration?(): number | null + + /** + * @description OPTIONAL (rides the fact-scan capability): the immutable, + * sealed fact-segment file paths covering `fromGeneration` — the zero-copy + * handoff. The mutable tail is excluded (read it via `scanFacts`). + */ + factSegmentPaths?(options?: { fromGeneration?: number }): string[] + /** * Save statistics data * @param statistics The statistics data to save @@ -1128,4 +1287,20 @@ export interface StorageAdapter { * @returns Promise that resolves to the total number of verbs */ getVerbCount(): Promise + + /** + * OPTIONAL — create a pre-upgrade backup of the whole store and return its + * location, or `null` when there is nothing to back up (empty store). On the + * filesystem adapter this is a zero-copy hard-link snapshot into a sibling + * directory; other backends may omit it. Callers feature-detect. Paired with + * {@link removeMigrationBackup}. Used by the 7.x → 8.0 migration lifecycle. + */ + createMigrationBackup?(): Promise + + /** + * OPTIONAL — remove a backup created by {@link createMigrationBackup}. + * Best-effort: a missing `location` is a no-op. Removing a hard-link snapshot + * never affects the live store (shared inodes; only the extra links are gone). + */ + removeMigrationBackup?(location: string): Promise } diff --git a/src/db/db.ts b/src/db/db.ts index ac6e4a2e..ac927fc5 100644 --- a/src/db/db.ts +++ b/src/db/db.ts @@ -144,6 +144,29 @@ export interface DbHost { * caller) and freed via the returned handle's `close()`. */ materializeAt(generation: number): Promise> + /** + * 8.0 #35 at-gen vector defer: true when the vector index is a + * {@link ../plugin.js VersionedIndexProvider} that advertised + * `isGenerationVisible(generation)` — i.e. it can serve the at-`generation` + * vector leg natively from retained segments, so a filtered semantic read needs + * NO O(n@G) materialization. False on the JS index (and whenever a native + * provider refuses the generation), so the caller falls back to `materializeAt`. + */ + canServeVectorAtGeneration(generation: number): boolean + /** + * Run the vector kNN for `params` (semantic or explicit-vector) AS OF + * `generation`, restricted to `allowedIds` (the at-gen metadata∩graph universe + * the caller resolved from the record layer). Returns `[id, distance]` pairs, + * descending relevance. Only reached when {@link DbHost.canServeVectorAtGeneration} + * is true — the provider serves the at-gen walk; Brainy composes the metadata + * half. `k` is the over-fetch (page + headroom). + */ + vectorSearchAtGeneration( + params: FindParams, + allowedIds: ReadonlySet, + k: number, + generation: number + ): Promise> /** Add one refcounted pin (store + versioned providers) on `generation`. */ pinGeneration(generation: number): void /** Release one refcounted pin (store + versioned providers) on `generation`. */ @@ -339,6 +362,12 @@ export class Db { if (this.overlay) { this.assertOverlayCompatibleFind(params) } else if (findRequiresIndexes(params)) { + // 8.0 #35: a FILTERED semantic/vector read at a historical generation can be + // served by a native at-gen vector provider (no O(n@G) materialization) — the + // metadata∩graph universe comes free from the record-overlay path, the vector + // leg from the provider. Falls through to materialization when not available. + const native = await this.tryAtGenerationVectorFind(params) + if (native !== null) return native const materialized = await this.materialize() return materialized.find(params) } @@ -346,9 +375,12 @@ export class Db { const limit = params.limit ?? 10 const offset = params.offset ?? 0 - // Ids whose live-index answer is NOT valid at this generation. + // Ids whose live-index answer is NOT valid at this generation. The upper + // bound is the full reserved watermark generation() — NOT committedGeneration() + // — so un-flushed single-op (Model-B) writes that changed an id after this + // pin are overlaid too (they participate in resolution via the pending tier). const changedNouns = historical - ? (await this.host.store.changedBetween(this.gen, this.host.store.committedGeneration())).nouns + ? (await this.host.store.changedBetween(this.gen, this.host.store.generation())).nouns : [] const overlayNouns = this.overlay?.nouns ?? new Map | null>() const excluded = new Set([...changedNouns, ...overlayNouns.keys()]) @@ -371,12 +403,12 @@ export class Db { entity = await this.host.entityFromRecord( id, { metadata: resolved.metadata, vector: resolved.vector }, - false + params.includeVectors ?? false ) } else if (resolved.source === 'current') { // Defensive: changed ids always resolve to a record or absent, but // a 'current' answer is still served correctly from the live path. - entity = await this.host.get(id) + entity = await this.host.get(id, { includeVectors: params.includeVectors ?? false }) } if (entity && entityMatchesFind(entity as Entity, params as FindParams)) { merged.push(resultFromEntity(entity)) @@ -413,6 +445,70 @@ export class Db { } } + /** + * @description 8.0 #35 — try to serve a FILTERED semantic/vector read at this + * historical generation via the native at-gen vector provider, with no O(n@G) + * materialization. Eligible only when: the query needs the vector leg + * (`query`/`vector`) WITH a metadata filter (the `allowedIds` source) and NO + * other index dimension (`near`/`connected`/`cursor`/`aggregate`/ + * `includeRelations`), AND the host's vector index can serve this generation + * ({@link DbHost.canServeVectorAtGeneration}). The at-gen metadata∩graph universe + * is resolved through this view's OWN record-overlay path (the metadata-only + * `find`, which costs no materialization); the provider then ranks the vector + * leg restricted to that universe, and rows are hydrated from the at-gen universe + * entities (so metadata reflects `generation`, not now). Returns `null` when + * ineligible — the caller falls back to materialization. + */ + private async tryAtGenerationVectorFind(params: FindParams): Promise[] | null> { + const wantsVector = params.query !== undefined || params.vector !== undefined + const hasOtherIndexDim = + params.near !== undefined || + params.connected !== undefined || + params.cursor !== undefined || + params.aggregate !== undefined || + params.includeRelations === true + const hasMetadataFilter = Boolean( + params.where || params.type || params.subtype || params.service || params.excludeVFS + ) + if (!wantsVector || hasOtherIndexDim || !hasMetadataFilter) return null + if (!this.host.canServeVectorAtGeneration(this.gen)) return null + + // At-gen metadata∩graph universe via the record-overlay path (no materialization): + // the same query with the vector legs stripped resolves through the metadata + // historical branch. Capped so a pathological filter can't materialize an + // unbounded universe; the provider walk is restricted to whatever the cap yields. + const universeParams: FindParams = { ...params } + delete universeParams.query + delete universeParams.vector + delete universeParams.searchMode + delete universeParams.orderBy + delete universeParams.order + universeParams.limit = AT_GEN_VECTOR_UNIVERSE_CAP + universeParams.offset = 0 + const universe = await this.find(universeParams) + if (universe.length === 0) return [] + + const limit = params.limit ?? 10 + const offset = params.offset ?? 0 + const allowedIds = new Set(universe.map((r) => r.id)) + const scored = await this.host.vectorSearchAtGeneration( + params, + allowedIds, + (offset + limit) * 2, + this.gen + ) + + // Compose: each at-gen metadata entity (from the universe) ranked by the + // provider's at-gen vector distance. + const byId = new Map(universe.map((r) => [r.id, r])) + const ranked: Result[] = [] + for (const [id, distance] of scored) { + const row = byId.get(id) + if (row) ranked.push({ ...row, score: Math.max(0, Math.min(1, 1 / (1 + distance))) }) + } + return ranked.slice(offset, offset + limit) + } + /** * @description Serialize part or all of this database value into a portable * `PortableGraph` document, read **as of this view's generation**. Because `export` @@ -460,7 +556,8 @@ export class Db { const params: RelatedParams = { ...rawParams, ...(rawParams.from !== undefined && { from: resolveEntityId(rawParams.from) }), - ...(rawParams.to !== undefined && { to: resolveEntityId(rawParams.to) }) + ...(rawParams.to !== undefined && { to: resolveEntityId(rawParams.to) }), + ...(rawParams.node !== undefined && { node: resolveEntityId(rawParams.node) }) } const historical = this.isHistorical() @@ -479,8 +576,11 @@ export class Db { const limit = params.limit ?? 100 const offset = params.offset ?? 0 + // Upper bound is the full reserved watermark generation() (see find()): an + // un-flushed single-op write that touched a relationship after this pin is + // overlaid too, via the pending tier. const changedVerbs = historical - ? (await this.host.store.changedBetween(this.gen, this.host.store.committedGeneration())).verbs + ? (await this.host.store.changedBetween(this.gen, this.host.store.generation())).verbs : [] const overlayVerbs = this.overlay?.verbs ?? new Map | null>() const excluded = new Set([...changedVerbs, ...overlayVerbs.keys()]) @@ -845,6 +945,13 @@ export class Db { * {@link SpeculativeOverlayError} (commit them with `brain.transact()` * first). * + * SPARSE FILES: a native accelerator's mmap index files can be sparse — + * huge apparent size, small allocated size. `persist()` handles them + * correctly (hard links share the allocation). But if you then archive the + * snapshot with EXTERNAL tools, use the sparse-aware flags (`tar czSf`, + * `rsync --sparse`, `cp --sparse=always`) or the copy materializes every + * hole — see docs/guides/external-backups-and-sparse-storage.md. + * * @param path - Absolute directory for the snapshot (created; must be * empty or absent). * @throws GenerationConflictError when this view is no longer the latest @@ -962,9 +1069,10 @@ function indexOnlyFindDimensions(params: FindParams): Array<[string, boolean]> { ['aggregation', params.aggregate !== undefined], ['relation expansion (includeRelations)', params.includeRelations === true], [ - `search mode '${params.mode ?? params.searchMode}'`, - (params.mode !== undefined && params.mode !== 'metadata' && params.mode !== 'auto') || - (params.searchMode !== undefined && params.searchMode !== 'auto') + `search mode '${params.searchMode}'`, + params.searchMode !== undefined && + params.searchMode !== 'auto' && + params.searchMode !== 'metadata' ] ] } @@ -979,6 +1087,14 @@ function findRequiresIndexes(params: FindParams): boolean { return indexOnlyFindDimensions(params).some(([, present]) => present) } +/** + * Cap on the at-gen metadata universe materialized for a native at-gen vector find + * (#35) — bounds a pathological filter; the provider's vector walk is restricted to + * whatever the cap yields. Only the metadata (no vectors) is held, so this is far + * cheaper than the full-corpus JS-HNSW rebuild it replaces. + */ +const AT_GEN_VECTOR_UNIVERSE_CAP = 100_000 + /** * @description Build a `Result` row from a record/overlay-resolved entity, * mirroring the live metadata-only find path (flattened fields, score `1.0`). @@ -1024,6 +1140,10 @@ function sortResultsBy(results: Result[], orderBy: string, order: 'asc' | * `to` → targetId, type/subtype set membership, service equality). */ function relationMatchesParams(relation: Relation, params: RelatedParams): boolean { + // `node` matches an edge incident in EITHER direction (the both-direction shorthand). + if (params.node !== undefined && relation.from !== params.node && relation.to !== params.node) { + return false + } if (params.from !== undefined && relation.from !== params.from) return false if (params.to !== undefined && relation.to !== params.to) return false if (params.type !== undefined) { diff --git a/src/db/errors.ts b/src/db/errors.ts index e5f70add..22f405be 100644 --- a/src/db/errors.ts +++ b/src/db/errors.ts @@ -159,3 +159,133 @@ export class GenerationCompactedError extends Error { this.horizon = horizon } } + +/** One entity/relationship left in an unreconciled state by a failed rollback. */ +export interface UnreconciledRecord { + /** The entity or relationship id. */ + id: string + /** `'noun'` (entity) or `'verb'` (relationship). */ + kind: 'noun' | 'verb' + /** + * `'orphan'` — the record is durably PRESENT but the transaction aborted + * (an add whose delete-undo failed); `'loss'` — the record is durably GONE + * or wrong when it should have been restored (a remove/update whose + * restore-undo failed — actual data loss). + */ + disposition: 'orphan' | 'loss' +} + +/** + * @description Thrown when a transaction's rollback could not be fully applied + * and the resulting inconsistency cannot be safely adopted forward — i.e. a + * multi-operation batch, or ANY case where a record was lost (a remove/update + * whose restore-undo failed). The store is left in a known-inconsistent state: + * the {@link records} name every entity/relationship whose canonical state no + * longer matches what the aborted transaction should have produced. + * + * On throw, the brain enters **write-quarantine** — reads continue to work, but + * further writes are refused until {@link } `repairIndex()` reconciles the + * derived indexes against canonical storage and lifts the quarantine. This is + * the honest, loud failure: a visible inconsistency the operator must repair, + * never a silent partial write. The generation counter is NOT advanced. + * + * (A SINGLE-op add whose only damage is a durably-present orphan is NOT this + * error: it is adopted forward as a committed generation and returned as a + * degraded-but-successful write — the record the caller asked for exists.) + * + * @example + * try { + * await brain.transact(ops) + * } catch (err) { + * if (err instanceof StoreInconsistentError) { + * console.error('store inconsistent:', err.records) // ids + dispositions + * await brain.repairIndex() // reconcile + lift the write-quarantine + * } + * } + */ +export class StoreInconsistentError extends Error { + /** Every record left in an unreconciled state by the failed rollback. */ + public readonly records: UnreconciledRecord[] + /** The original error that triggered the (then-failed) rollback. */ + public override readonly cause: Error + + /** + * @param records - The unreconciled entities/relationships (ids + disposition). + * @param cause - The error that triggered the rollback. + */ + constructor(records: UnreconciledRecord[], cause: Error) { + const orphans = records.filter((r) => r.disposition === 'orphan').length + const losses = records.filter((r) => r.disposition === 'loss').length + super( + `Store left inconsistent by a failed transaction rollback: ` + + `${records.length} record(s) could not be reconciled ` + + `(${orphans} orphaned, ${losses} lost). ` + + `The brain is now WRITE-QUARANTINED (reads still work) — run repairIndex() ` + + `to reconcile the derived indexes against canonical storage and lift the ` + + `quarantine. Ids: ${records.map((r) => `${r.id}:${r.disposition}`).join(', ')}. ` + + `Cause: ${cause.message}` + ) + this.name = 'StoreInconsistentError' + this.records = records + this.cause = cause + } +} + +/** + * @description Thrown by a write when the store cannot make single-op + * generation **history** durable: the asynchronous group-commit flush that + * persists buffered before-images to disk has failed repeatedly (a real + * storage fault — a full disk, an EIO, a permission loss — not a transient + * blip). Rather than keep accepting writes whose history silently piles up in + * memory and is never persisted — an invisible durability loss, and an + * unbounded leak — the store LATCHES this failure and refuses further writes + * until the pending tier drains. + * + * This is the loud, honest response to a broken history-durability path: + * - Live canonical data is unaffected (single-op live bytes are written before + * the history flush; only the immutable before-image history is stuck). + * - The background flush keeps retrying with backoff; when the underlying fault + * clears and a flush succeeds, the latch lifts and writes resume + * automatically. An explicit `flush()` also clears it on success. + * - {@link cause} is the underlying storage error from the last failed flush. + * + * A caller seeing this should treat it exactly like a full disk: stop writing, + * resolve the storage fault, and retry. It is NOT a data-corruption error — no + * committed generation is lost — it is a refusal to *promise* durability the + * store currently cannot deliver. + * + * @example + * try { + * await brain.add({ ... }) + * } catch (err) { + * if (err instanceof PendingFlushDurabilityError) { + * // History can't be persisted right now (disk fault). Resolve storage, + * // then retry — the store self-heals once a flush succeeds. + * console.error('history durability stalled:', err.cause) + * } + * } + */ +export class PendingFlushDurabilityError extends Error { + /** The underlying storage error from the most recent failed flush. */ + public override readonly cause: Error + /** How many consecutive flush attempts had failed when the latch tripped. */ + public readonly failedAttempts: number + + /** + * @param cause - The storage error from the last failed pending-tier flush. + * @param failedAttempts - Consecutive failed flush attempts at latch time. + */ + constructor(cause: Error, failedAttempts: number) { + super( + `Write refused: single-op generation history could not be made durable ` + + `after ${failedAttempts} consecutive flush attempts (${cause.message}). ` + + `Live data is intact, but buffered history is not yet persisted — the ` + + `store refuses further writes rather than silently accumulate ` + + `un-durable history. Resolve the underlying storage fault; the store ` + + `self-heals and resumes writes once a flush succeeds. Cause: ${cause.message}` + ) + this.name = 'PendingFlushDurabilityError' + this.cause = cause + this.failedAttempts = failedAttempts + } +} diff --git a/src/db/factLog.ts b/src/db/factLog.ts new file mode 100644 index 00000000..94e79700 --- /dev/null +++ b/src/db/factLog.ts @@ -0,0 +1,721 @@ +/** + * @module db/factLog + * @description The generation FACT LOG — an append-only, CRC-framed record of + * every committed generation as an AFTER-IMAGE "fact": what each touched + * entity/relationship BECAME (or a body-less tombstone when it was removed). + * This is the dual-write half of the log-canonical transition: today the + * before-image history + canonical tree remain authoritative; the fact log is + * appended at the same commit points and reconciled to committed truth at + * open, so consumers (index heals, replays, scans) can read one sequential, + * self-verifying stream instead of walking the entity tree. + * + * ## Wire format (frozen; additive-only within a major) + * + * Fact (msgpack, POSITIONAL array — the segment header's formatVersion + * governs the schema): + * + * fact := [ generation:u64, timestamp:u64, ops, meta|nil, blobHashes|nil ] + * op := [ kind:u8 (0=noun, 1=verb), id:bin16 (raw uuid bytes), + * record:[metaLeg, vecLeg] | nil ] // nil = TOMBSTONE + * + * Segment file (`_generations/facts/seg-.bfl`): + * + * header := magic "BFACTS\0\0" (8B) | formatVersion:u32 LE | + * firstGeneration:u64 LE | reserved 12B (ZEROED, verified) + * frame := length:u32 LE | crc32c:u32 LE (of payload) | payload + * + * A fact is never split across segments; a torn tail (length overrun or CRC + * mismatch) terminates that segment's scan — everything before it is intact. + * Zero-padded names make lexicographic order == generation order. + * + * ## Invariant + * + * After {@link FactLog.open}, the log contains EXACTLY the committed prefix: + * facts are appended BEFORE the commit point (inside the same durability + * window), so a crash can only leave the log AHEAD of committed truth — open + * truncates any fact beyond the committed generation. Absent generation = + * never committed; present = committed. A scan can never see an uncommitted + * fact. + * + * The manifest (`_generations/facts/manifest.json`, JSON — forensics stay + * terminal-readable) is the single source of truth for the segment SET; + * rotation flips it atomically (write-new → fsync → rename) BEFORE the new + * tail's first byte exists, so no segment file is ever unaccounted for. + */ +import { encode as defaultEncode, decode as defaultDecode } from '@msgpack/msgpack' +import { crc32c } from '../utils/crc32c.js' +import { prodLog } from '../utils/logger.js' + +// Swappable msgpack implementation — defaults to the JS codec; a native +// provider (registered via the plugin registry's 'msgpack' key) may replace +// it. Byte-compatibility is the contract (positional arrays, bin16 ids). +let msgpackEncode: (value: unknown) => Uint8Array = defaultEncode +let msgpackDecode: (bytes: Uint8Array) => unknown = defaultDecode + +/** Replace the msgpack encode/decode implementation at runtime. */ +export function setFactCodec(impl: { + encode: (value: unknown) => Uint8Array + decode: (bytes: Uint8Array) => unknown +}): void { + msgpackEncode = impl.encode + msgpackDecode = impl.decode +} + +/** Storage-root-relative home of the fact log. */ +export const FACTS_PREFIX = '_generations/facts' +/** The facts manifest path (JSON). */ +export const FACTS_MANIFEST_PATH = `${FACTS_PREFIX}/manifest.json` +/** Current segment format version (header field; additive-only within a major). */ +export const FACTS_FORMAT_VERSION = 1 +/** Rotation threshold: seal the tail segment once it exceeds this many bytes. */ +const SEGMENT_ROTATE_BYTES = 8 * 1024 * 1024 +/** Segment header: magic(8) + formatVersion(4) + firstGeneration(8) + reserved(12). */ +const HEADER_BYTES = 32 +const MAGIC = new Uint8Array([0x42, 0x46, 0x41, 0x43, 0x54, 0x53, 0x00, 0x00]) // "BFACTS\0\0" +/** Frame prefix: length(4) + crc32c(4). */ +const FRAME_PREFIX_BYTES = 8 + +/** One write inside a fact: what the id became (or a tombstone). */ +export interface FactOp { + kind: 'noun' | 'verb' + id: string + /** The AFTER-IMAGE legs, or `null` for a tombstone (the id was removed). */ + record: { metadata: unknown | null; vector: unknown | null } | null +} + +/** One committed generation, as scanned back out of the log. */ +export interface CommitFact { + generation: number + timestamp: number + ops: FactOp[] + meta?: Record + blobHashes?: string[] +} + +/** The telemetry a scan batch carries (frozen shape). */ +export interface FactScanBatch { + facts: CommitFact[] + firstGeneration: number + lastGeneration: number + factCount: number + byteSize: number + segmentId: string +} + +/** + * Liveness bound on a scan's FIRST batch (Stage-2 co-freeze, D1 contract): + * `batches()` must yield its first batch — or fail loudly — within this many + * ms of the first pull. A backlogged or damaged store may be SLOW, but it may + * never be SILENT: a consumer awaiting the first batch is otherwise + * indistinguishable from a wedge (the exact failure shape a production heal + * hit against a generations-backlogged brain). + */ +export const SCANFACTS_FIRST_BATCH_MS = 10_000 + +/** The telemetry a scan OPEN returns (frozen shape). */ +export interface FactScanHandle { + headGeneration: number + segmentCount: number + approxFactCount: number + /** + * Ordered batches; a detected gap aborts LOUDLY, never a silent skip. + * Liveness contract: the FIRST batch resolves or rejects within + * {@link SCANFACTS_FIRST_BATCH_MS} of the first pull — never a silent hang. + */ + batches: () => AsyncGenerator + /** Close telemetry — the invariant cross-check, valid after iteration ends. */ + summary: () => { factsYielded: number; segmentsRead: number } +} + +/** Manifest entry for a sealed segment. */ +interface SegmentEntry { + file: string + firstGeneration: number + lastGeneration: number + facts: number + bytes: number +} + +/** The facts manifest (JSON on disk). */ +interface FactsManifest { + formatVersion: number + segments: SegmentEntry[] + /** The append target. Its true content is established by scanning (crash tolerance). */ + tailSegment: string | null + updatedAt: string +} + +/** The narrow byte-level storage surface the fact log rides. */ +export interface FactLogStorage { + appendRawBytes(path: string, bytes: Uint8Array): Promise + readRawBytes(path: string): Promise + writeRawBytes(path: string, bytes: Uint8Array): Promise + rawByteSize(path: string): Promise + readRawObject(path: string): Promise + writeRawObject(path: string, data: any): Promise + syncRawObjects(paths: string[]): Promise + deleteRawObject(path: string): Promise +} + +/** True when the storage adapter exposes every primitive the fact log needs. */ +export function storageSupportsFactLog(storage: unknown): storage is FactLogStorage { + const s = storage as Record + return ( + typeof s.appendRawBytes === 'function' && + typeof s.readRawBytes === 'function' && + typeof s.writeRawBytes === 'function' && + typeof s.rawByteSize === 'function' + ) +} + +/** uuid string → 16 raw bytes (bin16 on the wire). */ +function uuidToBytes(id: string): Uint8Array { + const hex = id.replace(/-/g, '') + if (hex.length !== 32) { + // Non-uuid ids (legacy/natural keys) ride as UTF-8 with a length prefix + // marker impossible for uuids: we refuse instead — the write API has + // guaranteed uuid ids since 8.0, so anything else is a corruption signal. + throw new Error(`fact log: id is not a uuid: ${id}`) + } + const bytes = new Uint8Array(16) + for (let i = 0; i < 16; i++) { + bytes[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16) + } + return bytes +} + +/** 16 raw bytes → canonical lowercase uuid string. */ +function bytesToUuid(bytes: Uint8Array): string { + let hex = '' + for (let i = 0; i < 16; i++) hex += bytes[i].toString(16).padStart(2, '0') + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}` +} + +/** Zero-padded segment filename: lexicographic order == generation order. */ +function segmentFileName(firstGeneration: number): string { + return `seg-${String(firstGeneration).padStart(20, '0')}.bfl` +} + +/** Build a segment header. Reserved bytes are ZEROED (and verified on open). */ +function buildHeader(firstGeneration: number): Uint8Array { + const header = new Uint8Array(HEADER_BYTES) + header.set(MAGIC, 0) + const view = new DataView(header.buffer) + view.setUint32(8, FACTS_FORMAT_VERSION, true) + view.setBigUint64(12, BigInt(firstGeneration), true) + // bytes 20..31 stay zero (reserved) + return header +} + +/** Encode one fact into a framed record (length + crc32c + msgpack payload). */ +function encodeFrame(fact: CommitFact): Uint8Array { + const payload = msgpackEncode([ + fact.generation, + fact.timestamp, + fact.ops.map((op) => [ + op.kind === 'noun' ? 0 : 1, + uuidToBytes(op.id), + op.record === null ? null : [op.record.metadata, op.record.vector] + ]), + fact.meta ?? null, + fact.blobHashes && fact.blobHashes.length > 0 ? fact.blobHashes : null + ]) + const frame = new Uint8Array(FRAME_PREFIX_BYTES + payload.length) + const view = new DataView(frame.buffer) + view.setUint32(0, payload.length, true) + view.setUint32(4, crc32c(payload), true) + frame.set(payload, FRAME_PREFIX_BYTES) + return frame +} + +/** Decode one msgpack payload back into a CommitFact. */ +function decodeFact(payload: Uint8Array): CommitFact { + const raw = msgpackDecode(payload) as unknown[] + const [generation, timestamp, ops, meta, blobHashes] = raw as [ + number, + number, + Array<[number, Uint8Array, [unknown, unknown] | null]>, + Record | null, + string[] | null + ] + return { + generation: Number(generation), + timestamp: Number(timestamp), + ops: ops.map(([kind, idBytes, record]) => ({ + kind: kind === 0 ? ('noun' as const) : ('verb' as const), + id: bytesToUuid(idBytes), + record: record === null ? null : { metadata: record[0] ?? null, vector: record[1] ?? null } + })), + ...(meta ? { meta } : {}), + ...(blobHashes && blobHashes.length > 0 ? { blobHashes } : {}) + } +} + +/** + * Parse a segment's bytes: verify the header, then walk frames until the end + * or a torn tail (length overrun / CRC mismatch), which terminates the walk — + * everything before it is intact. Returns the decoded facts plus the byte + * length of the VALID prefix (header + intact frames), which reconciliation + * uses to cut a torn tail without re-encoding. + */ +function parseSegment( + file: string, + bytes: Uint8Array +): { facts: CommitFact[]; validBytes: number } { + if (bytes.length < HEADER_BYTES) { + prodLog.warn(`[FactLog] segment ${file} shorter than its header — treating as empty`) + return { facts: [], validBytes: 0 } + } + for (let i = 0; i < MAGIC.length; i++) { + if (bytes[i] !== MAGIC[i]) { + throw new Error(`fact log: segment ${file} has a bad magic — not a fact segment`) + } + } + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) + const version = view.getUint32(8, true) + if (version !== FACTS_FORMAT_VERSION) { + throw new Error( + `fact log: segment ${file} has formatVersion ${version}; this build reads ${FACTS_FORMAT_VERSION}` + ) + } + for (let i = 20; i < HEADER_BYTES; i++) { + if (bytes[i] !== 0) { + // Non-zero reserved bytes = a future format this build cannot verify. + throw new Error(`fact log: segment ${file} has non-zero reserved header bytes — unverifiable`) + } + } + + const facts: CommitFact[] = [] + let offset = HEADER_BYTES + while (offset + FRAME_PREFIX_BYTES <= bytes.length) { + const length = view.getUint32(offset, true) + const expectedCrc = view.getUint32(offset + 4, true) + const start = offset + FRAME_PREFIX_BYTES + const end = start + length + if (end > bytes.length) break // torn tail: frame length overruns the file + const payload = bytes.subarray(start, end) + if (crc32c(payload) !== expectedCrc) break // torn tail: payload CRC mismatch + facts.push(decodeFact(payload)) + offset = end + } + return { facts, validBytes: offset } +} + +/** + * The generation fact log. One instance per open store; every method assumes + * the single-writer discipline the generation store already enforces (calls + * arrive under its commit mutex). + */ +export class FactLog { + private readonly storage: FactLogStorage + /** Rotation threshold (bytes); tests may lower it to exercise rotation. */ + private readonly rotateBytes: number + private manifest: FactsManifest = { + formatVersion: FACTS_FORMAT_VERSION, + segments: [], + tailSegment: null, + updatedAt: new Date(0).toISOString() + } + /** Decoded facts of the TAIL segment (bounded by the rotation threshold). */ + private tailFacts: CommitFact[] = [] + /** Byte size of the tail segment file (valid prefix). */ + private tailBytes = 0 + /** Highest generation in the log (0 = empty). */ + private head = 0 + /** Segment paths appended since the last sync (the fsync batch). */ + private readonly dirtySegments = new Set() + + constructor(storage: FactLogStorage, options?: { rotateBytes?: number }) { + this.storage = storage + this.rotateBytes = options?.rotateBytes ?? SEGMENT_ROTATE_BYTES + } + + /** The highest committed generation the log holds (0 = empty). */ + headGeneration(): number { + return this.head + } + + /** + * Open the log and reconcile it to committed truth: read the manifest, + * establish the tail's intact content (torn-tail scan), then TRUNCATE any + * fact with `generation > committedGeneration` — those never committed (a + * crash between fact-append and the commit point). After open, the log is + * exactly the committed prefix. + */ + async open(committedGeneration: number): Promise { + const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null + if (stored && typeof stored === 'object' && Array.isArray(stored.segments)) { + if (stored.formatVersion !== FACTS_FORMAT_VERSION) { + throw new Error( + `fact log: manifest formatVersion ${stored.formatVersion}; this build reads ${FACTS_FORMAT_VERSION}` + ) + } + this.manifest = stored + } + + // Drop sealed segments that sit ENTIRELY beyond committed truth (a crash + // right after a rotation whose facts never committed), newest first. + while (this.manifest.segments.length > 0) { + const last = this.manifest.segments[this.manifest.segments.length - 1] + if (last.firstGeneration > committedGeneration) { + prodLog.warn( + `[FactLog] dropping sealed segment ${last.file} (generations ${last.firstGeneration}..` + + `${last.lastGeneration} never committed)` + ) + await this.storage.deleteRawObject(`${FACTS_PREFIX}/${last.file}`) + this.manifest.segments.pop() + await this.persistManifest() + } else if (last.lastGeneration > committedGeneration) { + // A sealed segment STRADDLING committed truth: cut it back. + await this.truncateSegmentTo(last.file, committedGeneration) + const cut = await this.reloadSegmentEntry(last.file) + this.manifest.segments[this.manifest.segments.length - 1] = cut + await this.persistManifest() + break + } else { + break + } + } + + // Establish the tail: scan its intact prefix, then truncate beyond + // committed truth (the common crash shape: buffered single-op facts whose + // counter never went durable). + if (this.manifest.tailSegment) { + const tailPath = `${FACTS_PREFIX}/${this.manifest.tailSegment}` + const bytes = await this.storage.readRawBytes(tailPath) + if (bytes === null) { + // Manifest named a tail whose first byte never landed — an empty tail. + this.tailFacts = [] + this.tailBytes = 0 + } else { + const { facts, validBytes } = parseSegment(this.manifest.tailSegment, bytes) + const kept = facts.filter((f) => f.generation <= committedGeneration) + if (kept.length !== facts.length || validBytes !== bytes.length) { + const dropped = facts.length - kept.length + if (dropped > 0) { + prodLog.warn( + `[FactLog] truncating ${dropped} uncommitted fact(s) beyond generation ` + + `${committedGeneration} from the tail (never committed)` + ) + } + await this.rewriteTail(kept) + } else { + this.tailFacts = facts + this.tailBytes = validBytes + } + } + } + + this.head = this.computeHead() + } + + /** + * Append one committed generation's fact. NOT durable until {@link sync} — + * the caller batches durability at its commit barrier (transact syncs in + * the same call; Model-B group-commit syncs at flush). + */ + async append(fact: CommitFact): Promise { + if (fact.generation <= this.head) { + throw new Error( + `fact log: non-monotonic append (generation ${fact.generation} ≤ head ${this.head})` + ) + } + if (this.manifest.tailSegment === null) { + await this.startTail(fact.generation) + } else if (this.tailBytes >= this.rotateBytes) { + await this.rotate(fact.generation) + } + const frame = encodeFrame(fact) + const tailPath = `${FACTS_PREFIX}/${this.manifest.tailSegment}` + await this.storage.appendRawBytes(tailPath, frame) + this.tailFacts.push(fact) + this.tailBytes += frame.length + this.head = fact.generation + this.dirtySegments.add(tailPath) + } + + /** Fsync every segment appended since the last sync. */ + async sync(): Promise { + if (this.dirtySegments.size === 0) return + const paths = [...this.dirtySegments] + this.dirtySegments.clear() + await this.storage.syncRawObjects(paths) + } + + /** + * Open a scan over committed facts. The scan runs against a MANIFEST + * SNAPSHOT (sealed segments + the tail's decoded facts at open) — exactly- + * once per fact, inclusive bounds, stable under concurrent appends. Gaps + * abort LOUDLY: a missing generation inside a segment's declared range is + * corruption, never silently skipped. + */ + scanFacts(options?: { + fromGeneration?: number + toGeneration?: number + kinds?: Array<'noun' | 'verb'> + batchSize?: number + /** Test override for the first-batch liveness bound (default {@link SCANFACTS_FIRST_BATCH_MS}). */ + firstBatchTimeoutMs?: number + }): FactScanHandle { + const from = options?.fromGeneration ?? 1 + const to = options?.toGeneration ?? this.head + const kinds = options?.kinds + const batchSize = Math.max(1, options?.batchSize ?? 256) + + // Snapshot: the segment list + tail content as of NOW. + const segments = this.manifest.segments.filter( + (s) => s.lastGeneration >= from && s.firstGeneration <= to + ) + const tailSnapshot = this.tailFacts.filter((f) => f.generation >= from && f.generation <= to) + const tailId = this.manifest.tailSegment ?? 'tail' + const approxFactCount = + segments.reduce((sum, s) => sum + s.facts, 0) + tailSnapshot.length + + let factsYielded = 0 + let segmentsRead = 0 + const storage = this.storage + + async function* batches(this: void): AsyncGenerator { + let expectedNext = 0 // gap detection: generations are monotonic, not necessarily dense + const emit = (facts: CommitFact[], segmentId: string, byteSize: number): FactScanBatch => ({ + facts, + firstGeneration: facts[0].generation, + lastGeneration: facts[facts.length - 1].generation, + factCount: facts.length, + byteSize, + segmentId + }) + const filterOps = (fact: CommitFact): CommitFact => + kinds + ? { ...fact, ops: fact.ops.filter((op) => kinds.includes(op.kind)) } + : fact + + for (const entry of segments) { + const bytes = await storage.readRawBytes(`${FACTS_PREFIX}/${entry.file}`) + if (bytes === null) { + throw new Error( + `fact log: sealed segment ${entry.file} is MISSING — the log is damaged; aborting scan` + ) + } + const { facts } = parseSegment(entry.file, bytes) + segmentsRead++ + const inRange = facts.filter((f) => f.generation >= from && f.generation <= to) + for (const f of inRange) { + if (f.generation <= expectedNext - 1) { + throw new Error(`fact log: out-of-order fact ${f.generation} in ${entry.file} — aborting scan`) + } + expectedNext = f.generation + 1 + } + for (let i = 0; i < inRange.length; i += batchSize) { + const slice = inRange.slice(i, i + batchSize).map(filterOps) + if (slice.length === 0) continue + factsYielded += slice.length + yield emit(slice, entry.file, slice.reduce((n, f) => n + encodeFrame(f).length, 0)) + } + } + + if (tailSnapshot.length > 0) { + segmentsRead++ + for (const f of tailSnapshot) { + if (f.generation <= expectedNext - 1) { + throw new Error(`fact log: out-of-order fact ${f.generation} in the tail — aborting scan`) + } + expectedNext = f.generation + 1 + } + for (let i = 0; i < tailSnapshot.length; i += batchSize) { + const slice = tailSnapshot.slice(i, i + batchSize).map(filterOps) + factsYielded += slice.length + yield emit(slice, tailId, slice.reduce((n, f) => n + encodeFrame(f).length, 0)) + } + } + } + + // Liveness wrapper: the FIRST pull races the contract deadline. Only the + // first — the bound is time-to-first-batch (proof the producer is alive), + // not per-batch pacing; and it runs only while a pull is actually pending, + // so consumer think-time between pulls never counts against the producer. + const firstBatchTimeoutMs = options?.firstBatchTimeoutMs ?? SCANFACTS_FIRST_BATCH_MS + async function* batchesWithLiveness(this: void): AsyncGenerator { + const inner = batches() + let timer: NodeJS.Timeout | undefined + try { + const deadline = new Promise((_, reject) => { + timer = setTimeout( + () => + reject( + new Error( + `fact log: scanFacts produced no first batch within ${firstBatchTimeoutMs}ms ` + + `(liveness contract) — the store is wedged or unreadably slow; aborting scan LOUDLY ` + + `instead of hanging the consumer.` + ) + ), + firstBatchTimeoutMs + ) + timer.unref?.() + }) + const first = await Promise.race([inner.next(), deadline]) + if (first.done) return + yield first.value + } finally { + clearTimeout(timer) + } + yield* inner + } + + return { + headGeneration: this.head, + segmentCount: segments.length + (tailSnapshot.length > 0 ? 1 : 0), + approxFactCount, + batches: batchesWithLiveness, + summary: () => ({ factsYielded, segmentsRead }) + } + } + + /** + * The mmap fast path (capability handoff): the immutable sealed segment + * files covering `fromGeneration`, in order. The TAIL is deliberately NOT + * included — it is append-mutable; consumers read it via {@link scanFacts}. + */ + segmentPaths(options?: { fromGeneration?: number }): string[] { + const from = options?.fromGeneration ?? 1 + return this.manifest.segments + .filter((s) => s.lastGeneration >= from) + .map((s) => `${FACTS_PREFIX}/${s.file}`) + } + + /** + * Drop every fact with `generation > keepThrough` — the in-session abort + * compensation: a transact appends its fact BEFORE the commit point, so a + * real (non-crash) abort after the append must take the fact back out. The + * dropped facts can only live in the TAIL (they were just appended); the + * rewrite is atomic and bounded by the rotation threshold. + */ + async dropAbove(keepThrough: number): Promise { + if (this.head <= keepThrough) return + const kept = this.tailFacts.filter((f) => f.generation <= keepThrough) + if (kept.length === this.tailFacts.length) { + throw new Error( + `fact log: dropAbove(${keepThrough}) found no droppable facts in the tail ` + + `(head ${this.head}) — the fact to drop was already sealed; the log needs reopen` + ) + } + await this.rewriteTail(kept) + this.head = this.computeHead() + } + + // -- internals ------------------------------------------------------------- + + private computeHead(): number { + if (this.tailFacts.length > 0) return this.tailFacts[this.tailFacts.length - 1].generation + const sealed = this.manifest.segments + if (sealed.length > 0) return sealed[sealed.length - 1].lastGeneration + return 0 + } + + /** Create the very first tail segment (manifest-first, then header bytes). */ + private async startTail(firstGeneration: number): Promise { + const file = segmentFileName(firstGeneration) + this.manifest.tailSegment = file + await this.persistManifest() + await this.storage.appendRawBytes(`${FACTS_PREFIX}/${file}`, buildHeader(firstGeneration)) + this.tailFacts = [] + this.tailBytes = HEADER_BYTES + } + + /** + * Seal the tail into the manifest and start a new one. Manifest-first: the + * flip both seals the old tail AND names the new one atomically, so no + * segment file ever exists unaccounted for. + */ + private async rotate(nextGeneration: number): Promise { + const sealedFile = this.manifest.tailSegment + if (!sealedFile) return + // Seal what the tail actually holds. + await this.sync() // sealed segments are always fully durable + const entry: SegmentEntry = { + file: sealedFile, + firstGeneration: this.tailFacts[0]?.generation ?? nextGeneration, + lastGeneration: this.tailFacts[this.tailFacts.length - 1]?.generation ?? nextGeneration - 1, + facts: this.tailFacts.length, + bytes: this.tailBytes + } + const newFile = segmentFileName(nextGeneration) + this.manifest.segments.push(entry) + this.manifest.tailSegment = newFile + await this.persistManifest() + await this.storage.appendRawBytes(`${FACTS_PREFIX}/${newFile}`, buildHeader(nextGeneration)) + this.tailFacts = [] + this.tailBytes = HEADER_BYTES + } + + /** Atomically persist the manifest (write-new → fsync → rename downstream). */ + private async persistManifest(): Promise { + this.manifest.updatedAt = new Date().toISOString() + await this.storage.writeRawObject(FACTS_MANIFEST_PATH, this.manifest) + await this.storage.syncRawObjects([FACTS_MANIFEST_PATH]) + } + + /** Rewrite the tail segment to hold exactly `facts` (atomic replace). */ + private async rewriteTail(facts: CommitFact[]): Promise { + const file = this.manifest.tailSegment + if (!file) return + const first = facts[0]?.generation ?? this.segmentFirstGenerationFromName(file) + const parts: Uint8Array[] = [buildHeader(first)] + for (const f of facts) parts.push(encodeFrame(f)) + const total = parts.reduce((n, p) => n + p.length, 0) + const merged = new Uint8Array(total) + let offset = 0 + for (const p of parts) { + merged.set(p, offset) + offset += p.length + } + await this.storage.writeRawBytes(`${FACTS_PREFIX}/${file}`, merged) + this.tailFacts = facts + this.tailBytes = total + } + + /** Cut a SEALED segment back to `committedGeneration` (atomic replace). */ + private async truncateSegmentTo(file: string, committedGeneration: number): Promise { + const path = `${FACTS_PREFIX}/${file}` + const bytes = await this.storage.readRawBytes(path) + if (bytes === null) return + const { facts } = parseSegment(file, bytes) + const kept = facts.filter((f) => f.generation <= committedGeneration) + prodLog.warn( + `[FactLog] truncating sealed segment ${file} to generation ${committedGeneration} ` + + `(${facts.length - kept.length} uncommitted fact(s) dropped)` + ) + const first = kept[0]?.generation ?? this.segmentFirstGenerationFromName(file) + const parts: Uint8Array[] = [buildHeader(first)] + for (const f of kept) parts.push(encodeFrame(f)) + const total = parts.reduce((n, p) => n + p.length, 0) + const merged = new Uint8Array(total) + let offset = 0 + for (const p of parts) { + merged.set(p, offset) + offset += p.length + } + await this.storage.writeRawBytes(path, merged) + } + + /** Re-derive a sealed segment's manifest entry from its actual bytes. */ + private async reloadSegmentEntry(file: string): Promise { + const bytes = await this.storage.readRawBytes(`${FACTS_PREFIX}/${file}`) + const { facts, validBytes } = bytes + ? parseSegment(file, bytes) + : { facts: [] as CommitFact[], validBytes: 0 } + return { + file, + firstGeneration: facts[0]?.generation ?? this.segmentFirstGenerationFromName(file), + lastGeneration: facts[facts.length - 1]?.generation ?? 0, + facts: facts.length, + bytes: validBytes + } + } + + /** Parse the zero-padded firstGeneration back out of a segment filename. */ + private segmentFirstGenerationFromName(file: string): number { + const match = /^seg-(\d{20})\.bfl$/.exec(file) + return match ? Number(match[1]) : 0 + } +} diff --git a/src/db/familyStamp.ts b/src/db/familyStamp.ts new file mode 100644 index 00000000..98342884 --- /dev/null +++ b/src/db/familyStamp.ts @@ -0,0 +1,152 @@ +/** + * @module db/familyStamp + * @description The generalized FAMILY STAMP — one JSON shape that declares, + * for any derived projection, WHICH source state it reflects and HOW to verify + * it is whole. The entity tree (canonical current-state files) carries the + * first brainy-side stamp; native index families carry the same shape. One + * verifier reads both member modes: + * + * - `enumerated` — bounded families: exact byte size per member file, + * verified at open. + * - `rollup` — unbounded families (the entity tree: millions of files): + * the verified surface is a small set of rollup invariants (entity/ + * relationship counts) plus `sourceGeneration`. + * + * `sourceGeneration` is the generation of the source-of-truth log this + * projection reflects — open-time coherence becomes a COMPARISON (stamp vs + * log head), not a walk: + * + * - equal + invariants hold → coherent, serve. + * - behind → the projection missed the tail (crash between commit and stamp); + * for the Stage-1 tree this is benign by construction (the tree is written + * BY the commit), so the stamp refreshes; a DERIVED projection would replay + * the gap instead. + * - invariants FAIL at equal generation → genuine incoherence: loud, and the + * repair ritual (`repairIndex()`, whose recount rebuilds the rollups from a + * canonical walk) heals it. + * + * Stamps are JSON on purpose — every incident gets debugged by reading a + * stamp in a terminal. + */ + +/** Storage-root-relative directory holding family stamps. */ +export const FAMILY_STAMPS_PREFIX = '_system/family-stamps' + +/** The entity tree's stamp path. */ +export const ENTITY_TREE_STAMP_PATH = `${FAMILY_STAMPS_PREFIX}/entity-tree.json` + +/** One enumerated member: a file and its exact expected byte size. */ +export interface EnumeratedMember { + path: string + bytes: number +} + +/** + * The stamp's verified surface, in one of the two member modes. Rollup + * invariant values may be numbers (counts, byte sizes) or strings (content + * fingerprints, e.g. a per-tree SHA-256) — the verifier compares by strict + * equality either way, so a type mismatch reads as incoherence, never a pass. + */ +export type StampMembers = + | { mode: 'enumerated'; files: EnumeratedMember[] } + | { mode: 'rollup'; invariants: Record } + +/** The generalized family stamp (one shape, one verifier, both engines). */ +export interface FamilyStamp { + /** Which projection this stamps (e.g. `'entity-tree'`). */ + family: string + /** Monotonic per-family stamp generation — bumps on every committed stamp. */ + generation: number + /** ISO timestamp of the stamp write. */ + committedAt: string + /** The source-of-truth generation this projection reflects. */ + sourceGeneration: number + /** The verified surface. */ + members: StampMembers +} + +/** The verdict of an open-time stamp verification. */ +export type StampVerdict = + | { state: 'coherent' } + | { state: 'absent' } // legacy store — first stamp writes at the next flush + | { state: 'behind'; stampSource: number; head: number } + | { state: 'incoherent'; failures: string[] } + | { state: 'unverifiable'; reason: string } // a FAULT reading the stamp — never conflated with absence + +/** The narrow storage surface stamps ride (JSON objects + fsync). */ +export interface StampStorage { + readRawObject(path: string): Promise + writeRawObject(path: string, data: any): Promise + syncRawObjects(paths: string[]): Promise +} + +/** Read a family's stamp; `null` when none was ever written. */ +export async function readFamilyStamp( + storage: StampStorage, + path: string +): Promise { + const stored = (await storage.readRawObject(path)) as FamilyStamp | null + if (!stored || typeof stored !== 'object' || typeof stored.family !== 'string') return null + return stored +} + +/** Write a family's stamp durably (atomic object write + fsync). */ +export async function writeFamilyStamp( + storage: StampStorage, + path: string, + stamp: Omit & { generation?: number } +): Promise { + const prior = await readFamilyStamp(storage, path) + const full: FamilyStamp = { + ...stamp, + generation: (prior?.generation ?? 0) + 1, + committedAt: new Date().toISOString() + } + await storage.writeRawObject(path, full) + await storage.syncRawObjects([path]) +} + +/** + * The ONE verifier, both member modes. `actual` supplies the observed rollup + * values (rollup mode) or file sizes (enumerated mode, keyed by path); + * `head` is the source-of-truth generation now. + */ +export function verifyFamilyStamp( + stamp: FamilyStamp | null, + head: number, + actual: Record +): StampVerdict { + if (stamp === null) return { state: 'absent' } + if (stamp.sourceGeneration > head) { + // A stamp AHEAD of the log claims state that never committed — the + // projection was stamped against truth that a crash rolled back. + return { + state: 'incoherent', + failures: [`sourceGeneration ${stamp.sourceGeneration} is ahead of the log head ${head}`] + } + } + if (stamp.sourceGeneration < head) { + return { state: 'behind', stampSource: stamp.sourceGeneration, head } + } + const failures: string[] = [] + if (stamp.members.mode === 'rollup') { + for (const [name, expected] of Object.entries(stamp.members.invariants)) { + const observed = actual[name] + if (observed === undefined) { + failures.push(`rollup invariant '${name}' has no observed value`) + } else if (observed !== expected) { + failures.push(`rollup invariant '${name}': stamped ${expected}, observed ${observed}`) + } + } + } else { + for (const member of stamp.members.files) { + const observed = actual[member.path] + if (observed === undefined) { + failures.push(`member '${member.path}' is missing`) + } else if (observed !== member.bytes) { + failures.push(`member '${member.path}': stamped ${member.bytes} bytes, observed ${observed}`) + } + } + } + return failures.length > 0 ? { state: 'incoherent', failures } : { state: 'coherent' } +} diff --git a/src/db/generationSegments.ts b/src/db/generationSegments.ts new file mode 100644 index 00000000..0c14b60c --- /dev/null +++ b/src/db/generationSegments.ts @@ -0,0 +1,459 @@ +/** + * @module db/generationSegments + * @description The generation-segment store — Stage-2 D1+D3+repacking's file + * format (co-frozen 2026-07-19; design: the d1-d3-repacking spec). + * + * Packs CONSECUTIVE cold generations' record-sets (before-images + delta) + * into append-once segment files with derived sidecar indexes, so history + * scales in SEGMENTS (tens) instead of FILES-PER-GENERATION (hundreds of + * thousands), and cold-open reads ONE manifest instead of listing the + * backlog. Layout under `_generations/segments/`: + * + * - `seg-.bgs` — magic "BGS1", then one frame per + * generation: `u32 payloadLen | u32 crc32c | msgpack payload`. Payload is + * POSITIONAL: `[generation, timestamp, delta, records[], flags]` with + * records `[kindByte, id, record]`. `flags` reserves encoding evolution + * (bit 0 = compressed payload — v1 always 0; a future writer upgrade, + * never a format break). Sealed segments are IMMUTABLE — the fact log's + * own law, generalized. + * - `seg-.idx` — DERIVED sidecar (msgpack): per-generation frame + * offsets (point reads = one ranged read, never a listing) + per-id + * generation postings (per-id chain rebuilds read only what they need). + * Corrupt/missing → rebuilt from its segment in one sequential read, + * loudly. + * - `manifest.json` — the segment catalogue + `compactedBelow` (D3's + * horizon marker). Cold-open reads THIS; the packed backlog is never + * listed. + * + * D3 semantics carried here: bounded-retention reclaim drops WHOLE segments + * at boundaries (O(1) per segment, no rewrite); under the archival profile + * (`retention: 'all'`) nothing here is ever dropped — folding is the only + * transform (re-representation, never deletion). + */ + +import { encode as msgpackEncode, decode as msgpackDecode } from '@msgpack/msgpack' +import { crc32c } from '../utils/crc32c.js' +import type { FactLogStorage } from './factLog.js' +import { prodLog } from '../utils/logger.js' + +/** Directory for segment files + manifest, under the generations prefix. */ +export const SEGMENTS_PREFIX = '_generations/segments' + +/** Target sealed-segment size (co-freeze proposal; tunable on evidence). */ +export const SEGMENT_TARGET_BYTES = 64 * 1024 * 1024 + +const MAGIC = new TextEncoder().encode('BGS1') +const FRAME_PREFIX_BYTES = 8 // u32 payloadLen + u32 crc32c +const MANIFEST_PATH = `${SEGMENTS_PREFIX}/manifest.json` + +/** One generation's fold input — exactly what the live tier holds for it. */ +export interface FoldGeneration { + generation: number + timestamp: number + /** The tx.json delta object, carried verbatim. */ + delta: unknown + /** The before-image record-set (empty for record-less generations). */ + records: Array<{ kind: 'noun' | 'verb'; id: string; record: unknown }> +} + +/** Manifest entry for one sealed segment. */ +export interface SegmentMeta { + file: string + firstGeneration: number + lastGeneration: number + frames: number + bytes: number + /** crc32c of the full segment byte stream — the digest chain's link. */ + checksum: number +} + +interface SegmentManifest { + version: 1 + compactedBelow: number + segments: SegmentMeta[] +} + +interface SidecarIndex { + version: 1 + /** [generation, frameOffset, frameLen] ascending by generation. */ + generations: Array<[number, number, number]> + /** `${kindByte}:${id}` → ascending generations holding a record for it. */ + ids: Record +} + +const segmentFileName = (firstGeneration: number): string => + `seg-${String(firstGeneration).padStart(20, '0')}.bgs` +const sidecarFileName = (firstGeneration: number): string => + `seg-${String(firstGeneration).padStart(20, '0')}.idx` + +/** + * The generation-segment store. Owns the packed tier ONLY — the live + * per-generation tier and the routing between tiers belong to + * `GenerationStore`. All mutating entry points here are called under the + * generation store's commit mutex. + */ +export class GenerationSegmentStore { + private readonly storage: FactLogStorage + private manifest: SegmentManifest = { version: 1, compactedBelow: 0, segments: [] } + /** Sidecar cache — segments are immutable, so entries never invalidate. */ + private readonly sidecars = new Map() + + constructor(storage: FactLogStorage) { + this.storage = storage + } + + /** Load the manifest (ONE read — never a directory listing). */ + async open(): Promise { + const raw = (await this.storage.readRawObject(MANIFEST_PATH)) as SegmentManifest | null + if (raw) { + if (raw.version !== 1) { + throw new Error( + `[GenerationSegments] manifest version ${String(raw.version)} is newer than this ` + + `engine understands — refusing to serve partial history. Upgrade the engine.` + ) + } + this.manifest = raw + } + } + + /** The packed tier's catalogue (ascending, immutable snapshot). */ + segments(): readonly SegmentMeta[] { + return this.manifest.segments + } + + /** D3's horizon marker: generations below this were reclaimed (bounded profiles only). */ + compactedBelow(): number { + return this.manifest.compactedBelow + } + + /** The covering sealed segment for `gen`, or null if it lives outside the packed tier. */ + private coveringSegment(gen: number): SegmentMeta | null { + // Manifest is ascending and ranges never overlap — binary search. + const segs = this.manifest.segments + let lo = 0 + let hi = segs.length - 1 + while (lo <= hi) { + const mid = (lo + hi) >> 1 + const s = segs[mid] + if (gen < s.firstGeneration) hi = mid - 1 + else if (gen > s.lastGeneration) lo = mid + 1 + else return s + } + return null + } + + /** True when `gen` is packed (readable from this tier). */ + hasGeneration(gen: number): boolean { + return this.coveringSegment(gen) !== null + } + + /** + * Fold consecutive generations into ONE new sealed segment + sidecar and + * append it to the manifest atomically. Caller guarantees: `gens` is + * ascending, contiguous with the packed tier (first = last packed + 1 when + * segments exist), and already durable in the live tier. Crash between the + * segment write and the caller's live-tier delete leaves a DUPLICATE + * representation — resolved live-tier-wins by the reader; never a gap. + */ + async fold(gens: FoldGeneration[]): Promise { + if (gens.length === 0) { + throw new Error('[GenerationSegments] fold() requires at least one generation') + } + for (let i = 1; i < gens.length; i++) { + if (gens[i].generation <= gens[i - 1].generation) { + throw new Error('[GenerationSegments] fold() input must be strictly ascending') + } + } + const last = this.manifest.segments[this.manifest.segments.length - 1] + if (last && gens[0].generation <= last.lastGeneration) { + throw new Error( + `[GenerationSegments] fold() overlaps the packed tier: ${gens[0].generation} ≤ ` + + `sealed ${last.lastGeneration} — segments are immutable, never rewritten` + ) + } + + const first = gens[0].generation + const file = segmentFileName(first) + const sidecar: SidecarIndex = { version: 1, generations: [], ids: {} } + + // Encode all frames, tracking offsets for the sidecar. + const parts: Uint8Array[] = [MAGIC] + let offset = MAGIC.length + for (const g of gens) { + const payload = msgpackEncode([ + g.generation, + g.timestamp, + g.delta, + g.records.map((r) => [r.kind === 'noun' ? 0 : 1, r.id, r.record]), + 0 // flags: v1 = uncompressed + ]) + const frame = new Uint8Array(FRAME_PREFIX_BYTES + payload.length) + const view = new DataView(frame.buffer) + view.setUint32(0, payload.length, true) + view.setUint32(4, crc32c(payload), true) + frame.set(payload, FRAME_PREFIX_BYTES) + sidecar.generations.push([g.generation, offset, frame.length]) + for (const r of g.records) { + const key = `${r.kind === 'noun' ? 0 : 1}:${r.id}` + ;(sidecar.ids[key] ??= []).push(g.generation) + } + parts.push(frame) + offset += frame.length + } + const total = parts.reduce((n, p) => n + p.length, 0) + const bytes = new Uint8Array(total) + let at = 0 + for (const p of parts) { + bytes.set(p, at) + at += p.length + } + + const meta: SegmentMeta = { + file, + firstGeneration: first, + lastGeneration: gens[gens.length - 1].generation, + frames: gens.length, + bytes: total, + checksum: crc32c(bytes) + } + + // Durability order: segment + sidecar fsync'd BEFORE the manifest names + // them (a crash before the manifest = invisible orphan files, harmless); + // manifest last, atomically. + const segPath = `${SEGMENTS_PREFIX}/${file}` + const idxPath = `${SEGMENTS_PREFIX}/${sidecarFileName(first)}` + await this.storage.writeRawBytes(segPath, bytes) + await this.storage.writeRawBytes(idxPath, msgpackEncode(sidecar)) + await this.storage.syncRawObjects([segPath, idxPath]) + const next: SegmentManifest = { + ...this.manifest, + segments: [...this.manifest.segments, meta] + } + await this.storage.writeRawObject(MANIFEST_PATH, next) + await this.storage.syncRawObjects([MANIFEST_PATH]) + this.manifest = next + this.sidecars.set(file, sidecar) + return meta + } + + /** Load (or rebuild, loudly) a segment's sidecar. */ + private async sidecarFor(meta: SegmentMeta): Promise { + const cached = this.sidecars.get(meta.file) + if (cached) return cached + const idxPath = `${SEGMENTS_PREFIX}/${sidecarFileName(meta.firstGeneration)}` + const raw = await this.storage.readRawBytes(idxPath) + if (raw) { + try { + const idx = msgpackDecode(raw) as SidecarIndex + if (idx.version === 1) { + this.sidecars.set(meta.file, idx) + return idx + } + } catch { + // fall through to rebuild + } + } + // Sidecars are DERIVED: rebuild from the segment, loudly — never serve + // wrong offsets silently. + prodLog.warn( + `[GenerationSegments] sidecar for ${meta.file} missing or unreadable — rebuilding from the segment` + ) + const rebuilt = await this.rebuildSidecar(meta) + await this.storage.writeRawBytes(idxPath, msgpackEncode(rebuilt)) + this.sidecars.set(meta.file, rebuilt) + return rebuilt + } + + /** One sequential read of the segment → a fresh sidecar. Verifies every frame CRC. */ + private async rebuildSidecar(meta: SegmentMeta): Promise { + const frames = await this.readAllFrames(meta) + const idx: SidecarIndex = { version: 1, generations: [], ids: {} } + for (const f of frames) { + idx.generations.push([f.generation, f.offset, f.frameLen]) + for (const r of f.records) { + const key = `${r.kind === 'noun' ? 0 : 1}:${r.id}` + ;(idx.ids[key] ??= []).push(f.generation) + } + } + return idx + } + + private decodeFrame( + payload: Uint8Array + ): { generation: number; timestamp: number; delta: unknown; records: FoldGeneration['records'] } { + const [generation, timestamp, delta, rawRecords] = msgpackDecode(payload) as [ + number, + number, + unknown, + Array<[number, string, unknown]>, + number + ] + return { + generation, + timestamp, + delta, + records: rawRecords.map(([kindByte, id, record]) => ({ + kind: kindByte === 0 ? ('noun' as const) : ('verb' as const), + id, + record + })) + } + } + + private async readAllFrames(meta: SegmentMeta): Promise< + Array & { offset: number; frameLen: number }> + > { + const bytes = await this.storage.readRawBytes(`${SEGMENTS_PREFIX}/${meta.file}`) + if (!bytes) { + throw new Error( + `[GenerationSegments] sealed segment ${meta.file} is MISSING — packed history is damaged; ` + + `refusing to continue silently` + ) + } + const out: Array & { offset: number; frameLen: number }> = [] + let at = MAGIC.length + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) + while (at + FRAME_PREFIX_BYTES <= bytes.length) { + const payloadLen = view.getUint32(at, true) + const crc = view.getUint32(at + 4, true) + const payload = bytes.subarray(at + FRAME_PREFIX_BYTES, at + FRAME_PREFIX_BYTES + payloadLen) + if (payload.length !== payloadLen || crc32c(payload) !== crc) { + throw new Error( + `[GenerationSegments] frame CRC mismatch in ${meta.file} at offset ${at} — ` + + `packed history is damaged; refusing to serve it` + ) + } + out.push({ ...this.decodeFrame(payload), offset: at, frameLen: FRAME_PREFIX_BYTES + payloadLen }) + at += FRAME_PREFIX_BYTES + payloadLen + } + return out + } + + /** Read one packed generation's frame via its sidecar offset (one ranged read). */ + private async readFrame( + gen: number + ): Promise | null> { + const meta = this.coveringSegment(gen) + if (!meta) return null + const idx = await this.sidecarFor(meta) + // generations ascending → binary search. + const gens = idx.generations + let lo = 0 + let hi = gens.length - 1 + while (lo <= hi) { + const mid = (lo + hi) >> 1 + if (gens[mid][0] < gen) lo = mid + 1 + else if (gens[mid][0] > gen) hi = mid - 1 + else { + const [, offset, frameLen] = gens[mid] + const bytes = await this.storage.readRawBytes(`${SEGMENTS_PREFIX}/${meta.file}`) + if (!bytes) { + throw new Error(`[GenerationSegments] sealed segment ${meta.file} is MISSING`) + } + const frame = bytes.subarray(offset, offset + frameLen) + const view = new DataView(frame.buffer, frame.byteOffset, frame.byteLength) + const payloadLen = view.getUint32(0, true) + const crc = view.getUint32(4, true) + const payload = frame.subarray(FRAME_PREFIX_BYTES, FRAME_PREFIX_BYTES + payloadLen) + if (payload.length !== payloadLen || crc32c(payload) !== crc) { + throw new Error( + `[GenerationSegments] frame CRC mismatch for generation ${gen} in ${meta.file} — ` + + `packed history is damaged; refusing to serve it` + ) + } + return this.decodeFrame(payload) + } + } + // In the covering range but not present: the packed tier is dense by + // construction (fold packs every generation it is handed, including + // record-less ones) — absence inside a sealed range is damage. + throw new Error( + `[GenerationSegments] generation ${gen} is inside sealed segment ${meta.file}'s declared ` + + `range but has no frame — packed history is damaged` + ) + } + + /** The packed tier's delta for `gen` (null = not packed). */ + async readDelta(gen: number): Promise<{ delta: unknown; timestamp: number } | null> { + const frame = await this.readFrame(gen) + return frame ? { delta: frame.delta, timestamp: frame.timestamp } : null + } + + /** The packed tier's full record-set for `gen` (null = not packed). */ + async readRecords(gen: number): Promise { + const frame = await this.readFrame(gen) + return frame ? frame.records : null + } + + /** One packed before-image (null = not packed OR no record for the id in that generation). */ + async readRecord(gen: number, kind: 'noun' | 'verb', id: string): Promise { + const frame = await this.readFrame(gen) + if (!frame) return null + const hit = frame.records.find((r) => r.kind === kind && r.id === id) + return hit ? hit.record : null + } + + /** + * D3 reclaim: drop WHOLE segments whose lastGeneration < `belowGeneration` + * and bump `compactedBelow`. Partial segments are never dropped — the + * boundary waits. NEVER called under the archival profile (the caller + * enforces retention semantics; this method only executes boundary drops). + */ + async dropSegmentsBelow(belowGeneration: number): Promise<{ dropped: number; compactedBelow: number }> { + const keep: SegmentMeta[] = [] + const drop: SegmentMeta[] = [] + for (const s of this.manifest.segments) { + ;(s.lastGeneration < belowGeneration ? drop : keep).push(s) + } + if (drop.length === 0) { + return { dropped: 0, compactedBelow: this.manifest.compactedBelow } + } + const compactedBelow = Math.max( + this.manifest.compactedBelow, + drop[drop.length - 1].lastGeneration + 1 + ) + // Manifest first (the drop is authoritative once named), then bytes — + // a crash between leaves orphan segment files invisible to the manifest, + // harmless and re-collectable. + const next: SegmentManifest = { ...this.manifest, compactedBelow, segments: keep } + await this.storage.writeRawObject(MANIFEST_PATH, next) + await this.storage.syncRawObjects([MANIFEST_PATH]) + this.manifest = next + for (const s of drop) { + await this.storage.deleteRawObject(`${SEGMENTS_PREFIX}/${s.file}`) + await this.storage.deleteRawObject(`${SEGMENTS_PREFIX}/${sidecarFileName(s.firstGeneration)}`) + this.sidecars.delete(s.file) + } + return { dropped: drop.length, compactedBelow } + } + + /** + * D8 rider — the packed portion of `generationDigest(g)`: a deterministic + * crc32c chain over sealed-segment checksums fully below `g`, plus the + * frame CRC of `g`'s own frame when `g` is mid-segment. O(segments), not + * O(generations); identical history ⇒ identical digest on any machine. + * The live-tier portion is composed by the caller. + */ + async digestThroughPacked(g: number): Promise { + let digest = 0 + let covered = false + for (const s of this.manifest.segments) { + if (s.lastGeneration <= g) { + digest = crc32c(new TextEncoder().encode(`${digest}:${s.checksum}`)) + if (s.lastGeneration === g) covered = true + } else if (s.firstGeneration <= g) { + // g is mid-segment: chain the partial prefix via g's frame CRC. + const frame = await this.readFrame(g) + if (frame === null) return null + const idx = await this.sidecarFor(s) + const upTo = idx.generations.filter(([gen]) => gen <= g) + for (const [gen, offset, frameLen] of upTo) { + digest = crc32c(new TextEncoder().encode(`${digest}:${gen}:${offset}:${frameLen}`)) + } + covered = true + break + } + } + return covered || this.manifest.segments.length > 0 ? digest : null + } +} diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index 364cdcef..aede17a4 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -32,7 +32,9 @@ */ import { prodLog } from '../utils/logger.js' -import { GenerationCompactedError, GenerationConflictError } from './errors.js' +import { GenerationCompactedError, GenerationConflictError, PendingFlushDurabilityError, StoreInconsistentError } from './errors.js' +import type { UnreconciledRecord } from './errors.js' +import { TransactionRollbackError } from '../transaction/errors.js' import type { ChangedIds, CompactHistoryOptions, @@ -43,6 +45,23 @@ import type { GenerationStorage, TxLogEntry } from './types.js' +import { FactLog, storageSupportsFactLog, type CommitFact, type FactOp } from './factLog.js' +import { GenerationSegmentStore, type FoldGeneration } from './generationSegments.js' +import { crc32c } from '../utils/crc32c.js' + +/** + * The byte-identical before-images of every id a commit touches, read UNDER + * the commit mutex immediately before the write applies. Passed to a commit's + * optional `precommit` hook so callers can enforce compare-and-swap + * preconditions (e.g. the per-entity `ifRev` check) atomically with the apply + * — the same conditional-commit idea as `ifAtGeneration`, generalized to + * arbitrary per-record predicates. An absent id maps to the create sentinel + * (`metadata: null, vector: null`). + */ +export interface CommitBeforeImages { + nouns: Map + verbs: Map +} /** Storage-root-relative path of the persisted generation counter. */ export const GENERATION_COUNTER_PATH = '_system/generation.json' @@ -105,6 +124,16 @@ export interface GenerationStoreOpenResult { export class GenerationStore { private readonly storage: GenerationStorage + /** + * The generation FACT LOG (dual-write transition) — an append-only, + * CRC-framed record of every committed generation as an AFTER-IMAGE fact. + * `null` when the storage layer lacks the binary raw-byte primitives. + * Appends ride the same commit protocol: a fact-append failure FAILS the + * write (loud — a silent fact gap would make the log a lie that a later + * replay discovers), and open() reconciles the log to committed truth. + */ + private factLog: FactLog | null = null + /** Latest reserved/observed generation (≥ {@link committed}). */ private counter = 0 /** Committed-transaction watermark (manifest generation). */ @@ -112,14 +141,206 @@ export class GenerationStore { /** Compaction horizon — record-sets ≤ this are reclaimed. */ private horizonGen = 0 - /** Sorted list of committed generations whose record dirs exist. */ - private committedGens: number[] = [] - /** Delta cache, keyed by generation (lazily loaded from `tx.json`). */ + /** + * Committed generations whose record dirs exist, stored as a SORTED, DISJOINT, + * ascending list of INCLUSIVE `[start, end]` intervals (a run-length set). + * + * Committed generations come from the monotonic `counter++`, so they are dense + * contiguous integers EXCEPT where {@link compact} punches a hole (reclaiming + * an oldest contiguous prefix). Storing them as intervals makes the resident + * size O(number-of-compaction-gaps) — typically a SINGLE interval + * `[firstGen, lastGen]` — instead of one array element per generation. A + * billion-insert corpus (every single-op reserves a distinct generation) would + * otherwise hold a ~1B-element array resident for the process lifetime; the + * interval form collapses that to O(1). All access goes through the helpers + * ({@link appendCommittedGen}, {@link committedGensAsc}, {@link committedCount}, + * {@link removeCommittedUpTo}, {@link lastCommittedGen}, + * {@link largestReservedAtOrBefore}) so the multiset and ordering are identical + * to the old flat array at every point. + */ + private committedRanges: Array<[number, number]> = [] + /** Delta cache, keyed by generation (lazily loaded from `tx.json`). `bytes` + * is the serialized record-set size, summed by {@link historyBytes} for the + * `maxBytes` / adaptive retention caps (0 for generations written before + * byte accounting, or for un-flushed pending generations). */ private readonly deltaCache = new Map< number, - { nouns: Set; verbs: Set; timestamp: number } + { nouns: Set; verbs: Set; timestamp: number; bytes: number } >() + /** + * Per-id inverted history index (Model-B scalability) — `id → ascending + * generations that touched it`, one map per kind. {@link resolveAt} binary- + * searches the id's OWN chain (O(log chain)) for the first generation after + * the pin, instead of linearly scanning the global committed-generation set + * ({@link committedRanges}) + * (which is O(database-age) — confirmed by the scalability spike: a read of an + * unchanged entity at an old pin scaled 11.9x for 10x history depth). This + * mirrors cor's `delta_history` chain. + * + * BOUNDED-RAM design (Approach C — hot-tail window + cold LRU). The old + * unbounded `Map` held ONE resident chain per id ever touched + * across retained history → O(distinct-ids-ever-touched) RAM, which defeats + * billion-scale time travel (the chains, not the data, become the heap floor). + * These three structures replace it with RAM bounded by the knobs W/L, + * INDEPENDENT of the corpus size: + * + * 1. {@link recentNounChains}/{@link recentVerbChains} — FULL per-id chains, + * but only over the HOT TAIL window `[windowLo, counter]` (the last + * {@link recentWindowGenerations} generations). Pins at-or-above + * `windowLo` (the overwhelming common case — recent history) resolve here + * with the same O(log chain) binary search as before. + * 2. {@link coldNounChains}/{@link coldVerbChains} — a bounded LRU of FULL + * per-id chains for DEEP pins (`gen < windowLo`). Reconstructed on demand + * from the persisted deltas ({@link reconstructColdChain}), cached so a + * re-read is O(log chain), evicted oldest-first past + * {@link coldChainLruMax}. Empty chains are cached too (bounded negative + * cache for untouched-in-cold-region ids). + * 3. {@link windowNounDeltas}/{@link windowVerbDeltas} — the window's INVERSE + * index (`gen → ids touched at that gen`), fed from the touched sets + * already handed to {@link extendChains}. It makes sliding `windowLo` + * forward an O(d̄) front-trim of exactly the ids leaving the window, with + * ZERO disk I/O — the write path never reads `tx.json` on a slide. + * + * INVARIANT — purely derived, NEVER persisted. All three structures (and the + * cold LRU) are reconstructable from the on-disk deltas at any time; eviction + * (cold LRU) and window slide-out drop entries that are then rebuilt on the + * next read that needs them. Correctness across eviction/reconstruction is + * guaranteed by pin-gating: a live historical `Db` pins at `g_old`, and the + * before-image a read needs lives at the FIRST generation after `g_old` + * (`> g_old ≥ minPinned`), which compaction can never reclaim while the pin is + * held — so a reconstructed chain yields the identical answer as the original. + */ + private readonly recentNounChains = new Map() + private readonly recentVerbChains = new Map() + /** Window inverse index (`gen → ids touched`), one map per kind — the O(d̄), + * I/O-free slide-trim driver. Holds exactly the gens in `[windowLo, counter]`. */ + private readonly windowNounDeltas = new Map() + private readonly windowVerbDeltas = new Map() + /** Bounded LRU of reconstructed deep-pin chains (`gen < windowLo`). JS Map + * insertion order is the LRU order: a hit re-inserts (delete+set) to mark + * MRU; eviction removes `keys().next().value` (the oldest) past the cap. */ + private readonly coldNounChains = new Map() + private readonly coldVerbChains = new Map() + /** Inclusive lower bound of the resident hot-tail window (`0` until built). */ + private windowLo = 0 + /** Whether the hot-tail window + its inverse index are built and current. */ + private windowReady = false + /** De-dupes concurrent first-callers of {@link ensureWindow}. */ + private windowBuilding: Promise | null = null + + /** + * Hot-tail window width W — how many of the newest generations keep a resident + * per-id chain ({@link recentNounChains}). Bounds recent-chain + window-delta + * RAM to O(W·d̄), independent of corpus size. Deep pins below the window fall to + * the cold LRU. Not `readonly` so tests can shrink it to exercise the + * cold/slide paths without building thousands of generations. + */ + private recentWindowGenerations = 4096 + + /** + * Cold-chain LRU capacity L — how many reconstructed deep-pin chains stay + * resident. Bounds cold-chain RAM to O(L·avg-chain-length), independent of how + * many distinct ids are deep-read. Not `readonly` so tests can shrink it to + * exercise eviction + reconstruction. + */ + private coldChainLruMax = 4096 + + /** + * Cap on resident {@link deltaCache} entries (Model-B RAM bound). The per-id + * {@link recentNounChains}/{@link recentVerbChains} window is the hot read + * structure; the raw + * deltas are only needed for range queries ({@link changedBetween}, `diff`, + * `since`) and chain (re)builds, and {@link getDelta} transparently re-reads an + * evicted delta from storage. Bounding this keeps a long-lived high-write + * process's heap O(cap) instead of O(generations) on the disk-backed path. + * Not `readonly` so tests can lower it to exercise the eviction/re-read path + * without building thousands of generations. + */ + private deltaCacheMax = 4096 + + /** + * Running total of on-disk history bytes across committed generations — + * `null` until {@link historyBytes} pays its one seeding walk. Maintained + * incrementally at commit/reclaim so the adaptive retention check on every + * flush() is O(1), never a tail re-walk. Never updated by cache re-reads + * ({@link setDelta} inserts are cache population, not new history). + */ + private historyBytesTotal: number | null = null + + /** + * The packed tier (D1+D3): sealed segments holding folded cold + * generations. Null until {@link open} wires it (and on storage adapters + * without raw-byte primitives — the live tier then carries everything, + * exactly as before the packed tier existed). + */ + private segments: GenerationSegmentStore | null = null + + /** + * Live-tier window: generations newer than `committed - REPACK_LIVE_WINDOW` + * are never folded — the hot tail stays in the per-generation layout the + * write path owns. Matches the resident chain window's scale. + */ + static readonly REPACK_LIVE_WINDOW = 1024 + + /** + * Model-B per-write group-commit — the in-memory PENDING tier. + * + * Each single-operation write reserves its OWN generation (the locked + * cross-project contract: "every write versioned; a distinct generation per + * single-op"), captures byte-identical before-images, and buffers them here + * instead of paying a synchronous per-write manifest fsync (the 3-5x write + * regression). {@link flushPendingSingleOps} persists the whole buffer to + * disk in ONE fsync (durability batching — NOT a generation collapse). + * + * Crucially, these pending generations participate in point-in-time + * resolution EXACTLY like committed ones ({@link resolveAt}, the per-id + * chains, {@link changedBetween}, {@link hasCommittedAfter} all read the + * union via {@link reservedGensAsc}; before-images resolve from this buffer + * while pending, from disk once flushed). That is what lets the SYNCHRONOUS + * `now()` pin freeze against un-flushed single-ops with no forced flush. + * + * Durability boundary: a hard crash before a flush loses only the buffered + * *history* of un-flushed writes (live data is already in canonical storage); + * a crash mid-flush is recovered by drop-without-restore (see + * {@link GenerationDelta.groupCommit}). + */ + private pendingGens: number[] = [] + private readonly pendingBuffer = new Map< + number, + { nouns: Map; verbs: Map; timestamp: number } + >() + /** Pending timer-coalesced flush handle (cleared on flush/close). */ + private pendingFlushTimer: ReturnType | null = null + /** + * Flush the pending tier once it holds this many generations (size trigger), + * complementing the {@link PENDING_FLUSH_DELAY_MS} timer trigger. Not + * `readonly` so tests can drive the boundary deterministically. + */ + private pendingFlushThreshold = 256 + /** Coalescing window (ms) before an idle pending tier is flushed. */ + private static readonly PENDING_FLUSH_DELAY_MS = 50 + + /** + * Latched pending-flush durability failure. Set once background flushes have + * failed {@link PENDING_FLUSH_FAILURE_THRESHOLD} consecutive times (a real, + * persistent storage fault — not a blip); while set, writes are REFUSED with + * a {@link PendingFlushDurabilityError} rather than silently accumulate + * un-durable history in memory. Cleared the moment a flush finally succeeds + * (the store self-heals). `null` = history-durability path is healthy. + */ + private pendingFlushError: Error | null = null + /** Consecutive failed background pending-flush attempts (resets on success). */ + private pendingFlushFailures = 0 + /** + * Consecutive failed flush attempts before the durability latch trips and + * writes start refusing. A small tolerance so a single transient fault does + * not trip the alarm, while a genuinely stuck disk stops the bleed fast. + */ + private static readonly PENDING_FLUSH_FAILURE_THRESHOLD = 3 + /** Upper bound (ms) on the exponential retry backoff between failed flushes. */ + private static readonly PENDING_FLUSH_RETRY_CAP_MS = 30_000 + /** Live pin refcounts, keyed by pinned generation. */ private readonly pins = new Map() @@ -182,20 +403,30 @@ export class GenerationStore { } let rolledBack = 0 - const committedGens: number[] = [] + // Coalesce the ascending on-disk committed gens into interval form: each + // contiguous run becomes a single `[start, end]` range (appendCommittedGen + // extends the last range on a +1 step, opens a new one across a gap). + this.committedRanges = [] for (const gen of [...seenGens].sort((a, b) => a - b)) { if (gen <= this.committed) { - committedGens.push(gen) + this.appendCommittedGen(gen) } else if (!options?.readOnly) { - // Uncommitted (crashed) transaction: restore before-images, drop the dir. - await this.rollBackUncommittedGeneration(gen) + // Uncommitted (crashed) generation. A transact generation is rolled + // back by RESTORING its before-images (its before-image + execute are + // one atomic unit). A single-op group-commit generation is DROPPED + // WITHOUT RESTORE — its live write already landed and was acknowledged + // before the flush, so restoring would silently revert it. The branch + // is decided by the persisted delta's `groupCommit` flag. + await this.recoverUncommittedGeneration(gen) rolledBack++ } // Reader mode: leave orphan dirs alone; resolution ignores them because - // committedGens only includes generations ≤ the manifest watermark. + // committedRanges only includes generations ≤ the manifest watermark. this.counter = Math.max(this.counter, gen) } - this.committedGens = committedGens + // The history window is rebuilt lazily from the (possibly changed) generation + // set on the next historical read — covers reopen and reopen-after-restore. + this.invalidateChains() if (rolledBack > 0) { prodLog.warn( @@ -206,6 +437,46 @@ export class GenerationStore { this.opened = true + // Generation FACT LOG (dual-write transition): when the storage layer + // exposes the binary raw-byte primitives, open the after-image fact log + // and reconcile it to committed truth — facts are appended BEFORE the + // commit point, so a crash can only leave the log AHEAD; open truncates + // any fact beyond `committed`. Storage without the primitives simply + // hosts no fact log (readers fall back to canonical enumeration). + if (storageSupportsFactLog(this.storage)) { + this.factLog = new FactLog(this.storage) + await this.factLog.open(this.committed) + } else { + this.factLog = null + } + + // PACKED TIER (D1+D3): same capability gate as the fact log. Opening + // reads ONE manifest — never a listing of the packed backlog — and seeds + // committedRanges with the sealed ranges so packed generations resolve + // exactly like live ones. + if (storageSupportsFactLog(this.storage)) { + this.segments = new GenerationSegmentStore(this.storage) + await this.segments.open() + const packedRanges = this.segments + .segments() + .map((s): [number, number] => [s.firstGeneration, Math.min(s.lastGeneration, this.committed)]) + .filter(([lo, hi]) => lo <= hi) + if (packedRanges.length > 0) { + // Merge packed (older) + live (newer) interval sets — both ascending; + // coalesce adjacency so range arithmetic stays interval-exact. + const merged: Array<[number, number]> = [] + for (const r of [...packedRanges, ...this.committedRanges].sort((a, b) => a[0] - b[0])) { + const last = merged[merged.length - 1] + if (last && r[0] <= last[1] + 1) last[1] = Math.max(last[1], r[1]) + else merged.push([r[0], r[1]]) + } + this.committedRanges = merged + } + this.horizonGen = Math.max(this.horizonGen, this.segments.compactedBelow() - 1) + } else { + this.segments = null + } + // Hook single-op write batches so generation() is always meaningful. // Suppressed while a transact batch executes (the batch is ONE generation). if (!options?.readOnly) { @@ -220,6 +491,10 @@ export class GenerationStore { * from `brain.close()`. */ async close(): Promise { + // Persist any buffered single-op history before detaching, so a clean close + // never loses generations the caller already observed. + this.clearPendingFlushTimer() + await this.flushPendingSingleOps() this.storage.setGenerationBumpHook(undefined) await this.persistCounterNow() } @@ -261,6 +536,110 @@ export class GenerationStore { return this.horizonGen } + /** + * @description Read-only history footprint for fleet audits: how much + * generational history this store holds on disk. `bytes` pays (and seeds) + * the one-time {@link historyBytes} walk on first call — subsequent calls + * are O(1). The oldest/newest timestamps come from those generations' + * deltas (cache-bounded reads). + * @returns Counts, bytes, generation range, and the compaction horizon. + */ + /** + * @description D8 (gate-to-generation provenance): a deterministic content + * digest of the generation log THROUGH `g` — identical history ⇒ identical + * digest on any machine; any divergence (different records, different + * order, reclaimed range) ⇒ different digest. Composed from the packed + * tier's sealed-segment checksum chain (O(segments)) plus the live tier's + * per-generation delta digests (O(live window at most)). Release gates pin + * {generation, digest} and verify both at execution time. + * @param g - The generation to digest through (≤ committed). + * @returns A hex digest string, stable across reopen and repacking states + * ONLY for fully-packed prefixes — repacking changes representation, so + * the composed digest is defined over CONTENT: live-tier gens hash their + * delta + record ids, packed gens hash via frame CRCs. A gate should pin + * after a repack pass for long-term stability, or re-pin on repack. + */ + async generationDigest(g: number): Promise { + if (!Number.isInteger(g) || g < 1 || g > this.committed) { + throw new RangeError( + `generationDigest(): generation ${g} is out of range [1, ${this.committed}]` + ) + } + if (g <= this.horizonGen) { + throw new GenerationCompactedError(g, this.horizonGen) + } + let digest = 0 + const enc = new TextEncoder() + if (this.segments) { + const packed = await this.segments.digestThroughPacked(g) + if (packed !== null) digest = packed + } + // Live-tier composition: every committed gen ≤ g not covered by a sealed + // segment hashes its delta content in ascending order. + for (const gen of this.committedGensAsc()) { + if (gen > g) break + if (this.segments?.hasGeneration(gen)) continue + const delta = await this.getDelta(gen) + digest = crc32c( + enc.encode( + `${digest}:${gen}:${delta.timestamp}:${[...delta.nouns].sort().join(',')}:${[...delta.verbs].sort().join(',')}` + ) + ) + } + return digest.toString(16).padStart(8, '0') + } + + async historyStats(): Promise<{ + generations: number + bytes: number + oldestGeneration: number | null + newestGeneration: number | null + oldestTimestamp: number | null + newestTimestamp: number | null + horizon: number + }> { + let oldest: number | null = null + let newest: number | null = null + for (const gen of this.committedGensAsc()) { + if (oldest === null) oldest = gen + newest = gen + } + return { + generations: this.committedCount(), + bytes: await this.historyBytes(), + oldestGeneration: oldest, + newestGeneration: newest, + oldestTimestamp: oldest !== null ? (await this.getDelta(oldest)).timestamp : null, + newestTimestamp: newest !== null ? (await this.getDelta(newest)).timestamp : null, + horizon: this.horizonGen + } + } + + /** + * @description Read one generation's persisted before-image records — the + * compaction fallback for generations written before deltas carried + * `blobHashes`. O(that generation's records). + * @param gen - The generation whose `prev/` records to read. + * @returns The record-set (empty when absent). + */ + private async readGenerationRecords(gen: number): Promise { + let paths: string[] = [] + try { + paths = await this.storage.listRawObjects(`${GENERATIONS_PREFIX}/${gen}/prev`) + } catch { + paths = [] + } + const records: GenerationRecord[] = [] + for (const p of paths) { + const record = (await this.storage.readRawObject(p)) as GenerationRecord | null + if (record) records.push(record) + } + if (records.length > 0) return records + // Two-tier: folded generations serve their record-set from the segment. + const packed = await this.segments?.readRecords(gen) + return packed ? (packed.map((r) => r.record) as GenerationRecord[]) : [] + } + /** * @description Single-operation write hook (registered with the storage * layer in {@link open}). Bumps the in-memory counter and schedules a @@ -380,13 +759,75 @@ export class GenerationStore { * @returns The committed generation and its commit timestamp. * @throws GenerationConflictError when the CAS expectation fails. */ + /** + * The generation fact log, or `null` when the storage layer cannot host one. + * Consumers scan committed facts through it (`scanFacts` / `segmentPaths`). + */ + getFactLog(): FactLog | null { + return this.factLog + } + + /** + * @description Build one commit's AFTER-IMAGE fact by reading canonical + * state back for every touched id — under the commit mutex, immediately + * after the operations applied, so canonical IS the after-image (and the + * reads are page-cache-warm: the operations just wrote these files). An + * absent id (both legs null) becomes a body-less TOMBSTONE — the delete + * fact needs no body, so removal never requires reading the removed thing. + * The fact's blobHashes are extracted from the AFTER records (the content + * this generation's state references), unlike the history path's + * before-image hashes. + */ + private async buildCommitFact(args: { + generation: number + timestamp: number + nouns: string[] + verbs: string[] + meta?: Record + }): Promise { + const ops: FactOp[] = [] + const afterRecords: GenerationRecord[] = [] + for (const id of args.nouns) { + const after = await this.storage.readNounRaw(id) + const absent = after.metadata === null && after.vector === null + ops.push({ kind: 'noun', id, record: absent ? null : after }) + if (!absent) afterRecords.push({ kind: 'noun', metadata: after.metadata, vector: after.vector }) + } + for (const id of args.verbs) { + const after = await this.storage.readVerbRaw(id) + const absent = after.metadata === null && after.vector === null + ops.push({ kind: 'verb', id, record: absent ? null : after }) + if (!absent) afterRecords.push({ kind: 'verb', metadata: after.metadata, vector: after.vector }) + } + const blobHashes = this.storage.extractBlobHashesFromRecords + ? this.storage.extractBlobHashesFromRecords(afterRecords) + : [] + return { + generation: args.generation, + timestamp: args.timestamp, + ops, + ...(args.meta ? { meta: args.meta } : {}), + ...(blobHashes.length > 0 ? { blobHashes } : {}) + } + } + async commitTransaction(args: { touched: TouchedIds meta?: Record ifAtGeneration?: number + /** Optional compare-and-swap precondition over the {@link CommitBeforeImages}, + * run under the commit mutex BEFORE anything is staged or applied — the + * per-record analogue of `ifAtGeneration`. A throw aborts the whole batch: + * the generation reservation is returned and no staging I/O has happened. */ + precommit?: (before: CommitBeforeImages) => void execute: () => Promise }): Promise<{ generation: number; timestamp: number }> { return this.withMutex(async () => { + // A latched history-durability failure compromises the whole generation + // spine — refuse a transact too (advancing the manifest past stuck, + // un-durable single-op generations would be inconsistent). Same loud + // error; self-clears when the pending tier drains. + this.assertHistoryDurable() if (args.ifAtGeneration !== undefined && args.ifAtGeneration !== this.counter) { throw new GenerationConflictError(args.ifAtGeneration, this.counter) } @@ -411,21 +852,61 @@ export class GenerationStore { } } + // Temporal-blob contract: hashes this record-set references, recorded + // before staging; scoped outside the try so an abort can compensate. + let txBlobHashes: string[] = [] + + // Hoisted so the catch can reconcile canonical state against them after a + // failed rollback (the trapdoor). Empty until populated below. + const nounBefore = new Map() + const verbBefore = new Map() + try { // -- 3. Before-images + delta (the durable undo log) ------------------ - const stagedPaths: string[] = [] + // Read every before-image FIRST, then run the caller's CAS + // precondition against them, and only then stage to disk — so a + // conflicting batch aborts with zero staging I/O. The maps hold the + // byte-identical records the staged files are written from. for (const id of nouns) { const prev = await this.storage.readNounRaw(id) - const recordPath = `${dir}/prev/${id}.json` - const record: GenerationRecord = { kind: 'noun', metadata: prev.metadata, vector: prev.vector } - await this.storage.writeRawObject(recordPath, record) - stagedPaths.push(recordPath) + nounBefore.set(id, { kind: 'noun', metadata: prev.metadata, vector: prev.vector }) } for (const id of verbs) { const prev = await this.storage.readVerbRaw(id) + verbBefore.set(id, { kind: 'verb', metadata: prev.metadata, vector: prev.vector }) + } + + // Conditional commit: the per-record CAS analogue of the + // `ifAtGeneration` check above, but against authoritative + // before-images under the mutex. A throw lands in the catch below, + // which returns the generation reservation; nothing was applied. + args.precommit?.({ nouns: nounBefore, verbs: verbBefore }) + + // Temporal-blob contract: count this record-set's content-hash + // references BEFORE it is staged (over-count-only crash ordering — + // see flushPendingSingleOps' matching note). + txBlobHashes = this.storage.extractBlobHashesFromRecords + ? this.storage.extractBlobHashesFromRecords([ + ...nounBefore.values(), + ...verbBefore.values() + ]) + : [] + if (txBlobHashes.length > 0 && this.storage.recordHistoryBlobReferences) { + await this.storage.recordHistoryBlobReferences(txBlobHashes) + } + + const stagedPaths: string[] = [] + let recordBytes = 0 // serialized record-set size, for retention accounting + for (const [id, record] of nounBefore) { const recordPath = `${dir}/prev/${id}.json` - const record: GenerationRecord = { kind: 'verb', metadata: prev.metadata, vector: prev.vector } await this.storage.writeRawObject(recordPath, record) + recordBytes += serializedBytes(record) + stagedPaths.push(recordPath) + } + for (const [id, record] of verbBefore) { + const recordPath = `${dir}/prev/${id}.json` + await this.storage.writeRawObject(recordPath, record) + recordBytes += serializedBytes(record) stagedPaths.push(recordPath) } const delta: GenerationDelta = { @@ -433,8 +914,12 @@ export class GenerationStore { timestamp, ...(args.meta && { meta: args.meta }), nouns, - verbs + verbs, + // Always present on new deltas (empty = "no blobs"), so compaction + // only falls back to record reads for pre-contract generations. + blobHashes: txBlobHashes } + delta.bytes = recordBytes + serializedBytes(delta) const deltaPath = `${dir}/tx.json` await this.storage.writeRawObject(deltaPath, delta) stagedPaths.push(deltaPath) @@ -442,14 +927,41 @@ export class GenerationStore { faultPoint('after-staging') // -- 4. Execute the planned batch ------------------------------------- + // Durability barrier: record every canonical write/delete the operations + // make, then fsync them BELOW (before the counter advances). Without + // this, a hard kill after commitTransaction returns could leave the + // fsync'd generation counter ahead of still-page-cached entity bytes — + // phantom progress for any generation-based consumer. + this.storage.beginWriteBarrier?.() this.inTransact = true try { await args.execute() } finally { this.inTransact = false } + // The transaction's entire canonical footprint is now durable, so the + // counter/manifest advance below can never outrun the entity bytes. + await this.storage.flushWriteBarrier?.() faultPoint('after-execute') + // Fact log (dual-write): append + fsync this generation's AFTER-IMAGE + // fact BEFORE the commit point, inside the same durability window — + // so a crash can only leave the log AHEAD (open truncates), never a + // committed generation without its fact. A real abort below this point + // compensates via dropAbove in the catch. An append failure fails the + // write, loudly — a silent fact gap would be a lie a replay discovers. + if (this.factLog) { + const fact = await this.buildCommitFact({ + generation: gen, + timestamp, + nouns, + verbs, + ...(args.meta ? { meta: args.meta } : {}) + }) + await this.factLog.append(fact) + await this.factLog.sync() + } + // -- 5. Counter + manifest rename (COMMIT POINT) ---------------------- await this.persistCounterUnlocked() faultPoint('before-manifest-rename') @@ -465,8 +977,17 @@ export class GenerationStore { // -- 6. Post-commit bookkeeping --------------------------------------- this.committed = gen - this.committedGens.push(gen) - this.deltaCache.set(gen, { nouns: new Set(nouns), verbs: new Set(verbs), timestamp }) + this.appendCommittedGen(gen) + this.setDelta(gen, { + nouns: new Set(nouns), + verbs: new Set(verbs), + timestamp, + bytes: delta.bytes ?? 0 + }) + if (this.historyBytesTotal !== null) { + this.historyBytesTotal += delta.bytes ?? 0 + } + this.extendChains(gen, nouns, verbs) const logEntry: TxLogEntry = { generation: gen, timestamp, ...(args.meta && { meta: args.meta }) } await this.storage.appendTxLogLine(JSON.stringify(logEntry)) @@ -478,6 +999,16 @@ export class GenerationStore { if (crashSimulated) { throw err } + // The trapdoor for a batch: if rollback FAILED to fully apply, canonical + // storage may be inconsistent. A batch is never adopted forward (its + // other ops were rolled back — partial commit would break atomicity), so + // any unreconciled record is a fail-loud StoreInconsistentError. Compute + // it here (before the staging cleanup, which is always safe to run) and + // throw it in place of the raw error at the end. + let inconsistent: UnreconciledRecord[] = [] + if (err instanceof TransactionRollbackError) { + inconsistent = await this.reconcileFailedRollback(nounBefore, verbBefore) + } // Failed before the manifest rename: nothing is committed. Remove the // staging directory; the TransactionManager already restored any // applied operation byte-identically. @@ -489,14 +1020,669 @@ export class GenerationStore { `${(cleanupErr as Error).message} (recovery will remove it on next open)` ) } + // Compensate the pre-staging history-reference increments. Best + // effort — a failure here only over-counts (a leak the scrub + // repairs). Reclaim inside is safe on an abort: the before-image + // hashes are the entities' still-live content, so live references + // block any physical delete. + if (txBlobHashes.length > 0 && this.storage.releaseHistoryBlobReferences) { + try { + await this.storage.releaseHistoryBlobReferences(txBlobHashes) + } catch { + // over-count-safe; the scrub restores exactness + } + } + // Fact-log compensation: a real (non-crash) abort after the fact was + // appended must take the fact back out — the generation never + // committed. A crash instead reaches open(), whose truncation does the + // same reconcile from disk. + if (this.factLog && this.factLog.headGeneration() >= gen) { + await this.factLog.dropAbove(gen - 1) + } // Return the reservation when no concurrent bump consumed a later // number, so a failed transaction leaves generation() unchanged. if (this.counter === gen) this.counter = gen - 1 + // A failed rollback that left canonical records unreconciled surfaces as + // a loud StoreInconsistentError (brainy quarantines writes until repair); + // otherwise the raw error (clean abort, or a derived-only undo failure + // the egress guard + rebuild handle). + if (inconsistent.length > 0) { + throw new StoreInconsistentError( + inconsistent, + (err as TransactionRollbackError).originalError + ) + } throw err } }) } + /** + * @description After a transaction's rollback FAILED to fully apply + * (`TransactionRollbackError`), determine which touched records are now + * unreconciled by comparing current canonical state to the before-images the + * commit captured. This observes reality rather than trusting the opaque undo + * closures — the authoritative signal for adopt-forward vs fail-loud. + * + * @param nounBefore - Byte-identical entity before-images captured pre-execute. + * @param verbBefore - Byte-identical relationship before-images. + * @returns Every record whose canonical state no longer matches its + * before-image, tagged `'orphan'` (durably present when it should be gone — + * an add whose delete-undo failed) or `'loss'` (durably gone/wrong when it + * should have been restored — a remove/update whose restore-undo failed). An + * empty array means canonical storage is cleanly rolled back (only a derived + * index undo failed — the egress guard + a rebuild handle that). + */ + private async reconcileFailedRollback( + nounBefore: Map, + verbBefore: Map + ): Promise { + const out: UnreconciledRecord[] = [] + const classify = ( + before: GenerationRecord, + current: { metadata: unknown | null; vector: unknown | null } + ): 'orphan' | 'loss' | null => { + const beforeEmpty = before.metadata == null && before.vector == null + const currentEmpty = current.metadata == null && current.vector == null + if (beforeEmpty && currentEmpty) return null // add cleanly undone + if (beforeEmpty) return 'orphan' // add's delete-undo failed → still present + if (currentEmpty) return 'loss' // remove's restore-undo failed → gone + // Both present: same value = reconciled; different = restore left a wrong value. + const same = + JSON.stringify({ m: before.metadata, v: before.vector }) === + JSON.stringify({ m: current.metadata, v: current.vector }) + return same ? null : 'loss' + } + for (const [id, before] of nounBefore) { + const d = classify(before, await this.storage.readNounRaw(id)) + if (d) out.push({ id, kind: 'noun', disposition: d }) + } + for (const [id, before] of verbBefore) { + const d = classify(before, await this.storage.readVerbRaw(id)) + if (d) out.push({ id, kind: 'verb', disposition: d }) + } + return out + } + + // ========================================================================== + // Single-operation commit (Model-B per-write group-commit) + // ========================================================================== + + /** + * @description Commit ONE single-operation write (`add`/`update`/`remove`/ + * `relate`/`updateRelation`/`unrelate` and the per-item calls of `addMany`/ + * `updateMany`/`relateMany`) as its own immutable generation — the Model-B + * "every write versioned" contract. (`removeMany` commits each delete *chunk* + * as a single generation for bulk efficiency — the one non-per-item path.) Structurally a one-operation {@link commitTransaction} with + * **deferred durability**: + * + * - Under the commit mutex it reserves generation `N`, captures byte-identical + * before-images of every touched id (so the pin/`asOf` read source is + * exact), runs `execute()` (the existing single-op `executeTransaction` + * batch) with the storage bump hook suppressed, then BUFFERS the + * before-images in the in-memory pending tier and returns — **no per-write + * manifest fsync** (that synchronous fsync is the 3-5x write regression this + * design avoids). + * - The generation is immediately visible to point-in-time reads (chains + + * {@link reservedGensAsc}), so a synchronous `now()` pin freezes against it + * with no forced flush. + * - {@link flushPendingSingleOps} later persists the buffer to disk in one + * fsync (async group-commit). A crash before that flush loses only the + * buffered *history*; live data is already in canonical storage. A crash + * mid-flush is recovered by drop-without-restore + * (see {@link GenerationDelta.groupCommit}). + * + * **Durability contract (single-op vs transact).** A single-op write's live + * canonical bytes are written via tmp+rename but NOT individually fsync'd — + * they are durable at the next {@link flushPendingSingleOps} or `close()`, not + * the instant the write resolves. On a hard kill before that flush, a + * single-op write can be lost even though the call returned; the group-commit + * counter is buffered alongside, so counter and data are lost together (no + * counter-ahead-of-state torn store). {@link commitTransaction} is the + * stronger contract: it runs a write barrier that fsyncs the whole batch's + * canonical footprint BEFORE advancing the generation counter, so a committed + * transact is durable on return. Callers that need per-write durability should + * use `transact()` (or `flush()` after a single-op); Model-B group-commit + * deliberately trades single-op fsync latency (a 3-5x write regression) for + * throughput. + * + * The per-write generation is read by graph-index ops through the same + * `generation()` watermark `transact()` uses — because this method, like + * {@link commitTransaction}, holds the mutex across the counter bump AND + * `execute()`, so the watermark equals `N` for the duration of the write (a + * distinct generation threaded to cor per single-op, the locked contract). + * + * @param args.touched - The ids this write creates/updates/deletes, by kind. + * @param args.execute - Runs the single-op's existing operation batch. + * @param args.precommit - Optional compare-and-swap precondition, invoked + * under the commit mutex with the just-read {@link CommitBeforeImages} and + * BEFORE `execute()`. A throw aborts the commit atomically: the generation + * reservation is returned and nothing is applied or buffered. This is the + * one place a per-record CAS (e.g. `ifRev`) is race-free — any check done + * before this mutex can interleave with a concurrent writer. + * @returns The reserved generation and its timestamp. + */ + async commitSingleOp(args: { + touched: { nouns?: string[]; verbs?: string[] } + execute: () => Promise + precommit?: (before: CommitBeforeImages) => void + }): Promise<{ generation: number; timestamp: number; degraded?: string[] }> { + return this.withMutex(async () => { + // Refuse to accept a write whose history we cannot make durable: if the + // pending-tier flush has latched a persistent failure, accepting more + // writes would silently pile un-durable before-images into memory. Fail + // loud instead (the latch self-clears when a flush finally succeeds). + this.assertHistoryDurable() + const nouns = args.touched.nouns ? [...new Set(args.touched.nouns)] : [] + const verbs = args.touched.verbs ? [...new Set(args.touched.verbs)] : [] + const gen = ++this.counter + const timestamp = Date.now() + + // Before-images BEFORE execute overwrites live storage (mirrors + // commitTransaction's staging, but buffered in memory — durability is + // deferred to the flush). readNounRaw/readVerbRaw on an absent id return + // {metadata:null, vector:null} = the create sentinel. + const nounBefore = new Map() + for (const id of nouns) { + const prev = await this.storage.readNounRaw(id) + nounBefore.set(id, { kind: 'noun', metadata: prev.metadata, vector: prev.vector }) + } + const verbBefore = new Map() + for (const id of verbs) { + const prev = await this.storage.readVerbRaw(id) + verbBefore.set(id, { kind: 'verb', metadata: prev.metadata, vector: prev.vector }) + } + + // Conditional commit: the caller's CAS precondition runs here — under + // the mutex, against the authoritative before-images, before any write. + // A throw aborts cleanly: return the generation reservation (nothing was + // applied or buffered) and surface the conflict to the caller. + if (args.precommit) { + try { + args.precommit({ nouns: nounBefore, verbs: verbBefore }) + } catch (err) { + if (this.counter === gen) this.counter = gen - 1 + throw err + } + } + + // Execute the live write. inTransact suppresses the storage bump hook so + // the counter is not double-advanced (this generation already reserved + // it) — the same suppression transact() uses. + this.inTransact = true + try { + await args.execute() + } catch (err) { + this.inTransact = false + // A failed rollback (TransactionRollbackError) may have left canonical + // storage inconsistent — the trapdoor. Reconcile against the + // before-images to decide the honest response (David's ruling: + // adopt-forward when safe, else fail loud). + if (err instanceof TransactionRollbackError) { + const records = await this.reconcileFailedRollback(nounBefore, verbBefore) + if (records.length > 0 && records.every((r) => r.disposition === 'orphan')) { + // ADOPT FORWARD: the durably-present orphan(s) are exactly what this + // single-op add wrote. Keep + buffer the generation so the record is + // legitimately committed and readable; the derived index may be + // incomplete for these ids until the next rebuild/repairIndex (the + // egress guard prevents wrong results meanwhile). Loud, honest, + // no double-write. + this.pendingBuffer.set(gen, { nouns: nounBefore, verbs: verbBefore, timestamp }) + this.pendingGens.push(gen) + this.extendChains(gen, nouns, verbs) + // The adopted generation is committed — it gets its fact like any + // other (durability rides the group-commit flush, same as the + // buffered history). + if (this.factLog) { + await this.factLog.append( + await this.buildCommitFact({ generation: gen, timestamp, nouns, verbs }) + ) + } + prodLog.warn( + `[GenerationStore] Recovered a failed rollback FORWARD: single-op write ` + + `committed as generation ${gen} because its canonical undo could not be ` + + `applied (record(s) ${records.map((r) => r.id).join(', ')} are durable). ` + + `The derived index may be incomplete for these ids — run repairIndex() to heal.` + ) + return { generation: gen, timestamp, degraded: records.map((r) => r.id) } + } + if (records.length > 0) { + // FAIL LOUD: a loss (or mixed) cannot be safely adopted. Return the + // reservation and throw — brainy quarantines writes until repair. + if (this.counter === gen) this.counter = gen - 1 + throw new StoreInconsistentError(records, err.originalError) + } + // records.length === 0: canonical is cleanly rolled back (only a + // derived undo failed) — fall through to the clean-abort path below. + } + // Live write failed with canonical cleanly rolled back: nothing buffered, + // nothing durable. Return the reservation (when no concurrent bump + // consumed a later number) so generation() is unchanged. + if (this.counter === gen) this.counter = gen - 1 + throw err + } + this.inTransact = false + + // Buffer the pending generation + make it instantly visible to reads. + this.pendingBuffer.set(gen, { nouns: nounBefore, verbs: verbBefore, timestamp }) + this.pendingGens.push(gen) + this.extendChains(gen, nouns, verbs) + // Fact log (dual-write): the acked write's AFTER-IMAGE fact, appended + // now (read back warm, under the mutex — group-commit means flush-time + // canonical only holds the LATEST state, so each generation's after-image + // exists only here). Durability rides the group-commit flush, exactly + // like the buffered before-image history: a crash before the flush loses + // the fact AND the generation together — never a torn state. + if (this.factLog) { + await this.factLog.append( + await this.buildCommitFact({ generation: gen, timestamp, nouns, verbs }) + ) + } + this.schedulePendingFlush() + return { generation: gen, timestamp } + }) + } + + /** + * @description Run a write WITHOUT creating a generation — for init-time / + * infrastructure writes (e.g. the VFS root directory) that constitute the + * generation-0 baseline of a freshly-materialized brain rather than a user + * mutation. The storage bump hook is suppressed (`inTransact`) so the + * generation counter does not advance, so a brand-new brain reports + * `generation() === 0` and an empty `transactionLog()`, and the first USER + * write is generation 1. Mirrors Datomic semantics where the empty database + * value is the starting point and bootstrap is not a transaction. + * @param execute - The infrastructure write to apply to the baseline. + */ + async runWithoutGeneration(execute: () => Promise): Promise { + this.inTransact = true + try { + await execute() + } finally { + this.inTransact = false + } + } + + /** + * @description Persist the in-memory pending single-op generations to disk in + * ONE fsync (async group-commit) and advance the committed watermark. Called + * on the size/timer triggers and forced by `flush()`/`close()`/`transact()`/ + * `compactHistory()` (so generations stay strictly ordered and durable before + * any of those). A no-op when the pending tier is empty. + * + * Each pending generation is written as a normal `_generations//` record + * set (`prev/.json` before-images + `tx.json` delta) but with the delta's + * `groupCommit: true` flag set — the drop-without-restore recovery marker. + * The single manifest rename to the highest generation commits the whole + * batch atomically (committed iff `gen ≤ manifest.generation`). + */ + async flushPendingSingleOps(): Promise { + if (this.pendingGens.length === 0) return + // Centralize durability accounting here so EVERY failure path — the + // background scheduler AND an explicit flush()/close() — latches + // consistently, and success (from any caller) clears the latch. The + // scheduler owns only the retry cadence. + try { + await this.flushPendingSingleOpsUnlocked() + this.onPendingFlushSuccess() + } catch (err) { + this.recordPendingFlushFailure(err as Error) + throw err + } + } + + /** The actual pending-tier flush under the mutex (see {@link flushPendingSingleOps}, + * which wraps this with durability accounting). */ + private async flushPendingSingleOpsUnlocked(): Promise { + return this.withMutex(async () => { + if (this.pendingGens.length === 0) return + this.clearPendingFlushTimer() + + const gens = [...this.pendingGens].sort((a, b) => a - b) + const stagedPaths: string[] = [] + const logEntries: TxLogEntry[] = [] + const genBytes = new Map() // for the post-commit setDelta + for (const gen of gens) { + const buf = this.pendingBuffer.get(gen) + if (!buf) continue + const dir = `${GENERATIONS_PREFIX}/${gen}` + + // Temporal-blob contract: count this record-set's content-hash + // references BEFORE persisting it (a crash between the two only + // over-counts — the scrub repairs a leak; under-counting could let + // compaction reclaim bytes a retained generation still needs). + const blobHashes = this.storage.extractBlobHashesFromRecords + ? this.storage.extractBlobHashesFromRecords([ + ...buf.nouns.values(), + ...buf.verbs.values() + ]) + : [] + if (blobHashes.length > 0 && this.storage.recordHistoryBlobReferences) { + await this.storage.recordHistoryBlobReferences(blobHashes) + } + + const nounIds: string[] = [] + const verbIds: string[] = [] + let recordBytes = 0 + for (const [id, record] of buf.nouns) { + const path = `${dir}/prev/${id}.json` + await this.storage.writeRawObject(path, record) + recordBytes += serializedBytes(record) + stagedPaths.push(path) + nounIds.push(id) + } + for (const [id, record] of buf.verbs) { + const path = `${dir}/prev/${id}.json` + await this.storage.writeRawObject(path, record) + recordBytes += serializedBytes(record) + stagedPaths.push(path) + verbIds.push(id) + } + const delta: GenerationDelta = { + generation: gen, + timestamp: buf.timestamp, + groupCommit: true, + nouns: nounIds, + verbs: verbIds, + // Always present on new deltas (empty = "no blobs"), so compaction + // only falls back to record reads for pre-contract generations. + blobHashes + } + delta.bytes = recordBytes + serializedBytes(delta) + genBytes.set(gen, delta.bytes) + const deltaPath = `${dir}/tx.json` + await this.storage.writeRawObject(deltaPath, delta) + stagedPaths.push(deltaPath) + logEntries.push({ generation: gen, timestamp: buf.timestamp }) + } + + // ONE fsync for the whole window — the durability-batching win. + await this.storage.syncRawObjects(stagedPaths) + + // Fact log (dual-write): make the window's buffered facts durable in the + // same batch, BEFORE the commit point below — so a crash can only leave + // the log AHEAD of the counter (open truncates), never a committed + // generation without its durable fact. + await this.factLog?.sync() + + // Test-only crash simulation: a throwing injector here leaves the staged + // group-commit generation dirs on disk with NO manifest advance — the + // exact "crashed mid-flush" state recovery must DROP-WITHOUT-RESTORE + // (the pending in-memory state would be lost in a real crash; we abandon + // this store and recover from disk on the next open()). + if (this.commitFaultInjector) this.commitFaultInjector('before-manifest-rename') + + // Commit point: persist the counter, then atomically rename the manifest + // to the highest pending generation — that commits ALL of them at once. + const highest = gens[gens.length - 1] + const highestTs = this.pendingBuffer.get(highest)?.timestamp ?? Date.now() + await this.persistCounterUnlocked() + const manifest: GenerationManifest = { + version: 1, + generation: highest, + committedAt: new Date(highestTs).toISOString(), + horizon: this.horizonGen + } + await this.storage.writeRawObject(MANIFEST_PATH, manifest) + await this.storage.syncRawObjects([MANIFEST_PATH]) + + // Move pending → committed. Chains already carry these gens; seed the + // bounded delta cache from the buffer so the next read avoids a re-read, + // then drop the in-memory before-images. + this.committed = highest + for (const gen of gens) { + const buf = this.pendingBuffer.get(gen) + if (!buf) continue + this.appendCommittedGen(gen) + this.setDelta(gen, { + nouns: new Set(buf.nouns.keys()), + verbs: new Set(buf.verbs.keys()), + timestamp: buf.timestamp, + bytes: genBytes.get(gen) ?? 0 + }) + if (this.historyBytesTotal !== null) { + this.historyBytesTotal += genBytes.get(gen) ?? 0 + } + this.pendingBuffer.delete(gen) + } + this.pendingGens = [] + + for (const entry of logEntries) { + await this.storage.appendTxLogLine(JSON.stringify(entry)) + } + }) + } + + /** + * @description Reset the durability-failure state after a successful flush. + * If a failure was latched (writes were being refused), log the recovery so + * the transition out of the loud state is as visible as the transition in. + */ + private onPendingFlushSuccess(): void { + if (this.pendingFlushError) { + prodLog.info( + `[GenerationStore] pending single-op history flush recovered after ` + + `${this.pendingFlushFailures} failed attempt(s); writes resume.` + ) + } + this.pendingFlushFailures = 0 + this.pendingFlushError = null + } + + /** + * @description Account for a failed pending-tier flush (called from + * {@link flushPendingSingleOps}' catch, so it fires on EVERY failure path — + * background scheduler and explicit flush()/close() alike). Escalates a + * transient blip (a warn, keep retrying) into a latched durability failure + * once {@link PENDING_FLUSH_FAILURE_THRESHOLD} consecutive attempts fail — + * from that point writes are refused with a {@link PendingFlushDurabilityError} + * until a flush finally succeeds. Always ensures a backoff retry is scheduled + * so a recovering disk self-heals without operator action, regardless of who + * triggered the failing flush. The pending before-images are retained (the + * failed flush never cleared them), so no history is lost — only not-yet-durable. + */ + private recordPendingFlushFailure(err: Error): void { + this.pendingFlushFailures++ + const attempts = this.pendingFlushFailures + if (attempts >= GenerationStore.PENDING_FLUSH_FAILURE_THRESHOLD) { + // Latch: refuse further writes rather than pile un-durable history. + this.pendingFlushError = err + prodLog.error( + `[GenerationStore] pending single-op history flush has FAILED ${attempts} ` + + `consecutive times (${err.message}). Generation history is not durable; ` + + `writes are now REFUSED until it drains. Live canonical data is intact. ` + + `Retrying with backoff; the store resumes writes when a flush succeeds.` + ) + } else { + prodLog.warn( + `[GenerationStore] pending single-op flush failed (attempt ${attempts}/` + + `${GenerationStore.PENDING_FLUSH_FAILURE_THRESHOLD}): ${err.message}; retrying with backoff.` + ) + } + this.scheduleFlushRetry(attempts) + } + + /** + * @description (Re)arm the pending-flush timer at a capped exponential + * backoff after a failure, so a persistent fault is retried at a decaying + * cadence instead of hot-looping. Shares the single {@link pendingFlushTimer} + * slot with {@link schedulePendingFlush} (only one flush is ever scheduled). + * The retry's own failure is re-accounted inside {@link flushPendingSingleOps}. + */ + private scheduleFlushRetry(attempt: number): void { + if (this.pendingFlushTimer !== null) return + const backoff = Math.min( + GenerationStore.PENDING_FLUSH_DELAY_MS * 2 ** Math.min(attempt, 6), + GenerationStore.PENDING_FLUSH_RETRY_CAP_MS + ) + this.pendingFlushTimer = setTimeout(() => { + this.pendingFlushTimer = null + // flushPendingSingleOps records the failure + reschedules internally. + void this.flushPendingSingleOps().catch(() => {}) + }, backoff) + const t = this.pendingFlushTimer as unknown as { unref?: () => void } + if (t && typeof t.unref === 'function') t.unref() + } + + /** + * @description Throw if the store has a latched history-durability failure. + * Called at the top of every write commit so a broken durability path fails + * loud and immediately instead of silently accumulating un-durable history. + */ + private assertHistoryDurable(): void { + if (this.pendingFlushError) { + throw new PendingFlushDurabilityError(this.pendingFlushError, this.pendingFlushFailures) + } + } + + /** Schedule a coalesced pending-tier flush (size trigger fires immediately on + * the next microtask; otherwise a {@link PENDING_FLUSH_DELAY_MS} timer). Both + * defer outside the current mutex section so the flush can re-acquire it. A + * failed flush is accounted + rescheduled inside {@link flushPendingSingleOps} + * (latch-on-persistent-failure), never swallowed as a bare log line. */ + private schedulePendingFlush(): void { + if (this.pendingGens.length >= this.pendingFlushThreshold) { + // Failure is recorded + retried inside flushPendingSingleOps. + void Promise.resolve() + .then(() => this.flushPendingSingleOps()) + .catch(() => {}) + return + } + if (this.pendingFlushTimer !== null) return + this.pendingFlushTimer = setTimeout(() => { + this.pendingFlushTimer = null + void this.flushPendingSingleOps().catch(() => {}) + }, GenerationStore.PENDING_FLUSH_DELAY_MS) + // A background flush must not keep the process alive (Node only). + const t = this.pendingFlushTimer as unknown as { unref?: () => void } + if (t && typeof t.unref === 'function') t.unref() + } + + /** Cancel any scheduled pending-tier flush timer. */ + private clearPendingFlushTimer(): void { + if (this.pendingFlushTimer !== null) { + clearTimeout(this.pendingFlushTimer) + this.pendingFlushTimer = null + } + } + + // ========================================================================== + // Committed-generation interval set (run-length representation) + // ========================================================================== + + /** + * @description Record a newly committed generation. `gen` is ALWAYS greater + * than every committed generation (the monotonic `counter++`), so it either + * extends the last interval (the normal contiguous append, `gen === + * lastEnd + 1`) or opens a fresh single-element interval (the first + * generation, or the first commit after a {@link compact} gap). + * @param gen - The just-committed generation. + */ + private appendCommittedGen(gen: number): void { + const ranges = this.committedRanges + const last = ranges[ranges.length - 1] + if (last !== undefined && gen === last[1] + 1) { + last[1] = gen + } else { + ranges.push([gen, gen]) + } + } + + /** @returns The newest committed generation, or `undefined` when none exist. */ + private lastCommittedGen(): number | undefined { + const ranges = this.committedRanges + return ranges.length ? ranges[ranges.length - 1][1] : undefined + } + + /** @returns How many committed generations the interval set represents. */ + private committedCount(): number { + let total = 0 + for (const [start, end] of this.committedRanges) total += end - start + 1 + return total + } + + /** + * @description Yield every committed generation ascending — the exact multiset + * (and order) the old flat `committedGens` array held. + */ + private *committedGensAsc(): IterableIterator { + for (const [start, end] of this.committedRanges) { + for (let g = start; g <= end; g++) yield g + } + } + + /** + * @description Yield committed ∪ pending generations, ascending. Pending + * generations are always greater than every committed one (reserved after the + * last commit; `transact()`/`compact()` flush the pending tier first), so the + * committed-then-pending concatenation is already sorted — identical to the old + * `[...committedGens, ...pendingGens]`. This is the union historical reads + * resolve over so un-flushed single-ops are visible to pins/`asOf`. + */ + private *reservedGensAsc(): IterableIterator { + yield* this.committedGensAsc() + for (const g of this.pendingGens) yield g + } + + /** + * @description Remove every committed generation `<= maxGen`. {@link compact} + * always reclaims an oldest CONTIGUOUS prefix of the committed gens, so this + * walk over the front of {@link committedRanges} is exact: drop whole ranges + * that end at or below `maxGen`, then clip the first surviving range up to + * `maxGen + 1` if `maxGen` falls inside it, and stop. + * @param maxGen - Inclusive upper bound of the reclaimed prefix. + */ + private removeCommittedUpTo(maxGen: number): void { + const ranges = this.committedRanges + let drop = 0 // count of leading ranges fully reclaimed + while (drop < ranges.length) { + const range = ranges[drop] + if (range[1] <= maxGen) { + drop++ // whole range is at or below the cut → reclaim it + } else { + if (range[0] <= maxGen) range[0] = maxGen + 1 // clip the straddling range + break // this range survives; nothing newer is below the cut either + } + } + if (drop > 0) ranges.splice(0, drop) + } + + /** + * @description The largest reserved (committed ∪ pending) generation + * `<= gen` — exactly what `largestAtOrBefore([...committedGens, ...pendingGens], + * gen)` returned, in O(log ranges + scanned pending) instead of materializing + * the union. Pending generations are the newest reserved gens (all greater than + * every committed one), so the largest pending `<= gen`, when one exists, + * dominates every committed gen and is the answer; otherwise the answer is the + * largest committed gen `<= gen`, found by binary-searching {@link committedRanges} + * for the rightmost range whose `start <= gen` and clamping `gen` into it. + * @param gen - The upper bound (inclusive). + * @returns The largest reserved generation `<= gen`, or `undefined` when every + * reserved generation is greater than `gen`. + */ + private largestReservedAtOrBefore(gen: number): number | undefined { + // Pending (ascending, all > every committed): the largest one ≤ gen wins. + let bestPending: number | undefined + for (const p of this.pendingGens) { + if (p <= gen) bestPending = p + else break + } + if (bestPending !== undefined) return bestPending + // No pending ≤ gen → the largest committed gen ≤ gen. Binary-search for the + // rightmost range whose start ≤ gen; the answer is min(gen, that range.end). + const ranges = this.committedRanges + let lo = 0 + let hi = ranges.length // first range index whose start is > gen + while (lo < hi) { + const mid = (lo + hi) >>> 1 + if (ranges[mid][0] <= gen) lo = mid + 1 + else hi = mid + } + if (lo === 0) return undefined // every committed range starts above gen + const [, end] = ranges[lo - 1] + return Math.min(gen, end) + } + // ========================================================================== // Point-in-time resolution // ========================================================================== @@ -508,7 +1694,13 @@ export class GenerationStore { * @param gen - The pinned generation. */ hasCommittedAfter(gen: number): boolean { - const last = this.committedGens[this.committedGens.length - 1] + // Pending single-op generations count: a now() pin must report itself + // historical the moment any later write (committed OR un-flushed) lands, + // else the Model-A "pin doesn't freeze against single-ops" hole reopens. + const last = + this.pendingGens.length > 0 + ? this.pendingGens[this.pendingGens.length - 1] + : this.lastCommittedGen() return last !== undefined && last > gen } @@ -535,28 +1727,336 @@ export class GenerationStore { | { source: 'absent' } | { source: 'record'; metadata: any; vector: any | null } > { - for (const candidate of this.committedGens) { - if (candidate <= gen) continue - const delta = await this.getDelta(candidate) - const touched = kind === 'noun' ? delta.nouns : delta.verbs - if (!touched.has(id)) continue - const record = (await this.storage.readRawObject( - `${GENERATIONS_PREFIX}/${candidate}/prev/${id}.json` - )) as GenerationRecord | null - if (record === null) { - // The record-set exists (candidate is committed) but the id's - // before-image is missing — only possible through external tampering. - throw new Error( - `Generation record missing: ${GENERATIONS_PREFIX}/${candidate}/prev/${id}.json ` + - `(store corrupted or records removed outside compactHistory())` - ) + await this.ensureWindow() + // Snapshot windowLo, then read the chain synchronously (no await between): + // a concurrent commit can only slide the window at the `await` above, so the + // routing decision and the chain it reads are a consistent pair. + const windowLo = this.windowLo + let candidate: number | undefined + if (gen >= windowLo) { + // HOT path — every generation that could hold the before-image + // (`> gen ≥ windowLo`) is inside the resident window, so the recent chain + // is complete for this pin. O(log chain). + const chain = (kind === 'noun' ? this.recentNounChains : this.recentVerbChains).get(id) + if (chain === undefined) return { source: 'current' } + candidate = firstGenerationAfter(chain, gen) + } else { + // COLD path — a deep pin below the window. Serve from the bounded LRU, + // reconstructing (lock-light) on a miss. Caching empty chains too keeps + // repeated deep reads of untouched-in-cold-region ids O(1). + const cold = kind === 'noun' ? this.coldNounChains : this.coldVerbChains + let chain = cold.get(id) + if (chain !== undefined) { + cold.delete(id) // re-insert to mark most-recently-used + cold.set(id, chain) + } else { + chain = await this.reconstructColdChain(kind, id) + cold.set(id, chain) + while (cold.size > this.coldChainLruMax) { + const oldest = cold.keys().next().value + if (oldest === undefined) break + cold.delete(oldest) + } } - if (record.metadata === null && record.vector === null) { - return { source: 'absent' } - } - return { source: 'record', metadata: record.metadata, vector: record.vector } + candidate = firstGenerationAfter(chain, gen) } - return { source: 'current' } + if (candidate === undefined) { + // Nothing after `gen` touched `id`; the live state is its state at `gen`. + return { source: 'current' } + } + const record = await this.readBeforeImage(kind, candidate, id) + if (record === null) { + // The record-set exists (candidate is committed) but the id's + // before-image is missing — only possible through external tampering. + throw new Error( + `Generation record missing: ${GENERATIONS_PREFIX}/${candidate}/prev/${id}.json ` + + `(store corrupted or records removed outside compactHistory())` + ) + } + if (record.metadata === null && record.vector === null) { + return { source: 'absent' } + } + return { source: 'record', metadata: record.metadata, vector: record.vector } + } + + /** + * @description Resolve the FIRST reserved generation after `gen` that touched + * each of `ids`, by kind — the bulk, MUTEX-FREE counterpart of + * {@link resolveAt}'s chain lookup. A SINGLE ascending pass over the reserved + * (committed ∪ pending) generations records `firstAfter[id] = g` for the first + * `g > gen` whose delta touched `id`, stopping early once every id is resolved. + * + * Ids absent from the returned map were never touched after `gen` → their live + * storage state IS their state at `gen` (the {@link resolveAt} `'current'` + * answer). Ids present map to the generation whose before-image + * ({@link readGenerationRecord}) holds their state as of `gen`. + * + * WHY MUTEX-FREE (load-bearing): {@link Brainy.materializeAtGeneration} runs its + * final reconciliation inside {@link snapshotWith} (which HOLDS the commit + * mutex). Resolving ids one-by-one through {@link resolveAt} there would (a) + * re-enter the mutex via {@link ensureWindow} → DEADLOCK, and (b) regress an + * O(R) bulk copy to O(N·R). This method takes no lock and makes one pass, so a + * deep-pin materialize over N ids stays O(R·d̄ + |ids|) with O(R) `getDelta` + * calls. It iterates SNAPSHOTS of the small committed-range list (O(gaps)) and + * the pending list (O(pending)) — NOT a materialized O(R) gen array — so it is + * safe against concurrent commits and bounded in memory. + * + * @param kind - `'noun'` for entities, `'verb'` for relationships. + * @param ids - The ids to resolve. + * @param gen - The pinned generation. + * @returns `id → first reserved generation after `gen` that touched it` for the + * subset of `ids` touched after `gen`. + */ + async resolveManyAt( + kind: 'noun' | 'verb', + ids: Set, + gen: number + ): Promise> { + const firstAfter = new Map() + if (ids.size === 0) return firstAfter + // Snapshot the small range list + pending list (NOT the expanded gens) so the + // pass is immune to a concurrent compaction splice / commit append and stays + // O(gaps + pending) in memory. + const ranges = this.committedRanges.map(([s, e]) => [s, e] as [number, number]) + const pending = [...this.pendingGens] + const record = async (g: number): Promise => { + const delta = await this.getDelta(g) + const touched = kind === 'noun' ? delta.nouns : delta.verbs + for (const id of touched) { + if (ids.has(id) && !firstAfter.has(id)) firstAfter.set(id, g) + } + return firstAfter.size === ids.size + } + for (const [s, e] of ranges) { + if (e <= gen) continue + for (let g = Math.max(s, gen + 1); g <= e; g++) { + if (await record(g)) return firstAfter + } + } + for (const g of pending) { + if (g <= gen) continue + if (await record(g)) return firstAfter + } + return firstAfter + } + + /** + * @description Read the before-image of `id` recorded by generation `gen` — the + * public, MUTEX-FREE accessor {@link Brainy.materializeAtGeneration}'s `copyAt` + * uses after {@link resolveManyAt} hands it the generation. Pairs with + * `resolveManyAt` so the bulk historical copy never touches the commit mutex + * (no deadlock inside {@link snapshotWith}, no per-id chain rebuild). + * @param kind - `'noun'` for entities, `'verb'` for relationships. + * @param gen - The generation whose before-image to read. + * @param id - The id to read. + * @returns The stored {@link GenerationRecord}, or `null` when the record-set + * has no before-image for `id`. + */ + async readGenerationRecord( + kind: 'noun' | 'verb', + gen: number, + id: string + ): Promise { + return this.readBeforeImage(kind, gen, id) + } + + /** + * @description Read the before-image of `id` stored by generation `gen` from + * the right tier: the in-memory pending buffer while the generation is + * un-flushed, or `_generations//prev/.json` on disk once committed. + * Returns `null` when the record-set has no before-image for `id`. + */ + private async readBeforeImage( + kind: 'noun' | 'verb', + gen: number, + id: string + ): Promise { + const pending = this.pendingBuffer.get(gen) + if (pending) { + return (kind === 'noun' ? pending.nouns : pending.verbs).get(id) ?? null + } + const live = (await this.storage.readRawObject( + `${GENERATIONS_PREFIX}/${gen}/prev/${id}.json` + )) as GenerationRecord | null + if (live) return live + // Two-tier: the packed tier serves folded generations (live-tier-wins). + if (this.segments?.hasGeneration(gen)) { + return (await this.segments.readRecord(gen, kind, id)) as GenerationRecord | null + } + return null + } + + /** + * @description Build the resident hot-tail window — the per-id chains + * ({@link recentNounChains}/{@link recentVerbChains}) and the inverse index + * ({@link windowNounDeltas}/{@link windowVerbDeltas}) over the last + * {@link recentWindowGenerations} reserved generations, `[windowLo, counter]`. + * Built once, lazily, under the commit mutex (so no concurrent commit mutates + * {@link committedRanges}/pending mid-build); thereafter maintained + * incrementally by {@link extendChains} and invalidated by {@link compact} / + * reopen. Concurrent first-callers share one build. O(W) `getDelta` reads the + * first time (only the window, NOT all of history); O(1) afterwards. Deep pins + * below `windowLo` are served by {@link reconstructColdChain} on demand. + */ + private async ensureWindow(): Promise { + if (this.windowReady) return + if (!this.windowBuilding) { + this.windowBuilding = this.withMutex(async () => { + if (this.windowReady) return + this.recentNounChains.clear() + this.recentVerbChains.clear() + this.windowNounDeltas.clear() + this.windowVerbDeltas.clear() + // The window is the newest W reserved generations, clamped above the + // compaction horizon (gens ≤ horizon are reclaimed — never resident). + this.windowLo = Math.max(this.horizonGen + 1, this.counter - this.recentWindowGenerations + 1) + // reservedGensAsc (committed ∪ pending) is ascending, so chains accrue in + // order and un-flushed single-ops in-window are indexed too. + for (const g of this.reservedGensAsc()) { + if (g < this.windowLo) continue + const delta = await this.getDelta(g) + const nouns = [...delta.nouns] + const verbs = [...delta.verbs] + for (const id of nouns) appendToChain(this.recentNounChains, id, g) + for (const id of verbs) appendToChain(this.recentVerbChains, id, g) + this.windowNounDeltas.set(g, nouns) + this.windowVerbDeltas.set(g, verbs) + } + this.windowReady = true + this.windowBuilding = null + }) + } + await this.windowBuilding + } + + /** + * @description Reconstruct the FULL per-id chain for a deep pin (`gen < + * windowLo`) — every generation that touched `id`, ascending. Built from the + * persisted deltas BELOW the window plus the resident in-window tail, then + * cached in the cold LRU by the caller. + * + * LOCK-LIGHT — deliberately does NOT hold the commit mutex (a deep read must + * never stall writers). Correctness without the lock: + * - The recent tail and `windowLo` are snapshotted SYNCHRONOUSLY up front, so + * even if the window slides during the (awaited) cold scan, the chain is the + * one consistent with the snapshot. New writes during the scan land ABOVE the + * snapshot and cannot change any deep pin's first-after answer; the cold + * entry is also dropped by {@link extendChains} the moment `id` is next + * touched, so it is never served stale. + * - The cold scan iterates a SNAPSHOT of the small committed-range list + * (O(gaps) memory, immune to a concurrent compaction splice). + * - A generation concurrently reclaimed by compaction surfaces as a missing + * delta. Such a gen is always `≤ minPinned` (compaction never reclaims above + * the lowest live pin) and therefore NEVER a live pin's answer (`first-after + * > pin ≥ minPinned`), so it is skipped, not raised. A missing delta ABOVE + * `minPinned` is genuine corruption and still throws. + * + * @param kind - `'noun'` for entities, `'verb'` for relationships. + * @param id - The id whose chain to reconstruct. + * @returns The id's full ascending generation chain (`[]` when never touched). + */ + private async reconstructColdChain(kind: 'noun' | 'verb', id: string): Promise { + // Synchronous, consistent snapshot of the window state. + const windowLo = this.windowLo + const recentTail = (kind === 'noun' ? this.recentNounChains : this.recentVerbChains).get(id) + const recentSnapshot = recentTail ? [...recentTail] : [] + const ranges = this.committedRanges.map(([s, e]) => [s, e] as [number, number]) + const pendingCold = this.pendingGens.filter((g) => g < windowLo) + + const cold: number[] = [] + const scan = async (g: number): Promise => { + let delta: { nouns: Set; verbs: Set } + try { + delta = await this.getDelta(g) + } catch (err) { + // Concurrent-compaction tolerance: a reclaimed gen (≤ minPinned, never a + // live pin's answer) is skipped; a missing gen above the lowest pin is + // real corruption and rethrows. + if (g <= this.minPinnedGeneration()) return + throw err + } + if ((kind === 'noun' ? delta.nouns : delta.verbs).has(id)) cold.push(g) + } + + for (const [s, e] of ranges) { + if (s >= windowLo) break // ascending → nothing newer is below the window + const hi = Math.min(e, windowLo - 1) + for (let g = s; g <= hi; g++) await scan(g) + } + // Cold pending gens (only when more than W writes are buffered, i.e. tests + // with a small window) sit between the committed gens and the window. + for (const g of pendingCold) await scan(g) + + // Cold gens (< windowLo) then the resident in-window tail (≥ windowLo) — both + // ascending and disjoint, so the concatenation is the full ascending chain. + return recentSnapshot.length ? cold.concat(recentSnapshot) : cold + } + + /** + * Record that generation `gen` (the newest) touched these ids and advance the + * hot-tail window. Keeps the resident window current after a commit without a + * rebuild. No-op until the window is built (the eventual build reads `gen` from + * {@link reservedGensAsc}). + * + * Three steps, all in-memory and I/O-FREE: + * 1. Append `gen` to each touched id's recent chain, and DROP each touched id + * from the cold LRU — a freshly-touched id's cached deep chain is now + * incomplete, so it must be reconstructed on the next deep read (else a + * deep pin whose first-after just became `gen` would wrongly read `current`). + * 2. Record the window inverse index entry `gen → touched ids` (the slide-trim + * driver). + * 3. Slide `windowLo` forward to `max(horizon+1, gen-W+1)`, front-trimming the + * chains of exactly the ids leaving the window using the inverse index — NO + * `getDelta`, so the write path never reads `tx.json` on a slide regardless + * of W vs the delta-cache size. + */ + private extendChains(gen: number, nouns: Iterable, verbs: Iterable): void { + if (!this.windowReady) return + const nounArr = [...nouns] + const verbArr = [...verbs] + for (const id of nounArr) { + appendToChain(this.recentNounChains, id, gen) + this.coldNounChains.delete(id) + } + for (const id of verbArr) { + appendToChain(this.recentVerbChains, id, gen) + this.coldVerbChains.delete(id) + } + this.windowNounDeltas.set(gen, nounArr) + this.windowVerbDeltas.set(gen, verbArr) + + const newLo = Math.max(this.horizonGen + 1, gen - this.recentWindowGenerations + 1) + if (newLo <= this.windowLo) return + // Collect the ids leaving the window from the inverse index (O(slid·d̄)), then + // front-trim their chains to the new low watermark (each id once). + const trimNouns = new Set() + const trimVerbs = new Set() + for (let g = this.windowLo; g < newLo; g++) { + const n = this.windowNounDeltas.get(g) + if (n) for (const id of n) trimNouns.add(id) + this.windowNounDeltas.delete(g) + const v = this.windowVerbDeltas.get(g) + if (v) for (const id of v) trimVerbs.add(id) + this.windowVerbDeltas.delete(g) + } + for (const id of trimNouns) dropChainFront(this.recentNounChains, id, newLo) + for (const id of trimVerbs) dropChainFront(this.recentVerbChains, id, newLo) + this.windowLo = newLo + } + + /** Drop the resident window + cold LRU so the next read rebuilds from the + * surviving generations (called after compaction removes record-sets / on + * reopen). */ + private invalidateChains(): void { + this.windowReady = false + this.windowBuilding = null + this.recentNounChains.clear() + this.recentVerbChains.clear() + this.windowNounDeltas.clear() + this.windowVerbDeltas.clear() + this.coldNounChains.clear() + this.coldVerbChains.clear() + this.windowLo = 0 } /** @@ -569,7 +2069,7 @@ export class GenerationStore { async changedBetween(fromGen: number, toGen: number): Promise { const nouns = new Set() const verbs = new Set() - for (const gen of this.committedGens) { + for (const gen of this.reservedGensAsc()) { if (gen <= fromGen || gen > toGen) continue const delta = await this.getDelta(gen) for (const id of delta.nouns) nouns.add(id) @@ -600,7 +2100,7 @@ export class GenerationStore { toGen: number ): Promise { const result: number[] = [] - for (const gen of this.committedGens) { + for (const gen of this.reservedGensAsc()) { if (gen <= fromGen || gen > toGen) continue const delta = await this.getDelta(gen) const touched = kind === 'noun' ? delta.nouns : delta.verbs @@ -637,9 +2137,10 @@ export class GenerationStore { /** * @description Resolve a wall-clock timestamp to the generation that was - * committed at-or-before it, via the tx-log. Resolution granularity is - * transact commits — single-operation writes between commits are not - * individually addressable (documented 8.0 history granularity). + * committed at-or-before it, via the tx-log. Under Model-B every write — + * `transact()` AND single-op — has its own tx-log entry, so resolution is + * per-write (the tx-log includes un-flushed single-op generations too, so + * `asOf(Date)` is consistent with the rest of the temporal surface). * @param timestampMs - Milliseconds since epoch. * @returns The resolved generation (`0` when `timestampMs` predates the * first commit) and the matched tx-log entry when one exists. @@ -677,6 +2178,16 @@ export class GenerationStore { } if (entry.generation <= this.committed) entries.push(entry) } + // Include un-flushed single-op generations so the tx-log is consistent with + // the rest of the temporal surface (`since`/`diff`/`asOf`/`changedBetween` + // already resolve over the pending tier). Reads are thus flush-agnostic — + // whether the async group-commit has run yet never changes results, only + // crash durability. Pending single-ops are always newer than every + // committed generation, so they extend the ascending list. + for (const gen of this.pendingGens) { + const buf = this.pendingBuffer.get(gen) + if (buf) entries.push({ generation: gen, timestamp: buf.timestamp }) + } return entries } @@ -687,24 +2198,53 @@ export class GenerationStore { * @param gen - The pinned generation. */ async commitTimestampAtOrBefore(gen: number): Promise { - for (let i = this.committedGens.length - 1; i >= 0; i--) { - const candidate = this.committedGens[i] - if (candidate > gen) continue - const delta = await this.getDelta(candidate) - return delta.timestamp - } - return null + // The largest reserved (committed ∪ pending) generation ≤ gen, found in + // O(log ranges) over the interval set (not an O(database-age) backward + // scan). Pending single-op generations carry timestamps too. + const candidate = this.largestReservedAtOrBefore(gen) + if (candidate === undefined) return null + const delta = await this.getDelta(candidate) + return delta.timestamp } private async getDelta( gen: number - ): Promise<{ nouns: Set; verbs: Set; timestamp: number }> { + ): Promise<{ nouns: Set; verbs: Set; timestamp: number; bytes: number }> { const cached = this.deltaCache.get(gen) if (cached) return cached + // A pending (un-flushed) generation has no tx.json on disk yet — derive its + // delta from the in-memory before-image buffer. Not cached: the bounded + // cache is for the disk path; the buffer is the source of truth here. + // bytes = 0: pending generations are not on disk, so they do not count + // toward the on-disk history byte budget ({@link historyBytes}). + const pending = this.pendingBuffer.get(gen) + if (pending) { + return { + nouns: new Set(pending.nouns.keys()), + verbs: new Set(pending.verbs.keys()), + timestamp: pending.timestamp, + bytes: 0 + } + } const delta = (await this.storage.readRawObject( `${GENERATIONS_PREFIX}/${gen}/tx.json` )) as GenerationDelta | null if (delta === null) { + // Two-tier read (D1+D3): not in the live tier → the packed tier. + // Live-tier-wins ordering (a crash mid-fold leaves a duplicate, never + // a gap), so the segment lookup runs only after the live miss. + const packed = await this.segments?.readDelta(gen) + if (packed) { + const d = packed.delta as GenerationDelta + const entry = { + nouns: new Set(d.nouns), + verbs: new Set(d.verbs), + timestamp: packed.timestamp, + bytes: d.bytes ?? 0 + } + this.setDelta(gen, entry) + return entry + } throw new Error( `Generation delta missing: ${GENERATIONS_PREFIX}/${gen}/tx.json ` + `(store corrupted or records removed outside compactHistory())` @@ -713,52 +2253,268 @@ export class GenerationStore { const entry = { nouns: new Set(delta.nouns), verbs: new Set(delta.verbs), - timestamp: delta.timestamp + timestamp: delta.timestamp, + bytes: delta.bytes ?? 0 } - this.deltaCache.set(gen, entry) + this.setDelta(gen, entry) return entry } + /** + * @description Insert a delta into {@link deltaCache}, evicting the oldest + * entries (FIFO, by insertion order) past {@link deltaCacheMax}. Eviction is + * safe: {@link getDelta} re-reads any evicted delta from storage on demand. + */ + private setDelta( + gen: number, + entry: { nouns: Set; verbs: Set; timestamp: number; bytes: number } + ): void { + this.deltaCache.set(gen, entry) + while (this.deltaCache.size > this.deltaCacheMax) { + const oldest = this.deltaCache.keys().next().value + if (oldest === undefined) break + this.deltaCache.delete(oldest) + } + } + // ========================================================================== // Compaction // ========================================================================== /** - * @description Reclaim generation record-sets past the retention policy. - * A generation directory `N` may be removed only when `N` is at or below - * **every** live pin (deleting `N` can only break readers pinned *below* - * `N`, because resolution reads before-images from generations strictly - * greater than the pin). With both retention options supplied, a - * generation must satisfy both to be eligible. + * @description Total serialized bytes of the ON-DISK generational history — + * the sum of every committed generation's recorded `bytes`. Backs the + * `maxBytes` and adaptive retention caps. O(1) after the first call: the + * total is computed by ONE walk over committed deltas, then maintained + * incrementally at every commit (+bytes) and reclaim (−bytes) and dropped on + * a wholesale state replacement (restore). Without the running total, the + * adaptive auto-compaction on every flush() re-walked the ENTIRE history — + * O(committed generations) file reads per flush past the delta-cache bound — + * which is how a 70k-generation production brain turned every write into a + * full-tail scan (SELF-GENERATIONS-GROWTH). Pending (un-flushed) generations + * are excluded (they are not on disk). + * @returns The total on-disk history byte count. + */ + async historyBytes(): Promise { + if (this.historyBytesTotal !== null) { + return this.historyBytesTotal + } + let total = 0 + for (const gen of this.committedGensAsc()) { + total += (await this.getDelta(gen)).bytes + } + this.historyBytesTotal = total + return total + } + + /** + * @description Reclaim generation record-sets per the retention CAPS. A + * generation is removed only when it is at or below **every** live pin + * (deleting `N` can only break readers pinned *below* `N`, because resolution + * reads before-images from generations strictly greater than the pin) — pins + * are ALWAYS exempt. Among the unpinned generations, the OLDEST are reclaimed + * while **ANY** supplied cap is exceeded: * - * @param options - Retention policy (see {@link CompactHistoryOptions}). + * - `maxGenerations` — reclaim while the committed-generation count exceeds it. + * - `maxAge` — reclaim generations older than `now − maxAge`. + * - `maxBytes` — reclaim while total on-disk history bytes exceed it. + * + * Because reclamation is oldest-first and every cap is monotone in age, once + * the oldest unpinned generation violates no cap, no newer one does either, so + * the scan stops. With no options, every unpinned generation is reclaimed. + * + * @param options - Retention caps (see {@link CompactHistoryOptions}). * @returns Count of removed record-sets and the new horizon. */ + /** + * @description The REPACKER (D1+D3+repacking): fold cold live-tier + * generations into sealed segments — re-representation, never deletion. + * Every record and delta stays readable (asOf/chains unchanged); the + * per-generation directories are deleted only AFTER their segment is + * durable (crash between = duplicate representation, resolved + * live-tier-wins by every reader; never a gap). This is the transform that + * takes a 70k-file history to tens of segment files, and the ONLY history + * transform permitted under the archival profile. + * + * Folds oldest-first, contiguous from the packed boundary, in batches, and + * stops at the live window ({@link GenerationStore.REPACK_LIVE_WINDOW}) + * or when `timeBudgetMs` is spent — an early stop is a consistent prefix; + * the next pass resumes. + */ + async repackHistory(options?: { timeBudgetMs?: number; batchGenerations?: number }): Promise<{ + foldedGenerations: number + segmentsCreated: number + }> { + if (!this.segments) return { foldedGenerations: 0, segmentsCreated: 0 } + const segments = this.segments + return this.withMutex(async () => { + const deadline = + options?.timeBudgetMs !== undefined ? Date.now() + options.timeBudgetMs : undefined + const batchSize = options?.batchGenerations ?? 512 + const coldCeiling = this.committed - GenerationStore.REPACK_LIVE_WINDOW + const packedThrough = + segments.segments().length > 0 + ? segments.segments()[segments.segments().length - 1].lastGeneration + : 0 + + // Cold, unpacked, committed generations — ascending, contiguous scan. + const eligible: number[] = [] + for (const gen of this.committedGensAsc()) { + if (gen > coldCeiling) break + if (gen <= packedThrough) continue // already packed (dup fold barred) + if (this.pendingBuffer.has(gen)) continue // un-flushed = live by definition + eligible.push(gen) + } + + let folded = 0 + let segmentsCreated = 0 + for (let i = 0; i < eligible.length; i += batchSize) { + if (deadline !== undefined && Date.now() >= deadline) break + const batch = eligible.slice(i, i + batchSize) + const foldInput: FoldGeneration[] = [] + for (const gen of batch) { + const delta = (await this.storage.readRawObject( + `${GENERATIONS_PREFIX}/${gen}/tx.json` + )) as GenerationDelta | null + if (delta === null) { + // Already folded by a prior crashed pass whose dirs were removed, + // or damage — getDelta's two-tier read decides which, loudly, + // when someone asks. Skip; never fold a generation we cannot read. + continue + } + const records: FoldGeneration['records'] = [] + for (const [kind, ids] of [ + ['noun', delta.nouns] as const, + ['verb', delta.verbs] as const + ]) { + for (const id of ids) { + const record = await this.storage.readRawObject( + `${GENERATIONS_PREFIX}/${gen}/prev/${id}.json` + ) + if (record) records.push({ kind, id, record }) + } + } + foldInput.push({ generation: gen, timestamp: delta.timestamp, delta, records }) + } + if (foldInput.length === 0) continue + await segments.fold(foldInput) + segmentsCreated++ + // Segment + manifest durable → the live copies retire. + for (const g of foldInput) { + await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`) + } + folded += foldInput.length + } + if (folded > 0) { + prodLog.info( + `[GenerationStore] repacked ${folded} cold generation(s) into ${segmentsCreated} segment(s) — history preserved, file count reduced` + ) + } + return { foldedGenerations: folded, segmentsCreated } + }) + } + async compact(options?: CompactHistoryOptions): Promise { return this.withMutex(async () => { const minPinned = this.minPinnedGeneration() - const retainGenerations = options?.retainGenerations ?? 0 - const countWatermark = this.committed - retainGenerations - const timeCutoff = - options?.retainMs !== undefined ? Date.now() - options.retainMs : Infinity + const maxGenerations = options?.maxGenerations + const maxAge = options?.maxAge + const maxBytes = options?.maxBytes + const ageCutoff = maxAge !== undefined ? Date.now() - maxAge : undefined + // Bounded maintenance pass (8.9.0): stop reclaiming once the budget is + // spent. Safe mid-loop — reclamation is oldest-first, so an early stop + // leaves a consistent contiguous prefix and the next pass resumes. + const deadline = + options?.timeBudgetMs !== undefined ? Date.now() + options.timeBudgetMs : undefined + const noCaps = + maxGenerations === undefined && maxAge === undefined && maxBytes === undefined + // Running totals the caps are evaluated against; updated as we reclaim. + let remainingCount = this.committedCount() + let remainingBytes = maxBytes !== undefined ? await this.historyBytes() : 0 + + // Snapshot the committed gens ascending so the reclaim loop iterates safely + // while removeCommittedUpTo mutates the interval set afterwards. const removed: number[] = [] - for (const gen of [...this.committedGens]) { - if (gen > countWatermark) continue - if (gen > minPinned) continue - if (timeCutoff !== Infinity) { - const delta = await this.getDelta(gen) - if (delta.timestamp >= timeCutoff) continue + for (const gen of [...this.committedGensAsc()]) { + // Pins are always exempt: never reclaim a generation a live pin needs. + if (gen > minPinned) break // committedGensAsc ascending → nothing newer is eligible either + if (deadline !== undefined && Date.now() >= deadline) break // budget spent — resume next pass + const delta = await this.getDelta(gen) + if (!noCaps) { + const violatesCount = maxGenerations !== undefined && remainingCount > maxGenerations + const violatesBytes = maxBytes !== undefined && remainingBytes > maxBytes + const violatesAge = ageCutoff !== undefined && delta.timestamp < ageCutoff + // Oldest-first: once the oldest unpinned gen trips no cap, none newer do. + if (!violatesCount && !violatesBytes && !violatesAge) break } + const genBytes = maxBytes !== undefined ? delta.bytes : 0 + + // Temporal-blob contract: resolve the content-blob hashes this + // generation's record-set references BEFORE deleting it. New + // generations carry the multiset in their persisted delta (empty + // array when none — distinguishing "new format, no blobs" from a + // pre-contract delta); legacy generations fall back to reading the + // records themselves. Skipped entirely on non-blob-aware storage. + let blobHashes: string[] | undefined + if (this.storage.releaseHistoryBlobReferences) { + const rawDelta = (await this.storage.readRawObject( + `${GENERATIONS_PREFIX}/${gen}/tx.json` + )) as GenerationDelta | null + blobHashes = rawDelta?.blobHashes + if (blobHashes === undefined && this.storage.extractBlobHashesFromRecords) { + blobHashes = this.storage.extractBlobHashesFromRecords( + await this.readGenerationRecords(gen) + ) + } + } + await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${gen}`) this.deltaCache.delete(gen) + if (this.historyBytesTotal !== null) { + this.historyBytesTotal -= delta.bytes + } + + // AFTER the record-set is gone (over-count-only crash ordering): + // release its history references and reclaim any blob left with zero + // live AND zero history references — the system's one byte-reclaim point. + if (blobHashes && blobHashes.length > 0 && this.storage.releaseHistoryBlobReferences) { + await this.storage.releaseHistoryBlobReferences(blobHashes) + } + + remainingCount-- + remainingBytes -= genBytes removed.push(gen) } if (removed.length > 0) { - const removedSet = new Set(removed) - this.committedGens = this.committedGens.filter((gen) => !removedSet.has(gen)) - this.horizonGen = Math.max(this.horizonGen, ...removed) + // The reclaim loop walks committedGensAsc() oldest-first and stops at the + // first survivor, so `removed` is the oldest CONTIGUOUS prefix of the + // committed gens — every committed gen ≤ max(removed) was reclaimed, which + // is exactly what removeCommittedUpTo strips. Guard that prefix invariant. + const highestRemoved = Math.max(...removed) + const countBefore = this.committedCount() + this.removeCommittedUpTo(highestRemoved) + if (this.committedCount() !== countBefore - removed.length) { + throw new Error( + `[GenerationStore] compaction invariant violated: reclaimed ${removed.length} ` + + `generation(s) but committedCount dropped by ${countBefore - this.committedCount()} ` + + `(the reclaimed set is not the oldest contiguous prefix)` + ) + } + // Reclaimed generations leave the per-id chains stale → rebuild on next read. + this.invalidateChains() + this.horizonGen = Math.max(this.horizonGen, highestRemoved) + // Packed-tier reclaim (D3): a packed generation's bytes live in a + // sealed segment — removeRawPrefix above was a no-op for it. Drop + // WHOLE segments now fully below the horizon; a partially-reclaimed + // segment keeps its bytes until the boundary passes it (the frozen + // partial-segments-wait rule; logical reclamation above still holds — + // the generations left committedRanges and asOf below the horizon + // throws regardless). + if (this.segments) { + await this.segments.dropSegmentsBelow(this.horizonGen + 1) + } const manifest: GenerationManifest = { version: 1, generation: this.committed, @@ -788,6 +2544,15 @@ export class GenerationStore { async reopenAfterRestore(floorGeneration: number): Promise { await this.withMutex(async () => { this.deltaCache.clear() + // The running history-byte total describes the REPLACED store — drop it; + // the next historyBytes() re-seeds with one walk over the new state. + this.historyBytesTotal = null + // A wholesale state replacement invalidates any buffered single-op + // history — discard the pending tier (its live writes are gone with the + // replaced store). + this.clearPendingFlushTimer() + this.pendingGens = [] + this.pendingBuffer.clear() this.opened = false // open() re-reads counter/manifest and re-registers the bump hook. await this.open() @@ -803,10 +2568,44 @@ export class GenerationStore { // ========================================================================== /** - * Restore the before-images of an uncommitted generation to the canonical - * entity paths, then remove its record directory. Restores are idempotent - * (writing identical bytes / deleting already-absent files), so recovery - * is safe at every crash point of the commit protocol. + * Recover one uncommitted (crashed) generation, branching on whether it is a + * single-op group-commit generation (drop-without-restore) or a `transact()` + * generation (restore before-images). The decision reads the generation's + * `tx.json` `groupCommit` flag: + * + * - **`groupCommit: true` → DROP** the directory, never restore. The single-op + * write already mutated canonical storage and was acknowledged to the caller + * before the flush; restoring its before-images would silently revert an + * acknowledged user write (the data-corruption trap). Only the *history* of + * the crashed flush window is lost — live data is correct as-is. + * - **absent/`false` (a transact generation) → RESTORE** then drop (the + * existing atomic-rollback path). + * - **`tx.json` unreadable/missing → DROP** (safe for both): a transact + * fsyncs `tx.json` BEFORE it executes, so an unreadable delta means the + * execute never ran → live storage is unchanged → dropping is a no-op on + * live state, while a partial group-commit must be dropped anyway. + */ + private async recoverUncommittedGeneration(gen: number): Promise { + const dir = `${GENERATIONS_PREFIX}/${gen}` + const delta = (await this.storage.readRawObject(`${dir}/tx.json`)) as GenerationDelta | null + if (delta === null || delta.groupCommit === true) { + // Drop-without-restore (group-commit, or an indeterminate partial dir). + await this.storage.removeRawPrefix(dir) + prodLog.warn( + `[GenerationStore] dropped uncommitted ${ + delta === null ? 'indeterminate' : 'group-commit' + } generation ${gen} (live data left intact; history of that flush window discarded)` + ) + return + } + await this.rollBackUncommittedGeneration(gen) + } + + /** + * Restore the before-images of an uncommitted `transact()` generation to the + * canonical entity paths, then remove its record directory. Restores are + * idempotent (writing identical bytes / deleting already-absent files), so + * recovery is safe at every crash point of the transact commit protocol. */ private async rollBackUncommittedGeneration(gen: number): Promise { const dir = `${GENERATIONS_PREFIX}/${gen}` @@ -858,6 +2657,22 @@ export class GenerationStore { } } +/** + * @description Approximate serialized byte size of a record or delta, for the + * Model-B retention byte caps (`maxBytes` / adaptive). The JSON string length + * (chars ≈ bytes) is a deliberately cheap soft estimate — retention is a budget, + * not exact accounting — and avoids a storage size API. Returns 0 on any + * serialization failure (e.g. a cyclic structure, which records never are). + * @param obj - The record or delta about to be persisted. + */ +function serializedBytes(obj: unknown): number { + try { + return JSON.stringify(obj)?.length ?? 0 + } catch { + return 0 + } +} + /** * @description Extract the generation number from a record path of the form * `_generations//...`. Returns `null` for paths that don't match. @@ -879,3 +2694,48 @@ function recordIdFromPath(path: string): string | null { const match = /[/\\]prev[/\\]([^/\\]+)\.json$/.exec(path) return match ? match[1] : null } + +/** + * @description Append `gen` to an id's ascending history chain in `chains`, + * creating the chain on first touch. Callers append in ascending generation + * order, so the chain stays sorted without a re-sort. + */ +function appendToChain(chains: Map, id: string, gen: number): void { + const chain = chains.get(id) + if (chain === undefined) { + chains.set(id, [gen]) + } else if (chain[chain.length - 1] !== gen) { + chain.push(gen) + } +} + +/** + * @description Drop the leading entries of an id's ascending chain that fall + * below `lo` (generations that slid out of the hot-tail window), deleting the + * chain entirely when it empties. Used by the O(d̄) window slide in + * {@link GenerationStore.extendChains}. Tolerates an already-trimmed front. + */ +function dropChainFront(chains: Map, id: string, lo: number): void { + const chain = chains.get(id) + if (chain === undefined) return + let i = 0 + while (i < chain.length && chain[i] < lo) i++ + if (i === chain.length) chains.delete(id) + else if (i > 0) chain.splice(0, i) +} + +/** + * @description Binary-search an ascending generation chain for the FIRST entry + * strictly greater than `gen` (the generation whose before-image holds the + * value as of `gen`). Returns `undefined` when every entry is ≤ `gen`. O(log n). + */ +function firstGenerationAfter(chain: number[], gen: number): number | undefined { + let lo = 0 + let hi = chain.length // upper-bound search over [lo, hi) + while (lo < hi) { + const mid = (lo + hi) >>> 1 + if (chain[mid] > gen) hi = mid + else lo = mid + 1 + } + return lo < chain.length ? chain[lo] : undefined +} diff --git a/src/db/types.ts b/src/db/types.ts index 31420b84..4c8a4957 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -116,6 +116,16 @@ export interface TransactOptions { * record is staged. */ ifAtGeneration?: number + /** + * Budget (ms) for the atomic apply phase. When omitted, the budget SCALES + * with the batch: `max(30 000, opCount × 2 000)` — production imports on + * network-attached disks measure ~2 s per operation, so a flat 30 s budget + * silently capped honest bulk work at ~15 operations. A tripped budget + * rolls the whole batch back and throws a retryable + * `TransactionTimeoutError` naming the operation it stopped at, the batch + * size, and the elapsed/budget times. + */ + timeoutMs?: number } /** @@ -138,21 +148,43 @@ export interface TransactReceipt { // ============================================================================ /** - * @description Options for `brain.compactHistory()`. When both options are - * supplied a generation must satisfy *both* retention rules to be reclaimed - * (the stricter retention wins). With no options, every generation not + * @description Options for `brain.compactHistory()` — explicit retention + * **caps** (Model-B `retention` knob). Each supplied field is an upper bound: + * the oldest unpinned generations are reclaimed while **ANY** supplied cap is + * exceeded (predictable ops — the more you constrain, the more is reclaimed). + * Live `Db` pins are ALWAYS exempt. With no options, every generation not * protected by a live pin is eligible. + * + * (8.0 rename: the pre-release `retainGenerations`/`retainMs` *floors* became + * `maxGenerations`/`maxAge` *caps* — the `max*` naming signals the semantics.) */ export interface CompactHistoryOptions { /** - * Keep at least this many of the most recent committed generations - * (`0` = keep none beyond what live pins protect). + * Keep at most this many of the most recent committed generations — reclaim + * the oldest unpinned generations while the count exceeds it (`0` = reclaim + * everything not protected by a live pin). */ - retainGenerations?: number + maxGenerations?: number /** - * Keep every generation committed within the last `retainMs` milliseconds. + * Keep only generations committed within the last `maxAge` milliseconds — + * reclaim unpinned generations older than the window. */ - retainMs?: number + maxAge?: number + /** + * Keep total generational-history bytes at or below this — reclaim the oldest + * unpinned generations while the total exceeds it. Byte accounting is the sum + * of each surviving generation's serialized record set (`GenerationDelta.bytes`). + */ + maxBytes?: number + /** + * Stop reclaiming after this many milliseconds even if caps are still + * exceeded (8.9.0). Compaction is maintenance — a bounded pass keeps + * `close()` (and any explicit maintenance window) from stalling on a large + * backlog; the next pass resumes where this one stopped (reclamation is + * oldest-first, so an early stop is always a consistent prefix). Unset = + * run to completion. + */ + timeBudgetMs?: number } /** @@ -170,17 +202,46 @@ export interface CompactHistoryResult { horizon: number } +/** + * @description Result of `brain.historyStats()` — the read-only generational + * history footprint, for fleet audits and ops doors. A pool operator runs this + * per brain to size retention exposure (how much MVCC history each brain + * carries and under which policy) without touching any data. + */ +export interface HistoryStats { + /** Committed generation record-sets currently on disk. */ + generations: number + /** Total on-disk history bytes across those record-sets. */ + bytes: number + /** Oldest committed generation still on disk (null when history is empty). */ + oldestGeneration: number | null + /** Newest committed generation (null when history is empty). */ + newestGeneration: number | null + /** Commit timestamp (ms) of the oldest on-disk generation. */ + oldestTimestamp: number | null + /** Commit timestamp (ms) of the newest on-disk generation. */ + newestTimestamp: number | null + /** Compaction horizon — generations below it were reclaimed. */ + horizon: number + /** The effective retention mode this brain runs under. */ + retentionMode: 'all' | 'adaptive' | 'explicit' + /** + * The adaptive byte budget in force (coordinator-driven or the local + * free-memory probe); null under 'all' or explicit caps. + */ + effectiveBudgetBytes: number | null +} + // ============================================================================ // Db surfaces // ============================================================================ /** * @description Result of `db.since(priorDb)` — the ids touched by committed - * transactions in the generation interval `(priorDb.generation, - * db.generation]`. Granularity note: single-operation writes performed - * outside `transact()` bump the generation counter but do not produce - * generation records, so they are not reflected here (documented 8.0 - * history granularity — see `docs/ADR-001-generational-mvcc.md`). + * generations in the interval `(priorDb.generation, db.generation]`. Under + * Model-B EVERY write is its own generation, so single-operation writes + * (`add`/`update`/`remove`/`relate` outside `transact()`) ARE reflected here, + * exactly like `transact()` commits. */ export interface ChangedIds { /** Entity (noun) ids touched in the interval, sorted. */ @@ -219,7 +280,8 @@ export interface DiffResult { * @description One version of a single entity or relationship in its * {@link EntityHistory} — the materialized state at a committed generation that * changed it. `value` is `null` when the id did not exist at that version - * (created-after / removed). Granularity is `transact()` commits. + * (created-after / removed). Granularity is per-write (every `transact()` AND + * single-op write is its own generation — Model-B). */ export interface HistoryVersion { /** The committed generation that produced this version. */ @@ -271,9 +333,11 @@ export interface GenerationRecord { /** * @description The per-generation delta persisted at - * `_generations//tx.json` — written and fsynced *before* any operation of - * the transaction executes, so crash recovery always knows exactly which ids - * to restore from before-images. + * `_generations//tx.json`. For a `transact()` generation it is written and + * fsynced *before* any operation of the transaction executes, so crash + * recovery always knows exactly which ids to restore from before-images. For a + * single-operation (Model-B) generation it is written at group-commit flush + * time with `groupCommit: true` — see that flag. */ export interface GenerationDelta { /** The generation this delta belongs to. */ @@ -286,6 +350,36 @@ export interface GenerationDelta { nouns: string[] /** Relationship ids touched by this generation. */ verbs: string[] + /** + * Content-blob hashes referenced by this generation's before-image records + * (a MULTISET — one entry per referencing record occurrence), captured at + * persist time so compaction can release the exact history references it + * reclaims without re-reading the records. Always present (possibly empty) + * on deltas written under the temporal-blob contract; absent only on + * pre-contract generations, for which compaction falls back to reading the + * record-set itself. + */ + blobHashes?: string[] + /** + * `true` for a Model-B single-operation generation persisted by the async + * group-commit flush (`GenerationStore.flushPendingSingleOps`). It marks the + * **drop-without-restore** crash-recovery contract: the live write already + * mutated canonical storage and was acknowledged to the caller BEFORE the + * flush, so an uncommitted (crashed-mid-flush) generation with this flag set + * must be DISCARDED — its before-images must NEVER be restored, or recovery + * would silently revert acknowledged user writes. Absent/`false` on a + * `transact()` generation, whose before-images ARE the durable undo log and + * are restored on rollback (the before-image + execute are one atomic unit). + */ + groupCommit?: boolean + /** + * Serialized byte size of this generation's record set (the `prev/.json` + * before-images plus this delta), recorded at write time so `maxBytes` / + * adaptive retention can bound total history without a storage size API. Read + * back lazily by `GenerationStore.historyBytes()`. Absent on generations + * written before byte accounting existed (treated as 0). + */ + bytes?: number } /** @@ -346,6 +440,24 @@ export interface GenerationStorage { /** Durability barrier: fsync the given object paths (no-op in memory). */ syncRawObjects(paths: string[]): Promise + /** + * OPTIONAL transaction durability barrier. `commitTransaction` calls + * {@link beginWriteBarrier} immediately before running the planned operations + * and {@link flushWriteBarrier} immediately after — BEFORE the generation + * counter and manifest are advanced. An adapter whose canonical writes are + * not synchronously durable (the filesystem adapter's tmp+rename lands in the + * page cache) MUST implement these so a transaction reported "committed" is + * durable before its generation stamp: otherwise a hard kill can leave the + * fsync'd counter ahead of the still-buffered entity bytes (phantom progress + * for any generation-based consumer). Adapters whose writes are already + * durable per-call (cloud object PUT) may leave these undefined — the store + * treats them as no-ops. `beginWriteBarrier` also resets any tracking left by + * an aborted prior transaction. + */ + beginWriteBarrier?(): void + /** @see beginWriteBarrier — fsync every canonical write since begin. */ + flushWriteBarrier?(): Promise + /** Read an entity's raw stored metadata+vector objects. */ readNounRaw(id: string): Promise<{ metadata: any | null; vector: any | null }> /** Restore an entity's raw stored objects (`null` part ⇒ delete that file). */ @@ -360,6 +472,44 @@ export interface GenerationStorage { /** Read all lines of `_system/tx-log.jsonl` (empty array if absent). */ readTxLogLines(): Promise + /** + * OPTIONAL binary raw-byte primitives — the substrate for the generation + * fact log's append-only CRC-framed segments. Feature-detected: a storage + * layer that omits them hosts no fact log (dual-write is skipped; readers + * fall back to canonical enumeration). Paths are used VERBATIM (no + * suffixing). Append durability rides `syncRawObjects` at the commit + * barrier, exactly like the staged history files. + */ + appendRawBytes?(path: string, bytes: Uint8Array): Promise + /** Read a raw binary file whole; absent → null; a real fault throws. */ + readRawBytes?(path: string): Promise + /** Replace a raw binary file atomically (tmp → fsync → rename). */ + writeRawBytes?(path: string, bytes: Uint8Array): Promise + /** Byte size of a raw binary file, or null when absent. */ + rawByteSize?(path: string): Promise + + /** + * OPTIONAL temporal-blob contract (implemented by blob-aware storage; the + * generation store treats the hashes as opaque strings). Extract the + * content-blob hashes a record-set references — a pure multiset extraction, + * no side effects. + */ + extractBlobHashesFromRecords?(records: GenerationRecord[]): string[] + /** + * OPTIONAL: record history references for the given hashes (one increment + * per occurrence). Called BEFORE the referencing record-set is persisted — + * a crash between the two can only over-count (a leak the scrub repairs), + * never under-count (which would risk reclaiming bytes history still needs). + */ + recordHistoryBlobReferences?(hashes: string[]): Promise + /** + * OPTIONAL: release history references for the given hashes (one decrement + * per occurrence) and physically reclaim any hash left with zero live AND + * zero history references. Called AFTER the referencing record-set is + * deleted by compaction — same over-count-only crash ordering. + */ + releaseHistoryBlobReferences?(hashes: string[]): Promise + /** * Register the generation-bump hook invoked on every entity-visible * single-operation write (see `BaseStorage.setGenerationBumpHook`). diff --git a/src/db/whereMatcher.ts b/src/db/whereMatcher.ts index 51c635a7..c5469209 100644 --- a/src/db/whereMatcher.ts +++ b/src/db/whereMatcher.ts @@ -39,9 +39,9 @@ export class UnsupportedWhereOperatorError extends Error { constructor(operator: string) { super( `The where-operator '${operator}' is not supported for historical/speculative ` + - `in-memory evaluation. Supported: eq/equals/is, ne/notEquals/isNot, in/oneOf, ` + - `gt/greaterThan, gte/greaterThanOrEqual/greaterEqual, lt/lessThan, ` + - `lte/lessThanOrEqual/lessEqual, between, contains, exists, missing, ` + + `in-memory evaluation. Supported: eq/equals, ne/notEquals, in/oneOf, ` + + `gt/greaterThan, gte/greaterThanOrEqual, lt/lessThan, ` + + `lte/lessThanOrEqual, between, contains, exists, missing, ` + `plus allOf/anyOf/not.` ) this.name = 'UnsupportedWhereOperatorError' @@ -150,12 +150,10 @@ function fieldConditionMatches(value: unknown, condition: unknown): boolean { for (const [op, operand] of Object.entries(condition as Record)) { let matches: boolean switch (op) { - case 'is': case 'equals': case 'eq': matches = eqMatches(value, operand) break - case 'isNot': case 'notEquals': case 'ne': matches = !eqMatches(value, operand) @@ -170,7 +168,6 @@ function fieldConditionMatches(value: unknown, condition: unknown): boolean { matches = cmp !== null && cmp > 0 break } - case 'greaterEqual': case 'greaterThanOrEqual': case 'gte': { const cmp = compare(value, operand) @@ -183,7 +180,6 @@ function fieldConditionMatches(value: unknown, condition: unknown): boolean { matches = cmp !== null && cmp < 0 break } - case 'lessEqual': case 'lessThanOrEqual': case 'lte': { const cmp = compare(value, operand) diff --git a/src/embeddings/CachedEmbeddings.ts b/src/embeddings/CachedEmbeddings.ts deleted file mode 100644 index 6108717a..00000000 --- a/src/embeddings/CachedEmbeddings.ts +++ /dev/null @@ -1,163 +0,0 @@ -/** - * Cached Embeddings - Performance Optimization Layer - * - * Provides pre-computed embeddings for common terms to avoid - * unnecessary model calls. Falls back to EmbeddingManager for - * unknown terms. - * - * This is purely a performance optimization - it doesn't affect - * the consistency or accuracy of embeddings. - */ - -import { Vector } from '../coreTypes.js' -import { embeddingManager } from './EmbeddingManager.js' - -// Pre-computed embeddings for top common terms -// In production, this could be loaded from a file or expanded significantly -const PRECOMPUTED_EMBEDDINGS: Record = { - // Programming languages - 'javascript': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.1)), - 'python': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.1)), - 'typescript': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.15)), - 'java': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.15)), - 'rust': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.2)), - 'go': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.2)), - 'c++': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.22)), - 'c#': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.22)), - - // Web frameworks - 'react': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.25)), - 'vue': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.25)), - 'angular': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.3)), - 'svelte': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.3)), - 'nextjs': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.32)), - 'nuxt': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.32)), - - // Databases - 'postgresql': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.35)), - 'mysql': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.35)), - 'mongodb': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.4)), - 'redis': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.4)), - 'elasticsearch': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.42)), - - // Common tech terms - 'database': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.45)), - 'api': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.45)), - 'server': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.5)), - 'client': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.5)), - 'frontend': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.55)), - 'backend': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.55)), - 'fullstack': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.57)), - 'devops': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.57)), - 'cloud': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.6)), - 'docker': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.6)), - 'kubernetes': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.62)), - 'microservices': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.62)), -} - -/** - * Simple character n-gram based embedding for short text - * This is much faster than using the model for simple terms - */ -function computeSimpleEmbedding(text: string): Vector { - const normalized = text.toLowerCase().trim() - const vector = new Array(384).fill(0) - - // Character trigrams for simple semantic similarity - for (let i = 0; i < normalized.length - 2; i++) { - const trigram = normalized.slice(i, i + 3) - const hash = trigram.charCodeAt(0) * 31 + - trigram.charCodeAt(1) * 7 + - trigram.charCodeAt(2) - const index = Math.abs(hash) % 384 - vector[index] += 1 / (normalized.length - 2) - } - - // Normalize vector - const magnitude = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0)) - if (magnitude > 0) { - for (let i = 0; i < vector.length; i++) { - vector[i] /= magnitude - } - } - - return vector -} - -/** - * Cached Embeddings with fallback to EmbeddingManager - */ -export class CachedEmbeddings { - private stats = { - cacheHits: 0, - simpleComputes: 0, - modelCalls: 0 - } - - /** - * Generate embedding with caching - */ - async embed(text: string | string[]): Promise { - if (Array.isArray(text)) { - return Promise.all(text.map(t => this.embedSingle(t))) - } - return this.embedSingle(text) - } - - /** - * Embed single text with cache lookup - */ - private async embedSingle(text: string): Promise { - const normalized = text.toLowerCase().trim() - - // 1. Check pre-computed cache (instant, zero cost) - if (PRECOMPUTED_EMBEDDINGS[normalized]) { - this.stats.cacheHits++ - return PRECOMPUTED_EMBEDDINGS[normalized] - } - - // 2. Check for partial matches in cache - for (const [term, embedding] of Object.entries(PRECOMPUTED_EMBEDDINGS)) { - if (normalized.includes(term) || term.includes(normalized)) { - this.stats.cacheHits++ - // Return slightly modified version to maintain uniqueness - return embedding.map(v => v * 0.95) - } - } - - // 3. For short text, use simple embedding (fast, low cost) - if (normalized.length < 50 && normalized.split(' ').length < 5) { - this.stats.simpleComputes++ - return computeSimpleEmbedding(normalized) - } - - // 4. Fall back to EmbeddingManager for complex text - this.stats.modelCalls++ - return await embeddingManager.embed(text) - } - - /** - * Get cache statistics - */ - getStats() { - return { - ...this.stats, - totalEmbeddings: this.stats.cacheHits + this.stats.simpleComputes + this.stats.modelCalls, - cacheHitRate: this.stats.cacheHits / - (this.stats.cacheHits + this.stats.simpleComputes + this.stats.modelCalls) || 0 - } - } - - /** - * Add custom pre-computed embeddings - */ - addPrecomputed(term: string, embedding: Vector) { - if (embedding.length !== 384) { - throw new Error('Embedding must have 384 dimensions') - } - PRECOMPUTED_EMBEDDINGS[term.toLowerCase()] = embedding - } -} - -// Export singleton instance -export const cachedEmbeddings = new CachedEmbeddings() \ No newline at end of file diff --git a/src/embeddings/EmbeddingManager.ts b/src/embeddings/EmbeddingManager.ts index 2efd84d8..83b42d82 100644 --- a/src/embeddings/EmbeddingManager.ts +++ b/src/embeddings/EmbeddingManager.ts @@ -60,7 +60,7 @@ export class EmbeddingManager { private constructor() { this.engine = WASMEmbeddingEngine.getInstance() // Log deferred to init() — at construction time we don't know if a plugin - // (like Cortex) will replace the WASM embedder with a native one. + // (like Cor) will replace the WASM embedder with a native one. } /** diff --git a/src/embeddings/index.ts b/src/embeddings/index.ts deleted file mode 100644 index e46062b5..00000000 --- a/src/embeddings/index.ts +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Embeddings Module - Clean, Unified Architecture - * - * This module provides all embedding functionality for Brainy. - * - * Main Components: - * - EmbeddingManager: Core embedding generation with Q8/FP32 support - * - CachedEmbeddings: Performance optimization layer with pre-computed embeddings - */ - -// Core embedding functionality -export { - EmbeddingManager, - embeddingManager, - embed, - getEmbeddingFunction, - getEmbeddingStats, - type ModelPrecision -} from './EmbeddingManager.js' - -// Cached embeddings for performance -export { - CachedEmbeddings, - cachedEmbeddings -} from './CachedEmbeddings.js' - -// Default export is the singleton manager -export { embeddingManager as default } from './EmbeddingManager.js' \ No newline at end of file diff --git a/src/embeddings/wasm/pkg/candle_embeddings.d.ts b/src/embeddings/wasm/pkg/candle_embeddings.d.ts new file mode 100644 index 00000000..90cfc974 --- /dev/null +++ b/src/embeddings/wasm/pkg/candle_embeddings.d.ts @@ -0,0 +1,100 @@ +/* tslint:disable */ +/* eslint-disable */ + +/** + * WASM-compatible embedding engine + */ +export class EmbeddingEngine { + free(): void; + [Symbol.dispose](): void; + /** + * Get the embedding dimension (384 for all-MiniLM-L6-v2) + */ + dimension(): number; + /** + * Generate embedding for a single text + * + * Returns a Float32Array of 384 dimensions + */ + embed(text: string): Float32Array; + /** + * Generate embeddings for multiple texts + * + * Takes a JavaScript Array of strings + * Returns a JavaScript Array of Float32Array + */ + embed_batch(texts: Array): Array; + /** + * Check if the engine is ready for inference + */ + is_ready(): boolean; + /** + * Load the model and tokenizer from bytes + * + * This is now the ONLY way to initialize the engine. + * Model weights are no longer embedded in WASM for faster initialization. + * + * # Arguments + * * `model_bytes` - SafeTensors format model weights + * * `tokenizer_bytes` - tokenizer.json contents + * * `config_bytes` - config.json contents + */ + load(model_bytes: Uint8Array, tokenizer_bytes: Uint8Array, config_bytes: Uint8Array): void; + /** + * Get the maximum sequence length + */ + max_sequence_length(): number; + /** + * Create a new embedding engine instance (not loaded) + */ + constructor(); +} + +/** + * Calculate cosine similarity between two embeddings + */ +export function cosine_similarity(a: Float32Array, b: Float32Array): number; + +export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module; + +export interface InitOutput { + readonly memory: WebAssembly.Memory; + readonly __wbg_embeddingengine_free: (a: number, b: number) => void; + readonly cosine_similarity: (a: number, b: number, c: number, d: number) => number; + readonly embeddingengine_dimension: (a: number) => number; + readonly embeddingengine_embed: (a: number, b: number, c: number) => [number, number, number]; + readonly embeddingengine_embed_batch: (a: number, b: any) => [number, number, number]; + readonly embeddingengine_is_ready: (a: number) => number; + readonly embeddingengine_load: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number]; + readonly embeddingengine_max_sequence_length: (a: number) => number; + readonly embeddingengine_new: () => number; + readonly __wbindgen_malloc: (a: number, b: number) => number; + readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number; + readonly __wbindgen_exn_store: (a: number) => void; + readonly __externref_table_alloc: () => number; + readonly __wbindgen_externrefs: WebAssembly.Table; + readonly __externref_table_dealloc: (a: number) => void; + readonly __wbindgen_start: () => void; +} + +export type SyncInitInput = BufferSource | WebAssembly.Module; + +/** + * Instantiates the given `module`, which can either be bytes or + * a precompiled `WebAssembly.Module`. + * + * @param {{ module: SyncInitInput }} module - Passing `SyncInitInput` directly is deprecated. + * + * @returns {InitOutput} + */ +export function initSync(module: { module: SyncInitInput } | SyncInitInput): InitOutput; + +/** + * If `module_or_path` is {RequestInfo} or {URL}, makes a request and + * for everything else, calls `WebAssembly.instantiate` directly. + * + * @param {{ module_or_path: InitInput | Promise }} module_or_path - Passing `InitInput` directly is deprecated. + * + * @returns {Promise} + */ +export default function __wbg_init (module_or_path?: { module_or_path: InitInput | Promise } | InitInput | Promise): Promise; diff --git a/src/embeddings/wasm/pkg/candle_embeddings.js b/src/embeddings/wasm/pkg/candle_embeddings.js new file mode 100644 index 00000000..736ad4e4 --- /dev/null +++ b/src/embeddings/wasm/pkg/candle_embeddings.js @@ -0,0 +1,525 @@ +/* @ts-self-types="./candle_embeddings.d.ts" */ + +/** + * WASM-compatible embedding engine + */ +export class EmbeddingEngine { + __destroy_into_raw() { + const ptr = this.__wbg_ptr; + this.__wbg_ptr = 0; + EmbeddingEngineFinalization.unregister(this); + return ptr; + } + free() { + const ptr = this.__destroy_into_raw(); + wasm.__wbg_embeddingengine_free(ptr, 0); + } + /** + * Get the embedding dimension (384 for all-MiniLM-L6-v2) + * @returns {number} + */ + dimension() { + const ret = wasm.embeddingengine_dimension(this.__wbg_ptr); + return ret >>> 0; + } + /** + * Generate embedding for a single text + * + * Returns a Float32Array of 384 dimensions + * @param {string} text + * @returns {Float32Array} + */ + embed(text) { + const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.embeddingengine_embed(this.__wbg_ptr, ptr0, len0); + if (ret[2]) { + throw takeFromExternrefTable0(ret[1]); + } + return takeFromExternrefTable0(ret[0]); + } + /** + * Generate embeddings for multiple texts + * + * Takes a JavaScript Array of strings + * Returns a JavaScript Array of Float32Array + * @param {Array} texts + * @returns {Array} + */ + embed_batch(texts) { + const ret = wasm.embeddingengine_embed_batch(this.__wbg_ptr, texts); + if (ret[2]) { + throw takeFromExternrefTable0(ret[1]); + } + return takeFromExternrefTable0(ret[0]); + } + /** + * Check if the engine is ready for inference + * @returns {boolean} + */ + is_ready() { + const ret = wasm.embeddingengine_is_ready(this.__wbg_ptr); + return ret !== 0; + } + /** + * Load the model and tokenizer from bytes + * + * This is now the ONLY way to initialize the engine. + * Model weights are no longer embedded in WASM for faster initialization. + * + * # Arguments + * * `model_bytes` - SafeTensors format model weights + * * `tokenizer_bytes` - tokenizer.json contents + * * `config_bytes` - config.json contents + * @param {Uint8Array} model_bytes + * @param {Uint8Array} tokenizer_bytes + * @param {Uint8Array} config_bytes + */ + load(model_bytes, tokenizer_bytes, config_bytes) { + const ptr0 = passArray8ToWasm0(model_bytes, wasm.__wbindgen_malloc); + const len0 = WASM_VECTOR_LEN; + const ptr1 = passArray8ToWasm0(tokenizer_bytes, wasm.__wbindgen_malloc); + const len1 = WASM_VECTOR_LEN; + const ptr2 = passArray8ToWasm0(config_bytes, wasm.__wbindgen_malloc); + const len2 = WASM_VECTOR_LEN; + const ret = wasm.embeddingengine_load(this.__wbg_ptr, ptr0, len0, ptr1, len1, ptr2, len2); + if (ret[1]) { + throw takeFromExternrefTable0(ret[0]); + } + } + /** + * Get the maximum sequence length + * @returns {number} + */ + max_sequence_length() { + const ret = wasm.embeddingengine_max_sequence_length(this.__wbg_ptr); + return ret >>> 0; + } + /** + * Create a new embedding engine instance (not loaded) + */ + constructor() { + const ret = wasm.embeddingengine_new(); + this.__wbg_ptr = ret >>> 0; + EmbeddingEngineFinalization.register(this, this.__wbg_ptr, this); + return this; + } +} +if (Symbol.dispose) EmbeddingEngine.prototype[Symbol.dispose] = EmbeddingEngine.prototype.free; + +/** + * Calculate cosine similarity between two embeddings + * @param {Float32Array} a + * @param {Float32Array} b + * @returns {number} + */ +export function cosine_similarity(a, b) { + const ptr0 = passArrayF32ToWasm0(a, wasm.__wbindgen_malloc); + const len0 = WASM_VECTOR_LEN; + const ptr1 = passArrayF32ToWasm0(b, wasm.__wbindgen_malloc); + const len1 = WASM_VECTOR_LEN; + const ret = wasm.cosine_similarity(ptr0, len0, ptr1, len1); + return ret; +} + +function __wbg_get_imports() { + const import0 = { + __proto__: null, + __wbg___wbindgen_is_function_0095a73b8b156f76: function(arg0) { + const ret = typeof(arg0) === 'function'; + return ret; + }, + __wbg___wbindgen_is_object_5ae8e5880f2c1fbd: function(arg0) { + const val = arg0; + const ret = typeof(val) === 'object' && val !== null; + return ret; + }, + __wbg___wbindgen_is_string_cd444516edc5b180: function(arg0) { + const ret = typeof(arg0) === 'string'; + return ret; + }, + __wbg___wbindgen_is_undefined_9e4d92534c42d778: function(arg0) { + const ret = arg0 === undefined; + return ret; + }, + __wbg___wbindgen_string_get_72fb696202c56729: function(arg0, arg1) { + const obj = arg1; + const ret = typeof(obj) === 'string' ? obj : undefined; + var ptr1 = isLikeNone(ret) ? 0 : passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + var len1 = WASM_VECTOR_LEN; + getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true); + getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true); + }, + __wbg___wbindgen_throw_be289d5034ed271b: function(arg0, arg1) { + throw new Error(getStringFromWasm0(arg0, arg1)); + }, + __wbg_call_389efe28435a9388: function() { return handleError(function (arg0, arg1) { + const ret = arg0.call(arg1); + return ret; + }, arguments); }, + __wbg_call_4708e0c13bdc8e95: function() { return handleError(function (arg0, arg1, arg2) { + const ret = arg0.call(arg1, arg2); + return ret; + }, arguments); }, + __wbg_crypto_86f2631e91b51511: function(arg0) { + const ret = arg0.crypto; + return ret; + }, + __wbg_getRandomValues_b3f15fcbfabb0f8b: function() { return handleError(function (arg0, arg1) { + arg0.getRandomValues(arg1); + }, arguments); }, + __wbg_get_9b94d73e6221f75c: function(arg0, arg1) { + const ret = arg0[arg1 >>> 0]; + return ret; + }, + __wbg_length_32ed9a279acd054c: function(arg0) { + const ret = arg0.length; + return ret; + }, + __wbg_length_35a7bace40f36eac: function(arg0) { + const ret = arg0.length; + return ret; + }, + __wbg_length_9a7876c9728a0979: function(arg0) { + const ret = arg0.length; + return ret; + }, + __wbg_msCrypto_d562bbe83e0d4b91: function(arg0) { + const ret = arg0.msCrypto; + return ret; + }, + __wbg_new_3eb36ae241fe6f44: function() { + const ret = new Array(); + return ret; + }, + __wbg_new_no_args_1c7c842f08d00ebb: function(arg0, arg1) { + const ret = new Function(getStringFromWasm0(arg0, arg1)); + return ret; + }, + __wbg_new_with_length_1763c527b2923202: function(arg0) { + const ret = new Array(arg0 >>> 0); + return ret; + }, + __wbg_new_with_length_63f2683cc2521026: function(arg0) { + const ret = new Float32Array(arg0 >>> 0); + return ret; + }, + __wbg_new_with_length_a2c39cbe88fd8ff1: function(arg0) { + const ret = new Uint8Array(arg0 >>> 0); + return ret; + }, + __wbg_node_e1f24f89a7336c2e: function(arg0) { + const ret = arg0.node; + return ret; + }, + __wbg_process_3975fd6c72f520aa: function(arg0) { + const ret = arg0.process; + return ret; + }, + __wbg_prototypesetcall_bdcdcc5842e4d77d: function(arg0, arg1, arg2) { + Uint8Array.prototype.set.call(getArrayU8FromWasm0(arg0, arg1), arg2); + }, + __wbg_randomFillSync_f8c153b79f285817: function() { return handleError(function (arg0, arg1) { + arg0.randomFillSync(arg1); + }, arguments); }, + __wbg_require_b74f47fc2d022fd6: function() { return handleError(function () { + const ret = module.require; + return ret; + }, arguments); }, + __wbg_set_f43e577aea94465b: function(arg0, arg1, arg2) { + arg0[arg1 >>> 0] = arg2; + }, + __wbg_set_f8edeec46569cc70: function(arg0, arg1, arg2) { + arg0.set(getArrayF32FromWasm0(arg1, arg2)); + }, + __wbg_static_accessor_GLOBAL_12837167ad935116: function() { + const ret = typeof global === 'undefined' ? null : global; + return isLikeNone(ret) ? 0 : addToExternrefTable0(ret); + }, + __wbg_static_accessor_GLOBAL_THIS_e628e89ab3b1c95f: function() { + const ret = typeof globalThis === 'undefined' ? null : globalThis; + return isLikeNone(ret) ? 0 : addToExternrefTable0(ret); + }, + __wbg_static_accessor_SELF_a621d3dfbb60d0ce: function() { + const ret = typeof self === 'undefined' ? null : self; + return isLikeNone(ret) ? 0 : addToExternrefTable0(ret); + }, + __wbg_static_accessor_WINDOW_f8727f0cf888e0bd: function() { + const ret = typeof window === 'undefined' ? null : window; + return isLikeNone(ret) ? 0 : addToExternrefTable0(ret); + }, + __wbg_subarray_a96e1fef17ed23cb: function(arg0, arg1, arg2) { + const ret = arg0.subarray(arg1 >>> 0, arg2 >>> 0); + return ret; + }, + __wbg_versions_4e31226f5e8dc909: function(arg0) { + const ret = arg0.versions; + return ret; + }, + __wbindgen_cast_0000000000000001: function(arg0, arg1) { + // Cast intrinsic for `Ref(Slice(U8)) -> NamedExternref("Uint8Array")`. + const ret = getArrayU8FromWasm0(arg0, arg1); + return ret; + }, + __wbindgen_cast_0000000000000002: function(arg0, arg1) { + // Cast intrinsic for `Ref(String) -> Externref`. + const ret = getStringFromWasm0(arg0, arg1); + return ret; + }, + __wbindgen_init_externref_table: function() { + const table = wasm.__wbindgen_externrefs; + const offset = table.grow(4); + table.set(0, undefined); + table.set(offset + 0, undefined); + table.set(offset + 1, null); + table.set(offset + 2, true); + table.set(offset + 3, false); + }, + }; + return { + __proto__: null, + "./candle_embeddings_bg.js": import0, + }; +} + +const EmbeddingEngineFinalization = (typeof FinalizationRegistry === 'undefined') + ? { register: () => {}, unregister: () => {} } + : new FinalizationRegistry(ptr => wasm.__wbg_embeddingengine_free(ptr >>> 0, 1)); + +function addToExternrefTable0(obj) { + const idx = wasm.__externref_table_alloc(); + wasm.__wbindgen_externrefs.set(idx, obj); + return idx; +} + +function getArrayF32FromWasm0(ptr, len) { + ptr = ptr >>> 0; + return getFloat32ArrayMemory0().subarray(ptr / 4, ptr / 4 + len); +} + +function getArrayU8FromWasm0(ptr, len) { + ptr = ptr >>> 0; + return getUint8ArrayMemory0().subarray(ptr / 1, ptr / 1 + len); +} + +let cachedDataViewMemory0 = null; +function getDataViewMemory0() { + if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) { + cachedDataViewMemory0 = new DataView(wasm.memory.buffer); + } + return cachedDataViewMemory0; +} + +let cachedFloat32ArrayMemory0 = null; +function getFloat32ArrayMemory0() { + if (cachedFloat32ArrayMemory0 === null || cachedFloat32ArrayMemory0.byteLength === 0) { + cachedFloat32ArrayMemory0 = new Float32Array(wasm.memory.buffer); + } + return cachedFloat32ArrayMemory0; +} + +function getStringFromWasm0(ptr, len) { + ptr = ptr >>> 0; + return decodeText(ptr, len); +} + +let cachedUint8ArrayMemory0 = null; +function getUint8ArrayMemory0() { + if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) { + cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer); + } + return cachedUint8ArrayMemory0; +} + +function handleError(f, args) { + try { + return f.apply(this, args); + } catch (e) { + const idx = addToExternrefTable0(e); + wasm.__wbindgen_exn_store(idx); + } +} + +function isLikeNone(x) { + return x === undefined || x === null; +} + +function passArray8ToWasm0(arg, malloc) { + const ptr = malloc(arg.length * 1, 1) >>> 0; + getUint8ArrayMemory0().set(arg, ptr / 1); + WASM_VECTOR_LEN = arg.length; + return ptr; +} + +function passArrayF32ToWasm0(arg, malloc) { + const ptr = malloc(arg.length * 4, 4) >>> 0; + getFloat32ArrayMemory0().set(arg, ptr / 4); + WASM_VECTOR_LEN = arg.length; + return ptr; +} + +function passStringToWasm0(arg, malloc, realloc) { + if (realloc === undefined) { + const buf = cachedTextEncoder.encode(arg); + const ptr = malloc(buf.length, 1) >>> 0; + getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf); + WASM_VECTOR_LEN = buf.length; + return ptr; + } + + let len = arg.length; + let ptr = malloc(len, 1) >>> 0; + + const mem = getUint8ArrayMemory0(); + + let offset = 0; + + for (; offset < len; offset++) { + const code = arg.charCodeAt(offset); + if (code > 0x7F) break; + mem[ptr + offset] = code; + } + if (offset !== len) { + if (offset !== 0) { + arg = arg.slice(offset); + } + ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0; + const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len); + const ret = cachedTextEncoder.encodeInto(arg, view); + + offset += ret.written; + ptr = realloc(ptr, len, offset, 1) >>> 0; + } + + WASM_VECTOR_LEN = offset; + return ptr; +} + +function takeFromExternrefTable0(idx) { + const value = wasm.__wbindgen_externrefs.get(idx); + wasm.__externref_table_dealloc(idx); + return value; +} + +let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }); +cachedTextDecoder.decode(); +const MAX_SAFARI_DECODE_BYTES = 2146435072; +let numBytesDecoded = 0; +function decodeText(ptr, len) { + numBytesDecoded += len; + if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) { + cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }); + cachedTextDecoder.decode(); + numBytesDecoded = len; + } + return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); +} + +const cachedTextEncoder = new TextEncoder(); + +if (!('encodeInto' in cachedTextEncoder)) { + cachedTextEncoder.encodeInto = function (arg, view) { + const buf = cachedTextEncoder.encode(arg); + view.set(buf); + return { + read: arg.length, + written: buf.length + }; + }; +} + +let WASM_VECTOR_LEN = 0; + +let wasmModule, wasm; +function __wbg_finalize_init(instance, module) { + wasm = instance.exports; + wasmModule = module; + cachedDataViewMemory0 = null; + cachedFloat32ArrayMemory0 = null; + cachedUint8ArrayMemory0 = null; + wasm.__wbindgen_start(); + return wasm; +} + +async function __wbg_load(module, imports) { + if (typeof Response === 'function' && module instanceof Response) { + if (typeof WebAssembly.instantiateStreaming === 'function') { + try { + return await WebAssembly.instantiateStreaming(module, imports); + } catch (e) { + const validResponse = module.ok && expectedResponseType(module.type); + + if (validResponse && module.headers.get('Content-Type') !== 'application/wasm') { + console.warn("`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n", e); + + } else { throw e; } + } + } + + const bytes = await module.arrayBuffer(); + return await WebAssembly.instantiate(bytes, imports); + } else { + const instance = await WebAssembly.instantiate(module, imports); + + if (instance instanceof WebAssembly.Instance) { + return { instance, module }; + } else { + return instance; + } + } + + function expectedResponseType(type) { + switch (type) { + case 'basic': case 'cors': case 'default': return true; + } + return false; + } +} + +function initSync(module) { + if (wasm !== undefined) return wasm; + + + if (module !== undefined) { + if (Object.getPrototypeOf(module) === Object.prototype) { + ({module} = module) + } else { + console.warn('using deprecated parameters for `initSync()`; pass a single object instead') + } + } + + const imports = __wbg_get_imports(); + if (!(module instanceof WebAssembly.Module)) { + module = new WebAssembly.Module(module); + } + const instance = new WebAssembly.Instance(module, imports); + return __wbg_finalize_init(instance, module); +} + +async function __wbg_init(module_or_path) { + if (wasm !== undefined) return wasm; + + + if (module_or_path !== undefined) { + if (Object.getPrototypeOf(module_or_path) === Object.prototype) { + ({module_or_path} = module_or_path) + } else { + console.warn('using deprecated parameters for the initialization function; pass a single object instead') + } + } + + if (module_or_path === undefined) { + module_or_path = new URL('candle_embeddings_bg.wasm', import.meta.url); + } + const imports = __wbg_get_imports(); + + if (typeof module_or_path === 'string' || (typeof Request === 'function' && module_or_path instanceof Request) || (typeof URL === 'function' && module_or_path instanceof URL)) { + module_or_path = fetch(module_or_path); + } + + const { instance, module } = await __wbg_load(await module_or_path, imports); + + return __wbg_finalize_init(instance, module); +} + +export { initSync, __wbg_init as default }; diff --git a/src/embeddings/wasm/pkg/candle_embeddings_bg.wasm b/src/embeddings/wasm/pkg/candle_embeddings_bg.wasm new file mode 100644 index 00000000..7cfdd5ea Binary files /dev/null and b/src/embeddings/wasm/pkg/candle_embeddings_bg.wasm differ diff --git a/src/embeddings/wasm/pkg/candle_embeddings_bg.wasm.d.ts b/src/embeddings/wasm/pkg/candle_embeddings_bg.wasm.d.ts new file mode 100644 index 00000000..46a79cb4 --- /dev/null +++ b/src/embeddings/wasm/pkg/candle_embeddings_bg.wasm.d.ts @@ -0,0 +1,19 @@ +/* tslint:disable */ +/* eslint-disable */ +export const memory: WebAssembly.Memory; +export const __wbg_embeddingengine_free: (a: number, b: number) => void; +export const cosine_similarity: (a: number, b: number, c: number, d: number) => number; +export const embeddingengine_dimension: (a: number) => number; +export const embeddingengine_embed: (a: number, b: number, c: number) => [number, number, number]; +export const embeddingengine_embed_batch: (a: number, b: any) => [number, number, number]; +export const embeddingengine_is_ready: (a: number) => number; +export const embeddingengine_load: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number]; +export const embeddingengine_max_sequence_length: (a: number) => number; +export const embeddingengine_new: () => number; +export const __wbindgen_malloc: (a: number, b: number) => number; +export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number; +export const __wbindgen_exn_store: (a: number) => void; +export const __externref_table_alloc: () => number; +export const __wbindgen_externrefs: WebAssembly.Table; +export const __externref_table_dealloc: (a: number) => void; +export const __wbindgen_start: () => void; diff --git a/src/errors/brainyError.ts b/src/errors/brainyError.ts index 350e98ba..a58236e3 100644 --- a/src/errors/brainyError.ts +++ b/src/errors/brainyError.ts @@ -10,7 +10,14 @@ export type BrainyErrorType = | 'NOT_FOUND' | 'RETRY_EXHAUSTED' | 'VALIDATION' + | 'INVALID_QUERY' | 'FIELD_NOT_INDEXED' + | 'GRAPH_INDEX_NOT_READY' + | 'METADATA_INDEX_NOT_READY' + | 'VECTOR_INDEX_NOT_READY' + | 'PROTECTED_ARTIFACT' + | 'DERIVED_ARTIFACT_MISSING' + | 'MIGRATION_IN_PROGRESS' /** * Custom error class for Brainy operations @@ -223,3 +230,178 @@ export class BrainyError extends Error { return BrainyError.storage(error.message, error) } } + +/** + * Thrown when the graph adjacency index reports that relationships exist (its + * persisted manifest/count loaded, or its readiness signal says otherwise) but + * the source→target adjacency itself did NOT load — so graph traversals + * (`find({ connected })`, `neighbors()`, `related()`) would otherwise return an + * EMPTY array indistinguishable from "no edges". + * + * On 8.0 brainy detects this on the first graph read via the provider's honest + * sync `isReady()` signal (true ONLY when the edges are loaded; see + * {@link import('../plugin.js').GraphIndexProvider.isReady}); for older providers + * that do not expose it, it falls back to a known-edge-sample probe (one persisted + * verb + one neighbor lookup). Either way it attempts a rebuild from storage and + * raises this LOUD, catchable error only if even that cannot make the adjacency + * ready — replacing silent data-invisibility with a clear failure. + * + * Observed with a native graph provider whose cold-open adjacency load is + * swallowed on certain storage adapters; the fix is upstream in the provider, + * but Brainy refuses to serve `[]` as if it were truth. + */ +export class GraphIndexNotReadyError extends BrainyError { + constructor(message: string, originalError?: Error) { + super(message, 'GRAPH_INDEX_NOT_READY', false, originalError) + this.name = 'GraphIndexNotReadyError' + if (Error.captureStackTrace) { + Error.captureStackTrace(this, GraphIndexNotReadyError) + } + } +} + +/** + * Thrown when the metadata field index reports data but cannot serve a KNOWN + * persisted field value even after a rebuild — i.e. the `where` / filter + * postings did not load on a cold open and could not be restored. The + * field-index counterpart of {@link GraphIndexNotReadyError}: it replaces the + * silent-empty failure mode (a cold `find({ where })` returning `[]` + * indistinguishable from "no such data") with a loud, catchable error, so a + * consumer never renders "nothing found" over data that is simply not-yet-warm. + * + * Detected once per brain by a known-value serving probe on the first filtered + * `find()`; brainy self-heals (rebuilds the index from the canonical records) + * first and only raises this if the rebuild still cannot serve the known value. + */ +export class MetadataIndexNotReadyError extends BrainyError { + constructor(message: string, originalError?: Error) { + super(message, 'METADATA_INDEX_NOT_READY', false, originalError) + this.name = 'MetadataIndexNotReadyError' + if (Error.captureStackTrace) { + Error.captureStackTrace(this, MetadataIndexNotReadyError) + } + } +} + +/** + * Thrown when the vector index reports vectors (`size() > 0` or, on a native + * provider, `isReady() === false`) but cannot return a KNOWN persisted vector + * even after a rebuild — i.e. the semantic serving structure did not load on a + * cold open and could not be restored. The vector-search counterpart of + * {@link GraphIndexNotReadyError} / {@link MetadataIndexNotReadyError}: it + * replaces the silent-empty failure mode (a cold `find({ query })` returning + * `[]` indistinguishable from "no similar data") with a loud, catchable error, + * so a consumer never renders "nothing found" over data that is simply + * not-yet-warm. + * + * Detected once per brain by a known-vector serving probe on the first + * semantic / proximity `find()`; brainy self-heals (rebuilds the index from the + * canonical records) first and only raises this if the rebuild still cannot + * serve the known vector. + */ +export class VectorIndexNotReadyError extends BrainyError { + constructor(message: string, originalError?: Error) { + super(message, 'VECTOR_INDEX_NOT_READY', false, originalError) + this.name = 'VectorIndexNotReadyError' + if (Error.captureStackTrace) { + Error.captureStackTrace(this, VectorIndexNotReadyError) + } + } +} + +/** + * Thrown when a delete (`deleteBinaryBlob` / `removeRawPrefix`) would remove a + * blob that is a declared member of a protected derived-index FAMILY (the + * registered-blob contract). Declared derived artifacts are undeletable through + * the storage layer — this makes an in-process GC / sweeper INCAPABLE of removing + * a load-bearing index file (the lost-`main.dkann` class). Intentional retirement + * is the explicit `unregisterDerivedFamily(name)` step, then the delete. + */ +export class ProtectedArtifactError extends BrainyError { + /** The blob key the delete targeted. */ + public readonly key: string + /** The protected family the key belongs to. */ + public readonly family: string + constructor(key: string, family: string) { + super( + `Refused to delete '${key}': it is a declared member of the protected ` + + `derived-index family '${family}'. Declared derived artifacts are undeletable ` + + `through the storage layer (COLD ≠ DEAD) — unregisterDerivedFamily('${family}') ` + + `first if retirement is intentional.`, + 'PROTECTED_ARTIFACT', + false + ) + this.name = 'ProtectedArtifactError' + this.key = key + this.family = family + } +} + +/** + * Raised (loudly) when a declared derived-index family is missing one or more of + * its members on open — i.e. a load-bearing blob was deleted OUTSIDE the write + * path (an external sweeper the in-process refusal cannot stop). The index must + * be rebuilt from canonical; "healthy-while-broken" is impossible because the + * missing member is named, not silently tolerated. + */ +export class DerivedArtifactMissingError extends BrainyError { + /** The family with missing members. */ + public readonly family: string + /** The member keys that are absent. */ + public readonly missing: string[] + constructor(family: string, missing: string[]) { + super( + `Derived-index family '${family}' is missing ${missing.length} declared ` + + `member(s) on open (${missing.join(', ')}) — deleted outside the write path. ` + + `The index must be rebuilt from canonical.`, + 'DERIVED_ARTIFACT_MISSING', + false + ) + this.name = 'DerivedArtifactMissingError' + this.family = family + this.missing = missing + } +} + +/** + * Thrown when a data-plane read or write is issued against a brain that is + * running its one-time, automatic 7.x → 8.0 on-disk upgrade — the coordinated + * migration LOCK. While a native provider rebuilds all derived indexes from the + * canonical records, brainy blocks reads and writes so no operation touches a + * half-built index; the caller waits for the correct answer rather than getting + * a partial one. This error is raised ONLY when the wait exceeds the configured + * window ({@link BrainyConfig.migrationWaitTimeoutMs}, default 30 s) — never + * instead of a partial/incorrect result. + * + * It is `retryable`: the upgrade continues in the background. Retry shortly, + * watch `brain.getIndexStatus().migration` for progress, or for a very large + * brain run the offline migrator. Data is safe; nothing is lost. Consumers can + * catch this (e.g. request middleware) and answer HTTP 503 + `Retry-After`. + * + * @example + * try { + * await brain.find({ query }) + * } catch (e) { + * if (e instanceof MigrationInProgressError) { + * res.set('Retry-After', '5').status(503).json({ upgrading: true, percent: e.percent }) + * return + * } + * throw e + * } + */ +export class MigrationInProgressError extends BrainyError { + /** Milliseconds the operation waited on the migration lock before timing out. */ + public readonly elapsedMs: number + /** Latest observed migration progress (0–100), when the provider reports it. */ + public readonly percent?: number + + constructor(message: string, elapsedMs: number, percent?: number, originalError?: Error) { + super(message, 'MIGRATION_IN_PROGRESS', true, originalError) + this.name = 'MigrationInProgressError' + this.elapsedMs = elapsedMs + this.percent = percent + if (Error.captureStackTrace) { + Error.captureStackTrace(this, MigrationInProgressError) + } + } +} diff --git a/src/events/changeFeed.ts b/src/events/changeFeed.ts new file mode 100644 index 00000000..9b9fda6f --- /dev/null +++ b/src/events/changeFeed.ts @@ -0,0 +1,171 @@ +/** + * @module events/changeFeed + * @description The in-process change feed behind {@link Brainy.onChange} — the + * authoritative "something committed" signal for every canonical mutation, + * regardless of origin (direct API calls, batches, transactions, imports, the + * VFS, or an accelerated native deployment: all of them funnel through the + * same generation-store commit points this feed is emitted from). + * + * Design properties: + * - **Post-commit only.** Events are enqueued after the commit succeeds, so an + * aborted commit (a losing `ifRev` CAS, a failed transaction) never emits. + * - **Commit-ordered.** Events are enqueued in commit order and dispatched + * FIFO, so a subscriber observes mutations in the order they became durable. + * - **Never blocks the write path.** Dispatch happens in a microtask after the + * committing call returns; a slow or throwing listener cannot delay or fail + * a write. Listener errors are isolated per listener and per event. + * - **Zero cost when unused.** Callers consult {@link ChangeFeed.hasListeners} + * before constructing event payloads; with no subscribers the write path + * does no event work at all. + * - **Fire-and-forget.** No acknowledgement, backpressure, or replay. Events + * carry the committed `generation`, so a consumer that needs catch-up + * semantics can pair the live feed with `transactionLog()` / `asOf()`. + */ + +/** The post-commit view of an entity carried by entity change events. */ +export interface ChangeEventEntity { + /** The entity's canonical UUID. */ + id: string + /** The entity's NounType string (e.g. `'person'`, `'document'`). */ + type: string + /** The entity's subtype, when set. */ + subtype?: string + /** The entity's custom (indexed) metadata fields. */ + metadata: Record + /** The writing service, when set. */ + service?: string +} + +/** The post-commit view of a relationship carried by relation change events. */ +export interface ChangeEventRelation { + /** The relationship's canonical UUID. */ + id: string + /** Source entity UUID. */ + from: string + /** Target entity UUID. */ + to: string + /** The relationship's VerbType string (e.g. `'contains'`). */ + type: string + /** The relationship's custom metadata fields, when present. */ + metadata?: Record +} + +/** + * One committed mutation, as delivered to {@link Brainy.onChange} listeners. + * + * Exactly one of `entity` / `relation` is populated for `kind: 'entity'` / + * `kind: 'relation'` events. `kind: 'store'` events (`clear` / `restore`) + * carry neither — they mean "the whole store changed; refetch what you care + * about". + * + * For `op: 'remove'` / `op: 'unrelate'` the payload is the record's LAST + * committed state (sourced from the commit's own before-image), so deletes are + * fully described rather than id-only. + */ +export interface BrainyChangeEvent { + /** What changed: one record, one relationship, or the whole store. */ + kind: 'entity' | 'relation' | 'store' + /** The mutation, in Brainy's own API vocabulary. */ + op: + | 'add' + | 'update' + | 'remove' + | 'relate' + | 'unrelate' + | 'updateRelation' + | 'clear' + | 'restore' + /** The mutated record's id (absent for store-level events). */ + id?: string + /** Post-commit entity view (entity events only). */ + entity?: ChangeEventEntity + /** Post-commit relation view (relation events only). */ + relation?: ChangeEventRelation + /** + * The committed generation this mutation belongs to (Model B: every write + * is a generation; a transaction's items share one). Absent for store-level + * events and init-time bootstrap writes, which are not generation-stamped. + */ + generation?: number + /** Commit timestamp (ms since epoch). */ + timestamp: number +} + +/** Listener signature for {@link Brainy.onChange}. */ +export type ChangeListener = (event: BrainyChangeEvent) => void + +/** + * A change event as constructed by a mutation method BEFORE its commit: the + * commit seam stamps `generation`/`timestamp` after the write becomes + * durable. An entity `remove` descriptor may omit `entity` — the seam fills + * it from the commit's own before-image (the record's last committed state). + */ +export type PendingChangeEvent = Omit + +/** + * @description Listener registry + commit-ordered async dispatcher for + * {@link BrainyChangeEvent}s. One instance per {@link Brainy}. + */ +export class ChangeFeed { + private listeners = new Set() + private queue: BrainyChangeEvent[] = [] + private draining = false + + /** Whether any listener is subscribed — the write path's zero-cost gate. */ + get hasListeners(): boolean { + return this.listeners.size > 0 + } + + /** + * @description Subscribe to committed mutations. + * @param listener - Called once per committed mutation, in commit order. + * @returns An unsubscribe function. + */ + subscribe(listener: ChangeListener): () => void { + this.listeners.add(listener) + return () => { + this.listeners.delete(listener) + } + } + + /** + * @description Enqueue committed events and schedule dispatch. Call ONLY + * after the commit has succeeded — this feed must never announce a write + * that did not become durable. Safe to call with an empty array. + * @param events - The committed mutations, in commit order. + */ + emit(events: BrainyChangeEvent[]): void { + if (events.length === 0 || this.listeners.size === 0) return + this.queue.push(...events) + if (!this.draining) { + this.draining = true + queueMicrotask(() => this.drain()) + } + } + + /** Deliver everything queued, FIFO, isolating listener errors. */ + private drain(): void { + try { + while (this.queue.length > 0) { + const event = this.queue.shift()! + for (const listener of this.listeners) { + try { + listener(event) + } catch (err) { + // A subscriber's bug must never affect the write path or its + // sibling subscribers. + console.error('[Brainy] onChange listener threw:', err) + } + } + } + } finally { + this.draining = false + } + } + + /** Drop all listeners (brain close/teardown). Queued events are discarded. */ + close(): void { + this.listeners.clear() + this.queue.length = 0 + } +} diff --git a/src/graph/analyticsFallback.ts b/src/graph/analyticsFallback.ts new file mode 100644 index 00000000..cf863d74 --- /dev/null +++ b/src/graph/analyticsFallback.ts @@ -0,0 +1,244 @@ +/** + * @module graph/analyticsFallback + * @description Pure-TS graph-analytics kernels — the fallback implementations + * behind `brain.graph.rank` / `brain.graph.communities` when no native + * {@link import('../plugin.js').GraphAccelerationProvider} is registered. Each + * operates on a dense integer adjacency (`outAdj[i]` = the target indices of node + * `i`'s out-edges, multiplicity preserved) so callers can build it once from the + * UUID graph and run any kernel. + * + * These are INTENT-level fallbacks: the public API promises the QUESTION + * ("which nodes matter most", "which things group together") — these answer it + * with the standard textbook algorithm (PageRank, connected components, + * Tarjan SCC). A native provider may answer the same intent with a different + * algorithm (personalized PageRank, Louvain, etc.); correctness of the INTENT, + * not the algorithm, is the contract. + * + * Correct at small/medium scale (the native provider is the at-scale path). All + * graph walks are iterative (explicit stacks/queues) so a deep or wide graph + * never blows the call stack. + */ + +/** Adjacency where `outAdj[i]` lists the target node indices of `i`'s out-edges. */ +export type DenseAdjacency = ReadonlyArray> + +/** + * @description Label each node with the id of its WEAKLY-connected component + * (edges treated as undirected) via union-find with path compression + a single + * pass over the adjacency. Two nodes share a label iff one can reach the other + * ignoring edge direction. + * @param outAdj - Dense out-adjacency. + * @returns `labels[i]` = the component representative of node `i` (labels are + * stable per component but NOT necessarily contiguous — group by equality). + */ +export function connectedComponents(outAdj: DenseAdjacency): number[] { + const n = outAdj.length + const parent = new Array(n) + for (let i = 0; i < n; i++) parent[i] = i + + const find = (x: number): number => { + let root = x + while (parent[root] !== root) root = parent[root] + // Path compression: point every node on the chain straight at the root. + while (parent[x] !== root) { + const next = parent[x] + parent[x] = root + x = next + } + return root + } + + for (let i = 0; i < n; i++) { + const neighbors = outAdj[i] + for (let k = 0; k < neighbors.length; k++) { + const ra = find(i) + const rb = find(neighbors[k]) + if (ra !== rb) parent[ra] = rb + } + } + + const labels = new Array(n) + for (let i = 0; i < n; i++) labels[i] = find(i) + return labels +} + +/** + * @description Label each node with its STRONGLY-connected component (directed — + * two nodes share a label iff each is reachable from the other following edge + * direction) via an iterative Tarjan's algorithm. + * @param outAdj - Dense out-adjacency. + * @returns `labels[i]` = the SCC index of node `i` (contiguous `0..count-1`). + */ +export function stronglyConnectedComponents(outAdj: DenseAdjacency): number[] { + const n = outAdj.length + const index = new Array(n).fill(-1) + const low = new Array(n).fill(0) + const onStack = new Array(n).fill(false) + const comp = new Array(n).fill(-1) + const sccStack: number[] = [] + let nextIndex = 0 + let compCount = 0 + + for (let start = 0; start < n; start++) { + if (index[start] !== -1) continue + // Explicit DFS stack of frames; `pi` is the next out-edge to visit for `node`. + const callStack: Array<{ node: number; pi: number }> = [{ node: start, pi: 0 }] + while (callStack.length > 0) { + const frame = callStack[callStack.length - 1] + const v = frame.node + if (frame.pi === 0) { + index[v] = low[v] = nextIndex++ + sccStack.push(v) + onStack[v] = true + } + if (frame.pi < outAdj[v].length) { + const w = outAdj[v][frame.pi] + frame.pi++ + if (index[w] === -1) { + callStack.push({ node: w, pi: 0 }) + } else if (onStack[w]) { + if (index[w] < low[v]) low[v] = index[w] + } + } else { + // All edges of v explored — if v roots an SCC, pop it off the SCC stack. + if (low[v] === index[v]) { + for (;;) { + const w = sccStack.pop() as number + onStack[w] = false + comp[w] = compCount + if (w === v) break + } + compCount++ + } + callStack.pop() + if (callStack.length > 0) { + const parent = callStack[callStack.length - 1].node + if (low[v] < low[parent]) low[parent] = low[v] + } + } + } + } + + return comp +} + +/** Tuning knobs for {@link pageRank}. */ +export interface PageRankOptions { + /** Teleport/damping factor (default `0.85`). */ + damping?: number + /** Max power-iterations before stopping (default `100`). */ + maxIterations?: number + /** L1 convergence threshold on the score vector (default `1e-6`). */ + tolerance?: number +} + +/** + * @description PageRank importance score per node via power-iteration. Dangling + * nodes (no out-edges) redistribute their mass uniformly so the score vector + * stays a probability distribution (sums to 1). Edge multiplicity is honored + * (a node with two edges to the same target sends it twice the share). + * @param outAdj - Dense out-adjacency. + * @param options - Damping / iteration / tolerance knobs. + * @returns `scores[i]` = node `i`'s PageRank (higher = more central). + */ +export function pageRank(outAdj: DenseAdjacency, options?: PageRankOptions): Float64Array { + const n = outAdj.length + if (n === 0) return new Float64Array(0) + + const damping = options?.damping ?? 0.85 + const maxIterations = options?.maxIterations ?? 100 + const tolerance = options?.tolerance ?? 1e-6 + + const outDegree = new Array(n) + for (let i = 0; i < n; i++) outDegree[i] = outAdj[i].length + + let rank = new Float64Array(n) + rank.fill(1 / n) + const teleport = (1 - damping) / n + + for (let iter = 0; iter < maxIterations; iter++) { + // Mass stranded on dangling nodes this round, spread uniformly to everyone. + let danglingMass = 0 + for (let i = 0; i < n; i++) if (outDegree[i] === 0) danglingMass += rank[i] + const danglingShare = (damping * danglingMass) / n + + const next = new Float64Array(n) + next.fill(teleport + danglingShare) + + for (let i = 0; i < n; i++) { + const degree = outDegree[i] + if (degree === 0) continue + const share = (damping * rank[i]) / degree + const neighbors = outAdj[i] + for (let k = 0; k < neighbors.length; k++) next[neighbors[k]] += share + } + + let delta = 0 + for (let i = 0; i < n; i++) delta += Math.abs(next[i] - rank[i]) + rank = next + if (delta < tolerance) break + } + + return rank +} + +/** + * @description A minimal binary min-heap (priority queue) — used by the weighted + * shortest-path (Dijkstra) fallback to pop the lowest-cost frontier node in + * O(log n). Stable enough for pathfinding; ties pop in unspecified order. + */ +export class MinHeap { + private readonly items: Array<{ value: T; priority: number }> = [] + + /** Number of queued items. */ + get size(): number { + return this.items.length + } + + /** + * @description Insert `value` with the given `priority` (lower pops first). + * @param value - The payload. + * @param priority - Ordering key (ascending). + */ + push(value: T, priority: number): void { + const items = this.items + items.push({ value, priority }) + let i = items.length - 1 + while (i > 0) { + const parent = (i - 1) >> 1 + if (items[parent].priority <= items[i].priority) break + const tmp = items[parent] + items[parent] = items[i] + items[i] = tmp + i = parent + } + } + + /** + * @description Remove and return the lowest-priority value. + * @returns The min value, or `undefined` when empty. + */ + pop(): T | undefined { + const items = this.items + if (items.length === 0) return undefined + const top = items[0].value + const last = items.pop() as { value: T; priority: number } + if (items.length > 0) { + items[0] = last + let i = 0 + for (;;) { + const left = 2 * i + 1 + const right = 2 * i + 2 + let smallest = i + if (left < items.length && items[left].priority < items[smallest].priority) smallest = left + if (right < items.length && items[right].priority < items[smallest].priority) smallest = right + if (smallest === i) break + const tmp = items[smallest] + items[smallest] = items[i] + items[i] = tmp + i = smallest + } + } + return top + } +} diff --git a/src/graph/graphAdjacencyIndex.ts b/src/graph/graphAdjacencyIndex.ts index eaf948c5..b37391aa 100644 --- a/src/graph/graphAdjacencyIndex.ts +++ b/src/graph/graphAdjacencyIndex.ts @@ -77,9 +77,10 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { private lsmTreeVerbsBySource: LSMTree // sourceId -> verbIds private lsmTreeVerbsByTarget: LSMTree // targetId -> verbIds - // ID-only tracking for billion-scale memory optimization - // Previous: Map stored full objects (128GB @ 1B verbs) - // Now: Set stores only IDs (~100KB @ 1B verbs) = 1,280,000x reduction + // ID-only membership tracking: a Set of verb ids rather than a + // Map of full objects, so per-verb resident memory is one id + // string instead of a whole relationship record. (Unmeasured order-of-magnitude + // figures dropped — the durable property is "ids, not objects".) private verbIdSet = new Set() // Verb-id interning for the BigInt boundary (8.0 u64 contract). @@ -194,6 +195,25 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { return next } + /** + * Eager cold-load of the persisted adjacency (readiness contract). + * + * Loads the LSM manifests + SSTables so `size()` reports the durable edge + * count at the rebuild gate — a warm reopen must load the persisted index, + * never re-derive it from a full canonical verb scan (the every-boot O(E) + * cost this method exists to eliminate). Idempotent; the lazy read paths + * call the same `ensureInitialized()` on demand. + * + * NOTE: the JS index deliberately does NOT expose `isReady()`. That signal + * (see `GraphIndexProvider.isReady`) asserts "traversals are trustworthy", + * which this side cannot honestly promise without comparing against the + * canonical store — the query-time known-edge probe + * (`verifyGraphAdjacencyLive` Strategy 2) remains the JS trust check. + */ + async init(): Promise { + await this.ensureInitialized() + } + /** * Initialize the graph index (lazy initialization) * Added defensive auto-rebuild check for verbIdSet consistency @@ -262,6 +282,16 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { } hasMore = result.hasMore + if (hasMore && (!result.nextCursor || result.nextCursor === cursor)) { + // A stalled cursor with hasMore=true would re-read the same page + // forever — a silent full-CPU loop at cold open. Abort loudly; a + // graph read failing beats a process that spins without a log line. + throw new Error( + `GraphAdjacencyIndex: verb walk stalled after ${count} verbs — storage returned ` + + `hasMore=true with ${result.nextCursor ? 'a non-advancing' : 'no'} cursor. ` + + `Aborting the cold-load; run brain.repairIndex() if this persists.` + ) + } cursor = result.nextCursor } @@ -877,6 +907,11 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { this.flushTimer = setInterval(async () => { await this.flush() }, this.config.flushInterval) + // Background maintenance must never keep the host process alive — + // close()/flush() handle durability; the interval is best-effort. + if (typeof this.flushTimer.unref === 'function') { + this.flushTimer.unref() + } } /** diff --git a/src/graph/graphAudit.ts b/src/graph/graphAudit.ts new file mode 100644 index 00000000..d0e44cd9 --- /dev/null +++ b/src/graph/graphAudit.ts @@ -0,0 +1,214 @@ +/** + * @module graph/graphAudit + * @description Read-only graph-truth audit — the graph sibling of `repairIndex()`'s + * diagnosis half. Verifies three layers against each other without mutating anything: + * + * 1. CANONICAL verb records (the storage walk — the source of truth) + * 2. the RELATIONSHIP READ PATH (`related()` with all visibility tiers — exactly + * what application reads like a VFS `readdir` consult) + * 3. ENTITY ENDPOINTS (does each verb's source/target still exist?) + * + * and classifies every discrepancy into the three failure families production + * incidents have shown: + * + * - `missingFromReads` — a canonical verb record the read path does NOT return + * for its source: PRESENT BUT INVISIBLE (adjacency/membership staleness). + * - `danglingEndpoints` — a canonical verb whose endpoint entity is gone: + * the SCAR class (write-path loss / partial delete). + * - `readOnlyVerbIds` — the read path returns an edge with NO canonical + * record: GHOST edges (stale index entries). + * + * `visibilityHiddenCount` is reported separately: an internal/system edge that is + * indexed and present but hidden from DEFAULT reads is working as designed — the + * audit reads with all tiers included so design-hiding is never misclassified as + * index loss. + * + * Full counts are always exact; only the example LISTS are capped (`maxExamples`) + * — a capped report says so via `truncatedExamples`, never silently. + */ + +import { prodLog } from '../utils/logger.js' + +/** One discrepant relationship, identified fully enough to inspect by hand. */ +export interface GraphAuditDiscrepancy { + verbId: string + from: string + to: string + type: string +} + +export interface GraphAuditReport { + /** True iff every discrepancy count is zero — `related()` returns canonical truth. */ + coherent: boolean + verbsInCanonical: number + entitiesInCanonical: number + /** Distinct source entities whose read path was actually consulted (coverage honesty). */ + sourcesChecked: number + + /** PRESENT BUT INVISIBLE: canonical records the read path omits. */ + missingFromReadsCount: number + missingFromReads: GraphAuditDiscrepancy[] + + /** SCAR CLASS: canonical verbs with a missing endpoint entity. */ + danglingEndpointsCount: number + danglingEndpoints: Array + + /** GHOST EDGES: read-path verb ids with no canonical record. */ + readOnlyCount: number + readOnlyVerbIds: string[] + + /** Canonical verbs hidden from DEFAULT reads by design (internal/system visibility). */ + visibilityHiddenCount: number + + /** Example lists above were capped at maxExamples; counts remain exact. */ + truncatedExamples: boolean + durationMs: number +} + +/** A canonical verb record, as the audit needs it. */ +export interface AuditVerbRecord { + id: string + type: string + sourceId: string + targetId: string + visibility?: string +} + +/** The seams the audit runs over — injected so the walk is testable in isolation. */ +export interface GraphAuditDeps { + /** Stream every canonical entity id (id-only; no per-entity reads needed). */ + eachNounId(consume: (id: string) => void): Promise + /** Stream every canonical verb record. */ + eachVerb(consume: (verb: AuditVerbRecord) => void): Promise + /** + * The END-TO-END relationship read for one source, ALL visibility tiers + * included — must be the same path application reads consult. + */ + readRelationsFrom(sourceId: string): Promise> +} + +export interface GraphAuditOptions { + /** Cap on entries per example list (counts stay exact). Default 100. */ + maxExamples?: number +} + +export async function runGraphAudit( + deps: GraphAuditDeps, + options: GraphAuditOptions = {} +): Promise { + const maxExamples = options.maxExamples ?? 100 + const started = Date.now() + + // 1. Canonical entity ids — endpoint existence oracle. + const entityIds = new Set() + await deps.eachNounId((id) => entityIds.add(id)) + + // 2. Canonical verb walk: group by source, check endpoints, note visibility. + const canonicalVerbIds = new Set() + const bySource = new Map() + let verbsInCanonical = 0 + let visibilityHiddenCount = 0 + let danglingEndpointsCount = 0 + const danglingEndpoints: GraphAuditReport['danglingEndpoints'] = [] + + await deps.eachVerb((verb) => { + verbsInCanonical++ + canonicalVerbIds.add(verb.id) + const list = bySource.get(verb.sourceId) + if (list) list.push(verb) + else bySource.set(verb.sourceId, [verb]) + + if (verb.visibility === 'internal' || verb.visibility === 'system') { + visibilityHiddenCount++ + } + + const fromMissing = !entityIds.has(verb.sourceId) + const toMissing = !entityIds.has(verb.targetId) + if (fromMissing || toMissing) { + danglingEndpointsCount++ + if (danglingEndpoints.length < maxExamples) { + danglingEndpoints.push({ + verbId: verb.id, + from: verb.sourceId, + to: verb.targetId, + type: verb.type, + missingEnd: fromMissing && toMissing ? 'both' : fromMissing ? 'from' : 'to' + }) + } + } + }) + + // 3. Per-source read-path comparison. A verb must be returned by the read + // path of ITS OWN source — the exact consult a readdir/traversal makes. + let missingFromReadsCount = 0 + const missingFromReads: GraphAuditDiscrepancy[] = [] + let readOnlyCount = 0 + const readOnlyVerbIds: string[] = [] + const readOnlySeen = new Set() + + for (const [sourceId, verbs] of bySource) { + const readIds = new Set((await deps.readRelationsFrom(sourceId)).map((r) => r.id)) + + for (const verb of verbs) { + if (!readIds.has(verb.id)) { + missingFromReadsCount++ + if (missingFromReads.length < maxExamples) { + missingFromReads.push({ + verbId: verb.id, + from: verb.sourceId, + to: verb.targetId, + type: verb.type + }) + } + } + } + + for (const readId of readIds) { + if (!canonicalVerbIds.has(readId) && !readOnlySeen.has(readId)) { + readOnlySeen.add(readId) + readOnlyCount++ + if (readOnlyVerbIds.length < maxExamples) { + readOnlyVerbIds.push(readId) + } + } + } + } + + const coherent = + missingFromReadsCount === 0 && danglingEndpointsCount === 0 && readOnlyCount === 0 + + const report: GraphAuditReport = { + coherent, + verbsInCanonical, + entitiesInCanonical: entityIds.size, + sourcesChecked: bySource.size, + missingFromReadsCount, + missingFromReads, + danglingEndpointsCount, + danglingEndpoints, + readOnlyCount, + readOnlyVerbIds, + visibilityHiddenCount, + truncatedExamples: + missingFromReadsCount > missingFromReads.length || + danglingEndpointsCount > danglingEndpoints.length || + readOnlyCount > readOnlyVerbIds.length, + durationMs: Date.now() - started + } + + if (coherent) { + prodLog.info( + `[GraphAudit] coherent: ${verbsInCanonical} verbs across ${bySource.size} sources — ` + + `the read path returns canonical truth (${report.durationMs}ms)` + ) + } else { + prodLog.warn( + `[GraphAudit] INCOHERENT: ${missingFromReadsCount} present-but-invisible, ` + + `${danglingEndpointsCount} dangling-endpoint, ${readOnlyCount} ghost ` + + `(of ${verbsInCanonical} canonical verbs, ${bySource.size} sources, ` + + `${visibilityHiddenCount} visibility-hidden by design) — ${report.durationMs}ms` + ) + } + + return report +} diff --git a/src/graph/lsm/LSMTree.ts b/src/graph/lsm/LSMTree.ts index 32f941bb..e19ec145 100644 --- a/src/graph/lsm/LSMTree.ts +++ b/src/graph/lsm/LSMTree.ts @@ -1,21 +1,21 @@ /** - * LSMTree - Log-Structured Merge Tree for Graph Storage - * - * Production-grade LSM-tree implementation that reduces memory usage - * from 500GB to 1.3GB for 1 billion relationships while maintaining - * sub-5ms read performance. + * @module graph/lsm/LSMTree + * @description Log-Structured Merge tree for the JS (open-core) graph store — the + * fallback used when no native graph provider is registered. Verb-id postings are + * buffered in an in-memory MemTable, flushed to immutable sorted SSTables, and + * background-compacted; bloom filters give fast negative lookups. * * Architecture: - * - MemTable: In-memory write buffer (100K relationships, ~24MB) - * - SSTables: Immutable sorted files on disk (10K relationships each) - * - Bloom Filters: In-memory filters for fast negative lookups - * - Compaction: Background merging of SSTables + * - MemTable: in-memory write buffer (flush threshold configurable, default 100K) + * - SSTables: immutable sorted segments, persisted via the StorageAdapter + * - Bloom filters: in-memory membership pre-checks + * - Compaction: background merge of SSTables (reclaims superseded segments) * - * Key Properties: - * - Write-optimized: O(1) writes to MemTable - * - Read-efficient: O(log n) reads with bloom filter optimization - * - Memory-efficient: 385x less memory than all-in-RAM approach - * - Storage-agnostic: Works with any StorageAdapter + * Complexity: O(1) MemTable writes; O(log n) reads with bloom-filter pre-filtering. + * Note: this JS implementation loads its SSTables into memory after open (it is the + * fallback engine). Billion-scale, on-disk-resident operation is the native graph + * provider's role behind the provider boundary, not this fallback's; no absolute + * memory/latency figures are claimed here without a cited benchmark. */ import { StorageAdapter } from '../../coreTypes.js' @@ -457,15 +457,20 @@ export class LSMTree { } }) - // Delete old SSTables from storage + // Reclaim the compacted-away SSTables: drop them from the manifest AND + // delete their persisted payloads, so the system channel does not grow + // unbounded with graph write volume. (deleteMetadata is idempotent, so a + // never-persisted memtable-only SSTable is a harmless no-op.) for (const sstable of sstables) { const oldKey = `${this.config.storagePrefix}-${sstable.metadata.id}` + this.manifest.sstables.delete(sstable.metadata.id) try { - // StorageAdapter doesn't have deleteMetadata, so we'll leave orphaned data - // In production, we'd add a cleanup mechanism - this.manifest.sstables.delete(sstable.metadata.id) + await this.storage.deleteMetadata(oldKey) } catch (error) { - prodLog.warn(`LSMTree: Failed to delete old SSTable ${sstable.metadata.id}`, error) + // A reclaim failure must not abort compaction (the merged SSTable is + // already durable and the manifest no longer references the old one); + // surface it so a persistent leak is visible rather than silent. + prodLog.warn(`LSMTree: failed to delete compacted SSTable ${sstable.metadata.id}`, error) } } @@ -512,6 +517,11 @@ export class LSMTree { } } }, this.config.compactionInterval) + // Background compaction must never keep the host process alive — + // close() compacts/flushes deterministically; this interval is best-effort. + if (this.compactionTimer && typeof this.compactionTimer.unref === 'function') { + this.compactionTimer.unref() + } } /** @@ -537,13 +547,26 @@ export class LSMTree { const data = metadata.data as PersistedManifestData this.manifest.sstables = new Map(Object.entries(data.sstables || {})) this.manifest.lastCompaction = data.lastCompaction || Date.now() - this.manifest.totalRelationships = data.totalRelationships || 0 - // Load SSTables from storage + // Load SSTables from storage BEFORE publishing the persisted count. + // If the SSTable load throws, `size()` must keep reporting 0 — a tree + // that claims its persisted relationships while holding none serves + // silent-empty traversals as truth (the cold-load swallow class), and + // downstream self-heal keys off the honest 0. await this.loadSSTables() + this.manifest.totalRelationships = data.totalRelationships || 0 } } catch (error) { - prodLog.debug('LSMTree: No existing manifest found, starting fresh') + // Reset anything partially loaded — an honest empty tree triggers the + // rebuild/self-heal paths; a half-loaded one masks them. (An absent + // manifest on a fresh store also lands here: empty is correct.) + this.manifest.sstables = new Map() + this.manifest.totalRelationships = 0 + this.sstablesByLevel.clear() + prodLog.debug( + `LSMTree(${this.config.storagePrefix}): no loadable manifest/SSTables — starting empty ` + + `(${error instanceof Error ? error.message : String(error)})` + ) } } @@ -551,6 +574,7 @@ export class LSMTree { * Load SSTables from storage based on manifest */ private async loadSSTables(): Promise { + const failures: string[] = [] const loadPromises: Promise[] = [] this.manifest.sstables.forEach((level, sstableId) => { @@ -575,7 +599,12 @@ export class LSMTree { } } } catch (error) { + // A per-SSTable load failure means the persisted adjacency is INCOMPLETE. + // Record it and fail the whole load closed (below): a partially-loaded + // tree that still publishes its full manifest count via size() would + // serve silent-empty traversals as truth (the cold-load swallow class). prodLog.warn(`LSMTree: Failed to load SSTable ${sstableId}`, error) + failures.push(sstableId) } })() @@ -583,6 +612,20 @@ export class LSMTree { }) await Promise.all(loadPromises) + + if (failures.length > 0) { + // Fail closed. loadManifest()'s catch resets sstables/totalRelationships/ + // sstablesByLevel to honest-empty, so size() reports 0 and the graph + // self-heal (_initializeGraphIndex size()===0 → rebuild) restores the index + // from the canonical records. Honest-partial is never published. + throw new Error( + `LSMTree(${this.config.storagePrefix}): ${failures.length} of ` + + `${this.manifest.sstables.size} SSTable(s) failed to load ` + + `(${failures.join(', ')}) — failing the load closed so size() reports 0 ` + + `and the graph self-heal rebuilds from canonical.` + ) + } + prodLog.info(`LSMTree: Loaded ${this.manifest.sstables.size} SSTables`) } diff --git a/src/graph/lsm/SSTable.ts b/src/graph/lsm/SSTable.ts index e7ce7c89..e7e21c01 100644 --- a/src/graph/lsm/SSTable.ts +++ b/src/graph/lsm/SSTable.ts @@ -20,13 +20,13 @@ import { BloomFilter, SerializedBloomFilter } from './BloomFilter.js' import { compareCodePoints } from '../../utils/collation.js' // Swappable msgpack implementation — defaults to @msgpack/msgpack JS, -// can be replaced with native msgpack (e.g., cortex's Rust-backed encoder) +// can be replaced with native msgpack (e.g., cor's Rust-backed encoder) let _encode: (data: unknown) => Uint8Array = defaultEncode as (data: unknown) => Uint8Array let _decode: (data: Uint8Array) => unknown = defaultDecode as (data: Uint8Array) => unknown /** * Replace the msgpack encode/decode implementation at runtime. - * Called by brainy.ts when a cortex 'msgpack' provider is registered. + * Called by brainy.ts when a cor 'msgpack' provider is registered. */ export function setMsgpackImplementation(impl: { encode: (data: unknown) => Uint8Array, decode: (data: Uint8Array) => unknown }) { _encode = impl.encode diff --git a/src/graph/pathfinding.ts b/src/graph/pathfinding.ts deleted file mode 100644 index 46f26e9c..00000000 --- a/src/graph/pathfinding.ts +++ /dev/null @@ -1,523 +0,0 @@ -/** - * Advanced Graph Pathfinding Algorithms - * Provides shortest path, multi-hop traversal, and path ranking - */ - -// Graph pathfinding doesn't need to import from coreTypes - -export interface GraphNode { - id: string - [key: string]: any -} - -export interface GraphEdge { - source: string - target: string - type: string - weight: number - metadata?: any -} - -export interface Path { - nodes: string[] - edges: GraphEdge[] - totalWeight: number - length: number -} - -export interface PathfindingOptions { - maxDepth?: number - maxPaths?: number - bidirectional?: boolean - weightField?: string - relationshipTypes?: string[] - nodeFilter?: (node: GraphNode) => boolean - edgeFilter?: (edge: GraphEdge) => boolean -} - -export class GraphPathfinding { - private adjacencyList: Map> = new Map() - private nodes: Map = new Map() - - /** - * Add a node to the graph - */ - public addNode(node: GraphNode): void { - this.nodes.set(node.id, node) - if (!this.adjacencyList.has(node.id)) { - this.adjacencyList.set(node.id, new Map()) - } - } - - /** - * Add an edge to the graph - */ - public addEdge(edge: GraphEdge): void { - // Ensure nodes exist - if (!this.adjacencyList.has(edge.source)) { - this.adjacencyList.set(edge.source, new Map()) - } - if (!this.adjacencyList.has(edge.target)) { - this.adjacencyList.set(edge.target, new Map()) - } - - // Add edge to adjacency list - const sourceEdges = this.adjacencyList.get(edge.source)! - if (!sourceEdges.has(edge.target)) { - sourceEdges.set(edge.target, []) - } - sourceEdges.get(edge.target)!.push(edge) - } - - /** - * Find shortest path using Dijkstra's algorithm - * O((V + E) log V) with binary heap - */ - public shortestPath( - start: string, - end: string, - options: PathfindingOptions = {} - ): Path | null { - const { - maxDepth = Infinity, - relationshipTypes, - edgeFilter - } = options - - // Priority queue: [nodeId, distance, path] - const pq: Array<[string, number, string[], GraphEdge[]]> = [[start, 0, [start], []]] - const visited = new Set() - const distances = new Map([[start, 0]]) - - while (pq.length > 0) { - // Sort by distance (simple array, could optimize with heap) - pq.sort((a, b) => a[1] - b[1]) - const [current, distance, path, edges] = pq.shift()! - - if (visited.has(current)) continue - visited.add(current) - - // Found target - if (current === end) { - return { - nodes: path, - edges, - totalWeight: distance, - length: path.length - 1 - } - } - - // Max depth reached - if (path.length > maxDepth) continue - - // Explore neighbors - const neighbors = this.adjacencyList.get(current) - if (!neighbors) continue - - for (const [neighbor, edgeList] of neighbors) { - if (visited.has(neighbor)) continue - - // Find best edge to neighbor - let bestEdge: GraphEdge | null = null - let bestWeight = Infinity - - for (const edge of edgeList) { - // Apply filters - if (relationshipTypes && !relationshipTypes.includes(edge.type)) continue - if (edgeFilter && !edgeFilter(edge)) continue - - if (edge.weight < bestWeight) { - bestWeight = edge.weight - bestEdge = edge - } - } - - if (!bestEdge) continue - - const newDistance = distance + bestWeight - const currentBest = distances.get(neighbor) ?? Infinity - - if (newDistance < currentBest) { - distances.set(neighbor, newDistance) - pq.push([ - neighbor, - newDistance, - [...path, neighbor], - [...edges, bestEdge] - ]) - } - } - } - - return null // No path found - } - - /** - * Find all paths between two nodes - * Uses DFS with cycle detection - */ - public allPaths( - start: string, - end: string, - options: PathfindingOptions = {} - ): Path[] { - const { - maxDepth = 10, - maxPaths = 100, - relationshipTypes, - edgeFilter - } = options - - const paths: Path[] = [] - const visited = new Set() - - const dfs = ( - current: string, - path: string[], - edges: GraphEdge[], - weight: number - ): void => { - if (paths.length >= maxPaths) return - if (path.length > maxDepth) return - - if (current === end && path.length > 1) { - paths.push({ - nodes: [...path], - edges: [...edges], - totalWeight: weight, - length: path.length - 1 - }) - return - } - - visited.add(current) - - const neighbors = this.adjacencyList.get(current) - if (neighbors) { - for (const [neighbor, edgeList] of neighbors) { - if (visited.has(neighbor)) continue - - for (const edge of edgeList) { - // Apply filters - if (relationshipTypes && !relationshipTypes.includes(edge.type)) continue - if (edgeFilter && !edgeFilter(edge)) continue - - dfs( - neighbor, - [...path, neighbor], - [...edges, edge], - weight + edge.weight - ) - } - } - } - - visited.delete(current) - } - - dfs(start, [start], [], 0) - - // Sort paths by weight - paths.sort((a, b) => a.totalWeight - b.totalWeight) - - return paths - } - - /** - * Bidirectional search for faster pathfinding - * Searches from both start and end simultaneously - */ - public bidirectionalSearch( - start: string, - end: string, - options: PathfindingOptions = {} - ): Path | null { - const { maxDepth = 10 } = options - - // Two search frontiers - const forwardVisited = new Map() - const backwardVisited = new Map() - - forwardVisited.set(start, { path: [start], edges: [], weight: 0 }) - backwardVisited.set(end, { path: [end], edges: [], weight: 0 }) - - const forwardQueue = [start] - const backwardQueue = [end] - - let depth = 0 - - while ( - (forwardQueue.length > 0 || backwardQueue.length > 0) && - depth < maxDepth - ) { - // Expand forward frontier - const forwardNext: string[] = [] - for (const current of forwardQueue) { - const currentData = forwardVisited.get(current)! - const neighbors = this.adjacencyList.get(current) - - if (neighbors) { - for (const [neighbor, edges] of neighbors) { - if (forwardVisited.has(neighbor)) continue - - // Select edge with lowest weight for optimal path - const bestEdge = edges.reduce((best, edge) => - edge.weight < best.weight ? edge : best, edges[0]) - forwardVisited.set(neighbor, { - path: [...currentData.path, neighbor], - edges: [...currentData.edges, bestEdge], - weight: currentData.weight + bestEdge.weight - }) - - // Check if we met the backward search - if (backwardVisited.has(neighbor)) { - const forward = forwardVisited.get(neighbor)! - const backward = backwardVisited.get(neighbor)! - - // Combine paths - const fullPath = [ - ...forward.path, - ...backward.path.slice(1).reverse() - ] - - // Reverse backward edges and combine - const backwardEdgesReversed = backward.edges - .map(e => ({ - ...e, - source: e.target, - target: e.source - })) - .reverse() - - return { - nodes: fullPath, - edges: [...forward.edges, ...backwardEdgesReversed], - totalWeight: forward.weight + backward.weight, - length: fullPath.length - 1 - } - } - - forwardNext.push(neighbor) - } - } - } - - // Expand backward frontier - const backwardNext: string[] = [] - for (const current of backwardQueue) { - const currentData = backwardVisited.get(current)! - - // For backward search, we need to look at incoming edges - for (const [nodeId, neighbors] of this.adjacencyList) { - const edges = neighbors.get(current) - if (!edges) continue - - if (backwardVisited.has(nodeId)) continue - - // Select edge with lowest weight for optimal path - const bestEdge = edges.reduce((best, edge) => - edge.weight < best.weight ? edge : best, edges[0]) - backwardVisited.set(nodeId, { - path: [...currentData.path, nodeId], - edges: [...currentData.edges, bestEdge], - weight: currentData.weight + bestEdge.weight - }) - - // Check if we met the forward search - if (forwardVisited.has(nodeId)) { - const forward = forwardVisited.get(nodeId)! - const backward = backwardVisited.get(nodeId)! - - // Combine paths - const fullPath = [ - ...forward.path, - ...backward.path.slice(1).reverse() - ] - - // Reverse backward edges and combine - const backwardEdgesReversed = backward.edges - .map(e => ({ - ...e, - source: e.target, - target: e.source - })) - .reverse() - - return { - nodes: fullPath, - edges: [...forward.edges, ...backwardEdgesReversed], - totalWeight: forward.weight + backward.weight, - length: fullPath.length - 1 - } - } - - backwardNext.push(nodeId) - } - } - - forwardQueue.splice(0, forwardQueue.length, ...forwardNext) - backwardQueue.splice(0, backwardQueue.length, ...backwardNext) - depth++ - } - - return null - } - - /** - * Multi-hop traversal (e.g., friends of friends) - * Returns all nodes within N hops - */ - public multiHopTraversal( - start: string, - hops: number, - options: PathfindingOptions = {} - ): Map { - const { relationshipTypes, nodeFilter, edgeFilter } = options - - const results = new Map() - const visited = new Set() - const queue: Array<{ node: string, distance: number, path: string[], edges: GraphEdge[] }> = [ - { node: start, distance: 0, path: [start], edges: [] } - ] - - while (queue.length > 0) { - const { node, distance, path, edges } = queue.shift()! - - if (distance > hops) continue - - // Record this node - if (!results.has(node)) { - results.set(node, { distance, paths: [] }) - } - results.get(node)!.paths.push({ - nodes: path, - edges, - totalWeight: edges.reduce((sum, e) => sum + e.weight, 0), - length: path.length - 1 - }) - - if (distance === hops) continue - - // Explore neighbors - const neighbors = this.adjacencyList.get(node) - if (neighbors) { - for (const [neighbor, edgeList] of neighbors) { - // Apply node filter - if (nodeFilter) { - const neighborNode = this.nodes.get(neighbor) - if (neighborNode && !nodeFilter(neighborNode)) continue - } - - for (const edge of edgeList) { - // Apply filters - if (relationshipTypes && !relationshipTypes.includes(edge.type)) continue - if (edgeFilter && !edgeFilter(edge)) continue - - queue.push({ - node: neighbor, - distance: distance + 1, - path: [...path, neighbor], - edges: [...edges, edge] - }) - } - } - } - } - - return results - } - - /** - * Find connected components using DFS - */ - public connectedComponents(): Array> { - const visited = new Set() - const components: Array> = [] - - const dfs = (node: string, component: Set): void => { - visited.add(node) - component.add(node) - - const neighbors = this.adjacencyList.get(node) - if (neighbors) { - for (const neighbor of neighbors.keys()) { - if (!visited.has(neighbor)) { - dfs(neighbor, component) - } - } - } - } - - for (const node of this.adjacencyList.keys()) { - if (!visited.has(node)) { - const component = new Set() - dfs(node, component) - components.push(component) - } - } - - return components - } - - /** - * Calculate PageRank for all nodes - * Useful for ranking importance in the graph - */ - public pageRank(iterations: number = 100, damping: number = 0.85): Map { - const nodes = Array.from(this.adjacencyList.keys()) - const n = nodes.length - - if (n === 0) return new Map() - - // Initialize ranks - const ranks = new Map() - for (const node of nodes) { - ranks.set(node, 1 / n) - } - - // Calculate outgoing edge counts - const outDegree = new Map() - for (const [node, neighbors] of this.adjacencyList) { - let count = 0 - for (const edges of neighbors.values()) { - count += edges.length - } - outDegree.set(node, count) - } - - // Iterate PageRank algorithm - for (let i = 0; i < iterations; i++) { - const newRanks = new Map() - - for (const node of nodes) { - let rank = (1 - damping) / n - - // Sum contributions from incoming edges - for (const [source, neighbors] of this.adjacencyList) { - if (neighbors.has(node)) { - const sourceRank = ranks.get(source) ?? 0 - const sourceOutDegree = outDegree.get(source) ?? 1 - rank += damping * (sourceRank / sourceOutDegree) - } - } - - newRanks.set(node, rank) - } - - // Update ranks - for (const [node, rank] of newRanks) { - ranks.set(node, rank) - } - } - - return ranks - } - - /** - * Clear the graph - */ - public clear(): void { - this.adjacencyList.clear() - this.nodes.clear() - } -} \ No newline at end of file diff --git a/src/hnsw/connectionsCodec.ts b/src/hnsw/connectionsCodec.ts index 91dc00b5..318e8d41 100644 --- a/src/hnsw/connectionsCodec.ts +++ b/src/hnsw/connectionsCodec.ts @@ -1,7 +1,7 @@ /** * @module hnsw/connectionsCodec * @description Bridge between brainy's UUID-keyed HNSW connection sets and - * cortex's int-keyed delta-varint encode/decode (the `graph:compression` + * cor's int-keyed delta-varint encode/decode (the `graph:compression` * provider). Translates UUIDs to stable int slots via the post-2.4.0 #1 * `EntityIdMapper`, batches all of a node's per-level connection lists into * a single compact buffer, and reverses the path on load. @@ -130,7 +130,7 @@ export class ConnectionsCodec { * @description Build the binary-blob storage key for a node's compressed * connections. Suffix-free; the adapter appends its own. Keys live under * `_hnsw_conn/` so they don't collide with the existing `_column_index/` - * blobs and so a cortex-side reader knows exactly where to look. + * blobs and so a cor-side reader knows exactly where to look. */ export function compressedConnectionsKey(nodeId: string): string { return `_hnsw_conn/${nodeId}` diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index 757ece69..605d2a0b 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -14,7 +14,7 @@ import { euclideanDistance, calculateDistancesBatch } from '../utils/index.js' import type { BaseStorage } from '../storage/baseStorage.js' import { getGlobalCache, UnifiedCache } from '../utils/unifiedCache.js' import { prodLog } from '../utils/logger.js' -import type { VectorIndexProvider } from '../plugin.js' +import type { VectorIndexProvider, OpaqueIdSet, AtGenerationVectors } from '../plugin.js' import { ConnectionsCodec, compressedConnectionsKey } from './connectionsCodec.js' // Default HNSW parameters @@ -25,6 +25,29 @@ const DEFAULT_CONFIG: HNSWConfig = { ml: 16 // Max level } +/** + * @description Thrown by {@link JsHnswVectorIndex.flush} when one or more dirty + * nodes (or the system record) could not be persisted. The failed nodes remain + * in the dirty set for the next flush; this error tells the caller the flush did + * NOT achieve durability instead of a node-count that lies. Mandate: loud + * errors, never quiet losses. + */ +export class HnswFlushError extends Error { + constructor( + public readonly failedNodeCount: number, + public readonly systemFailed: boolean, + public override readonly cause?: Error + ) { + super( + `HNSW flush did not achieve durability: ${failedNodeCount} node(s) failed to ` + + `persist${systemFailed ? ' and the system record (entryPoint/maxLevel) failed' : ''}. ` + + `Failed nodes remain dirty for retry.` + + (cause ? ` First error: ${cause.message}` : '') + ) + this.name = 'HnswFlushError' + } +} + /** * Implements {@link VectorIndexProvider}: the vector-index surface Brainy calls * on whatever the `'vector'` factory returns (its own `JsHnswVectorIndex`, or a native @@ -32,6 +55,14 @@ const DEFAULT_CONFIG: HNSWConfig = { */ export class JsHnswVectorIndex implements VectorIndexProvider { private nouns: Map = new Map() + /** + * Reverse adjacency: `target id → (level → set of node ids that link TO target)`. + * Lets `removeItem` find a node's in-neighbors in O(in-degree) instead of scanning + * the whole corpus (which made bulk delete O(N²)). Lazily built on first need and + * maintained incrementally on link/prune/remove; `null` means "stale — rebuild on + * next use" (set by every bulk path: cold-load restore + `clear()`). + */ + private incoming: Map>> | null = null private entryPointId: string | null = null private maxLevel = 0 // Track high-level nodes for O(1) entry point selection @@ -142,41 +173,69 @@ export class JsHnswVectorIndex implements VectorIndexProvider { const startTime = Date.now() const nodeCount = this.dirtyNodes.size - // Batch persist all dirty nodes concurrently + // Batch persist all dirty nodes concurrently. A node whose connections FAIL + // to persist must stay dirty (retried on the next flush) — clearing it would + // silently drop the write forever. Track failures; only successfully- + // persisted (or deleted) nodes leave the dirty set, so nodes added to it + // during this flush are preserved. + const failedNodes = new Set() + let firstError: Error | null = null + if (this.dirtyNodes.size > 0) { const batchSize = 50 // Reasonable batch size for cloud storage const nodeIds = Array.from(this.dirtyNodes) for (let i = 0; i < nodeIds.length; i += batchSize) { const batch = nodeIds.slice(i, i + batchSize) - const promises = batch.map(nodeId => { + const promises = batch.map(async nodeId => { const noun = this.nouns.get(nodeId) - if (!noun) return Promise.resolve() // Node was deleted - - return this.persistNodeConnections(nodeId, noun).catch(error => { - console.error(`[HNSW flush] Failed to persist node ${nodeId}:`, error) - }) + if (!noun) return // Node was deleted — drop it from the dirty set. + try { + await this.persistNodeConnections(nodeId, noun) + } catch (error) { + failedNodes.add(nodeId) + if (firstError === null) firstError = error as Error + prodLog.error(`[HNSW flush] Failed to persist node ${nodeId}: ${(error as Error).message}`) + } }) - await Promise.allSettled(promises) + await Promise.all(promises) } - this.dirtyNodes.clear() + // Remove only nodes that were persisted (or deleted mid-flush); keep the + // failed ones dirty for the next attempt. + for (const nodeId of nodeIds) { + if (!failedNodes.has(nodeId)) this.dirtyNodes.delete(nodeId) + } } - // Persist system data if dirty + // Persist system data if dirty — keep it dirty on failure so the next flush + // retries rather than losing the entry-point/maxLevel update. + let systemFailed = false if (this.dirtySystem) { - await this.storage.saveHNSWSystem({ - entryPointId: this.entryPointId, - maxLevel: this.maxLevel - }).catch(error => { - console.error('[HNSW flush] Failed to persist system data:', error) - }) - - this.dirtySystem = false + try { + await this.storage.saveHNSWSystem({ + entryPointId: this.entryPointId, + maxLevel: this.maxLevel + }) + this.dirtySystem = false + } catch (error) { + systemFailed = true + if (firstError === null) firstError = error as Error + prodLog.error(`[HNSW flush] Failed to persist system data: ${(error as Error).message}`) + } } const duration = Date.now() - startTime + + // Loud failure: if ANY node or the system record could not be persisted the + // flush did not achieve durability. Throw so callers (close(), explicit + // flush(), the flush-request watcher) see the failure instead of a success + // count that lies. The failed nodes/system stay dirty for retry. + if (failedNodes.size > 0 || systemFailed) { + throw new HnswFlushError(failedNodes.size, systemFailed, firstError ?? undefined) + } + if (nodeCount > 0) { prodLog.info(`[HNSW] Flushed ${nodeCount} dirty nodes in ${duration}ms`) } @@ -245,6 +304,9 @@ export class JsHnswVectorIndex implements VectorIndexProvider { hnswData: { level: number; connections: Record }, noun: HNSWNoun ): Promise { + // Bulk load sets connections directly (bypassing the link path that maintains + // the reverse index) — mark it stale so the next removeItem rebuilds it. + this.incoming = null const storageWithBlob = this.storage as unknown as { loadBinaryBlob?: (key: string) => Promise } @@ -385,13 +447,14 @@ export class JsHnswVectorIndex implements VectorIndexProvider { this.maxLevel = nounLevel this.nouns.set(id, noun) - // Persist system data for first noun (previously skipped) + // Persist system data for first noun (previously skipped). Surface a + // persist failure loudly — the entry point is the root of the whole index; + // silently dropping it while addItem() returns the id would strand every + // future search on a rootless index. Mandate: never a quiet loss. if (this.storage && this.persistMode === 'immediate') { await this.storage.saveHNSWSystem({ entryPointId: this.entryPointId, maxLevel: this.maxLevel - }).catch(error => { - console.error('Failed to persist initial HNSW system data:', error) }) } else if (this.persistMode === 'deferred') { this.dirtySystem = true @@ -489,12 +552,14 @@ export class JsHnswVectorIndex implements VectorIndexProvider { noun.connections.get(level)!.add(neighborId) + this.addIncoming(neighborId, level, id) // forward edge id → neighborId // Add reverse connection if (!neighbor.connections.has(level)) { neighbor.connections.set(level, new Set()) } neighbor.connections.get(level)!.add(id) + this.addIncoming(id, level, neighborId) // forward edge neighborId → id // Ensure neighbor doesn't have too many connections if (neighbor.connections.get(level)!.size > this.config.M) { @@ -656,23 +721,61 @@ export class JsHnswVectorIndex implements VectorIndexProvider { * @param queryVector Query vector * @param k Number of results to return * @param filter Optional filter function - * @param options Additional search options (`candidateIds` to restrict the - * search to a pre-filtered set; `rerank` is provider-only, ignored here) + * @param options Additional search options (`candidateIds` / `allowedIds` to + * restrict the search to a pre-filtered set; `rerank` is provider-only, ignored here) */ public async search( queryVector: Vector, k: number = 10, filter?: (id: string) => Promise, - options?: { rerank?: { multiplier: number }; candidateIds?: string[] } + options?: { + rerank?: { multiplier: number } + candidateIds?: string[] + allowedIds?: OpaqueIdSet | ReadonlySet + // 8.0 #35 at-gen seam: the JS index has no retained per-generation segments + // (it only ever holds "now"), so it ignores `generation` — exactly the + // refuse/fall-back posture the contract requires. Brainy never routes a + // historical read here (its defer-gate checks `isGenerationVisible` first), + // so an ignored `generation` cannot silently return now-vectors-as-at-gen. + generation?: bigint + // 8.0 #35 part-3: at-gen candidate vectors for the native exact-rerank. The JS + // index serves "now" only and never receives this (gated off upstream); ignored. + atGenerationVectors?: AtGenerationVectors + } ): Promise> { if (this.nouns.size === 0) { return [] } - // Metadata-first: convert candidateIds to filter function if no explicit filter - if (!filter && options?.candidateIds && options.candidateIds.length > 0) { - const candidateSet = new Set(options.candidateIds) - filter = async (id: string) => candidateSet.has(id) + // Metadata-first candidate restriction. Build a filter from the pre-resolved + // universe(s) when the caller didn't pass an explicit one. Both `candidateIds` + // (legacy string[]) and the 8.0 `allowedIds` predicate-pushdown param restrict + // the SAME way here — applied INSIDE the beam walk (searchLayer keeps every + // neighbor as a traversal candidate but only COLLECTS allowed ids), which is + // what recovers the filtered recall that post-filtering loses. An `allowedIds` + // that arrives as an OpaqueIdSet (a native/cor roaring Buffer) is opaque to the + // JS index — it cannot decode it — so it is ignored here; only the native + // provider consumes that form. A `ReadonlySet` is honored. When both + // `candidateIds` and a Set `allowedIds` are present they AND together. + if (!filter) { + const universes: ReadonlySet[] = [] + if (options?.candidateIds && options.candidateIds.length > 0) { + universes.push(new Set(options.candidateIds)) + } + const allowed = options?.allowedIds + if ( + allowed && + !(allowed instanceof Uint8Array) && + typeof (allowed as ReadonlySet).has === 'function' + ) { + universes.push(allowed as ReadonlySet) + } + if (universes.length === 1) { + const universe = universes[0] + filter = async (id: string) => universe.has(id) + } else if (universes.length > 1) { + filter = async (id: string) => universes.every((u) => u.has(id)) + } } // Check if query vector is defined @@ -803,6 +906,44 @@ export class JsHnswVectorIndex implements VectorIndexProvider { return [...nearestNouns].slice(0, k) } + /** + * Build (or return the cached) reverse-adjacency index. O(N + E) the first time + * after a bulk load/clear; O(1) thereafter while it stays live. + */ + private ensureIncoming(): Map>> { + if (this.incoming !== null) return this.incoming + const inc = new Map>>() + for (const [nodeId, node] of this.nouns) { + for (const [level, targets] of node.connections) { + for (const target of targets) { + let byLevel = inc.get(target) + if (!byLevel) { byLevel = new Map(); inc.set(target, byLevel) } + let set = byLevel.get(level) + if (!set) { set = new Set(); byLevel.set(level, set) } + set.add(nodeId) + } + } + } + this.incoming = inc + return inc + } + + /** Record a new forward edge `source → target` at `level` (no-op while the index is stale). */ + private addIncoming(target: string, level: number, source: string): void { + if (this.incoming === null) return + let byLevel = this.incoming.get(target) + if (!byLevel) { byLevel = new Map(); this.incoming.set(target, byLevel) } + let set = byLevel.get(level) + if (!set) { set = new Set(); byLevel.set(level, set) } + set.add(source) + } + + /** Record that the forward edge `source → target` at `level` is gone. */ + private removeIncoming(target: string, level: number, source: string): void { + if (this.incoming === null) return + this.incoming.get(target)?.get(level)?.delete(source) + } + /** * Remove an item from the index */ @@ -814,41 +955,38 @@ export class JsHnswVectorIndex implements VectorIndexProvider { const noun = this.nouns.get(id)! - // Remove connections to this noun from all neighbors - for (const [level, connections] of noun.connections.entries()) { - for (const neighborId of connections) { - - const neighbor = this.nouns.get(neighborId) - if (!neighbor) { - // Skip neighbors that don't exist (expected during rapid additions/deletions) - continue - } - if (neighbor.connections.has(level)) { - neighbor.connections.get(level)!.delete(id) - - // Prune connections after removing this noun to ensure consistency - await this.pruneConnections(neighbor, level) + // Reverse-adjacency lets us touch ONLY the nodes that actually reference `id` + // (its in-neighbors) rather than scanning the whole corpus — turning a delete + // from O(N) into O(in-degree) and a bulk delete from O(N²) into O(N·degree). + // Snapshot each referrer set because pruneConnections mutates the index. + const incoming = this.ensureIncoming() + const referrers = incoming.get(id) + if (referrers) { + for (const [level, refSet] of referrers) { + for (const refId of Array.from(refSet)) { + const ref = this.nouns.get(refId) + if (ref && ref.connections.has(level)) { + // Drop the forward edge ref → id, then re-prune ref so the graph stays + // navigable. (id's own reverse entry is dropped wholesale below, so we + // intentionally do not maintain incoming[id] inside this loop.) + ref.connections.get(level)!.delete(id) + await this.pruneConnections(ref, level) + } } } } - // Also check all other nouns for references to this noun and remove them - for (const [nounId, otherNoun] of this.nouns.entries()) { - if (nounId === id) continue // Skip the noun being removed - - - for (const [level, connections] of otherNoun.connections.entries()) { - if (connections.has(id)) { - connections.delete(id) - - // Prune connections after removing this reference - await this.pruneConnections(otherNoun, level) - } + // id's OUTGOING edges disappear with it → drop id from each out-neighbor's + // reverse set so no stale referrer survives. + for (const [level, targets] of noun.connections) { + for (const target of targets) { + this.removeIncoming(target, level, id) } } - // Remove the noun + // Remove the noun + its reverse-index entry. this.nouns.delete(id) + this.incoming?.delete(id) // If we removed the entry point, find a new one if (this.entryPointId === id) { @@ -928,6 +1066,7 @@ export class JsHnswVectorIndex implements VectorIndexProvider { */ public clear(): void { this.nouns.clear() + this.incoming = null // reverse index no longer reflects the (now empty) graph this.entryPointId = null this.maxLevel = 0 } @@ -1724,7 +1863,11 @@ export class JsHnswVectorIndex implements VectorIndexProvider { // Only proceed if we have valid neighbors if (distances.size === 0) { - // If no valid neighbors, clear connections at this level + // If no valid neighbors, clear connections at this level. Every current + // connection is dropped — unlink each from the reverse index. + if (this.incoming !== null) { + for (const dropped of connections) this.removeIncoming(dropped, level, noun.id) + } noun.connections.set(level, new Set()) return } @@ -1736,8 +1879,16 @@ export class JsHnswVectorIndex implements VectorIndexProvider { this.config.M ) - // Update connections with only valid neighbors - noun.connections.set(level, new Set(selectedNeighbors.keys())) + // Update connections with only valid neighbors. Pruning only ever drops + // edges (the selected set is a subset of the current connections), so unlink + // each dropped neighbor from the reverse index. + const kept = new Set(selectedNeighbors.keys()) + if (this.incoming !== null) { + for (const prev of connections) { + if (!kept.has(prev)) this.removeIncoming(prev, level, noun.id) + } + } + noun.connections.set(level, kept) } /** diff --git a/src/import/BackgroundDeduplicator.ts b/src/import/BackgroundDeduplicator.ts index 53b28262..073fbd9e 100644 --- a/src/import/BackgroundDeduplicator.ts +++ b/src/import/BackgroundDeduplicator.ts @@ -41,6 +41,14 @@ export interface DeduplicationStats { * - Import-scoped deduplication (no cross-contamination) * - 3-tier strategy (ID → Name → Similarity) * - Uses existing indexes (EntityIdMapper, MetadataIndexManager, TypeAware HNSW) + * + * Lifecycle: ONE instance per brain, owned by Brainy (getBackgroundDeduplicator) + * so the debounce genuinely spans imports and brain.close() cancels pending + * work via cancelPending() — this pass merge-DELETES duplicate entities, so it + * must never fire against a closed brain. The enableDeduplication gate lives + * at the scheduling call site (ImportCoordinator); scheduleDedup itself is + * unconditional. The timer is unref'd — a pending pass never holds the + * process open. */ export class BackgroundDeduplicator { private brain: Brainy @@ -67,12 +75,15 @@ export class BackgroundDeduplicator { clearTimeout(this.debounceTimer) } - // Schedule for 5 minutes from now + // Schedule for 5 minutes from now. unref'd: a pending dedup pass must + // never hold the process open (exit-hang class) — if the process exits + // first, the pass simply never runs; imports are already durable. this.debounceTimer = setTimeout(() => { this.runBatchDedup().catch(error => { prodLog.error('[BackgroundDedup] Batch dedup failed:', error) }) }, 5 * 60 * 1000) + this.debounceTimer.unref?.() } /** diff --git a/src/import/EntityDeduplicator.ts b/src/import/EntityDeduplicator.ts deleted file mode 100644 index 306e349a..00000000 --- a/src/import/EntityDeduplicator.ts +++ /dev/null @@ -1,354 +0,0 @@ -/** - * Entity Deduplicator - * - * Finds and merges duplicate entities across imports using: - * - Embedding-based similarity matching - * - Type-aware comparison - * - Confidence-weighted merging - * - Provenance tracking - * - * NO MOCKS - Production-ready implementation - */ - -import { Brainy } from '../brainy.js' -import { NounType } from '../types/graphTypes.js' - -export interface EntityCandidate { - id?: string - name: string - type: NounType - description: string - confidence: number - metadata: Record -} - -export interface DuplicateMatch { - existingId: string - existingName: string - similarity: number - shouldMerge: boolean - reason: string -} - -export interface EntityDeduplicationOptions { - /** Similarity threshold for considering entities as duplicates (0-1) */ - similarityThreshold?: number - - /** Only match entities of the same type */ - strictTypeMatching?: boolean - - /** Enable fuzzy name matching */ - enableFuzzyMatching?: boolean - - /** Minimum confidence to consider for merging */ - minConfidence?: number -} - -export interface MergeResult { - mergedEntityId: string - wasMerged: boolean - mergedWith?: string - confidence: number - provenance: string[] -} - -/** - * EntityDeduplicator - Prevents duplicate entities across imports - */ -export class EntityDeduplicator { - private brain: Brainy - - constructor(brain: Brainy) { - this.brain = brain - } - - /** - * Find duplicate entities in the knowledge graph - */ - async findDuplicates( - candidate: EntityCandidate, - options: EntityDeduplicationOptions = {} - ): Promise { - const opts = { - similarityThreshold: options.similarityThreshold || 0.85, - strictTypeMatching: options.strictTypeMatching !== false, - enableFuzzyMatching: options.enableFuzzyMatching !== false, - minConfidence: options.minConfidence || 0.6 - } - - // Skip low-confidence candidates - if (candidate.confidence < opts.minConfidence) { - return null - } - - // Search for similar entities by name and description - const searchText = `${candidate.name} ${candidate.description}`.trim() - - try { - const results = await this.brain.find({ - query: searchText, - limit: 5, - where: opts.strictTypeMatching ? { type: candidate.type } : undefined - }) - - // Check each result for potential duplicates - for (const result of results) { - const similarity = result.score || 0 - const existingName = result.entity.metadata?.name || result.id - const existingType = result.entity.metadata?.type || result.entity.metadata?.nounType || result.entity.type - - // Skip if below similarity threshold - if (similarity < opts.similarityThreshold) { - continue - } - - // Type matching check - if (opts.strictTypeMatching && existingType !== candidate.type) { - continue - } - - // Exact name match (case-insensitive) - if (this.normalizeString(candidate.name) === this.normalizeString(existingName)) { - return { - existingId: result.id, - existingName, - similarity: 1.0, - shouldMerge: true, - reason: 'Exact name match' - } - } - - // High similarity match - if (similarity >= opts.similarityThreshold) { - // Additional validation for fuzzy matching - if (opts.enableFuzzyMatching && this.areSimilarNames(candidate.name, existingName)) { - return { - existingId: result.id, - existingName, - similarity, - shouldMerge: true, - reason: `High similarity (${(similarity * 100).toFixed(1)}%)` - } - } - } - } - } catch (error) { - // If search fails, assume no duplicates - return null - } - - return null - } - - /** - * Merge entity data with existing entity - */ - async mergeEntity( - existingId: string, - candidate: EntityCandidate, - importSource: string - ): Promise { - try { - // Get existing entity - const existing = await this.brain.get(existingId) - if (!existing) { - throw new Error(`Entity ${existingId} not found`) - } - - // Update confidence (weighted average) — `confidence` is a reserved - // top-level field, so it travels via the dedicated update() param, not - // the metadata bag. - const mergedConfidence = this.mergeConfidence( - existing.confidence ?? 0.5, - candidate.confidence - ) - - // Merge metadata (custom fields only — reserved fields are top-level) - const mergedMetadata = { - ...existing.metadata, - // Track provenance - imports: [ - ...(existing.metadata?.imports || []), - importSource - ], - // Merge VFS paths - vfsPaths: [ - ...(existing.metadata?.vfsPaths || [existing.metadata?.vfsPath]).filter(Boolean), - candidate.metadata?.vfsPath - ].filter(Boolean), - // Merge other metadata - ...this.mergeMetadataFields(existing.metadata, candidate.metadata), - // Track last update - lastUpdated: Date.now(), - mergeCount: (existing.metadata?.mergeCount || 0) + 1 - } - - // Update entity - await this.brain.update({ - id: existingId, - confidence: mergedConfidence, - metadata: mergedMetadata, - merge: true - }) - - return { - mergedEntityId: existingId, - wasMerged: true, - mergedWith: existing.metadata?.name || existingId, - confidence: mergedConfidence, - provenance: mergedMetadata.imports - } - } catch (error) { - throw new Error(`Failed to merge entity: ${error instanceof Error ? error.message : String(error)}`) - } - } - - /** - * Create or merge entity with deduplication - */ - async createOrMerge( - candidate: EntityCandidate, - importSource: string, - options: EntityDeduplicationOptions = {} - ): Promise { - // Check for duplicates - const duplicate = await this.findDuplicates(candidate, options) - - if (duplicate && duplicate.shouldMerge) { - // Merge with existing entity - return await this.mergeEntity(duplicate.existingId, candidate, importSource) - } - - // No duplicate found, create new entity. Preserve any subtype the candidate - // carried (set by the extractor or upstream importer), else fall back to - // `'imported'` so enforcement doesn't fire (added 7.30.1). - // `confidence` is a reserved top-level field (dedicated add() param); - // creation time is system-managed — neither belongs in the metadata bag. - const entityId = await this.brain.add({ - data: candidate.description || candidate.name, - type: candidate.type, - subtype: - (candidate as EntityCandidate & { subtype?: string }).subtype ?? - 'imported', - confidence: candidate.confidence, - metadata: { - ...candidate.metadata, - name: candidate.name, - imports: [importSource], - vfsPaths: [candidate.metadata?.vfsPath].filter(Boolean), - mergeCount: 0 - } - }) - - // Update candidate with new ID - candidate.id = entityId - - return { - mergedEntityId: entityId, - wasMerged: false, - confidence: candidate.confidence, - provenance: [importSource] - } - } - - /** - * Normalize string for comparison - */ - private normalizeString(str: string): string { - return str - .toLowerCase() - .trim() - .replace(/[^a-z0-9]/g, '') - } - - /** - * Check if two names are similar (fuzzy matching) - */ - private areSimilarNames(name1: string, name2: string): boolean { - const n1 = this.normalizeString(name1) - const n2 = this.normalizeString(name2) - - // Exact match - if (n1 === n2) return true - - // Length difference check - const lengthDiff = Math.abs(n1.length - n2.length) - if (lengthDiff > 3) return false - - // Levenshtein distance - const distance = this.levenshteinDistance(n1, n2) - const maxLength = Math.max(n1.length, n2.length) - const similarity = 1 - (distance / maxLength) - - return similarity >= 0.85 - } - - /** - * Calculate Levenshtein distance between two strings - */ - private levenshteinDistance(str1: string, str2: string): number { - const m = str1.length - const n = str2.length - const dp: number[][] = Array(m + 1).fill(null).map(() => Array(n + 1).fill(0)) - - for (let i = 0; i <= m; i++) dp[i][0] = i - for (let j = 0; j <= n; j++) dp[0][j] = j - - for (let i = 1; i <= m; i++) { - for (let j = 1; j <= n; j++) { - if (str1[i - 1] === str2[j - 1]) { - dp[i][j] = dp[i - 1][j - 1] - } else { - dp[i][j] = Math.min( - dp[i - 1][j] + 1, // deletion - dp[i][j - 1] + 1, // insertion - dp[i - 1][j - 1] + 1 // substitution - ) - } - } - } - - return dp[m][n] - } - - /** - * Merge confidence scores (weighted average favoring higher confidence) - */ - private mergeConfidence(existing: number, incoming: number): number { - // Weight higher confidence more heavily - const weights = existing > incoming ? [0.6, 0.4] : [0.4, 0.6] - return existing * weights[0] + incoming * weights[1] - } - - /** - * Merge metadata fields intelligently - */ - private mergeMetadataFields( - existing: Record, - incoming: Record - ): Record { - const merged: Record = {} - - // Merge arrays - const arrayFields = ['concepts', 'tags', 'categories'] - for (const field of arrayFields) { - if (existing[field] || incoming[field]) { - const combined = [ - ...(existing[field] || []), - ...(incoming[field] || []) - ] - // Deduplicate - merged[field] = [...new Set(combined)] - } - } - - // Prefer longer descriptions - if (existing.description || incoming.description) { - merged.description = (existing.description || '').length > (incoming.description || '').length - ? existing.description - : incoming.description - } - - return merged - } -} diff --git a/src/import/ImportCoordinator.ts b/src/import/ImportCoordinator.ts index c2eaaef5..1e1316b7 100644 --- a/src/import/ImportCoordinator.ts +++ b/src/import/ImportCoordinator.ts @@ -13,7 +13,6 @@ import { Brainy } from '../brainy.js' import { FormatDetector, SupportedFormat } from './FormatDetector.js' import { ImportHistory, type ImportHistoryEntry } from './ImportHistory.js' -import { BackgroundDeduplicator } from './BackgroundDeduplicator.js' import { SmartExcelImporter } from '../importers/SmartExcelImporter.js' import { SmartPDFImporter } from '../importers/SmartPDFImporter.js' import { SmartCSVImporter } from '../importers/SmartCSVImporter.js' @@ -23,6 +22,7 @@ import { SmartYAMLImporter } from '../importers/SmartYAMLImporter.js' import { SmartDOCXImporter } from '../importers/SmartDOCXImporter.js' import { VFSStructureGenerator } from '../importers/VFSStructureGenerator.js' import { NounType, VerbType } from '../types/graphTypes.js' +import { splitNounMetadataRecord, splitVerbMetadataRecord } from '../types/reservedFields.js' import { v4 as uuidv4 } from '../universal/uuid.js' import * as fs from 'fs' import * as path from 'path' @@ -111,7 +111,12 @@ export interface ValidImportOptions { /** Confidence threshold for entities */ confidenceThreshold?: number - /** Enable entity deduplication across imports */ + /** + * Enable entity deduplication (default: true). Gates BOTH passes: the + * inline merge during import AND the debounced background pass that runs + * ~5 minutes after the last import (which merge-DELETES duplicate entities). + * Set false for deployments that must never auto-remove records. + */ enableDeduplication?: boolean /** Similarity threshold for deduplication (0-1) */ @@ -285,7 +290,6 @@ export class ImportCoordinator { private brain: Brainy private detector: FormatDetector private history: ImportHistory - private backgroundDedup: BackgroundDeduplicator private excelImporter: SmartExcelImporter private pdfImporter: SmartPDFImporter private csvImporter: SmartCSVImporter @@ -299,7 +303,6 @@ export class ImportCoordinator { this.brain = brain this.detector = new FormatDetector() this.history = new ImportHistory(brain) - this.backgroundDedup = new BackgroundDeduplicator(brain) this.excelImporter = new SmartExcelImporter(brain) this.pdfImporter = new SmartPDFImporter(brain) this.csvImporter = new SmartCSVImporter(brain) @@ -834,7 +837,11 @@ export class ImportCoordinator { type: 'media', description: '', confidence: 1.0, - metadata: { subtype: 'image' } + // `subtype` is a reserved field — keep it top-level on the extractor + // entity (read into the `subtype` param at add() time), never inside + // the metadata bag that gets spread into add({ metadata }). + subtype: 'image', + metadata: {} }, relatedEntities: [], relationships: [] @@ -843,7 +850,8 @@ export class ImportCoordinator { id: imageId, name: imageName, type: 'media', - metadata: { subtype: 'image' } + subtype: 'image', + metadata: {} }], relationships: [], metadata: {}, @@ -862,6 +870,38 @@ export class ImportCoordinator { } } + /** + * Strip Brainy-reserved entity keys out of an extractor-supplied metadata bag. + * + * Extractors (and consumer `customMetadata`) can carry reserved keys + * (`confidence`, `subtype`, `weight`, …) inside `metadata`. Brainy 8.0's + * default `reservedFieldPolicy` is `'throw'`, so spreading such a bag into + * `add({ metadata })` would reject the whole import. The import pipeline owns + * the correct write path: user-mutable reserved values are passed as dedicated + * `AddParams` params (see the call sites), so here we simply drop the reserved + * half of the bag and keep only the custom fields that belong in `metadata`. + * + * @param bag - The extractor/consumer metadata bag (may be undefined). + * @returns The custom-only metadata (reserved keys removed). + */ + private stripReservedFromBag(bag: Record | undefined | null): Record { + if (!bag || typeof bag !== 'object') return {} + return splitNounMetadataRecord(bag).custom + } + + /** + * Relationship mirror of {@link stripReservedFromBag} — strips reserved verb + * keys (`verb`, `confidence`, `weight`, `subtype`, …) out of an edge metadata + * bag so it carries only custom fields. Reserved values that have a dedicated + * `RelateParams` param are passed there by the call site instead. + * @param bag - The extractor/consumer edge metadata bag (may be undefined). + * @returns The custom-only edge metadata (reserved keys removed). + */ + private stripReservedFromRelationBag(bag: Record | undefined | null): Record { + if (!bag || typeof bag !== 'object') return {} + return splitVerbMetadataRecord(bag).custom + } + /** * Create entities and relationships in knowledge graph * Added sourceInfo parameter for document entity creation @@ -977,7 +1017,7 @@ export class ImportCoordinator { importedAt: trackingContext.importedAt, importFormat: trackingContext.importFormat, importSource: trackingContext.importSource, - ...trackingContext.customMetadata + ...this.stripReservedFromBag(trackingContext.customMetadata) }) } }) @@ -1005,10 +1045,14 @@ export class ImportCoordinator { data: entity.description || entity.name, type: entity.type, subtype: entity.subtype ?? options.defaultSubtype ?? 'imported', + // `confidence` is a reserved field — pass it as the dedicated param, + // never inside the metadata bag (8.0 reservedFieldPolicy defaults to 'throw'). + confidence: entity.confidence, metadata: { - ...entity.metadata, + // Extractor/consumer bags may smuggle reserved keys — strip them so + // the bag carries only custom fields. + ...this.stripReservedFromBag(entity.metadata), name: entity.name, - confidence: entity.confidence, vfsPath: vfsFile?.path, importedFrom: 'import-coordinator', imports: [importSource], @@ -1020,7 +1064,7 @@ export class ImportCoordinator { importSource: trackingContext.importSource, sourceRow: row.rowNumber, sourceSheet: row.sheet, - ...trackingContext.customMetadata + ...this.stripReservedFromBag(trackingContext.customMetadata) }) } } @@ -1101,7 +1145,7 @@ export class ImportCoordinator { importIds: [trackingContext.importId], projectId: trackingContext.projectId, importFormat: trackingContext.importFormat, - ...trackingContext.customMetadata + ...this.stripReservedFromRelationBag(trackingContext.customMetadata) }) } } @@ -1132,10 +1176,12 @@ export class ImportCoordinator { data: entity.description || entity.name, type: entity.type, subtype: entity.subtype ?? options.defaultSubtype ?? 'imported', + // `confidence` is a reserved field — dedicated param, not metadata. + confidence: entity.confidence, metadata: { - ...entity.metadata, + // Strip any reserved keys an extractor smuggled into the bag. + ...this.stripReservedFromBag(entity.metadata), name: entity.name, - confidence: entity.confidence, vfsPath: vfsFile?.path, importedFrom: 'import-coordinator', // Import tracking metadata @@ -1148,7 +1194,7 @@ export class ImportCoordinator { importSource: trackingContext.importSource, sourceRow: row.rowNumber, sourceSheet: row.sheet, - ...trackingContext.customMetadata + ...this.stripReservedFromBag(trackingContext.customMetadata) }) } }) @@ -1188,7 +1234,7 @@ export class ImportCoordinator { importIds: [trackingContext.importId], projectId: trackingContext.projectId, importFormat: trackingContext.importFormat, - ...trackingContext.customMetadata + ...this.stripReservedFromRelationBag(trackingContext.customMetadata) }) } }) @@ -1243,7 +1289,7 @@ export class ImportCoordinator { projectId: trackingContext.projectId, importedAt: trackingContext.importedAt, importFormat: trackingContext.importFormat, - ...trackingContext.customMetadata + ...this.stripReservedFromBag(trackingContext.customMetadata) }) } }) @@ -1273,7 +1319,7 @@ export class ImportCoordinator { projectId: trackingContext.projectId, importedAt: trackingContext.importedAt, importFormat: trackingContext.importFormat, - ...trackingContext.customMetadata + ...this.stripReservedFromRelationBag(trackingContext.customMetadata) }) } }) @@ -1371,8 +1417,12 @@ export class ImportCoordinator { from: rel.from, to: rel.to, type: verbType, // Enhanced type + // confidence/weight are reserved — carry them as dedicated params, + // never inside the bag (they were collected top-level on each rel). + ...(typeof (rel as any).confidence === 'number' && { confidence: (rel as any).confidence }), + ...(typeof (rel as any).weight === 'number' && { weight: (rel as any).weight }), metadata: { - ...(rel.metadata || {}), + ...this.stripReservedFromRelationBag(rel.metadata), relationshipType: 'semantic', // Distinguish from VFS/provenance inferredType: verbType !== rel.type, // Track if type was enhanced originalType: rel.type @@ -1411,9 +1461,16 @@ export class ImportCoordinator { } } - // Schedule background deduplication (debounced 5 minutes) - if (trackingContext && trackingContext.importId) { - this.backgroundDedup.scheduleDedup(trackingContext.importId) + // Schedule background deduplication (debounced 5 minutes, brain-owned so + // close() can cancel it). Honors the same enableDeduplication gate as the + // inline pass — false means NO dedup, inline or background. + if ( + trackingContext && + trackingContext.importId && + options.enableDeduplication !== false + ) { + const backgroundDedup = await this.brain.getBackgroundDeduplicator() + backgroundDedup.scheduleDedup(trackingContext.importId) } return { diff --git a/src/import/InstancePool.ts b/src/import/InstancePool.ts deleted file mode 100644 index 2ac26120..00000000 --- a/src/import/InstancePool.ts +++ /dev/null @@ -1,269 +0,0 @@ -/** - * InstancePool - Shared instance management for memory efficiency - * - * Production-grade instance pooling to prevent memory leaks during imports. - * Critical for scaling to billions of entities. - * - * Problem: Creating new NLP/Extractor instances in loops → memory leak - * Solution: Reuse shared instances across entire import session - * - * Memory savings: - * - Without pooling: 100K rows × 50MB per instance = 5TB RAM (OOM!) - * - With pooling: 50MB total (shared across all rows) - */ - -import { Brainy } from '../brainy.js' -import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js' -import { NeuralEntityExtractor } from '../neural/entityExtractor.js' - -/** - * InstancePool - Manages shared instances for memory efficiency - * - * Lifecycle: - * 1. Create pool at import start - * 2. Reuse instances across all rows - * 3. Pool is garbage collected when import completes - * - * Thread safety: Not thread-safe (single import session per pool) - */ -export class InstancePool { - private brain: Brainy - - // Shared instances (created lazily) - private nlpInstance: NaturalLanguageProcessor | null = null - private extractorInstance: NeuralEntityExtractor | null = null - - // Initialization state - private nlpInitialized = false - private initializationPromise: Promise | null = null - - // Statistics - private stats = { - nlpReuses: 0, - extractorReuses: 0, - creationTime: 0 - } - - constructor(brain: Brainy) { - this.brain = brain - } - - /** - * Get shared NaturalLanguageProcessor instance - * - * Lazy initialization - created on first access - * All subsequent calls return same instance - * - * @returns Shared NLP instance - */ - async getNLP(): Promise { - if (!this.nlpInstance) { - const startTime = Date.now() - this.nlpInstance = new NaturalLanguageProcessor(this.brain) - this.stats.creationTime += Date.now() - startTime - } - - // Ensure initialized before returning - if (!this.nlpInitialized) { - await this.ensureNLPInitialized() - } - - this.stats.nlpReuses++ - return this.nlpInstance - } - - /** - * Get shared NeuralEntityExtractor instance - * - * Lazy initialization - created on first access - * All subsequent calls return same instance - * - * @returns Shared extractor instance - */ - getExtractor(): NeuralEntityExtractor { - if (!this.extractorInstance) { - const startTime = Date.now() - this.extractorInstance = new NeuralEntityExtractor(this.brain) - this.stats.creationTime += Date.now() - startTime - } - - this.stats.extractorReuses++ - return this.extractorInstance - } - - /** - * Get shared NLP instance (synchronous, may return uninitialized) - * - * Use when you need NLP synchronously and will handle initialization yourself. - * Prefer getNLP() for async code. - * - * @returns Shared NLP instance (possibly uninitialized) - */ - getNLPSync(): NaturalLanguageProcessor { - if (!this.nlpInstance) { - this.nlpInstance = new NaturalLanguageProcessor(this.brain) - } - - this.stats.nlpReuses++ - return this.nlpInstance - } - - /** - * Initialize all instances upfront - * - * Call at start of import to avoid lazy initialization overhead - * during processing. Improves predictability and first-row performance. - * - * @returns Promise that resolves when all instances are ready - */ - async init(): Promise { - // Prevent duplicate initialization - if (this.initializationPromise) { - return this.initializationPromise - } - - this.initializationPromise = this.initializeInternal() - return this.initializationPromise - } - - /** - * Internal initialization implementation - */ - private async initializeInternal(): Promise { - const startTime = Date.now() - - // Create instances - if (!this.nlpInstance) { - this.nlpInstance = new NaturalLanguageProcessor(this.brain) - } - if (!this.extractorInstance) { - this.extractorInstance = new NeuralEntityExtractor(this.brain) - } - - // Initialize NLP (loads pattern library) - await this.ensureNLPInitialized() - - this.stats.creationTime = Date.now() - startTime - } - - /** - * Ensure NLP is initialized (loads 220 patterns) - * - * Handles concurrent initialization requests safely - */ - private async ensureNLPInitialized(): Promise { - if (this.nlpInitialized) { - return - } - - if (!this.nlpInstance) { - throw new Error('NLP instance not created yet') - } - - await this.nlpInstance.init() - this.nlpInitialized = true - } - - /** - * Check if instances are initialized - * - * @returns True if NLP is initialized and ready to use - */ - isInitialized(): boolean { - return this.nlpInitialized && this.nlpInstance !== null - } - - /** - * Get pool statistics - * - * Useful for performance monitoring and memory leak detection - * - * @returns Statistics about instance reuse - */ - getStats() { - return { - ...this.stats, - nlpCreated: this.nlpInstance !== null, - extractorCreated: this.extractorInstance !== null, - initialized: this.isInitialized(), - // Memory savings estimate - memorySaved: this.calculateMemorySaved() - } - } - - /** - * Calculate estimated memory saved by pooling - * - * Assumes ~50MB per NLP instance, ~10MB per extractor instance - * - * @returns Estimated memory saved in bytes - */ - private calculateMemorySaved(): number { - const nlpSize = 50 * 1024 * 1024 // 50MB per instance - const extractorSize = 10 * 1024 * 1024 // 10MB per instance - - // Without pooling: size × reuses - // With pooling: size × 1 - // Saved: size × (reuses - 1) - - const nlpSaved = nlpSize * Math.max(0, this.stats.nlpReuses - 1) - const extractorSaved = extractorSize * Math.max(0, this.stats.extractorReuses - 1) - - return nlpSaved + extractorSaved - } - - /** - * Reset statistics (useful for testing) - */ - resetStats(): void { - this.stats = { - nlpReuses: 0, - extractorReuses: 0, - creationTime: 0 - } - } - - /** - * Get string representation (for debugging) - */ - toString(): string { - const stats = this.getStats() - return `InstancePool(nlp=${stats.nlpCreated}, extractor=${stats.extractorCreated}, initialized=${stats.initialized}, nlpReuses=${stats.nlpReuses}, extractorReuses=${stats.extractorReuses})` - } - - /** - * Cleanup method (for explicit resource management) - * - * Note: Usually not needed - pool is garbage collected when import completes. - * Use only if you need explicit cleanup for some reason. - */ - cleanup(): void { - // Clear references to allow garbage collection - this.nlpInstance = null - this.extractorInstance = null - this.nlpInitialized = false - this.initializationPromise = null - } -} - -/** - * Create a new instance pool - * - * Convenience factory function - * - * @param brain Brainy instance - * @param autoInit Whether to initialize instances immediately - * @returns Instance pool - */ -export async function createInstancePool( - brain: Brainy, - autoInit = true -): Promise { - const pool = new InstancePool(brain) - - if (autoInit) { - await pool.init() - } - - return pool -} diff --git a/src/import/index.ts b/src/import/index.ts deleted file mode 100644 index 8aed2f41..00000000 --- a/src/import/index.ts +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Unified Import System - * - * Single entry point for importing any file format into Brainy with: - * - Auto-detection of formats - * - Dual storage (VFS + Knowledge Graph) - * - Shared entities across imports (deduplication) - * - Simple, powerful API - */ - -export { ImportCoordinator } from './ImportCoordinator.js' -export { FormatDetector, SupportedFormat, DetectionResult } from './FormatDetector.js' -export { EntityDeduplicator } from './EntityDeduplicator.js' -export { BackgroundDeduplicator } from './BackgroundDeduplicator.js' -export { ImportHistory } from './ImportHistory.js' - -export type { - ImportSource, - ImportOptions, - ImportProgress, - ImportResult -} from './ImportCoordinator.js' - -export type { - EntityCandidate, - DuplicateMatch, - EntityDeduplicationOptions, - MergeResult -} from './EntityDeduplicator.js' - -export type { - DeduplicationStats -} from './BackgroundDeduplicator.js' - -export type { - ImportHistoryEntry, - RollbackResult -} from './ImportHistory.js' diff --git a/src/importers/SmartImportOrchestrator.ts b/src/importers/SmartImportOrchestrator.ts deleted file mode 100644 index d9157852..00000000 --- a/src/importers/SmartImportOrchestrator.ts +++ /dev/null @@ -1,786 +0,0 @@ -/** - * Smart Import Orchestrator - * - * Coordinates the entire smart import pipeline: - * 1. Extract entities/relationships using SmartExcelImporter - * 2. Create entities and relationships in Brainy - * 3. Organize into VFS structure using VFSStructureGenerator - * - * NO MOCKS - Production-ready implementation - */ - -import { Brainy } from '../brainy.js' -import { VirtualFileSystem } from '../vfs/VirtualFileSystem.js' -import { NounType, VerbType } from '../types/graphTypes.js' -import { SmartExcelImporter, SmartExcelOptions, SmartExcelResult } from './SmartExcelImporter.js' -import { SmartPDFImporter, SmartPDFOptions, SmartPDFResult } from './SmartPDFImporter.js' -import { SmartCSVImporter, SmartCSVOptions, SmartCSVResult } from './SmartCSVImporter.js' -import { SmartJSONImporter, SmartJSONOptions, SmartJSONResult } from './SmartJSONImporter.js' -import { SmartMarkdownImporter, SmartMarkdownOptions, SmartMarkdownResult } from './SmartMarkdownImporter.js' -import { VFSStructureGenerator, VFSStructureOptions } from './VFSStructureGenerator.js' - -export interface SmartImportOptions extends SmartExcelOptions { - /** Create VFS structure */ - createVFSStructure?: boolean - - /** VFS root path */ - vfsRootPath?: string - - /** VFS grouping strategy */ - vfsGroupBy?: 'type' | 'sheet' | 'flat' | 'custom' - - /** Create entities in Brainy */ - createEntities?: boolean - - /** Create relationships in Brainy */ - createRelationships?: boolean - - /** Source filename */ - filename?: string - - /** - * Default subtype tag for entities + relationships this importer creates when - * the extractor doesn't set one. See `ValidImportOptions.defaultSubtype` — - * same semantics, same precedence (extractor > caller default > `'imported'`). - * Added 7.30.1. - */ - defaultSubtype?: string -} - -export interface SmartImportProgress { - phase: 'parsing' | 'extracting' | 'creating' | 'relationships' | 'organizing' | 'complete' - message: string - processed: number - total: number - entities: number - relationships: number -} - -export interface SmartImportResult { - success: boolean - - /** Extraction results */ - extraction: SmartExcelResult - - /** Created entity IDs */ - entityIds: string[] - - /** Created relationship IDs */ - relationshipIds: string[] - - /** VFS structure created */ - vfsStructure?: { - rootPath: string - directories: string[] - files: number - } - - /** Overall statistics */ - stats: { - rowsProcessed: number - entitiesCreated: number - relationshipsCreated: number - filesCreated: number - totalTime: number - } - - /** Any errors encountered */ - errors: string[] -} - -/** - * SmartImportOrchestrator - Main entry point for smart imports - */ -export class SmartImportOrchestrator { - private brain: Brainy - private excelImporter: SmartExcelImporter - private pdfImporter: SmartPDFImporter - private csvImporter: SmartCSVImporter - private jsonImporter: SmartJSONImporter - private markdownImporter: SmartMarkdownImporter - private vfsGenerator: VFSStructureGenerator - - constructor(brain: Brainy) { - this.brain = brain - this.excelImporter = new SmartExcelImporter(brain) - this.pdfImporter = new SmartPDFImporter(brain) - this.csvImporter = new SmartCSVImporter(brain) - this.jsonImporter = new SmartJSONImporter(brain) - this.markdownImporter = new SmartMarkdownImporter(brain) - this.vfsGenerator = new VFSStructureGenerator(brain) - } - - /** - * Initialize the orchestrator - */ - async init(): Promise { - await this.excelImporter.init() - await this.pdfImporter.init() - await this.csvImporter.init() - await this.jsonImporter.init() - await this.markdownImporter.init() - await this.vfsGenerator.init() - } - - /** - * Import Excel file with full pipeline - */ - async importExcel( - buffer: Buffer, - options: SmartImportOptions = {}, - onProgress?: (progress: SmartImportProgress) => void - ): Promise { - const startTime = Date.now() - const result: SmartImportResult = { - success: false, - // Typed boundary: populated in the extraction phase below. If extraction - // throws, the error path returns with this still null (pre-existing - // contract — consumers check `success`/`errors` before reading it). - extraction: null as unknown as SmartExcelResult, - entityIds: [], - relationshipIds: [], - stats: { - rowsProcessed: 0, - entitiesCreated: 0, - relationshipsCreated: 0, - filesCreated: 0, - totalTime: 0 - }, - errors: [] - } - - try { - // Phase 1: Extract entities and relationships - onProgress?.({ - phase: 'extracting', - message: 'Extracting entities and relationships...', - processed: 0, - total: 0, - entities: 0, - relationships: 0 - }) - - result.extraction = await this.excelImporter.extract(buffer, { - ...options, - onProgress: (stats) => { - onProgress?.({ - phase: 'extracting', - message: `Processing row ${stats.processed}/${stats.total}...`, - processed: stats.processed, - total: stats.total, - entities: stats.entities, - relationships: stats.relationships - }) - } - }) - - result.stats.rowsProcessed = result.extraction.rowsProcessed - - // Phase 2: Create entities in Brainy - if (options.createEntities !== false) { - onProgress?.({ - phase: 'creating', - message: 'Creating entities in knowledge graph...', - processed: 0, - total: result.extraction.rows.length, - entities: 0, - relationships: 0 - }) - - for (let i = 0; i < result.extraction.rows.length; i++) { - const extracted = result.extraction.rows[i] - - try { - // Create main entity. Subtype precedence: extractor-set → caller default - // → Brainy default `'imported'` (added 7.30.1). - const entityId = await this.brain.add({ - data: extracted.entity.description, - type: extracted.entity.type, - subtype: - (extracted.entity as typeof extracted.entity & { subtype?: string }) - .subtype ?? options.defaultSubtype ?? 'imported', - confidence: extracted.entity.confidence, // reserved field — dedicated param, not metadata - metadata: { - ...extracted.entity.metadata, - name: extracted.entity.name, - importedFrom: 'smart-import' - } - }) - - result.entityIds.push(entityId) - result.stats.entitiesCreated++ - - // Update entity ID in extraction result - extracted.entity.id = entityId - - onProgress?.({ - phase: 'creating', - message: `Created entity: ${extracted.entity.name}`, - processed: i + 1, - total: result.extraction.rows.length, - entities: result.entityIds.length, - relationships: result.relationshipIds.length - }) - } catch (error: any) { - result.errors.push(`Failed to create entity ${extracted.entity.name}: ${error.message}`) - } - } - } - - // Phase 3: Create relationships - if (options.createRelationships !== false && options.createEntities !== false) { - onProgress?.({ - phase: 'creating', - message: 'Preparing relationships...', - processed: 0, - total: result.extraction.rows.length, - entities: result.entityIds.length, - relationships: 0 - }) - - // Build entity name -> ID map - const entityMap = new Map() - for (const extracted of result.extraction.rows) { - entityMap.set(extracted.entity.name.toLowerCase(), extracted.entity.id) - } - - // Collect all relationship parameters - const relationshipParams: Array<{from: string; to: string; type: VerbType; subtype?: string; metadata?: any}> = [] - - for (const extracted of result.extraction.rows) { - for (const rel of extracted.relationships) { - try { - // Find target entity ID - let toEntityId: string | undefined - - // Try to find by name in our extracted entities - for (const otherExtracted of result.extraction.rows) { - if (rel.to.toLowerCase().includes(otherExtracted.entity.name.toLowerCase()) || - otherExtracted.entity.name.toLowerCase().includes(rel.to.toLowerCase())) { - toEntityId = otherExtracted.entity.id - break - } - } - - // If not found, create a placeholder entity. `import-placeholder` marks - // these as synthetic targets so consumers can distinguish them from real - // imports and downstream dedup can consolidate (added 7.30.1). - if (!toEntityId) { - toEntityId = await this.brain.add({ - data: rel.to, - type: NounType.Thing, - subtype: 'import-placeholder', - metadata: { - name: rel.to, - placeholder: true, - extractedFrom: extracted.entity.name - } - }) - result.entityIds.push(toEntityId) - } - - // Collect relationship parameter. Subtype precedence: extractor-set rel - // subtype → caller default → Brainy default `'imported'` (added 7.30.1). - relationshipParams.push({ - from: extracted.entity.id, - to: toEntityId, - type: rel.type, - subtype: - (rel as typeof rel & { subtype?: string }).subtype ?? - options.defaultSubtype ?? 'imported', - metadata: { - confidence: rel.confidence, - evidence: rel.evidence - } - }) - } catch (error: any) { - result.errors.push(`Failed to prepare relationship: ${error.message}`) - } - } - } - - // Batch create all relationships with progress - if (relationshipParams.length > 0) { - onProgress?.({ - phase: 'relationships', - message: 'Building relationships...', - processed: 0, - total: relationshipParams.length, - entities: result.entityIds.length, - relationships: 0 - }) - - try { - const relationshipIds = await this.brain.relateMany({ - items: relationshipParams, - parallel: true, - chunkSize: 100, - continueOnError: true, - onProgress: (done, total) => { - onProgress?.({ - phase: 'relationships', - message: `Building relationships: ${done}/${total}`, - processed: done, - total: total, - entities: result.entityIds.length, - relationships: done - }) - } - }) - - result.relationshipIds = relationshipIds - result.stats.relationshipsCreated = relationshipIds.length - } catch (error: any) { - result.errors.push(`Failed to create relationships: ${error.message}`) - } - } - } - - // Phase 4: Create VFS structure - if (options.createVFSStructure !== false) { - onProgress?.({ - phase: 'organizing', - message: 'Organizing into file structure...', - processed: 0, - total: result.extraction.rows.length, - entities: result.entityIds.length, - relationships: result.relationshipIds.length - }) - - const vfsOptions: VFSStructureOptions = { - rootPath: options.vfsRootPath || '/imports/' + (options.filename || 'import'), - groupBy: options.vfsGroupBy || 'type', - preserveSource: true, - sourceBuffer: buffer, - sourceFilename: options.filename || 'import.xlsx', - createRelationshipFile: true, - createMetadataFile: true - } - - const vfsResult = await this.vfsGenerator.generate(result.extraction, vfsOptions) - - result.vfsStructure = { - rootPath: vfsResult.rootPath, - directories: vfsResult.directories, - files: vfsResult.files.length - } - - result.stats.filesCreated = vfsResult.files.length - } - - // Complete - result.success = result.errors.length === 0 - result.stats.totalTime = Date.now() - startTime - - onProgress?.({ - phase: 'complete', - message: `Import complete: ${result.stats.entitiesCreated} entities, ${result.stats.relationshipsCreated} relationships`, - processed: result.extraction.rows.length, - total: result.extraction.rows.length, - entities: result.stats.entitiesCreated, - relationships: result.stats.relationshipsCreated - }) - - } catch (error: any) { - result.errors.push(`Import failed: ${error.message}`) - result.success = false - } - - return result - } - - /** - * Import PDF file with full pipeline - */ - async importPDF( - buffer: Buffer, - options: SmartImportOptions & SmartPDFOptions = {}, - onProgress?: (progress: SmartImportProgress) => void - ): Promise { - const startTime = Date.now() - const result: SmartImportResult = { - success: false, - // Typed boundary: populated after extraction (see importExcel). - extraction: null as unknown as SmartExcelResult, - entityIds: [], - relationshipIds: [], - stats: { - rowsProcessed: 0, - entitiesCreated: 0, - relationshipsCreated: 0, - filesCreated: 0, - totalTime: 0 - }, - errors: [] - } - - try { - // Phase 1: Extract from PDF - onProgress?.({ phase: 'extracting', message: 'Extracting from PDF...', processed: 0, total: 0, entities: 0, relationships: 0 }) - - const pdfResult = await this.pdfImporter.extract(buffer, options) - - // Convert PDF result to Excel-like format for processing - result.extraction = this.convertPDFToExcelFormat(pdfResult) - result.stats.rowsProcessed = pdfResult.sectionsProcessed - - // Phase 2 & 3: Create entities and relationships - await this.createEntitiesAndRelationships(result, options, onProgress) - - // Phase 4: Create VFS structure - if (options.createVFSStructure !== false) { - const vfsOptions: VFSStructureOptions = { - rootPath: options.vfsRootPath || '/imports/' + (options.filename || 'import'), - groupBy: options.vfsGroupBy || 'type', - preserveSource: true, - sourceBuffer: buffer, - sourceFilename: options.filename || 'import.pdf', - createRelationshipFile: true, - createMetadataFile: true - } - const vfsResult = await this.vfsGenerator.generate(result.extraction, vfsOptions) - result.vfsStructure = { rootPath: vfsResult.rootPath, directories: vfsResult.directories, files: vfsResult.files.length } - result.stats.filesCreated = vfsResult.files.length - } - - result.success = result.errors.length === 0 - result.stats.totalTime = Date.now() - startTime - onProgress?.({ phase: 'complete', message: `Import complete: ${result.stats.entitiesCreated} entities, ${result.stats.relationshipsCreated} relationships`, processed: result.stats.rowsProcessed, total: result.stats.rowsProcessed, entities: result.stats.entitiesCreated, relationships: result.stats.relationshipsCreated }) - - } catch (error: any) { - result.errors.push(`PDF import failed: ${error.message}`) - result.success = false - } - - return result - } - - /** - * Import CSV file with full pipeline - */ - async importCSV( - buffer: Buffer, - options: SmartImportOptions & SmartCSVOptions = {}, - onProgress?: (progress: SmartImportProgress) => void - ): Promise { - // CSV is very similar to Excel, can reuse importExcel logic - return this.importExcel(buffer, options, onProgress) - } - - /** - * Import JSON data with full pipeline - */ - async importJSON( - data: any, - options: SmartImportOptions & SmartJSONOptions = {}, - onProgress?: (progress: SmartImportProgress) => void - ): Promise { - const startTime = Date.now() - const result: SmartImportResult = { - success: false, - // Typed boundary: populated after extraction (see importExcel). - extraction: null as unknown as SmartExcelResult, - entityIds: [], - relationshipIds: [], - stats: { - rowsProcessed: 0, - entitiesCreated: 0, - relationshipsCreated: 0, - filesCreated: 0, - totalTime: 0 - }, - errors: [] - } - - try { - onProgress?.({ phase: 'extracting', message: 'Extracting from JSON...', processed: 0, total: 0, entities: 0, relationships: 0 }) - - const jsonResult = await this.jsonImporter.extract(data, options) - - result.extraction = this.convertJSONToExcelFormat(jsonResult) - result.stats.rowsProcessed = jsonResult.nodesProcessed - - await this.createEntitiesAndRelationships(result, options, onProgress) - - if (options.createVFSStructure !== false) { - const sourceBuffer = Buffer.from(typeof data === 'string' ? data : JSON.stringify(data, null, 2)) - const vfsOptions: VFSStructureOptions = { - rootPath: options.vfsRootPath || '/imports/' + (options.filename || 'import'), - groupBy: options.vfsGroupBy || 'type', - preserveSource: true, - sourceBuffer, - sourceFilename: options.filename || 'import.json', - createRelationshipFile: true, - createMetadataFile: true - } - const vfsResult = await this.vfsGenerator.generate(result.extraction, vfsOptions) - result.vfsStructure = { rootPath: vfsResult.rootPath, directories: vfsResult.directories, files: vfsResult.files.length } - result.stats.filesCreated = vfsResult.files.length - } - - result.success = result.errors.length === 0 - result.stats.totalTime = Date.now() - startTime - onProgress?.({ phase: 'complete', message: `Import complete: ${result.stats.entitiesCreated} entities, ${result.stats.relationshipsCreated} relationships`, processed: result.stats.rowsProcessed, total: result.stats.rowsProcessed, entities: result.stats.entitiesCreated, relationships: result.stats.relationshipsCreated }) - - } catch (error: any) { - result.errors.push(`JSON import failed: ${error.message}`) - result.success = false - } - - return result - } - - /** - * Import Markdown content with full pipeline - */ - async importMarkdown( - markdown: string, - options: SmartImportOptions & SmartMarkdownOptions = {}, - onProgress?: (progress: SmartImportProgress) => void - ): Promise { - const startTime = Date.now() - const result: SmartImportResult = { - success: false, - // Typed boundary: populated after extraction (see importExcel). - extraction: null as unknown as SmartExcelResult, - entityIds: [], - relationshipIds: [], - stats: { - rowsProcessed: 0, - entitiesCreated: 0, - relationshipsCreated: 0, - filesCreated: 0, - totalTime: 0 - }, - errors: [] - } - - try { - onProgress?.({ phase: 'extracting', message: 'Extracting from Markdown...', processed: 0, total: 0, entities: 0, relationships: 0 }) - - const mdResult = await this.markdownImporter.extract(markdown, options) - - result.extraction = this.convertMarkdownToExcelFormat(mdResult) - result.stats.rowsProcessed = mdResult.sectionsProcessed - - await this.createEntitiesAndRelationships(result, options, onProgress) - - if (options.createVFSStructure !== false) { - const sourceBuffer = Buffer.from(markdown, 'utf-8') - const vfsOptions: VFSStructureOptions = { - rootPath: options.vfsRootPath || '/imports/' + (options.filename || 'import'), - groupBy: options.vfsGroupBy || 'type', - preserveSource: true, - sourceBuffer, - sourceFilename: options.filename || 'import.md', - createRelationshipFile: true, - createMetadataFile: true - } - const vfsResult = await this.vfsGenerator.generate(result.extraction, vfsOptions) - result.vfsStructure = { rootPath: vfsResult.rootPath, directories: vfsResult.directories, files: vfsResult.files.length } - result.stats.filesCreated = vfsResult.files.length - } - - result.success = result.errors.length === 0 - result.stats.totalTime = Date.now() - startTime - onProgress?.({ phase: 'complete', message: `Import complete: ${result.stats.entitiesCreated} entities, ${result.stats.relationshipsCreated} relationships`, processed: result.stats.rowsProcessed, total: result.stats.rowsProcessed, entities: result.stats.entitiesCreated, relationships: result.stats.relationshipsCreated }) - - } catch (error: any) { - result.errors.push(`Markdown import failed: ${error.message}`) - result.success = false - } - - return result - } - - /** - * Helper: Create entities and relationships from extraction result - */ - private async createEntitiesAndRelationships( - result: SmartImportResult, - options: SmartImportOptions, - onProgress?: (progress: SmartImportProgress) => void - ): Promise { - if (options.createEntities !== false) { - onProgress?.({ phase: 'creating', message: 'Creating entities in knowledge graph...', processed: 0, total: result.extraction.rows.length, entities: 0, relationships: 0 }) - - for (let i = 0; i < result.extraction.rows.length; i++) { - const extracted = result.extraction.rows[i] - try { - // Subtype precedence: extractor → caller default → `'imported'` (7.30.1). - const entityId = await this.brain.add({ - data: extracted.entity.description, - type: extracted.entity.type, - subtype: - (extracted.entity as typeof extracted.entity & { subtype?: string }) - .subtype ?? options.defaultSubtype ?? 'imported', - confidence: extracted.entity.confidence, // reserved field — dedicated param, not metadata - metadata: { ...extracted.entity.metadata, name: extracted.entity.name, importedFrom: 'smart-import' } - }) - result.entityIds.push(entityId) - result.stats.entitiesCreated++ - extracted.entity.id = entityId - } catch (error: any) { - result.errors.push(`Failed to create entity ${extracted.entity.name}: ${error.message}`) - } - } - } - - if (options.createRelationships !== false && options.createEntities !== false) { - onProgress?.({ phase: 'creating', message: 'Preparing relationships...', processed: 0, total: result.extraction.rows.length, entities: result.entityIds.length, relationships: 0 }) - - // Collect all relationship parameters - const relationshipParams: Array<{from: string; to: string; type: VerbType; subtype?: string; confidence?: number; metadata?: any}> = [] - - for (const extracted of result.extraction.rows) { - for (const rel of extracted.relationships) { - try { - let toEntityId: string | undefined - for (const otherExtracted of result.extraction.rows) { - if (rel.to.toLowerCase().includes(otherExtracted.entity.name.toLowerCase()) || otherExtracted.entity.name.toLowerCase().includes(rel.to.toLowerCase())) { - toEntityId = otherExtracted.entity.id - break - } - } - if (!toEntityId) { - // Subtype `import-placeholder` marks synthetic targets (7.30.1). - toEntityId = await this.brain.add({ data: rel.to, type: NounType.Thing, subtype: 'import-placeholder', metadata: { name: rel.to, placeholder: true, extractedFrom: extracted.entity.name } }) - result.entityIds.push(toEntityId) - } - // Relationship subtype precedence: extractor → caller default → `'imported'` (7.30.1). - // `confidence` is a reserved top-level field — dedicated relate() param, not metadata - relationshipParams.push({ from: extracted.entity.id, to: toEntityId, type: rel.type, subtype: (rel as typeof rel & { subtype?: string }).subtype ?? options.defaultSubtype ?? 'imported', confidence: rel.confidence, metadata: { evidence: rel.evidence } }) - } catch (error: any) { - result.errors.push(`Failed to prepare relationship: ${error.message}`) - } - } - } - - // Batch create all relationships with progress - if (relationshipParams.length > 0) { - onProgress?.({ phase: 'relationships', message: 'Building relationships...', processed: 0, total: relationshipParams.length, entities: result.entityIds.length, relationships: 0 }) - - try { - const relationshipIds = await this.brain.relateMany({ - items: relationshipParams, - parallel: true, - chunkSize: 100, - continueOnError: true, - onProgress: (done, total) => { - onProgress?.({ phase: 'relationships', message: `Building relationships: ${done}/${total}`, processed: done, total: total, entities: result.entityIds.length, relationships: done }) - } - }) - - result.relationshipIds = relationshipIds - result.stats.relationshipsCreated = relationshipIds.length - } catch (error: any) { - result.errors.push(`Failed to create relationships: ${error.message}`) - } - } - } - } - - /** - * Helper: Convert PDF result to Excel-like format - */ - private convertPDFToExcelFormat(pdfResult: SmartPDFResult): Omit & { rows: any[] } { - const rows = pdfResult.sections.flatMap(section => - section.entities.map(entity => ({ - entity, - relatedEntities: [], - relationships: section.relationships.filter(r => r.from === entity.id), - concepts: section.concepts - })) - ) - - return { - rowsProcessed: pdfResult.sectionsProcessed, - entitiesExtracted: pdfResult.entitiesExtracted, - relationshipsInferred: pdfResult.relationshipsInferred, - rows, - entityMap: pdfResult.entityMap, - processingTime: pdfResult.processingTime, - stats: pdfResult.stats - } - } - - /** - * Helper: Convert JSON result to Excel-like format - */ - private convertJSONToExcelFormat(jsonResult: SmartJSONResult): Omit & { rows: any[] } { - const rows = jsonResult.entities.map(entity => ({ - entity, - relatedEntities: [], - relationships: jsonResult.relationships.filter(r => r.from === entity.id), - concepts: entity.metadata.concepts || [] - })) - - return { - rowsProcessed: jsonResult.nodesProcessed, - entitiesExtracted: jsonResult.entitiesExtracted, - relationshipsInferred: jsonResult.relationshipsInferred, - rows, - entityMap: jsonResult.entityMap, - processingTime: jsonResult.processingTime, - stats: jsonResult.stats - } - } - - /** - * Helper: Convert Markdown result to Excel-like format - */ - private convertMarkdownToExcelFormat(mdResult: SmartMarkdownResult): Omit & { rows: any[] } { - const rows = mdResult.sections.flatMap(section => - section.entities.map(entity => ({ - entity, - relatedEntities: [], - relationships: section.relationships.filter(r => r.from === entity.id), - concepts: section.concepts - })) - ) - - return { - rowsProcessed: mdResult.sectionsProcessed, - entitiesExtracted: mdResult.entitiesExtracted, - relationshipsInferred: mdResult.relationshipsInferred, - rows, - entityMap: mdResult.entityMap, - processingTime: mdResult.processingTime, - stats: mdResult.stats - } - } - - /** - * Get import statistics - */ - async getImportStatistics(vfsRootPath: string): Promise<{ - entitiesInGraph: number - relationshipsInGraph: number - filesInVFS: number - lastImport?: Date - }> { - // Read metadata file - const vfs = new VirtualFileSystem(this.brain) - await vfs.init() - - const metadataPath = `${vfsRootPath}/_metadata.json` - - try { - const metadataBuffer = await vfs.readFile(metadataPath) - const metadata = JSON.parse(metadataBuffer.toString('utf-8')) - - return { - entitiesInGraph: metadata.import.stats.entitiesExtracted, - relationshipsInGraph: metadata.import.stats.relationshipsInferred, - filesInVFS: metadata.structure.fileCount, - lastImport: new Date(metadata.import.timestamp) - } - } catch (error) { - return { - entitiesInGraph: 0, - relationshipsInGraph: 0, - filesInVFS: 0 - } - } - } -} diff --git a/src/importers/VFSStructureGenerator.ts b/src/importers/VFSStructureGenerator.ts index 46b74cf5..e638d747 100644 --- a/src/importers/VFSStructureGenerator.ts +++ b/src/importers/VFSStructureGenerator.ts @@ -189,8 +189,7 @@ export class VFSStructureGenerator { reportProgress('metadata', `Preserved source file: ${options.sourceFilename}`) } - // Note: groups already calculated above for progress tracking - // const groups = this.groupEntities(importResult, options) + // `groups` was already computed above (for progress tracking) and is reused here. // Create directories and files for each group for (const [groupName, entities] of groups.entries()) { diff --git a/src/importers/index.ts b/src/importers/index.ts deleted file mode 100644 index 12d0465d..00000000 --- a/src/importers/index.ts +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Smart Import System - * - * Production-ready entity and relationship extraction from multiple formats: - * - Excel (.xlsx) - * - PDF (.pdf) - * - CSV (.csv) - * - JSON (.json) - * - Markdown (.md) - * - * Uses brainy's built-in NeuralEntityExtractor and NaturalLanguageProcessor - * - * NO MOCKS - Real working implementation - */ - -// Excel Importer -export { SmartExcelImporter } from './SmartExcelImporter.js' -export type { - SmartExcelOptions, - ExtractedRow, - SmartExcelResult -} from './SmartExcelImporter.js' - -// PDF Importer -export { SmartPDFImporter } from './SmartPDFImporter.js' -export type { - SmartPDFOptions, - ExtractedSection, - SmartPDFResult -} from './SmartPDFImporter.js' - -// CSV Importer -export { SmartCSVImporter } from './SmartCSVImporter.js' -export type { - SmartCSVOptions, - SmartCSVResult -} from './SmartCSVImporter.js' - -// JSON Importer -export { SmartJSONImporter } from './SmartJSONImporter.js' -export type { - SmartJSONOptions, - ExtractedJSONEntity, - ExtractedJSONRelationship, - SmartJSONResult -} from './SmartJSONImporter.js' - -// Markdown Importer -export { SmartMarkdownImporter } from './SmartMarkdownImporter.js' -export type { - SmartMarkdownOptions, - MarkdownSection, - SmartMarkdownResult -} from './SmartMarkdownImporter.js' - -// VFS Structure Generator -export { VFSStructureGenerator } from './VFSStructureGenerator.js' -export type { - VFSStructureOptions, - VFSStructureResult -} from './VFSStructureGenerator.js' - -// Orchestrator (Main entry point) -export { SmartImportOrchestrator } from './SmartImportOrchestrator.js' -export type { - SmartImportOptions, - SmartImportProgress, - SmartImportResult -} from './SmartImportOrchestrator.js' diff --git a/src/index.ts b/src/index.ts index b563d2df..b978b9fd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,7 +6,7 @@ * - Brainy: The unified database with Triple Intelligence * - Triple Intelligence: Seamless fusion of vector + graph + field search * - Db: Immutable, generation-pinned database values (now/transact/asOf/with) - * - Plugins: Extensible plugin system (cortex, storage adapters) + * - Plugins: Extensible plugin system (cor, storage adapters) * - Neural Import: AI-powered entity extraction & smart data import */ @@ -15,8 +15,29 @@ import { Brainy } from './brainy.js' export { Brainy } +// The in-process change feed (brain.onChange) — event + listener types. +export type { + BrainyChangeEvent, + ChangeEventEntity, + ChangeEventRelation, + ChangeListener +} from './events/changeFeed.js' + +// Temporal VFS — a file version entry (vfs.history / readFile({ asOf })). +export type { FileVersion } from './vfs/types.js' + // Export diagnostics result type export type { DiagnosticsResult } from './brainy.js' +export type { + GraphAuditReport, + GraphAuditDiscrepancy +} from './graph/graphAudit.js' +export { + checkOsLimits, + NOFILE_POOL_FLOOR, + MAX_MAP_COUNT_POOL_FLOOR +} from './utils/osLimits.js' +export type { OsLimitsReport } from './utils/osLimits.js' // Export Brainy configuration and types export type { @@ -137,6 +158,11 @@ export { RevisionConflictError } from './transaction/RevisionConflictError.js' // transact/with() when a referenced entity or relation does not exist. export { EntityNotFoundError, RelationNotFoundError } from './errors/notFound.js' +// Base error + typed migration-lock error — thrown by any data-plane call while a +// brain runs its one-time 7.x→8.0 upgrade; catch to answer HTTP 503 + Retry-After. +export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError, ProtectedArtifactError, DerivedArtifactMissingError } from './errors/brainyError.js' +export type { BrainyErrorType } from './errors/brainyError.js' + // ============= 8.0 Db API — generational MVCC ============= // Immutable database values: brain.now() / brain.transact() / brain.asOf() / // db.with() / db.persist() / Brainy.load(). See src/db/ for the record layer. @@ -158,8 +184,11 @@ export type { export { GenerationConflictError, SpeculativeOverlayError, - GenerationCompactedError + GenerationCompactedError, + StoreInconsistentError, + PendingFlushDurabilityError } from './db/errors.js' +export type { UnreconciledRecord } from './db/errors.js' export type { TxOperation, TxAddOperation, @@ -172,14 +201,52 @@ export type { TxLogEntry, CompactHistoryOptions, CompactHistoryResult, + HistoryStats, ChangedIds, DiffResult, HistoryVersion, EntityHistory } from './db/types.js' +// The generation fact log — sequential after-image scan surface +// (brain.scanFacts / brain.factSegmentPaths) for index heals and replays. +export type { + CommitFact, + FactOp, + FactScanBatch, + SCANFACTS_FIRST_BATCH_MS, + FactScanHandle +} from './db/factLog.js' +// The generalized family stamp — which source generation a projection +// reflects + the surface that verifies it whole; one verifier, both member +// modes (enumerated byte-exact / rollup invariants). +export { readFamilyStamp, verifyFamilyStamp, ENTITY_TREE_STAMP_PATH } from './db/familyStamp.js' +export type { FamilyStamp, StampMembers, StampVerdict } from './db/familyStamp.js' // Optional provider capability for generation-aware native indexes export { isVersionedIndexProvider } from './plugin.js' export type { VersionedIndexProvider } from './plugin.js' +export type { ProviderInvariantReport, InvariantResult, InvariantHeal } from './plugin.js' +// Optional native graph-acceleration engine (cor 3.0) — the published provider +// contract + its columnar wire types. Brainy feature-detects an implementation +// and falls back to its pure-TS adjacency when absent. +export type { + GraphAccelerationProvider, + Subgraph, + OpaqueIdSet, + GraphTraversalDirection, + TraverseOptions, + EdgesForNodeOptions, + GraphCursorHandle, + GraphCursorOptions, + GraphCursorChunk, + GraphScores, + GraphCommunities, + GraphPath, + RankOptions, + CommunitiesOptions, + PathOptions, + SampleOptions, + MostConnectedOptions +} from './plugin.js' // Export embedding functionality import { @@ -259,7 +326,8 @@ import type { HNSWNoun, HNSWVerb, HNSWConfig, - StorageAdapter + StorageAdapter, + DerivedFamilyDeclaration } from './coreTypes.js' // Export vector index implementation (the JS HNSW path) @@ -277,7 +345,8 @@ export type { HNSWNoun, HNSWVerb, HNSWConfig, - StorageAdapter + StorageAdapter, + DerivedFamilyDeclaration } // Export graph types diff --git a/src/indexes/columnStore/ColumnSegmentCursor.ts b/src/indexes/columnStore/ColumnSegmentCursor.ts index 550d0773..79e78f39 100644 --- a/src/indexes/columnStore/ColumnSegmentCursor.ts +++ b/src/indexes/columnStore/ColumnSegmentCursor.ts @@ -7,7 +7,7 @@ * Multiple cursors can read the same segment concurrently (e.g. during k-way merge). * * For the TS baseline, segments are loaded fully into memory and cached via - * UnifiedCache. The Cortex Rust implementation mmaps the segment file and + * UnifiedCache. The Cor Rust implementation mmaps the segment file and * indexes into it directly — same read interface, zero-copy access. */ diff --git a/src/indexes/columnStore/ColumnStore.ts b/src/indexes/columnStore/ColumnStore.ts index 6c8a7d8b..d33c05c4 100644 --- a/src/indexes/columnStore/ColumnStore.ts +++ b/src/indexes/columnStore/ColumnStore.ts @@ -69,6 +69,34 @@ interface HeapEntry { * await store.flush() * const newest = await store.sortTopK('createdAt', 'desc', 10) // bigint[] */ +/** + * @description Thrown when a segment the manifest lists cannot be loaded — + * either it yields no bytes (missing/empty on disk) or its bytes are + * undecodable (corruption). Previously such a segment was silently skipped, + * dropping ALL of its entities from every `filter`/`rangeQuery`/`sortTopK` with + * no error — a manifest↔segments divergence that looked like a short result. + * Surfacing it loudly makes the divergence visible and repairable. (A genuine + * storage IO fault is a different class — it propagates as the underlying error, + * not wrapped in this.) + */ +export class ColumnSegmentLoadError extends Error { + /** The indexed field whose segment failed to load. */ + public readonly field: string + /** The manifest-listed segment id that could not be loaded. */ + public readonly segmentId: number | string + constructor(field: string, segmentId: number | string, reason: string) { + super( + `ColumnStore segment ${field}:${segmentId} is listed in the manifest but ` + + `could not be loaded (${reason}). The index is inconsistent with its ` + + `manifest — rebuild/repair the metadata index rather than trusting a ` + + `short query result.` + ) + this.name = 'ColumnSegmentLoadError' + this.field = field + this.segmentId = segmentId + } +} + export class ColumnStore implements ColumnStoreProvider { private storage!: StorageAdapter private idMapper!: EntityIdMapper @@ -476,9 +504,9 @@ export class ColumnStore implements ColumnStoreProvider { * Storage path (2.4.0 #4 / cortex-interchange contract): * When the storage adapter exposes the binary-blob primitive * (`saveBinaryBlob` — every brainy adapter ≥7.25.0 does), the segment - * bytes are written as a raw blob at the shared cortex key + * bytes are written as a raw blob at the shared cor key * `_column_index//L-NNNNNN` (suffix-free; the adapter - * appends its own). This matches byte-for-byte what cortex's + * appends its own). This matches byte-for-byte what cor's * `NativeColumnStore` writes, so JS- and native-written indexes * interchange without re-encoding. * @@ -522,7 +550,7 @@ export class ColumnStore implements ColumnStoreProvider { ) if (canUseBlob) { - // Raw blob, cortex-shared key convention. + // Raw blob, cor-shared key convention. const key = `${this.basePath}/${field}/L0-${String(segId).padStart(6, '0')}` await storage.saveBinaryBlob!(key, segBuffer) } else { @@ -561,7 +589,7 @@ export class ColumnStore implements ColumnStoreProvider { } // Persist the global deleted bitmap alongside the manifest. Same - // raw-blob preference as segments — shared cortex key `//DELETED`. + // raw-blob preference as segments — shared cor key `//DELETED`. const deleted = this.deletedEntities.get(field) if (deleted && deleted.size > 0) { const serialized = deleted.serialize(true) @@ -594,14 +622,15 @@ export class ColumnStore implements ColumnStoreProvider { let cursor = this.segmentCache.get(cacheKey) if (!cursor) { - const loaded = await this.loadSegmentCursor(field, seg) - if (loaded) { - cursor = loaded - this.segmentCache.set(cacheKey, cursor) - } + // loadSegmentCursor either returns a cursor or THROWS — a corrupt / + // missing manifest-listed segment raises ColumnSegmentLoadError and a + // real storage fault propagates, so a listed segment is never silently + // dropped from the result set. + cursor = await this.loadSegmentCursor(field, seg) + this.segmentCache.set(cacheKey, cursor) } - if (cursor) cursors.push(cursor) + cursors.push(cursor) } return cursors @@ -615,39 +644,51 @@ export class ColumnStore implements ColumnStoreProvider { * `.cidx` object-path so indexes written before the format unification keep * loading correctly. Mirror of cortex's 2.3.1 read-side fallback. */ - private async loadSegmentCursor(field: string, seg: SegmentMeta): Promise { + private async loadSegmentCursor(field: string, seg: SegmentMeta): Promise { const manifest = this.manifests.get(field) - if (!manifest) return null + if (!manifest) { + // Defensive: getSegmentCursors guards on the manifest before calling. + throw new ColumnSegmentLoadError(field, seg.id, 'no manifest for field') + } - try { - let buf: Buffer | null = null + let buf: Buffer | null = null - const storage = this.storage as unknown as { - loadBinaryBlob?: (key: string) => Promise - readObjectFromPath: (path: string) => Promise - } + const storage = this.storage as unknown as { + loadBinaryBlob?: (key: string) => Promise + readObjectFromPath: (path: string) => Promise + } - // Preferred: raw blob at the cortex-shared key. - if (typeof storage.loadBinaryBlob === 'function') { - const key = `${this.basePath}/${field}/L${seg.level}-${String(seg.id).padStart(6, '0')}` - const blob = await storage.loadBinaryBlob(key) - if (blob && blob.length > 0) buf = blob - } + // Preferred: raw blob at the cor-shared key. A real IO fault PROPAGATES — + // loadBinaryBlob throws on a fault and returns null only for genuine absence + // (a present-but-unreadable segment must not read as "missing"). + if (typeof storage.loadBinaryBlob === 'function') { + const key = `${this.basePath}/${field}/L${seg.level}-${String(seg.id).padStart(6, '0')}` + const blob = await storage.loadBinaryBlob(key) + if (blob && blob.length > 0) buf = blob + } - // Legacy fallback: `{ _binary, base64 }` envelope at the .cidx object-path. - if (!buf) { - const segPath = manifest.segmentPath(seg.level, seg.id) - const stored = await storage.readObjectFromPath(segPath) - if (!stored) return null + // Legacy fallback: `{ _binary, base64 }` envelope at the .cidx object-path. + if (!buf) { + const segPath = manifest.segmentPath(seg.level, seg.id) + const stored = await storage.readObjectFromPath(segPath) + if (stored) { if (stored._binary && stored.data) { buf = Buffer.from(stored.data, 'base64') } else if (Buffer.isBuffer(stored)) { buf = stored - } else { - return null } } + } + // A manifest-listed segment that yields NO loadable bytes is corruption, not + // benign absence: swallowing it silently dropped all of the segment's + // entities from every filter/rangeQuery/sortTopK with no error. Surface it + // loudly so the manifest↔segments divergence is visible (and repairable). + if (!buf) { + throw new ColumnSegmentLoadError(field, seg.id, 'manifest-listed segment has no loadable bytes') + } + + try { const parsed = readSegmentFromBuffer(buf) return new ColumnSegmentCursor( parsed.header, @@ -655,8 +696,13 @@ export class ColumnStore implements ColumnStoreProvider { parsed.entityIds, parsed.tombstones ) - } catch { - return null + } catch (err) { + // Undecodable bytes for a listed segment — corruption, not a short result. + throw new ColumnSegmentLoadError( + field, + seg.id, + `segment decode failed: ${(err as Error).message}` + ) } } diff --git a/src/indexes/columnStore/index.ts b/src/indexes/columnStore/index.ts deleted file mode 100644 index 1f9bf344..00000000 --- a/src/indexes/columnStore/index.ts +++ /dev/null @@ -1,38 +0,0 @@ -/** - * @module columnStore - * @description Unified column store for filtering, sorting, range queries, - * and text search on any field type with exact precision at billion scale. - * - * Replaces MetadataIndex's sparse chunk/bloom filter/bucketing internals - * with per-field sorted columns. Design lineage: Lucene doc values + roaring - * bitmap acceleration. - */ - -export { ColumnStore } from './ColumnStore.js' -export type { ColumnStoreConfig } from './ColumnStore.js' -export { ColumnTailBuffer } from './ColumnTailBuffer.js' -export { ColumnManifest } from './ColumnManifest.js' -export { ColumnSegmentCursor, TailBufferCursor } from './ColumnSegmentCursor.js' -export { - writeSegmentToBuffer, - readSegmentFromBuffer, - writeHeader, - readHeader, - crc32 -} from './ColumnSegmentFormat.js' -export { - ValueType, - CIDX_MAGIC, - CIDX_VERSION, - HEADER_SIZE, - FOOTER_SIZE, - DEFAULT_FLUSH_THRESHOLD, - FLAG_MULTI_VALUE -} from './types.js' -export type { - ColumnStoreProvider, - SegmentHeader, - SegmentFooter, - SegmentMeta, - ManifestData -} from './types.js' diff --git a/src/indexes/columnStore/types.ts b/src/indexes/columnStore/types.ts index a9f79059..71dd99a0 100644 --- a/src/indexes/columnStore/types.ts +++ b/src/indexes/columnStore/types.ts @@ -168,7 +168,7 @@ export const FLAG_MULTI_VALUE = 0x01 * The plugin-provider contract for the column store. * * Brainy ships a TypeScript baseline that implements this interface. - * Cortex registers a Rust-accelerated implementation at higher priority. + * Cor registers a Rust-accelerated implementation at higher priority. * The MetadataIndex coordinator calls whichever is registered. * * **8.0 u64 contract — BigInt entity ints at the boundary.** Entity ints flow diff --git a/src/integrations/core/IntegrationLoader.ts b/src/integrations/core/IntegrationLoader.ts index 519948b7..68aa84ba 100644 --- a/src/integrations/core/IntegrationLoader.ts +++ b/src/integrations/core/IntegrationLoader.ts @@ -83,6 +83,10 @@ export function detectEnvironment(): RuntimeEnvironment { Deno?: unknown Bun?: unknown HTMLRewriter?: unknown + // `caches` (Cloudflare Workers / ServiceWorker global) is no longer in `lib` + // now that DOM is dropped (8.0 is Node/Bun/Deno-only) — declare it locally so + // the edge-runtime probe below still type-checks. + caches?: unknown } // Deno diff --git a/src/integrations/odata/ODataIntegration.ts b/src/integrations/odata/ODataIntegration.ts index cd6a0cfd..1dd44bfe 100644 --- a/src/integrations/odata/ODataIntegration.ts +++ b/src/integrations/odata/ODataIntegration.ts @@ -253,7 +253,7 @@ export class ODataIntegration extends IntegrationBase implements HTTPIntegration /** * Get augmentation manifest */ - getManifest(): Record { + override getManifest(): Record { return { id: 'odata', name: 'OData Integration', diff --git a/src/integrations/sheets/GoogleSheetsIntegration.ts b/src/integrations/sheets/GoogleSheetsIntegration.ts index 2e368fcf..d22d0267 100644 --- a/src/integrations/sheets/GoogleSheetsIntegration.ts +++ b/src/integrations/sheets/GoogleSheetsIntegration.ts @@ -283,7 +283,7 @@ export class GoogleSheetsIntegration /** * Get manifest */ - getManifest(): Record { + override getManifest(): Record { return { id: 'sheets', name: 'Google Sheets Integration', diff --git a/src/integrations/sse/SSEIntegration.ts b/src/integrations/sse/SSEIntegration.ts index d0d347f8..67af21a9 100644 --- a/src/integrations/sse/SSEIntegration.ts +++ b/src/integrations/sse/SSEIntegration.ts @@ -292,7 +292,7 @@ export class SSEIntegration /** * Get manifest */ - getManifest(): Record { + override getManifest(): Record { return { id: 'sse', name: 'SSE Streaming', diff --git a/src/integrations/webhooks/WebhookIntegration.ts b/src/integrations/webhooks/WebhookIntegration.ts index 1455a202..1170880e 100644 --- a/src/integrations/webhooks/WebhookIntegration.ts +++ b/src/integrations/webhooks/WebhookIntegration.ts @@ -262,7 +262,7 @@ export class WebhookIntegration extends IntegrationBase { /** * Get manifest */ - getManifest(): Record { + override getManifest(): Record { return { id: 'webhooks', name: 'Webhooks', diff --git a/src/internals.ts b/src/internals.ts index bcd067cf..009124fe 100644 --- a/src/internals.ts +++ b/src/internals.ts @@ -16,6 +16,32 @@ export type { EntityIdMapperOptions, EntityIdMapperData } from './utils/entityId export { getRecommendedCacheConfig, formatBytes, checkMemoryPressure } from './utils/memoryDetection.js' export type { MemoryInfo, CacheAllocationStrategy } from './utils/memoryDetection.js' // Entity field resolution — single source of truth for reading fields off -// HNSWNounWithMetadata. First-party plugins (Cortex) use this to stay in +// HNSWNounWithMetadata. First-party plugins (Cor) use this to stay in // lockstep with the entity shape contract. export { resolveEntityField, STANDARD_ENTITY_FIELDS } from './coreTypes.js' + +// The generalized family stamp — ONE verifier, both member modes, shared +// verbatim with native providers so stamp verification is literally one +// function, never two synchronized copies. Providers write the same shape +// (their set-swap rewrites stamps, so adoption is migration-free). +export { + readFamilyStamp, + writeFamilyStamp, + verifyFamilyStamp, + FAMILY_STAMPS_PREFIX, + ENTITY_TREE_STAMP_PATH +} from './db/familyStamp.js' +export type { + FamilyStamp, + StampMembers, + StampVerdict, + EnumeratedMember, + StampStorage +} from './db/familyStamp.js' + +// Generation fact-log types — the scan surface a provider reaches through the +// storage capability (`storage.scanFacts` / `storage.factLogHeadGeneration` / +// `storage.factSegmentPaths`, wired by the host brain at init). Providers +// never construct a FactLog themselves: open() is writer-side (it reconciles +// by truncating/rewriting) and there is exactly one writer. +export type { CommitFact, FactOp, FactScanBatch, FactScanHandle } from './db/factLog.js' diff --git a/src/mcp/brainyMCPBroadcast.ts b/src/mcp/brainyMCPBroadcast.ts deleted file mode 100644 index 314e891f..00000000 --- a/src/mcp/brainyMCPBroadcast.ts +++ /dev/null @@ -1,363 +0,0 @@ -/** - * BrainyMCPBroadcast - * - * Enhanced MCP service with real-time WebSocket broadcasting capabilities - * for multi-agent coordination (Jarvis ↔ Picasso communication) - * - * Features: - * - WebSocket server for real-time push notifications - * - Subscription management for multiple Claude instances - * - Message broadcasting to all connected agents - * - Works both locally and with cloud deployment - */ - -import { WebSocketServer, WebSocket } from 'ws' -import { createServer, IncomingMessage } from 'node:http' -import { BrainyMCPService } from './brainyMCPService.js' -import { BrainyInterface } from '../types/brainyInterface.js' -import { MCPServiceOptions } from '../types/mcpTypes.js' -import { v4 as uuidv4 } from '../universal/uuid.js' - -interface BroadcastMessage { - id: string - from: string - to?: string | string[] - type: 'message' | 'notification' | 'sync' | 'heartbeat' | 'identify' - event?: string - data: any - timestamp: number -} - -interface ConnectedAgent { - id: string - name: string - role: string - socket: WebSocket - lastSeen: number -} - -export class BrainyMCPBroadcast extends BrainyMCPService { - private wsServer?: WebSocketServer - private httpServer?: any - private agents: Map = new Map() - private messageHistory: BroadcastMessage[] = [] - private maxHistorySize = 100 - - constructor( - brainyData: BrainyInterface, - options: MCPServiceOptions & { - broadcastPort?: number - cloudUrl?: string - } = {} - ) { - super(brainyData, options) - } - - /** - * Start the WebSocket broadcast server - * @param port Port to listen on (default: 8765) - * @param isCloud Whether this is a cloud deployment - */ - async startBroadcastServer(port = 8765, isCloud = false): Promise { - return new Promise((resolve, reject) => { - try { - // Create HTTP server - this.httpServer = createServer((req, res) => { - // Health check endpoint - if (req.url === '/health') { - res.writeHead(200, { 'Content-Type': 'application/json' }) - res.end(JSON.stringify({ - status: 'healthy', - agents: Array.from(this.agents.values()).map(a => ({ - id: a.id, - name: a.name, - role: a.role, - connected: true - })), - uptime: process.uptime() - })) - } else { - res.writeHead(404) - res.end('Not found') - } - }) - - // Create WebSocket server - this.wsServer = new WebSocketServer({ - server: this.httpServer, - perMessageDeflate: false // Better performance - }) - - this.wsServer.on('connection', (socket, request) => { - this.handleNewConnection(socket, request) - }) - - // Start listening - this.httpServer.listen(port, () => { - console.log(`🧠 Brain Jar Broadcast Server running on ${isCloud ? 'cloud' : 'local'} port ${port}`) - console.log(`📡 WebSocket: ws://localhost:${port}`) - console.log(`🔍 Health: http://localhost:${port}/health`) - resolve() - }) - - // Heartbeat to keep connections alive - setInterval(() => { - this.agents.forEach((agent) => { - if (Date.now() - agent.lastSeen > 30000) { - // Remove inactive agents - this.removeAgent(agent.id) - } else { - // Send heartbeat - this.sendToAgent(agent.id, { - id: uuidv4(), - from: 'server', - type: 'heartbeat', - data: { timestamp: Date.now() }, - timestamp: Date.now() - }) - } - }) - }, 15000) - - } catch (error) { - reject(error) - } - }) - } - - /** - * Handle new WebSocket connection - */ - private handleNewConnection(socket: WebSocket, request: IncomingMessage) { - const agentId = uuidv4() - - // Send welcome message - socket.send(JSON.stringify({ - id: uuidv4(), - from: 'server', - type: 'notification', - event: 'welcome', - data: { - agentId, - message: 'Connected to Brain Jar Broadcast Server', - agents: Array.from(this.agents.values()).map(a => ({ - id: a.id, - name: a.name, - role: a.role - })) - }, - timestamp: Date.now() - })) - - // Handle messages from this agent - socket.on('message', (data) => { - try { - const message = JSON.parse(data.toString()) - this.handleAgentMessage(agentId, message) - } catch (error) { - console.error('Invalid message from agent:', error) - } - }) - - // Handle disconnection - socket.on('close', () => { - this.removeAgent(agentId) - }) - - // Handle errors - socket.on('error', (error) => { - console.error(`Agent ${agentId} error:`, error) - }) - - // Store temporary connection until identified - this.agents.set(agentId, { - id: agentId, - name: 'Unknown', - role: 'Unknown', - socket, - lastSeen: Date.now() - }) - } - - /** - * Handle message from an agent - */ - private handleAgentMessage(agentId: string, message: any) { - const agent = this.agents.get(agentId) - if (!agent) return - - // Update last seen - agent.lastSeen = Date.now() - - // Handle identification - if (message.type === 'identify') { - agent.name = message.name || agent.name - agent.role = message.role || agent.role - - // Notify all agents about new member - this.broadcast({ - id: uuidv4(), - from: 'server', - type: 'notification', - event: 'agent_joined', - data: { - agent: { - id: agent.id, - name: agent.name, - role: agent.role - } - }, - timestamp: Date.now() - }, agentId) // Exclude the joining agent - - // Send recent history to new agent - if (this.messageHistory.length > 0) { - this.sendToAgent(agentId, { - id: uuidv4(), - from: 'server', - type: 'sync', - data: { - history: this.messageHistory.slice(-20) // Last 20 messages - }, - timestamp: Date.now() - }) - } - - return - } - - // Create broadcast message - const broadcastMsg: BroadcastMessage = { - id: message.id || uuidv4(), - from: agent.name, - to: message.to, - type: message.type || 'message', - event: message.event, - data: message.data, - timestamp: Date.now() - } - - // Store in history - this.addToHistory(broadcastMsg) - - // Broadcast based on recipient - if (message.to) { - // Send to specific agent(s) - const recipients = Array.isArray(message.to) ? message.to : [message.to] - recipients.forEach((recipientName: string) => { - const recipient = Array.from(this.agents.values()).find( - a => a.name === recipientName - ) - if (recipient) { - this.sendToAgent(recipient.id, broadcastMsg) - } - }) - } else { - // Broadcast to all agents except sender - this.broadcast(broadcastMsg, agentId) - } - } - - /** - * Broadcast message to all connected agents - */ - broadcast(message: BroadcastMessage, excludeId?: string) { - const messageStr = JSON.stringify(message) - - this.agents.forEach((agent) => { - if (agent.id !== excludeId && agent.socket.readyState === WebSocket.OPEN) { - agent.socket.send(messageStr) - } - }) - } - - /** - * Send message to specific agent - */ - private sendToAgent(agentId: string, message: BroadcastMessage) { - const agent = this.agents.get(agentId) - if (agent && agent.socket.readyState === WebSocket.OPEN) { - agent.socket.send(JSON.stringify(message)) - } - } - - /** - * Remove agent from connected list - */ - private removeAgent(agentId: string) { - const agent = this.agents.get(agentId) - if (agent) { - // Notify others about disconnection - this.broadcast({ - id: uuidv4(), - from: 'server', - type: 'notification', - event: 'agent_left', - data: { - agent: { - id: agent.id, - name: agent.name, - role: agent.role - } - }, - timestamp: Date.now() - }) - - this.agents.delete(agentId) - } - } - - /** - * Add message to history - */ - private addToHistory(message: BroadcastMessage) { - this.messageHistory.push(message) - - // Trim history if too large - if (this.messageHistory.length > this.maxHistorySize) { - this.messageHistory = this.messageHistory.slice(-this.maxHistorySize) - } - } - - /** - * Stop the broadcast server - */ - async stopBroadcastServer(): Promise { - // Close all agent connections - this.agents.forEach(agent => { - agent.socket.close(1000, 'Server shutting down') - }) - this.agents.clear() - - // Close WebSocket server - if (this.wsServer) { - this.wsServer.close() - } - - // Close HTTP server - if (this.httpServer) { - this.httpServer.close() - } - } - - /** - * Get connected agents - */ - getConnectedAgents(): Array<{ id: string; name: string; role: string }> { - return Array.from(this.agents.values()).map(a => ({ - id: a.id, - name: a.name, - role: a.role - })) - } - - /** - * Get message history - */ - getMessageHistory(): BroadcastMessage[] { - return [...this.messageHistory] - } -} - -// Export for both environments -export default BrainyMCPBroadcast \ No newline at end of file diff --git a/src/mcp/brainyMCPClient.ts b/src/mcp/brainyMCPClient.ts deleted file mode 100644 index 85ee1eb6..00000000 --- a/src/mcp/brainyMCPClient.ts +++ /dev/null @@ -1,322 +0,0 @@ -/** - * BrainyMCPClient - * - * Client for connecting Claude instances to the Brain Jar Broadcast Server - * Utilizes Brainy for persistent memory and vector search capabilities - */ - -import WebSocket from 'ws' -import { Brainy } from '../brainy.js' -import { NounType } from '../types/graphTypes.js' -import { v4 as uuidv4 } from '../universal/uuid.js' - -interface ClientOptions { - name: string // e.g., 'Jarvis' or 'Picasso' - role: string // e.g., 'Backend Systems' or 'Frontend Design' - serverUrl?: string // Default: ws://localhost:8765 - autoReconnect?: boolean - useBrainyMemory?: boolean // Store messages in Brainy for persistence -} - -interface Message { - id: string - from: string - to?: string | string[] - type: 'message' | 'notification' | 'sync' | 'heartbeat' | 'identify' - event?: string - data: any - timestamp: number -} - -export class BrainyMCPClient { - private socket?: WebSocket - private options: Required - private brainy?: Brainy - private messageHandlers: Map void> = new Map() - private reconnectTimeout?: NodeJS.Timeout - private isConnected = false - - constructor(options: ClientOptions) { - this.options = { - serverUrl: 'ws://localhost:8765', - autoReconnect: true, - useBrainyMemory: true, - ...options - } - } - - /** - * Initialize Brainy for persistent memory - */ - private async initBrainy() { - if (this.options.useBrainyMemory && !this.brainy) { - this.brainy = new Brainy({ - storage: { - requestPersistentStorage: true - } - }) - await this.brainy.init() - console.log(`🧠 Brainy memory initialized for ${this.options.name}`) - } - } - - /** - * Connect to the broadcast server - */ - async connect(): Promise { - // Initialize Brainy first - await this.initBrainy() - - return new Promise((resolve, reject) => { - try { - this.socket = new WebSocket(this.options.serverUrl) - - this.socket.on('open', () => { - console.log(`✅ ${this.options.name} connected to Brain Jar Broadcast`) - this.isConnected = true - - // Identify ourselves - this.send({ - type: 'identify', - data: { - name: this.options.name, - role: this.options.role - } - }) - - resolve() - }) - - this.socket.on('message', async (data) => { - try { - const message = JSON.parse(data.toString()) as Message - await this.handleMessage(message) - } catch (error) { - console.error('Error parsing message:', error) - } - }) - - this.socket.on('close', () => { - console.log(`❌ ${this.options.name} disconnected from Brain Jar`) - this.isConnected = false - - if (this.options.autoReconnect) { - this.scheduleReconnect() - } - }) - - this.socket.on('error', (error) => { - console.error(`Connection error for ${this.options.name}:`, error) - reject(error) - }) - - } catch (error) { - reject(error) - } - }) - } - - /** - * Handle incoming message - */ - private async handleMessage(message: Message) { - // Store in Brainy for persistent memory. Subtype `'mcp-message'` marks - // these as MCP-protocol messages so consumers can filter / count them via - // `find({ type: NounType.Message, subtype: 'mcp-message' })` and so - // enforcement consumers registering a vocabulary on NounType.Message don't - // reject MCP traffic (added 7.30.1; also fixes the pre-existing missing - // `data` field by aliasing from the prior `text` field). - if (this.brainy && message.type === 'message') { - try { - await this.brainy.add({ - data: `${message.from}: ${JSON.stringify(message.data)}`, - type: NounType.Message, - subtype: 'mcp-message', - metadata: { - messageId: message.id, - from: message.from, - to: message.to, - timestamp: message.timestamp, - messageType: message.type, - event: message.event - } - }) - } catch (error) { - console.error('Error storing message in Brainy:', error) - } - } - - // Handle sync messages (receive history) - if (message.type === 'sync' && message.data.history) { - console.log(`📜 ${this.options.name} received ${message.data.history.length} historical messages`) - - // Store history in Brainy with the same subtype as live messages. - if (this.brainy) { - for (const histMsg of message.data.history) { - await this.brainy.add({ - data: `${histMsg.from}: ${JSON.stringify(histMsg.data)}`, - type: NounType.Message, - subtype: 'mcp-message', - metadata: { - ...histMsg - } - }) - } - } - } - - // Call registered handlers - const handler = this.messageHandlers.get(message.type) - if (handler) { - handler(message) - } - - // Call universal handler - const universalHandler = this.messageHandlers.get('*') - if (universalHandler) { - universalHandler(message) - } - } - - /** - * Send a message - */ - send(message: Partial) { - if (!this.socket || this.socket.readyState !== WebSocket.OPEN) { - console.error(`${this.options.name} is not connected`) - return - } - - const fullMessage: Message = { - id: message.id || uuidv4(), - from: this.options.name, - type: message.type || 'message', - data: message.data || {}, - timestamp: Date.now(), - ...message - } - - this.socket.send(JSON.stringify(fullMessage)) - } - - /** - * Send a message to specific agent(s) - */ - sendTo(recipient: string | string[], data: any) { - this.send({ - to: recipient, - type: 'message', - data - }) - } - - /** - * Broadcast to all agents - */ - broadcast(data: any) { - this.send({ - type: 'message', - data - }) - } - - /** - * Register a message handler - */ - on(type: string, handler: (message: Message) => void) { - this.messageHandlers.set(type, handler) - } - - /** - * Remove a message handler - */ - off(type: string) { - this.messageHandlers.delete(type) - } - - /** - * Search historical messages using Brainy's vector search - */ - async searchMemory(query: string, limit = 10): Promise { - if (!this.brainy) { - console.warn('Brainy memory not initialized') - return [] - } - - const results = await this.brainy.find({ query, limit }) - return results.map(r => ({ - ...r.metadata, - relevance: r.score - })) - } - - /** - * Get recent messages from Brainy memory - */ - async getRecentMessages(limit = 20): Promise { - if (!this.brainy) { - console.warn('Brainy memory not initialized') - return [] - } - - // Search for recent activity - const results = await this.brainy.find({ query: 'recent messages communication', limit }) - return results - .map(r => r.metadata) - .sort((a: any, b: any) => b.timestamp - a.timestamp) - } - - /** - * Schedule reconnection attempt - */ - private scheduleReconnect() { - if (this.reconnectTimeout) { - clearTimeout(this.reconnectTimeout) - } - - this.reconnectTimeout = setTimeout(() => { - console.log(`🔄 ${this.options.name} attempting to reconnect...`) - this.connect().catch(error => { - console.error('Reconnection failed:', error) - this.scheduleReconnect() - }) - }, 5000) - } - - /** - * Disconnect from server - */ - disconnect() { - if (this.reconnectTimeout) { - clearTimeout(this.reconnectTimeout) - } - - if (this.socket) { - this.socket.close(1000, 'Client disconnecting') - this.socket = undefined - } - - this.isConnected = false - } - - /** - * Check if connected - */ - getIsConnected(): boolean { - return this.isConnected - } - - /** - * Get agent info - */ - getAgentInfo() { - return { - name: this.options.name, - role: this.options.role, - connected: this.isConnected - } - } -} - -// Export for both environments -export default BrainyMCPClient \ No newline at end of file diff --git a/src/neural/embeddedPatterns.ts b/src/neural/embeddedPatterns.ts index a8ece789..c15447e7 100644 --- a/src/neural/embeddedPatterns.ts +++ b/src/neural/embeddedPatterns.ts @@ -2,7 +2,7 @@ * 🧠 BRAINY EMBEDDED PATTERNS * * AUTO-GENERATED - DO NOT EDIT - * Generated: 2026-06-11T21:32:56.112Z + * Generated: 2026-07-02T21:43:26.976Z * Patterns: 220 * Coverage: 94-98% of all queries * diff --git a/src/neural/naturalLanguageProcessorStatic.ts b/src/neural/naturalLanguageProcessorStatic.ts deleted file mode 100644 index c7bea9ff..00000000 --- a/src/neural/naturalLanguageProcessorStatic.ts +++ /dev/null @@ -1,200 +0,0 @@ -/** - * 🧠 Natural Language Query Processor - STATIC VERSION - * No runtime initialization, no memory leaks, patterns pre-built at compile time - * - * Uses static pattern matching with 220 pre-built patterns - */ - -import { Vector } from '../coreTypes.js' -import { TripleQuery } from '../triple/TripleIntelligence.js' -import { patternMatchQuery, PATTERN_STATS } from './staticPatternMatcher.js' - -export interface NaturalQueryIntent { - type: 'vector' | 'field' | 'graph' | 'combined' - confidence: number - extractedTerms: { - entities?: string[] - fields?: string[] - relationships?: string[] - modifiers?: string[] - } -} - -export class NaturalLanguageProcessor { - private queryHistory: Array<{ query: string; result: TripleQuery; success: boolean }> - - constructor() { - this.queryHistory = [] - // Patterns are static - no initialization needed! - } - - /** - * No initialization needed - patterns are pre-built! - */ - async init(): Promise { - // Nothing to do - patterns are compiled into the code - return Promise.resolve() - } - - /** - * Process natural language query into structured Triple Intelligence query - * @param naturalQuery The natural language query string - * @param queryEmbedding Pre-computed embedding from Brainy (passed in to avoid circular dependency) - */ - async processNaturalQuery(naturalQuery: string, queryEmbedding?: Vector): Promise { - // Use static pattern matcher (no async, no memory allocation!) - const structuredQuery = patternMatchQuery(naturalQuery, queryEmbedding) - - // Step 3: Enhance with intent analysis if needed - if (!structuredQuery.where && !structuredQuery.connected) { - const intent = await this.analyzeIntent(naturalQuery) - - // Add metadata based on intent - if (intent.type === 'field' && intent.extractedTerms.fields) { - structuredQuery.where = this.buildFieldConstraints(intent.extractedTerms.fields) - } - } - - // Track for learning (but don't create new Brainy!) - this.queryHistory.push({ - query: naturalQuery, - result: structuredQuery, - success: false // Will be updated based on user interaction - }) - - // Keep history limited to prevent memory growth - if (this.queryHistory.length > 100) { - this.queryHistory.shift() - } - - return structuredQuery - } - - /** - * Analyze query intent using keywords - */ - private async analyzeIntent(query: string): Promise { - const lowerQuery = query.toLowerCase() - - // Check for field-specific keywords - const fieldKeywords = ['where', 'filter', 'with', 'has', 'contains', 'equals', 'greater', 'less', 'between'] - const hasFieldIntent = fieldKeywords.some(kw => lowerQuery.includes(kw)) - - // Check for graph keywords - const graphKeywords = ['related', 'connected', 'linked', 'associated', 'references'] - const hasGraphIntent = graphKeywords.some(kw => lowerQuery.includes(kw)) - - // Determine type - let type: NaturalQueryIntent['type'] = 'vector' - if (hasFieldIntent && hasGraphIntent) { - type = 'combined' - } else if (hasFieldIntent) { - type = 'field' - } else if (hasGraphIntent) { - type = 'graph' - } - - return { - type, - confidence: 0.8, - extractedTerms: { - fields: hasFieldIntent ? this.extractFieldTerms(query) : undefined, - relationships: hasGraphIntent ? this.extractRelationshipTerms(query) : undefined - } - } - } - - /** - * Extract field terms from query - */ - private extractFieldTerms(query: string): string[] { - const terms: string[] = [] - - // Simple extraction of potential field names - const words = query.split(/\s+/) - const fieldIndicators = ['year', 'date', 'author', 'type', 'category', 'status', 'price'] - - for (const word of words) { - if (fieldIndicators.includes(word.toLowerCase())) { - terms.push(word.toLowerCase()) - } - } - - return terms - } - - /** - * Extract relationship terms - */ - private extractRelationshipTerms(query: string): string[] { - const terms: string[] = [] - const relationshipWords = ['related', 'connected', 'linked', 'references', 'cites'] - - const words = query.toLowerCase().split(/\s+/) - for (const word of words) { - if (relationshipWords.includes(word)) { - terms.push(word) - } - } - - return terms - } - - /** - * Build field constraints from extracted terms - */ - private buildFieldConstraints(fields: string[]): Record { - const constraints: Record = {} - - // Simple mapping for common fields - for (const field of fields) { - // This would be enhanced with actual value extraction - constraints[field] = { exists: true } - } - - return constraints - } - - /** - * Find similar queries from history (without using Brainy) - * NOTE: Currently unused - reserved for future query caching optimization - */ - private findSimilarQueries(embedding: Vector): Array<{ - query: string - result: TripleQuery - similarity: number - }> { - // Not implemented - not required for core functionality - // Would implement cosine similarity against queryHistory if needed - return [] - } - - /** - * Adapt a previous query for new input - */ - private adaptQuery(newQuery: string, previousResult: TripleQuery): TripleQuery { - return previousResult - } - - /** - * Extract entities from query - */ - private async extractEntities(query: string): Promise { - // Could use the Entity Registry here if available - return [] - } - - /** - * Build query from components - */ - private buildQuery( - query: string, - intent: NaturalQueryIntent, - entities: string[] - ): TripleQuery { - return { - like: query, - limit: 10 - } - } -} \ No newline at end of file diff --git a/src/neural/neuralImport.ts b/src/neural/neuralImport.ts index 8a2ba843..c8d19eb5 100644 --- a/src/neural/neuralImport.ts +++ b/src/neural/neuralImport.ts @@ -7,6 +7,7 @@ import { Brainy } from '../brainy.js' import { NounType, VerbType } from '../types/graphTypes.js' +import { splitNounMetadataRecord, splitVerbMetadataRecord } from '../types/reservedFields.js' import * as fs from '../universal/fs.js' import * as path from '../universal/path.js' // @ts-ignore @@ -802,9 +803,12 @@ export class NeuralImport { data: this.extractMainText(entity.originalData), type: entity.nounType as NounType, subtype: entity.subtype ?? options.defaultSubtype ?? 'extracted', + // `confidence` is a reserved field — dedicated param, not metadata + // (8.0 reservedFieldPolicy defaults to 'throw'). + confidence: entity.confidence, metadata: { - ...entity.originalData, - confidence: entity.confidence, + // Strip any reserved keys the source data smuggled into the bag. + ...splitNounMetadataRecord(entity.originalData).custom, id: entity.suggestedId } }) @@ -821,7 +825,8 @@ export class NeuralImport { confidence: relationship.confidence, // reserved field — dedicated param, not metadata metadata: { context: relationship.context, - ...relationship.metadata + // Strip any reserved keys smuggled into the edge metadata bag. + ...splitVerbMetadataRecord(relationship.metadata).custom } }) } diff --git a/src/neural/neuralImportAugmentation.ts b/src/neural/neuralImportAugmentation.ts deleted file mode 100644 index 2942d884..00000000 --- a/src/neural/neuralImportAugmentation.ts +++ /dev/null @@ -1,633 +0,0 @@ -/** - * Neural Import - AI-Powered Data Understanding - * - * Standalone implementation for intelligent data processing. - */ - -import { NounType, VerbType } from '../types/graphTypes.js' -import * as fs from '../universal/fs.js' -import * as path from '../universal/path.js' -import { prodLog } from '../utils/logger.js' - -// Neural Import Analysis Types -export interface NeuralAnalysisResult { - detectedEntities: DetectedEntity[] - detectedRelationships: DetectedRelationship[] - confidence: number - insights: NeuralInsight[] -} - -export interface DetectedEntity { - originalData: any - nounType: string - confidence: number - suggestedId: string - reasoning: string - alternativeTypes: Array<{ type: string, confidence: number }> -} - -export interface DetectedRelationship { - sourceId: string - targetId: string - verbType: string - confidence: number - weight: number - reasoning: string - context: string - metadata?: Record -} - -export interface NeuralInsight { - type: 'hierarchy' | 'cluster' | 'pattern' | 'anomaly' | 'opportunity' - description: string - confidence: number - affectedEntities: string[] - recommendation?: string -} - -export interface NeuralImportConfig { - confidenceThreshold: number - enableWeights: boolean - skipDuplicates: boolean - categoryFilter?: string[] - dataType?: string -} - -/** - * Neural Import Augmentation - Unified Implementation - * Processes data with AI before storage operations - */ -export class NeuralImportAugmentation { - readonly name = 'neural-import' - private operations = ['add', 'addNoun', 'addVerb', 'all'] - - protected config: NeuralImportConfig - private analysisCache = new Map() - private context?: { brain: any } - - constructor(config: Partial = {}) { - this.config = { - confidenceThreshold: 0.7, - enableWeights: true, - skipDuplicates: true, - dataType: 'json', - ...config - } - } - - async init(): Promise { - // No external dependencies to initialize - } - - private log(message: string, _level?: string): void { - // Silent by default - } - - /** - * Execute augmentation - process data with AI before storage - */ - async execute( - operation: string, - params: any, - next: () => Promise - ): Promise { - // Only process on add operations - if (!this.operations.includes(operation)) { - return next() - } - - try { - // Extract data from params based on operation - const rawData = this.extractRawData(operation, params) - if (!rawData) { - return next() - } - - // Perform neural analysis - const analysis = await this.performNeuralAnalysis(rawData, this.config) - - // Enhance params with neural insights - if (params.metadata) { - params.metadata._neuralProcessed = true - params.metadata._neuralConfidence = analysis.confidence - params.metadata._detectedEntities = analysis.detectedEntities.length - params.metadata._detectedRelationships = analysis.detectedRelationships.length - params.metadata._neuralInsights = analysis.insights - } else if (typeof params === 'object') { - params.metadata = { - _neuralProcessed: true, - _neuralConfidence: analysis.confidence, - _detectedEntities: analysis.detectedEntities.length, - _detectedRelationships: analysis.detectedRelationships.length, - _neuralInsights: analysis.insights - } - } - - // Store neural analysis for later retrieval - await this.storeNeuralAnalysis(analysis) - - // If we detected entities/relationships, potentially add them - if (this.context?.brain && analysis.detectedEntities.length > 0) { - // This could automatically create entities/relationships - // But for now, just enhance the metadata - this.log(`Detected ${analysis.detectedEntities.length} entities and ${analysis.detectedRelationships.length} relationships`) - } - - // Continue with enhanced data - return next() - } catch (error) { - this.log(`Neural analysis failed: ${error}`, 'warn') - // Continue without neural processing - return next() - } - } - - /** - * Extract raw data from operation params - */ - private extractRawData(operation: string, params: any): any { - switch (operation) { - case 'add': - return params.content || params.data || params - case 'addNoun': - return params.noun || params.data || params - case 'addVerb': - return params.verb || params - case 'addBatch': - return params.items || params.batch || params - default: - return null - } - } - - /** - * Get the full neural analysis result (for external use) - */ - async getNeuralAnalysis(rawData: Buffer | string, dataType?: string): Promise { - const parsedData = await this.parseRawData(rawData, dataType || this.config.dataType || 'json') - return await this.performNeuralAnalysis(parsedData, this.config) - } - - /** - * Parse raw data based on type - */ - private async parseRawData(rawData: Buffer | string, dataType: string): Promise { - const content = typeof rawData === 'string' ? rawData : rawData.toString('utf8') - - switch (dataType.toLowerCase()) { - case 'json': - try { - const jsonData = JSON.parse(content) - return Array.isArray(jsonData) ? jsonData : [jsonData] - } catch { - // If JSON parse fails, treat as text - return [{ text: content }] - } - - case 'csv': - return this.parseCSV(content) - - case 'yaml': - case 'yml': - return this.parseYAML(content) - - case 'txt': - case 'text': - // Split text into sentences/paragraphs for analysis - return content.split(/\n+/).filter(line => line.trim()).map(line => ({ text: line })) - - default: - // Unknown type, treat as text - return [{ text: content }] - } - } - - /** - * Parse CSV data - handles quoted values, escaped quotes, and edge cases - */ - private parseCSV(content: string): any[] { - const lines = content.split('\n') - if (lines.length === 0) return [] - - // Parse a CSV line handling quotes - const parseLine = (line: string): string[] => { - const result: string[] = [] - let current = '' - let inQuotes = false - let i = 0 - - while (i < line.length) { - const char = line[i] - const nextChar = line[i + 1] - - if (char === '"') { - if (inQuotes && nextChar === '"') { - // Escaped quote - current += '"' - i += 2 - } else { - // Toggle quote mode - inQuotes = !inQuotes - i++ - } - } else if (char === ',' && !inQuotes) { - // Field separator - result.push(current.trim()) - current = '' - i++ - } else { - current += char - i++ - } - } - - // Add last field - result.push(current.trim()) - return result - } - - // Parse headers - const headers = parseLine(lines[0]) - const data = [] - - // Parse data rows - for (let i = 1; i < lines.length; i++) { - const line = lines[i].trim() - if (!line) continue // Skip empty lines - - const values = parseLine(line) - const row: any = {} - - headers.forEach((header, index) => { - const value = values[index] || '' - // Try to parse numbers - const num = Number(value) - row[header] = !isNaN(num) && value !== '' ? num : value - }) - - data.push(row) - } - - return data - } - - /** - * Parse YAML data - */ - private parseYAML(content: string): any[] { - try { - // Simple YAML parser for basic structures - // For full YAML support, we'd use js-yaml library - const lines = content.split('\n') - const result: any[] = [] - let currentObject: any = null - let currentIndent = 0 - - for (const line of lines) { - const trimmed = line.trim() - if (!trimmed || trimmed.startsWith('#')) continue // Skip empty lines and comments - - // Calculate indentation - const indent = line.length - line.trimStart().length - - // Check for array item - if (trimmed.startsWith('- ')) { - const value = trimmed.substring(2).trim() - if (indent === 0) { - // Top-level array item - if (value.includes(':')) { - // Object in array - currentObject = {} - result.push(currentObject) - const [key, val] = value.split(':').map(s => s.trim()) - currentObject[key] = this.parseYAMLValue(val) - } else { - result.push(this.parseYAMLValue(value)) - } - } else if (currentObject) { - // Nested array - const lastKey = Object.keys(currentObject).pop() - if (lastKey) { - if (!Array.isArray(currentObject[lastKey])) { - currentObject[lastKey] = [] - } - currentObject[lastKey].push(this.parseYAMLValue(value)) - } - } - } else if (trimmed.includes(':')) { - // Key-value pair - const colonIndex = trimmed.indexOf(':') - const key = trimmed.substring(0, colonIndex).trim() - const value = trimmed.substring(colonIndex + 1).trim() - - if (indent === 0) { - // Top-level object - if (!currentObject) { - currentObject = {} - result.push(currentObject) - } - currentObject[key] = this.parseYAMLValue(value) - currentIndent = 0 - } else if (currentObject) { - // Nested object - if (indent > currentIndent && !value) { - // Start of nested object - const lastKey = Object.keys(currentObject).pop() - if (lastKey) { - currentObject[lastKey] = { [key]: '' } - } - } else { - currentObject[key] = this.parseYAMLValue(value) - } - currentIndent = indent - } - } - } - - // If we built a single object and not an array, wrap it - if (result.length === 0 && currentObject) { - result.push(currentObject) - } - - return result.length > 0 ? result : [{ text: content }] - } catch (error) { - prodLog.warn('YAML parsing failed, treating as text:', error) - return [{ text: content }] - } - } - - /** - * Parse a YAML value (handle strings, numbers, booleans, null) - */ - private parseYAMLValue(value: string): any { - if (!value || value === '~' || value === 'null') return null - if (value === 'true') return true - if (value === 'false') return false - - // Remove quotes if present - if ((value.startsWith('"') && value.endsWith('"')) || - (value.startsWith("'") && value.endsWith("'"))) { - return value.slice(1, -1) - } - - // Try to parse as number - const num = Number(value) - if (!isNaN(num) && value !== '') return num - - return value - } - - /** - * Perform neural analysis on parsed data - */ - private async performNeuralAnalysis(data: any[], config?: any): Promise { - const detectedEntities: DetectedEntity[] = [] - const detectedRelationships: DetectedRelationship[] = [] - const insights: NeuralInsight[] = [] - - // Simple entity detection (in real implementation, would use ML) - for (const item of data) { - if (typeof item === 'object') { - // Detect entities from object properties - const entityId = item.id || item.name || item.title || `entity_${Date.now()}_${Math.random()}` - - detectedEntities.push({ - originalData: item, - nounType: await this.inferNounType(item), - confidence: 0.85, - suggestedId: String(entityId), - reasoning: 'Detected from structured data', - alternativeTypes: [] - }) - - // Detect relationships from references - await this.detectRelationships(item, entityId, detectedRelationships) - } - } - - // Generate insights - if (detectedEntities.length > 10) { - insights.push({ - type: 'pattern', - description: `Large dataset with ${detectedEntities.length} entities detected`, - confidence: 0.9, - affectedEntities: detectedEntities.slice(0, 5).map(e => e.suggestedId), - recommendation: 'Consider batch processing for optimal performance' - }) - } - - // Look for clusters - const typeGroups = this.groupByType(detectedEntities) - if (Object.keys(typeGroups).length > 1) { - insights.push({ - type: 'cluster', - description: `Multiple entity types detected: ${Object.keys(typeGroups).join(', ')}`, - confidence: 0.8, - affectedEntities: [], - recommendation: 'Data contains diverse entity types suitable for graph analysis' - }) - } - - return { - detectedEntities, - detectedRelationships, - confidence: detectedEntities.length > 0 ? 0.85 : 0.5, - insights - } - } - - /** - * Infer noun type from object structure using field heuristics - */ - private async inferNounType(obj: any): Promise { - if (typeof obj !== 'object' || obj === null) return NounType.Thing - - // Check for explicit type field - if (obj.type && typeof obj.type === 'string') { - const normalized = obj.type.charAt(0).toUpperCase() + obj.type.slice(1) - if (Object.values(NounType).includes(normalized as NounType)) { - return normalized as NounType - } - } - - if (obj.email || obj.firstName || obj.lastName || obj.username) return NounType.Person - if (obj.companyName || obj.organizationId || obj.employees) return NounType.Organization - if (obj.latitude || obj.longitude || obj.address || obj.city) return NounType.Location - if ((obj.content && (obj.title || obj.author)) || obj.pages) return NounType.Document - if (obj.startTime || obj.endTime || obj.date || obj.attendees) return NounType.Event - if (obj.price || obj.sku || obj.productId) return NounType.Product - if ((obj.status && obj.assignee) || obj.priority) return NounType.Task - if (Array.isArray(obj.data) || obj.rows || obj.columns) return NounType.Dataset - - return NounType.Thing - } - - /** - * Detect relationships from object references - */ - private async detectRelationships(obj: any, sourceId: string, relationships: DetectedRelationship[]): Promise { - // Look for reference patterns - for (const [key, value] of Object.entries(obj)) { - if (key.endsWith('Id') || key.endsWith('_id') || key === 'parentId' || key === 'userId') { - relationships.push({ - sourceId, - targetId: String(value), - verbType: await this.inferVerbType(key, obj, { id: value }), - confidence: 0.75, - weight: 1, - reasoning: `Reference detected in field: ${key}`, - context: key - }) - } - - // Array of IDs - if (Array.isArray(value) && value.length > 0 && typeof value[0] === 'string') { - if (key.endsWith('Ids') || key.endsWith('_ids')) { - for (const targetId of value) { - relationships.push({ - sourceId, - targetId: String(targetId), - verbType: await this.inferVerbType(key, obj, { id: targetId }), - confidence: 0.7, - weight: 1, - reasoning: `Array reference in field: ${key}`, - context: key - }) - } - } - } - } - } - - /** - * Infer verb type from field name using common patterns - */ - private async inferVerbType(fieldName: string, _sourceObj?: any, _targetObj?: any): Promise { - const field = fieldName.toLowerCase() - - if (field.includes('parent') || field.includes('child') || field.includes('contain')) { - return VerbType.Contains - } - if (field.includes('owner') || field.includes('created') || field.includes('author')) { - return VerbType.Creates - } - if (field.includes('member') || field.includes('belong')) { - return VerbType.MemberOf - } - if (field.includes('depend') || field.includes('require')) { - return VerbType.DependsOn - } - if (field.includes('ref') || field.includes('link') || field.includes('source')) { - return VerbType.References - } - - return VerbType.RelatedTo - } - - /** - * Group entities by type - */ - private groupByType(entities: DetectedEntity[]): Record { - const groups: Record = {} - - for (const entity of entities) { - if (!groups[entity.nounType]) { - groups[entity.nounType] = [] - } - groups[entity.nounType].push(entity) - } - - return groups - } - - /** - * Store neural analysis results - */ - private async storeNeuralAnalysis(analysis: NeuralAnalysisResult): Promise { - // Cache the analysis for potential later use - const key = `analysis_${Date.now()}` - this.analysisCache.set(key, analysis) - - // Limit cache size - if (this.analysisCache.size > 100) { - const firstKey = this.analysisCache.keys().next().value - if (firstKey) { - this.analysisCache.delete(firstKey) - } - } - } - - /** - * Helper to get data type from file path - */ - private getDataTypeFromPath(filePath: string): string { - const ext = path.extname(filePath).toLowerCase() - switch (ext) { - case '.json': return 'json' - case '.csv': return 'csv' - case '.txt': return 'text' - case '.yaml': - case '.yml': return 'yaml' - default: return 'text' - } - } - - /** - * PUBLIC API: Process raw data (for external use, like Synapses) - * This maintains compatibility with code that wants to use Neural Import directly - */ - async processRawData( - rawData: Buffer | string, - dataType: string, - options?: Record - ): Promise<{ - success: boolean - data: { - nouns: string[] - verbs: string[] - confidence?: number - insights?: Array<{ - type: string - description: string - confidence: number - }> - metadata?: Record - } - error?: string - }> { - try { - const analysis = await this.getNeuralAnalysis(rawData, dataType) - - // Convert to legacy format for compatibility - const nouns = analysis.detectedEntities.map(e => e.suggestedId) - const verbs = analysis.detectedRelationships.map(r => - `${r.sourceId}->${r.verbType}->${r.targetId}` - ) - - return { - success: true, - data: { - nouns, - verbs, - confidence: analysis.confidence, - insights: analysis.insights.map(i => ({ - type: i.type, - description: i.description, - confidence: i.confidence - })), - metadata: { - detectedEntities: analysis.detectedEntities.length, - detectedRelationships: analysis.detectedRelationships.length, - timestamp: new Date().toISOString() - } - } - } - } catch (error) { - return { - success: false, - data: { nouns: [], verbs: [] }, - error: error instanceof Error ? error.message : 'Neural analysis failed' - } - } - } -} \ No newline at end of file diff --git a/src/neural/presets.ts b/src/neural/presets.ts deleted file mode 100644 index 6d309aed..00000000 --- a/src/neural/presets.ts +++ /dev/null @@ -1,479 +0,0 @@ -/** - * Smart Import Presets - Zero-Configuration Auto-Detection - * - * Automatically selects optimal import strategy based on: - * - File type (Excel, CSV, PDF, Markdown, JSON) - * - File size and row count - * - Column structure (explicit relationships vs narrative) - * - Available memory and performance requirements - * - * Production-ready: Handles billions of entities with optimal performance - */ - -import { NounType, VerbType } from '../types/graphTypes.js' - -/** - * Signal types used for entity classification - */ -export type SignalType = 'embedding' | 'exact' | 'pattern' | 'context' - -/** - * Strategy types used for relationship extraction - */ -export type StrategyType = 'explicit' | 'pattern' | 'embedding' - -/** - * Import context for preset auto-detection - */ -export interface ImportContext { - fileType?: 'excel' | 'csv' | 'json' | 'pdf' | 'markdown' | 'unknown' - fileSize?: number // bytes - rowCount?: number - hasExplicitColumns?: boolean // Has "Related Terms" or similar columns - hasNarrativeContent?: boolean // Has long-form text/descriptions - avgDefinitionLength?: number // Average length of definitions - memoryAvailable?: number // bytes -} - -/** - * Signal configuration with weights - */ -export interface SignalConfig { - enabled: SignalType[] - weights: Record - timeout: number // milliseconds -} - -/** - * Strategy configuration with priorities - */ -export interface StrategyConfig { - enabled: StrategyType[] - timeout: number // milliseconds - earlyTermination: boolean - minConfidence: number -} - -/** - * Complete preset configuration - */ -export interface PresetConfig { - name: string - description: string - signals: SignalConfig - strategies: StrategyConfig - streaming: boolean - batchSize: number -} - -/** - * Fast Preset - For large imports (>10K rows) - * - * Optimized for speed over accuracy: - * - Only exact match and pattern signals - * - Only explicit strategy (O(1) lookups) - * - Streaming enabled for memory efficiency - * - Early termination on first high-confidence match - * - * Use case: Bulk imports, data migrations - * Performance: ~10ms per row - * Accuracy: ~85% - */ -export const FAST_PRESET: PresetConfig = { - name: 'fast', - description: 'Fast bulk import for large datasets', - signals: { - enabled: ['exact', 'pattern'], - weights: { - exact: 0.70, - pattern: 0.30, - embedding: 0, - context: 0 - }, - timeout: 50 - }, - strategies: { - enabled: ['explicit'], - timeout: 100, - earlyTermination: true, - minConfidence: 0.70 - }, - streaming: true, - batchSize: 1000 -} - -/** - * Balanced Preset - Default for most imports - * - * Good balance of speed and accuracy: - * - All signals except context (embedding, exact, pattern) - * - All strategies with smart ordering - * - Moderate timeouts - * - Early termination after high-confidence matches - * - * Use case: Standard imports, general glossaries - * Performance: ~30ms per row - * Accuracy: ~92% - */ -export const BALANCED_PRESET: PresetConfig = { - name: 'balanced', - description: 'Balanced speed and accuracy for most imports', - signals: { - enabled: ['exact', 'embedding', 'pattern'], - weights: { - exact: 0.40, - embedding: 0.35, - pattern: 0.25, - context: 0 - }, - timeout: 100 - }, - strategies: { - enabled: ['explicit', 'pattern', 'embedding'], - timeout: 200, - earlyTermination: true, - minConfidence: 0.65 - }, - streaming: false, - batchSize: 500 -} - -/** - * Accurate Preset - For small, critical imports - * - * Optimized for accuracy over speed: - * - All signals including context - * - All strategies, no early termination - * - Longer timeouts for thorough analysis - * - Lower confidence threshold (accept more matches) - * - * Use case: Knowledge bases, critical taxonomies - * Performance: ~100ms per row - * Accuracy: ~97% - */ -export const ACCURATE_PRESET: PresetConfig = { - name: 'accurate', - description: 'Maximum accuracy for critical imports', - signals: { - enabled: ['exact', 'embedding', 'pattern', 'context'], - weights: { - exact: 0.40, - embedding: 0.35, - pattern: 0.20, - context: 0.05 - }, - timeout: 500 - }, - strategies: { - enabled: ['explicit', 'pattern', 'embedding'], - timeout: 1000, - earlyTermination: false, - minConfidence: 0.50 - }, - streaming: false, - batchSize: 100 -} - -/** - * Explicit Preset - For glossaries with relationship columns - * - * Optimized for structured data with explicit relationships: - * - Only exact match signals (no AI needed) - * - Only explicit and pattern strategies - * - Fast, deterministic results - * - Perfect for Excel/CSV with "Related Terms" columns - * - * Use case: structured taxonomies and glossaries from spreadsheet sources - * Performance: ~5ms per row - * Accuracy: ~99% (high confidence) - */ -export const EXPLICIT_PRESET: PresetConfig = { - name: 'explicit', - description: 'For glossaries with explicit relationship columns', - signals: { - enabled: ['exact', 'pattern'], - weights: { - exact: 0.70, - pattern: 0.30, - embedding: 0, - context: 0 - }, - timeout: 50 - }, - strategies: { - enabled: ['explicit', 'pattern'], - timeout: 100, - earlyTermination: true, - minConfidence: 0.80 - }, - streaming: false, - batchSize: 500 -} - -/** - * Pattern Preset - For documents with narrative content - * - * Optimized for unstructured text with rich patterns: - * - Embedding and pattern signals (semantic understanding) - * - Pattern and embedding strategies - * - Good for PDFs, articles, documentation - * - * Use case: PDF imports, markdown docs, articles - * Performance: ~50ms per row - * Accuracy: ~90% - */ -export const PATTERN_PRESET: PresetConfig = { - name: 'pattern', - description: 'For documents with narrative content', - signals: { - enabled: ['embedding', 'pattern', 'context'], - weights: { - embedding: 0.50, - pattern: 0.40, - context: 0.10, - exact: 0 - }, - timeout: 200 - }, - strategies: { - enabled: ['pattern', 'embedding'], - timeout: 300, - earlyTermination: false, - minConfidence: 0.60 - }, - streaming: false, - batchSize: 200 -} - -/** - * All available presets - */ -export const PRESETS: Record = { - fast: FAST_PRESET, - balanced: BALANCED_PRESET, - accurate: ACCURATE_PRESET, - explicit: EXPLICIT_PRESET, - pattern: PATTERN_PRESET -} - -/** - * Auto-detect optimal preset based on import context - * - * Decision tree: - * 1. Large dataset (>10K rows or >10MB) → fast - * 2. Small dataset (<100 rows) → accurate - * 3. Excel/CSV with explicit columns → explicit - * 4. PDF/Markdown with long content → pattern - * 5. Default → balanced - * - * @param context Import context (file type, size, structure) - * @returns Optimal preset configuration - */ -export function autoDetectPreset(context: ImportContext = {}): PresetConfig { - const { - fileType = 'unknown', - fileSize = 0, - rowCount = 0, - hasExplicitColumns = false, - hasNarrativeContent = false, - avgDefinitionLength = 0 - } = context - - // Rule 1: Large imports → fast preset (prioritize speed) - if (rowCount > 10000 || fileSize > 10_000_000) { - return FAST_PRESET - } - - // Rule 2: Small critical imports → accurate preset (prioritize accuracy) - if (rowCount > 0 && rowCount < 100) { - return ACCURATE_PRESET - } - - // Rule 3: Structured data with explicit relationships → explicit preset - // (Handles spreadsheet imports where relationships are encoded in columns.) - if (hasExplicitColumns && (fileType === 'excel' || fileType === 'csv')) { - return EXPLICIT_PRESET - } - - // Rule 4: Narrative content → pattern preset - // Good for PDFs, articles, documentation - if ( - hasNarrativeContent || - fileType === 'pdf' || - fileType === 'markdown' || - avgDefinitionLength > 500 - ) { - return PATTERN_PRESET - } - - // Rule 5: JSON data → balanced preset - if (fileType === 'json') { - return BALANCED_PRESET - } - - // Default: balanced preset - return BALANCED_PRESET -} - -/** - * Get preset by name - * - * @param name Preset name (fast, balanced, accurate, explicit, pattern) - * @returns Preset configuration - * @throws Error if preset not found - */ -export function getPreset(name: string): PresetConfig { - const preset = PRESETS[name.toLowerCase()] - if (!preset) { - throw new Error(`Unknown preset: ${name}. Available: ${Object.keys(PRESETS).join(', ')}`) - } - return preset -} - -/** - * Get all available preset names - * - * @returns Array of preset names - */ -export function getPresetNames(): string[] { - return Object.keys(PRESETS) -} - -/** - * Explain why a preset was selected - * - * @param context Import context - * @returns Human-readable explanation - */ -export function explainPresetChoice(context: ImportContext = {}): string { - const { - fileType = 'unknown', - fileSize = 0, - rowCount = 0, - hasExplicitColumns = false, - hasNarrativeContent = false, - avgDefinitionLength = 0 - } = context - - if (rowCount > 10000 || fileSize > 10_000_000) { - return `Large dataset (${rowCount} rows, ${(fileSize / 1_000_000).toFixed(1)}MB) → fast preset for optimal performance` - } - - if (rowCount > 0 && rowCount < 100) { - return `Small critical dataset (${rowCount} rows) → accurate preset for maximum accuracy` - } - - if (hasExplicitColumns && (fileType === 'excel' || fileType === 'csv')) { - return `${fileType.toUpperCase()} with explicit relationship columns → explicit preset for deterministic results` - } - - if (hasNarrativeContent || fileType === 'pdf' || fileType === 'markdown') { - return `Narrative content (${fileType}) → pattern preset for semantic understanding` - } - - if (fileType === 'json') { - return `JSON data → balanced preset for structured imports` - } - - return `Standard import → balanced preset (default)` -} - -/** - * Create custom preset by merging with base preset - * - * @param baseName Base preset name - * @param overrides Custom overrides - * @returns Custom preset configuration - */ -export function createCustomPreset( - baseName: string, - overrides: Partial -): PresetConfig { - const base = getPreset(baseName) - - return { - ...base, - ...overrides, - signals: { - ...base.signals, - ...(overrides.signals || {}) - }, - strategies: { - ...base.strategies, - ...(overrides.strategies || {}) - } - } -} - -/** - * Validate preset configuration - * - * @param preset Preset to validate - * @returns True if valid, throws error otherwise - */ -export function validatePreset(preset: PresetConfig): boolean { - // Validate signals - if (preset.signals.enabled.length === 0) { - throw new Error('Preset must have at least one enabled signal') - } - - // Validate strategies - if (preset.strategies.enabled.length === 0) { - throw new Error('Preset must have at least one enabled strategy') - } - - // Validate weights sum to ~1.0 - const enabledSignals = preset.signals.enabled - const totalWeight = enabledSignals.reduce( - (sum, signal) => sum + preset.signals.weights[signal], - 0 - ) - - if (Math.abs(totalWeight - 1.0) > 0.01) { - throw new Error( - `Signal weights must sum to 1.0, got ${totalWeight.toFixed(2)}` - ) - } - - // Validate timeouts - if (preset.signals.timeout <= 0 || preset.strategies.timeout <= 0) { - throw new Error('Timeouts must be positive') - } - - // Validate batch size - if (preset.batchSize <= 0) { - throw new Error('Batch size must be positive') - } - - return true -} - -/** - * Format preset for display - * - * @param preset Preset configuration - * @returns Human-readable preset summary - */ -export function formatPreset(preset: PresetConfig): string { - const lines = [ - `Preset: ${preset.name}`, - `Description: ${preset.description}`, - '', - 'Signals:', - ...preset.signals.enabled.map( - (s) => ` - ${s}: ${(preset.signals.weights[s] * 100).toFixed(0)}%` - ), - ` Timeout: ${preset.signals.timeout}ms`, - '', - 'Strategies:', - ...preset.strategies.enabled.map((s) => ` - ${s}`), - ` Timeout: ${preset.strategies.timeout}ms`, - ` Early termination: ${preset.strategies.earlyTermination}`, - ` Min confidence: ${preset.strategies.minConfidence}`, - '', - `Streaming: ${preset.streaming}`, - `Batch size: ${preset.batchSize}` - ] - - return lines.join('\n') -} diff --git a/src/neural/relationshipConfidence.ts b/src/neural/relationshipConfidence.ts deleted file mode 100644 index df6a61c8..00000000 --- a/src/neural/relationshipConfidence.ts +++ /dev/null @@ -1,309 +0,0 @@ -/** - * Relationship Confidence Scoring - * - * Scores the confidence of detected relationships based on multiple factors: - * - Entity proximity in text - * - Entity confidence scores - * - Pattern matches - * - Structural analysis - */ - -import { ExtractedEntity } from './entityExtractor.js' -import { VerbType } from '../types/graphTypes.js' -import { RelationEvidence } from '../types/brainy.types.js' - -/** - * Detected relationship with confidence - */ -export interface DetectedRelationship { - sourceEntity: ExtractedEntity - targetEntity: ExtractedEntity - verbType: VerbType - confidence: number - evidence: RelationEvidence -} - -/** - * Configuration for relationship detection - */ -export interface RelationshipDetectionConfig { - minConfidence?: number // Minimum confidence to return (default: 0.5) - maxDistance?: number // Maximum token distance between entities (default: 50) - useProximityBoost?: boolean // Boost score based on proximity (default: true) - usePatternMatching?: boolean // Use verb pattern matching (default: true) - useStructuralAnalysis?: boolean // Analyze sentence structure (default: true) -} - -/** - * Relationship confidence scorer - */ -export class RelationshipConfidenceScorer { - private config: Required - - constructor(config: RelationshipDetectionConfig = {}) { - this.config = { - minConfidence: config.minConfidence || 0.5, - maxDistance: config.maxDistance || 50, - useProximityBoost: config.useProximityBoost !== false, - usePatternMatching: config.usePatternMatching !== false, - useStructuralAnalysis: config.useStructuralAnalysis !== false - } - } - - /** - * Score a potential relationship between two entities - */ - scoreRelationship( - source: ExtractedEntity, - target: ExtractedEntity, - verbType: VerbType, - context: string - ): { confidence: number, evidence: RelationEvidence } { - let confidence = 0.5 // Base confidence - - // Evidence tracking - const reasoningParts: string[] = [] - - // Factor 1: Proximity boost (closer entities = higher confidence) - if (this.config.useProximityBoost) { - const proximityBoost = this.calculateProximityBoost(source, target) - confidence += proximityBoost - if (proximityBoost > 0) { - reasoningParts.push( - `Entities are close together (boost: +${proximityBoost.toFixed(2)})` - ) - } - } - - // Factor 2: Entity confidence boost - const entityConfidence = (source.confidence + target.confidence) / 2 - const entityBoost = (entityConfidence - 0.5) * 0.2 // Scale to 0-0.2 - confidence *= (1 + entityBoost) - if (entityBoost > 0) { - reasoningParts.push( - `High entity confidence (boost: ${entityBoost.toFixed(2)})` - ) - } - - // Factor 3: Pattern match boost - if (this.config.usePatternMatching) { - const patternBoost = this.checkVerbPattern(source, target, verbType, context) - confidence += patternBoost - if (patternBoost > 0) { - reasoningParts.push( - `Matches relationship pattern (boost: +${patternBoost.toFixed(2)})` - ) - } - } - - // Factor 4: Structural boost (same sentence, clause, etc.) - if (this.config.useStructuralAnalysis) { - const structuralBoost = this.analyzeStructure(source, target, context) - confidence += structuralBoost - if (structuralBoost > 0) { - reasoningParts.push( - `Structural relationship (boost: +${structuralBoost.toFixed(2)})` - ) - } - } - - // Cap confidence at 1.0 - confidence = Math.min(confidence, 1.0) - - // Extract source text evidence - const start = Math.min(source.position.start, target.position.start) - const end = Math.max(source.position.end, target.position.end) - - const evidence: RelationEvidence = { - sourceText: context.substring(start, end), - position: { start, end }, - method: 'neural', - reasoning: reasoningParts.join('; ') - } - - return { confidence, evidence } - } - - /** - * Calculate proximity boost based on distance between entities - */ - private calculateProximityBoost( - source: ExtractedEntity, - target: ExtractedEntity - ): number { - const distance = Math.abs(source.position.start - target.position.start) - - if (distance === 0) return 0 // Same position, not meaningful - - // Very close (< 20 chars): +0.2 - if (distance < 20) return 0.2 - - // Close (< 50 chars): +0.1 - if (distance < 50) return 0.1 - - // Medium (< 100 chars): +0.05 - if (distance < 100) return 0.05 - - // Far (> 100 chars): no boost - return 0 - } - - /** - * Check if entities match a verb pattern - */ - private checkVerbPattern( - source: ExtractedEntity, - target: ExtractedEntity, - verbType: VerbType, - context: string - ): number { - const contextBetween = this.getContextBetween(source, target, context) - const contextLower = contextBetween.toLowerCase() - - // Verb-specific patterns - const patterns: Record = { - [VerbType.Creates]: ['creates', 'made', 'built', 'developed', 'produces'], - [VerbType.Owns]: ['owns', 'belongs to', 'possessed by', 'has'], - [VerbType.Contains]: ['contains', 'includes', 'has', 'holds'], - [VerbType.Requires]: ['requires', 'needs', 'depends on', 'relies on'], - [VerbType.Uses]: ['uses', 'utilizes', 'employs', 'applies'], - [VerbType.ReportsTo]: ['manages', 'oversees', 'supervises', 'controls'], - [VerbType.Causes]: ['influences', 'affects', 'impacts', 'shapes', 'causes'], - [VerbType.DependsOn]: ['depends on', 'relies on', 'based on'], - [VerbType.Modifies]: ['modifies', 'changes', 'alters', 'updates'], - [VerbType.References]: ['references', 'cites', 'mentions', 'refers to'] - } - - const verbPatterns = patterns[verbType] || [] - - for (const pattern of verbPatterns) { - if (contextLower.includes(pattern)) { - return 0.2 // Strong pattern match - } - } - - return 0 // No pattern match - } - - /** - * Analyze structural relationship - */ - private analyzeStructure( - source: ExtractedEntity, - target: ExtractedEntity, - context: string - ): number { - const contextBetween = this.getContextBetween(source, target, context) - - // Same sentence (no sentence-ending punctuation between them) - if (!contextBetween.match(/[.!?]/)) { - return 0.1 - } - - // Same paragraph (single newline between them) - if (!contextBetween.match(/\n\n/)) { - return 0.05 - } - - return 0 - } - - /** - * Get context text between two entities - */ - private getContextBetween( - source: ExtractedEntity, - target: ExtractedEntity, - context: string - ): string { - const start = Math.min(source.position.end, target.position.end) - const end = Math.max(source.position.start, target.position.start) - - if (start >= end) return '' - - return context.substring(start, end) - } - - /** - * Detect relationships between a list of entities - */ - detectRelationships( - entities: ExtractedEntity[], - context: string, - verbHints?: VerbType[] - ): DetectedRelationship[] { - const relationships: DetectedRelationship[] = [] - const verbs = verbHints || [ - VerbType.Creates, - VerbType.Uses, - VerbType.Contains, - VerbType.Requires, - VerbType.RelatedTo - ] - - // Check all entity pairs - for (let i = 0; i < entities.length; i++) { - for (let j = i + 1; j < entities.length; j++) { - const source = entities[i] - const target = entities[j] - - // Check distance - const distance = Math.abs(source.position.start - target.position.start) - if (distance > this.config.maxDistance) { - continue // Too far apart - } - - // Try each verb type - for (const verbType of verbs) { - const { confidence, evidence } = this.scoreRelationship( - source, - target, - verbType, - context - ) - - if (confidence >= this.config.minConfidence) { - relationships.push({ - sourceEntity: source, - targetEntity: target, - verbType, - confidence, - evidence - }) - } - } - } - } - - // Sort by confidence (highest first) - relationships.sort((a, b) => b.confidence - a.confidence) - - return relationships - } -} - -/** - * Convenience function to score a single relationship - */ -export function scoreRelationshipConfidence( - source: ExtractedEntity, - target: ExtractedEntity, - verbType: VerbType, - context: string, - config?: RelationshipDetectionConfig -): { confidence: number, evidence: RelationEvidence } { - const scorer = new RelationshipConfidenceScorer(config) - return scorer.scoreRelationship(source, target, verbType, context) -} - -/** - * Convenience function to detect all relationships in text - */ -export function detectRelationshipsWithConfidence( - entities: ExtractedEntity[], - context: string, - config?: RelationshipDetectionConfig -): DetectedRelationship[] { - const scorer = new RelationshipConfidenceScorer(config) - return scorer.detectRelationships(entities, context) -} diff --git a/src/neural/staticPatternMatcher.ts b/src/neural/staticPatternMatcher.ts deleted file mode 100644 index b64aaf2e..00000000 --- a/src/neural/staticPatternMatcher.ts +++ /dev/null @@ -1,188 +0,0 @@ -/** - * Static Pattern Matcher - NO runtime initialization, NO Brainy needed - * - * All patterns and embeddings are pre-computed at build time - * This is pure pattern matching with zero dependencies - */ - -import { EMBEDDED_PATTERNS, getPatternEmbeddings } from './embeddedPatterns.js' -import type { Vector } from '../coreTypes.js' -import type { TripleQuery } from '../triple/TripleIntelligence.js' - -// Pre-load patterns and embeddings at module load time (happens once) -const patterns = new Map(EMBEDDED_PATTERNS.map(p => [p.id, p])) -const patternEmbeddings = getPatternEmbeddings() - -/** - * Cosine similarity between two vectors. - * Accepts any number-indexed sequence so pre-computed Float32Array pattern - * embeddings can be compared against number[] query vectors without copying. - */ -function cosineSimilarity(a: ArrayLike, b: ArrayLike): number { - if (!a || !b || a.length !== b.length) return 0 - - let dotProduct = 0 - let normA = 0 - let normB = 0 - - for (let i = 0; i < a.length; i++) { - dotProduct += a[i] * b[i] - normA += a[i] * a[i] - normB += b[i] * b[i] - } - - const denominator = Math.sqrt(normA) * Math.sqrt(normB) - return denominator === 0 ? 0 : dotProduct / denominator -} - -/** - * Extract slots from matched pattern - */ -function extractSlots(query: string, pattern: string): Record | null { - try { - const regex = new RegExp(pattern, 'i') - const match = query.match(regex) - - if (!match) return null - - const slots: Record = {} - for (let i = 1; i < match.length; i++) { - if (match[i]) { - slots[`$${i}`] = match[i] - } - } - - return Object.keys(slots).length > 0 ? slots : null - } catch { - return null - } -} - -/** - * Apply template with extracted slots - */ -function applyTemplate(template: any, slots: Record): any { - if (!template || !slots) return template - - const result = JSON.parse(JSON.stringify(template)) - const applySlots = (obj: any): any => { - if (typeof obj === 'string') { - return obj.replace(/\$\{(\d+)\}/g, (_, num) => slots[`$${num}`] || '') - } - if (Array.isArray(obj)) { - return obj.map(applySlots) - } - if (typeof obj === 'object' && obj !== null) { - const newObj: any = {} - for (const [key, value] of Object.entries(obj)) { - newObj[key] = applySlots(value) - } - return newObj - } - return obj - } - - return applySlots(result) -} - -/** - * Match query against all patterns using embeddings - */ -export function findBestPatterns( - queryEmbedding: Vector, - k: number = 3 -): Array<{ pattern: typeof EMBEDDED_PATTERNS[0]; similarity: number }> { - - const matches: Array<{ pattern: typeof EMBEDDED_PATTERNS[0]; similarity: number }> = [] - - for (const pattern of EMBEDDED_PATTERNS) { - - const patternEmbedding = patternEmbeddings.get(pattern.id) - if (!patternEmbedding) continue - - // Pass Float32Array directly, no need for Array.from()! - const similarity = cosineSimilarity(queryEmbedding, patternEmbedding) - if (similarity > 0.5) { // Threshold for relevance - matches.push({ pattern, similarity }) - } - } - - // Sort by similarity and return top k - return matches - .sort((a, b) => b.similarity - a.similarity) - .slice(0, k) -} - -/** - * Match query against patterns using regex - */ -export function matchPatternByRegex(query: string): { - pattern: typeof EMBEDDED_PATTERNS[0] - slots: Record - query: TripleQuery -} | null { - // Try direct regex matching first (fastest) - for (const pattern of EMBEDDED_PATTERNS) { - const slots = extractSlots(query, pattern.pattern) - if (slots) { - const templatedQuery = applyTemplate(pattern.template, slots) - return { - pattern, - slots, - query: templatedQuery - } - } - } - - return null -} - -/** - * Convert natural language to structured query using STATIC patterns - * NO initialization needed, NO Brainy required - */ -export function patternMatchQuery( - query: string, - queryEmbedding?: Vector -): TripleQuery { - - // ALWAYS use vector similarity when we have embeddings (which we always do!) - if (queryEmbedding && queryEmbedding.length === 384) { - const bestPatterns = findBestPatterns(queryEmbedding, 5) // Get top 5 matches - - // Try to extract slots from best matching patterns - for (const { pattern, similarity } of bestPatterns) { - // Only try patterns with good similarity - if (similarity < 0.7) break - - const slots = extractSlots(query, pattern.pattern) - if (slots) { - // Found a good match with extractable slots! - const result = applyTemplate(pattern.template, slots) - console.log('[NLP] Applied template with slots:', JSON.stringify(result)) - return result - } - } - - // If no slots extracted but we have a good match, use the template as-is - if (bestPatterns.length > 0 && bestPatterns[0].similarity > 0.75) { - console.log('[NLP] Returning template as-is:', JSON.stringify(bestPatterns[0].pattern.template)) - return bestPatterns[0].pattern.template - } - } - - // Fallback: simple vector search (should rarely happen) - console.log('[NLP] Fallback - returning simple query') - return { - like: query, - limit: 10 - } -} - -// Export pattern statistics for monitoring -export const PATTERN_STATS = { - totalPatterns: EMBEDDED_PATTERNS.length, - categories: [...new Set(EMBEDDED_PATTERNS.map(p => p.category))], - domains: [...new Set(EMBEDDED_PATTERNS.filter(p => p.domain).map(p => p.domain!))], - hasEmbeddings: patternEmbeddings.size > 0 -} \ No newline at end of file diff --git a/src/plugin.ts b/src/plugin.ts index 100e5282..df7c7224 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -2,7 +2,7 @@ * Brainy Plugin System * * Simple plugin architecture for two use cases: - * 1. Native acceleration (@soulcraft/cortex) + * 1. Native acceleration (@soulcraft/cor) * 2. Custom storage adapters (e.g., Redis, DynamoDB, custom backends) * * Plugins are loaded from an explicit `plugins: [...]` config list or @@ -20,7 +20,7 @@ import type { MetadataIndexStats } from './utils/metadataIndex.js' import type { GraphIndexStats } from './graph/graphAdjacencyIndex.js' // Re-export the provider contracts that already live closer to their -// implementations so a plugin author (Cortex) can import the *entire* +// implementations so a plugin author (Cor) can import the *entire* // provider surface from one stable entrypoint: `@soulcraft/brainy/plugin`. export type { ColumnStoreProvider } from './indexes/columnStore/types.js' export type { @@ -75,7 +75,7 @@ export interface BrainyPluginContext { /** * Register a provider for a named subsystem. * - * Well-known provider keys (used by cortex): + * Well-known provider keys (used by cor): * - 'metadataIndex' — MetadataIndexManager replacement * - 'graphIndex' — GraphAdjacencyIndex replacement * - 'entityIdMapper' — EntityIdMapper replacement @@ -103,7 +103,7 @@ export interface BrainyPluginContext { // // These interfaces are the type-level half of the provider-parity guarantee. // They capture only what Brainy actually invokes, so a native accelerator -// (Cortex) that declares `implements MetadataIndexProvider` gets a compile +// (Cor) that declares `implements MetadataIndexProvider` gets a compile // error the moment a method Brainy depends on is dropped or its signature // drifts. Brainy's own baseline classes (`MetadataIndexManager`, // `GraphAdjacencyIndex`, `JsHnswVectorIndex`, `EntityIdMapper`, `UnifiedCache`) @@ -115,6 +115,62 @@ export interface BrainyPluginContext { // implementation then fails to compile until it provides the member. // =========================================================================== +/** + * @description How a failed provider invariant should be remediated: + * - `'none'` — informational; the invariant held or nothing to do. + * - `'repair'` — a targeted, cheap fix exists (e.g. re-derive a count/manifest field). + * - `'rebuild'` — the derived state must be rebuilt from canonical (`provider.rebuild()`). + */ +export type InvariantHeal = 'none' | 'repair' | 'rebuild' + +/** + * @description The result of ONE provider invariant check. A failure + * (`holds === false`) NAMES what diverged, with numbers, so it is diagnosable + * from the report alone — never a bare boolean. `name` is a stable kebab-case id + * for telemetry / remediation routing. + */ +export interface InvariantResult { + /** Stable kebab-case id, e.g. `'manifest-residency'` / `'posted-count-floor'`. */ + name: string + /** `true` when the invariant holds. */ + holds: boolean + /** Human-readable detail; on failure, names the divergence WITH numbers. */ + detail: string + /** The value the invariant expected (optional, for diagnosis). */ + expected?: unknown + /** The value actually observed (optional, for diagnosis). */ + actual?: unknown + /** How a failure should be remediated. Ignored when `holds === true`. */ + heal: InvariantHeal +} + +/** + * @description A provider's self-report of its own cross-layer invariants + * (the `validateInvariants()` hook). Contract: + * - It NEVER throws — a failure is DATA (`healthy: false` + a failing invariant), + * not an exception. + * - It is BOUNDED (<50ms): residency checks + O(1) counts only, NO canonical + * walks — safe to call on a live brain, repeatedly. + * - `serving` = can the provider answer queries right now (the `isReady()` truth); + * `healthy` = do ALL invariants hold. A provider can be `serving` while an + * invariant flags a latent divergence, or `healthy` but not-yet-`serving` on a + * cold open. + */ +export interface ProviderInvariantReport { + /** Which provider produced this report, e.g. `'vector'` / `'graph'` / `'metadata'` / `'column'`. */ + provider: string + /** `true` iff every invariant in {@link invariants} holds. */ + healthy: boolean + /** `true` iff the provider can serve queries now (the `isReady()` truth). */ + serving: boolean + /** Each checked invariant and its verdict. */ + invariants: InvariantResult[] + /** Epoch millis when the check ran. */ + checkedAt: number + /** How long the check took (must stay well under 50ms). */ + durationMs: number +} + /** * The `'metadataIndex'` provider — a drop-in for `MetadataIndexManager`. * Brainy calls this surface via `this.metadataIndex.*` (see `brainy.ts`) and @@ -125,13 +181,77 @@ export interface MetadataIndexProvider { flush(): Promise rebuild(): Promise + /** + * @description OPTIONAL honest durability signal (readiness contract, + * mirrors `isReady?()` on the graph and vector providers). `true` ⇔ the + * persisted field postings are loaded (or cheaply demand-loadable) and + * consistent with what the provider last persisted — a rebuild from the + * canonical records would be redundant. When exposed, the rebuild gate + * defers to this signal INSTEAD of the `getStats().totalEntries === 0` + * heuristic (a durable provider may legitimately report 0 resident entries + * on a cold open while its postings sit loadable on disk). Absent → the + * gate keeps the count heuristic. Never return `true` when the durable + * state failed to load. + */ + isReady?(): boolean + + /** + * @description OPTIONAL. The provider's self-report of its own + * cross-layer invariants (manifest ↔ segments ↔ counts residency/coherence). + * MUST NOT throw — a failure is DATA (`healthy: false` + a failing invariant). + * MUST be bounded (<50ms): residency + O(1) counts only, NO canonical walks, so + * brainy's {@link } `validateIndexConsistency()` can call it on a live brain. + * Absent → brainy skips this provider in the cross-layer check (feature-detected). + * `repairIndex()` maps any failing invariant with `heal: 'rebuild'` to this + * provider's `rebuild()`. + */ + validateInvariants?(): Promise + + /** + * @description OPTIONAL. A native provider returns true from the moment its + * `init()` detects a large epoch-drift until its background + * build-new→verify→swap has verified-and-swapped. While true, brainy SKIPS its + * own rebuild for this provider and lets the provider's non-blocking background + * migration own the index (the no-freeze path); the provider serves correct + * reads from canonical meanwhile. Mirrors {@link MetadataIndexProvider.init} + * (and `isReady?()` on the graph provider). + */ + isMigrating?(): boolean + addToIndex(id: string, entityOrMetadata: any, skipFlush?: boolean, deferWrites?: boolean): Promise removeFromIndex(id: string, metadata?: any): Promise getIds(field: string, value: any): Promise - getIdsForFilter(filter: any): Promise + /** + * Resolve a `where` filter to its matching ids. + * @param filter - The filter shape (eq / allOf / anyOf / ne / range / exists / …). + * @param opts - OPTIONAL page bound for the UNSORTED `find({ type, where, limit })` + * path. Brainy passes `{ limit: offset+limit (+hidden over-fetch), offset: 0 }` and + * ALWAYS re-windows the result itself (visibility filter + `slice`), so a provider + * that honors `opts` should early-stop and return the `[0, limit)` PREFIX (it must + * NOT pre-apply `offset`). The JS index ignores `opts` and returns all matches — + * so honoring it is a pure native-side optimization that removes the O(N) FFI + * marshal at billion scale. Pairs with {@link getIdSetForFilter}. + */ + getIdsForFilter(filter: any, opts?: { limit?: number; offset?: number }): Promise + /** + * @description OPTIONAL: resolve a `where` filter to its matching id universe as + * an {@link OpaqueIdSet} (a serialized roaring `Buffer`) WITHOUT materializing + * the ids in TypeScript — the producer half of the predicate-pushdown + * (`CTX-PUSHDOWN-ALLOWEDIDS`) and graph query→expand fusion. Brainy forwards the + * returned set opaquely to `VectorIndexProvider.search({ allowedIds })` and + * `GraphAccelerationProvider.traverse(seeds)`; it NEVER inspects it. A native + * (cor) metadata index returns its roaring filter result directly (zero + * crossing); the JS index does not implement this — Brainy falls back to + * {@link MetadataIndexProvider.getIdsForFilter} + a `ReadonlySet` when + * absent. Present ⟺ the native stack is the active provider, so the matching + * native vector/graph engines can decode the same envelope. + * @param filter - The same filter shape accepted by `getIdsForFilter`. + * @returns The matching id universe as an opaque set. + */ + getIdSetForFilter?(filter: any): Promise getIdsForTextQuery(query: string): Promise> - getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc'): Promise + getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise getFilterValues(field: string): Promise getFilterFields(): Promise getFieldValueForEntity(entityId: string, field: string): Promise @@ -155,6 +275,17 @@ export interface MetadataIndexProvider { getAllVFSEntityCounts(): Promise> detectAndRepairCorruption(): Promise + /** + * OPTIONAL cheap (O(1)) cold-open consistency probe — the counterpart of the + * graph cold-load guard. Returns `true` if a sampled `(field, value, int)` is + * clean, `false` if it detects the cross-bucket phantom signature (an int whose + * current value for `field` no longer equals `value`). Brainy calls it ONCE per + * brain on the first read and, on `false`, runs {@link detectAndRepairCorruption} + * to self-heal — so an already-poisoned index repairs itself on open without the + * cost of the full-scan {@link validateConsistency}. A provider that omits it is + * simply never auto-probed (no behavior change). + */ + probeConsistency?(): Promise validateConsistency(): Promise<{ healthy: boolean avgEntriesPerEntity: number @@ -203,9 +334,63 @@ export interface MetadataIndexProvider { * contract at this surface. */ export interface GraphIndexProvider { - /** `false` until the index has loaded; Brainy probes this before fast paths. */ + /** + * `false` until the provider has loaded its persisted state. A provider should + * self-load on first read; this flag lets it report readiness for diagnostics + * and tooling. (Brainy's graph reads route through the provider's own methods, + * which are expected to be self-loading — it is the {@link GraphAccelerationProvider} + * fast paths that gate on `isInitialized`.) + */ readonly isInitialized: boolean + /** + * @description Returns true ONLY when the source→target EDGES are loaded (NOT + * membership/manifest count) — the honest cold-load readiness signal. brainy + * gates the graph rebuild + the connected read-time guard on it: a `false` + * forces a rebuild from storage, and a still-`false` after that rebuild raises + * a loud {@link import('./errors/brainyError.js').GraphIndexNotReadyError} + * rather than serving an empty traversal as truth. cortex >= 2.7.8 (2.x) / 3.0 + * exposes it; ABSENT on older providers, where brainy falls back to a + * known-edge-sample probe. + * @returns `true` when the persisted adjacency is loaded and traversals are + * trustworthy; `false` when only the count/manifest loaded (cold-open). + */ + isReady?(): boolean + + /** + * @description OPTIONAL. The provider's self-report of its own + * cross-layer invariants (manifest ↔ segments ↔ counts residency/coherence). + * MUST NOT throw — a failure is DATA (`healthy: false` + a failing invariant). + * MUST be bounded (<50ms): residency + O(1) counts only, NO canonical walks, so + * brainy's {@link } `validateIndexConsistency()` can call it on a live brain. + * Absent → brainy skips this provider in the cross-layer check (feature-detected). + * `repairIndex()` maps any failing invariant with `heal: 'rebuild'` to this + * provider's `rebuild()`. + */ + validateInvariants?(): Promise + + /** + * @description OPTIONAL eager cold-load. Called once during brain init — AFTER + * the metadata provider's `init()` (so the id-mapper is hydrated; a native int + * adjacency resolves endpoints through it — the id-mapper-before-adjacency + * order) and BEFORE the rebuild gate — so a native provider loads its + * source→target adjacency from storage and reports `isReady() === true` at the + * gate, with no spurious rebuild (the §7.1 `rebuild()==0` acceptance). Mirrors + * {@link MetadataIndexProvider.init}. The JS graph index omits it and + * self-loads its adjacency on demand. + */ + init?(): Promise + + /** + * @description OPTIONAL. A native provider returns true from the moment its + * `init()` detects a large epoch-drift until its background + * build-new→verify→swap has verified-and-swapped. While true, brainy SKIPS its + * own rebuild for this provider and lets the provider's non-blocking background + * migration own the index (the no-freeze path); the provider serves correct + * reads from canonical meanwhile. Mirrors `isReady?()` / `init?()`. + */ + isMigrating?(): boolean + /** * @description Entity ints reachable from `id` (1 hop), deduped. * @param id - The entity's interned int (from the shared idMapper). @@ -287,6 +472,314 @@ export interface GraphIndexProvider { getAllRelationshipCounts(): Map } +// ============= Graph Acceleration (optional native engine) ============= + +/** + * Opaque serialized id-set — a cor-internal roaring payload (a Node `Buffer` at + * runtime when the native engine is present), passed straight from a `find()` + * universe into `traverse` / `VectorIndexProvider.search` with NO id + * materialization in TypeScript (the O(1)-crossing query→expand win). Brainy + * NEVER inspects it; the provider version-tags the envelope and throws on a + * format mismatch (it owns integrity + the id-space-width guarantee). Typed as + * `Uint8Array` (which a Node `Buffer` satisfies) so the universal build stays + * browser-safe. + */ +export type OpaqueIdSet = Uint8Array + +/** Direction of a graph traversal relative to each frontier node. */ +export type GraphTraversalDirection = 'in' | 'out' | 'both' + +/** + * Columnar subgraph — the shared wire format for every native graph read + * (`traverse`, `edgesForNode`, cursor chunks). Parallel typed arrays, never an + * array-of-objects, so the NAPI boundary transfers them near-zero-copy and + * Brainy maps u64↔UUID **lazily** (only for the rows a caller actually renders, + * via `EntityIdMapperProvider.entityIntsToUuids` + `GraphIndexProvider.verbIntsToIds`). + * The three `edge*` arrays are parallel (index `i` is one edge); `nodes` / + * `nodeDepth` are parallel. + */ +export interface Subgraph { + /** Discovered entity ints; resolve via `entityIntsToUuids`. */ + nodes: BigInt64Array + /** Hop distance of each node from the nearest seed (parallel to `nodes`). Present iff `includeDepth`. */ + nodeDepth?: Uint8Array + /** Edge source entity ints. */ + edgeSources: BigInt64Array + /** Edge target entity ints. */ + edgeTargets: BigInt64Array + /** Edge verb ints; resolve via `verbIntsToIds`. */ + edgeVerbInts: BigInt64Array + /** Edge verb-type indices (stable TypeIdx; resolve via `TypeUtils.getVerbFromIndex`). */ + edgeTypes: Uint16Array + /** + * `true` when a `maxNodes` / `maxEdges` cap truncated the result. The returned + * rows are the deterministic BFS-order prefix; page the remainder with a + * `graphCursor` (traverse stays stateless/bounded — no continuation token). + */ + truncated?: boolean + /** When `truncated`, how many nodes/edges were cut (for "+N more" affordances). */ + truncatedNodeCount?: number + truncatedEdgeCount?: number +} + +/** Knobs shared by the bounded traversals. */ +export interface TraverseOptions { + /** Max hop distance from any seed. */ + depth: number + /** Edge direction to follow (default `'both'`). */ + direction?: GraphTraversalDirection + /** Restrict traversed edges to these verb-type indices (TypeIdx). */ + verbTypes?: number[] + /** Restrict traversed edges to these subtypes. */ + subtypes?: string[] + /** + * Visibility tiers to EXCLUDE from the frontier (default: `['internal','system']`, + * matching `related()`). Pass `[]` for an all-tiers (admin) view. + */ + excludeVisibility?: string[] + /** Cap on returned nodes (deterministic BFS-order prefix; sets `Subgraph.truncated`). */ + maxNodes?: number + /** Cap on returned edges. */ + maxEdges?: number + /** Include the edge arrays (`false` = nodes-only reachability). Default `true`. */ + includeEdges?: boolean + /** Populate `Subgraph.nodeDepth`. Default `false`. */ + includeDepth?: boolean +} + +/** Options for the both-direction single-node edge read. */ +export interface EdgesForNodeOptions { + /** Edge direction (default `'both'`). */ + direction?: GraphTraversalDirection + verbTypes?: number[] + subtypes?: string[] + excludeVisibility?: string[] + limit?: number +} + +/** Opaque server-side cursor handle (TTL-bounded; pinned to a generation at open). */ +export type GraphCursorHandle = string + +/** Options for opening a streaming whole-graph (or seeded) cursor. */ +export interface GraphCursorOptions { + direction?: GraphTraversalDirection + excludeVisibility?: string[] + /** `'light'` = ids/types only, no metadata-join columns (viz default); `'full'` = with joins. */ + projection?: 'light' | 'full' + /** Seed set to bound the walk; omitted = the whole graph. */ + seeds?: bigint[] | OpaqueIdSet + /** + * Resume token from a prior chunk — used to continue after a `SnapshotExpired` + * (the pinned generation's TTL lapsed): reopen with this to resume the walk. + */ + cursor?: string +} + +/** One streamed chunk of a graph cursor walk. */ +export interface GraphCursorChunk { + /** The chunk's nodes + edges, columnar. */ + subgraph: Subgraph + /** Opaque resume token (pass to `graphCursorOpen({ cursor })` after `SnapshotExpired`). */ + cursor?: string + /** `true` once the walk is exhausted; `graphCursorNext` should not be called again. */ + done: boolean +} + +/** Per-node scores returned by `rank` / `mostConnected`; `scores[i]` belongs to `nodeInts[i]` (descending). */ +export interface GraphScores { + nodeInts: BigInt64Array + scores: Float64Array +} + +/** Community/cluster labelling; `communityIds[i]` is the group of `nodeInts[i]`. */ +export interface GraphCommunities { + nodeInts: BigInt64Array + communityIds: Uint32Array + communityCount: number +} + +/** A path between two nodes — the node sequence + the edges between them (or `null` if unreachable). */ +export interface GraphPath { + nodeInts: BigInt64Array + edgeVerbInts: BigInt64Array + /** Cost of the path: number of hops, or summed edge weight when `by: 'weight'`. */ + cost: number +} + +/** + * Options for `rank` (importance / influence ranking). INTENT-level only — the + * ranking ALGORITHM is the provider's choice (the JS fallback uses PageRank; a + * native provider may use personalized PageRank, eigenvector centrality, etc.), + * so no algorithm-tuning knobs are exposed here. + */ +export interface RankOptions { + /** Return only the top-K nodes by score (default: all). */ + topK?: number + /** Visibility tiers to EXCLUDE from the ranked set (default `['internal','system']`). */ + excludeVisibility?: string[] +} + +/** Options for `communities` (grouping related nodes). The grouping algorithm is the provider's choice. */ +export interface CommunitiesOptions { + /** Treat the graph as directed when grouping (default `false` — direction-agnostic). */ + directed?: boolean + excludeVisibility?: string[] +} + +/** Options for `path` (best route between two nodes). */ +export interface PathOptions { + direction?: GraphTraversalDirection + /** Optimize for fewest `'hops'` (default) or least summed edge `'weight'`. */ + by?: 'hops' | 'weight' + /** Restrict the route to these verb-type indices (TypeIdx). */ + verbTypes?: number[] + excludeVisibility?: string[] + /** Abandon the search past this many hops. */ + maxDepth?: number +} + +/** Options for `sample` (a representative neighborhood sample for dense-graph viz). */ +export interface SampleOptions { + /** Max hop distance from any seed. */ + depth: number + direction?: GraphTraversalDirection + /** Max neighbors kept per node (random, seeded for reproducibility). */ + fanout: number + /** RNG seed so a given (seeds, opts, seed) yields a stable sample. */ + seed?: number + excludeVisibility?: string[] + maxNodes?: number +} + +/** Options for `mostConnected` (the most-connected nodes). */ +export interface MostConnectedOptions { + /** Return the top-K most-connected nodes. */ + topK: number + direction?: GraphTraversalDirection + excludeVisibility?: string[] +} + +/** + * The `'graphAcceleration'` provider — an OPTIONAL native graph engine (the + * cor 3.0 acceleration layer). Brainy feature-detects it on the registered + * providers: when present, the public `brain.graph.*` surface and the + * `related({ node })` / `neighbors({ depth })` / `find({ connected })` paths + * route here; when absent, Brainy serves the same operations from its pure-TS + * adjacency fallback (correct, small-graph-scale). It is SEPARATE from the + * required {@link GraphIndexProvider} (which stays the bigint-adjacency + * contract) so "optional native acceleration" is explicit at the seam. + * + * **Generation-aware (8.0 time-travel).** Every read takes an optional trailing + * `generation?: bigint`; omitted = the current generation ("now"). With a + * generation, the provider resolves the graph **as of** that generation + * (`db.asOf(g).graph.*`), reading its generation-filtered adjacency rather than + * the now-only fast path. + * + * **Columnar + opaque-id-set.** Reads return the columnar {@link Subgraph}; + * `traverse` / sample accept seeds as `bigint[]` OR an {@link OpaqueIdSet} + * (a `find()` universe forwarded with no id materialization — the query→expand + * fusion). Visibility tiers are excluded at the index by default. + */ +export interface GraphAccelerationProvider { + /** `false` until the native engine has loaded; Brainy probes before routing. */ + readonly isInitialized: boolean + + /** + * @description Bounded multi-hop expansion from `seeds`, returning the reachable + * subgraph. One call replaces Brainy's per-hop BFS round-trips. + * @param seeds - Start nodes as entity ints, OR an {@link OpaqueIdSet} (a + * `find()` universe — the frontier is intersected in id-space, zero crossing). + * @param options - Depth, direction, edge/visibility filters, and caps. + * @param generation - Optional as-of generation (omitted = now). + * @returns The reachable {@link Subgraph} (BFS-order; `truncated` set if capped). + */ + traverse(seeds: bigint[] | OpaqueIdSet, options: TraverseOptions, generation?: bigint): Promise + + /** + * @description All edges incident to one node, both directions, as structure + + * cor-indexed fields (Brainy hydrates full edge metadata lazily — the provider + * cannot recover verb-id strings from its interned form). + * @param nodeInt - The node's interned entity int. + * @param options - Direction, edge/visibility filters, limit. + * @param generation - Optional as-of generation. + * @returns A {@link Subgraph} of the node's incident edges. + */ + edgesForNode(nodeInt: bigint, options: EdgesForNodeOptions, generation?: bigint): Promise + + /** + * @description Open a snapshot-consistent streaming walk of the whole graph (or a + * seeded region) for O(N) viz loads. Pins a generation at open; the walk reads + * as-of that pin (no dup/skip under concurrent writes). Handles are TTL-bounded — + * `graphCursorNext` after expiry rejects with `SnapshotExpired`; reopen with the + * last chunk's `cursor` to resume. Call `graphCursorClose` on normal completion. + * @param options - Direction, visibility, projection mode, optional seeds / resume cursor. + * @param generation - Optional explicit as-of generation to pin (else pins current). + * @returns A handle for `graphCursorNext` / `graphCursorClose`. + */ + graphCursorOpen(options: GraphCursorOptions, generation?: bigint): Promise + /** + * @description Pull the next chunk from an open cursor. + * @param handle - From `graphCursorOpen`. + * @param chunkSize - Target number of nodes/edges in this chunk. + * @returns The next {@link GraphCursorChunk}; `done: true` when exhausted. + */ + graphCursorNext(handle: GraphCursorHandle, chunkSize: number): Promise + /** + * @description Release an open cursor's pinned generation and server-side state. + * @param handle - From `graphCursorOpen`. + */ + graphCursorClose(handle: GraphCursorHandle): Promise + + /** + * @description Rank nodes by influence/importance over the VISIBLE graph (hidden + * tiers excluded by default, so the ranking reflects the public view). The + * ranking ALGORITHM is the provider's choice — the JS fallback uses PageRank; a + * native provider may use personalized PageRank, eigenvector centrality, etc. The + * intent ("which nodes matter most") is the contract, not the algorithm. + * @param options - Top-K + visibility (intent-level; no algorithm tuning). + * @param generation - Optional as-of generation. + * @returns Per-node scores, descending. + */ + rank(options: RankOptions, generation?: bigint): Promise + /** + * @description Group related nodes into communities over the VISIBLE graph. The + * grouping ALGORITHM is the provider's choice — the JS fallback uses connected + * components; a native provider may use community detection (e.g. Louvain/Leiden). + * @param options - Directed-grouping toggle + visibility. + * @param generation - Optional as-of generation. + * @returns Per-node community labels + community count. + */ + communities(options: CommunitiesOptions, generation?: bigint): Promise + /** + * @description Find the best path between two nodes — fewest hops by default, or + * least summed edge weight with `by: 'weight'`. The pathfinding ALGORITHM is the + * provider's choice (JS fallback: BFS / Dijkstra). + * @param fromInt - Start node int. + * @param toInt - End node int. + * @param options - Direction, cost basis (`by`), verb-type filter, max depth, visibility. + * @param generation - Optional as-of generation. + * @returns The path, or `null` if `toInt` is unreachable from `fromInt`. + */ + path(fromInt: bigint, toInt: bigint, options: PathOptions, generation?: bigint): Promise + /** + * @description A bounded, representative SAMPLE of the neighborhood around the seeds + * (capped fan-out per node) — for dense-graph viz. Seeded for reproducibility. + * @param seeds - Start nodes (ints or an {@link OpaqueIdSet}). + * @param options - Depth, fan-out, RNG seed, visibility, node cap. + * @param generation - Optional as-of generation. + * @returns The sampled {@link Subgraph}. + */ + sample(seeds: bigint[] | OpaqueIdSet, options: SampleOptions, generation?: bigint): Promise + /** + * @description The top-K most-connected nodes over the VISIBLE graph. The + * connectivity metric is the provider's choice (JS fallback: edge-count degree). + * @param options - Top-K, direction, visibility. + * @param generation - Optional as-of generation. + * @returns Per-node connectivity scores, descending. + */ + mostConnected(options: MostConnectedOptions, generation?: bigint): Promise +} + /** * Optional capability interface for index providers that maintain versioned * (generation-aware) internal state — the provider-side half of Brainy 8.0's @@ -389,10 +882,59 @@ export function isVersionedIndexProvider( ) } +/** + * @description Feature-detect an optional {@link GraphAccelerationProvider} (the + * native graph engine). Brainy probes the registered `'graphAcceleration'` + * provider with this: present ⇒ `brain.graph.*` routes to native `traverse` / + * cursor / analytics; absent ⇒ Brainy serves the same operations from its + * pure-TS adjacency fallback. Checks the two anchor methods (`traverse` + + * `graphCursorOpen`) — a partial implementation that lacks them is treated as + * absent (fall back rather than crash). + * @param candidate - The value registered under `'graphAcceleration'`, or undefined. + * @returns Whether `candidate` is a usable {@link GraphAccelerationProvider}. + */ +export function isGraphAccelerationProvider( + candidate: unknown +): candidate is GraphAccelerationProvider { + if (candidate === null || typeof candidate !== 'object') { + return false + } + const c = candidate as Record + return typeof c.traverse === 'function' && typeof c.graphCursorOpen === 'function' +} + +/** + * Columnar at-`generation` candidate vectors for the 8.0 #35 FILTERED exact-rerank. + * Brainy resolves the per-generation vector before-images for the at-gen filtered + * universe (from its canonical generation records) and ships them flat, so a native + * provider reranks against the historically-correct vectors with ZERO per-vector FFI + * crossing — without the provider duplicating the per-gen retention Brainy already does. + * + * - **Row-major:** row `i` is `vectors[i*dim .. (i+1)*dim]` and belongs to `ids[i]`. + * INVARIANT: `vectors.length === ids.length * dim` (the provider asserts + refuses on mismatch). + * - **`ids` IS the candidate set:** entity ints (interned via the shared mapper) for the + * at-gen metadata∩graph filtered universe. The provider reranks EXACTLY these + * `(id, vector)` pairs and returns top-k by distance; any `allowedIds` passed alongside + * is redundant on this path (the provider may AND it as belt-and-suspenders). + * - **`dim`** must equal the provider's configured index dimension (asserted; mismatch → refuse). + * + * Supplied only when there is a BOUNDED filtered universe; the unfiltered-deep at-gen + * case stays the provider's own retained-segment ANN ({@link VersionedIndexProvider.isGenerationVisible} + * reports false there, so Brainy falls back to its materialization overlay). + */ +export interface AtGenerationVectors { + /** Entity ints (shared-mapper interned), one per candidate; the candidate set. */ + ids: BigInt64Array + /** Flat row-major vectors: `vectors[i*dim .. (i+1)*dim]` is `ids[i]`'s at-gen vector. */ + vectors: Float32Array + /** Vector dimension; MUST equal the provider's configured index dim. */ + dim: number +} + /** * The object returned by the `'vector'` provider factory — Brainy's vector * index contract. Implementations include Brainy's own JS HNSW index and any - * native acceleration provider (e.g. cortex's Adaptive DiskANN). + * native acceleration provider (e.g. cor's Adaptive DiskANN). * * Brainy calls this surface via `this.index.*` plus the transactional add/remove * operations. `enableCOW`, `getItem`, and `setPersistMode` are intentionally @@ -411,13 +953,98 @@ export interface VectorIndexProvider { queryVector: Vector, k?: number, filter?: (id: string) => Promise, - options?: { rerank?: { multiplier: number }; candidateIds?: string[] } + options?: { + rerank?: { multiplier: number } + candidateIds?: string[] + /** + * Predicate-pushdown universe — restrict the result to this id-set, applied + * INSIDE the beam walk (walk traverses all nodes, collects only allowed), which + * recovers the filtered recall that post-filtering loses. A native provider with + * cor as the metadata index gets the `find()` universe as an {@link OpaqueIdSet} + * (zero id materialization); the pure-JS path gets a `ReadonlySet`. Absent + * = no restriction. Optional — a provider that doesn't pushdown ignores it and + * falls back to `filter`. + */ + allowedIds?: OpaqueIdSet | ReadonlySet + /** + * As-of generation for historical (time-travel) reads — the vector-side + * mirror of the {@link GraphAccelerationProvider} trailing `generation?`. + * Omitted = "now" (the live index). When set, a {@link VersionedIndexProvider} + * serves the kNN / exact-rerank over its retained at-`generation` segments + * instead of the current vectors, so `db.asOf(g)` semantic queries need no + * O(n@G) JS-HNSW rebuild. Brainy only passes it when the provider advertised + * `isGenerationVisible(generation)` at pin time; a provider that does not + * honor it MUST refuse/fall back (never score current vectors as if at-gen) — + * Brainy then serves the historical leg from its own materialization. Brainy's + * `generation` is the same u64 counter handed to the graph index on writes. + */ + generation?: bigint + /** + * 8.0 #35 part-3: the at-`generation` candidate vectors for the FILTERED + * exact-rerank — Brainy supplies the historically-correct vectors so the + * provider need not retain them. Present only alongside `generation` on the + * filtered at-gen path; the provider reranks `atGenerationVectors.ids` by + * distance and returns top-k. See {@link AtGenerationVectors}. The built-in + * JS index ignores it (it serves "now" only). + */ + atGenerationVectors?: AtGenerationVectors + } ): Promise> size(): number clear(): void rebuild(options?: any): Promise flush(): Promise getPersistMode(): 'immediate' | 'deferred' + + /** + * @description OPTIONAL eager cold-load (readiness contract, mirrors + * {@link GraphIndexProvider.init}). Called once during brain init — AFTER the + * metadata provider's `init()` (the id-mapper is hydrated first, so a + * provider whose vector slots resolve through interned ints reads a complete + * mapping) and BEFORE the rebuild gate — so a durable provider loads (or + * verifies it can demand-load) its persisted index and reports + * `isReady() === true` at the gate instead of eating a spurious + * rebuild-from-canonical on every open. The built-in JS index omits it: + * `rebuild()` IS its load path. + */ + init?(): Promise + + /** + * @description OPTIONAL honest durability signal (readiness contract, + * mirrors {@link GraphIndexProvider.isReady}). `true` ⇔ the persisted + * derived index is loaded (or cheaply demand-loadable) and consistent with + * what the provider last persisted — a rebuild from the canonical records + * would be redundant work. When exposed, the rebuild gate defers to this + * signal INSTEAD of the `size() === 0` heuristic (an mmap/disk-native index + * may legitimately report 0 resident entries while fully durable). Absent → + * the gate keeps the size heuristic. Never return `true` when the durable + * state failed to load — that converts a recoverable rebuild into silent + * empty results. + */ + isReady?(): boolean + + /** + * @description OPTIONAL. The provider's self-report of its own + * cross-layer invariants (manifest ↔ segments ↔ counts residency/coherence). + * MUST NOT throw — a failure is DATA (`healthy: false` + a failing invariant). + * MUST be bounded (<50ms): residency + O(1) counts only, NO canonical walks, so + * brainy's {@link } `validateIndexConsistency()` can call it on a live brain. + * Absent → brainy skips this provider in the cross-layer check (feature-detected). + * `repairIndex()` maps any failing invariant with `heal: 'rebuild'` to this + * provider's `rebuild()`. + */ + validateInvariants?(): Promise + + /** + * @description OPTIONAL. A native provider returns true from the moment its + * `init()` detects a large epoch-drift until its background + * build-new→verify→swap has verified-and-swapped. While true, brainy SKIPS its + * own rebuild for this provider and lets the provider's non-blocking background + * migration own the index (the no-freeze path); the provider serves correct + * reads from canonical meanwhile. Mirrors `isReady?()` / `init?()` on the + * other index providers. + */ + isMigrating?(): boolean } @@ -429,6 +1056,18 @@ export interface VectorIndexProvider { */ export interface EntityIdMapperProvider { init(): Promise + /** + * @description OPTIONAL: reload the uuid↔int mapping from the CURRENT storage, + * discarding in-memory state — called by `brain.restore()` AFTER the storage + * has been replaced from a snapshot and BEFORE the graph index rebuilds, so + * the graph resolves each verb endpoint through a mapper that reflects the + * snapshot's assignments (without it, native adjacency edges — keyed on these + * ints — resolve to stale/missing ints and are silently dropped). A native + * mapper reloads from its restored binary KV; the JS fallback re-reads its + * persisted metadata. Optional so a mapper that already reloads via `init()` + * stays compatible — `restore()` falls back to `init()` when this is absent. + */ + rebuild?(): Promise getOrAssign(uuid: string): number getUuid(intId: number): string | undefined getInt(uuid: string): number | undefined @@ -437,6 +1076,20 @@ export interface EntityIdMapperProvider { clear(): Promise getAllIntIds(): number[] intsIterableToUuids(ints: Iterable): string[] + /** + * @description Batch reverse-resolve u64 entity ints → UUID strings — the + * bigint counterpart of {@link EntityIdMapperProvider.intsIterableToUuids}, + * for the native graph engine ({@link GraphAccelerationProvider}, whose + * `Subgraph.nodes` is a `BigInt64Array`). Brainy resolves lazily — only the + * rows a caller renders — but viz/export render many at once, so one batch + * crossing beats N. Order-preserving (one entry per input). A native mapper + * resolves every int from its binary int↔uuid store; the JS fallback resolves + * assigned ints and yields `''` for a never-assigned int (which does not occur + * for graph-engine results, since those only reference assigned ints). + * @param nodeInts - Entity ints as returned in a {@link Subgraph}. + * @returns One UUID per input, in order. + */ + entityIntsToUuids(nodeInts: BigInt64Array): string[] readonly size: number } @@ -460,7 +1113,7 @@ export interface CacheProvider { /** * The `'graph:compression'` provider — pure-function encode/decode for HNSW - * connection lists as compact delta-varint byte sequences (cortex's + * connection lists as compact delta-varint byte sequences (cor's * `encodeConnections` / `decodeConnections`). * * Brainy's `JsHnswVectorIndex` consumes this via a `ConnectionsCodec` that translates diff --git a/src/storage/adapters/baseStorageAdapter.ts b/src/storage/adapters/baseStorageAdapter.ts index 2b54f076..a76d22df 100644 --- a/src/storage/adapters/baseStorageAdapter.ts +++ b/src/storage/adapters/baseStorageAdapter.ts @@ -53,13 +53,15 @@ export abstract class BaseStorageAdapter implements StorageAdapter { abstract getMetadata(id: string): Promise + abstract deleteMetadata(id: string): Promise + abstract getNounMetadata(id: string): Promise abstract getVerbMetadata(id: string): Promise // Vector Index Persistence (Brainy 8.0 — algorithm-neutral surface). // Concrete adapters persist the per-entity HNSW graph node (level + connections) - // under these methods. Cortex's DiskANN-style native vector index doesn't use + // under these methods. Cor's DiskANN-style native vector index doesn't use // them (it persists its own single mmap'd `.dkann` file under // `_system/vector-index/.dkann` per BRAINY-8.0-RENAME-COORDINATION § B.2). @@ -429,6 +431,12 @@ export abstract class BaseStorageAdapter implements StorageAdapter { this.statisticsBatchUpdateTimerId = setTimeout(() => { this.flushStatistics() }, delayMs) + // Best-effort statistics flush — must not keep the process alive + // (close() flushes counts deterministically). + const statsTimer = this.statisticsBatchUpdateTimerId as unknown as { unref?: () => void } + if (statsTimer && typeof statsTimer.unref === 'function') { + statsTimer.unref() + } } /** diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 3f8f5118..5eb4785a 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -5,9 +5,7 @@ import { HNSWNoun, - HNSWVerb, HNSWNounWithMetadata, - HNSWVerbWithMetadata, StatisticsData, NounType } from '../../coreTypes.js' @@ -19,10 +17,7 @@ import { WriterLockInfo } from '../baseStorage.js' import { getBrainyVersion } from '../../utils/index.js' - -// Type aliases for better readability -type HNSWNode = HNSWNoun -type Edge = HNSWVerb +import { isAbsentError } from '../../utils/errorClassification.js' // Node.js modules - dynamically imported to avoid issues in browser environments let fs: any @@ -75,8 +70,6 @@ export class FileSystemStorage extends BaseStorage { // - Handles 2.5M+ entities with < 10K files per shard // - Eliminates dynamic depth changes that cause path mismatch bugs private readonly SHARDING_DEPTH = 1 as const - private readonly MAX_SHARDS = 256 // Hex range: 00-ff - private cachedShardingDepth: number = this.SHARDING_DEPTH // Always use fixed depth protected rootDir: string private nounsDir!: string private verbsDir!: string @@ -103,6 +96,14 @@ export class FileSystemStorage extends BaseStorage { private static readonly WRITER_STALE_THRESHOLD_MS = 60_000 private writerLockHeartbeat?: NodeJS.Timeout private writerLockInfo?: WriterLockInfo + /** + * The currently-executing heartbeat refresh, if any. `releaseWriterLock()` + * awaits it before unlinking: clearInterval() stops FUTURE ticks but not a + * tick already in flight, and a straggler landing after the unlink would + * RE-CREATE the lock file — a phantom lock blocking the next writer until + * the stale TTL expires (the pool-eviction reopen case). + */ + private writerHeartbeatInFlight?: Promise // Flush-request RPC state. The writer polls `locks/_flush_requests/` for // new `.req` files and emits `.ack` files in `locks/_flush_responses/` after @@ -127,6 +128,18 @@ export class FileSystemStorage extends BaseStorage { private compressionEnabled: boolean = true // Enable gzip compression by default for 60-80% disk savings private compressionLevel: number = 6 // zlib compression level (1-9, default: 6 = balanced) + // Transaction durability barrier (see GenerationStorage.beginWriteBarrier). + // Non-null ONLY between beginWriteBarrier() and flushWriteBarrier(). Because + // canonical writes are tmp+rename (durable only in the page cache until an + // fsync), the generation store fsyncs everything a transaction wrote before + // it advances the generation counter. `writeBarrierPaths` collects the + // root-relative object paths written; `writeBarrierDeleteDirs` the parent + // dirs of deleted objects (an unlink is durable only once its directory is + // fsync'd). Both are null outside a transaction, so the tracking `add`s below + // are free on the single-op and non-transactional write paths. + private writeBarrierPaths: Set | null = null + private writeBarrierDeleteDirs: Set | null = null + /** * Initialize the storage adapter * @param rootDirectory The root directory for storage @@ -164,7 +177,7 @@ export class FileSystemStorage extends BaseStorage { * * @returns FileSystem-optimized batch configuration */ - public getBatchConfig(): StorageBatchConfig { + public override getBatchConfig(): StorageBatchConfig { return { maxBatchSize: 500, batchDelayMs: 0, @@ -180,7 +193,7 @@ export class FileSystemStorage extends BaseStorage { /** * Initialize the storage adapter */ - public async init(): Promise { + public override async init(): Promise { if (this.isInitialized) { return } @@ -219,6 +232,11 @@ export class FileSystemStorage extends BaseStorage { // Create the root directory if it doesn't exist await this.ensureDirectoryExists(this.rootDir) + // Finish any restore interrupted by a crash (resume the staged swap, or + // discard an uncommitted staging area) BEFORE counts/derived state load, + // so the rest of startup sees the completed store. + await this.completeInterruptedRestore() + // Create the nouns directory if it doesn't exist await this.ensureDirectoryExists(this.nounsDir) @@ -251,29 +269,20 @@ export class FileSystemStorage extends BaseStorage { this.countsFilePath = path.join(this.systemDir, 'counts.json') await this.initializeCounts() - // Detect existing sharding structure and migrate if needed - const detectedDepth = await this.detectExistingShardingDepth() - - if (detectedDepth !== null && detectedDepth !== this.SHARDING_DEPTH) { - // Migration needed: existing structure doesn't match our fixed depth - console.log(`📦 Brainy Storage Migration`) - console.log(` Current structure: depth ${detectedDepth}`) - console.log(` Target structure: depth ${this.SHARDING_DEPTH}`) - console.log(` Entities to migrate: ${this.totalNounCount}`) - - await this.migrateShardingStructure(detectedDepth, this.SHARDING_DEPTH) - - console.log(`✅ Migration complete - now using depth ${this.SHARDING_DEPTH} sharding`) - } else if (detectedDepth === null) { - // New installation - console.log(`📁 New installation: using depth ${this.SHARDING_DEPTH} sharding (optimal for 1-2.5M entities)`) - } else { - // Already using correct depth - console.log(`📁 Using depth ${this.SHARDING_DEPTH} sharding (${this.totalNounCount} entities)`) - } - - // Always use fixed depth after migration/detection - this.cachedShardingDepth = this.SHARDING_DEPTH + // Boot log: new-vs-established, decided from the canonical layout the + // database actually reads and writes (`entities/nouns///`) + // plus the known noun count. The legacy hnsw sharding-depth probe and + // its depth-migration machinery are gone: the 8.0 write path never + // populated the directory they inspected, so the probe concluded "new + // installation" for every store on every boot and the migration branch + // was unreachable. + const established = + this.totalNounCount > 0 || (await this.hasCanonicalEntities()) + console.log( + established + ? `📁 Using depth ${this.SHARDING_DEPTH} sharding (${this.totalNounCount} entities)` + : `📁 New installation: using depth ${this.SHARDING_DEPTH} sharding (optimal for 1-2.5M entities)` + ) // Initialize GraphAdjacencyIndex and type statistics await super.init() @@ -309,419 +318,15 @@ export class FileSystemStorage extends BaseStorage { } } - /** - * Save a node to storage - * CRITICAL FIX: Added atomic write pattern to prevent file corruption during concurrent imports - */ - protected async saveNode(node: HNSWNode): Promise { - await this.ensureInitialized() - // Convert connections Map to a serializable format - // CRITICAL: Only save lightweight vector data (no metadata) - // Metadata is saved separately via saveNounMetadata() (2-file system) - const serializableNode = { - id: node.id, - vector: node.vector, - connections: this.mapToObject(node.connections, (set) => - Array.from(set as Set) - ), - level: node.level || 0 - // NO metadata field - saved separately for scalability - } - const filePath = this.getNodePath(node.id) - const tempPath = `${filePath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}` - try { - // ATOMIC WRITE SEQUENCE: - // 1. Write to temp file - await this.ensureDirectoryExists(path.dirname(tempPath)) - await fs.promises.writeFile(tempPath, JSON.stringify(serializableNode, null, 2)) - // 2. Atomic rename temp → final (crash-safe, prevents truncation during concurrent writes) - await fs.promises.rename(tempPath, filePath) - } catch (error: any) { - // Clean up temp file on any error - try { - await fs.promises.unlink(tempPath) - } catch (cleanupError) { - // Ignore cleanup errors - } - throw error - } - // Count tracking happens in baseStorage.saveNounMetadata_internal - // This fixes the race condition where metadata didn't exist yet - } - /** - * Get a node from storage - */ - protected async getNode(id: string): Promise { - await this.ensureInitialized() - // Clean, predictable path - no backward compatibility needed - const filePath = this.getNodePath(id) - try { - const data = await fs.promises.readFile(filePath, 'utf-8') - const parsedNode = JSON.parse(data) - // Convert serialized connections back to Map> - const connections = new Map>() - for (const [level, nodeIds] of Object.entries(parsedNode.connections)) { - connections.set(Number(level), new Set(nodeIds as string[])) - } - // CRITICAL: Only return lightweight vector data (no metadata) - // Metadata is retrieved separately via getNounMetadata() (2-file system) - return { - id: parsedNode.id, - vector: parsedNode.vector, - connections, - level: parsedNode.level || 0 - // NO metadata field - retrieved separately for scalability - } - } catch (error: any) { - if (error.code !== 'ENOENT') { - console.error(`Error reading node ${id}:`, error) - } - return null - } - } - - /** - * Get all nodes from storage - * CRITICAL FIX: Now scans sharded subdirectories (depth=1) - * Previously only scanned flat directory, causing rebuild to find 0 entities - */ - protected async getAllNodes(): Promise { - await this.ensureInitialized() - - const allNodes: HNSWNode[] = [] - try { - // FIX: Use sharded file discovery instead of flat directory read - // This scans all 256 shard subdirectories (00-ff) to find actual files - const files = await this.getAllShardedFiles(this.nounsDir) - - for (const file of files) { - // Extract ID from filename and use sharded path - const id = file.replace('.json', '') - const filePath = this.getNodePath(id) - - const data = await fs.promises.readFile(filePath, 'utf-8') - const parsedNode = JSON.parse(data) - - // Convert serialized connections back to Map> - const connections = new Map>() - for (const [level, nodeIds] of Object.entries( - parsedNode.connections - )) { - connections.set(Number(level), new Set(nodeIds as string[])) - } - - allNodes.push({ - id: parsedNode.id, - vector: parsedNode.vector, - connections, - level: parsedNode.level || 0 - }) - } - } catch (error: any) { - if (error.code !== 'ENOENT') { - console.error(`Error reading directory ${this.nounsDir}:`, error) - } - } - return allNodes - } - - /** - * Get nodes by noun type - * CRITICAL FIX: Now scans sharded subdirectories (depth=1) - * @param nounType The noun type to filter by - * @returns Promise that resolves to an array of nodes of the specified noun type - */ - protected async getNodesByNounType(nounType: string): Promise { - await this.ensureInitialized() - - const nouns: HNSWNode[] = [] - try { - // FIX: Use sharded file discovery instead of flat directory read - const files = await this.getAllShardedFiles(this.nounsDir) - - for (const file of files) { - // Extract ID from filename and use sharded path - const nodeId = file.replace('.json', '') - const filePath = this.getNodePath(nodeId) - - const data = await fs.promises.readFile(filePath, 'utf-8') - const parsedNode = JSON.parse(data) - - // Filter by noun type using metadata - const metadata = await this.getMetadata(nodeId) - if (metadata && metadata.noun === nounType) { - // Convert serialized connections back to Map> - const connections = new Map>() - for (const [level, nodeIds] of Object.entries( - parsedNode.connections - )) { - connections.set(Number(level), new Set(nodeIds as string[])) - } - - nouns.push({ - id: parsedNode.id, - vector: parsedNode.vector, - connections, - level: parsedNode.level || 0 - }) - } - } - } catch (error: any) { - if (error.code !== 'ENOENT') { - console.error(`Error reading directory ${this.nounsDir}:`, error) - } - } - - return nouns - } - - /** - * Delete a node from storage - */ - protected async deleteNode(id: string): Promise { - await this.ensureInitialized() - - const filePath = this.getNodePath(id) - - // Load metadata to get type for count update (separate storage) - try { - const metadata = await this.getNounMetadata(id) - if (metadata) { - const type = metadata.noun || 'default' - this.decrementEntityCount(type) - } - } catch { - // Metadata might not exist, that's ok - } - - try { - await fs.promises.unlink(filePath) - - // Persist counts periodically - if (this.totalNounCount % 10 === 0) { - await this.persistCounts() - } - } catch (error: any) { - if (error.code !== 'ENOENT') { - console.error(`Error deleting node file ${filePath}:`, error) - throw error - } - } - } - - /** - * Save an edge to storage - * CRITICAL FIX: Added atomic write pattern to prevent file corruption during concurrent imports - */ - protected async saveEdge(edge: Edge): Promise { - await this.ensureInitialized() - - // Convert connections Map to a serializable format - // ARCHITECTURAL FIX: Include core relational fields in verb vector file - // These fields are essential for 90% of operations - no metadata lookup needed - const serializableEdge = { - id: edge.id, - vector: edge.vector, - connections: this.mapToObject(edge.connections, (set) => - Array.from(set as Set) - ), - - // CORE RELATIONAL DATA - verb: edge.verb, - sourceId: edge.sourceId, - targetId: edge.targetId, - - // User metadata (if any) - saved separately for scalability - // metadata field is saved separately via saveVerbMetadata() - } - - const filePath = this.getVerbPath(edge.id) - const tempPath = `${filePath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}` - - try { - // ATOMIC WRITE SEQUENCE: - // 1. Write to temp file - await this.ensureDirectoryExists(path.dirname(tempPath)) - await fs.promises.writeFile(tempPath, JSON.stringify(serializableEdge, null, 2)) - - // 2. Atomic rename temp → final (crash-safe, prevents truncation during concurrent writes) - await fs.promises.rename(tempPath, filePath) - } catch (error: any) { - // Clean up temp file on any error - try { - await fs.promises.unlink(tempPath) - } catch (cleanupError) { - // Ignore cleanup errors - } - throw error - } - - // Count tracking happens in baseStorage.saveVerbMetadata_internal - // This fixes the race condition where metadata didn't exist yet - } - - /** - * Get an edge from storage - */ - protected async getEdge(id: string): Promise { - await this.ensureInitialized() - - const filePath = this.getVerbPath(id) - try { - const data = await fs.promises.readFile(filePath, 'utf-8') - const parsedEdge = JSON.parse(data) - - // Convert serialized connections back to Map> - const connections = new Map>() - for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) { - connections.set(Number(level), new Set(nodeIds as string[])) - } - - // Return HNSWVerb with core relational fields (NO metadata field) - return { - id: parsedEdge.id, - vector: parsedEdge.vector, - connections, - - // CORE RELATIONAL DATA (read from vector file) - verb: parsedEdge.verb, - sourceId: parsedEdge.sourceId, - targetId: parsedEdge.targetId - - // ✅ NO metadata field - // User metadata retrieved separately via getVerbMetadata() - } - } catch (error: any) { - if (error.code !== 'ENOENT') { - console.error(`Error reading edge ${id}:`, error) - } - return null - } - } - - /** - * Get all edges from storage - * CRITICAL FIX: Now scans sharded subdirectories (depth=1) - * Previously only scanned flat directory, causing rebuild to find 0 relationships - */ - protected async getAllEdges(): Promise { - await this.ensureInitialized() - - const allEdges: Edge[] = [] - try { - // FIX: Use sharded file discovery instead of flat directory read - // This scans all 256 shard subdirectories (00-ff) to find actual files - const files = await this.getAllShardedFiles(this.verbsDir) - - for (const file of files) { - // Extract ID from filename and use sharded path - const id = file.replace('.json', '') - const filePath = this.getVerbPath(id) - - const data = await fs.promises.readFile(filePath, 'utf-8') - const parsedEdge = JSON.parse(data) - - // Convert serialized connections back to Map> - const connections = new Map>() - for (const [level, nodeIds] of Object.entries( - parsedEdge.connections - )) { - connections.set(Number(level), new Set(nodeIds as string[])) - } - - // Include core relational fields (NO metadata field) - allEdges.push({ - id: parsedEdge.id, - vector: parsedEdge.vector, - connections, - - // CORE RELATIONAL DATA - verb: parsedEdge.verb, - sourceId: parsedEdge.sourceId, - targetId: parsedEdge.targetId - - // ✅ NO metadata field - // User metadata retrieved separately via getVerbMetadata() - }) - } - } catch (error: any) { - if (error.code !== 'ENOENT') { - console.error(`Error reading directory ${this.verbsDir}:`, error) - } - } - return allEdges - } - - /** - * Get edges by source - */ - protected async getEdgesBySource(sourceId: string): Promise { - // This method is deprecated and would require loading metadata for each edge - // For now, return empty array since this is not efficiently implementable with new storage pattern - console.warn('getEdgesBySource is deprecated and not efficiently supported in new storage pattern') - return [] - } - - /** - * Get edges by target - */ - protected async getEdgesByTarget(targetId: string): Promise { - // This method is deprecated and would require loading metadata for each edge - // For now, return empty array since this is not efficiently implementable with new storage pattern - console.warn('getEdgesByTarget is deprecated and not efficiently supported in new storage pattern') - return [] - } - - /** - * Get edges by type - */ - protected async getEdgesByType(type: string): Promise { - // This method is deprecated and would require loading metadata for each edge - // For now, return empty array since this is not efficiently implementable with new storage pattern - console.warn('getEdgesByType is deprecated and not efficiently supported in new storage pattern') - return [] - } - - /** - * Delete an edge from storage - */ - protected async deleteEdge(id: string): Promise { - await this.ensureInitialized() - - // Delete the HNSWVerb file using sharded path - const filePath = this.getVerbPath(id) - try { - await fs.promises.unlink(filePath) - } catch (error: any) { - if (error.code !== 'ENOENT') { - console.error(`Error deleting edge file ${filePath}:`, error) - throw error - } - } - - // CRITICAL: Also delete verb metadata - this is what getVerbs() uses to find verbs - // Without this, getVerbsBySource() will still find "deleted" verbs via their metadata - try { - const metadata = await this.getVerbMetadata(id) - if (metadata) { - const verbType = (metadata.verb || metadata.type || 'default') as string - this.decrementVerbCount(verbType) - await this.deleteVerbMetadata(id) - } - } catch (error) { - // Ignore metadata deletion errors - verb file is already deleted - console.warn(`Failed to delete verb metadata for ${id}:`, error) - } - } /** * Primitive operation: Write object to path @@ -794,6 +399,12 @@ export class FileSystemStorage extends BaseStorage { throw error } } + + // Transaction durability barrier: record the write so the generation store + // can fsync it before advancing the counter. Reached only on a successful + // rename; the logical (non-`.gz`) path is stored — flushWriteBarrier's + // syncRawObjects resolves the compressed variant. + this.writeBarrierPaths?.add(pathStr) } /** @@ -844,8 +455,12 @@ export class FileSystemStorage extends BaseStorage { return null } - console.error(`Error reading object from ${pathStr}:`, error) - return null + // A real storage fault (EIO/EACCES/EMFILE/…) is NOT "object absent". The + // ENOENT branch (above) already returns null, and the corrupted-JSON + // branch (above) is a deliberate concurrent-write tolerance; a genuine + // fault reaching here must propagate loudly rather than masquerade as a + // missing object — which would corrupt reads and drive needless rebuilds. + throw error } } @@ -888,6 +503,109 @@ export class FileSystemStorage extends BaseStorage { if (deletedCount === 0) { // File doesn't exist - this is fine } + + // Transaction durability barrier: an unlink is durable only once its parent + // directory is fsync'd. Record the dir (root-relative; '.' for a top-level + // object) so flushWriteBarrier can sync it before the counter advances. + this.writeBarrierDeleteDirs?.add(path.dirname(pathStr)) + } + + /** + * @description Remove the entity container directory after both canonical legs + * were deleted (full-removal delete). `objectLegPath` is one leg's + * storage-root-relative path (e.g. `entities/nouns///vectors.json`); + * its parent is the `/` entity directory. `rmdir` removes it ONLY if empty + * — both legs are gone by this point, so it should be. A non-empty dir means an + * unexpected leftover (a leg whose delete faulted — already surfaced upstream — + * or foreign data): we do NOT recursively nuke it (loud errors, never quiet + * losses); we log and leave it for the orphan repair sweep (`repairIndex`). + */ + protected override async removeCanonicalContainer(objectLegPath: string): Promise { + const relDir = path.dirname(objectLegPath) + const absDir = path.join(this.rootDir, relDir) + try { + await fs.promises.rmdir(absDir) + // The dir removal is durable once its PARENT (the shard dir) is fsync'd. + this.writeBarrierDeleteDirs?.add(path.dirname(relDir)) + } catch (error: any) { + if (error?.code === 'ENOENT') return // container already gone — fine + if (error?.code === 'ENOTEMPTY' || error?.code === 'EEXIST') { + console.warn( + `[FileSystemStorage] entity container ${relDir} not empty after delete — ` + + `leaving it for the orphan repair sweep (brain.repairIndex()).` + ) + return + } + throw error + } + } + + /** + * @description Prune orphaned entity directories left by the pre-8.3.1 + * partial-delete defect. A delete used to remove the metadata (content) leg + * but leave the `vectors.json` leg + the `/` directory (a "ghost"), or — + * on the direct storage path — remove both legs but leave an empty directory + * (a "scar"). Neither is a live entity (`getNoun` needs the metadata content + * leg) yet each lingers on disk, inflating enumerated counts and confusing + * locator resolution. + * + * CONSERVATIVE by design: removes ONLY dirs with NO metadata content leg — a + * directory that still holds its content is left untouched. LOUD: logs every + * removed orphan. Operator-invoked via {@link Brainy.repairIndex}; never runs + * automatically. Returns the pruned container ids so the caller can recompute + * counts. + */ + public async pruneOrphanedEntities(): Promise<{ nouns: string[]; verbs: string[] }> { + await this.ensureInitialized() + const pruned: { nouns: string[]; verbs: string[] } = { nouns: [], verbs: [] } + + for (const [kind, root] of [ + ['nouns', 'entities/nouns'], + ['verbs', 'entities/verbs'] + ] as const) { + const rootAbs = path.join(this.rootDir, root) + let shards: string[] + try { + shards = await fs.promises.readdir(rootAbs) + } catch (error: any) { + if (error?.code === 'ENOENT') continue // no entities of this kind yet + throw error + } + + for (const shard of shards) { + const shardAbs = path.join(rootAbs, shard) + let entries: import('fs').Dirent[] + try { + entries = await fs.promises.readdir(shardAbs, { withFileTypes: true }) + } catch (error: any) { + if (error?.code === 'ENOENT') continue + throw error + } + + for (const entry of entries) { + if (!entry.isDirectory()) continue + const idAbs = path.join(shardAbs, entry.name) + let legs: string[] + try { + legs = await fs.promises.readdir(idAbs) + } catch (error: any) { + if (error?.code === 'ENOENT') continue + throw error + } + // A live entity has its metadata content leg. No content leg → a + // vector-only ghost or an empty scar → prune the whole container. + if (legs.some((f) => f.startsWith('metadata.json'))) continue + await fs.promises.rm(idAbs, { recursive: true, force: true }) + pruned[kind].push(entry.name) + console.warn( + `[FileSystemStorage] pruned orphaned ${kind === 'nouns' ? 'noun' : 'verb'} ` + + `container ${root}/${shard}/${entry.name} (no metadata content leg)` + ) + } + } + } + + return pruned } /** @@ -970,10 +688,56 @@ export class FileSystemStorage extends BaseStorage { ]) /** - * Top-level directories excluded from snapshots: process-local lock state - * (writer lock, flush-request RPC files) must never travel with the data. + * Top-level directories whose EVERY file is mutated in place (not tmp+rename) + * and must therefore be byte-copied into a snapshot, not hard-linked — the + * directory analogue of {@link SNAPSHOT_BYTE_COPY_PATHS}. + * + * `_id_mapper` holds the native provider's shared mmap `BinaryIdMapper`, which + * `flush()` msyncs and a migration rebuild can truncate+re-inject IN PLACE + * (confirmed by the native side). A hard link shares the inode, so an in-place + * msync/truncate on the live file would reach through into the pre-upgrade + * backup — byte-copy keeps the snapshot a faithful, independent copy. It is + * bounded (the id map), not the large index files, so the copy cost is small. */ - private static readonly SNAPSHOT_EXCLUDED_TOP_DIRS = new Set(['locks']) + private static readonly SNAPSHOT_BYTE_COPY_DIRS = new Set(['_id_mapper']) + + /** + * Nested path PREFIXES whose files are byte-copied into snapshots, not + * hard-linked — for append-in-place files below the top level. The + * generation fact log's tail segment is appended in place between rotations; + * a hard-linked tail would let post-snapshot appends reach through into the + * snapshot. (Sealed segments are immutable and would be link-safe, but the + * prefix rule keeps the discipline simple; segments are bounded by the + * rotation threshold, so the copy cost is small.) + */ + private static readonly SNAPSHOT_BYTE_COPY_PREFIXES: string[] = ['_generations/facts/'] + + /** + * Top-level directories excluded from snapshots: process-local lock state + * (writer lock, flush-request RPC files) must never travel with the data, and + * the restore staging area ({@link RESTORE_STAGING_DIR}) is transient scratch + * that must never be captured or restored. + */ + private static readonly SNAPSHOT_EXCLUDED_TOP_DIRS = new Set(['locks', '_restore_staging']) + + /** + * Transient top-level directory holding a restore-in-progress: the snapshot is + * fully copied here (sparse-aware) BEFORE any live data is touched, then an + * atomic per-entry swap moves it into place. Its presence + the completion + * marker let {@link completeInterruptedRestore} resume a crashed restore. + */ + private static readonly RESTORE_STAGING_DIR = '_restore_staging' + /** + * Written+fsync'd inside the staging dir ONLY after the whole snapshot has + * copied successfully. Its presence authorizes the swap (and its resume): a + * staging dir WITHOUT this marker is an interrupted copy — discardable debris, + * live data still authoritative. + */ + private static readonly RESTORE_MARKER = '.restore-manifest.json' + /** Chunk size for sparse-aware copying (holes are preserved at this grain). */ + private static readonly SPARSE_CHUNK_BYTES = 4 * 1024 * 1024 + /** A zero buffer the size of one sparse chunk, for all-zero (hole) detection. */ + private static readonly SPARSE_ZERO_CHUNK = Buffer.alloc(4 * 1024 * 1024) /** * Remove every object under a storage-root-relative prefix — one recursive @@ -981,8 +745,11 @@ export class FileSystemStorage extends BaseStorage { * * @param prefix - Storage-root-relative directory prefix to remove. */ - public async removeRawPrefix(prefix: string): Promise { + public override async removeRawPrefix(prefix: string): Promise { await this.ensureInitialized() + // Registered-blob contract: a prefix-nuke must not take out a protected + // family member. Throws if the prefix intersects one. + await this.assertPrefixNotProtected(prefix) await fs.promises.rm(path.join(this.rootDir, prefix), { recursive: true, force: true }) } @@ -995,7 +762,7 @@ export class FileSystemStorage extends BaseStorage { * * @param paths - Storage-root-relative object paths previously written. */ - public async syncRawObjects(paths: string[]): Promise { + public override async syncRawObjects(paths: string[]): Promise { await this.ensureInitialized() const parentDirs = new Set() @@ -1037,6 +804,61 @@ export class FileSystemStorage extends BaseStorage { } } + /** + * Begin a transaction durability barrier: start recording every canonical + * object write and delete so {@link flushWriteBarrier} can fsync them before + * the generation counter advances. Resets unconditionally, discarding any + * tracking left by a transaction that aborted without flushing. + * + * @see GenerationStorage.beginWriteBarrier + */ + public beginWriteBarrier(): void { + this.writeBarrierPaths = new Set() + this.writeBarrierDeleteDirs = new Set() + } + + /** + * Flush the transaction durability barrier: fsync every canonical write since + * {@link beginWriteBarrier} (file contents AND the rename directory entries, + * via {@link syncRawObjects}), then fsync the parent directory of every + * canonical delete so the unlinks are durable too. Clears the tracking. After + * this resolves, the transaction's entire canonical footprint is on disk, so + * the generation counter/manifest can be advanced without risking a + * counter-ahead-of-state torn store on a hard kill. + * + * @see GenerationStorage.flushWriteBarrier + */ + public async flushWriteBarrier(): Promise { + const paths = this.writeBarrierPaths + const deleteDirs = this.writeBarrierDeleteDirs + this.writeBarrierPaths = null + this.writeBarrierDeleteDirs = null + + if (paths && paths.size > 0) { + // syncRawObjects fsyncs each file and its parent directory. + await this.syncRawObjects([...paths]) + } + + if (deleteDirs && deleteDirs.size > 0) { + for (const relDir of deleteDirs) { + const dirPath = path.join(this.rootDir, relDir) + let handle: any + try { + handle = await fs.promises.open(dirPath, 'r') + } catch { + continue // directory vanished or platform disallows opening dirs + } + try { + await handle.sync() + } catch { + // Some platforms/filesystems reject directory fsync — best effort. + } finally { + await handle.close() + } + } + } + } + /** * Append one line to `_system/tx-log.jsonl`. Plain `appendFile` — the * tx-log is the one append-in-place file in the store (and is byte-copied, @@ -1067,6 +889,75 @@ export class FileSystemStorage extends BaseStorage { } } + // ========================================================================== + // Binary raw-byte primitives — the substrate for append-only log-structured + // files (the generation fact log's CRC-framed segments). Paths are used + // VERBATIM (no .gz/.bin suffixing). Append durability rides syncRawObjects + // at the commit barrier, like every other staged write. + // ========================================================================== + + /** + * Append bytes to a raw binary file, creating it (and parent directories) + * when absent. NOT fsync'd here — the caller batches durability via + * `syncRawObjects` at its commit barrier. + */ + public async appendRawBytes(rawPath: string, bytes: Uint8Array): Promise { + await this.ensureInitialized() + const fullPath = path.join(this.rootDir, rawPath) + await fs.promises.mkdir(path.dirname(fullPath), { recursive: true }) + await fs.promises.appendFile(fullPath, bytes) + } + + /** + * Read a raw binary file whole. Absent → `null`; a real IO fault throws — + * a present-but-unreadable log segment must never read as "no facts". + */ + public async readRawBytes(rawPath: string): Promise { + await this.ensureInitialized() + try { + const buf: Buffer = await fs.promises.readFile(path.join(this.rootDir, rawPath)) + return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength) + } catch (error: any) { + if (isAbsentError(error)) return null + throw error + } + } + + /** + * Replace a raw binary file atomically: write-new → fsync → rename. The + * reconcile primitive (e.g. truncating a fact-log tail back to committed + * truth after a crash) — a crash mid-replace leaves either the old file or + * the new one, never a mix. + */ + public async writeRawBytes(rawPath: string, bytes: Uint8Array): Promise { + await this.ensureInitialized() + const fullPath = path.join(this.rootDir, rawPath) + await fs.promises.mkdir(path.dirname(fullPath), { recursive: true }) + const tmpPath = `${fullPath}.tmp.${Date.now()}.${Math.random().toString(36).slice(2)}` + const handle = await fs.promises.open(tmpPath, 'w') + try { + await handle.writeFile(bytes) + await handle.sync() + } finally { + await handle.close() + } + await fs.promises.rename(tmpPath, fullPath) + } + + /** + * Byte size of a raw binary file, or `null` when absent. + */ + public async rawByteSize(rawPath: string): Promise { + await this.ensureInitialized() + try { + const stat = await fs.promises.stat(path.join(this.rootDir, rawPath)) + return stat.size + } catch (error: any) { + if (isAbsentError(error)) return null + throw error + } + } + /** * Snapshot the entire store into `targetPath` as a hard-link farm * (Cassandra-style: instant, space-shared). Safe because every data file @@ -1110,7 +1001,15 @@ export class FileSystemStorage extends BaseStorage { // relative path with separators unified, so `_system/tx-log.jsonl` // matches on every platform. const normalized = relPath.split(path.sep).join('/') - if (FileSystemStorage.SNAPSHOT_BYTE_COPY_PATHS.has(normalized)) { + // Byte-copy (never hard-link) files that are mutated in place: the exact + // append-in-place paths, and every file under a mmap-mutated directory + // (e.g. the native `_id_mapper/*`). A shared inode would let a live + // msync/truncate reach through into the snapshot. + if ( + FileSystemStorage.SNAPSHOT_BYTE_COPY_PATHS.has(normalized) || + FileSystemStorage.SNAPSHOT_BYTE_COPY_DIRS.has(normalized.split('/')[0]) || + FileSystemStorage.SNAPSHOT_BYTE_COPY_PREFIXES.some((p) => normalized.startsWith(p)) + ) { await fs.promises.copyFile(sourceFile, targetFile) continue } @@ -1135,6 +1034,56 @@ export class FileSystemStorage extends BaseStorage { } } + /** + * @description Pre-upgrade backup: a hard-link snapshot of the whole store into + * a SIBLING directory (`.migration-backup`, outside `rootDir` so the + * snapshot never recurses into itself), taken before a 7.x → 8.0 upgrade + * rebuilds the derived indexes. Zero-copy (shared inodes; the store is + * immutable-by-rename) and instant even at scale. Returns the backup path, or + * `null` when the store holds no canonical nouns (nothing to protect). If a + * backup already exists from a prior FAILED upgrade, it is REUSED as-is (that + * is the true pre-upgrade state; a retry must not overwrite it). + */ + public async createMigrationBackup(): Promise { + await this.ensureInitialized() + + // Nothing to protect if the store has no canonical nouns (e.g. a brand-new + // brain whose marker is simply absent — `epochStale` is true but there is no + // 7.x data to migrate). + const probe = await this.getNouns({ pagination: { limit: 1 } }) + if ((probe.totalCount || 0) === 0 && probe.items.length === 0) { + return null + } + + const backupPath = `${this.rootDir}.migration-backup` + + // Reuse an existing backup (a prior failed upgrade's pre-state) rather than + // overwrite it — snapshotToDirectory also refuses a non-empty target. + try { + const existing = await fs.promises.readdir(backupPath) + if (existing.length > 0) return backupPath + } catch (error: any) { + if (error.code !== 'ENOENT') throw error + } + + await this.snapshotToDirectory(backupPath) + return backupPath + } + + /** + * @description Remove a {@link createMigrationBackup} snapshot. Best-effort: a + * missing path is a no-op, and any error is swallowed (a leftover backup dir is + * harmless — the operator can delete it). Removing the hard-links never touches + * the live store's bytes (shared inodes; only the extra links go away). + */ + public async removeMigrationBackup(location: string): Promise { + try { + await fs.promises.rm(location, { recursive: true, force: true }) + } catch { + // best-effort — leaving the backup behind is safe + } + } + /** * Recursively collect snapshot-eligible files under `dirAbs`, excluding the * lock directory and in-flight `*.tmp.*` write files. @@ -1171,6 +1120,16 @@ export class FileSystemStorage extends BaseStorage { * * @param sourcePath - Absolute path of a directory produced by * {@link FileSystemStorage.snapshotToDirectory}. + * + * Non-destructive: the snapshot is copied into a staging area (sparse-aware, + * so a store of mostly-hole mmap blobs cannot balloon and ENOSPC) BEFORE any + * live data is touched. Only once the full copy has succeeded and a completion + * marker is fsync'd does an atomic per-entry swap move it into place. A copy + * failure (including ENOSPC) leaves the live store exactly as it was; a crash + * mid-swap is resumed forward on the next {@link init} by + * {@link completeInterruptedRestore}. The previous implementation removed the + * live store first and then `fs.cp`'d — a copy failure destroyed the brain it + * was meant to recover. */ public async restoreFromDirectory(sourcePath: string): Promise { await this.ensureInitialized() @@ -1180,23 +1139,194 @@ export class FileSystemStorage extends BaseStorage { throw new Error(`restoreFromDirectory: ${sourcePath} is not a directory`) } - // Clear current contents, preserving live lock state. - const currentEntries = await fs.promises.readdir(this.rootDir) - for (const entry of currentEntries) { + const staging = path.join(this.rootDir, FileSystemStorage.RESTORE_STAGING_DIR) + + // Discard any staging left by a prior aborted restore, then stage fresh. + await fs.promises.rm(staging, { recursive: true, force: true }) + await fs.promises.mkdir(staging, { recursive: true }) + + // Copy the snapshot into staging (sparse-aware). ANY failure here — most + // importantly ENOSPC — leaves the live store untouched: we remove only the + // half-written staging area and re-throw. + const staged: string[] = [] + try { + const sourceEntries = await fs.promises.readdir(sourcePath) + for (const entry of sourceEntries) { + if (FileSystemStorage.SNAPSHOT_EXCLUDED_TOP_DIRS.has(entry)) continue + await this.copyTreeSparse( + path.join(sourcePath, entry), + path.join(staging, entry) + ) + staged.push(entry) + } + // Commit point: fsync a marker naming the fully-staged entries. Only after + // this does the swap (and its resume) become authorized. + const markerPath = path.join(staging, FileSystemStorage.RESTORE_MARKER) + await fs.promises.writeFile(markerPath, JSON.stringify({ entries: staged })) + const mfh = await fs.promises.open(markerPath, 'r') + try { + await mfh.sync() + } finally { + await mfh.close() + } + const dfh = await fs.promises.open(staging, 'r').catch(() => null) + if (dfh) { + try { + await dfh.sync() + } catch { + // platform may reject directory fsync — best effort + } finally { + await dfh.close() + } + } + } catch (error: any) { + await fs.promises.rm(staging, { recursive: true, force: true }).catch(() => {}) + throw new Error( + `restoreFromDirectory: staging copy failed, live store left untouched: ${error.message}` + ) + } + + // Atomic per-entry swap (metadata-only renames within rootDir — cannot ENOSPC). + await this.swapStagedRestoreIn() + await this.reloadDerivedState() + } + + /** + * Move a fully-staged, marker-committed restore into place, then clear the + * staging area. Idempotent and resumable: driven by the marker's entry list + * and by which staged entries remain, so a crash at any point is completed by + * simply calling it again (from {@link completeInterruptedRestore} on the next + * open). Every step is a same-filesystem rename or a remove — no operation can + * fail for disk space, so once the marker exists the store is guaranteed to + * reach the restored state. + */ + private async swapStagedRestoreIn(): Promise { + const staging = path.join(this.rootDir, FileSystemStorage.RESTORE_STAGING_DIR) + const markerPath = path.join(staging, FileSystemStorage.RESTORE_MARKER) + + let staged: string[] + try { + const marker = JSON.parse(await fs.promises.readFile(markerPath, 'utf-8')) + staged = Array.isArray(marker?.entries) ? marker.entries : [] + } catch { + return // no committed marker — nothing to swap + } + const stagedSet = new Set(staged) + + // 1. Remove stale live entries the snapshot does not contain (excluding + // process-local dirs and the staging area itself). An already-placed + // staged entry is in stagedSet, so it is kept. + for (const entry of await fs.promises.readdir(this.rootDir)) { if (FileSystemStorage.SNAPSHOT_EXCLUDED_TOP_DIRS.has(entry)) continue + if (entry === FileSystemStorage.RESTORE_STAGING_DIR) continue + if (stagedSet.has(entry)) continue await fs.promises.rm(path.join(this.rootDir, entry), { recursive: true, force: true }) } - // Byte-copy the snapshot in (defensively skipping lock dirs in old snapshots). - const sourceEntries = await fs.promises.readdir(sourcePath) - for (const entry of sourceEntries) { - if (FileSystemStorage.SNAPSHOT_EXCLUDED_TOP_DIRS.has(entry)) continue - await fs.promises.cp(path.join(sourcePath, entry), path.join(this.rootDir, entry), { - recursive: true - }) + // 2. Place each staged entry (idempotent: a prior attempt that already moved + // it leaves staging/entry absent, so we skip). `rm` the old first — a + // rename onto an existing non-empty directory is not allowed; the staged + // copy is the durable source until it is placed, so a crash between the + // rm and the rename is recovered forward on the next call. + for (const entry of staged) { + const from = path.join(staging, entry) + const to = path.join(this.rootDir, entry) + const exists = await fs.promises.lstat(from).then(() => true, () => false) + if (!exists) continue + await fs.promises.rm(to, { recursive: true, force: true }) + await fs.promises.rename(from, to) } - await this.reloadDerivedState() + // 3. Clear the staging area (marker last-standing entry). + await fs.promises.rm(staging, { recursive: true, force: true }) + } + + /** + * On open, finish any restore interrupted by a crash. A staging dir WITH the + * completion marker means the copy had succeeded — resume the swap forward + * (loudly). A staging dir WITHOUT the marker is an interrupted copy — pure + * debris; the live store is authoritative, so discard it. Called from + * {@link init} before counts/derived state load, so recovery is invisible to + * the rest of startup. Returns `true` if a swap was resumed. + */ + private async completeInterruptedRestore(): Promise { + const staging = path.join(this.rootDir, FileSystemStorage.RESTORE_STAGING_DIR) + const stagingStat = await fs.promises.stat(staging).catch(() => null) + if (!stagingStat || !stagingStat.isDirectory()) return false + + const markerPath = path.join(staging, FileSystemStorage.RESTORE_MARKER) + const hasMarker = await fs.promises + .stat(markerPath) + .then(() => true, () => false) + + if (!hasMarker) { + // Interrupted before the copy committed — live data untouched, discard. + await fs.promises.rm(staging, { recursive: true, force: true }).catch(() => {}) + return false + } + + console.log('♻️ Resuming an interrupted restore (completing the staged swap)') + await this.swapStagedRestoreIn() + return true + } + + /** + * Recursively copy `src` to `dest`, preserving holes (sparse regions). Files + * are copied chunk-by-chunk skipping all-zero chunks, so a store of + * mostly-hole mmap blobs restores at its true allocated size instead of + * materializing every hole (the failure that made `fs.cp` ENOSPC a restore + * that would otherwise fit). Directories recurse; symlinks are recreated. + */ + private async copyTreeSparse(src: string, dest: string): Promise { + const stat = await fs.promises.lstat(src) + if (stat.isDirectory()) { + await fs.promises.mkdir(dest, { recursive: true }) + for (const child of await fs.promises.readdir(src)) { + await this.copyTreeSparse(path.join(src, child), path.join(dest, child)) + } + } else if (stat.isSymbolicLink()) { + await fs.promises.symlink(await fs.promises.readlink(src), dest) + } else if (stat.isFile()) { + await this.copyFileSparse(src, dest, stat.size, stat.mode) + } + // Other node types (sockets, devices) do not occur in a brain store. + } + + /** Sparse-aware single-file copy — see {@link copyTreeSparse}. */ + private async copyFileSparse( + src: string, + dest: string, + size: number, + mode: number + ): Promise { + const CHUNK = FileSystemStorage.SPARSE_CHUNK_BYTES + const srcFh = await fs.promises.open(src, 'r') + try { + const destFh = await fs.promises.open(dest, 'w', mode) + try { + // Pre-size the destination so unwritten regions are holes. + await destFh.truncate(size) + const buf = Buffer.allocUnsafe(CHUNK) + let pos = 0 + while (pos < size) { + const { bytesRead } = await srcFh.read(buf, 0, CHUNK, pos) + if (bytesRead === 0) break + const chunk = buf.subarray(0, bytesRead) + // Skip all-zero chunks: leaving them unwritten preserves the hole. + const isHole = chunk.equals( + FileSystemStorage.SPARSE_ZERO_CHUNK.subarray(0, bytesRead) + ) + if (!isHole) { + await destFh.write(chunk, 0, bytesRead, pos) + } + pos += bytesRead + } + } finally { + await destFh.close() + } + } finally { + await srcFh.close() + } } // =========================================================================== @@ -1209,7 +1339,7 @@ export class FileSystemStorage extends BaseStorage { * The key's "/"-separated segments become nested directories and the file is * suffixed with `.bin`, e.g. `"graph-lsm/source/sstable-123"` → * `/_blobs/graph-lsm/source/sstable-123.bin`. This convention is - * shared with cortex's `MmapFileSystemStorage` so native code and the TS layer + * shared with cor's `MmapFileSystemStorage` so native code and the TS layer * agree on exactly where each blob lives. * * @param key - The blob key. @@ -1238,27 +1368,44 @@ export class FileSystemStorage extends BaseStorage { await this.ensureInitialized() const filePath = this.blobPath(key) await this.ensureDirectoryExists(path.dirname(filePath)) - // Unique per-writer temp suffix — matches the pattern used at every other - // atomic-write site in this file (lines 336, 551, 744, 781, 1529, 2908). - // Without a unique suffix, two concurrent saveBinaryBlob() calls for the - // same key collide on `${filePath}.tmp`: both writeFile, the first rename - // succeeds, the second fires against a missing temp and throws ENOENT. - // Reproduced in production: column-store compaction running alongside an - // explicit flush() repeatedly raced on `_column_index//DELETED.bin`. - const tmpPath = `${filePath}.tmp.${process.pid}.${Date.now()}.${Math.random().toString(36).slice(2)}` - await fs.promises.writeFile(tmpPath, data) - try { - await fs.promises.rename(tmpPath, filePath) - } catch (err) { - const code = (err as NodeJS.ErrnoException).code - // The writes are idempotent for a given key — every caller persists the - // same logical bytes for that key — so ENOENT on rename means the temp - // is already gone (rare with the unique suffix, but defensive against - // crash-resume cleanup paths and external file-system sweepers). - if (code === 'ENOENT') return - // Any other failure: clean up our own temp to avoid orphans before rethrow. - await fs.promises.unlink(tmpPath).catch(() => {}) - throw err + + // Atomic write via a UNIQUE per-writer temp suffix — matches the pattern + // used at every other atomic-write site in this file (lines 336, 551, 744, + // 781, 1529, 2908). The unique suffix (pid+time+random) means NO other + // writer ever touches this temp, so two concurrent saveBinaryBlob() calls + // for the same key can never collide on the temp path (the shared-`.tmp` + // race that once threw ENOENT — column-store compaction vs an explicit + // flush() on `_column_index//DELETED.bin` — is gone). + // + // Crucially, BECAUSE the temp is unique, a rename ENOENT can no longer mean + // "a concurrent idempotent writer already renamed it": nobody else has this + // temp. It means OUR just-written temp vanished before the rename, so the + // bytes did NOT land — returning success would acknowledge a write that + // stored nothing (the native provider mmaps these blobs; a phantom-acked + // blob is exactly the silent-loss class). The only way that happens with a + // unique temp is an external sweeper / crash-cleanup removing it mid-write, + // so retry ONCE with a fresh temp; if it vanishes again, FAIL LOUD. + const writeOnce = async (): Promise<'ok' | 'temp-vanished'> => { + const tmpPath = `${filePath}.tmp.${process.pid}.${Date.now()}.${Math.random().toString(36).slice(2)}` + await fs.promises.writeFile(tmpPath, data) + try { + await fs.promises.rename(tmpPath, filePath) + return 'ok' + } catch (err) { + // Clean up our own temp (best-effort) so no failure path orphans it. + await fs.promises.unlink(tmpPath).catch(() => {}) + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return 'temp-vanished' + throw err + } + } + + if ((await writeOnce()) === 'temp-vanished' && (await writeOnce()) === 'temp-vanished') { + throw new Error( + `saveBinaryBlob('${key}'): the temp file was removed before rename on two ` + + `successive attempts — the blob did NOT persist. Some external process is ` + + `deleting files under ${this.blobsDir} mid-write. Failing loud rather than ` + + `acknowledging a durable write that stored nothing.` + ) } } @@ -1272,8 +1419,17 @@ export class FileSystemStorage extends BaseStorage { await this.ensureInitialized() try { return await fs.promises.readFile(this.blobPath(key)) - } catch { - return null + } catch (err) { + // Absent blob → null (the documented contract). A real fault + // (EIO/EACCES/EMFILE/…) must NOT be masked as "absent": doing so makes a + // present-but-unreadable blob look missing and drives a needless rebuild + // or an empty read (the native provider consumes this). Mandate: loud + // errors, never quiet losses. Fault-propagation restored lockstep with + // cortex 3.0.13, whose two column-store call sites now handle the throw + // (they mark the field unavailable + throw a named error) instead of + // relying on null-on-error. + if (isAbsentError(err)) return null + throw err } } @@ -1284,6 +1440,11 @@ export class FileSystemStorage extends BaseStorage { */ public async deleteBinaryBlob(key: string): Promise { await this.ensureInitialized() + // Registered-blob contract: refuse to delete a declared + // derived-index family member — an in-process GC/sweeper cannot remove a + // load-bearing index file. Throws ProtectedArtifactError; no-op when no + // families are registered. + await this.assertBlobKeyDeletable(key) try { await fs.promises.unlink(this.blobPath(key)) } catch { @@ -1403,6 +1564,24 @@ export class FileSystemStorage extends BaseStorage { await fs.promises.rm(casDir, { recursive: true, force: true }) } + // Remove the raw-blob + native-shared + column-index footprint. These + // top-level trees were NOT wiped before, so a cleared brain re-read stale + // native blobs (HNSW/LSM segments, the native dkann index), a stale native + // id-mapper, and — worst — orphaned column manifests. `_column_index` holds + // the column-store MANIFEST.json files while their segment bytes live under + // `_blobs/_column_index/...`; removing one without the other would strand a + // manifest listing segments that no longer exist, which the load path now + // (correctly) refuses with ColumnSegmentLoadError. They must fall together, + // as a set, exactly as `_cas` does — the complete derived footprint, not a + // subset. (`locks/` is deliberately left: it is live coordination state, + // not data.) + for (const nativeDir of ['_blobs', '_id_mapper', '_column_index']) { + const dir = path.join(this.rootDir, nativeDir) + if (await this.directoryExists(dir)) { + await fs.promises.rm(dir, { recursive: true, force: true }) + } + } + // Reset ALL shared derived state via the same path restore uses: the // write-through cache, id→type/subtype caches, per-type/subtype count // rollups, statistics cache, graph-index singleton, and the BlobStorage @@ -1568,7 +1747,7 @@ export class FileSystemStorage extends BaseStorage { // Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation // Removed 2 pagination methods (getNounsWithPagination, getVerbsWithPagination) - use BaseStorage's implementation - public supportsMultiProcessLocking(): boolean { + public override supportsMultiProcessLocking(): boolean { return true } @@ -1586,93 +1765,181 @@ export class FileSystemStorage extends BaseStorage { * `lastHeartbeat` is older than `WRITER_STALE_THRESHOLD_MS` (60s). * Stale locks are overwritten with a warning. `force: true` overrides even live locks. */ - public async acquireWriterLock(options?: { force?: boolean }): Promise { + public override async acquireWriterLock(options?: { force?: boolean }): Promise { await this.ensureInitialized() await this.ensureDirectoryExists(this.lockDir) const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE) const os = await import('node:os') const hostname = os.hostname() - const now = new Date().toISOString() + const myPid = typeof process !== 'undefined' && process.pid ? process.pid : 0 - const existing = await this.readWriterLock() - if (existing && !options?.force) { - // Same-process re-open: a second Brainy instance in this Node process - // (e.g. test "simulate server restart" patterns, or a consumer that - // explicitly re-instantiates without closing first). This isn't the - // dangerous cross-process case the lock exists to prevent — the two - // instances share a memory space and can't silently diverge from each - // other beyond what their callers already see. Warn and take over the lock. - if (existing.pid === (typeof process !== 'undefined' ? process.pid : 0) && - existing.hostname === hostname) { - console.warn( - `[brainy] Re-acquiring writer lock for ${this.rootDir} held by the same process (PID ${existing.pid}). ` + - `If you intended to keep the previous Brainy instance alive, this is a bug — close it first.` - ) - } else { - const stale = await this.isWriterLockStale(existing) - if (!stale) { + // Bounded acquire loop. The CLAIM itself is an atomic create-exclusive + // write (O_EXCL) — two processes racing an ABSENT lock can never both + // succeed, which closes the read-then-write window where the loser used + // to keep running unlocked, silently. An EEXIST loser loops, re-reads, + // and handles whatever it finds honestly (fresh foreign lock → loud + // throw; stale/forced → verified takeover). + const MAX_ATTEMPTS = 3 + for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) { + const now = new Date().toISOString() + const existing = await this.readWriterLock() + + if (existing) { + // Same-process re-open: a second Brainy instance in this Node process + // (e.g. test "simulate server restart" patterns, or a consumer that + // explicitly re-instantiates without closing first). This isn't the + // dangerous cross-process case the lock exists to prevent — the two + // instances share a memory space and can't silently diverge from each + // other beyond what their callers already see. Warn and take over. + if (existing.pid === myPid && existing.hostname === hostname && !options?.force) { + console.warn( + `[brainy] Re-acquiring writer lock for ${this.rootDir} held by the same process (PID ${existing.pid}). ` + + `If you intended to keep the previous Brainy instance alive, this is a bug — close it first.` + ) + const info: WriterLockInfo = { + pid: myPid, + hostname, + startedAt: now, + lastHeartbeat: now, + version: getBrainyVersion(), + rootDir: this.rootDir + } + await this.writeFileAtomic(lockFile, JSON.stringify(info, null, 2)) + this.installWriterLock(info) + return info + } + + const stale = !options?.force && (await this.isWriterLockStale(existing)) + if (!options?.force && !stale) { // Consumer-facing error contract: callers detect this case via // err.code and read the holder's details from err.lockInfo. - const err = new Error( - `Another writer holds this Brainy directory.\n` + - ` PID: ${existing.pid} on host ${existing.hostname}\n` + - ` Started: ${existing.startedAt}\n` + - ` Heartbeat: ${existing.lastHeartbeat}\n` + - ` Version: ${existing.version}\n` + - ` Directory: ${this.rootDir}\n\n` + - `For diagnostic queries against this live store, use:\n` + - ` const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: '${this.rootDir}' } })\n\n` + - `If you have verified the existing lock is stale (e.g. a crashed writer on a different host that PID liveness cannot reach), pass { force: true }.` - ) as Error & { code: string; lockInfo: WriterLockInfo } - err.code = 'BRAINY_WRITER_LOCKED' - err.lockInfo = existing - throw err + throw this.writerLockedError(existing) } + console.warn( - `[brainy] Overwriting stale writer lock for ${this.rootDir} ` + - `(PID ${existing.pid} on ${existing.hostname} appears dead).` + options?.force + ? `[brainy] Force-overwriting writer lock for ${this.rootDir} ` + + `(was held by PID ${existing.pid} on ${existing.hostname}).` + : `[brainy] Overwriting stale writer lock for ${this.rootDir} ` + + `(PID ${existing.pid} on ${existing.hostname} appears dead).` ) + // Takeover: verify the file still holds the lock we judged (a live + // successor may have claimed meanwhile), then remove it and fall + // through to the atomic claim below. A racing claimer who beats us to + // the create simply wins — our next loop iteration reads their fresh + // lock and throws honestly. (Advisory file locking has no + // compare-and-delete; staleness requiring a 60s-old heartbeat keeps + // the residual verify-to-unlink window practically unreachable.) + const recheck = await this.readWriterLock() + if ( + recheck && + (recheck.pid !== existing.pid || + recheck.startedAt !== existing.startedAt || + recheck.lastHeartbeat !== existing.lastHeartbeat) + ) { + continue // the lock changed hands while we deliberated — re-evaluate + } + try { + await fs.promises.unlink(lockFile) + } catch (err: any) { + if (err.code !== 'ENOENT') throw err + } } - } else if (existing && options?.force) { - console.warn( - `[brainy] Force-overwriting writer lock for ${this.rootDir} ` + - `(was held by PID ${existing.pid} on ${existing.hostname}).` - ) + + const info: WriterLockInfo = { + pid: myPid, + hostname, + startedAt: existing && options?.force ? existing.startedAt : now, + lastHeartbeat: now, + version: getBrainyVersion(), + rootDir: this.rootDir + } + + // The atomic claim: create-exclusive, so exactly ONE racer wins. + try { + await fs.promises.writeFile(lockFile, JSON.stringify(info, null, 2), { flag: 'wx' }) + } catch (err: any) { + if (err.code === 'EEXIST') { + continue // someone else claimed between our read and create — re-evaluate + } + throw err + } + + this.installWriterLock(info) + return info } - const info: WriterLockInfo = { - pid: typeof process !== 'undefined' && process.pid ? process.pid : 0, - hostname, - startedAt: existing && options?.force ? existing.startedAt : now, - lastHeartbeat: now, - version: getBrainyVersion(), - rootDir: this.rootDir - } + // Attempts exhausted: something is claiming this directory faster than we + // can evaluate it. Read whoever holds it now and fail loudly with their + // details rather than degrading into a lockless open. + const holder = await this.readWriterLock() + if (holder) throw this.writerLockedError(holder) + throw new Error( + `Failed to acquire the writer lock for ${this.rootDir} after ${MAX_ATTEMPTS} attempts — ` + + `the lock file is being contended. Retry, or inspect ${lockFile}.` + ) + } - await this.writeFileAtomic(lockFile, JSON.stringify(info, null, 2)) + /** Record lock ownership + start the unref'd heartbeat. */ + private installWriterLock(info: WriterLockInfo): void { this.writerLockInfo = info // Heartbeat — rewrite lastHeartbeat every WRITER_HEARTBEAT_MS so other // processes can tell a live writer from one that crashed without releasing. this.writerLockHeartbeat = setInterval(() => { - this.refreshWriterLockHeartbeat().catch((err) => { - console.warn('[brainy] Failed to refresh writer lock heartbeat:', err) + const tick = this.refreshWriterLockHeartbeat().catch((err) => { + // ENOENT = the lock (or its directory) vanished mid-refresh — the + // store was released or removed under us; the next acquire recreates + // it. Benign by construction; anything else stays loud. + if ((err as NodeJS.ErrnoException)?.code !== 'ENOENT') { + console.warn('[brainy] Failed to refresh writer lock heartbeat:', err) + } + }) + this.writerHeartbeatInFlight = tick.finally(() => { + if (this.writerHeartbeatInFlight === tick) { + this.writerHeartbeatInFlight = undefined + } }) }, FileSystemStorage.WRITER_HEARTBEAT_MS) if (typeof this.writerLockHeartbeat.unref === 'function') { // Don't keep the event loop alive just for the heartbeat. this.writerLockHeartbeat.unref() } - - return info } - public async releaseWriterLock(): Promise { + /** The consumer-facing BRAINY_WRITER_LOCKED error, holder details attached. */ + private writerLockedError(existing: WriterLockInfo): Error { + const err = new Error( + `Another writer holds this Brainy directory.\n` + + ` PID: ${existing.pid} on host ${existing.hostname}\n` + + ` Started: ${existing.startedAt}\n` + + ` Heartbeat: ${existing.lastHeartbeat}\n` + + ` Version: ${existing.version}\n` + + ` Directory: ${this.rootDir}\n\n` + + `For diagnostic queries against this live store, use:\n` + + ` const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: '${this.rootDir}' } })\n\n` + + `If you have verified the existing lock is stale (e.g. a crashed writer on a different host that PID liveness cannot reach), pass { force: true }.` + ) as Error & { code: string; lockInfo: WriterLockInfo } + err.code = 'BRAINY_WRITER_LOCKED' + err.lockInfo = existing + return err + } + + public override async releaseWriterLock(): Promise { if (this.writerLockHeartbeat) { clearInterval(this.writerLockHeartbeat) this.writerLockHeartbeat = undefined } + // Drain an in-flight heartbeat tick BEFORE unlinking: clearInterval stops + // future ticks only, and a straggler write landing after the unlink would + // re-create the lock as a phantom (blocking the next writer until the + // stale TTL). After the drain, any refresh is either fully landed (we + // unlink its output below) or not started (it sees writerLockInfo + // undefined and returns). + if (this.writerHeartbeatInFlight) { + await this.writerHeartbeatInFlight + } if (!this.writerLockInfo) { return } @@ -1693,7 +1960,7 @@ export class FileSystemStorage extends BaseStorage { } } - public async readWriterLock(): Promise { + public override async readWriterLock(): Promise { await this.ensureInitialized() const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE) try { @@ -1778,7 +2045,7 @@ export class FileSystemStorage extends BaseStorage { * `locks/_flush_responses/` with the same request ID. Stale `.req` files * (>FLUSH_REQUEST_TTL_MS) are garbage-collected on every tick. */ - public startFlushRequestWatcher(onRequest: () => Promise): void { + public override startFlushRequestWatcher(onRequest: () => Promise): void { if (this.flushWatcherInterval) return // already watching this.flushWatcherOnRequest = onRequest @@ -1801,7 +2068,7 @@ export class FileSystemStorage extends BaseStorage { } } - public stopFlushRequestWatcher(): void { + public override stopFlushRequestWatcher(): void { if (this.flushWatcherInterval) { clearInterval(this.flushWatcherInterval) this.flushWatcherInterval = undefined @@ -1880,7 +2147,7 @@ export class FileSystemStorage extends BaseStorage { * Inspector side: drop a `.req` file and poll for the corresponding `.ack`. * Returns true if the writer acknowledged in time, false on timeout. */ - public async requestFlushOverFilesystem(timeoutMs: number): Promise { + public override async requestFlushOverFilesystem(timeoutMs: number): Promise { await this.ensureInitialized() const reqDir = path.join(this.lockDir, FileSystemStorage.FLUSH_REQUEST_DIR) const ackDir = path.join(this.lockDir, FileSystemStorage.FLUSH_RESPONSE_DIR) @@ -2192,39 +2459,33 @@ export class FileSystemStorage extends BaseStorage { */ private async initializeCountsFromDisk(): Promise { try { - // CRITICAL: Detect existing depth before counting - // Can't use getAllShardedFiles() which assumes depth=1 - const existingDepth = await this.detectExistingShardingDepth() - const depthToUse = existingDepth !== null ? existingDepth : this.SHARDING_DEPTH + // Count the CANONICAL 8.0 layout (`entities////…`) — + // the tree saveNoun/getNouns actually read and write. The previous scan + // counted the vestigial `entities/*/hnsw` directories, which the 8.0 + // write path never populates, so a store recovering from a lost or + // corrupted counts.json re-initialized every counter to ZERO on real + // data (wrong stats/boot logs and a mis-sized rebuild-strategy + // decision at open). + const nouns = await this.scanCanonicalEntities('nouns') + this.totalNounCount = nouns.count + const verbs = await this.scanCanonicalEntities('verbs') + this.totalVerbCount = verbs.count - // Count nouns using detected depth - const validNounFiles = await this.getAllFilesAtDepth(this.nounsDir, depthToUse) - this.totalNounCount = validNounFiles.length - - // Count verbs using detected depth - const validVerbFiles = await this.getAllFilesAtDepth(this.verbsDir, depthToUse) - this.totalVerbCount = validVerbFiles.length - - // Sample some files to get type distribution (don't read all) - // Load metadata separately for type information - const sampleSize = Math.min(100, validNounFiles.length) - for (let i = 0; i < sampleSize; i++) { - try { - const file = validNounFiles[i] - const id = file.replace('.json', '') - - // Load metadata from separate storage for type info - const metadata = await this.getNounMetadata(id) - if (metadata) { - const type = metadata.noun || 'default' - this.entityCounts.set(type, (this.entityCounts.get(type) || 0) + 1) - } - } catch { - // Skip invalid files or missing metadata + // Sample some entities for the type distribution (don't read all). + // Read the metadata files DIRECTLY with fs — this runs inside init(), + // and every guarded accessor (getNounMetadata → ensureInitialized) + // re-enters init() from here, deadlocking the open. (Latent in the old + // code too: its scan of the empty hnsw dirs just never sampled.) + for (const entityDir of nouns.sampleDirs) { + const metadata = await this.readEntityMetadataRaw(entityDir) + if (metadata) { + const type = metadata.noun || 'default' + this.entityCounts.set(type, (this.entityCounts.get(type) || 0) + 1) } } // Extrapolate counts if we sampled + const sampleSize = nouns.sampleDirs.length if (sampleSize < this.totalNounCount && sampleSize > 0) { const multiplier = this.totalNounCount / sampleSize for (const [type, count] of this.entityCounts.entries()) { @@ -2238,6 +2499,62 @@ export class FileSystemStorage extends BaseStorage { } } + /** + * Walk the canonical `entities//<2-hex-shard>//` tree, counting + * one entity per id directory (the layout `getNounVectorPath`/`getNouns` + * use). Returns up to 100 sampled entity directories (absolute paths) — + * nouns feed the type-distribution estimate above. An absent tree (fresh + * store) counts zero. + */ + private async scanCanonicalEntities( + kind: 'nouns' | 'verbs' + ): Promise<{ count: number; sampleDirs: string[] }> { + const base = path.join(this.rootDir, 'entities', kind) + const SAMPLE_MAX = 100 + let count = 0 + const sampleDirs: string[] = [] + try { + const shards = await fs.promises.readdir(base, { withFileTypes: true }) + for (const shard of shards) { + if (!shard.isDirectory() || !/^[0-9a-f]{2}$/i.test(shard.name)) continue + const shardPath = path.join(base, shard.name) + const ids = await fs.promises.readdir(shardPath, { withFileTypes: true }) + for (const entry of ids) { + if (!entry.isDirectory()) continue + count++ + if (sampleDirs.length < SAMPLE_MAX) { + sampleDirs.push(path.join(shardPath, entry.name)) + } + } + } + } catch (error: any) { + if (error?.code !== 'ENOENT') throw error + } + return { count, sampleDirs } + } + + /** + * Read one canonical entity's `metadata.json` (or `.json.gz`) directly with + * fs — NO guarded accessors. Used only by the init-time count recovery, + * where `getNounMetadata`'s `ensureInitialized()` would re-enter `init()`. + * @param entityDir - Absolute `entities///` directory. + * @returns The parsed metadata, or null when absent/unreadable. + */ + private async readEntityMetadataRaw(entityDir: string): Promise { + const base = path.join(entityDir, 'metadata.json') + try { + return JSON.parse(await fs.promises.readFile(base, 'utf-8')) + } catch { + // fall through to the compressed variant + } + try { + const gz = await fs.promises.readFile(`${base}.gz`) + return JSON.parse(zlib.gunzipSync(gz).toString('utf-8')) + } catch { + return null + } + } + /** * Persist counts to filesystem storage */ @@ -2268,307 +2585,9 @@ export class FileSystemStorage extends BaseStorage { // Intelligent Directory Sharding // ============================================= - /** - * Migrate files from one sharding depth to another - * Handles: 0→1 (flat to single-level), 2→1 (deep to single-level) - * Uses atomic file operations and comprehensive error handling - * - * @param fromDepth - Source sharding depth - * @param toDepth - Target sharding depth (must be 1) - */ - private async migrateShardingStructure(fromDepth: number, toDepth: number): Promise { - // Validation - if (fromDepth === toDepth) { - throw new Error(`Migration not needed: already at depth ${toDepth}`) - } - if (toDepth !== 1) { - throw new Error(`Migration only supports target depth 1 (got ${toDepth})`) - } - if (fromDepth !== 0 && fromDepth !== 2) { - throw new Error(`Migration only supports source depth 0 or 2 (got ${fromDepth})`) - } - // Create migration lock to prevent concurrent migrations - const lockFile = path.join(this.systemDir, '.migration-lock') - const lockExists = await this.fileExists(lockFile) - - if (lockExists) { - // Check if lock is stale (> 1 hour old) - try { - const stats = await fs.promises.stat(lockFile) - const lockAge = Date.now() - stats.mtimeMs - const ONE_HOUR = 60 * 60 * 1000 - - if (lockAge < ONE_HOUR) { - throw new Error( - 'Migration already in progress. If this is incorrect, delete .migration-lock file.' - ) - } - - // Lock is stale, remove it - console.log('⚠️ Removing stale migration lock (> 1 hour old)') - await fs.promises.unlink(lockFile) - } catch (error: any) { - if (error.code !== 'ENOENT') { - throw error - } - } - } - - try { - // Create lock file - await fs.promises.writeFile(lockFile, JSON.stringify({ - startedAt: new Date().toISOString(), - fromDepth, - toDepth, - pid: process.pid - })) - - // Discover all files to migrate - console.log('📊 Discovering files to migrate...') - const filesToMigrate = await this.discoverFilesForMigration(fromDepth) - - if (filesToMigrate.length === 0) { - console.log('ℹ️ No files to migrate') - return - } - - console.log(`📦 Migrating ${filesToMigrate.length} files...`) - - // Create all target shard directories upfront - await this.createAllShardDirectories(this.nounsDir) - await this.createAllShardDirectories(this.verbsDir) - - // Migrate files with progress tracking - let migratedCount = 0 - let skippedCount = 0 - const errors: Array<{ file: string; error: string }> = [] - - for (const fileInfo of filesToMigrate) { - try { - await this.migrateFile(fileInfo, fromDepth, toDepth) - migratedCount++ - - // Progress update every 1000 files - if (migratedCount % 1000 === 0) { - const percent = ((migratedCount / filesToMigrate.length) * 100).toFixed(1) - console.log(` 📊 Progress: ${migratedCount}/${filesToMigrate.length} (${percent}%)`) - } - - // Yield to event loop every 100 files to prevent blocking - if (migratedCount % 100 === 0) { - await new Promise(resolve => setImmediate(resolve)) - } - } catch (error: any) { - skippedCount++ - errors.push({ - file: fileInfo.oldPath, - error: error.message - }) - - // Log first few errors - if (errors.length <= 5) { - console.warn(`⚠️ Skipped ${fileInfo.oldPath}: ${error.message}`) - } - } - } - - // Final summary - console.log(`\n✅ Migration Results:`) - console.log(` Migrated: ${migratedCount} files`) - console.log(` Skipped: ${skippedCount} files`) - - if (errors.length > 0) { - console.warn(`\n⚠️ ${errors.length} files could not be migrated`) - if (errors.length > 5) { - console.warn(` (First 5 errors shown above, ${errors.length - 5} more occurred)`) - } - } - - // Cleanup: Remove empty old directories - if (fromDepth === 0) { - // No subdirectories to clean for flat structure - } else if (fromDepth === 2) { - await this.cleanupEmptyDirectories(this.nounsDir, fromDepth) - await this.cleanupEmptyDirectories(this.verbsDir, fromDepth) - } - - // Verification: Count files in new structure - const verifyCount = await this.countFilesInStructure(toDepth) - console.log(`\n🔍 Verification: ${verifyCount} files in new structure`) - - if (verifyCount < migratedCount) { - console.warn(`⚠️ Warning: Verification count (${verifyCount}) < migrated count (${migratedCount})`) - } - } finally { - // Always remove lock file - try { - await fs.promises.unlink(lockFile) - } catch (error) { - // Ignore error if lock file doesn't exist - } - } - } - - /** - * Discover all files that need to be migrated - * Constructs correct oldPath based on source depth - */ - private async discoverFilesForMigration(fromDepth: number): Promise> { - const files: Array<{ oldPath: string; id: string; type: 'noun' | 'verb' }> = [] - - // Discover noun files - const nounFiles = await this.getAllFilesAtDepth(this.nounsDir, fromDepth) - for (const filename of nounFiles) { - const id = filename.replace('.json', '') - - // Construct correct oldPath based on fromDepth - let oldPath: string - switch (fromDepth) { - case 0: - // Flat: nouns/uuid.json - oldPath = path.join(this.nounsDir, `${id}.json`) - break - case 1: - // Single-level: nouns/ab/uuid.json - oldPath = path.join(this.nounsDir, id.substring(0, 2), `${id}.json`) - break - case 2: - // Deep: nouns/ab/cd/uuid.json - oldPath = path.join(this.nounsDir, id.substring(0, 2), id.substring(2, 4), `${id}.json`) - break - default: - throw new Error(`Unsupported fromDepth: ${fromDepth}`) - } - - files.push({ oldPath, id, type: 'noun' }) - } - - // Discover verb files - const verbFiles = await this.getAllFilesAtDepth(this.verbsDir, fromDepth) - for (const filename of verbFiles) { - const id = filename.replace('.json', '') - - // Construct correct oldPath based on fromDepth - let oldPath: string - switch (fromDepth) { - case 0: - // Flat: verbs/uuid.json - oldPath = path.join(this.verbsDir, `${id}.json`) - break - case 1: - // Single-level: verbs/ab/uuid.json - oldPath = path.join(this.verbsDir, id.substring(0, 2), `${id}.json`) - break - case 2: - // Deep: verbs/ab/cd/uuid.json - oldPath = path.join(this.verbsDir, id.substring(0, 2), id.substring(2, 4), `${id}.json`) - break - default: - throw new Error(`Unsupported fromDepth: ${fromDepth}`) - } - - files.push({ oldPath, id, type: 'verb' }) - } - - return files - } - - /** - * Get all files at a specific depth - */ - private async getAllFilesAtDepth(baseDir: string, depth: number): Promise { - const allFiles: string[] = [] - - try { - const dirExists = await this.directoryExists(baseDir) - if (!dirExists) { - return [] - } - - switch (depth) { - case 0: - // Flat: files directly in baseDir - const entries = await fs.promises.readdir(baseDir) - for (const entry of entries) { - if (entry.endsWith('.json')) { - allFiles.push(entry) - } - } - break - - case 1: - // Single-level: baseDir/ab/uuid.json - const shardDirs = await fs.promises.readdir(baseDir) - for (const shard of shardDirs) { - const shardPath = path.join(baseDir, shard) - try { - const stat = await fs.promises.stat(shardPath) - if (stat.isDirectory()) { - const shardFiles = await fs.promises.readdir(shardPath) - for (const file of shardFiles) { - if (file.endsWith('.json')) { - allFiles.push(file) - } - } - } - } catch (error) { - // Skip inaccessible directories - } - } - break - - case 2: - // Deep: baseDir/ab/cd/uuid.json - const level1Dirs = await fs.promises.readdir(baseDir) - for (const level1 of level1Dirs) { - const level1Path = path.join(baseDir, level1) - try { - const level1Stat = await fs.promises.stat(level1Path) - if (level1Stat.isDirectory()) { - const level2Dirs = await fs.promises.readdir(level1Path) - for (const level2 of level2Dirs) { - const level2Path = path.join(level1Path, level2) - try { - const level2Stat = await fs.promises.stat(level2Path) - if (level2Stat.isDirectory()) { - const files = await fs.promises.readdir(level2Path) - for (const file of files) { - if (file.endsWith('.json')) { - allFiles.push(file) - } - } - } - } catch (error) { - // Skip inaccessible directories - } - } - } - } catch (error) { - // Skip inaccessible directories - } - } - break - } - } catch (error) { - // Directory doesn't exist or not accessible - } - - return allFiles - } - - /** - * Create all 256 shard directories (00-ff) - */ - private async createAllShardDirectories(baseDir: string): Promise { - for (let i = 0; i < this.MAX_SHARDS; i++) { - const shard = i.toString(16).padStart(2, '0') - const shardDir = path.join(baseDir, shard) - await this.ensureDirectoryExists(shardDir) - } - } /** * Migrate a single file atomically @@ -2597,403 +2616,41 @@ export class FileSystemStorage extends BaseStorage { await fs.promises.rename(oldPath, newPath) } + + + /** - * Clean up empty directories after migration + * Whether this store already holds canonical 8.0 entities. + * + * 8.0 writes nouns to `entities/nouns///vectors.json` (see + * `getNounVectorPath`). This checks that canonical shard tree — the one the + * DB actually reads and writes (the same `entities/nouns/` shards + * `getNounsWithPagination` walks) — so the new-vs-established boot log is + * truthful. (The 7.x hnsw sharding probe this replaced inspected a directory + * the 8.0 write path never populated, so it mislabeled every established + * store "New installation" on every boot; that probe and its depth-migration + * machinery are removed.) + * + * @returns true if at least one 2-hex shard directory (00–ff) exists under + * `entities/nouns/`, i.e. the store has previously persisted entities. */ - private async cleanupEmptyDirectories(baseDir: string, depth: number): Promise { + private async hasCanonicalEntities(): Promise { + const canonicalNounsDir = path.join(this.rootDir, 'entities', 'nouns') try { - if (depth === 2) { - // Clean up level2 and level1 directories - const level1Dirs = await fs.promises.readdir(baseDir) - for (const level1 of level1Dirs) { - const level1Path = path.join(baseDir, level1) - try { - const level1Stat = await fs.promises.stat(level1Path) - if (level1Stat.isDirectory()) { - const level2Dirs = await fs.promises.readdir(level1Path) - for (const level2 of level2Dirs) { - const level2Path = path.join(level1Path, level2) - try { - // Try to remove level2 directory (will fail if not empty) - await fs.promises.rmdir(level2Path) - } catch (error) { - // Directory not empty or other error - ignore - } - } - - // Try to remove level1 directory - await fs.promises.rmdir(level1Path) - } - } catch (error) { - // Directory not empty or other error - ignore - } - } - } - } catch (error) { - // Cleanup is best-effort, don't throw - } - } - - /** - * Count files in the current structure - */ - private async countFilesInStructure(depth: number): Promise { - let count = 0 - - count += (await this.getAllFilesAtDepth(this.nounsDir, depth)).length - count += (await this.getAllFilesAtDepth(this.verbsDir, depth)).length - - return count - } - - /** - * Detect the actual sharding depth used by existing files - * Examines directory structure to determine current sharding strategy - * Returns null if no files exist yet (new installation) - */ - private async detectExistingShardingDepth(): Promise { - try { - // Check if nouns directory exists and has content - const dirExists = await this.directoryExists(this.nounsDir) - if (!dirExists) { - return null // New installation - } - - const entries = await fs.promises.readdir(this.nounsDir, { withFileTypes: true }) - - // Check if there are any .json files directly in nounsDir (flat structure) - const hasDirectJsonFiles = entries.some((e: any) => e.isFile() && e.name.endsWith('.json')) - if (hasDirectJsonFiles) { - return 0 // Flat structure: nouns/uuid.json - } - - // Check for subdirectories with hex names (sharding directories) - const subdirs = entries.filter((e: any) => e.isDirectory() && /^[0-9a-f]{2}$/i.test(e.name)) - if (subdirs.length === 0) { - return null // No files yet - } - - // Check first subdir to see if it has files or more subdirs - const firstSubdir = subdirs[0].name - const subdirPath = path.join(this.nounsDir, firstSubdir) - const subdirEntries = await fs.promises.readdir(subdirPath, { withFileTypes: true }) - - const hasJsonFiles = subdirEntries.some((e: any) => e.isFile() && e.name.endsWith('.json')) - if (hasJsonFiles) { - return 1 // Single-level sharding: nouns/ab/uuid.json - } - - const hasSubSubdirs = subdirEntries.some((e: any) => e.isDirectory() && /^[0-9a-f]{2}$/i.test(e.name)) - if (hasSubSubdirs) { - return 2 // Deep sharding: nouns/ab/cd/uuid.json - } - - return 1 // Default to single-level if structure is unclear - } catch (error) { - // If we can't read the directory, assume new installation - return null - } - } - - /** - * Get sharding depth - * Always returns 1 (single-level sharding) for optimal balance of - * simplicity, performance, and reliability across all dataset sizes - * - * Single-level sharding (depth=1): - * - 256 shard directories (00-ff) - * - Handles 2.5M+ entities with excellent performance - * - No dynamic depth changes = no path mismatch bugs - * - Industry standard approach (Git uses similar) - */ - private getOptimalShardingDepth(): number { - return this.SHARDING_DEPTH - } - - /** - * Get the path for a node with consistent sharding strategy - * Clean, predictable path generation - */ - private getNodePath(id: string): string { - return this.getShardedPath(this.nounsDir, id) - } - - /** - * Get the path for a verb with consistent sharding strategy - */ - private getVerbPath(id: string): string { - return this.getShardedPath(this.verbsDir, id) - } - - /** - * Universal sharded path generator - * Always uses depth=1 (single-level sharding) for consistency - * - * Format: baseDir/ab/uuid.json - * Where 'ab' = first 2 hex characters of UUID (lowercase) - * - * Validates UUID format and throws descriptive errors - */ - private getShardedPath(baseDir: string, id: string): string { - // Extract first 2 characters for shard directory - const shard = id.substring(0, 2).toLowerCase() - - // Validate shard is valid hex (00-ff) - if (!/^[0-9a-f]{2}$/.test(shard)) { - throw new Error( - `Invalid entity ID format: ${id}. ` + - `Expected UUID starting with 2 hex characters, got '${shard}'. ` + - `IDs must be UUIDs or hex strings.` + const entries = await fs.promises.readdir(canonicalNounsDir, { + withFileTypes: true + }) + // A populated store has ≥1 two-hex shard dir (00–ff). The vestigial + // `hnsw` subdir is 4 chars and correctly excluded by the hex test. + return entries.some( + (e: any) => e.isDirectory() && /^[0-9a-f]{2}$/i.test(e.name) ) - } - - // Single-level sharding: baseDir/ab/uuid.json - return path.join(baseDir, shard, `${id}.json`) - } - - /** - * Get all JSON files from the single-level sharded directory structure - * Traverses all shard subdirectories (00-ff) - */ - private async getAllShardedFiles(baseDir: string): Promise { - const allFiles: string[] = [] - - try { - const shardDirs = await fs.promises.readdir(baseDir) - - for (const shardDir of shardDirs) { - const shardPath = path.join(baseDir, shardDir) - - try { - const stat = await fs.promises.stat(shardPath) - - if (stat.isDirectory()) { - const shardFiles = await fs.promises.readdir(shardPath) - for (const file of shardFiles) { - if (file.endsWith('.json')) { - allFiles.push(file) - } - } - } - } catch (shardError) { - // Skip inaccessible shard directories - continue - } - } - - // Sort for consistent ordering - allFiles.sort() - return allFiles - - } catch (error: any) { - if (error.code === 'ENOENT') { - // Directory doesn't exist yet - return [] - } - throw error + } catch { + // Directory absent (fresh store) or unreadable → no persisted entities. + return false } } - /** - * Production-scale streaming pagination for very large datasets - * Avoids loading all filenames into memory - */ - private async getVerbsWithPaginationStreaming( - options: { - limit?: number - cursor?: string - filter?: { - verbType?: string | string[] - sourceId?: string | string[] - targetId?: string | string[] - service?: string | string[] - metadata?: Record - } - }, - startIndex: number, - limit: number - ): Promise<{ - items: HNSWVerbWithMetadata[] - totalCount?: number - hasMore: boolean - nextCursor?: string - }> { - const verbs: HNSWVerbWithMetadata[] = [] - let processedCount = 0 - let skippedCount = 0 - let resultCount = 0 - - const depth = this.cachedShardingDepth ?? this.getOptimalShardingDepth() - - try { - // Stream through sharded directories efficiently - // hasMore=false means we reached the end of files, hasMore=true means streaming stopped early - const streamingHasMore = await this.streamShardedFiles( - this.verbsDir, - depth, - async (filename: string, filePath: string) => { - // Skip files until we reach start index - if (skippedCount < startIndex) { - skippedCount++ - return true // continue - } - - // Stop if we have enough results - if (resultCount >= limit) { - return false // stop streaming - more files exist - } - - try { - const id = filename.replace('.json', '') - - // Read verb data and metadata - const data = await fs.promises.readFile(filePath, 'utf-8') - const edge = JSON.parse(data) - const metadata = await this.getVerbMetadata(id) - - // Don't skip verbs without metadata - metadata is optional - // FIX: This was the root cause of the VFS bug (11 versions) - // Verbs can exist without metadata files (e.g., from imports/migrations) - - // Convert connections if needed - let connections = edge.connections - if (connections && typeof connections === 'object' && !(connections instanceof Map)) { - const connectionsMap = new Map>() - for (const [level, nodeIds] of Object.entries(connections)) { - connectionsMap.set(Number(level), new Set(nodeIds as string[])) - } - connections = connectionsMap - } - - // Canonical hydration (single source of truth: - // src/types/reservedFields.ts) — reserved fields top-level, ONLY - // custom fields in `metadata`. The previous hand-rolled - // destructure here missed `subtype` (so streamed verbs lost it) - // and `verb` (so the type key echoed inside custom metadata). - const verbWithMetadata: HNSWVerbWithMetadata = this.hydrateVerbWithMetadata( - { - id: edge.id, - vector: edge.vector, - connections: connections || new Map(), - verb: edge.verb, - sourceId: edge.sourceId, - targetId: edge.targetId - }, - metadata - ) - - // Apply filters - if (options.filter) { - const filter = options.filter - - if (filter.verbType) { - const types = Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType] - if (!types.includes(verbWithMetadata.verb)) return true // continue - } - - if (filter.sourceId) { - const sources = Array.isArray(filter.sourceId) ? filter.sourceId : [filter.sourceId] - if (!sources.includes(verbWithMetadata.sourceId)) return true // continue - } - - if (filter.targetId) { - const targets = Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId] - if (!targets.includes(verbWithMetadata.targetId)) return true // continue - } - } - - verbs.push(verbWithMetadata) - resultCount++ - processedCount++ - return true // continue - - } catch (error) { - console.warn(`Failed to read verb from ${filePath}:`, error) - processedCount++ - return true // continue - } - } - ) - - // CRITICAL FIX: Use streaming result for hasMore, not cached totalVerbCount - // streamingHasMore=false means we exhausted all files - // Also verify we loaded items to prevent infinite loops - const finalHasMore = streamingHasMore && (resultCount > 0 || startIndex === 0) - - return { - items: verbs, - totalCount: this.totalVerbCount || undefined, // Return cached count as hint only - hasMore: finalHasMore, - nextCursor: finalHasMore ? String(startIndex + resultCount) : undefined - } - - } catch (error: any) { - if (error.code === 'ENOENT') { - return { - items: [], - totalCount: 0, - hasMore: false - } - } - throw error - } - } - - /** - * Stream through sharded files without loading all names into memory - * Production-scale implementation for millions of files - */ - /** - * Stream through files in single-level sharded structure - * Calls processor for each file until processor returns false - * Returns true if more files exist (processor stopped early), false if all processed - */ - private async streamShardedFiles( - baseDir: string, - depth: number, - processor: (filename: string, fullPath: string) => Promise - ): Promise { - let hasMore = true - - // Single-level sharding (depth=1): baseDir/ab/uuid.json - try { - const shardDirs = await fs.promises.readdir(baseDir) - const sortedShardDirs = shardDirs.sort() - - for (const shardDir of sortedShardDirs) { - const shardPath = path.join(baseDir, shardDir) - - try { - const stat = await fs.promises.stat(shardPath) - - if (stat.isDirectory()) { - const files = await fs.promises.readdir(shardPath) - const sortedFiles = files.filter((f: string) => f.endsWith('.json')).sort() - - for (const file of sortedFiles) { - const shouldContinue = await processor(file, path.join(shardPath, file)) - - if (!shouldContinue) { - hasMore = false - break - } - } - - if (!hasMore) break - } - } catch (shardError) { - // Skip inaccessible shard directories - continue - } - } - } catch (error: any) { - if (error.code === 'ENOENT') { - hasMore = false - } - } - - return hasMore - } /** * Check if a file exists (handles both sharded and non-sharded) diff --git a/src/storage/adapters/memoryStorage.ts b/src/storage/adapters/memoryStorage.ts index 9f6410cf..1b1f412e 100644 --- a/src/storage/adapters/memoryStorage.ts +++ b/src/storage/adapters/memoryStorage.ts @@ -69,7 +69,7 @@ export class MemoryStorage extends BaseStorage { * * @returns Memory-optimized batch configuration */ - public getBatchConfig(): StorageBatchConfig { + public override getBatchConfig(): StorageBatchConfig { return { maxBatchSize: 1000, batchDelayMs: 0, @@ -86,7 +86,7 @@ export class MemoryStorage extends BaseStorage { * Initialize the storage adapter * Calls super.init() to initialize GraphAdjacencyIndex and type statistics */ - public async init(): Promise { + public override async init(): Promise { await super.init() } @@ -133,6 +133,11 @@ export class MemoryStorage extends BaseStorage { */ protected async deleteObjectFromPath(path: string): Promise { this.objectStore.delete(path) + // Filesystem parity: on disk, objects and raw BYTE files are both just + // files — unlink removes whichever exists. Without this, deleteRawObject + // on a raw-bytes path (fact-log/generation segments) silently no-ops on + // memory storage: the delete "succeeds" and the bytes remain. + this.rawBytesStore.delete(path) } /** @@ -183,6 +188,9 @@ export class MemoryStorage extends BaseStorage { * @param key - The blob key. */ public async deleteBinaryBlob(key: string): Promise { + // Registered-blob contract — parity with the filesystem adapter: + // a declared family member is undeletable (throws ProtectedArtifactError). + await this.assertBlobKeyDeletable(key) this.blobStore.delete(key) } @@ -219,6 +227,45 @@ export class MemoryStorage extends BaseStorage { return [...this.txLogLines] } + // =========================================================================== + // Binary raw-byte primitives — in-memory mirror of the filesystem adapter's + // append-only substrate (the generation fact log's segments), so memory + // brains dual-write facts too and the compat suite runs on both adapters. + // =========================================================================== + + /** Raw binary files, keyed by verbatim path. */ + private rawBytesStore: Map = new Map() + + /** Append bytes to a raw binary file, creating it when absent. */ + public async appendRawBytes(rawPath: string, bytes: Uint8Array): Promise { + const existing = this.rawBytesStore.get(rawPath) + if (!existing) { + this.rawBytesStore.set(rawPath, bytes.slice()) + return + } + const merged = new Uint8Array(existing.length + bytes.length) + merged.set(existing, 0) + merged.set(bytes, existing.length) + this.rawBytesStore.set(rawPath, merged) + } + + /** Read a raw binary file whole (a copy); absent → null. */ + public async readRawBytes(rawPath: string): Promise { + const bytes = this.rawBytesStore.get(rawPath) + return bytes ? bytes.slice() : null + } + + /** Replace a raw binary file (atomic by construction in memory). */ + public async writeRawBytes(rawPath: string, bytes: Uint8Array): Promise { + this.rawBytesStore.set(rawPath, bytes.slice()) + } + + /** Byte size of a raw binary file, or null when absent. */ + public async rawByteSize(rawPath: string): Promise { + const bytes = this.rawBytesStore.get(rawPath) + return bytes ? bytes.length : null + } + /** * Serialize the entire in-memory store to a directory in the exact layout * the filesystem adapter uses (uncompressed JSON objects, `_blobs/*.bin` @@ -351,6 +398,7 @@ export class MemoryStorage extends BaseStorage { public async clear(): Promise { this.objectStore.clear() this.blobStore.clear() + this.rawBytesStore.clear() this.txLogLines = [] this.statistics = null diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index 88fb17e6..6daf09c0 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -5,6 +5,7 @@ import { GraphAdjacencyIndex } from '../graph/graphAdjacencyIndex.js' import type { GraphEntityIdResolver } from '../graph/graphAdjacencyIndex.js' +import { assessIndexReadiness } from '../utils/indexReadiness.js' import { GraphVerb, @@ -14,9 +15,9 @@ import { VerbMetadata, HNSWNounWithMetadata, HNSWVerbWithMetadata, - StatisticsData + StatisticsData, + DerivedFamilyDeclaration } from '../coreTypes.js' -import type { EntityVisibility } from '../coreTypes.js' import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js' import { validateNounType, validateVerbType } from '../utils/typeValidation.js' import { @@ -30,6 +31,8 @@ import { getShardId } from './sharding.js' import { BlobStorage, type BlobStoreAdapter } from './blobStorage.js' import { unwrapBinaryData } from './binaryDataCodec.js' import { prodLog } from '../utils/logger.js' +import { isAbsentError } from '../utils/errorClassification.js' +import { BrainyError, ProtectedArtifactError, DerivedArtifactMissingError } from '../errors/brainyError.js' import { MetadataWriteBuffer } from '../utils/metadataWriteBuffer.js' import { splitNounMetadataRecord, @@ -184,6 +187,20 @@ function getVerbVectorPath(id: string): string { return `entities/verbs/${shard}/${id}/vectors.json` } +/** + * @description Extract the entity id embedded in a vector path + * (`entities/{nouns|verbs}/{shard}/{id}/vectors.json`). Used by the cursored + * noun/verb walks to order and skip candidates by id WITHOUT reading each file, + * which is what keeps a full cursored pagination O(N) instead of O(N²). + * @param path - A vector path (full or prefix-relative; must end with `/vectors.json`). + * @returns The entity id (the path segment immediately before `/vectors.json`). + */ +function idFromVectorPath(path: string): string { + const withoutSuffix = path.replace(/\/vectors\.json$/, '') + const lastSlash = withoutSuffix.lastIndexOf('/') + return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix +} + /** * Get ID-first path for verb metadata * No type parameter needed - direct O(1) lookup by ID @@ -193,18 +210,6 @@ function getVerbMetadataPath(id: string): string { return `entities/verbs/${shard}/${id}/metadata.json` } -/** - * Extract the entity id from an ID-first metadata path. Both noun and verb - * metadata live at `entities////metadata.json`, so the id is - * the second-to-last path segment. - * @param path - A metadata.json path produced by `getNounMetadataPath` / `getVerbMetadataPath`. - * @returns The id segment, or `undefined` if the path doesn't have the expected shape. - */ -function idFromMetadataPath(path: string): string | undefined { - const segments = path.split('/') - return segments[segments.length - 2] -} - /** * Optional count capabilities probed via duck typing by getNouns()/getVerbs(). * Adapters with a native O(1) count API may implement these; they are not part @@ -245,6 +250,27 @@ function isCountedVisibility(visibility: unknown): boolean { */ export abstract class BaseStorage extends BaseStorageAdapter { protected isInitialized = false + /** One-shot guard so the graph fast-path cold-load probe runs once per adapter. */ + private _graphFastPathProbed = false + /** + * Registered-blob contract: declared derived-index families, keyed + * by name. Members are undeletable through the blob delete seams. Loaded lazily + * from `_system/derived-artifacts.json` and re-persisted on every change. + */ + private _derivedFamilies = new Map() + /** One-shot guard for loading the persisted family registry. */ + private _derivedFamiliesLoaded = false + /** Storage-root-relative path of the persisted family registry. */ + private static readonly DERIVED_FAMILIES_KEY = '_system/derived-artifacts.json' + /** + * Bounded concurrency for hydrating enumerated nouns during a pagination walk + * (readCanonicalObject + getNounMetadata per item). A canonical enumeration — + * which every index heal performs — otherwise pays N×per-op-latency serially; + * 16-way matches the wave the native rebuild uses so heal wall-clock is + * ~2×N/16×per-op instead of N×per-op. Bounded so a huge dataset can't spawn a + * read per entity at once. + */ + private static readonly HYDRATE_CONCURRENCY = 16 protected graphIndex?: GraphAdjacencyIndex protected graphIndexPromise?: Promise /** @@ -307,54 +333,16 @@ export abstract class BaseStorage extends BaseStorageAdapter { */ protected verbSubtypeCountsByType = new Map>() - /** - * In-memory map from noun id → its `noun` (NounType) value, populated when - * `saveNounMetadata_internal()` runs. `saveNoun_internal()` consults this - * to attribute the entity to the correct slot in `nounCountsByType` so the - * persisted `_system/type-statistics.json` is honest. - * - * History: a previous cache was removed in commit `42ae5be` and replaced - * with a hardcoded `return 'thing'` from `getNounType()`, which silently - * poisoned the on-disk type statistics. This cache restores the correct - * behavior. Memory footprint is one Map entry per live noun id (typically - * tens of bytes each); the writer process keeps it for the duration of - * its lifetime and prunes on delete. - */ - protected nounTypeByIdCache = new Map() - - /** - * In-memory map from noun id → its `subtype` string (when set). Parallel to - * `nounTypeByIdCache`. Lets `deleteNounMetadata()` decrement the correct - * subtype bucket without re-reading metadata. Sparse — only ids with a - * non-empty subtype get an entry. - */ - protected nounSubtypeByIdCache = new Map() - - /** - * In-memory map from verb id → `{ verb, subtype }` pair. Verb-side mirror of - * `nounTypeByIdCache` + `nounSubtypeByIdCache`. Lets `deleteVerbMetadata` and - * `updateRelation` decrement the right bucket without a re-read of metadata. - * Sparse — only ids with a non-empty subtype get an entry. - */ - protected verbSubtypeByIdCache = new Map() - // Total: 676 bytes (99.2% reduction vs Map-based tracking) - - /** - * In-memory map from noun id → its non-public `visibility` tier ('internal' | - * 'system'). Parallel to `nounTypeByIdCache`. Lets `saveNoun_internal()` - * (which has no metadata access in its signature) skip the user-facing - * `nounCountsByType` increment for hidden entities, and lets - * `deleteNounMetadata()` skip the matching decrement — keeping the counts - * symmetric. Sparse by design: public entities (the common case) get NO - * entry, so the absence of an id means "public, counted". - */ - protected nounVisibilityByIdCache = new Map() - - /** - * In-memory map from verb id → its non-public `visibility` tier. Verb-side - * mirror of `nounVisibilityByIdCache`. Sparse — public edges get no entry. - */ - protected verbVisibilityByIdCache = new Map() + // Count attribution (type / subtype / visibility) is sourced directly from the + // canonical metadata RECORD, never from id-keyed in-memory caches. The metadata + // save/delete paths already read the prior record (`existingMetadata` on write, + // read-before-delete on remove), so the entity's `noun`/`verb` type, `subtype`, + // and `visibility` are in hand exactly where a count must change — there is no + // need for a parallel O(N) `id → type/subtype/visibility` map resident on the + // writer. The five such caches that used to live here were removed in the 8.0 + // billion-scale RAM pass; `nounCountsByType` / `verbCountsByType` (fixed + // type-indexed Uint32Arrays) and `subtypeCountsByType` / `verbSubtypeCountsByType` + // (bounded by distinct subtype labels) remain because they are NOT id-keyed. // Type caches REMOVED - ID-first paths eliminate need for type lookups! // With ID-first architecture, we construct paths directly from IDs: {SHARD}/{ID}/metadata.json @@ -389,7 +377,14 @@ export abstract class BaseStorage extends BaseStorageAdapter { id.startsWith('statistics_') || id === 'statistics' || id.startsWith('__chunk__') || // Metadata index chunks (roaring bitmap data) - id.startsWith('__sparse_index__') // Metadata sparse indices (zone maps + bloom filters) + id.startsWith('__sparse_index__') || // Metadata sparse indices (zone maps + bloom filters) + id.startsWith('__aggregation_') || // Aggregation engine definitions + state (routing is + // identical to the unknown-key fallback these keys hit + // before being listed here — this only kills the + // per-boot "Unknown key format" warning) + isSingletonSystemKey(id) // Known singletons (e.g. brainy:entityIdMapper) hit the + // same warn-then-route fallback without this — the + // routing below already handles them identically if (isSystemKey) { if (isSingletonSystemKey(id)) { @@ -716,6 +711,91 @@ export abstract class BaseStorage extends BaseStorageAdapter { this.blobStorage = new BlobStorage(casAdapter) } + /** + * @description Adopt orphaned 7.x copy-on-write VFS content blobs into the 8.0 + * content-addressed store. 7.x kept VFS blobs (`blob:` bytes + + * `blob-meta:` JSON) under the copy-on-write area (`_cow/`) of the + * branch/versioning system that 8.0 removed. The 7→8 layout migration + * collapses entity files but historically did NOT bridge these blobs, so a VFS + * read of a still-in-`_cow/` blob throws "Blob metadata not found" and every + * page backed by it 500s. + * + * This scans `_cow/` and, for every blob whose `blob:`/`blob-meta:` pair is not + * already present in the 8.0 store (`_cas/`), copies BOTH objects across + * verbatim. Copying preserves the exact content bytes (so the content hash and + * every VFS reference still resolve) and the exact metadata (so the 7.x + * refCount is retained) — the only first-party, index-consistent recovery. + * Hand-moving files at the OS level bypasses the paired-metadata contract and + * is unsafe; this goes through the adapter's object primitives. + * + * Idempotent and non-destructive: a pair already in `_cas/` is left untouched + * (counted `alreadyPresent`); the `_cow/` originals are never deleted, so a + * re-run — or a rollback — is always possible. A no-op on memory storage and + * on any brain without a `_cow/` area (returns zeros). Bytes are written before + * metadata so an interruption can only leave the pre-adoption "no metadata" + * state (retryable), never a metadata-without-bytes blob. + * + * @returns `cowBlobs` (distinct blob hashes found in `_cow/`), `adopted` + * (newly copied into `_cas/`), `alreadyPresent` (already in `_cas/`), + * `incomplete` (a `_cow/` blob missing its bytes or its metadata — skipped + * and reported rather than half-adopted). + */ + public async adoptLegacyCowBlobs(): Promise<{ + cowBlobs: number + adopted: number + alreadyPresent: number + incomplete: number + }> { + let cowPaths: string[] + try { + cowPaths = await this.listObjectsUnderPath('_cow/') + } catch { + // No `_cow/` area (fresh/native-8.0 brain, or a non-listing backend). + return { cowBlobs: 0, adopted: 0, alreadyPresent: 0, incomplete: 0 } + } + + // Distinct content-blob hashes from `_cow/blob:` keys. Non-blob `_cow/` + // entries (7.x branch COW state) are ignored — only VFS content is adopted. + const hashes = new Set() + for (const p of cowPaths) { + const key = p.replace(/^_cow\//, '') + const m = /^blob:(.+)$/.exec(key) + if (m) hashes.add(m[1]) + } + + let adopted = 0 + let alreadyPresent = 0 + let incomplete = 0 + for (const hash of hashes) { + // A blob counts as present only when BOTH its bytes and its metadata + // already live in `_cas/`. A half-adopted blob (bytes without meta — the + // exact "Blob metadata not found" state) is re-adopted. + const casBlob = await this.readObjectFromPath(`_cas/blob:${hash}`) + const casMeta = await this.readObjectFromPath(`_cas/blob-meta:${hash}`) + if (casBlob !== null && casMeta !== null) { + alreadyPresent++ + continue + } + + const cowBlob = await this.readObjectFromPath(`_cow/blob:${hash}`) + const cowMeta = await this.readObjectFromPath(`_cow/blob-meta:${hash}`) + if (cowBlob === null || cowMeta === null) { + // Can't register a blob the store can't fully describe — report it so an + // operator investigates rather than silently half-adopting. + incomplete++ + continue + } + + // Bytes first, then metadata: a VFS read checks metadata before bytes, so + // metadata's presence must imply the bytes are already there. + await this.writeObjectToPath(`_cas/blob:${hash}`, cowBlob) + await this.writeObjectToPath(`_cas/blob-meta:${hash}`, cowMeta) + adopted++ + } + + return { cowBlobs: hashes.size, adopted, alreadyPresent, incomplete } + } + /** * @description Write a canonical object (write-cache coherent). The object * lands in the write-through cache before the asynchronous write starts, so @@ -769,6 +849,82 @@ export abstract class BaseStorage extends BaseStorageAdapter { return this.deleteObjectFromPath(path) } + // ========================================================================== + // Fact-scan capability — the seam through which an index provider holding + // only `storage` reaches the generation fact log. The HOST brain wires the + // source at init (a closure over its live fact log, so restore/reopen stays + // transparent); providers call storage.scanFacts?.(...) and fall back to the + // enumeration walk when it returns null. Providers never construct a + // fact-log reader themselves — the log's open path is writer-side. + // ========================================================================== + + /** The host-wired fact-scan source (closures over the live generation state). */ + private _factScanSource: { + factLog: () => import('../db/factLog.js').FactLog | null + committedGeneration: () => number + } | null = null + + /** HOST-ONLY: wire (or clear) the fact-scan capability's source. */ + public setFactScanSource( + source: { + factLog: () => import('../db/factLog.js').FactLog | null + committedGeneration: () => number + } | null + ): void { + this._factScanSource = source + } + + /** Open a scan over committed facts, or `null` when no fact log exists. */ + public scanFacts(options?: { + fromGeneration?: number + toGeneration?: number + kinds?: Array<'noun' | 'verb'> + batchSize?: number + }): import('../db/factLog.js').FactScanHandle | null { + const log = this._factScanSource?.factLog() + return log ? log.scanFacts(options) : null + } + + /** The fact log's head generation, or `null` when no fact log exists. */ + public factLogHeadGeneration(): number | null { + const log = this._factScanSource?.factLog() + return log ? log.headGeneration() : null + } + + /** + * The COMMITTED generation watermark (the manifest truth) — the value a + * projection's `sourceGeneration` compares against. Exposed here so a + * provider never parses the store's private manifest format. `null` when + * the capability is unwired (no host, or a bare adapter). + */ + public committedGeneration(): number | null { + return this._factScanSource ? this._factScanSource.committedGeneration() : null + } + + /** Sealed fact-segment paths (zero-copy handoff); empty when none. */ + public factSegmentPaths(options?: { fromGeneration?: number }): string[] { + const log = this._factScanSource?.factLog() + return log ? log.segmentPaths(options) : [] + } + + /** + * @description Remove the container that held a canonical entity's leg files, + * called after both legs are deleted so a delete leaves NOTHING behind — no + * orphan "scar" directory. `objectLegPath` is any one of the entity's leg + * paths (e.g. its `vectors.json`); the container is that path's parent. + * + * Default: a no-op. Key/prefix-addressed stores (in-memory, cloud object + * stores) have no empty-container concept — deleting the leg keys already + * removes the entity entirely. The filesystem adapter overrides this to + * `rmdir` the emptied entity directory. + * + * @param objectLegPath - Storage-root-relative path of one of the entity's legs. + * @protected + */ + protected async removeCanonicalContainer(objectLegPath: string): Promise { + void objectLegPath + } + /** * @description List canonical objects under a storage-root-relative prefix. * @@ -820,6 +976,148 @@ export abstract class BaseStorage extends BaseStorageAdapter { this.generationBumpHook = hook } + // ========================================================================== + // Temporal-blob contract (the GenerationStorage optional methods) + // + // Content blobs join the Model-B immutability model through these hooks: + // the generation store counts a history reference per before-image record + // that carries a content hash, and compaction — the ONE reclamation point — + // releases those references and physically deletes bytes only at zero live + // AND zero history references. Crash ordering is over-count-only (record + // BEFORE the record-set persists, release AFTER it is deleted), so a crash + // can leak bytes until the scrub recounts but can never reclaim bytes a + // retained generation still needs. + // ========================================================================== + + /** Set when the open-time backfill/scrub could not verify history reference + * counts. While true, the temporal-blob hooks stop mutating counts and + * compaction stops reclaiming blob bytes — pure leak-safe mode until a + * successful {@link scrubBlobHistoryRefCounts} restores exactness. */ + private blobHistoryRefsUnverified = false + + /** + * @description Extract the content-blob hashes a generation record-set + * references — a pure MULTISET extraction (one entry per referencing record + * occurrence), no side effects. Only entity records can reference VFS + * content (`metadata.storage.type === 'blob'`). + * @param records - The record-set's before-image records. + * @returns The referenced hashes, duplicates preserved. + */ + public extractBlobHashesFromRecords( + records: Array<{ kind: string; metadata: unknown }> + ): string[] { + const hashes: string[] = [] + for (const record of records) { + if (record.kind !== 'noun') continue + const storage = (record.metadata as { storage?: { type?: string; hash?: unknown } } | null) + ?.storage + if (storage?.type === 'blob' && typeof storage.hash === 'string') { + hashes.push(storage.hash) + } + } + return hashes + } + + /** + * @description Record one history reference per hash occurrence (see the + * contract note above — called BEFORE the referencing record-set persists). + * No-op without a blob store or while counts are unverified. + * @param hashes - Hash multiset from {@link extractBlobHashesFromRecords}. + */ + public async recordHistoryBlobReferences(hashes: string[]): Promise { + if (!this.blobStorage || this.blobHistoryRefsUnverified || hashes.length === 0) return + for (const hash of hashes) { + await this.blobStorage.recordHistoryReference(hash) + } + } + + /** + * @description Release one history reference per hash occurrence and + * physically reclaim any hash left with zero live AND zero history + * references — compaction's blob-reclamation step (called AFTER the + * referencing record-set is deleted). No-op without a blob store or while + * counts are unverified (leak-safe: nothing is reclaimed on guesses). + * @param hashes - Hash multiset recorded when the record-set was persisted. + */ + public async releaseHistoryBlobReferences(hashes: string[]): Promise { + if (!this.blobStorage || this.blobHistoryRefsUnverified || hashes.length === 0) return + for (const hash of hashes) { + await this.blobStorage.releaseHistoryReference(hash) + } + for (const hash of new Set(hashes)) { + await this.blobStorage.reclaimIfUnreferenced(hash) + } + } + + /** + * @description One-time (marker-gated) backfill of blob history reference + * counts for stores whose generation history predates the temporal-blob + * contract. Runs the scrub, then stamps `_system/blob-history-refs.json` so + * later opens skip the walk. On scrub failure the store enters leak-safe + * mode (counts untouched, reclamation disabled) rather than risking a + * premature delete on wrong counts. + */ + public async backfillBlobHistoryRefCountsIfNeeded(): Promise { + if (!this.blobStorage) return + const MARKER = '_system/blob-history-refs.json' + try { + const marker = (await this.readObjectFromPath(MARKER)) as { version?: number } | null + if (marker?.version === 1) return + } catch { + // no marker — proceed to scrub + } + try { + await this.scrubBlobHistoryRefCounts() + await this.writeObjectToPath(MARKER, { version: 1, verifiedAt: new Date().toISOString() }) + } catch (err) { + this.blobHistoryRefsUnverified = true + console.error( + '[Brainy] blob history-reference backfill failed — temporal-blob ' + + 'reclamation disabled for this session (leak-safe); history reads ' + + 'are unaffected. Re-open to retry.', + err + ) + } + } + + /** + * @description Recount every blob's history references from the actual + * generation record-sets and set the counts ABSOLUTELY (uncounted blobs are + * zeroed) — the idempotent repair that restores exactness after any crash + * that over-counted. O(history records + stored blobs). + * @returns Blobs counted and records walked, for observability. + */ + public async scrubBlobHistoryRefCounts(): Promise<{ blobs: number; records: number }> { + if (!this.blobStorage) return { blobs: 0, records: 0 } + const counts = new Map() + let records = 0 + let paths: string[] = [] + try { + paths = await this.listObjectsUnderPath('_generations') + } catch { + paths = [] // no history yet + } + for (const p of paths) { + if (!p.includes('/prev/')) continue + const record = (await this.readObjectFromPath(p)) as + | { kind?: string; metadata?: unknown } + | null + if (!record) continue + records++ + for (const hash of this.extractBlobHashesFromRecords([ + { kind: record.kind ?? '', metadata: record.metadata } + ])) { + counts.set(hash, (counts.get(hash) ?? 0) + 1) + } + } + const allHashes = await this.blobStorage.listHashes() + for (const hash of allHashes) { + await this.blobStorage.setHistoryRefCount(hash, counts.get(hash) ?? 0) + } + this.blobHistoryRefsUnverified = false + return { blobs: allHashes.length, records } + } + /** * Read a raw object at a storage-root-relative path. Bypasses the write * cache (record-layer files are written through @@ -846,6 +1144,154 @@ export abstract class BaseStorage extends BaseStorageAdapter { await this.writeObjectToPath(path, data) } + // ========================================================================== + // Registered-blob contract — declared derived-index families are + // undeletable through the blob delete seams. Shared here so every adapter that + // extends BaseStorage inherits the same enforcement; the concrete + // deleteBinaryBlob / removeRawPrefix call assertBlobKeyDeletable / + // assertPrefixNotProtected before removing anything. + // ========================================================================== + + /** Load the persisted family registry once (lazy). */ + private async ensureDerivedFamiliesLoaded(): Promise { + if (this._derivedFamiliesLoaded) return + const stored = await this.readRawObject(BaseStorage.DERIVED_FAMILIES_KEY).catch(() => null) + const families = (stored as { families?: DerivedFamilyDeclaration[] } | null)?.families + if (Array.isArray(families)) { + for (const f of families) { + if (f && typeof f.name === 'string' && Array.isArray(f.members)) this._derivedFamilies.set(f.name, f) + } + } + this._derivedFamiliesLoaded = true + } + + /** Persist the current family registry (fsync'd via the raw-object write). */ + private async persistDerivedFamilies(): Promise { + await this.writeRawObject(BaseStorage.DERIVED_FAMILIES_KEY, { + families: [...this._derivedFamilies.values()] + }) + } + + public async registerDerivedFamily(family: DerivedFamilyDeclaration): Promise { + await this.ensureInitialized() + await this.ensureDerivedFamiliesLoaded() + this._derivedFamilies.set(family.name, { + name: family.name, + members: [...family.members], + ...(family.namespace !== undefined ? { namespace: family.namespace } : {}), + ...(family.rebuildable !== undefined ? { rebuildable: family.rebuildable } : {}) + }) + await this.persistDerivedFamilies() + } + + public async unregisterDerivedFamily(name: string): Promise { + await this.ensureInitialized() + await this.ensureDerivedFamiliesLoaded() + if (this._derivedFamilies.delete(name)) await this.persistDerivedFamilies() + } + + public async listDerivedFamilies(): Promise { + await this.ensureInitialized() + await this.ensureDerivedFamiliesLoaded() + return [...this._derivedFamilies.values()] + } + + /** + * @description A blob key that is a transient write-scratch file (a `*.tmp.*` + * temp, a `*.rebuild-tmp`, a `*.rotate-tmp`). Its OWNER renames/removes it as + * part of an atomic write; it is never a protected family member. + */ + private isTransientBlobKey(key: string): boolean { + return /\.rebuild-tmp$|\.rotate-tmp$|\.tmp(\.|$)/.test(key) + } + + /** + * @description The name of the protected family a blob key belongs to, or + * `null`. A `namespace` member protects every key beneath it; a plain member + * protects that exact key. + */ + private protectingFamilyOf(key: string): string | null { + for (const family of this._derivedFamilies.values()) { + for (const member of family.members) { + if (family.namespace ? key === member || key.startsWith(member) : key === member) { + return family.name + } + } + } + return null + } + + /** + * @description Enforcement point for `deleteBinaryBlob`: refuse (throw + * {@link ProtectedArtifactError}) when the key is a declared family member. + * Transients pass through. An undeclared blob delete under an ACTIVE contract + * (families are registered) is logged loudly — nothing under `_blobs/` should + * vanish unremarked once the contract is in force. + */ + protected async assertBlobKeyDeletable(key: string): Promise { + await this.ensureDerivedFamiliesLoaded() + if (this._derivedFamilies.size === 0) return // contract inactive — no enforcement + if (this.isTransientBlobKey(key)) return + const family = this.protectingFamilyOf(key) + if (family) throw new ProtectedArtifactError(key, family) + prodLog.warn( + `[BaseStorage] deleteBinaryBlob('${key}') removes an UNDECLARED blob while the ` + + `registered-blob contract is active — permitted, but surfaced so no _blobs/ file ` + + `disappears silently.` + ) + } + + /** + * @description Enforcement point for `removeRawPrefix`: refuse when the prefix + * would take out a protected family member (a prefix-nuke must not remove a + * declared blob). Transients are ignored. + */ + protected async assertPrefixNotProtected(prefix: string): Promise { + await this.ensureDerivedFamiliesLoaded() + if (this._derivedFamilies.size === 0) return + for (const family of this._derivedFamilies.values()) { + for (const member of family.members) { + // Under the shared `_blobs/` root, member keys resolve beneath it; a + // prefix intersects a member when either contains the other. + const memberBlobPath = `_blobs/${member}` + if ( + memberBlobPath.startsWith(prefix) || + prefix.startsWith(memberBlobPath) || + member.startsWith(prefix) || + prefix.startsWith(member) + ) { + if (!this.isTransientBlobKey(member)) throw new ProtectedArtifactError(member, family.name) + } + } + } + } + + /** + * @description Verify every declared family has all its members present on + * disk (missing-on-open catch for an EXTERNAL deleter that bypasses + * the in-process refusal). Returns the incomplete families (name + missing + * members) — the caller decides how to heal (rebuild from canonical). Loud by + * construction: a missing load-bearing blob is named, never silently tolerated. + */ + public async checkDerivedFamiliesPresent(): Promise> { + await this.ensureInitialized() + await this.ensureDerivedFamiliesLoaded() + const incomplete: Array<{ name: string; missing: string[]; rebuildable: boolean }> = [] + for (const family of this._derivedFamilies.values()) { + if (family.namespace) continue // a growing prefix has no fixed member set to verify + const missing: string[] = [] + for (const member of family.members) { + const blob = await this.loadBinaryBlob(member).catch(() => null) + if (!blob) missing.push(member) + } + if (missing.length > 0) { + prodLog.warn(new DerivedArtifactMissingError(family.name, missing).message) + incomplete.push({ name: family.name, missing, rebuildable: family.rebuildable !== false }) + } + } + return incomplete + } + /** * Delete a raw object at a storage-root-relative path (no-op if absent). * @@ -1022,15 +1468,19 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Reset and reload every piece of adapter-internal derived state after the * underlying objects changed wholesale (restore-from-snapshot): the - * write-through cache, the id→type/subtype caches, type/subtype statistics, - * total counts, and the graph-index singleton (invalidated so the next - * accessor rebuilds from the restored verbs). + * write-through cache, type/subtype statistics, total counts, and the + * graph-index singleton (invalidated so the next accessor rebuilds from the + * restored verbs). Count attribution reads the canonical metadata record, so + * there are no id-keyed caches to clear here. */ protected async reloadDerivedState(): Promise { this.clearWriteCache() - this.nounTypeByIdCache.clear() - this.nounSubtypeByIdCache.clear() - this.verbSubtypeByIdCache.clear() + // Registered-blob registry: clear() wipes the persisted `_system/` copy and + // the family members under `_blobs/`, so drop the in-memory cache too — a + // reset brain must not keep stale family protection or report their members + // "missing" on the next check. + this._derivedFamilies.clear() + this._derivedFamiliesLoaded = false this.nounCountsByType.fill(0) this.verbCountsByType.fill(0) this.subtypeCountsByType.clear() @@ -1086,6 +1536,11 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Standard fields at top-level type: (reserved.noun as NounType) || NounType.Thing, subtype: reserved.subtype as string | undefined, + // visibility is a reserved top-level field (absent === 'public'). Surfacing it + // here lets visibility-aware reads (export node streaming, count/find candidate + // filters) see a noun's tier from getNouns without re-reading metadata — the + // noun mirror of the verb-hydration fix. + visibility: reserved.visibility as HNSWNounWithMetadata['visibility'], createdAt: normalizeStoredTimestamp(reserved.createdAt), updatedAt: normalizeStoredTimestamp(reserved.updatedAt), confidence: reserved.confidence as number | undefined, @@ -1119,6 +1574,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { ...verb, // Standard fields at top-level subtype: reserved.subtype as string | undefined, + // visibility is a reserved top-level field (the verb mirror of Entity.visibility). + // Surfacing it here lets the graph-index fast paths apply visibility filtering on + // their already-hydrated results (so default related() stays O(degree) instead of + // falling through to a full scan), and lets related({ includeInternal }) results + // actually report which edges are internal. + visibility: reserved.visibility as HNSWVerbWithMetadata['visibility'], createdAt: normalizeStoredTimestamp(reserved.createdAt), updatedAt: normalizeStoredTimestamp(reserved.updatedAt), confidence: reserved.confidence as number | undefined, @@ -1131,6 +1592,45 @@ export abstract class BaseStorage extends BaseStorageAdapter { } } + /** + * @description Apply the metadata-derived verb filters (`subtype`, + * `excludeVisibility`) that the graph-index fast paths can satisfy on their + * already-hydrated O(degree) / O(type) candidate set — matching the semantics + * of the full-scan fallback in {@link getVerbsWithPagination}. This is what + * keeps default `related({ from/to })` (which always sets the visibility + * exclusion) on the fast adjacency path instead of forcing a full O(E) scan. + * + * - `subtype`: keeps only verbs carrying a matching subtype; a verb with no + * subtype is excluded when a subtype filter is set (same as the fallback). + * - `excludeVisibility`: drops verbs whose stored visibility tier is in the + * excluded set; an absent tier is `'public'` and is always kept. + * + * @param verbs - Hydrated candidate verbs from a fast-path lookup. + * @param filter - The `getVerbs` filter (only `subtype` / `excludeVisibility` + * are read here; the structural match was already done by the caller). + * @returns The candidates with the metadata filters applied (input order preserved). + */ + protected applyVerbMetadataFilters( + verbs: HNSWVerbWithMetadata[], + filter?: { + subtype?: string | string[] + excludeVisibility?: string[] + } + ): HNSWVerbWithMetadata[] { + let out = verbs + const subtypeFilter = filter?.subtype + if (subtypeFilter) { + const allowed = new Set(Array.isArray(subtypeFilter) ? subtypeFilter : [subtypeFilter]) + out = out.filter((v) => v.subtype !== undefined && allowed.has(v.subtype)) + } + const excludeVisibility = filter?.excludeVisibility + if (excludeVisibility && excludeVisibility.length > 0) { + const excluded = new Set(excludeVisibility) + out = out.filter((v) => !(v.visibility && excluded.has(v.visibility))) + } + return out + } + /** * Get a noun from storage (returns combined HNSWNounWithMetadata) * @param id Entity ID @@ -1181,19 +1681,27 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Delete a noun from storage */ - public async deleteNoun(id: string): Promise { + public async deleteNoun(id: string, priorMetadata?: NounMetadata | null): Promise { await this.ensureInitialized() - // Delete both the vector file and metadata file (2-file system) - await this.deleteNoun_internal(id) + // FULL removal (live-HEAD hygiene): remove BOTH canonical legs AND the + // entity's container, so a delete leaves nothing behind — no orphan/scar + // directory to inflate the enumerated count or ghost resolveLocator. The + // generation store holds the immutable before-image, so asOf() still + // reconstructs the deleted entity until retention expires. + await this.deleteNoun_internal(id) // vectors.json leg - // Delete metadata file (if it exists) - try { - await this.deleteNounMetadata(id) - } catch (error) { - // Ignore if metadata file doesn't exist - prodLog.debug(`No metadata file to delete for noun ${id}`) - } + // Metadata leg + count decrement. A genuine "already absent" is a no-op + // downstream (readCanonicalObject / deleteObjectFromPath both treat ENOENT as + // success); a REAL fault must surface loudly (loud errors, never quiet + // losses) and must never silently skip the count decrement — so this is NO + // LONGER wrapped in a blind catch that masked faults as "file didn't exist". + // `priorMetadata` (the caller's pre-delete read) keeps the decrement honest + // even when the canonical read inside returns null (replace race / ghost). + await this.deleteNounMetadata(id, priorMetadata) + + // Remove the now-empty entity container (a no-op for key/prefix stores). + await this.removeCanonicalContainer(getNounVectorPath(id)) } /** @@ -1504,23 +2012,24 @@ export abstract class BaseStorage extends BaseStorageAdapter { } } - // Storage adapter does not support pagination - prodLog.error( - 'Storage adapter does not support pagination. The deprecated getAllNouns_internal() method has been removed. Please implement getNounsWithPagination() in your storage adapter.' + // Storage adapter does not support pagination. This is a hard + // misconfiguration — find()/rebuild/aggregation all read through this path, + // so returning an empty page would silently present a misconfigured adapter + // as an empty database. Fail loud instead. + throw BrainyError.storage( + 'Storage adapter does not implement getNounsWithPagination(). The deprecated getAllNouns_internal() fallback has been removed; implement pagination in your storage adapter.' ) - - return { - items: [], - totalCount: 0, - hasMore: false - } } catch (error) { - prodLog.error('Error getting nouns with pagination:', error) - return { - items: [], - totalCount: 0, - hasMore: false - } + // Never convert a genuine read failure into a success-shaped empty page: + // this is the highest-fan-in read path (find() fallback, cold-start rebuild + // gating, aggregation backfill, getNounsByNounType), so a swallowed error + // would propagate as "zero rows" everywhere — indistinguishable from a truly + // empty store, and the exact silent-failure class the 8.0 contract forbids. + if (error instanceof BrainyError) throw error + throw BrainyError.storage( + 'getNouns pagination read failed', + error instanceof Error ? error : undefined + ) } } @@ -1538,10 +2047,10 @@ export abstract class BaseStorage extends BaseStorageAdapter { * - Early termination when offset + limit entities collected * - Memory efficient: Never loads full dataset */ - public async getNounsWithPagination(options: { + public override async getNounsWithPagination(options: { limit: number offset: number - cursor?: string // Currently ignored (offset-based pagination). Cursor support planned + cursor?: string // Opaque resume token from a prior page's nextCursor; when set it supersedes offset (O(N) walk, no re-scan) filter?: { nounType?: string | string[] service?: string | string[] @@ -1556,58 +2065,102 @@ export abstract class BaseStorage extends BaseStorageAdapter { await this.ensureInitialized() const { limit, offset = 0, filter } = options - const collectedNouns: HNSWNounWithMetadata[] = [] - // Collect ONE item past the requested window so `hasMore` is decidable: - // stopping exactly at offset+limit cannot distinguish "page full, nothing - // after it" from "page full, more behind it" — which made hasMore - // permanently false and silently truncated every multi-page walk - // (audit/migrateField/fillSubtypes, graph verb-id recovery, count rebuilds). - const targetCount = offset + limit - const peekCount = targetCount + 1 + // Cursor (8.0): resume token carrying the (shard, nounId) of the last returned + // noun — the noun mirror of getVerbsWithPagination. When present it supersedes + // `offset` and resumes the shard walk immediately AFTER that position, so a full + // walk is O(N) instead of the O(N²) of offset paging. (Previously the cursor was + // ignored, which was latent — the only multi-page consumer used a single big + // page — until small chunk sizes needed page 2 and an offset-0-on-every-call + // walk never terminated.) + const cursor = this.decodeNounWalkCursor(options.cursor) + if (options.cursor && cursor === null) { + // A supplied-but-undecodable resume token must FAIL, not silently restart + // at offset 0 — to a while(hasMore) caller the silent fallback re-serves + // page 1 forever: an unbounded CPU loop wearing pagination's clothes. + throw BrainyError.storage( + `getNouns: invalid pagination cursor '${options.cursor}' — cannot resume this walk. ` + + `Restart it without a cursor.` + ) + } + const collected: Array<{ noun: HNSWNounWithMetadata; shard: number }> = [] - // Iterate by shards (0x00-0xFF) instead of types - for (let shard = 0; shard < 256 && collectedNouns.length < peekCount; shard++) { + // Peek one past the window so `hasMore` is decidable. Cursor mode collects one + // page (+1); offset mode keeps the full [0, offset+limit] window (+1). + const peekCount = cursor ? limit + 1 : offset + limit + 1 + const startShard = cursor ? cursor.shard : 0 + + // Iterate by shards (0x00-0xFF), early-terminating at peekCount. + for (let shard = startShard; shard < 256 && collected.length < peekCount; shard++) { const shardHex = shard.toString(16).padStart(2, '0') const shardDir = `entities/nouns/${shardHex}` try { const nounFiles = await this.listCanonicalObjects(shardDir) - for (const nounPath of nounFiles) { - if (collectedNouns.length >= peekCount) break - if (!nounPath.includes('/vectors.json')) continue + // Stable within-shard order (by noun id) so offset windows and cursor resume + // are deterministic; ids come from the path so skipped nouns are never read. + const entries = nounFiles + .filter((p) => p.includes('/vectors.json')) + .map((p) => ({ path: p, id: idFromVectorPath(p) })) + .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)) - try { - const noun = await this.readCanonicalObject(nounPath) - if (noun) { - const deserialized = this.deserializeNoun(noun) - const metadata = await this.getNounMetadata(deserialized.id) + // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor + // id BEFORE hydrating — a cheap id compare (from the path), so skipped + // nouns are never read. + const toHydrate = + cursor && shard === cursor.shard + ? entries.filter((e) => e.id > cursor.id) + : entries - if (metadata) { - // Apply type filter - if (filter?.nounType && metadata.noun) { - const types = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType] - if (!types.includes(metadata.noun)) { - continue - } - } - - // Apply service filter - if (filter?.service) { - const services = Array.isArray(filter.service) ? filter.service : [filter.service] - if (metadata.service && !services.includes(metadata.service)) { - continue - } - } - - // Combine noun + metadata via the canonical hydration helper — - // reserved fields top-level, ONLY custom fields in `metadata` - // (this site previously echoed the full flat record). - collectedNouns.push(this.hydrateNounWithMetadata(deserialized, metadata)) + // Hydrate in bounded-concurrency batches (16-way) instead of one-at-a-time. + // Every canonical enumeration — and every index heal enumerates canonical — + // otherwise pays N×per-op-latency SERIALLY (the dominant heal-time term). + // Order is preserved (the batch is a slice of the sorted entries and its + // results are consumed in order), so offset windows / cursor resume stay + // deterministic. peekCount stops the walk; the final batch over-hydrates by + // at most BASE_STORAGE_HYDRATE_CONCURRENCY entries (bounded, acceptable). + for ( + let i = 0; + i < toHydrate.length && collected.length < peekCount; + i += BaseStorage.HYDRATE_CONCURRENCY + ) { + const batch = toHydrate.slice(i, i + BaseStorage.HYDRATE_CONCURRENCY) + const hydrated = await Promise.all( + batch.map(async ({ path: nounPath }) => { + try { + const noun = await this.readCanonicalObject(nounPath) + if (!noun) return null + const deserialized = this.deserializeNoun(noun) + const metadata = await this.getNounMetadata(deserialized.id) + if (!metadata) return null + return { deserialized, metadata } + } catch (error) { + // Skip nouns that fail to load + return null } + }) + ) + + for (const h of hydrated) { + if (collected.length >= peekCount) break + if (!h) continue + const { deserialized, metadata } = h + + // Apply type filter + if (filter?.nounType && metadata.noun) { + const types = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType] + if (!types.includes(metadata.noun)) continue } - } catch (error) { - // Skip nouns that fail to load + + // Apply service filter + if (filter?.service) { + const services = Array.isArray(filter.service) ? filter.service : [filter.service] + if (metadata.service && !services.includes(metadata.service)) continue + } + + // Combine noun + metadata via the canonical hydration helper — + // reserved fields top-level, ONLY custom fields in `metadata`. + collected.push({ noun: this.hydrateNounWithMetadata(deserialized, metadata), shard }) } } } catch (error) { @@ -1615,35 +2168,182 @@ export abstract class BaseStorage extends BaseStorageAdapter { } } - // Apply pagination. The peeked extra item (if any) is dropped by the slice; - // its existence is exactly what makes hasMore true. - const paginatedNouns = collectedNouns.slice(offset, offset + limit) - const hasMore = collectedNouns.length > targetCount + // Window selection. Cursor mode already starts at the resume point (window + // [0, limit)); offset mode slices [offset, offset+limit). The peeked extra + // entry (if any) is dropped — its existence is exactly what makes hasMore true. + const windowStart = cursor ? 0 : offset + const pagePairs = collected.slice(windowStart, windowStart + limit) + const paginatedNouns = pagePairs.map((p) => p.noun) + const hasMore = collected.length > windowStart + limit - // totalCount must be the TRUE dataset total, not the size of this (peeked) - // page. The shard scan early-terminates at `peekCount = offset + limit + 1` - // for memory efficiency, so `collectedNouns.length` only ever reaches the - // page size + 1 — returning it as `totalCount` made every non-empty brain - // look like it held ~`limit` items (e.g. `getNouns({ pagination: { limit: 1 } - // }).totalCount` was 2, tripping the index-rebuild gate's "Small dataset" - // path). For the unfiltered case the authoritative total is the O(1) counter - // maintained on every add/delete and rehydrated on init; `Math.max` guards a - // stale counter from under-reporting below what we collected. A filtered scan - // has no cheap exact total, so it keeps the collected length (a lower bound). + // totalCount must be the TRUE dataset total, not this peeked page. For the + // unfiltered case the authoritative total is the O(1) counter maintained on + // every add/delete (rehydrated on init); `Math.max` guards a stale counter. A + // filtered scan has no cheap exact total, so it keeps the collected length. const totalCount = filter - ? collectedNouns.length - : Math.max(this.totalNounCount, collectedNouns.length) + ? collected.length + : Math.max(this.totalNounCount, collected.length) + + // nextCursor = the (shard, id) of the last RETURNED noun, so the next call + // resumes immediately after it (works for both cursor and offset callers). + let nextCursor: string | undefined = undefined + if (hasMore && pagePairs.length > 0) { + const lastPair = pagePairs[pagePairs.length - 1] + nextCursor = this.encodeNounWalkCursor(lastPair.shard, lastPair.noun.id) + } return { items: paginatedNouns, totalCount, hasMore, - nextCursor: hasMore && paginatedNouns.length > 0 - ? paginatedNouns[paginatedNouns.length - 1].id - : undefined + nextCursor } } + /** + * @description Enumerate noun IDS ONLY, without hydrating each entity's vector + * and metadata — the opt-out for callers (e.g. an index heal) that own their + * own IO schedule and index straight from a stream. Shares the exact + * shard-walk, cursor, offset and `nextCursor` contract of + * {@link getNounsWithPagination}, so the two are page-compatible. + * + * - UNFILTERED (the heal case): ids come straight from the shard paths — ZERO + * per-entity reads. A full enumeration is O(entries listed), not O(N reads). + * - FILTERED: the type/service filter needs metadata, so only the metadata is + * hydrated (16-way bounded concurrency), never the full noun. + * + * @param options - `limit`/`offset`/`cursor`/`filter` — same semantics as + * {@link getNounsWithPagination}. + * @returns The page of ids plus `totalCount` / `hasMore` / `nextCursor`. + */ + public async getNounIdsWithPagination(options: { + limit: number + offset?: number + cursor?: string + filter?: { + nounType?: string | string[] + service?: string | string[] + metadata?: Record + } + }): Promise<{ ids: string[]; totalCount: number; hasMore: boolean; nextCursor?: string }> { + await this.ensureInitialized() + + const { limit, offset = 0, filter } = options + const cursor = this.decodeNounWalkCursor(options.cursor) + if (options.cursor && cursor === null) { + // Same law as getNouns/getVerbs: an undecodable resume token FAILS + // instead of silently restarting the walk at offset 0. + throw BrainyError.storage( + `getNounIds: invalid pagination cursor '${options.cursor}' — cannot resume this walk. ` + + `Restart it without a cursor.` + ) + } + const collected: Array<{ id: string; shard: number }> = [] + const peekCount = cursor ? limit + 1 : offset + limit + 1 + const startShard = cursor ? cursor.shard : 0 + + for (let shard = startShard; shard < 256 && collected.length < peekCount; shard++) { + const shardHex = shard.toString(16).padStart(2, '0') + const shardDir = `entities/nouns/${shardHex}` + try { + const nounFiles = await this.listCanonicalObjects(shardDir) + const entries = nounFiles + .filter((p) => p.includes('/vectors.json')) + .map((p) => idFromVectorPath(p)) + .sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)) + const toWalk = + cursor && shard === cursor.shard ? entries.filter((id) => id > cursor.id) : entries + + if (!filter) { + // Unfiltered — ids straight from the paths, no reads at all. + for (const id of toWalk) { + if (collected.length >= peekCount) break + collected.push({ id, shard }) + } + } else { + // Filtered — hydrate metadata ONLY (16-way) to apply the filter. + for ( + let i = 0; + i < toWalk.length && collected.length < peekCount; + i += BaseStorage.HYDRATE_CONCURRENCY + ) { + const batch = toWalk.slice(i, i + BaseStorage.HYDRATE_CONCURRENCY) + const metas = await Promise.all( + batch.map(async (id) => { + try { + return { id, metadata: await this.getNounMetadata(id) } + } catch { + return null + } + }) + ) + for (const m of metas) { + if (collected.length >= peekCount) break + if (!m || !m.metadata) continue + const metadata = m.metadata + if (filter.nounType && metadata.noun) { + const types = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType] + if (!types.includes(metadata.noun)) continue + } + if (filter.service) { + const services = Array.isArray(filter.service) ? filter.service : [filter.service] + if (metadata.service && !services.includes(metadata.service)) continue + } + collected.push({ id: m.id, shard }) + } + } + } + } catch (error) { + // Skip shards with no data + } + } + + const windowStart = cursor ? 0 : offset + const pagePairs = collected.slice(windowStart, windowStart + limit) + const ids = pagePairs.map((p) => p.id) + const hasMore = collected.length > windowStart + limit + const totalCount = filter ? collected.length : Math.max(this.totalNounCount, collected.length) + + let nextCursor: string | undefined = undefined + if (hasMore && pagePairs.length > 0) { + const lastPair = pagePairs[pagePairs.length - 1] + nextCursor = this.encodeNounWalkCursor(lastPair.shard, lastPair.id) + } + + return { ids, totalCount, hasMore, nextCursor } + } + + /** + * @description Encode a noun-walk resume cursor — the `(shard, nounId)` of the + * last returned noun — as an opaque, version-tagged token (`cn1:` prefix lets + * {@link decodeNounWalkCursor} reject foreign tokens, e.g. a bare-id cursor). + * `nounId` is placed last and the decoder re-joins on `:` so any id format survives. + * @param shard - The shard (0–255) the noun lives in. + * @param id - The noun id. + * @returns The opaque cursor token. + */ + private encodeNounWalkCursor(shard: number, id: string): string { + return `cn1:${shard}:${id}` + } + + /** + * @description Decode a noun-walk cursor from {@link encodeNounWalkCursor}; + * returns `null` for an absent / malformed / foreign token (caller falls back + * to offset paging rather than mis-resuming). + * @param cursor - The opaque cursor token, or undefined. + * @returns `{ shard, id }` resume position, or `null`. + */ + private decodeNounWalkCursor(cursor?: string): { shard: number; id: string } | null { + if (!cursor) return null + const parts = cursor.split(':') + if (parts.length < 3 || parts[0] !== 'cn1') return null + const shard = Number(parts[1]) + if (!Number.isInteger(shard) || shard < 0 || shard > 255) return null + const id = parts.slice(2).join(':') + if (id.length === 0) return null + return { shard, id } + } + /** * Get verbs with pagination (Type-first implementation with billion-scale optimizations) * @@ -1659,10 +2359,10 @@ export abstract class BaseStorage extends BaseStorageAdapter { * - Memory efficient: Never loads full dataset * - Inline filtering for sourceId, targetId, verbType */ - public async getVerbsWithPagination(options: { + public override async getVerbsWithPagination(options: { limit: number offset: number - cursor?: string // Currently ignored (offset-based pagination). Cursor support planned + cursor?: string // Opaque resume token from a prior page's nextCursor; when set it supersedes offset (O(N) walk, no re-scan) filter?: { verbType?: string | string[] sourceId?: string | string[] @@ -1678,13 +2378,26 @@ export abstract class BaseStorage extends BaseStorageAdapter { }> { await this.ensureInitialized() - const { limit, offset = 0, filter } = options // cursor intentionally not extracted (not yet implemented) - const collectedVerbs: HNSWVerbWithMetadata[] = [] - // Same peek-one-past-the-window strategy as getNounsWithPagination — see - // the comment there. Without the extra item, hasMore is undecidable and - // was permanently false (silent truncation of every multi-page walk). - const targetCount = offset + limit // Requested window end - const peekCount = targetCount + 1 // Early termination target (window + 1 peek) + const { limit, offset = 0, filter } = options + // Cursor (8.0): an opaque resume token (see encodeVerbWalkCursor) carrying the + // (shard, verbId) of the last returned verb. When present it SUPERSEDES `offset` + // and resumes the shard walk immediately AFTER that position, so a full walk is + // O(N) total instead of the O(N²) of offset paging (which re-scans from shard 0 + // every page). + const cursor = this.decodeVerbWalkCursor(options.cursor) + if (options.cursor && cursor === null) { + // A supplied-but-undecodable resume token must FAIL, not silently restart + // at offset 0 — to a while(hasMore) caller the silent fallback re-serves + // page 1 forever: an unbounded CPU loop wearing pagination's clothes. + throw BrainyError.storage( + `getVerbs: invalid pagination cursor '${options.cursor}' — cannot resume this walk. ` + + `Restart it without a cursor.` + ) + } + + // Each collected entry remembers its shard so nextCursor can point at the exact + // (shard, id) resume position. + const collected: Array<{ verb: HNSWVerbWithMetadata; shard: number }> = [] // Prepare filter sets for efficient lookup const filterVerbTypes = filter?.verbType @@ -1716,17 +2429,34 @@ export abstract class BaseStorage extends BaseStorageAdapter { ? new Set(excludeVisibility) : null - // Iterate by shards (0x00-0xFF) instead of types - single pass! - for (let shard = 0; shard < 256 && collectedVerbs.length < peekCount; shard++) { + // Peek one past the window so hasMore is decidable. Cursor mode collects exactly + // one page (+1); offset mode keeps the full [0, offset+limit] window (+1). + const peekCount = cursor ? limit + 1 : offset + limit + 1 + // Cursor resume skips every shard BEFORE the cursor's shard outright (the core of + // the O(N) win); offset mode always starts at shard 0. + const startShard = cursor ? cursor.shard : 0 + + // Iterate by shards (0x00-0xFF) — single pass, early-terminating at peekCount. + for (let shard = startShard; shard < 256 && collected.length < peekCount; shard++) { const shardHex = shard.toString(16).padStart(2, '0') const shardDir = `entities/verbs/${shardHex}` try { const verbFiles = await this.listCanonicalObjects(shardDir) - for (const verbPath of verbFiles) { - if (collectedVerbs.length >= peekCount) break - if (!verbPath.includes('/vectors.json')) continue + // Stable within-shard order (by verb id) so offset windows and cursor resume + // are deterministic and consistent across calls. Ids come from the path, so + // verbs skipped by the cursor are never read. + const entries = verbFiles + .filter((p) => p.includes('/vectors.json')) + .map((p) => ({ path: p, id: idFromVectorPath(p) })) + .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)) + + for (const { path: verbPath, id: verbId } of entries) { + if (collected.length >= peekCount) break + // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor id + // (later shards are processed in full). No read for skipped verbs. + if (cursor && shard === cursor.shard && verbId <= cursor.id) continue try { const rawVerb = await this.readCanonicalObject(verbPath) @@ -1771,9 +2501,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { } // Combine verb + metadata via the canonical hydration helper — - // reserved fields top-level, ONLY custom fields in `metadata` - // (this site previously echoed the full flat record). - collectedVerbs.push(this.hydrateVerbWithMetadata(verb, metadata)) + // reserved fields top-level, ONLY custom fields in `metadata`. + collected.push({ verb: this.hydrateVerbWithMetadata(verb, metadata), shard }) } catch (error) { // Skip verbs that fail to load } @@ -1783,35 +2512,72 @@ export abstract class BaseStorage extends BaseStorageAdapter { } } - // Apply pagination. The peeked extra item (if any) is dropped by the slice; - // its existence is exactly what makes hasMore true. `>` (not `>=`) keeps the - // exact-boundary case (total == offset+limit) from looping forever. - const paginatedVerbs = collectedVerbs.slice(offset, offset + limit) - const hasMore = collectedVerbs.length > targetCount + // Window selection. Cursor mode already starts at the resume point, so its window + // is [0, limit); offset mode slices [offset, offset+limit). The peeked extra entry + // (if any) is dropped here — its existence is exactly what makes hasMore true. + const windowStart = cursor ? 0 : offset + const pagePairs = collected.slice(windowStart, windowStart + limit) + const paginatedVerbs = pagePairs.map((p) => p.verb) + const hasMore = collected.length > windowStart + limit - // totalCount must be the TRUE dataset total, not this peeked page. The verb - // type-scan early-terminates at `peekCount = offset + limit + 1`, so - // `collectedVerbs.length` only ever reaches the page size + 1 — returning it - // made `getVerbs({ pagination: { limit: 1 } }).totalCount` read 1 (or 2) for - // any non-empty brain. This is the verb mirror of the noun fix (b2005ff): for - // the unfiltered scan the authoritative total is the O(1) `totalVerbCount` - // counter (isNew-gated, visibility-filtered, rehydrated on init); `Math.max` - // guards a stale counter from under-reporting. A filtered scan has no cheap - // exact total, so it keeps the collected length (a lower bound). + // totalCount must be the TRUE dataset total, not this peeked page. For the + // unfiltered scan the authoritative total is the O(1) `totalVerbCount` counter + // (isNew-gated, visibility-filtered, rehydrated on init); `Math.max` guards a + // stale counter from under-reporting. A filtered scan has no cheap exact total, + // so it keeps the collected length (a lower bound). const totalCount = filter - ? collectedVerbs.length - : Math.max(this.totalVerbCount, collectedVerbs.length) + ? collected.length + : Math.max(this.totalVerbCount, collected.length) + + // nextCursor encodes the (shard, id) of the LAST RETURNED verb so the next call + // resumes immediately after it — for both cursor and offset callers (an offset + // caller can switch to cursor paging to escape the O(N²)). + let nextCursor: string | undefined = undefined + if (hasMore && pagePairs.length > 0) { + const lastPair = pagePairs[pagePairs.length - 1] + nextCursor = this.encodeVerbWalkCursor(lastPair.shard, lastPair.verb.id) + } return { items: paginatedVerbs, totalCount, hasMore, - nextCursor: hasMore && paginatedVerbs.length > 0 - ? paginatedVerbs[paginatedVerbs.length - 1].id - : undefined + nextCursor } } + /** + * @description Encode a verb-walk resume cursor — the `(shard, verbId)` of the + * last returned verb — as an opaque, version-tagged token. The `cv1:` prefix + * lets {@link decodeVerbWalkCursor} reject foreign tokens (e.g. the bare-id + * cursors the graph-index fast paths emit). `verbId` is placed last and the + * decoder re-joins on `:` so any id format survives the round-trip. + * @param shard - The shard (0–255) the verb lives in. + * @param id - The verb id. + * @returns The opaque cursor token. + */ + private encodeVerbWalkCursor(shard: number, id: string): string { + return `cv1:${shard}:${id}` + } + + /** + * @description Decode a verb-walk cursor produced by {@link encodeVerbWalkCursor}. + * Returns `null` for an absent, malformed, or foreign token so the caller falls + * back to offset-based paging rather than mis-resuming. + * @param cursor - The opaque cursor token, or undefined. + * @returns `{ shard, id }` resume position, or `null`. + */ + private decodeVerbWalkCursor(cursor?: string): { shard: number; id: string } | null { + if (!cursor) return null + const parts = cursor.split(':') + if (parts.length < 3 || parts[0] !== 'cv1') return null + const shard = Number(parts[1]) + if (!Number.isInteger(shard) || shard < 0 || shard > 255) return null + const id = parts.slice(2).join(':') + if (id.length === 0) return null + return { shard, id } + } + /** * Get verbs with pagination and filtering * @param options Pagination and filtering options @@ -1851,17 +2617,15 @@ export abstract class BaseStorage extends BaseStorageAdapter { const cursor = pagination.cursor // Optimize for common filter cases to avoid loading all verbs. - // NOTE: subtype filter intentionally disqualifies the fast paths and forces - // fallthrough to the full shard scan (which loads metadata and applies the - // subtype check after the load). Subtype is not stored on the raw HNSWVerb; - // it's a metadata field. The graph-index fast paths return verbs without - // metadata, so they can't apply subtype filtering correctly. The 8.0 - // `excludeVisibility` filter (also a metadata field) disqualifies them the same way. - if ( - options?.filter && - !(options.filter as { verbType?: string | string[]; subtype?: string | string[] }).subtype && - !(options.filter as { excludeVisibility?: string[] }).excludeVisibility - ) { + // The graph-index fast paths (getVerbsBySource/Target/Type_internal) hydrate + // each candidate's metadata, so `subtype` and the 8.0 `excludeVisibility` + // filters — both metadata fields — are applied on the small O(degree) / + // O(type) candidate set via applyVerbMetadataFilters() BEFORE pagination. + // (They used to disqualify the fast paths and force a full O(E) shard scan, + // which made default related({ from/to }) — which always sets the visibility + // exclusion — scan the entire graph per node.) An arbitrary `metadata` filter + // still falls through, since each block guards `!options.filter.metadata`. + if (options?.filter) { // CRITICAL VFS FIX: If filtering by sourceId + verbType (most common VFS pattern!) // This is the query PathResolver.getChildren() uses: related({ from: dirId, type: VerbType.Contains }) if ( @@ -1879,9 +2643,13 @@ export abstract class BaseStorage extends BaseStorageAdapter { ? options.filter.verbType[0] : options.filter.verbType - // Get verbs by source, then filter by type (O(1) graph lookup + O(n) type filter) + // Get verbs by source, then filter by type (O(1) graph lookup + O(n) type filter), + // then apply the subtype / visibility metadata filters on the candidate set. const verbsBySource = await this.getVerbsBySource_internal(sourceId) - const filteredVerbs = verbsBySource.filter(v => v.verb === verbType) + const filteredVerbs = this.applyVerbMetadataFilters( + verbsBySource.filter(v => v.verb === verbType), + options.filter + ) // Apply pagination const paginatedVerbs = filteredVerbs.slice(offset, offset + limit) @@ -1914,8 +2682,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { ? options.filter.sourceId[0] : options.filter.sourceId - // Get verbs by source directly - const verbsBySource = await this.getVerbsBySource_internal(sourceId) + // Get verbs by source directly (hydrated with metadata), then apply the + // subtype / visibility metadata filters on the O(degree) candidate set. + const verbsBySource = this.applyVerbMetadataFilters( + await this.getVerbsBySource_internal(sourceId), + options.filter + ) // Apply pagination const paginatedVerbs = verbsBySource.slice(offset, offset + limit) @@ -1948,8 +2720,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { ? options.filter.targetId[0] : options.filter.targetId - // Get verbs by target directly - const verbsByTarget = await this.getVerbsByTarget_internal(targetId) + // Get verbs by target directly (hydrated with metadata), then apply the + // subtype / visibility metadata filters on the O(degree) candidate set. + const verbsByTarget = this.applyVerbMetadataFilters( + await this.getVerbsByTarget_internal(targetId), + options.filter + ) // Apply pagination const paginatedVerbs = verbsByTarget.slice(offset, offset + limit) @@ -1982,8 +2758,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { ? options.filter.verbType[0] : options.filter.verbType - // Get verbs by type directly - const verbsByType = await this.getVerbsByType_internal(verbType) + // Get verbs by type directly (hydrated with metadata), then apply the + // subtype / visibility metadata filters on the candidate set. + const verbsByType = this.applyVerbMetadataFilters( + await this.getVerbsByType_internal(verbType), + options.filter + ) // Apply pagination const paginatedVerbs = verbsByType.slice(offset, offset + limit) @@ -2026,8 +2806,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Get verbs by source (uses GraphAdjacencyIndex if available) const verbsBySource = await this.getVerbsBySource_internal(sourceId) - // Filter by verbType in memory (fast - usually small number of verbs per source) - const filtered = verbsBySource.filter(v => verbTypes.includes(v.verb)) + // Filter by verbType in memory (fast - usually small number of verbs per source), + // then apply the subtype / visibility metadata filters on the candidate set. + const filtered = this.applyVerbMetadataFilters( + verbsBySource.filter(v => verbTypes.includes(v.verb)), + options.filter + ) // Apply pagination const paginatedVerbs = filtered.slice(offset, offset + limit) @@ -2231,31 +3015,30 @@ export abstract class BaseStorage extends BaseStorageAdapter { : undefined } } catch (error) { - prodLog.error('Error getting verbs with pagination:', error) - return { - items: [], - totalCount: 0, - hasMore: false - } + // Same no-silent-failure contract as getNouns: a genuine read failure must + // surface as a named, catchable error, not a success-shaped empty page that + // callers read as "this graph has no verbs." + if (error instanceof BrainyError) throw error + throw BrainyError.storage( + 'getVerbs pagination read failed', + error instanceof Error ? error : undefined + ) } } /** * Delete a verb from storage */ - public async deleteVerb(id: string): Promise { + public async deleteVerb(id: string, priorMetadata?: VerbMetadata | null): Promise { await this.ensureInitialized() - // Delete both the vector file and metadata file (2-file system) - await this.deleteVerb_internal(id) - - // Delete metadata file (if it exists) - try { - await this.deleteVerbMetadata(id) - } catch (error) { - // Ignore if metadata file doesn't exist - prodLog.debug(`No metadata file to delete for verb ${id}`) - } + // FULL removal (see deleteNoun): both canonical legs + the verb container. + // The blind catch that masked real faults as "no metadata file" is gone — a + // genuine absence is already a no-op downstream; a real fault surfaces loudly. + // `priorMetadata` keeps the count decrement honest on a null internal read. + await this.deleteVerb_internal(id) // vectors.json leg + await this.deleteVerbMetadata(id, priorMetadata) // metadata leg + count decrement + await this.removeCanonicalContainer(getVerbVectorPath(id)) } /** * Get graph index (lazy initialization with concurrent access protection) @@ -2294,26 +3077,96 @@ export abstract class BaseStorage extends BaseStorageAdapter { // invalidateGraphIndex); on first init Brainy wires it right after. this.graphIndex = new GraphAdjacencyIndex(this, {}, this.graphEntityIdResolver) - // Check if we need to rebuild from existing data - const sampleVerbs = await this.getVerbs({ pagination: { limit: 1 } }) - if (sampleVerbs.items.length > 0) { - prodLog.info('Found existing verbs, rebuilding graph index...') - await this.graphIndex.rebuild() + // Load the PERSISTED adjacency first (LSM manifests + SSTables). A warm + // reopen must load the durable index it already built — the previous + // "any verb exists → rebuild()" check here re-derived the whole graph + // from a full canonical verb scan on EVERY boot, an O(E) cost that + // dominated real deployments' startup. + await this.graphIndex.init() + + // Self-heal only when the durable state is genuinely missing: canonical + // records exist but the loaded index is empty (first open on pre-index + // data, a deleted/corrupt _graph dir, or the LSM load failing loud). + // One O(1) probe replaces the unconditional O(E) re-derive. + if (this.graphIndex.size() === 0) { + const sampleVerbs = await this.getVerbs({ pagination: { limit: 1 } }) + if (sampleVerbs.items.length > 0) { + prodLog.warn( + 'GraphAdjacencyIndex: canonical verbs exist but the persisted adjacency is empty — ' + + 'rebuilding from storage (one-time self-heal).' + ) + await this.graphIndex.rebuild() + } } return this.graphIndex } + + /** + * @description One-shot cold-load self-heal for a graph provider that does NOT + * expose the honest `isReady()` signal. A native provider wired via + * {@link setGraphIndex} bypasses `_initializeGraphIndex`'s `size()===0` + * self-heal, so a cold-open that loaded the COUNT but not the source→target + * adjacency would let the fast path return a silent `[]`. This probe (run once, + * before the fast path) samples ONE known persisted edge: if the index claims + * relationships (`size() > 0`) yet that edge's source resolves to no verb ints, + * the adjacency did not load → rebuild once from storage. Providers that expose + * `isReady()` are covered by the honest gate in + * getVerbsBy{Source,Target}_internal and skip this probe; the JS index (which + * self-heals in `_initializeGraphIndex`) resolves its known edge and no-ops. + */ + private async ensureGraphFastPathProbed(): Promise { + if (this._graphFastPathProbed) return + const index = this.graphIndex + const resolver = this.graphEntityIdResolver + if (!index || !index.isInitialized || !resolver) return + // isReady()-capable providers report serving honestly — the gate handles them. + if (assessIndexReadiness(index) !== 'unknown') { + this._graphFastPathProbed = true + return + } + if (index.size() <= 0) { + this._graphFastPathProbed = true + return // no edges claimed — nothing to verify + } + try { + const sample = await this.getVerbs({ pagination: { limit: 1 } }) + const verb = sample.items?.[0] + if (!verb || !verb.sourceId) { + this._graphFastPathProbed = true + return // no edges in storage — stale count, harmless + } + const sourceInt = resolver.getInt(verb.sourceId) + if (sourceInt === undefined) { + this._graphFastPathProbed = true + return // foreign / unmapped sample — inconclusive, never a false rebuild + } + const verbInts = await index.getVerbIdsBySource(BigInt(sourceInt)) + if (verbInts.length === 0) { + prodLog.warn( + `[BaseStorage] Graph fast path: index reports ${index.size()} relationship(s) but a ` + + `known persisted edge resolves to none — the persisted adjacency did not load. ` + + `Rebuilding once from storage.` + ) + await index.rebuild() + } + this._graphFastPathProbed = true + } catch (error) { + // Transient probe/rebuild failure must not break the read; re-arm for next call. + prodLog.debug(`[BaseStorage] Graph fast-path probe skipped (transient): ${error}`) + } + } /** * Clear all data from storage * This method should be implemented by each specific adapter */ - public abstract clear(): Promise + public abstract override clear(): Promise /** * Get information about storage usage and capacity * This method should be implemented by each specific adapter */ - public abstract getStorageStatus(): Promise<{ + public abstract override getStorageStatus(): Promise<{ type: string used: number quota: number | null @@ -2392,6 +3245,26 @@ export abstract class BaseStorage extends BaseStorageAdapter { return null } + /** + * Delete a system/metadata object previously written with {@link saveMetadata}. + * The exact inverse of save/get: routes through `analyzeKey(id, 'system')` and + * removes the canonical object (plus the legacy flat path that `getMetadata` + * also reads, so a sharded key leaves nothing behind). Lets keyed-payload + * owners — e.g. the LSM graph store reclaiming its compacted-away SSTables — + * free storage instead of orphaning it. Idempotent: deleting a missing path is + * a no-op. + */ + public async deleteMetadata(id: string): Promise { + await this.ensureInitialized() + const keyInfo = this.analyzeKey(id, 'system') + await this.deleteCanonicalObject(keyInfo.fullPath) + // Mirror getMetadata's legacy fallback so an older flat-path payload for a + // sharded key is also reclaimed. + if (keyInfo.shardId !== null) { + await this.deleteCanonicalObject(`${SYSTEM_DIR}/${id}.json`) + } + } + /** * Save noun metadata to storage (now typed) * Routes to correct sharded location based on UUID @@ -2425,19 +3298,16 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Save the metadata (write-cache coherent canonical write) await this.writeCanonicalObject(path, metadata) - // Record the id→type mapping so `saveNoun_internal()` (which receives only - // an HNSWNoun and has no metadata access in its signature) can attribute - // the entity to the right slot in `nounCountsByType`. This is what makes - // `_system/type-statistics.json` honest. Updates the cache even for - // existing entities so a type change via `update()` is reflected. - if (metadata.noun) { - this.nounTypeByIdCache.set(id, metadata.noun as NounType) - } - // Track subtype changes: on type or subtype change via update(), decrement - // the prior bucket before incrementing the new one. Symmetric with the - // delete-path decrement in `deleteNounMetadata()`. - const priorSubtype = this.nounSubtypeByIdCache.get(id) + // the prior bucket before incrementing the new one. The prior (type, subtype) + // comes straight from the canonical record (`existingMetadata`, already loaded + // above) — there is no id-keyed subtype cache. Symmetric with the delete-path + // decrement in `deleteNounMetadata()`. + const priorSubtype = isNew + ? undefined + : (typeof existingMetadata?.subtype === 'string' && (existingMetadata.subtype as string).length > 0 + ? (existingMetadata.subtype as string) + : undefined) const priorTypeForSubtype = isNew ? undefined : (existingMetadata?.noun as NounType | undefined) const newSubtype = typeof metadata.subtype === 'string' && metadata.subtype.length > 0 ? metadata.subtype as string @@ -2449,24 +3319,14 @@ export abstract class BaseStorage extends BaseStorageAdapter { } if (newSubtype && newType && (isNew || priorSubtype !== newSubtype || priorTypeForSubtype !== newType)) { this.incrementSubtypeCount(newType, newSubtype) - this.nounSubtypeByIdCache.set(id, newSubtype) - } else if (!newSubtype && priorSubtype) { - // Subtype cleared by an update — drop the cache entry (decrement already done above) - this.nounSubtypeByIdCache.delete(id) } // Visibility (8.0): only public entities count toward the user-facing totals. - // Warm `nounVisibilityByIdCache` so `saveNoun_internal()` (no metadata access) - // can gate the `nounCountsByType` increment, and `deleteNounMetadata()` can gate - // the matching decrement. Sparse — public ids get no entry. + // The gate reads `metadata.visibility` (new) and `existingMetadata?.visibility` + // (prior) directly off the record — no id-keyed visibility cache. const newVisibility = metadata.visibility const wasCounted = isNew ? false : isCountedVisibility(existingMetadata?.visibility) const isCounted = isCountedVisibility(newVisibility) - if (isCountedVisibility(newVisibility)) { - this.nounVisibilityByIdCache.delete(id) - } else { - this.nounVisibilityByIdCache.set(id, newVisibility as EntityVisibility) - } // CRITICAL FIX: Increment count for new entities // This runs AFTER metadata is saved, guaranteeing type information is available @@ -2480,11 +3340,21 @@ export abstract class BaseStorage extends BaseStorageAdapter { // used to be bumped unconditionally in saveNoun_internal(), but the HNSW index // re-saves a node on every neighbor-link change, so that inflated the per-type // counts with graph connectivity (e.g. 8 documents could read as 44). - this.nounCountsByType[TypeUtils.getNounIndex(metadata.noun as NounType)]++ + const typeIdx = TypeUtils.getNounIndex(metadata.noun as NounType) + this.nounCountsByType[typeIdx]++ // Persist counts asynchronously (fire and forget) this.scheduleCountPersist().catch(() => { // Ignore persist errors - will retry on next operation }) + // Persist type-statistics on the first entity of a type and every 100th + // thereafter. This trigger used to live in saveNoun_internal(), which had to + // call getNounType() purely to recover the type index; sourcing the type from + // the metadata record here keeps the hot vector-save path free of any type + // lookup. The "only when counted" half of the heuristic holds by construction + // inside this branch. + if (this.nounCountsByType[typeIdx] === 1 || this.nounCountsByType[typeIdx] % 100 === 0) { + await this.saveTypeStatistics() + } } else if (!isNew && metadata.noun && wasCounted !== isCounted) { // Visibility flipped on update(): move the entity in/out of the user-facing // total (counts.json / getNounCount()) AND the per-type counter together, so @@ -2493,6 +3363,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { if (isCounted) { this.incrementEntityCount(metadata.noun) this.nounCountsByType[typeIdx]++ + // Same cadence-gated type-statistics persist as the fresh-add branch — only + // fires when the entity is now counted (public), matching the original + // `counted && (count === 1 || count % 100 === 0)` heuristic. + if (this.nounCountsByType[typeIdx] === 1 || this.nounCountsByType[typeIdx] % 100 === 0) { + await this.saveTypeStatistics() + } } else { this.decrementEntityCount(metadata.noun) if (this.nounCountsByType[typeIdx] > 0) this.nounCountsByType[typeIdx]-- @@ -2799,7 +3675,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { * * @public - Inherited from BaseStorageAdapter */ - public getBatchConfig(): StorageBatchConfig { + public override getBatchConfig(): StorageBatchConfig { // Conservative defaults - adapters should override with their actual limits return { maxBatchSize: 100, @@ -2814,37 +3690,62 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Delete noun metadata from storage (ID-first, O(1) delete) + * Delete noun metadata from storage (ID-first, O(1) delete). + * + * @param id - The entity id. + * @param priorRecord - OPTIONAL already-known metadata of the entity being + * removed (e.g. the pre-delete read `remove()` performs, or a captured + * before-image). The count decrement must never REQUIRE re-reading the + * thing being removed: when the canonical read here returns `null` (a + * replace race, or a partial-delete ghost from an earlier version) the + * decrement falls back to this record instead of being silently skipped — + * the skip permanently inflated the persisted totals (adds counted, paired + * removals not decremented), and `Math.max(totalNounCount, scanned)` made + * the inflation unfixable by any disk cleanup. */ - public async deleteNounMetadata(id: string): Promise { + public async deleteNounMetadata(id: string, priorRecord?: NounMetadata | null): Promise { await this.ensureInitialized() - // Direct O(1) delete with ID-first path + // Direct O(1) delete with ID-first path. Read the canonical record BEFORE + // removing it: the per-type and subtype decrements are sourced from the + // entity's own metadata (`noun` type, `subtype`, `visibility`) rather than an + // id-keyed cache, keeping type-statistics honest across deletes — symmetric + // with the increments in `saveNounMetadata_internal()`. A null read falls + // back to the caller-provided prior record (see @param priorRecord). const path = getNounMetadataPath(id) + const read = await this.readCanonicalObject(path) await this.deleteCanonicalObject(path) + const record = read ?? priorRecord - // Prune the id→type cache so a future re-add of the same id (e.g. churn - // during tests) doesn't see a stale type. Lookup the prior type and - // decrement `nounCountsByType` so type-statistics stay honest across - // deletes; symmetric with the increment in `saveNounMetadata_internal()`. - const priorType = this.nounTypeByIdCache.get(id) + const priorType = record?.noun as NounType | undefined // 8.0 visibility: an internal/system entity was never added to `nounCountsByType` - // (gated in `saveNoun_internal()`), so it must not be decremented here either. - const priorCounted = isCountedVisibility(this.nounVisibilityByIdCache.get(id)) - this.nounVisibilityByIdCache.delete(id) + // (gated in `saveNounMetadata_internal()`), so it must not be decremented here either. + const priorCounted = isCountedVisibility(record?.visibility) if (priorType) { - this.nounTypeByIdCache.delete(id) if (priorCounted) { + // Symmetric with the counted-add increment in saveNounMetadata_internal(): + // decrement BOTH the user-facing scalar total (getNounCount / counts.json) AND + // the per-type bucket, then persist. The scalar decrement was previously + // omitted here, so deletes permanently inflated getNounCount() — the stale + // scalar wins pagination via Math.max(totalNounCount, collected.length) and is + // persisted by scheduleCountPersist(). With this, the invariant + // `totalNounCount === Σ nounCountsByType` holds across add / update-flip / delete. + this.decrementEntityCount(priorType) const idx = TypeUtils.getNounIndex(priorType) if (this.nounCountsByType[idx] > 0) { this.nounCountsByType[idx]-- } + this.scheduleCountPersist().catch(() => { + // Ignore persist errors — the in-memory count is authoritative; a later + // operation retries the persist. + }) } - // Symmetric subtype decrement - const priorSubtype = this.nounSubtypeByIdCache.get(id) + // Symmetric subtype decrement — same non-empty-string guard as the write path. + const priorSubtype = typeof record?.subtype === 'string' && (record.subtype as string).length > 0 + ? (record.subtype as string) + : undefined if (priorSubtype) { - this.nounSubtypeByIdCache.delete(id) this.decrementSubtypeCount(priorType, priorSubtype) } } @@ -2900,39 +3801,31 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Save the metadata (write-cache coherent canonical write) await this.writeCanonicalObject(path, metadata) - // Cache verb type for faster lookups - // Track verb subtype changes: on type or subtype change via updateRelation(), - // decrement the prior bucket before incrementing the new one. Symmetric with - // the delete-path decrement in `deleteVerbMetadata()`. - const priorEntry = this.verbSubtypeByIdCache.get(id) + // decrement the prior bucket before incrementing the new one. The prior + // (verb, subtype) is read straight from the canonical record (`existingMetadata`, + // loaded above) — there is no id-keyed verb-subtype cache. Symmetric with the + // delete-path decrement in `deleteVerbMetadata()`. const priorVerbForSubtype = isNew ? undefined : (existingMetadata?.verb as VerbType | undefined) + const priorSubtype = isNew + ? undefined + : (typeof existingMetadata?.subtype === 'string' && (existingMetadata.subtype as string).length > 0 + ? (existingMetadata.subtype as string) + : undefined) const newSubtype = typeof metadata.subtype === 'string' && metadata.subtype.length > 0 ? metadata.subtype as string : undefined - if (priorEntry && (priorEntry.subtype !== newSubtype || priorEntry.verb !== verbType)) { - this.decrementVerbSubtypeCount(priorEntry.verb, priorEntry.subtype) - } else if (!priorEntry && priorVerbForSubtype && !isNew) { - // Edge case: cache miss but metadata existed with a subtype (e.g. reader process startup). - const priorSubFromMeta = existingMetadata?.subtype as string | undefined - if (priorSubFromMeta && (priorSubFromMeta !== newSubtype || priorVerbForSubtype !== verbType)) { - this.decrementVerbSubtypeCount(priorVerbForSubtype, priorSubFromMeta) - } + if (priorSubtype && priorVerbForSubtype && (priorSubtype !== newSubtype || priorVerbForSubtype !== verbType)) { + this.decrementVerbSubtypeCount(priorVerbForSubtype, priorSubtype) } - if (newSubtype) { - if (!priorEntry || priorEntry.subtype !== newSubtype || priorEntry.verb !== verbType) { - this.incrementVerbSubtypeCount(verbType, newSubtype) - } - this.verbSubtypeByIdCache.set(id, { verb: verbType, subtype: newSubtype }) - } else if (priorEntry) { - // Subtype cleared by an update — drop the cache entry (decrement done above) - this.verbSubtypeByIdCache.delete(id) + if (newSubtype && (isNew || priorSubtype !== newSubtype || priorVerbForSubtype !== verbType)) { + this.incrementVerbSubtypeCount(verbType, newSubtype) } - // Visibility (8.0): verb mirror of the noun count gating. Warm - // `verbVisibilityByIdCache` so `deleteVerbMetadata()` can prune it. Sparse — - // public edges get no entry. + // Visibility (8.0): verb mirror of the noun count gating. The gate reads + // `metadata.visibility` (new) and `existingMetadata?.visibility` (prior) + // directly off the record — no id-keyed visibility cache. // // NOTE on `verbCountsByType`: unlike the noun path, `saveVerb_internal()` runs // BEFORE this method (relate() saves the verb vector first) and has already done @@ -2942,11 +3835,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { const newVisibility = metadata.visibility const wasCounted = isNew ? false : isCountedVisibility(existingMetadata?.visibility) const isCounted = isCountedVisibility(newVisibility) - if (isCounted) { - this.verbVisibilityByIdCache.delete(id) - } else { - this.verbVisibilityByIdCache.set(id, newVisibility as EntityVisibility) - } const verbTypeIdx = TypeUtils.getVerbIndex(verbType) // CRITICAL FIX: Increment verb count for new relationships @@ -2991,36 +3879,54 @@ export abstract class BaseStorage extends BaseStorageAdapter { await this.ensureInitialized() // Direct O(1) lookup with ID-first paths - no type search needed! + // Symmetric with getNounMetadata: readCanonicalObject already returns null for + // a genuine not-found, so a real storage fault (permission/corruption/IO) must + // propagate rather than be masked as "this verb has no metadata". const path = getVerbMetadataPath(id) - - try { - const metadata = await this.readCanonicalObject(path) - return metadata || null - } catch (error) { - // Entity not found - return null - } + return this.readCanonicalObject(path) } /** * Delete verb metadata from storage (ID-first, O(1) delete) */ - public async deleteVerbMetadata(id: string): Promise { + public async deleteVerbMetadata(id: string, priorRecord?: VerbMetadata | null): Promise { await this.ensureInitialized() - // Direct O(1) delete with ID-first path + // Direct O(1) delete with ID-first path. Read the canonical record BEFORE + // removing it so every decrement is sourced from the edge's own metadata + // (`verb` type, `subtype`, `visibility`) rather than an id-keyed cache — + // symmetric with the increments in `saveVerbMetadata_internal()`. A null + // read falls back to the caller-provided prior record: the decrement must + // never REQUIRE re-reading the thing being removed (a silent skip minted + // permanent counter inflation — see deleteNounMetadata). const path = getVerbMetadataPath(id) + const read = await this.readCanonicalObject(path) await this.deleteCanonicalObject(path) + const record = read ?? priorRecord - // Symmetric verb subtype decrement - const priorEntry = this.verbSubtypeByIdCache.get(id) - if (priorEntry) { - this.verbSubtypeByIdCache.delete(id) - this.decrementVerbSubtypeCount(priorEntry.verb, priorEntry.subtype) + const priorVerb = record?.verb as VerbType | undefined + // Symmetric count decrement (previously OMITTED — verb deletes touched neither the + // scalar total nor the per-type bucket, so both inflated permanently). A COUNTED + // edge bumped BOTH the scalar (incrementVerbCount in saveVerbMetadata_internal) and + // the per-type bucket (the unconditional bump in saveVerb_internal that the metadata + // path keeps for counted edges). Delete must undo both, gated on the SAME visibility + // as the add, so `totalVerbCount === Σ verbCountsByType` holds across delete. + const priorCounted = isCountedVisibility(record?.visibility) + if (priorVerb && priorCounted) { + this.decrementVerbCount(priorVerb) + const idx = TypeUtils.getVerbIndex(priorVerb) + if (this.verbCountsByType[idx] > 0) this.verbCountsByType[idx]-- + this.scheduleCountPersist().catch(() => { + // Ignore persist errors — in-memory count is authoritative; a later op retries. + }) + } + + const priorSubtype = typeof record?.subtype === 'string' && (record.subtype as string).length > 0 + ? (record.subtype as string) + : undefined + if (priorVerb && priorSubtype) { + this.decrementVerbSubtypeCount(priorVerb, priorSubtype) } - // 8.0 visibility: prune the cache entry (verb deletes don't touch verbCountsByType - // in this path, mirroring the existing verb-subtype delete semantics). - this.verbVisibilityByIdCache.delete(id) // 8.0 MVCC: entity-visible write — advance the generation watermark // (suppressed inside transact batches by the generation store). @@ -3217,7 +4123,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { public async rebuildSubtypeCounts(): Promise { prodLog.info('[BaseStorage] Rebuilding subtype counts from storage...') this.subtypeCountsByType.clear() - this.nounSubtypeByIdCache.clear() for (let shard = 0; shard < 256; shard++) { const shardHex = shard.toString(16).padStart(2, '0') @@ -3230,10 +4135,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { const metadata = await this.readCanonicalObject(path) if (metadata && metadata.noun && typeof metadata.subtype === 'string' && metadata.subtype.length > 0) { this.incrementSubtypeCount(metadata.noun as NounType, metadata.subtype) - // Path is `entities/nouns///metadata.json` — extract id segment. - const segments = path.split('/') - const idSeg = segments[segments.length - 2] - if (idSeg) this.nounSubtypeByIdCache.set(idSeg, metadata.subtype) } } catch { /* skip unreadable entities */ } } @@ -3334,7 +4235,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { public async rebuildVerbSubtypeCounts(): Promise { prodLog.info('[BaseStorage] Rebuilding verb subtype counts from storage...') this.verbSubtypeCountsByType.clear() - this.verbSubtypeByIdCache.clear() for (let shard = 0; shard < 256; shard++) { const shardHex = shard.toString(16).padStart(2, '0') @@ -3349,9 +4249,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { const verb = metadata.verb as VerbType const subtype = metadata.subtype as string this.incrementVerbSubtypeCount(verb, subtype) - const segments = path.split('/') - const idSeg = segments[segments.length - 2] - if (idSeg) this.verbSubtypeByIdCache.set(idSeg, { verb, subtype }) } } catch { /* skip unreadable verbs */ } } @@ -3377,7 +4274,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { * writer flush — the same silent-stale failure mode that once produced * zero counts from `brain.stats()`. */ - public async flushCounts(): Promise { + public override async flushCounts(): Promise { await super.flushCounts() await this.saveTypeStatistics() await this.saveSubtypeStatistics() @@ -3442,6 +4339,17 @@ export abstract class BaseStorage extends BaseStorageAdapter { this.nounCountsByType = new Uint32Array(NOUN_TYPE_COUNT) this.verbCountsByType = new Uint32Array(VERB_TYPE_COUNT) + // The SAME walk also rebuilds the user-facing scalar totals + per-type maps + // persisted in counts.json (totalNounCount / totalVerbCount / entityCounts / + // verbCounts). Previously only the type-statistics arrays were rebuilt and + // the total was computed just to LOG it — so a drifted persisted scalar + // (deletes whose decrement was skipped) survived every "rebuild" forever, + // and Math.max(totalNounCount, scanned) made the inflation unfixable by any + // disk cleanup. This method is now the SANCTIONED RECOUNT: one canonical + // walk, every counter rollup rebuilt and persisted from it. + const countedNouns = new Map() + const countedVerbs = new Map() + // Scan noun shards for (let shard = 0; shard < 256; shard++) { const shardHex = shard.toString(16).padStart(2, '0') @@ -3456,17 +4364,13 @@ export abstract class BaseStorage extends BaseStorageAdapter { try { const metadata = await this.readCanonicalObject(path) if (metadata && metadata.noun) { - // 8.0 visibility: rebuild only public entities into the user-facing - // per-type stat; warm the cache so later writes/deletes stay symmetric. - const id = idFromMetadataPath(path) + // 8.0 visibility: rebuild only public entities into the user-facing per-type stat. if (isCountedVisibility(metadata.visibility)) { const typeIndex = TypeUtils.getNounIndex(metadata.noun) if (typeIndex >= 0 && typeIndex < NOUN_TYPE_COUNT) { this.nounCountsByType[typeIndex]++ } - if (id) this.nounVisibilityByIdCache.delete(id) - } else if (id) { - this.nounVisibilityByIdCache.set(id, metadata.visibility as EntityVisibility) + countedNouns.set(metadata.noun, (countedNouns.get(metadata.noun) || 0) + 1) } } } catch (error) { @@ -3493,15 +4397,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { const metadata = await this.readCanonicalObject(path) if (metadata && metadata.verb) { // 8.0 visibility: rebuild only public edges into the user-facing per-type stat. - const id = idFromMetadataPath(path) if (isCountedVisibility(metadata.visibility)) { const typeIndex = TypeUtils.getVerbIndex(metadata.verb) if (typeIndex >= 0 && typeIndex < VERB_TYPE_COUNT) { this.verbCountsByType[typeIndex]++ } - if (id) this.verbVisibilityByIdCache.delete(id) - } else if (id) { - this.verbVisibilityByIdCache.set(id, metadata.visibility as EntityVisibility) + countedVerbs.set(metadata.verb, (countedVerbs.get(metadata.verb) || 0) + 1) } } } catch (error) { @@ -3518,60 +4419,39 @@ export abstract class BaseStorage extends BaseStorageAdapter { const totalVerbs = this.verbCountsByType.reduce((sum, count) => sum + count, 0) const totalNouns = this.nounCountsByType.reduce((sum, count) => sum + count, 0) - prodLog.info(`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs`) + + // The sanctioned recount half: replace the user-facing scalar totals + the + // per-type maps with the walk's truth and PERSIST them (counts.json), so an + // inflated persisted counter is actually corrected — not merely out-voted + // in memory until the next reopen rehydrates the stale file. + this.entityCounts = countedNouns + this.verbCounts = countedVerbs + this.totalNounCount = totalNouns + this.totalVerbCount = totalVerbs + this.countCache.clear() + await this.persistCounts() + + prodLog.info(`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs (scalar + per-type persisted)`) } /** - * Resolve the `NounType` for a noun, used to attribute the entity to the - * correct slot in `nounCountsByType` (which backs `_system/type-statistics.json` - * and `brain.stats().entitiesByType`). + * Resolve a noun's `NounType` straight from its canonical metadata record on + * disk. Used by the poisoned-statistics detector (`detectPoisonedTypeStatistics()`) + * to confirm whether on-disk types disagree with a `'thing'`-only persisted + * rollup before triggering a full `rebuildTypeCounts()`. Returns `null` when + * metadata genuinely doesn't exist or the read fails; the caller decides whether + * to skip the entity. There is no in-memory id→type cache — the record is the + * single source of truth. * - * Lookup order: - * 1. `nounTypeByIdCache`, populated by `saveNounMetadata_internal()` - * whenever a noun's metadata is written. This is the common path — - * add() saves metadata before the HNSW noun, so by the time we get - * here the cache is warm. - * 2. `'thing'` as a final fallback when metadata genuinely doesn't exist - * (a noun added without metadata — irregular but tolerated for back- - * compat). Logged so a real bug doesn't go unnoticed. - * - * Note this is intentionally synchronous because `saveNoun_internal()` and - * its callers are not async-friendly at this layer. For cases where only - * an id is known after a process restart and the cache is cold, - * `getNounTypeFromStorageAsync()` is available for callers that can await. - */ - protected getNounType(noun: HNSWNoun): NounType { - const cached = this.nounTypeByIdCache.get(noun.id) - if (cached) return cached - // Cache miss on a noun we're about to write means metadata was not saved - // first. Brainy.add() always saves metadata before HNSW, so this only - // fires in unusual code paths (raw `storage.saveNoun()` without metadata). - prodLog.warn( - `[BaseStorage] getNounType: no type cached for noun ${noun.id}. ` + - `Type-statistics will attribute this entity to 'thing'. ` + - `Call saveNounMetadata() before saveNoun() to avoid stat drift.` - ) - return 'thing' - } - - /** - * Async variant of `getNounType()` that consults disk if the in-memory - * cache is cold (e.g. after a process restart with deferred work). Used by - * `rebuildTypeCounts()` and other callers that can await an IO. Returns - * `null` if metadata genuinely doesn't exist; callers decide whether to - * fall back to 'thing' or skip the entity. + * @param id - The noun id whose type to resolve. + * @returns The stored `NounType`, or `null` if absent/unreadable. */ protected async getNounTypeFromStorageAsync(id: string): Promise { - const cached = this.nounTypeByIdCache.get(id) - if (cached) return cached try { const metadataPath = getNounMetadataPath(id) const metadata = await this.readCanonicalObject(metadataPath) if (metadata && (metadata as NounMetadata).noun) { - const type = (metadata as NounMetadata).noun as NounType - // Warm the cache so subsequent sync calls hit fast. - this.nounTypeByIdCache.set(id, type) - return type + return (metadata as NounMetadata).noun as NounType } } catch { // Storage error — treat as unknown. @@ -3675,28 +4555,16 @@ export abstract class BaseStorage extends BaseStorageAdapter { * Save a noun to storage (ID-first path) */ protected async saveNoun_internal(noun: HNSWNoun): Promise { - const type = this.getNounType(noun) const path = getNounVectorPath(noun.id) - // Per-type stats counter (`nounCountsByType`, read by stats().entitiesByType) - // is maintained in saveNounMetadata_internal(), gated on isNew + visibility — - // NOT here. saveNoun_internal() also runs on HNSW neighbor-link re-saves, so - // incrementing here inflated the per-type counts with graph connectivity. We - // only READ the (externally-maintained) count below to decide when to persist. - const typeIndex = TypeUtils.getNounIndex(type) - const counted = isCountedVisibility(this.nounVisibilityByIdCache.get(noun.id)) - - // Write-cache coherent canonical write + // Hot path: write the vector record only. Per-type counters + // (`nounCountsByType`, read by stats().entitiesByType) AND the periodic + // type-statistics persist trigger are both maintained in + // saveNounMetadata_internal(), which holds the canonical metadata record — + // and therefore the NounType + visibility — at the exact point a count + // changes. saveNoun_internal() also re-runs on every HNSW neighbor-link + // re-save, so it deliberately performs NO type lookup and NO count work here. await this.writeCanonicalObject(path, noun) - - // Periodically save statistics - // Also save on first noun of each type to ensure low-count types are tracked - const shouldSave = counted && - (this.nounCountsByType[typeIndex] === 1 || // First noun of type - this.nounCountsByType[typeIndex] % 100 === 0) // Every 100th - if (shouldSave) { - await this.saveTypeStatistics() - } } /** @@ -3714,8 +4582,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { return this.deserializeNoun(noun) } } catch (error) { - // Entity not found - return null + // A real storage/deserialize fault is NOT "entity not found": + // readCanonicalObject returns null for genuine absence and only throws on + // a real fault, so masking it here reports a present-but-unreadable entity + // as missing. Absence → null, fault → propagate loudly. + if (isAbsentError(error)) return null + throw error } return null @@ -3825,8 +4697,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { return this.deserializeVerb(verb) } } catch (error) { - // Entity not found - return null + // A real storage/deserialize fault is NOT "relationship not found": + // readCanonicalObject returns null for genuine absence and only throws on + // a real fault, so masking it here reports a present-but-unreadable edge + // as missing. Absence → null, fault → propagate loudly. + if (isAbsentError(error)) return null + throw error } return null @@ -3840,13 +4716,22 @@ export abstract class BaseStorage extends BaseStorageAdapter { sourceId: string ): Promise { await this.ensureInitialized() + await this.ensureGraphFastPathProbed() prodLog.debug(`[BaseStorage] getVerbsBySource_internal: sourceId=${sourceId}, graphIndex=${!!this.graphIndex}, isInitialized=${this.graphIndex?.isInitialized}`) // Fast path - use GraphAdjacencyIndex if available (lazy-loaded). // 8.0 BigInt boundary: convert the UUID to an entity int up front and // resolve returned verb ints back to verb-id strings. - if (this.graphIndex && this.graphIndex.isInitialized && this.graphEntityIdResolver) { + // Honest gate: a provider that exposes isReady() and reports not-ready + // (count/manifest loaded but source→target edges NOT) is SKIPPED so we fall + // to the correct-but-slower canonical shard scan below instead of a silent []. + if ( + this.graphIndex && + this.graphIndex.isInitialized && + this.graphEntityIdResolver && + assessIndexReadiness(this.graphIndex) !== 'not-ready' + ) { try { const sourceInt = this.graphEntityIdResolver.getInt(sourceId) if (sourceInt === undefined) { @@ -4065,11 +4950,19 @@ export abstract class BaseStorage extends BaseStorageAdapter { targetId: string ): Promise { await this.ensureInitialized() + await this.ensureGraphFastPathProbed() // Fast path - use GraphAdjacencyIndex if available (lazy-loaded). // 8.0 BigInt boundary: convert the UUID to an entity int up front and // resolve returned verb ints back to verb-id strings. - if (this.graphIndex && this.graphIndex.isInitialized && this.graphEntityIdResolver) { + // Honest gate: a not-ready provider (count loaded but edges NOT) is SKIPPED so + // we fall to the correct-but-slower canonical shard scan instead of a silent []. + if ( + this.graphIndex && + this.graphIndex.isInitialized && + this.graphEntityIdResolver && + assessIndexReadiness(this.graphIndex) !== 'not-ready' + ) { try { const targetInt = this.graphEntityIdResolver.getInt(targetId) if (targetInt === undefined) { @@ -4215,7 +5108,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { * Save statistics data to storage (public interface) * @param statistics The statistics data to save */ - public async saveStatistics(statistics: StatisticsData): Promise { + public override async saveStatistics(statistics: StatisticsData): Promise { return this.saveStatisticsData(statistics) } @@ -4223,7 +5116,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { * Get statistics data from storage (public interface) * @returns Promise that resolves to the statistics data or null if not found */ - public async getStatistics(): Promise { + public override async getStatistics(): Promise { return this.getStatisticsData() } @@ -4232,7 +5125,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { * This method should be implemented by each specific adapter * @param statistics The statistics data to save */ - protected abstract saveStatisticsData( + protected abstract override saveStatisticsData( statistics: StatisticsData ): Promise @@ -4241,5 +5134,5 @@ export abstract class BaseStorage extends BaseStorageAdapter { * This method should be implemented by each specific adapter * @returns Promise that resolves to the statistics data or null if not found */ - protected abstract getStatisticsData(): Promise + protected abstract override getStatisticsData(): Promise } diff --git a/src/storage/blobStorage.ts b/src/storage/blobStorage.ts index 4d98c0bc..f83511f9 100644 --- a/src/storage/blobStorage.ts +++ b/src/storage/blobStorage.ts @@ -15,6 +15,7 @@ import { createHash } from 'crypto' import { unwrapBinaryData } from './binaryDataCodec.js' +import { InMemoryMutex } from '../utils/mutex.js' /** * @description Key-value bridge the blob store persists through. Implemented @@ -47,8 +48,19 @@ export interface BlobMetadata { compression: 'none' | 'zstd' /** Creation timestamp (epoch ms). */ createdAt: number - /** Number of logical references to this blob (deduplicated writes). */ + /** Number of LIVE logical references to this blob (deduplicated writes). */ refCount: number + /** + * Number of persisted generation record-sets (Model-B before-images) that + * reference this hash — the blob's membership in the temporal history. + * Bytes are physically reclaimed only when BOTH counts are zero, and only + * by history compaction: live references protect the present, history + * references protect every `asOf` read inside the retention window (pins + * ride generation pinning, which compaction already respects). Absent on + * metas written before the temporal contract existed (treated as 0; the + * one-time open-time backfill makes legacy stores exact). + */ + historyRefCount?: number } /** @@ -147,7 +159,9 @@ interface CacheEntry { * @example * const hash = await blobStorage.write(buffer, { mimeType: 'image/png' }) * const bytes = await blobStorage.read(hash) // verified against the hash - * await blobStorage.delete(hash) // decrements refCount first + * await blobStorage.release(hash) // drop one LIVE reference + * // bytes are physically reclaimed only by history compaction, once no + * // live reference AND no in-window generation references the hash */ export class BlobStorage { private adapter: BlobStoreAdapter @@ -164,6 +178,19 @@ export class BlobStorage { private readonly CACHE_MAX_SIZE = 100 * 1024 * 1024 // 100MB default private readonly COMPRESSION_THRESHOLD = 1024 // 1KB - don't compress smaller + /** + * Per-hash write serialization. Every reference-count-bearing mutation + * (`write`'s dedup check-then-act, `delete`'s decrement-then-maybe-remove) + * is a read-modify-write over `blob-meta:` — unserialized, two + * concurrent writes of identical content both saw "absent" and both wrote + * `refCount: 1` (one reference lost → a later delete removed bytes another + * file still referenced), and concurrent increments/decrements could drop + * counts. Keyed by hash, so distinct content never contends; the process + * is the whole concurrency domain (storage enforces single-writer per + * directory). + */ + private readonly hashLocks = new InMemoryMutex() + /** * @param adapter - Key-value bridge to persist through. * @param options - `cacheMaxSize` bounds the LRU read cache (bytes, @@ -222,45 +249,53 @@ export class BlobStorage { async write(data: Buffer, options: BlobWriteOptions = {}): Promise { const hash = BlobStorage.hash(data) - // Deduplication: identical content already stored — just add a reference. - if (await this.has(hash)) { - await this.incrementRefCount(hash) + // The dedup decision (exists → add a reference; absent → create with + // refCount 1) is check-then-act over the same metadata a concurrent + // same-content write mutates — serialized per hash so N concurrent + // writes of identical content yield exactly N references, never a lost + // count (a lost reference turns a later delete into premature removal + // of bytes another file still needs). + return this.hashLocks.runExclusive(hash, async () => { + // Deduplication: identical content already stored — just add a reference. + if (await this.has(hash)) { + await this.incrementRefCount(hash) + return hash + } + + await this.ensureCompressionReady() + + // Determine compression strategy + const compression = this.selectCompression(data, options) + + // Compress if needed + let finalData = data + let compressedSize = data.length + + if (compression === 'zstd' && this.zstdCompress) { + finalData = await this.zstdCompress(data) + compressedSize = finalData.length + } + + // Record the ACTUAL compression state, not the intended one — prevents + // corruption if compression failed to initialize. + const actualCompression = finalData === data ? 'none' : compression + const metadata: BlobMetadata = { + hash, + size: data.length, + compressedSize, + compression: actualCompression, + createdAt: Date.now(), + refCount: 1 + } + + await this.adapter.put(`blob:${hash}`, finalData) + await this.adapter.put(`blob-meta:${hash}`, Buffer.from(JSON.stringify(metadata))) + + // Write-through cache (caches the ORIGINAL bytes, not the compressed form) + this.addToCache(hash, data, metadata) + return hash - } - - await this.ensureCompressionReady() - - // Determine compression strategy - const compression = this.selectCompression(data, options) - - // Compress if needed - let finalData = data - let compressedSize = data.length - - if (compression === 'zstd' && this.zstdCompress) { - finalData = await this.zstdCompress(data) - compressedSize = finalData.length - } - - // Record the ACTUAL compression state, not the intended one — prevents - // corruption if compression failed to initialize. - const actualCompression = finalData === data ? 'none' : compression - const metadata: BlobMetadata = { - hash, - size: data.length, - compressedSize, - compression: actualCompression, - createdAt: Date.now(), - refCount: 1 - } - - await this.adapter.put(`blob:${hash}`, finalData) - await this.adapter.put(`blob-meta:${hash}`, Buffer.from(JSON.stringify(metadata))) - - // Write-through cache (caches the ORIGINAL bytes, not the compressed form) - this.addToCache(hash, data, metadata) - - return hash + }) } /** @@ -334,23 +369,105 @@ export class BlobStorage { } /** - * @description Drop one reference to the blob. The stored bytes and - * metadata are physically deleted only when the reference count reaches - * zero — deduplicated content shared by other writers survives. + * @description Drop one LIVE reference to the blob. Never deletes bytes — + * blob content is immutable under the temporal model, exactly like every + * other record: a past generation's `asOf` read may still need these bytes + * even when no live file references them. Physical reclamation happens in + * ONE place only — history compaction via {@link reclaimIfUnreferenced}, + * once no live reference AND no retained generation references the hash. * * @param hash - The blob's SHA-256 hash. */ - async delete(hash: string): Promise { - const refCount = await this.decrementRefCount(hash) + async release(hash: string): Promise { + await this.hashLocks.runExclusive(hash, async () => { + await this.decrementRefCount(hash) + }) + } - // Only delete if no references remain - if (refCount > 0) { - return - } + /** + * @description Record that one persisted generation record-set references + * this hash (called by the commit path BEFORE the record-set is written — + * a crash between the two can only over-count, which leaks until the scrub + * recounts; it can never under-count, which would risk premature deletion). + * A missing meta (bytes never stored or already gone) is skipped with a + * warning — counting it could not make its bytes readable. + * @param hash - The blob's SHA-256 hash. + */ + async recordHistoryReference(hash: string): Promise { + await this.hashLocks.runExclusive(hash, async () => { + const metadata = await this.getMetadata(hash) + if (!metadata) { + console.warn( + `[BlobStorage] history reference recorded for absent blob ${hash} — skipped` + ) + return + } + metadata.historyRefCount = (metadata.historyRefCount ?? 0) + 1 + await this.adapter.put(`blob-meta:${hash}`, Buffer.from(JSON.stringify(metadata))) + }) + } - await this.adapter.delete(`blob:${hash}`) - await this.adapter.delete(`blob-meta:${hash}`) - this.removeFromCache(hash) + /** + * @description Drop one history reference (called by compaction AFTER the + * referencing generation record-set is deleted — the safe ordering: a crash + * between the two over-counts, never under-counts). Floored at zero. + * @param hash - The blob's SHA-256 hash. + */ + async releaseHistoryReference(hash: string): Promise { + await this.hashLocks.runExclusive(hash, async () => { + const metadata = await this.getMetadata(hash) + if (!metadata) return + metadata.historyRefCount = Math.max(0, (metadata.historyRefCount ?? 0) - 1) + await this.adapter.put(`blob-meta:${hash}`, Buffer.from(JSON.stringify(metadata))) + }) + } + + /** + * @description Physically delete the blob's bytes + metadata IFF nothing + * references it: zero live references AND zero history references. The one + * reclamation point in the system, invoked by history compaction after it + * releases the reclaimed generations' references. Atomic per hash. + * @param hash - The blob's SHA-256 hash. + * @returns `true` when the bytes were reclaimed. + */ + async reclaimIfUnreferenced(hash: string): Promise { + return this.hashLocks.runExclusive(hash, async () => { + const metadata = await this.getMetadata(hash) + if (!metadata) return false + if ((metadata.refCount ?? 0) > 0 || (metadata.historyRefCount ?? 0) > 0) { + return false + } + await this.adapter.delete(`blob:${hash}`) + await this.adapter.delete(`blob-meta:${hash}`) + this.removeFromCache(hash) + return true + }) + } + + /** + * @description Set the history reference count to an absolute value — the + * backfill/scrub primitive (recounts derived from the actual generation + * records replace whatever the incremental counters hold). Idempotent. + * @param hash - The blob's SHA-256 hash. + * @param count - The exact history reference count. + */ + async setHistoryRefCount(hash: string, count: number): Promise { + await this.hashLocks.runExclusive(hash, async () => { + const metadata = await this.getMetadata(hash) + if (!metadata) return + metadata.historyRefCount = Math.max(0, count) + await this.adapter.put(`blob-meta:${hash}`, Buffer.from(JSON.stringify(metadata))) + }) + } + + /** + * @description Enumerate every stored blob hash (from the metadata keys) — + * the backfill/scrub walk. O(stored blobs). + * @returns All hashes with a stored metadata record. + */ + async listHashes(): Promise { + const keys = await this.adapter.list('blob-meta:') + return keys.map((k) => k.slice('blob-meta:'.length)) } /** @@ -403,6 +520,8 @@ export class BlobStorage { /** * Increment the reference count for an existing blob. + * Caller MUST hold the per-hash lock ({@link hashLocks}) — this is a raw + * read-modify-write with no serialization of its own. */ private async incrementRefCount(hash: string): Promise { const metadata = await this.getMetadata(hash) @@ -417,6 +536,8 @@ export class BlobStorage { /** * Decrement the reference count for a blob (floored at zero). + * Caller MUST hold the per-hash lock ({@link hashLocks}) — this is a raw + * read-modify-write with no serialization of its own. */ private async decrementRefCount(hash: string): Promise { const metadata = await this.getMetadata(hash) diff --git a/src/storage/brainFormat.ts b/src/storage/brainFormat.ts new file mode 100644 index 00000000..2e6488e9 --- /dev/null +++ b/src/storage/brainFormat.ts @@ -0,0 +1,136 @@ +/** + * @module storage/brainFormat + * @description The 7.x → 8.0 version-handshake marker: the small persisted + * artifact at `_system/brain-format.json` that lets Brainy and a native + * metadata/index provider agree, at open time, on the on-disk format of the + * brain AND of its derived indexes. + * + * Two fields, two owners: + * + * - `dataFormat` — **Brainy-owned**. The data-layer version string (the + * canonical entity / relationship / generation-record layout). Brainy bumps + * it when that data layout changes; a native provider reads it only to + * confirm the major line it is binding against (e.g. "this is an 8.0 brain"). + * - `indexEpoch` — **SHARED**. A monotonic integer that Brainy and the native + * provider bump TOGETHER, in lockstep, on any coordinated release where the + * on-disk format of ANY derived index (the HNSW vectors, the metadata + * postings, or the graph adjacency) changes. It is the single switch that + * declares "every derived index written by an older build is stale and must + * be rebuilt from the canonical records." + * + * Open-time handshake. On open, Brainy compares the on-disk `indexEpoch` + * against the compiled {@link EXPECTED_INDEX_EPOCH}. A mismatch — OR an absent + * marker (a pre-handshake brain, or a brand-new store) — means the derived + * indexes on disk predate this build, so Brainy rebuilds them from the + * canonical records and only THEN re-stamps the marker. The provider reads the + * same surface synchronously through `brain.formatInfo()` at its own provider + * init() to confirm "running data-format X, index epoch N" before binding its + * native readers. + * + * Lockstep contract. NEVER bump {@link EXPECTED_INDEX_EPOCH} unilaterally. It + * advances only on a coordinated release whose on-disk derived-index format + * actually changed, and both projects ship the SAME new value in the same + * release — so a brain written by either side is recognised as current by the + * other, and a brain written by an older build of either side is recognised as + * stale and rebuilt. The constant living here makes this module the single + * source of truth both sides reference. + * + * Non-destructive stamping. The marker is the LAST thing written, AFTER the + * rebuild has verified. A crash between the rebuild and the stamp leaves the + * old / absent marker on disk, so the next open re-detects the drift and + * re-runs the (idempotent) rebuild — the marker is never advanced ahead of the + * indexes it certifies. This mirrors the generational record layer's + * "build-new → verify → atomic-rename" commit discipline + * (`src/db/generationStore.ts`). + */ + +/** + * @description The narrow storage surface the marker helpers need — the + * raw-object read/write primitives every `BaseStorage` adapter implements (the + * same surface the generational record layer uses for `_system/` artifacts). + * Declared structurally so this module carries no runtime dependency on the + * adapter class. + */ +export interface BrainFormatStorage { + /** Read a raw object at a storage-root-relative path (`null` if absent). */ + readRawObject(path: string): Promise + /** Write a raw object at a storage-root-relative path (atomic tmp+rename on disk). */ + writeRawObject(path: string, data: unknown): Promise +} + +/** Storage-root-relative path of the version-handshake marker. */ +export const BRAIN_FORMAT_PATH = '_system/brain-format.json' + +/** + * @description The compiled derived-index format epoch this build expects on + * disk. SHARED and lockstep-bumped with the native provider: advance it (in + * BOTH projects, to the same value, in a coordinated release) on any change to + * the on-disk format of any derived index — never on one side alone. Start = 1 + * (the 8.0 GA baseline). An on-disk `indexEpoch` that differs from this — or an + * absent marker — triggers a full derived-index rebuild on open. + */ +export const EXPECTED_INDEX_EPOCH = 1 + +/** + * @description The data-layer format string this build writes and runs as. + * Brainy-owned; bumped when the canonical data layout changes. Start = '8.0'. + */ +export const CURRENT_DATA_FORMAT = '8.0' + +/** + * @description The persisted shape of `_system/brain-format.json` — the + * value `brain.formatInfo()` returns for the running brain, and the value + * read back from disk to drive the epoch-drift rebuild trigger. + */ +export interface BrainFormat { + /** Brainy-owned data-layer version string (e.g. `'8.0'`). */ + dataFormat: string + /** Shared, lockstep-bumped derived-index format epoch (e.g. `1`). */ + indexEpoch: number +} + +/** + * @description Read the on-disk version-handshake marker, or `null` when it is + * absent (a pre-handshake brain, or a brand-new store). A malformed marker + * (not an object, missing either field, or a non-finite epoch) is also treated + * as `null` — a corrupt marker forces a safe rebuild rather than trusting a bad + * epoch. + * @param storage - The brain's storage adapter (raw-object surface). + * @returns The parsed {@link BrainFormat}, or `null`. + * @example + * const onDisk = await readBrainFormat(brain.storage) + * const stale = onDisk === null || onDisk.indexEpoch !== EXPECTED_INDEX_EPOCH + */ +export async function readBrainFormat( + storage: Pick +): Promise { + const raw = (await storage.readRawObject(BRAIN_FORMAT_PATH)) as Partial | null + if (raw === null || typeof raw !== 'object') return null + if ( + typeof raw.dataFormat !== 'string' || + typeof raw.indexEpoch !== 'number' || + !Number.isFinite(raw.indexEpoch) + ) { + return null + } + return { dataFormat: raw.dataFormat, indexEpoch: raw.indexEpoch } +} + +/** + * @description Stamp this build's {@link CURRENT_DATA_FORMAT} / + * {@link EXPECTED_INDEX_EPOCH} to `_system/brain-format.json` (an atomic + * tmp+rename on the filesystem adapter). MUST be called only AFTER the + * derived-index rebuild has verified: the marker certifies the indexes on + * disk, so advancing it ahead of them would suppress the rebuild a future open + * needs (the non-destructive contract — see the module docs). + * @param storage - The brain's storage adapter (raw-object surface). + */ +export async function writeBrainFormat( + storage: Pick +): Promise { + const marker: BrainFormat = { + dataFormat: CURRENT_DATA_FORMAT, + indexEpoch: EXPECTED_INDEX_EPOCH + } + await storage.writeRawObject(BRAIN_FORMAT_PATH, marker) +} diff --git a/src/storage/cacheManager.ts b/src/storage/cacheManager.ts deleted file mode 100644 index cfeb61d7..00000000 --- a/src/storage/cacheManager.ts +++ /dev/null @@ -1,1257 +0,0 @@ -/** - * Multi-level Cache Manager - * - * Implements a three-level caching strategy: - * - Level 1: Hot cache (most accessed nodes) - RAM (automatically detecting and adjusting in each environment) - * - Level 2: Warm cache (recent nodes) - OPFS, Filesystem or S3 depending on environment - * - Level 3: Cold storage (all nodes) - OPFS, Filesystem or S3 depending on environment - */ - -import { HNSWNoun, GraphVerb, HNSWVerb } from '../coreTypes.js' -import { BrainyError } from '../errors/brainyError.js' - -// Type aliases for better readability -type HNSWNode = HNSWNoun -type Edge = GraphVerb - -// Cache entry with metadata for LRU and TTL management -interface CacheEntry { - data: T - lastAccessed: number - accessCount: number - expiresAt: number | null -} - -// Cache statistics for monitoring and tuning -interface CacheStats { - hits: number - misses: number - evictions: number - size: number - maxSize: number - hotCacheSize: number - warmCacheSize: number - hotCacheHits: number - hotCacheMisses: number - warmCacheHits: number - warmCacheMisses: number -} - -// Storage type for warm and cold caches. Brainy 8.0 only ships filesystem + -// memory tiers, but the enum is retained for diagnostic/log output. -enum StorageType { - MEMORY, - FILESYSTEM -} - -/** - * Multi-level cache manager for efficient data access - */ -export class CacheManager { - // Hot cache (RAM) - private hotCache = new Map>() - - // Cache statistics - private stats: CacheStats = { - hits: 0, - misses: 0, - evictions: 0, - size: 0, - maxSize: 0, - hotCacheSize: 0, - warmCacheSize: 0, - hotCacheHits: 0, - hotCacheMisses: 0, - warmCacheHits: 0, - warmCacheMisses: 0 - } - - // Storage configuration - private warmStorageType: StorageType - private coldStorageType: StorageType - - // Cache configuration - private hotCacheMaxSize: number - private hotCacheEvictionThreshold: number - private warmCacheTTL: number - private batchSize: number - - // Auto-tuning configuration - private autoTune: boolean - private lastAutoTuneTime: number = 0 - private autoTuneInterval: number = 5 * 60 * 1000 // 5 minutes - private storageStatistics: any = null - - // Storage adapters for warm and cold caches - private warmStorage: any - private coldStorage: any - - // Store options for later reference - private options: { - hotCacheMaxSize?: number - hotCacheEvictionThreshold?: number - warmCacheTTL?: number - batchSize?: number - autoTune?: boolean - warmStorage?: any - coldStorage?: any - readOnly?: boolean - environmentConfig?: { - node?: { - hotCacheMaxSize?: number - hotCacheEvictionThreshold?: number - warmCacheTTL?: number - batchSize?: number - } - [key: string]: { - hotCacheMaxSize?: number - hotCacheEvictionThreshold?: number - warmCacheTTL?: number - batchSize?: number - } | undefined - } - } - - /** - * Initialize the cache manager - * @param options Configuration options - */ - constructor(options: { - hotCacheMaxSize?: number - hotCacheEvictionThreshold?: number - warmCacheTTL?: number - batchSize?: number - autoTune?: boolean - warmStorage?: any - coldStorage?: any - readOnly?: boolean - environmentConfig?: { - node?: { - hotCacheMaxSize?: number - hotCacheEvictionThreshold?: number - warmCacheTTL?: number - batchSize?: number - } - [key: string]: { - hotCacheMaxSize?: number - hotCacheEvictionThreshold?: number - warmCacheTTL?: number - batchSize?: number - } | undefined - } - } = {}) { - // Store options for later reference - this.options = options - - // Set storage types (Brainy 8.0 ships filesystem only) - this.warmStorageType = this.detectWarmStorageType() - this.coldStorageType = this.detectColdStorageType() - - // Initialize storage adapters - this.warmStorage = options.warmStorage || this.initializeWarmStorage() - this.coldStorage = options.coldStorage || this.initializeColdStorage() - - // Set auto-tuning flag - this.autoTune = options.autoTune !== undefined ? options.autoTune : true - - // Brainy 8.0 only runs on Node-like runtimes, so only the `node` slot of - // environmentConfig is honored. - const envConfig = options.environmentConfig?.node - - // Set default values or use environment-specific values or global values - this.hotCacheMaxSize = envConfig?.hotCacheMaxSize || options.hotCacheMaxSize || this.detectOptimalCacheSize() - this.hotCacheEvictionThreshold = envConfig?.hotCacheEvictionThreshold || options.hotCacheEvictionThreshold || 0.8 - this.warmCacheTTL = envConfig?.warmCacheTTL || options.warmCacheTTL || 24 * 60 * 60 * 1000 // 24 hours - this.batchSize = envConfig?.batchSize || options.batchSize || 10 - - // If auto-tuning is enabled, perform initial tuning - if (this.autoTune) { - this.tuneParameters() - } - - // Log configuration - if (process.env.DEBUG) { - console.log('Cache Manager initialized with configuration:', { - environment: 'node', - hotCacheMaxSize: this.hotCacheMaxSize, - hotCacheEvictionThreshold: this.hotCacheEvictionThreshold, - warmCacheTTL: this.warmCacheTTL, - batchSize: this.batchSize, - autoTune: this.autoTune, - warmStorageType: StorageType[this.warmStorageType], - coldStorageType: StorageType[this.coldStorageType] - }) - } - } - - - /** - * Detect the optimal cache size based on available memory and operating mode - * - * Enhanced to better handle large datasets in S3 or other storage: - * - Increases cache size for read-only mode - * - Adjusts based on total dataset size when available - * - Provides more aggressive caching for large datasets - * - Optimizes memory usage based on environment - */ - private detectOptimalCacheSize(): number { - try { - // Default to a conservative value - const defaultSize = 1000 - - // Get the total dataset size if available - const totalItems = this.storageStatistics ? - (this.storageStatistics.totalNodes || 0) + (this.storageStatistics.totalEdges || 0) : 0 - - // Determine if we're dealing with a large dataset (>100K items) - const isLargeDataset = totalItems > 100000 - - // Check if we're in read-only mode (from parent Brainy instance) - const isReadOnly = this.options?.readOnly || false - - try { - // Synchronous path can't use dynamic imports, so we use conservative - // assumed defaults. The async variant (detectAvailableMemory) reads - // real values via `node:os` when possible. - const estimatedFreeMemory = 4 * 1024 * 1024 * 1024 // Assume 4GB free - - // Estimate average entry size (in bytes) - // This is a conservative estimate for complex objects with vectors - const ESTIMATED_BYTES_PER_ENTRY = 1024 // 1KB per entry - - // Base memory percentage - 10% by default - let memoryPercentage = 0.1 - - // Adjust based on operating mode and dataset size - if (isReadOnly) { - // In read-only mode, we can use more memory for caching - memoryPercentage = 0.25 // 25% of free memory - - // For large datasets in read-only mode, be even more aggressive - if (isLargeDataset) { - memoryPercentage = 0.4 // 40% of free memory - } - } else if (isLargeDataset) { - // For large datasets in normal mode, increase slightly - memoryPercentage = 0.15 // 15% of free memory - } - - // Calculate optimal size based on adjusted percentage - const optimalSize = Math.max( - Math.floor(estimatedFreeMemory * memoryPercentage / ESTIMATED_BYTES_PER_ENTRY), - 1000 - ) - - // If we know the total dataset size, cap at a reasonable percentage - if (totalItems > 0) { - // In read-only mode, we can cache a larger percentage - const maxPercentage = isReadOnly ? 0.5 : 0.3 - const maxItems = Math.ceil(totalItems * maxPercentage) - - // Return the smaller of the two to avoid excessive memory usage - return Math.min(optimalSize, maxItems) - } - - return optimalSize - } catch (error) { - console.warn('Failed to detect optimal cache size:', error) - return defaultSize - } - } catch (error) { - console.warn('Error detecting optimal cache size:', error) - return 1000 // Conservative default - } - } - - /** - * Async version of detectOptimalCacheSize that uses dynamic imports - * to access system information in Node.js environments - * - * This method provides more accurate memory detection by using - * the OS module's dynamic import in Node.js environments - */ - private async detectOptimalCacheSizeAsync(): Promise { - try { - // Default to a conservative value - const defaultSize = 1000 - - // Get the total dataset size if available - const totalItems = this.storageStatistics ? - (this.storageStatistics.totalNodes || 0) + (this.storageStatistics.totalEdges || 0) : 0 - - // Determine if we're dealing with a large dataset (>100K items) - const isLargeDataset = totalItems > 100000 - - // Check if we're in read-only mode (from parent Brainy instance) - const isReadOnly = this.options?.readOnly || false - - // Get memory information based on environment - const memoryInfo = await this.detectAvailableMemory() - - // If memory detection failed, use the synchronous method - if (!memoryInfo) { - return this.detectOptimalCacheSize() - } - - // Estimate average entry size (in bytes) - // This is a conservative estimate for complex objects with vectors - const ESTIMATED_BYTES_PER_ENTRY = 1024 // 1KB per entry - - // Base memory percentage - 10% by default - let memoryPercentage = 0.1 - - // Adjust based on operating mode and dataset size - if (isReadOnly) { - // In read-only mode, we can use more memory for caching - memoryPercentage = 0.25 // 25% of free memory - - // For large datasets in read-only mode, be even more aggressive - if (isLargeDataset) { - memoryPercentage = 0.4 // 40% of free memory - } - } else if (isLargeDataset) { - // For large datasets in normal mode, increase slightly - memoryPercentage = 0.15 // 15% of free memory - } - - // Calculate optimal size based on adjusted percentage - const optimalSize = Math.max( - Math.floor(memoryInfo.freeMemory * memoryPercentage / ESTIMATED_BYTES_PER_ENTRY), - 1000 - ) - - // If we know the total dataset size, cap at a reasonable percentage - if (totalItems > 0) { - // In read-only mode, we can cache a larger percentage - const maxPercentage = isReadOnly ? 0.5 : 0.3 - const maxItems = Math.ceil(totalItems * maxPercentage) - - // Return the smaller of the two to avoid excessive memory usage - return Math.min(optimalSize, maxItems) - } - - return optimalSize - } catch (error) { - console.warn('Error detecting optimal cache size asynchronously:', error) - return 1000 // Conservative default - } - } - - /** - * Detect available memory using Node-like runtime primitives. - * - * Brainy 8.0 only runs on Node.js, Bun, and Deno, so this method reads real - * values from `node:os` (`process.memoryUsage()` is not used here because we - * want system-wide free memory, not just the V8 heap). If the import fails, - * we fall back to conservative defaults that match the synchronous path. - * - * @returns An object with totalMemory and freeMemory in bytes, or null if detection fails - */ - private async detectAvailableMemory(): Promise<{ totalMemory: number, freeMemory: number } | null> { - try { - try { - // Use dynamic import for OS module - const os = await import('node:os') - - // Get actual system memory information - const totalMemory = os.totalmem() - const freeMemory = os.freemem() - - return { totalMemory, freeMemory } - } catch (error) { - console.warn('Failed to detect memory via node:os:', error) - } - - // If detection failed, use conservative defaults - return { - totalMemory: 8 * 1024 * 1024 * 1024, // Assume 8GB total - freeMemory: 4 * 1024 * 1024 * 1024 // Assume 4GB free - } - } catch (error) { - console.warn('Memory detection failed:', error) - return null - } - } - - /** - * Tune cache parameters based on statistics and environment - * This method is called periodically if auto-tuning is enabled - * - * The auto-tuning process: - * 1. Retrieves storage statistics if available - * 2. Tunes each parameter based on statistics and environment - * 3. Logs the tuned parameters if debug is enabled - * - * Auto-tuning helps optimize cache performance by adapting to: - * - The current environment (Node.js, browser, worker) - * - Available system resources (memory, CPU) - * - Usage patterns (read-heavy vs. write-heavy workloads) - * - Cache efficiency (hit/miss ratios) - */ - private async tuneParameters(): Promise { - // Skip if auto-tuning is disabled - if (!this.autoTune) return - - // Check if it's time to tune parameters - const now = Date.now() - if (now - this.lastAutoTuneTime < this.autoTuneInterval) return - - // Update last tune time - this.lastAutoTuneTime = now - - try { - // Get storage statistics if available - if (this.coldStorage && typeof this.coldStorage.getStatistics === 'function') { - this.storageStatistics = await this.coldStorage.getStatistics() - } - - // Get cache statistics for adaptive tuning - const cacheStats = this.getStats() - - // Use the async version of tuneHotCacheSize which uses detectOptimalCacheSizeAsync - await this.tuneHotCacheSize() - - // Tune eviction threshold based on hit/miss ratio - this.tuneEvictionThreshold(cacheStats) - - // Tune warm cache TTL based on access patterns - this.tuneWarmCacheTTL(cacheStats) - - // Tune batch size based on access patterns and storage type - this.tuneBatchSize(cacheStats) - - // Log tuned parameters if debug is enabled - if (process.env.DEBUG) { - console.log('Cache parameters auto-tuned:', { - hotCacheMaxSize: this.hotCacheMaxSize, - hotCacheEvictionThreshold: this.hotCacheEvictionThreshold, - warmCacheTTL: this.warmCacheTTL, - batchSize: this.batchSize, - cacheStats: { - hotCacheSize: cacheStats.hotCacheSize, - warmCacheSize: cacheStats.warmCacheSize, - hotCacheHits: cacheStats.hotCacheHits, - hotCacheMisses: cacheStats.hotCacheMisses, - warmCacheHits: cacheStats.warmCacheHits, - warmCacheMisses: cacheStats.warmCacheMisses - } - }) - } - } catch (error) { - console.warn('Error during cache parameter auto-tuning:', error) - } - } - - /** - * Tune hot cache size based on statistics, environment, and operating mode - * - * The hot cache size is tuned based on: - * 1. Available memory in the current environment - * 2. Total number of nodes and edges in the system - * 3. Cache hit/miss ratio - * 4. Operating mode (read-only vs. read-write) - * 5. Storage type (S3, filesystem, memory) - * - * Enhanced algorithm: - * - Start with a size based on available memory and operating mode - * - For large datasets in S3 or other remote storage, use more aggressive caching - * - Adjust based on access patterns (read-heavy vs. write-heavy) - * - For read-only mode, prioritize cache size over eviction speed - * - Dynamically adjust based on hit/miss ratio and query patterns - */ - private async tuneHotCacheSize(): Promise { - // Use the async version to get more accurate memory information - let optimalSize = await this.detectOptimalCacheSizeAsync() - - const isReadOnly = this.options?.readOnly || false - - // Adjust by total entity count if we have storage statistics. - if (this.storageStatistics) { - const totalItems = (this.storageStatistics.totalNodes || 0) + - (this.storageStatistics.totalEdges || 0) - - if (totalItems > 0) { - let percentageToCache = isReadOnly ? 0.3 : 0.2 - - if (totalItems > 1_000_000) { - percentageToCache = Math.min(percentageToCache, 0.15) - } else if (totalItems > 100_000) { - percentageToCache = Math.min(percentageToCache, 0.25) - } - - const statisticsBasedSize = Math.ceil(totalItems * percentageToCache) - optimalSize = Math.min(optimalSize, statisticsBasedSize) - } - } - - // Adjust by observed hit ratio. - const totalAccesses = this.stats.hits + this.stats.misses - if (totalAccesses > 100) { - const hitRatio = this.stats.hits / totalAccesses - - if (hitRatio < 0.5) { - const baseAdjustment = 0.5 - hitRatio - const hitRatioFactor = isReadOnly ? 1 + baseAdjustment * 1.5 : 1 + baseAdjustment - optimalSize = Math.ceil(optimalSize * hitRatioFactor) - } else if (hitRatio > 0.9 && !isReadOnly) { - optimalSize = Math.ceil(optimalSize * 0.9) - } - } - - // Read-heavy workloads warrant a slightly larger cache. - if (this.storageStatistics?.operations) { - const ops = this.storageStatistics.operations - const totalOps = ops.total || 1 - const readOps = (ops.search || 0) + (ops.get || 0) - - if (totalOps > 100 && readOps / totalOps > 0.8) { - optimalSize = Math.ceil(optimalSize * 1.2) - } - } - - const minSize = isReadOnly ? 2000 : 1000 - optimalSize = Math.max(optimalSize, minSize) - - // Update the hot cache max size - this.hotCacheMaxSize = optimalSize - this.stats.maxSize = optimalSize - } - - /** - * Tune eviction threshold based on statistics - * - * The eviction threshold determines when items start being evicted from the hot cache. - * It is tuned based on: - * 1. Cache hit/miss ratio - * 2. Operation patterns (read-heavy vs. write-heavy workloads) - * 3. Memory pressure and available resources - * - * Algorithm: - * - Start with a default threshold of 0.8 (80% of max size) - * - For high hit ratios, increase the threshold to keep more items in cache - * - For low hit ratios, decrease the threshold to evict items more aggressively - * - For read-heavy workloads, use a higher threshold - * - For write-heavy workloads, use a lower threshold - * - Under memory pressure, use a lower threshold to conserve resources - * - * @param cacheStats Optional cache statistics for more adaptive tuning - */ - private tuneEvictionThreshold(cacheStats?: CacheStats): void { - // Default threshold - let threshold = 0.8 - - // Use provided cache stats or internal stats - const stats = cacheStats || this.getStats() - - // Adjust based on hit/miss ratio if we have enough data - const totalHotAccesses = stats.hotCacheHits + stats.hotCacheMisses - if (totalHotAccesses > 100) { - const hotHitRatio = stats.hotCacheHits / totalHotAccesses - - // If hit ratio is high, we can use a higher threshold - // If hit ratio is low, we should use a lower threshold to evict more aggressively - if (hotHitRatio > 0.8) { - // High hit ratio, increase threshold (up to 0.9) - threshold = Math.min(0.9, 0.8 + (hotHitRatio - 0.8) * 0.5) - } else if (hotHitRatio < 0.5) { - // Low hit ratio, decrease threshold (down to 0.6) - threshold = Math.max(0.6, 0.8 - (0.5 - hotHitRatio) * 0.5) - } - } - - // If we have storage statistics with operation counts, adjust based on operation patterns - if (this.storageStatistics && this.storageStatistics.operations) { - const ops = this.storageStatistics.operations - const totalOps = ops.total || 1 - - // Calculate read/write ratio - const readOps = ops.search || 0 - const writeOps = (ops.add || 0) + (ops.update || 0) + (ops.delete || 0) - - if (totalOps > 100) { - const readRatio = readOps / totalOps - const writeRatio = writeOps / totalOps - - // For read-heavy workloads, use higher threshold - // For write-heavy workloads, use lower threshold - if (readRatio > 0.8) { - // Read-heavy, increase threshold slightly - threshold = Math.min(0.9, threshold + 0.05) - } else if (writeRatio > 0.5) { - // Write-heavy, decrease threshold - threshold = Math.max(0.6, threshold - 0.1) - } - } - } - - // Check memory pressure - if hot cache is growing too fast relative to hits, - // reduce the threshold to conserve memory - if (stats.hotCacheSize > 0 && totalHotAccesses > 0) { - const sizeToAccessRatio = stats.hotCacheSize / totalHotAccesses - - // If the ratio is high, it means we're caching a lot but not getting many hits - if (sizeToAccessRatio > 10) { - // Reduce threshold more aggressively under high memory pressure - threshold = Math.max(0.5, threshold - 0.1) - } - } - - // If we're in read-only mode, we can be more aggressive with caching - const isReadOnly = this.options?.readOnly || false - if (isReadOnly) { - threshold = Math.min(0.95, threshold + 0.05) - } - - // Update the eviction threshold - this.hotCacheEvictionThreshold = threshold - } - - /** - * Tune warm cache TTL based on statistics - * - * The warm cache TTL determines how long items remain in the warm cache. - * It is tuned based on: - * 1. Update frequency from operation statistics - * 2. Warm cache hit/miss ratio - * 3. Access patterns and frequency - * 4. Available storage resources - * - * Algorithm: - * - Start with a default TTL of 24 hours - * - For frequently updated data, use a shorter TTL - * - For rarely updated data, use a longer TTL - * - For frequently accessed data, use a longer TTL - * - For rarely accessed data, use a shorter TTL - * - Under storage pressure, use a shorter TTL - * - * @param cacheStats Optional cache statistics for more adaptive tuning - */ - private tuneWarmCacheTTL(cacheStats?: CacheStats): void { - // Default TTL (24 hours) - let ttl = 24 * 60 * 60 * 1000 - - // Use provided cache stats or internal stats - const stats = cacheStats || this.getStats() - - // Adjust based on warm cache hit/miss ratio if we have enough data - const totalWarmAccesses = stats.warmCacheHits + stats.warmCacheMisses - if (totalWarmAccesses > 50) { - const warmHitRatio = stats.warmCacheHits / totalWarmAccesses - - // If warm cache hit ratio is high, items in warm cache are useful - // so we should keep them longer - if (warmHitRatio > 0.7) { - // High hit ratio, increase TTL (up to 36 hours) - ttl = Math.min(36 * 60 * 60 * 1000, ttl * (1 + (warmHitRatio - 0.7))) - } else if (warmHitRatio < 0.3) { - // Low hit ratio, decrease TTL (down to 12 hours) - ttl = Math.max(12 * 60 * 60 * 1000, ttl * (0.8 - (0.3 - warmHitRatio))) - } - } - - // If we have storage statistics with operation counts, adjust based on update frequency - if (this.storageStatistics && this.storageStatistics.operations) { - const ops = this.storageStatistics.operations - const totalOps = ops.total || 1 - const updateOps = (ops.update || 0) - - if (totalOps > 100) { - const updateRatio = updateOps / totalOps - - // For frequently updated data, use shorter TTL - // For rarely updated data, use longer TTL - if (updateRatio > 0.3) { - // Frequently updated, decrease TTL (down to 6 hours) - ttl = Math.max(6 * 60 * 60 * 1000, ttl * (1 - updateRatio * 0.5)) - } else if (updateRatio < 0.1) { - // Rarely updated, increase TTL (up to 48 hours) - ttl = Math.min(48 * 60 * 60 * 1000, ttl * (1.2 - updateRatio)) - } - } - } - - // Check warm cache size relative to hot cache size - // If warm cache is much larger than hot cache, reduce TTL to prevent excessive storage use - if (stats.warmCacheSize > 0 && stats.hotCacheSize > 0) { - const warmToHotRatio = stats.warmCacheSize / stats.hotCacheSize - - if (warmToHotRatio > 5) { - // Warm cache is much larger than hot cache, reduce TTL - ttl = Math.max(6 * 60 * 60 * 1000, ttl * (0.9 - Math.min(0.3, (warmToHotRatio - 5) / 20))) - } - } - - // If we're in read-only mode, we can use a longer TTL - const isReadOnly = this.options?.readOnly || false - if (isReadOnly) { - ttl = Math.min(72 * 60 * 60 * 1000, ttl * 1.5) - } - - // Update the warm cache TTL - this.warmCacheTTL = ttl - } - - /** - * Tune batch size based on environment, statistics, and operating mode - * - * The batch size determines how many items are processed in a single batch - * for operations like prefetching. It is tuned based on: - * 1. Current environment (Node.js, browser, worker) - * 2. Available memory - * 3. Operation patterns - * 4. Cache hit/miss ratio - * 5. Operating mode (read-only vs. read-write) - * 6. Storage type (S3, filesystem, memory) - * 7. Dataset size - * 8. Cache efficiency and access patterns - * - * Enhanced algorithm: - * - Start with a default based on the environment - * - For large datasets in S3 or other remote storage, use larger batches - * - For read-only mode, use larger batches to improve throughput - * - Dynamically adjust based on network latency and throughput - * - Balance between memory usage and performance - * - Adapt to cache hit/miss patterns - * - * @param cacheStats Optional cache statistics for more adaptive tuning - */ - private tuneBatchSize(cacheStats?: CacheStats): void { - let batchSize = 10 - const stats = cacheStats || this.getStats() - const isReadOnly = this.options?.readOnly || false - - const totalItems = this.storageStatistics - ? (this.storageStatistics.totalNodes || 0) + (this.storageStatistics.totalEdges || 0) - : 0 - const isLargeDataset = totalItems > 100_000 - const isVeryLargeDataset = totalItems > 1_000_000 - - // Brainy 8.0 ships Node-like runtimes only (Bun, Deno, Node). All paths - // funnel into the same batch-size envelope. - batchSize = isReadOnly ? 30 : 20 - if (isLargeDataset) batchSize = Math.min(100, batchSize * 1.5) - if (isVeryLargeDataset) batchSize = Math.min(200, batchSize * 2) - - // Adjust by hit-ratio observations. - const totalHotAccesses = stats.hotCacheHits + stats.hotCacheMisses - const totalWarmAccesses = stats.warmCacheHits + stats.warmCacheMisses - - if (totalHotAccesses > 100) { - const hotHitRatio = stats.hotCacheHits / totalHotAccesses - if (hotHitRatio > 0.8) batchSize = Math.min(batchSize * 1.5, 150) - else if (hotHitRatio < 0.4) batchSize = Math.max(5, batchSize * 0.8) - } - - if (totalWarmAccesses > 50) { - const warmHitRatio = stats.warmCacheHits / totalWarmAccesses - if (warmHitRatio > 0.7) batchSize = Math.min(batchSize * 1.3, 120) - else if (warmHitRatio < 0.3) batchSize = Math.max(5, batchSize * 0.9) - } - - // If we have storage statistics with operation counts, adjust based on operation patterns - if (this.storageStatistics && this.storageStatistics.operations) { - const ops = this.storageStatistics.operations - const totalOps = ops.total || 1 - const searchOps = (ops.search || 0) - const getOps = (ops.get || 0) - - if (totalOps > 100) { - const searchRatio = searchOps / totalOps - const getRatio = getOps / totalOps - - if (searchRatio > 0.6) { - batchSize = Math.min(100, Math.ceil(batchSize * 1.5)) - } - if (getRatio > 0.6) { - batchSize = Math.max(10, Math.ceil(batchSize * 0.9)) - } - } - } - - // Memory-pressure trim. - if (stats.hotCacheSize > 0 && this.hotCacheMaxSize > 0) { - const cacheUtilization = stats.hotCacheSize / this.hotCacheMaxSize - if (cacheUtilization > 0.85) { - batchSize = Math.max(5, Math.floor(batchSize * 0.8)) - } - } - - // Combined hot+warm hit-ratio adjustment. - const totalAccesses = stats.hotCacheHits + stats.hotCacheMisses + stats.warmCacheHits + stats.warmCacheMisses - if (totalAccesses > 100) { - const hitRatio = (stats.hotCacheHits + stats.warmCacheHits) / totalAccesses - const increaseFactorForLowHitRatio = isReadOnly ? 1.5 : 1.2 - const decreaseFactorForHighHitRatio = isReadOnly ? 0.9 : 0.8 - - if (hitRatio > 0.8 && !isVeryLargeDataset) { - batchSize = Math.max(isReadOnly ? 10 : 5, Math.floor(batchSize * decreaseFactorForHighHitRatio)) - } else if (hitRatio < 0.5) { - const maxBatchSize = isVeryLargeDataset ? 150 : 100 - batchSize = Math.min(maxBatchSize, Math.ceil(batchSize * increaseFactorForLowHitRatio)) - } - } - - // Min/max envelope. 8.0 is Node-like only. - const minBatchSize = isReadOnly ? 10 : 5 - batchSize = Math.max(minBatchSize, batchSize) - batchSize = Math.min(150, batchSize) - - // Update the batch size with the adaptively tuned value - this.batchSize = Math.round(batchSize) - } - - /** - * Resolve the warm-tier storage type. Brainy 8.0 ships filesystem + - * memory only. - */ - private detectWarmStorageType(): StorageType { - return StorageType.FILESYSTEM - } - - /** - * Resolve the cold-tier storage type. Brainy 8.0 ships filesystem + - * memory only. - */ - private detectColdStorageType(): StorageType { - return StorageType.FILESYSTEM - } - - /** - * Initialize warm storage adapter - */ - private initializeWarmStorage(): any { - // Implementation depends on the detected storage type - // For now, return null as this will be provided by the storage adapter - return null - } - - /** - * Initialize cold storage adapter - */ - private initializeColdStorage(): any { - // Implementation depends on the detected storage type - // For now, return null as this will be provided by the storage adapter - return null - } - - /** - * Get an item from cache, trying each level in order - * @param id The item ID - * @returns The cached item or null if not found - */ - public async get(id: string): Promise { - // Check if it's time to tune parameters - await this.checkAndTuneParameters() - - // Try hot cache first (fastest) - const hotCacheEntry = this.hotCache.get(id) - if (hotCacheEntry) { - // Update access metadata - hotCacheEntry.lastAccessed = Date.now() - hotCacheEntry.accessCount++ - - // Update stats - this.stats.hits++ - - return hotCacheEntry.data - } - - // Try warm cache next - try { - const warmCacheItem = await this.getFromWarmCache(id) - if (warmCacheItem) { - // Promote to hot cache - this.addToHotCache(id, warmCacheItem) - - // Update stats - this.stats.hits++ - - return warmCacheItem - } - } catch (error) { - console.warn(`Error accessing warm cache for ${id}:`, error) - } - - // Finally, try cold storage - try { - const coldStorageItem = await this.getFromColdStorage(id) - if (coldStorageItem) { - // Promote to hot and warm caches - this.addToHotCache(id, coldStorageItem) - await this.addToWarmCache(id, coldStorageItem) - - // Update stats - this.stats.misses++ - - return coldStorageItem - } - } catch (error) { - console.warn(`Error accessing cold storage for ${id}:`, error) - } - - // Item not found in any cache level - this.stats.misses++ - return null - } - - /** - * Get an item from warm cache - * @param id The item ID - * @returns The cached item or null if not found - */ - private async getFromWarmCache(id: string): Promise { - if (!this.warmStorage) return null - - try { - return await this.warmStorage.get(id) - } catch (error) { - console.warn(`Error getting item ${id} from warm cache:`, error) - return null - } - } - - /** - * Get an item from cold storage - * @param id The item ID - * @returns The item or null if not found - */ - private async getFromColdStorage(id: string): Promise { - if (!this.coldStorage) return null - - try { - return await this.coldStorage.get(id) - } catch (error) { - console.warn(`Error getting item ${id} from cold storage:`, error) - return null - } - } - - /** - * Add an item to hot cache - * @param id The item ID - * @param item The item to cache - */ - private addToHotCache(id: string, item: T): void { - // Check if we need to evict items - if (this.hotCache.size >= this.hotCacheMaxSize * this.hotCacheEvictionThreshold) { - this.evictFromHotCache() - } - - // Add to hot cache - this.hotCache.set(id, { - data: item, - lastAccessed: Date.now(), - accessCount: 1, - expiresAt: null // Hot cache items don't expire - }) - - // Update stats - this.stats.size = this.hotCache.size - } - - /** - * Add an item to warm cache - * @param id The item ID - * @param item The item to cache - */ - private async addToWarmCache(id: string, item: T): Promise { - if (!this.warmStorage) return - - try { - // Add to warm cache with TTL - await this.warmStorage.set(id, item, { - ttl: this.warmCacheTTL - }) - } catch (error) { - console.warn(`Error adding item ${id} to warm cache:`, error) - } - } - - /** - * Evict items from hot cache based on LRU policy - */ - private evictFromHotCache(): void { - // Find the least recently used items - const entries = Array.from(this.hotCache.entries()) - - // Sort by last accessed time (oldest first) - entries.sort((a, b) => a[1].lastAccessed - b[1].lastAccessed) - - // Remove the oldest 20% of items - const itemsToRemove = Math.ceil(this.hotCache.size * 0.2) - for (let i = 0; i < itemsToRemove && i < entries.length; i++) { - this.hotCache.delete(entries[i][0]) - this.stats.evictions++ - } - - // Update stats - this.stats.size = this.hotCache.size - - if (process.env.DEBUG) { - console.log(`Evicted ${itemsToRemove} items from hot cache, new size: ${this.hotCache.size}`) - } - } - - /** - * Set an item in all cache levels - * @param id The item ID - * @param item The item to cache - */ - public async set(id: string, item: T): Promise { - // Add to hot cache - this.addToHotCache(id, item) - - // Add to warm cache - await this.addToWarmCache(id, item) - - // Add to cold storage - if (this.coldStorage) { - try { - await this.coldStorage.set(id, item) - } catch (error) { - console.warn(`Error adding item ${id} to cold storage:`, error) - } - } - } - - /** - * Delete an item from all cache levels - * @param id The item ID to delete - */ - public async delete(id: string): Promise { - // Remove from hot cache - this.hotCache.delete(id) - - // Remove from warm cache - if (this.warmStorage) { - try { - await this.warmStorage.delete(id) - } catch (error) { - console.warn(`Error deleting item ${id} from warm cache:`, error) - } - } - - // Remove from cold storage - if (this.coldStorage) { - try { - await this.coldStorage.delete(id) - } catch (error) { - console.warn(`Error deleting item ${id} from cold storage:`, error) - } - } - - // Update stats - this.stats.size = this.hotCache.size - } - - /** - * Clear all cache levels - */ - public async clear(): Promise { - // Clear hot cache - this.hotCache.clear() - - // Clear warm cache - if (this.warmStorage) { - try { - await this.warmStorage.clear() - } catch (error) { - console.warn('Error clearing warm cache:', error) - } - } - - // Clear cold storage - if (this.coldStorage) { - try { - await this.coldStorage.clear() - } catch (error) { - console.warn('Error clearing cold storage:', error) - } - } - - // Reset stats - this.stats = { - hits: 0, - misses: 0, - evictions: 0, - size: 0, - maxSize: this.hotCacheMaxSize, - hotCacheSize: 0, - warmCacheSize: 0, - hotCacheHits: 0, - hotCacheMisses: 0, - warmCacheHits: 0, - warmCacheMisses: 0 - } - } - - /** - * Get cache statistics - * @returns Cache statistics - */ - public getStats(): CacheStats { - return { ...this.stats } - } - - /** - * Prefetch items based on ID patterns or relationships - * @param ids Array of IDs to prefetch - */ - public async prefetch(ids: string[]): Promise { - // Check if it's time to tune parameters - await this.checkAndTuneParameters() - - // Prefetch in batches to avoid overwhelming the system - const batches: string[][] = [] - - // Split into batches using the configurable batch size - for (let i = 0; i < ids.length; i += this.batchSize) { - const batch = ids.slice(i, i + this.batchSize) - batches.push(batch) - } - - // Process each batch - for (const batch of batches) { - await Promise.all( - batch.map(async (id) => { - // Skip if already in hot cache - if (this.hotCache.has(id)) return - - try { - // Try to get from any cache level - await this.get(id) - } catch (error) { - // Ignore errors during prefetching - if (process.env.DEBUG) { - console.warn(`Error prefetching ${id}:`, error) - } - } - }) - ) - } - } - - /** - * Check if it's time to tune parameters and do so if needed - * This is called before operations that might benefit from tuned parameters - * - * This method serves as a checkpoint for auto-tuning, ensuring that: - * 1. Parameters are tuned periodically based on the auto-tune interval - * 2. Tuning happens before critical operations that would benefit from optimized parameters - * 3. Tuning doesn't happen too frequently, which could impact performance - * - * By calling this method before get(), getMany(), and prefetch() operations, - * we ensure that the cache parameters are optimized for the current workload - * without adding unnecessary overhead to every operation. - */ - private async checkAndTuneParameters(): Promise { - // Skip if auto-tuning is disabled - if (!this.autoTune) return - - // Check if it's time to tune parameters - const now = Date.now() - if (now - this.lastAutoTuneTime >= this.autoTuneInterval) { - await this.tuneParameters() - } - } - - /** - * Get multiple items at once, optimizing for batch retrieval - * @param ids Array of IDs to get - * @returns Map of ID to item - */ - public async getMany(ids: string[]): Promise> { - // Check if it's time to tune parameters - await this.checkAndTuneParameters() - - const result = new Map() - - // First check hot cache for all IDs - const missingIds: string[] = [] - for (const id of ids) { - const hotCacheEntry = this.hotCache.get(id) - if (hotCacheEntry) { - // Update access metadata - hotCacheEntry.lastAccessed = Date.now() - hotCacheEntry.accessCount++ - - // Add to result - result.set(id, hotCacheEntry.data) - - // Update stats - this.stats.hits++ - } else { - missingIds.push(id) - } - } - - if (missingIds.length === 0) { - return result - } - - // Try to get missing items from warm cache - if (this.warmStorage) { - try { - const warmCacheItems = await this.warmStorage.getMany(missingIds) - for (const [id, item] of warmCacheItems.entries()) { - if (item) { - // Promote to hot cache - this.addToHotCache(id, item) - - // Add to result - result.set(id, item) - - // Update stats - this.stats.hits++ - - // Remove from missing IDs - const index = missingIds.indexOf(id) - if (index !== -1) { - missingIds.splice(index, 1) - } - } - } - } catch (error) { - console.warn('Error accessing warm cache for batch:', error) - } - } - - if (missingIds.length === 0) { - return result - } - - // Try to get remaining missing items from cold storage - if (this.coldStorage) { - try { - const coldStorageItems = await this.coldStorage.getMany(missingIds) - for (const [id, item] of coldStorageItems.entries()) { - if (item) { - // Promote to hot and warm caches - this.addToHotCache(id, item) - await this.addToWarmCache(id, item) - - // Add to result - result.set(id, item) - - // Update stats - this.stats.misses++ - } - } - } catch (error) { - console.warn('Error accessing cold storage for batch:', error) - } - } - - return result - } - - /** - * Set the storage adapters for warm and cold caches - * @param warmStorage Warm cache storage adapter - * @param coldStorage Cold storage adapter - */ - public setStorageAdapters(warmStorage: any, coldStorage: any): void { - this.warmStorage = warmStorage - this.coldStorage = coldStorage - } -} diff --git a/src/transaction/Transaction.ts b/src/transaction/Transaction.ts index ff19b571..092a2a25 100644 --- a/src/transaction/Transaction.ts +++ b/src/transaction/Transaction.ts @@ -37,6 +37,24 @@ const DEFAULT_OPTIONS: Required = { maxRollbackRetries: 3 } +/** + * The apply-phase budget for a batch of `opCount` operations. + * + * An explicit override wins untouched. Otherwise the budget SCALES with the + * batch: `max(30 000 ms, opCount × 2 000 ms)`. The per-op term is calibrated + * from field data — bulk imports on network-attached disks measure ~2 s per + * operation (each op pays canonical writes + fsync + index maintenance) — so + * a flat 30 s budget silently capped honest work at ~15 operations while + * looking generous for small batches. Scaling keeps small transacts + * fast-failing and gives bulk ones a budget proportional to the work they + * actually asked for; a trip still rolls back atomically and throws a + * retryable, fully-labeled TransactionTimeoutError. + */ +export function transactTimeoutBudget(opCount: number, override?: number): number { + if (override !== undefined) return override + return Math.max(30_000, opCount * 2_000) +} + /** * Transaction class */ @@ -98,49 +116,64 @@ export class Transaction implements TransactionContext { } try { - // Execute each operation in order + // Execute each operation in order. This loop is the sole rollback-guarded + // region: ANY error that escapes it — an operation failure OR a + // mid-flight timeout — is caught below and rolls back every operation + // applied so far, in reverse order. That single guarantee is the + // transaction's atomicity contract, and the generation-store commit path + // depends on it: its abort cleanup assumes a throw from here already + // restored the applied operations byte-identically (it only discards the + // uncommitted staging directory). A rollback per exit path — the previous + // design — let the timeout throw slip past rollback and strand the + // already-applied writes as torn, generation-less state in canonical + // storage. for (let i = 0; i < this.operations.length; i++) { - // Check timeout + // Budget check BEFORE starting the next operation. A trip here throws + // into the catch below and rolls back like any other failure — it must + // never bypass rollback. if (Date.now() - this.startTime > this.options.timeout) { - throw new TransactionTimeoutError(this.options.timeout, i) + throw new TransactionTimeoutError(this.options.timeout, i, { + elapsedMs: Date.now() - this.startTime, + totalOperations: this.operations.length, + operationName: this.operations[i]?.name + }) } const operation = this.operations[i] + if (this.options.logging) { + prodLog.info(`[Transaction] Executing operation ${i}: ${operation.name || 'unnamed'}`) + } + + let rollbackAction: RollbackAction | undefined try { - if (this.options.logging) { - prodLog.info(`[Transaction] Executing operation ${i}: ${operation.name || 'unnamed'}`) - } - - // Execute operation - const rollbackAction = await operation.execute() - - // Record rollback action (if provided) - if (rollbackAction) { - this.rollbackActions.push(rollbackAction) - } - + rollbackAction = await operation.execute() } catch (error) { - // Operation failed - rollback and re-throw - const executionError = new TransactionExecutionError( + // Normalize an operation failure to a TransactionExecutionError; the + // outer catch performs the (single) rollback and surfaces it. + throw new TransactionExecutionError( `Operation ${i} failed: ${(error as Error).message}`, i, operation.name, error as Error ) + } - await this.rollback(executionError) - throw executionError + // Record rollback action (if the operation provided one) + if (rollbackAction) { + this.rollbackActions.push(rollbackAction) } } - - // All operations succeeded - commit - this.commit() - } catch (error) { - // Error already handled in rollback + // Single rollback point: undo everything applied so far, in reverse + // order, then surface the original error. A rollback failure supersedes + // it (rollback() throws TransactionRollbackError wrapping this error). + await this.rollback(error as Error) throw error } + + // All operations succeeded — commit. + this.commit() } /** @@ -208,15 +241,22 @@ export class Transaction implements TransactionContext { } } - this.state = 'rolled_back' + // Honest terminal state: only claim 'rolled_back' when EVERY undo applied. + // If any undo failed, the store is not cleanly rolled back — mark + // 'inconsistent' so the commit orchestration reconciles rather than trusts + // a false 'rolled_back'. (Fixes the state half of the post-commit response + // lie: the record whose undo failed is still durable.) + this.state = rollbackErrors.length > 0 ? 'inconsistent' : 'rolled_back' this.endTime = Date.now() if (this.options.logging) { const duration = this.endTime - (this.startTime || this.endTime) - prodLog.info(`[Transaction] Rolled back in ${duration}ms`) + prodLog.info(`[Transaction] ${this.state === 'inconsistent' ? 'Rollback INCOMPLETE' : 'Rolled back'} in ${duration}ms`) } - // If rollback encountered errors, wrap them with original error + // If rollback encountered errors, wrap them with original error. The + // orchestration keys on `instanceof TransactionRollbackError` to know the + // undo did not fully apply and reconciliation is required. if (rollbackErrors.length > 0) { throw new TransactionRollbackError( `Transaction rollback encountered ${rollbackErrors.length} errors during cleanup`, diff --git a/src/transaction/errors.ts b/src/transaction/errors.ts index 1cb49efd..c270d0ed 100644 --- a/src/transaction/errors.ts +++ b/src/transaction/errors.ts @@ -26,7 +26,7 @@ export class TransactionExecutionError extends TransactionError { message: string, public readonly operationIndex: number, public readonly operationName: string | undefined, - public readonly cause: Error + public override readonly cause: Error ) { super(message, { operationIndex, @@ -77,11 +77,24 @@ export class InvalidTransactionStateError extends TransactionError { export class TransactionTimeoutError extends TransactionError { constructor( timeoutMs: number, - operationIndex: number + operationIndex: number, + telemetry?: { + elapsedMs?: number + totalOperations?: number + operationName?: string + } ) { + const progress = + telemetry?.totalOperations !== undefined + ? `${operationIndex}/${telemetry.totalOperations}` + : String(operationIndex) + const name = telemetry?.operationName ? ` ('${telemetry.operationName}')` : '' + const elapsed = + telemetry?.elapsedMs !== undefined ? `${telemetry.elapsedMs}ms elapsed, ` : '' super( - `Transaction timed out after ${timeoutMs}ms at operation ${operationIndex}`, - { timeoutMs, operationIndex } + `Transaction timed out at operation ${progress}${name} — ${elapsed}budget ${timeoutMs}ms. ` + + `The batch rolled back atomically; retry with a higher timeoutMs or a smaller batch.`, + { timeoutMs, operationIndex, ...telemetry } ) this.name = 'TransactionTimeoutError' } diff --git a/src/transaction/index.ts b/src/transaction/index.ts deleted file mode 100644 index a09eb234..00000000 --- a/src/transaction/index.ts +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Transaction System - * - * Provides atomicity for Brainy operations. - * All operations succeed or all rollback - no partial failures. - * - * @module transaction - */ - -export * from './types.js' -export * from './errors.js' -export { Transaction } from './Transaction.js' -export { TransactionManager, type TransactionStats } from './TransactionManager.js' - -// Re-export for convenience -export type { - Operation, - RollbackAction, - TransactionState, - TransactionContext, - TransactionFunction, - TransactionResult, - TransactionOptions -} from './types.js' - -export { - TransactionError, - TransactionExecutionError, - TransactionRollbackError, - InvalidTransactionStateError, - TransactionTimeoutError -} from './errors.js' - -export { RevisionConflictError } from './RevisionConflictError.js' diff --git a/src/transaction/operations/IndexOperations.ts b/src/transaction/operations/IndexOperations.ts index f0bede05..97c39270 100644 --- a/src/transaction/operations/IndexOperations.ts +++ b/src/transaction/operations/IndexOperations.ts @@ -176,14 +176,36 @@ export class RemoveFromMetadataIndexOperation implements Operation { * generation is reused for the rollback removal, so an add and its undo * reference one watermark in a provider's per-generation edge chain. */ +/** + * A verb's interned endpoint ints — eager (already resolved), or a thunk + * evaluated when the operation EXECUTES. The lazy form exists for + * `transact()` forward references: a relate whose endpoint is added in the + * SAME batch cannot resolve ints at plan time (the entity does not exist yet + * — a strict native id mapper rightly refuses to assign, and even a permissive + * one would leak the assignment if the batch is rejected at precommit). + * Deferring to execute time resolves after the batch's add operations have + * applied, mirroring how `generationFn` is already evaluated lazily. + */ +export type VerbEndpointInts = + | { sourceInt: bigint; targetInt: bigint } + | (() => { sourceInt: bigint; targetInt: bigint }) + +/** Resolve a {@link VerbEndpointInts} at execution time. */ +function resolveEndpointInts( + endpoints: VerbEndpointInts +): { sourceInt: bigint; targetInt: bigint } { + return typeof endpoints === 'function' ? endpoints() : endpoints +} + export class AddToGraphIndexOperation implements Operation { readonly name = 'AddToGraphIndex' /** * @param index - The graph-index provider (JS baseline or native). * @param verb - The verb to index (`sourceInt`/`targetInt` mirrored on it). - * @param sourceInt - The source entity's interned int. - * @param targetInt - The target entity's interned int. + * @param endpointInts - The endpoints' interned ints — eager, or a thunk + * evaluated at execute time (REQUIRED for transact forward references; + * see {@link VerbEndpointInts}). * @param generationFn - Resolves the commit generation to stamp this edge at, * evaluated when the operation executes (see class note). * @param onVerbInt - Optional hook invoked with the interned verb int @@ -192,17 +214,18 @@ export class AddToGraphIndexOperation implements Operation { constructor( private readonly index: GraphIndexProvider, private readonly verb: GraphVerb, - private readonly sourceInt: bigint, - private readonly targetInt: bigint, + private readonly endpointInts: VerbEndpointInts, private readonly generationFn: () => bigint, private readonly onVerbInt?: (verbInt: bigint) => void ) {} async execute(): Promise { // Stamp this edge at the in-flight commit generation; reuse it for the - // rollback so add + undo reference the same watermark. + // rollback so add + undo reference the same watermark. Endpoint ints + // resolve HERE — after any same-batch adds have applied. const generation = this.generationFn() - const verbInt = await this.index.addVerb(this.verb, this.sourceInt, this.targetInt, generation) + const { sourceInt, targetInt } = resolveEndpointInts(this.endpointInts) + const verbInt = await this.index.addVerb(this.verb, sourceInt, targetInt, generation) this.onVerbInt?.(verbInt) // Return rollback action @@ -231,28 +254,32 @@ export class RemoveFromGraphIndexOperation implements Operation { /** * @param index - The graph-index provider (JS baseline or native). * @param verb - The verb being removed (required for rollback re-add). - * @param sourceInt - The source entity's interned int (rollback re-add). - * @param targetInt - The target entity's interned int (rollback re-add). + * @param endpointInts - The endpoints' interned ints for the rollback + * re-add — eager, or a thunk evaluated at execute time (required when the + * verb or its endpoints were created in the SAME transact batch; see + * {@link VerbEndpointInts}). * @param generationFn - Resolves the commit generation for this removal, * evaluated when the operation executes. */ constructor( private readonly index: GraphIndexProvider, private readonly verb: GraphVerb, // Required for rollback - private readonly sourceInt: bigint, - private readonly targetInt: bigint, + private readonly endpointInts: VerbEndpointInts, private readonly generationFn: () => bigint ) {} async execute(): Promise { // Resolve the removal generation once; reuse it for the rollback re-add. + // Endpoint ints resolve HERE (after any same-batch adds applied) and are + // captured for the rollback, whose re-add must use the same mappings. const generation = this.generationFn() + const { sourceInt, targetInt } = resolveEndpointInts(this.endpointInts) await this.index.removeVerb(this.verb.id, generation) // Return rollback action return async () => { // Re-add verb with original data - await this.index.addVerb(this.verb, this.sourceInt, this.targetInt, generation) + await this.index.addVerb(this.verb, sourceInt, targetInt, generation) } } } diff --git a/src/transaction/operations/StorageOperations.ts b/src/transaction/operations/StorageOperations.ts index ca513cd4..316f1ac0 100644 --- a/src/transaction/operations/StorageOperations.ts +++ b/src/transaction/operations/StorageOperations.ts @@ -104,35 +104,72 @@ export class SaveNounOperation implements Operation { } /** - * Delete noun metadata with rollback support + * Delete a noun — FULL canonical removal, with rollback support. + * + * Despite the historical name, this removes the WHOLE entity: both canonical + * legs (metadata + vector) AND the entity's container. Previously it deleted + * only the metadata leg via `deleteNounMetadata`, leaving the canonical + * `vectors.json` leg and the `/` directory orphaned on disk — a "ghost" + * that reads as absent (getNoun needs both legs) yet inflates the enumerated + * count and can never be told apart from a damage scar. Routing through + * `storage.deleteNoun` removes both legs + the container in one place. + * + * Immutability is preserved: this cleans only the live-HEAD projection; the + * generation store retains the before-image so `asOf()` still reconstructs the + * deleted entity until retention expires. * * Rollback strategy: - * - Restore deleted metadata + * - Restore BOTH legs from the before-image (vector leg raw, metadata leg via + * the count-aware save so deleteNoun()'s decrement is reversed). */ export class DeleteNounMetadataOperation implements Operation { - readonly name = 'DeleteNounMetadata' + readonly name = 'DeleteNoun' constructor( private readonly storage: StorageAdapter, - private readonly id: string + private readonly id: string, + /** + * OPTIONAL already-known metadata of the entity being removed (the caller's + * pre-delete read). Removal must never REQUIRE re-reading the thing being + * removed: if the reads here return null (replace race, or a ghost left by + * an earlier version), the count decrement downstream falls back to this + * record instead of being silently skipped — the skip minted permanent + * counter inflation (adds counted, paired removals not decremented). + */ + private readonly priorMetadata?: NounMetadata | null ) {} async execute(): Promise { - // Get metadata before deletion (for rollback) - const previousMetadata = await this.storage.getNounMetadata(this.id) + // Capture the FULL before-image (both legs) so the undo restores the whole + // entity — a metadata-only rollback would leave the vector leg unrestored. + // A null metadata read falls back to the caller's pre-delete read. + const previousNoun = await this.storage.getNoun(this.id) + const previousMetadata = (await this.storage.getNounMetadata(this.id)) ?? this.priorMetadata ?? null - if (!previousMetadata) { + if (!previousNoun && !previousMetadata) { // Nothing to delete - no rollback needed return async () => {} } - // Delete metadata - await this.storage.deleteNounMetadata(this.id) + // Full removal: both canonical legs + the entity container + count decrement + // (the prior record keeps the decrement honest on a null canonical read). + await this.storage.deleteNoun(this.id, previousMetadata) // Return rollback action return async () => { - // Restore deleted metadata - await this.storage.saveNounMetadata(this.id, previousMetadata) + // Restore the vector leg, then the metadata leg through the count-aware + // save so deleteNoun()'s decrement is reversed. + if (previousNoun) { + await this.storage.saveNoun({ + id: previousNoun.id, + vector: previousNoun.vector, + connections: previousNoun.connections || new Map(), + level: previousNoun.level || 0 + }) + } + if (previousMetadata) { + await this.storage.saveNounMetadata(this.id, previousMetadata) + } } } } @@ -242,9 +279,9 @@ export class DeleteVerbMetadataOperation implements Operation { return async () => {} } - // Delete verb (metadata + vector) - // Note: StorageAdapter has deleteVerb but not deleteVerbMetadata - await this.storage.deleteVerb(this.id) + // Delete verb (metadata + vector). The pre-read rides along so the count + // decrement never depends on re-reading the record being removed. + await this.storage.deleteVerb(this.id, previousMetadata) // Return rollback action return async () => { diff --git a/src/transaction/types.ts b/src/transaction/types.ts index 5917ea4c..9a3a2eaa 100644 --- a/src/transaction/types.ts +++ b/src/transaction/types.ts @@ -14,6 +14,10 @@ export type TransactionState = | 'committed' // Successfully committed | 'rolling_back' // Rolling back due to failure | 'rolled_back' // Successfully rolled back + | 'inconsistent' // Rollback ran but ≥1 undo could not be applied — the store + // is NOT cleanly rolled back; the commit orchestration must + // reconcile (adopt-forward or fail-loud). Never claim + // 'rolled_back' when an undo failed. /** * Rollback action - undoes an operation diff --git a/src/types/apiTypes.ts b/src/types/apiTypes.ts deleted file mode 100644 index d788d60e..00000000 --- a/src/types/apiTypes.ts +++ /dev/null @@ -1,326 +0,0 @@ -/** - * Consistent API Types for Brainy - * - * These types provide a uniform interface for all public methods, - * using object parameters for consistency and extensibility. - */ - -import type { Vector } from '../coreTypes.js' -import type { NounType, VerbType } from './graphTypes.js' - -// ============= NOUN OPERATIONS ============= - -/** - * Parameters for adding a noun - */ -export interface AddNounParams { - data: any | Vector // Content or pre-computed vector - type: NounType | string // Noun type (required) - metadata?: any // Optional metadata - id?: string // Optional custom ID - service?: string // Optional service identifier -} - -/** - * Parameters for updating a noun - */ -export interface UpdateNounParams { - id: string // Noun ID to update - data?: any // New data - metadata?: any // New metadata - type?: NounType | string // New type -} - -/** - * Parameters for getting nouns - */ -export interface GetNounsParams { - ids?: string[] // Specific IDs to fetch - type?: NounType | string | string[] // Filter by type(s) - limit?: number // Maximum results - offset?: number // Pagination offset - cursor?: string // Pagination cursor - filter?: Record // Metadata filters - service?: string // Service filter -} - -// ============= VERB OPERATIONS ============= - -/** - * Parameters for adding a verb (relationship) - */ -export interface AddVerbParams { - source: string // Source noun ID - target: string // Target noun ID - type: VerbType | string // Verb type (required) - weight?: number // Relationship weight (0-1) - metadata?: any // Optional metadata - service?: string // Optional service identifier -} - -/** - * Parameters for getting verbs - */ -export interface GetVerbsParams { - source?: string // Filter by source - target?: string // Filter by target - type?: VerbType | string | string[] // Filter by type(s) - limit?: number // Maximum results - offset?: number // Pagination offset - cursor?: string // Pagination cursor - filter?: Record // Metadata filters - service?: string // Service filter -} - -// ============= SEARCH OPERATIONS ============= - -/** - * Unified search parameters - */ -export interface SearchParams { - query: string | Vector // Text query or vector - limit?: number // Maximum results (default: 10) - threshold?: number // Similarity threshold (0-1) - filter?: { - type?: NounType | string | string[] // Filter by noun type(s) - metadata?: Record // Metadata filters - service?: string // Service filter - } - includeMetadata?: boolean // Include metadata in results - includeVectors?: boolean // Include vectors in results -} - -/** - * Parameters for similarity search - */ -export interface SimilarityParams { - id?: string // Find similar to this ID - data?: any | Vector // Or find similar to this data - limit?: number // Maximum results (default: 10) - threshold?: number // Similarity threshold (0-1) - filter?: { - type?: NounType | string | string[] - metadata?: Record - service?: string - } -} - -/** - * Parameters for related items search - */ -export interface RelatedParams { - id: string // Starting noun ID - depth?: number // Traversal depth (default: 1) - limit?: number // Max results per level - types?: VerbType[] | string[] // Relationship types to follow - direction?: 'outgoing' | 'incoming' | 'both' -} - -// ============= BATCH OPERATIONS ============= - -/** - * Parameters for batch noun operations - */ -export interface BatchNounsParams { - items: AddNounParams[] // Array of nouns to add - parallel?: boolean // Process in parallel - chunkSize?: number // Batch size for processing - onProgress?: (completed: number, total: number) => void -} - -/** - * Parameters for batch verb operations - */ -export interface BatchVerbsParams { - items: AddVerbParams[] // Array of verbs to add - parallel?: boolean // Process in parallel - chunkSize?: number // Batch size for processing - onProgress?: (completed: number, total: number) => void -} - -// ============= STATISTICS & METADATA ============= - -/** - * Parameters for statistics queries - */ -export interface StatisticsParams { - detailed?: boolean // Include detailed breakdown - includeAugmentations?: boolean // Include augmentation stats - includeMemory?: boolean // Include memory usage - service?: string // Filter by service -} - -/** - * Parameters for metadata operations - */ -export interface MetadataParams { - id: string // Entity ID - metadata: any // Metadata to set/update - merge?: boolean // Merge with existing (vs replace) -} - -// ============= CONFIGURATION ============= - -/** - * Dynamic configuration update parameters - */ -export interface ConfigUpdateParams { - embeddings?: { - model?: string - precision?: 'q8' - cache?: boolean - } - augmentations?: { - [name: string]: boolean | Record - } - storage?: { - type?: string - config?: any - } - performance?: { - batchSize?: number - maxConcurrency?: number - cacheSize?: number - } -} - -// ============= TRIPLE INTELLIGENCE API ============= - -/** - * API for Triple Intelligence Engine to access Brainy internals - * This provides type-safe access without unchecked casts - */ -export interface TripleIntelligenceAPI { - // Vector operations - vectorSearch(vector: Vector | string, limit: number): Promise> - - // Graph operations - graphTraversal(options: { - start: string | string[] - type?: string | string[] - direction?: 'in' | 'out' | 'both' - maxDepth?: number - }): Promise> - - // Metadata operations - metadataQuery(where: Record): Promise> - getEntity(id: string): Promise - - // Storage operations - getVerbsBySource(sourceId: string): Promise - getVerbsByTarget(targetId: string): Promise - - // Statistics - getStatistics(): Promise<{ - totalCount: number - fieldStats: Record - }> - - // Index access - getAllNouns(): Map - hasMetadataIndex(): boolean -} - -// ============= RESULTS ============= - -/** - * Unified search result - */ -export interface SearchResult { - id: string - score: number // Similarity score (0-1) - data?: T // Original data - metadata?: any // Metadata if requested - vector?: Vector // Vector if requested - type?: string // Noun type - distance?: number // Raw distance metric -} - -/** - * Paginated result wrapper - */ -export interface PaginatedResult { - items: T[] - total?: number - hasMore: boolean - nextCursor?: string - previousCursor?: string -} - -/** - * Batch operation result - */ -export interface BatchResult { - successful: string[] // Successfully processed IDs - failed: Array<{ - index: number - error: string - item?: any - }> - total: number - duration: number // Total time in ms -} - -/** - * Statistics result - */ -export interface StatisticsResult { - nouns: { - total: number - byType: Record - } - verbs: { - total: number - byType: Record - } - storage: { - used: number - type: string - } - performance?: { - avgLatency: number - throughput: number - cacheHitRate?: number - } - augmentations?: Record - memory?: { - used: number - limit: number - } -} - -// ============= ERRORS ============= - -/** - * Structured error for API operations - */ -export class BrainyAPIError extends Error { - constructor( - message: string, - public code: string, - public statusCode: number = 400, - public details?: any - ) { - super(message) - this.name = 'BrainyAPIError' - } -} - -// Error codes -export const ErrorCodes = { - INVALID_TYPE: 'INVALID_TYPE', - NOT_FOUND: 'NOT_FOUND', - DUPLICATE_ID: 'DUPLICATE_ID', - INVALID_VECTOR: 'INVALID_VECTOR', - STORAGE_ERROR: 'STORAGE_ERROR', - EMBEDDING_ERROR: 'EMBEDDING_ERROR', - AUGMENTATION_ERROR: 'AUGMENTATION_ERROR', - VALIDATION_ERROR: 'VALIDATION_ERROR', - QUOTA_EXCEEDED: 'QUOTA_EXCEEDED', - UNAUTHORIZED: 'UNAUTHORIZED' -} as const \ No newline at end of file diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index 61e6a86a..1ec4c4c3 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -1,6 +1,10 @@ /** - * 🧠 Brainy 3.0 Type Definitions - * + * @module types/brainy.types + * @description Public type definitions for the Brainy API — the entity/relation + * shapes, the `find()`/`add()`/`relate()` parameter and result types, and the + * configuration surface. (Version-agnostic header: these track the package + * version, not a fixed number.) + * * Beautiful, consistent, type-safe interfaces for the future of neural databases */ @@ -27,7 +31,7 @@ import type { * alongside user metadata but extracted to top-level Entity fields on read. */ export interface Entity { - /** Unique identifier (UUID v4) */ + /** Unique identifier — a UUID (auto-generated as a time-ordered v7; a supplied natural key is normalized to a stable v5). */ id: string /** Embedding vector (384-dim by default). Empty array when loaded without includeVectors. */ vector: Vector @@ -140,8 +144,8 @@ export interface RelationEvidence { /** * Search result with similarity score * - * Flattens commonly-used entity fields to top level for convenience, - * while preserving full entity in 'entity' field for backward compatibility. + * Flattens commonly-used entity fields to the top level for convenience, while + * keeping the complete record under `entity` for callers that need the full shape. */ export interface Result { // Search metadata @@ -158,7 +162,7 @@ export interface Result { weight?: number // Entity importance (from entity.weight) _rev?: number // Monotonic revision counter (from entity._rev) — pass back as update({ ifRev }) for CAS - // Full entity (preserved for backward compatibility) + // Full entity record (the flattened fields above are projections of this) entity: Entity // Score transparency @@ -173,8 +177,8 @@ export interface Result { // Aggregation: present only on rows returned by find({ aggregate }). These mirror the // documented AggregateResult shape so callers can read group/metric data at the top level - // instead of digging into `metadata`. (The same values are also flattened into `metadata` - // for backward compatibility.) + // instead of digging into `metadata`. (The same values are also mirrored into `metadata` + // so aggregate-unaware callers still see them.) groupKey?: Record // Group-by key values for this aggregate row metrics?: Record // Computed metric values (sum/avg/min/max/count) count?: number // Total entity count in this group @@ -327,7 +331,7 @@ export interface AddParams { * a one-shot warning. */ metadata?: EntityMetadataInput - /** Custom entity ID (auto-generated UUID v4 if not provided) */ + /** Custom entity ID. When omitted, a time-ordered UUID v7 is generated; a supplied natural-key string is normalized to a stable UUID v5. */ id?: string /** Pre-computed embedding vector (skips auto-embedding when provided) */ vector?: Vector @@ -346,6 +350,24 @@ export interface AddParams { * Ignored when `id` is omitted (a freshly generated UUID can never collide). */ ifAbsent?: boolean + /** + * Create-or-update in a single call. When `true` AND a custom `id` is supplied AND + * an entity with that `id` already exists, `add()` applies the supplied fields as an + * UPDATE instead of the default destructive overwrite: metadata is MERGED (not + * replaced), `data` re-embeds only when it changed, `_rev` is bumped, and the + * original `createdAt` is PRESERVED. When no entity with that `id` exists, `add()` + * inserts normally (a fresh `_rev` of 1). Ignored — behaves as a plain insert — when + * `id` is omitted, since a freshly generated UUID can never collide. + * + * Mutually exclusive with {@link AddParams.ifAbsent}: `ifAbsent` SKIPS an existing + * entity, `upsert` MERGES into it — supplying both throws. + * + * @example + * // First call inserts; a later call with the same id merges + bumps _rev. + * await brain.add({ id: 'customer-42', type: 'customer', data: 'Acme', upsert: true }) + * await brain.add({ id: 'customer-42', type: 'customer', metadata: { tier: 'gold' }, upsert: true }) + */ + upsert?: boolean } /** @@ -533,14 +555,25 @@ export interface FindParams { order?: 'asc' | 'desc' // Sort direction: 'asc' (default) or 'desc' // Advanced options - mode?: SearchMode // Search strategy - explain?: boolean // Return scoring explanation includeRelations?: boolean // Include entity relationships excludeVFS?: boolean // Exclude VFS entities from results (default: false - VFS included) service?: string // Multi-tenancy filter + /** + * Return each result's stored embedding under `entity.vector`. Defaults to `false`, + * in which case `entity.vector` is an empty array (the perf default — vectors are + * large and most reads don't need them). Set `true` to get the persisted vector + * back from `find()` without a recompute, e.g. to feed downstream similarity math. + * Mirrors {@link GetOptions.includeVectors} on `get()`. + */ + includeVectors?: boolean // Hybrid search options - searchMode?: 'auto' | 'text' | 'semantic' | 'vector' | 'hybrid' // Search strategy: auto (default), text-only, semantic/vector-only, or explicit hybrid + /** + * Search strategy — the single canonical search-mode knob (the redundant `mode` + * alias was removed in 8.0). `'auto'` (default) blends text + semantic; the + * other members force a single intelligence. See {@link SearchMode}. + */ + searchMode?: SearchMode hybridAlpha?: number // Weight between text (0.0) and semantic (1.0) search, default: auto-detected by query length // Triple Intelligence Fusion @@ -567,8 +600,8 @@ export interface FindParams { export interface GraphConstraints { to?: string // Connected to this entity from?: string // Connected from this entity - via?: VerbType | VerbType[] // Via these relationship types - type?: VerbType | VerbType[] // Alias for via + via?: VerbType | VerbType[] // Traverse via these relationship types (the canonical key) + type?: VerbType | VerbType[] // Accepted alias for `via` (resolved identically: `via ?? type`) /** * Filter traversal edges by VerbType subtype. Single string for equality, * array for set membership. Composes with `via` — `{ via: 'manages', subtype: @@ -578,7 +611,6 @@ export interface GraphConstraints { subtype?: string | string[] depth?: number // Max traversal depth (default: 1) direction?: 'in' | 'out' | 'both' // Direction of traversal (default: 'both') - bidirectional?: boolean // Consider both directions } /** @@ -658,6 +690,21 @@ export interface RelatedParams { */ to?: string + /** + * Filter by INCIDENT entity ID — every relationship touching this entity in + * EITHER direction (where it is the source OR the target), deduped. The + * one-call "all edges on a noun" query: equivalent to merging + * `related({ from: id })` and `related({ to: id })` but in a single + * O(degree) call. Mutually exclusive with `from` / `to` (pass `node` for + * both directions, or `from` / `to` for one). Composes with `type` / + * `subtype` / visibility filters. + * + * @example + * // Everything connected to this person, in or out. + * const edges = await brain.related({ node: personId }) + */ + node?: string + /** * Filter by relationship type(s) * @@ -722,6 +769,245 @@ export interface RelatedParams { service?: string } +// ============= Graph views (brain.graph.*) ============= + +/** + * A node in a {@link GraphView} — a lightweight reference (id + classification + + * hop depth), NOT a full {@link Entity}. Graph reads are structure-first; resolve + * a node to its full entity with `get(id)` when you need its `data` / metadata. + */ +export interface GraphNode { + /** Entity id. */ + id: string + /** Entity type (present when the view hydrates node classification). */ + type?: NounType + /** Per-product subtype (present when hydrated). */ + subtype?: string + /** Hop distance from the nearest seed (0 = a seed); present on subgraph results. */ + depth?: number +} + +/** + * A hydrated view of a region of the graph — the discovered nodes plus the edges + * among them, in id form (the public face of the provider's columnar int form). + */ +export interface GraphView { + /** The nodes in this region (seeds + everything reached). */ + nodes: GraphNode[] + /** The edges among `nodes`, as full {@link Relation} objects. */ + edges: Relation[] + /** `true` when a `maxNodes` / `maxEdges` cap truncated the result. */ + truncated?: boolean +} + +/** + * Options for {@link GraphApi.subgraph}. + */ +export interface SubgraphOptions { + /** Max hop distance from any seed (default `1`). */ + depth?: number + /** Edge direction to follow (default `'both'`). */ + direction?: 'in' | 'out' | 'both' + /** Restrict traversed edges to these relationship type(s). */ + type?: VerbType | VerbType[] + /** Restrict traversed edges to these subtype(s). */ + subtype?: string | string[] + /** Also traverse through `internal`-visibility edges/nodes (hidden by default). */ + includeInternal?: boolean + /** Also traverse through `system`-visibility edges/nodes (hidden by default). */ + includeSystem?: boolean + /** Cap on returned nodes (sets `GraphView.truncated`). */ + maxNodes?: number + /** Cap on returned edges. */ + maxEdges?: number + /** + * Hydrate each node's `type` / `subtype` (one batch read). Default `true`; + * set `false` to skip it when you only need graph structure (ids + edges). + */ + hydrateNodes?: boolean +} + +/** + * Options for {@link GraphApi.export}. + */ +export interface GraphExportOptions { + /** Nodes/edges per streamed chunk (default `1000`). */ + chunkSize?: number + /** Also include `internal`-visibility nodes/edges (hidden by default). */ + includeInternal?: boolean + /** Also include `system`-visibility nodes/edges (hidden by default). */ + includeSystem?: boolean + /** Stream the nodes (default `true`; set `false` to stream only edges). */ + includeNodes?: boolean + /** Stream the edges (default `true`; set `false` to stream only nodes). */ + includeEdges?: boolean +} + +/** + * Options for {@link GraphApi.communities}. + */ +export interface GraphCommunitiesOptions { + /** Treat edges as directed when grouping (default `false` — direction-agnostic). */ + directed?: boolean + /** Also group through `internal`-visibility nodes/edges (hidden by default). */ + includeInternal?: boolean + /** Also group through `system`-visibility nodes/edges (hidden by default). */ + includeSystem?: boolean +} + +/** + * Result of {@link GraphApi.communities} — the graph partitioned into connected + * groups of entity ids. + */ +export interface GraphCommunitiesResult { + /** Each group is the member entity ids; largest group first. */ + groups: string[][] + /** Number of distinct groups (`= groups.length`). */ + count: number +} + +/** + * Options for {@link GraphApi.rank} (importance ranking). INTENT-level only — the + * ranking ALGORITHM is the provider's choice (the TS fallback uses PageRank), so + * no algorithm-tuning knobs are exposed. + */ +export interface GraphRankOptions { + /** Return only the top-K nodes by score (default: all, descending). */ + topK?: number + /** Also rank through `internal`-visibility nodes/edges (hidden by default). */ + includeInternal?: boolean + /** Also rank through `system`-visibility nodes/edges (hidden by default). */ + includeSystem?: boolean +} + +/** One ranked entity from {@link GraphApi.rank}, descending by `score`. */ +export interface GraphRankEntry { + /** The entity id. */ + id: string + /** The importance score (relative; higher = more central). */ + score: number +} + +/** + * Options for {@link GraphApi.path} (best route between two entities). + */ +export interface GraphPathOptions { + /** Edge direction to follow (default `'both'`). */ + direction?: 'in' | 'out' | 'both' + /** Optimize for fewest `'hops'` (default) or least summed edge `'weight'`. */ + by?: 'hops' | 'weight' + /** Restrict the route to these relationship type(s). */ + type?: VerbType | VerbType[] + /** Also route through `internal`-visibility nodes/edges (hidden by default). */ + includeInternal?: boolean + /** Also route through `system`-visibility nodes/edges (hidden by default). */ + includeSystem?: boolean + /** Abandon the search past this many hops. */ + maxDepth?: number +} + +/** + * Result of {@link GraphApi.path} — the route from `from` to `to`, or `null` when + * unreachable. + */ +export interface GraphPathResult { + /** The entity ids along the route, `from` … `to` inclusive. */ + nodes: string[] + /** The relationship (verb) ids traversed, one fewer than `nodes`. */ + relationships: string[] + /** Route cost: number of hops, or summed edge weight when `by: 'weight'`. */ + cost: number +} + +/** + * What {@link GraphApi.subgraph} expands from — "the things to grow a neighborhood + * around". One of: + * - an entity id, or an array of them (the explicit id set), + * - a {@link Result} array (a `find()` result — its entities become the seeds), or + * - a {@link FindParams} query (the query→expand fusion: run the query, expand from + * every match). With the native engine a metadata-only query's matched universe is + * forwarded to the traversal as an opaque set with no id materialization. + */ +export type SubgraphSelector = string | string[] | Result[] | FindParams + +/** + * The `brain.graph` namespace — graph-shaped reads over the knowledge graph. + * Routes to a native {@link import('../plugin.js').GraphAccelerationProvider} + * when one is registered, otherwise serves the same results from Brainy's + * pure-TS adjacency (correct at small/medium scale). + */ +export interface GraphApi { + /** + * @description Extract the subgraph reachable from a seed selector — the bounded + * multi-hop neighborhood, as nodes + edges. The one-call answer to "show me + * everything around this, N hops out". The seed can be entity id(s), a `find()` + * result, or a {@link FindParams} query (query→expand: run the query, then expand + * from every match — with the native engine the matched universe never leaves + * the engine's representation). + * @param selector - Seed id(s), a `find()` result, or a query. See {@link SubgraphSelector}. + * @param options - Depth, direction, edge/visibility filters, and caps. + * @returns The hydrated {@link GraphView}. + * @example + * const view = await brain.graph.subgraph(personId, { depth: 2 }) + * // view.nodes = the 2-hop neighborhood; view.edges = the relations among them + * @example + * // Query→expand: the neighborhood of everything matching a filter. + * const cluster = await brain.graph.subgraph({ where: { team: 'platform' } }, { depth: 1 }) + */ + subgraph(selector: SubgraphSelector, options?: SubgraphOptions): Promise> + + /** + * @description Stream the WHOLE graph in one O(N+E) pass as a sequence of + * {@link GraphView} chunks — the right primitive for visualizing all data + * (vs. paging per node). Node chunks come first, then edge chunks; assemble + * them on the consumer side. Backed by cursor pagination, so a full walk is + * O(N+E) at any chunk size (no re-scan). + * @param options - Chunk size, visibility, and node/edge inclusion toggles. + * @returns An async-iterable of graph chunks. + * @example + * for await (const chunk of brain.graph.export()) { + * addNodes(chunk.nodes); addEdges(chunk.edges) + * } + */ + export(options?: GraphExportOptions): AsyncIterable> + + /** + * @description Rank entities by graph importance (influence / centrality) — the + * one-call answer to "which nodes matter most". The TS fallback uses PageRank; + * a native provider may use personalized PageRank / eigenvector centrality. + * @param options - `topK`, visibility opt-ins. + * @returns Entities with scores, DESCENDING (most important first). + * @example + * const top = await brain.graph.rank({ topK: 10 }) + * top[0] // { id, score } — the most central entity + */ + rank(options?: GraphRankOptions): Promise + + /** + * @description Partition the graph into connected communities (clusters of + * related entities) — "which things group together". + * @param options - `directed`, visibility opt-ins. + * @returns The groups (member ids) + their count. + * @example + * const { groups, count } = await brain.graph.communities() + */ + communities(options?: GraphCommunitiesOptions): Promise + + /** + * @description Find the best route between two entities — fewest hops (default) + * or least summed edge weight (`by: 'weight'`). The pathfinding ALGORITHM is + * the provider's choice (TS fallback: BFS for hops, Dijkstra for weight). + * @param from - Start entity id (natural keys are resolved). + * @param to - End entity id. + * @param options - Direction, `by`, type filter, `maxDepth`, visibility opt-ins. + * @returns The route (`nodes` + `relationships` + `cost`), or `null` if unreachable. + * @example + * const route = await brain.graph.path(aliceId, bobId) + * route?.nodes // [aliceId, …, bobId] + */ + path(from: string, to: string, options?: GraphPathOptions): Promise +} + // ============= Batch Operations ============= /** @@ -738,6 +1024,14 @@ export interface AddManyParams { * on each item individually. Item-level `ifAbsent` overrides the batch flag. */ ifAbsent?: boolean + /** + * Create-or-update applied to every item. Equivalent to setting `upsert: true` on + * each item individually (see {@link AddParams.upsert}): an item whose custom `id` + * already exists is MERGED as an update rather than overwritten. Item-level `upsert` + * overrides the batch flag. Mutually exclusive with {@link AddManyParams.ifAbsent} + * at the item level — an item resolving to both throws. + */ + upsert?: boolean } /** @@ -759,6 +1053,14 @@ export interface RemoveManyParams { type?: NounType // Remove all of type where?: any // Remove by metadata limit?: number // Max to remove (safety) + /** + * Entities removed per transaction chunk. Each chunk is one atomic transaction — + * a chunk commits or rolls back together, and failures are isolated to their chunk. + * Defaults to the storage adapter's `maxBatchSize` (memory: 1000, S3/R2: 100, + * GCS: 50), so zero-config removals match the adapter's characteristics. Override + * only to tune throughput vs. transaction granularity. + */ + chunkSize?: number onProgress?: (done: number, total: number) => void continueOnError?: boolean // Continue processing if a removal fails } @@ -1082,7 +1384,7 @@ export interface MetricState { /** * Value multiset (String(value) → occurrence count) for exact percentile + distinctCount. * Maintained only for `percentile`/`distinctCount` metrics. JSON-serializable; mirrors - * Cortex's `value_counts` so JS and native agree bit-for-bit. + * Cor's `value_counts` so JS and native agree bit-for-bit. */ valueCounts?: Record } @@ -1141,7 +1443,7 @@ export interface AggregateResult { } /** - * Provider interface for Cortex-accelerated aggregation. + * Provider interface for Cor-accelerated aggregation. * When registered as 'aggregation' provider, Brainy delegates to this. */ export interface AggregationProvider { @@ -1273,6 +1575,31 @@ export interface BrainyStats { /** * Brainy configuration */ +/** + * Structured progress of the one-time, automatic 7.x → 8.0 migration (the + * coordinated LOCK). Relayed verbatim from the native provider's optional + * `migrationStatus()` and surfaced on `getIndexStatus().migration` while the + * brain is upgrading. Every field is optional — a provider may report only a + * phase, or nothing (in which case `getIndexStatus()` shows `migrating: true` + * plus `elapsedMs` alone). + */ +export interface MigrationProgress { + /** Coarse stage, e.g. `'rebuilding'` | `'verifying'`. */ + phase?: string + /** Which derived index is currently rebuilding. */ + index?: 'metadata' | 'vector' | 'graph' + /** Overall progress, 0–100. */ + percent?: number + /** Canonical entities processed so far. */ + entitiesDone?: number + /** Total canonical entities to process. */ + entitiesTotal?: number + /** When the migration was first observed (epoch ms). */ + startedAt?: number + /** Milliseconds elapsed since the migration was first observed. */ + elapsedMs?: number +} + export interface BrainyConfig { /** * Storage configuration: either a config object resolved through the @@ -1324,6 +1651,43 @@ export interface BrainyConfig { */ disableAutoRebuild?: boolean + /** + * How long (ms) an operation **waits** on the coordinated 7.x → 8.0 migration + * before throwing a retryable `MigrationInProgressError`. + * + * IMPORTANT: this bounds the *caller's wait*, NOT the migration. The migration + * itself is **unbounded** — a native provider rebuilds a billion-scale brain + * for as long as it needs, and this timeout never interrupts it. While the + * rebuild runs, reads and writes (and `init()` before it serves) block so no + * operation touches a half-built index: + * - **Small/medium upgrade (< this window):** the caller waits, then resumes + * transparently — no error. + * - **Large upgrade (> this window):** the caller gets a retryable + * `MigrationInProgressError` (the rebuild continues in the background); + * `getIndexStatus().migrating` — never gated — is the readiness-probe signal + * that maps to HTTP 503 + Retry-After so an orchestrator waits rather than + * routing traffic in. + * + * Raise it for a latency-tolerant batch job (wait longer / effectively wait + * through); lower it to fail fast on a request path. Default: `30000` (30 s). + */ + migrationWaitTimeoutMs?: number + + /** + * Take an automatic **pre-upgrade backup** before a one-time 7.x → 8.0 + * migration rebuilds the derived indexes, and remove it once the upgrade + * verifies (retain it on failure, for rollback). On the filesystem adapter + * this is a **hard-link snapshot** of the brain directory — near-zero cost and + * space (shared inodes; the store is immutable-by-rename), even at scale. + * The migration is already structurally safe (it only reads canonical records, + * derived indexes are fully reconstructable, and a failed upgrade self-heals on + * re-open), so this is catastrophe-insurance against a migration *bug*, not a + * data-loss guard — and NOT a substitute for an off-device backup. No-op for + * non-filesystem storage and for a brain with no persisted data. + * Default: `true`. Set `false` to opt out (e.g. you run your own backup). + */ + migrationBackup?: boolean + /** * Vector index configuration (Brainy 8.0). * @@ -1335,7 +1699,7 @@ export interface BrainyConfig { * byte-for-byte so a `'balanced'`-default upgrade is a no-op. * * **Closed-form contract** locked in handoff thread - * BRAINY-8.0-RENAME-COORDINATION § A.2 + § G.2 (cortex-confirmed 2026-06-09). + * BRAINY-8.0-RENAME-COORDINATION § A.2 + § G.2 (cor-confirmed 2026-06-09). */ vector?: { /** @@ -1368,33 +1732,52 @@ export interface BrainyConfig { } /** - * Generational-history retention policy (8.0 MVCC). + * Generational-history retention policy (8.0 MVCC — the `retention` knob). * - * Every `transact()` produces an immutable generation record-set that - * serves historical reads (`asOf()`, pinned `Db` values). Without - * compaction those record-sets accumulate forever, so Brainy - * **auto-compacts on every `flush()` and `close()`** with this policy. - * A generation is reclaimed only when it is older than BOTH bounds - * (and never while a live `Db` pin protects it): + * Under Model-B EVERY write (`transact()` AND single-op `add`/`update`/ + * `remove`/`relate`) produces an immutable generation record-set serving + * historical reads (`asOf()`, pinned `Db` values). Without compaction those + * accumulate, so Brainy **auto-compacts at `close()`** (time-bounded per + * pass; 8.9.0 removed compaction from `flush()` — flush is durability work + * and never pays maintenance costs). A long-lived writer that never closes + * accumulates history until its next explicit `compactHistory()` call. + * Live `Db` pins are ALWAYS exempt from reclamation, in every mode. * - * - `retainGenerations` — keep at least the N most recent committed - * generations (default: `100`). - * - `retainMs` — keep everything committed within the window - * (default: 7 days). + * Modes: + * - **unset → ADAPTIVE (default):** the retention horizon tracks + * disk/RAM pressure. A machine-level byte budget governs how much history + * is kept — driven by `budgetBytes` when a coordinator (e.g. cor's + * `ResourceManager`, fair-shared across co-located instances) sets it, else + * a local `os.freemem`/filesystem-free probe. Oldest-unpinned history is + * reclaimed when the budget is exceeded. Zero-config. + * - **`'all'` → unbounded:** never reclaim history (index compaction for + * query speed still runs — the decouple). The legacy keep-everything + * behavior, now explicit opt-in. + * - **`{ maxGenerations?, maxAge?, maxBytes? }` → explicit CAPS:** reclaim + * the oldest unpinned generations while ANY supplied cap is exceeded + * (predictable ops). `maxAge` in ms; `maxBytes` total history bytes. * - * Escape hatches: raise either bound for longer time-travel windows, - * or set `autoCompact: false` to manage history manually via - * `brain.compactHistory()`. Long-term archives belong in - * `db.persist(path)` snapshots, which compaction never touches. + * `autoCompact: false` disables the automatic close() compaction (manage + * manually via `brain.compactHistory()`). `budgetBytes` is the settable + * adaptive byte budget a coordinator drives (also via `brain.setRetentionBudget()`). + * Long-term archives belong in `db.persist(path)` snapshots, which compaction + * never touches. */ - history?: { - /** Keep at least this many recent committed generations (default: 100). */ - retainGenerations?: number - /** Keep generations committed within this window in ms (default: 604800000 = 7 days). */ - retainMs?: number - /** Run compaction automatically on flush()/close() (default: true). */ - autoCompact?: boolean - } + retention?: + | 'all' + | 'adaptive' + | { + /** Keep at most this many recent committed generations (cap). */ + maxGenerations?: number + /** Keep generations committed within this window in ms (cap). */ + maxAge?: number + /** Keep total generational-history bytes at or below this (cap). */ + maxBytes?: number + /** Adaptive byte budget for this brain, driven by a coordinator (e.g. cor). */ + budgetBytes?: number + /** Run compaction automatically at close() (default: true; 8.9.0 — flush() never compacts). */ + autoCompact?: boolean + } // Memory management options maxQueryLimit?: number // Override auto-detected query result limit (max: 100000) @@ -1424,11 +1807,15 @@ export interface BrainyConfig { eagerEmbeddings?: boolean // Plugin configuration - // Controls which plugins are loaded during init() - // - undefined (default): Auto-detect installed plugins (@soulcraft/cor, etc.) - // - false: No plugins — skip auto-detection entirely - // - []: No plugins — skip auto-detection entirely - // - ['@soulcraft/cor']: Load only specified plugins, no auto-detection + // Controls which plugins are loaded during init(). + // - undefined (default): guarded auto-detection of the first-party + // accelerator (@soulcraft/cor) — installing the package IS the opt-in. + // Not installed → plain brainy, silently. Installed → it loads and + // announces itself. Installed but broken → init() THROWS (an installed + // accelerator never silently vanishes behind the JS engines). + // - false / []: no plugins, no detection (explicit opt-out) + // - ['@soulcraft/cor']: load exactly these packages; a listed plugin that + // fails to load or is invalid THROWS (loud, never a silent JS fallback) plugins?: string[] | false // Logging configuration @@ -1496,6 +1883,33 @@ export interface BrainyConfig { * (PID liveness + heartbeat) cannot prove it. Logs a warning regardless. */ force?: boolean + + /** + * How write paths react when an untyped (JavaScript) caller smuggles a + * Brainy-reserved field (`RESERVED_ENTITY_FIELDS` / `RESERVED_RELATION_FIELDS` + * — `confidence`, `weight`, `subtype`, `visibility`, `service`, `createdBy`, + * `noun`/`verb`, `data`, `createdAt`, `updatedAt`, `_rev`) **inside the + * `metadata` bag** of `add()` / `update()` / `relate()` / `updateRelation()` + * (and their `transact()` / `with()` mirrors). TypeScript callers can't write + * these shapes at all — the compile-time guard on the metadata param types + * (`NoReservedEntityKeys` / `NoReservedRelationKeys`) rejects a literal + * reserved key — so this policy only governs untyped callers that slip one + * past the compiler. + * + * - `'throw'` (**default, 8.0**): a reserved key in the bag throws a clear + * `Error` naming the offending key(s) and the correct write path. No silent + * remap, no data loss, no surprise. This is the 8.0 "no silent failures" + * contract. + * - `'warn'`: legacy remapping with a loud, one-shot (per key, per process) + * warning for EVERY reserved key found — user-mutable fields are remapped to + * their dedicated top-level param (top-level wins when both are supplied), + * system-managed fields are dropped. Use while migrating untyped call sites. + * - `'remap'`: the pre-8.0 silent remapping, no warning. Last-resort + * compatibility hatch for code that intentionally relies on the bag path. + * + * @default 'throw' + */ + reservedFieldPolicy?: 'throw' | 'warn' | 'remap' } // ============= Neural API Types ============= diff --git a/src/types/fileSystemTypes.ts b/src/types/fileSystemTypes.ts deleted file mode 100644 index 4e927516..00000000 --- a/src/types/fileSystemTypes.ts +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Type declarations for the File System Access API - * Extends the FileSystemDirectoryHandle interface to include the [Symbol.asyncIterator] method - * and FileSystemHandle to include getFile() method for TypeScript compatibility - */ - -// Extend the FileSystemDirectoryHandle interface -interface FileSystemDirectoryHandle { - [Symbol.asyncIterator](): AsyncIterableIterator<[string, FileSystemHandle]>; - - keys(): AsyncIterableIterator; - - entries(): AsyncIterableIterator<[string, FileSystemHandle]>; -} - -// Extend the FileSystemHandle interface to include getFile method -// This is needed because TypeScript doesn't recognize that a FileSystemHandle -// can be a FileSystemFileHandle which has the getFile method -interface FileSystemHandle { - getFile?(): Promise; -} - -// Export something to make this a module -export const fileSystemTypesLoaded = true diff --git a/src/types/graphTypes.ts b/src/types/graphTypes.ts index 6e456a02..9fa7dd4c 100644 --- a/src/types/graphTypes.ts +++ b/src/types/graphTypes.ts @@ -399,10 +399,12 @@ export interface GraphVerb { } /** - * Version of GraphVerb for embedded relationships - * Used when the source is implicit from the parent document + * A {@link GraphVerb} for relationships embedded under a parent noun, where the + * source is implicit (the parent), so `sourceId` is dropped. (`GraphVerb` has no + * `source` field — the prior `Omit` was a no-op that left the + * shape identical to `GraphVerb`; this omits the real `sourceId`.) */ -export type EmbeddedGraphVerb = Omit +export type EmbeddedGraphVerb = Omit // Proper Noun interfaces - extend GraphNoun with specific noun types diff --git a/src/types/progress.types.ts b/src/types/progress.types.ts deleted file mode 100644 index b8444d01..00000000 --- a/src/types/progress.types.ts +++ /dev/null @@ -1,288 +0,0 @@ -/** - * Standardized Progress Reporting - * - * Provides unified progress tracking across all long-running operations - * in Brainy (imports, clustering, large searches, etc.) - */ - -/** - * Progress status states - */ -export type ProgressStatus = 'pending' | 'running' | 'completed' | 'failed' | 'cancelled' - -/** - * Standardized progress report - */ -export interface BrainyProgress { - // Core status - status: ProgressStatus - - // Progress percentage (0-100) - progress: number - - // Human-readable message - message: string - - // Detailed metadata - metadata: { - itemsProcessed: number - itemsTotal: number - currentItem?: string - estimatedTimeRemaining?: number // milliseconds - startedAt: number - completedAt?: number - throughput?: number // items per second - } - - // Result when completed - result?: T - - // Error when failed - error?: Error -} - -/** - * Progress tracker with automatic time estimation - */ -export class ProgressTracker { - private status: ProgressStatus = 'pending' - private processed = 0 - private total: number - private startedAt?: number - private completedAt?: number - private currentItem?: string - private result?: T - private error?: Error - private processingTimes: number[] = [] // Track last N processing times for estimation - - constructor(total: number) { - if (total < 0) { - throw new Error('Total must be non-negative') - } - this.total = total - } - - /** - * Factory method for creating progress trackers - */ - static create(total: number): ProgressTracker { - return new ProgressTracker(total) - } - - /** - * Start tracking progress - */ - start(): BrainyProgress { - this.status = 'running' - this.startedAt = Date.now() - return this.current() - } - - /** - * Update progress - */ - update(processed: number, currentItem?: string): BrainyProgress { - if (processed < 0) { - throw new Error('Processed count must be non-negative') - } - if (processed > this.total) { - throw new Error(`Processed count (${processed}) exceeds total (${this.total})`) - } - - const previousProcessed = this.processed - this.processed = processed - this.currentItem = currentItem - - // Track processing time for estimation - if (this.startedAt && previousProcessed < processed) { - const itemsProcessed = processed - previousProcessed - const timeTaken = Date.now() - this.startedAt - const avgTimePerItem = timeTaken / processed - this.processingTimes.push(avgTimePerItem) - - // Keep only last 100 measurements for rolling average - if (this.processingTimes.length > 100) { - this.processingTimes.shift() - } - } - - return this.current() - } - - /** - * Increment progress by 1 - */ - increment(currentItem?: string): BrainyProgress { - return this.update(this.processed + 1, currentItem) - } - - /** - * Mark as completed - */ - complete(result: T): BrainyProgress { - this.status = 'completed' - this.completedAt = Date.now() - this.processed = this.total - this.result = result - return this.current() - } - - /** - * Mark as failed - */ - fail(error: Error): BrainyProgress { - this.status = 'failed' - this.completedAt = Date.now() - this.error = error - return this.current() - } - - /** - * Mark as cancelled - */ - cancel(): BrainyProgress { - this.status = 'cancelled' - this.completedAt = Date.now() - return this.current() - } - - /** - * Get current progress state - */ - current(): BrainyProgress { - const progress = this.total > 0 ? Math.round((this.processed / this.total) * 100) : 0 - - // Generate message based on status - let message: string - switch (this.status) { - case 'pending': - message = `Ready to process ${this.total} items` - break - case 'running': - message = this.currentItem - ? `Processing: ${this.currentItem} (${this.processed}/${this.total})` - : `Processing ${this.processed}/${this.total} items` - break - case 'completed': - message = `Completed ${this.total} items` - break - case 'failed': - message = `Failed after ${this.processed} items: ${this.error?.message || 'Unknown error'}` - break - case 'cancelled': - message = `Cancelled after ${this.processed} items` - break - } - - return { - status: this.status, - progress, - message, - metadata: { - itemsProcessed: this.processed, - itemsTotal: this.total, - currentItem: this.currentItem, - estimatedTimeRemaining: this.estimateTimeRemaining(), - startedAt: this.startedAt || Date.now(), - completedAt: this.completedAt, - throughput: this.calculateThroughput() - }, - result: this.result, - error: this.error - } - } - - /** - * Estimate time remaining based on processing history - */ - private estimateTimeRemaining(): number | undefined { - if (this.status !== 'running' || !this.startedAt || this.processed === 0) { - return undefined - } - - const remaining = this.total - this.processed - if (remaining === 0) { - return 0 - } - - // Use rolling average if we have enough samples - if (this.processingTimes.length > 0) { - const avgTimePerItem = this.processingTimes.reduce((a, b) => a + b, 0) / this.processingTimes.length - return Math.round(avgTimePerItem * remaining) - } - - // Fallback to simple calculation - const elapsed = Date.now() - this.startedAt - const avgTimePerItem = elapsed / this.processed - return Math.round(avgTimePerItem * remaining) - } - - /** - * Calculate current throughput (items/second) - */ - private calculateThroughput(): number | undefined { - if (!this.startedAt || this.processed === 0) { - return undefined - } - - const elapsed = Date.now() - this.startedAt - const seconds = elapsed / 1000 - return seconds > 0 ? Math.round((this.processed / seconds) * 100) / 100 : undefined - } - - /** - * Get progress statistics - */ - getStats() { - const elapsed = this.startedAt ? Date.now() - this.startedAt : 0 - return { - status: this.status, - processed: this.processed, - total: this.total, - remaining: this.total - this.processed, - progress: this.total > 0 ? this.processed / this.total : 0, - elapsed, - estimatedTotal: elapsed > 0 && this.processed > 0 - ? Math.round((elapsed / this.processed) * this.total) - : undefined, - throughput: this.calculateThroughput() - } - } -} - -/** - * Helper to format time duration - */ -export function formatDuration(ms: number): string { - const seconds = Math.floor(ms / 1000) - const minutes = Math.floor(seconds / 60) - const hours = Math.floor(minutes / 60) - - if (hours > 0) { - return `${hours}h ${minutes % 60}m` - } else if (minutes > 0) { - return `${minutes}m ${seconds % 60}s` - } else { - return `${seconds}s` - } -} - -/** - * Helper to format progress percentage - */ -export function formatProgress(progress: BrainyProgress): string { - const { status, progress: pct, metadata } = progress - const remaining = metadata.estimatedTimeRemaining - - let str = `[${status.toUpperCase()}] ${pct}% (${metadata.itemsProcessed}/${metadata.itemsTotal})` - - if (metadata.throughput) { - str += ` - ${metadata.throughput} items/s` - } - - if (remaining && remaining > 0) { - str += ` - ${formatDuration(remaining)} remaining` - } - - return str -} diff --git a/src/types/typeMigration.ts b/src/types/typeMigration.ts deleted file mode 100644 index de4d3e06..00000000 --- a/src/types/typeMigration.ts +++ /dev/null @@ -1,174 +0,0 @@ -/** - * Type Migration Utilities for Stage 3 Taxonomy - * - * Provides migration helpers for code using removed types from v5.x - * - * ## Removed Types - * - * ### Nouns (2 removed): - * - `user` → merged into `person` - * - `topic` → merged into `concept` - * - * ### Verbs (4 removed): - * - `succeeds` → use inverse of `precedes` - * - `belongsTo` → use inverse of `owns` - * - `createdBy` → use inverse of `creates` - * - `supervises` → use inverse of `reportsTo` - */ - -import { NounType, VerbType } from './graphTypes.js' - -/** - * Migration mapping for removed noun types - */ -export const REMOVED_NOUN_TYPES = { - user: NounType.Person, - topic: NounType.Concept -} as const - -/** - * Migration mapping for removed verb types - * Note: Some verbs should use inverse relationships in Stage 3 - */ -export const REMOVED_VERB_TYPES = { - succeeds: VerbType.Precedes, // Use with inverted source/target - belongsTo: VerbType.Owns, // Use with inverted source/target - createdBy: VerbType.Creates, // Use with inverted source/target - supervises: VerbType.ReportsTo // Use with inverted source/target -} as const - -/** - * Check if a type was removed in Stage 3 - */ -export function isRemovedNounType(type: string): type is keyof typeof REMOVED_NOUN_TYPES { - return type in REMOVED_NOUN_TYPES -} - -/** - * Check if a verb type was removed in Stage 3 - */ -export function isRemovedVerbType(type: string): type is keyof typeof REMOVED_VERB_TYPES { - return type in REMOVED_VERB_TYPES -} - -/** - * Migrate a noun type (Stage 3) - * Returns the migrated type or the original if no migration needed - */ -export function migrateNounType(type: string): NounType { - if (isRemovedNounType(type)) { - console.warn(`⚠️ NounType "${type}" was removed in v6.0. Migrating to "${REMOVED_NOUN_TYPES[type]}"`) - return REMOVED_NOUN_TYPES[type] - } - return type as NounType -} - -/** - * Migrate a verb type (Stage 3) - * Returns the migrated type or the original if no migration needed - * - * WARNING: Some verbs require inverting source/target relationships! - * See VERB_REQUIRES_INVERSION for details. - */ -export function migrateVerbType(type: string): VerbType { - if (isRemovedVerbType(type)) { - console.warn(`⚠️ VerbType "${type}" was removed in v6.0. Migrating to "${REMOVED_VERB_TYPES[type]}" (may require source/target inversion)`) - return REMOVED_VERB_TYPES[type] - } - return type as VerbType -} - -/** - * Verbs that require inverting source/target when migrating - * - * Example: - * - Old: `A createdBy B` → New: `B creates A` - * - Old: `A belongsTo B` → New: `B owns A` - * - Old: `A supervises B` → New: `B reportsTo A` - * - Old: `A succeeds B` → New: `B precedes A` - */ -export const VERB_REQUIRES_INVERSION = new Set([ - 'succeeds', - 'belongsTo', - 'createdBy', - 'supervises' -]) - -/** - * Check if a verb type requires inverting source/target during migration - */ -export function requiresInversion(oldVerbType: string): boolean { - return VERB_REQUIRES_INVERSION.has(oldVerbType) -} - -/** - * Migrate a relationship, handling source/target inversion if needed - * - * @returns Object with migrated verb and potentially inverted source/target - */ -export function migrateRelationship(params: { - verb: string - source: string - target: string -}): { - verb: VerbType - source: string - target: string - inverted: boolean -} { - const verb = migrateVerbType(params.verb) - const inverted = requiresInversion(params.verb) - - if (inverted) { - return { - verb, - source: params.target, // Swap source and target - target: params.source, - inverted: true - } - } - - return { - verb, - source: params.source, - target: params.target, - inverted: false - } -} - -/** - * Stage 3 Type Compatibility Check - * Helps developers identify code that needs updating - */ -export function checkTypeCompatibility(nounTypes: string[], verbTypes: string[]): { - valid: boolean - removedNouns: string[] - removedVerbs: string[] - warnings: string[] -} { - const removedNouns = nounTypes.filter(isRemovedNounType) - const removedVerbs = verbTypes.filter(isRemovedVerbType) - const warnings: string[] = [] - - if (removedNouns.length > 0) { - warnings.push( - `Found ${removedNouns.length} removed noun type(s): ${removedNouns.join(', ')}. ` + - `These were merged in Stage 3. Use Person instead of User, Concept instead of Topic.` - ) - } - - if (removedVerbs.length > 0) { - warnings.push( - `Found ${removedVerbs.length} removed verb type(s): ${removedVerbs.join(', ')}. ` + - `These require using inverse relationships in Stage 3. ` + - `Example: "A createdBy B" becomes "B creates A".` - ) - } - - return { - valid: removedNouns.length === 0 && removedVerbs.length === 0, - removedNouns, - removedVerbs, - warnings - } -} diff --git a/src/unified.ts b/src/unified.ts deleted file mode 100644 index 987dd867..00000000 --- a/src/unified.ts +++ /dev/null @@ -1,39 +0,0 @@ -/** - * @module unified - * @description Unified entry point for Brainy. Re-exports the public surface - * from `index.js` and surfaces a tiny runtime-environment object the rest of - * the library can read. - * - * 8.0 supports Node-like runtimes only (Node.js, Bun, Deno); the browser - * detection helpers and Worker Thread plumbing are gone. - */ - -import './setup.js' - -import { isNode } from './utils/environment.js' - -export const environment = { - get isNode() { - return isNode() - } -} - -declare global { - /** - * Runtime-environment marker published by the unified entry point so - * embedders and diagnostics can detect the active Brainy environment. - * Ambient `var` is required — `let`/`const` in `declare global` do not - * attach to `globalThis`. - */ - var __ENV__: typeof environment | undefined -} - -if (typeof globalThis !== 'undefined') { - globalThis.__ENV__ = environment -} - -console.log( - `Brainy running on ${environment.isNode ? 'Node-like runtime' : 'unknown runtime'}` -) - -export * from './index.js' diff --git a/src/universal/crypto.ts b/src/universal/crypto.ts deleted file mode 100644 index 4c117c1f..00000000 --- a/src/universal/crypto.ts +++ /dev/null @@ -1,159 +0,0 @@ -/** - * Universal Crypto implementation - * Framework-friendly: Trusts that frameworks provide crypto polyfills - * Works in all environments: Browser (via framework), Node.js, Serverless - */ - -import { isNode } from '../utils/environment.js' - -let nodeCrypto: any = null - -// Dynamic import for Node.js crypto (only in Node.js environment) -if (isNode()) { - try { - // Use node: protocol to prevent bundler polyfilling (requires Node 22+) - nodeCrypto = await import('node:crypto') - } catch { - // Ignore import errors in non-Node environments - } -} - -/** - * Generate random bytes - * Framework-friendly: Assumes crypto API is available via framework polyfills - */ -export function randomBytes(size: number): Uint8Array { - if (typeof crypto !== 'undefined') { - // Use Web Crypto API (available in browsers via framework polyfills and modern Node.js) - const array = new Uint8Array(size) - crypto.getRandomValues(array) - return array - } else if (nodeCrypto) { - // Use Node.js crypto - return new Uint8Array(nodeCrypto.randomBytes(size)) - } else { - throw new Error('Crypto API not available. Framework bundlers should provide crypto polyfills.') - } -} - -/** - * Generate random UUID - * Framework-friendly: Assumes crypto.randomUUID is available via framework polyfills - */ -export function randomUUID(): string { - if (typeof crypto !== 'undefined' && crypto.randomUUID) { - return crypto.randomUUID() - } else if (nodeCrypto && nodeCrypto.randomUUID) { - return nodeCrypto.randomUUID() - } else { - throw new Error('crypto.randomUUID not available. Framework bundlers should provide crypto polyfills.') - } -} - -/** - * Create hash (simplified interface) - * Framework-friendly: Relies on Node.js crypto or framework-provided implementations - */ -export function createHash(algorithm: string): { - update: (data: string | Uint8Array) => any - digest: (encoding: string) => string -} { - if (nodeCrypto && nodeCrypto.createHash) { - return nodeCrypto.createHash(algorithm) - } else { - throw new Error(`createHash not available. For browser environments, frameworks should provide crypto polyfills or use Web Crypto API directly.`) - } -} - -/** - * Create HMAC - * Framework-friendly: Relies on Node.js crypto or framework-provided implementations - */ -export function createHmac(algorithm: string, key: string | Uint8Array): { - update: (data: string | Uint8Array) => any - digest: (encoding: string) => string -} { - if (nodeCrypto && nodeCrypto.createHmac) { - return nodeCrypto.createHmac(algorithm, key) - } else { - throw new Error(`createHmac not available. For browser environments, frameworks should provide crypto polyfills or use Web Crypto API directly.`) - } -} - -/** - * PBKDF2 synchronous - * Framework-friendly: Relies on Node.js crypto or framework-provided implementations - */ -export function pbkdf2Sync(password: string | Uint8Array, salt: string | Uint8Array, iterations: number, keylen: number, digest: string): Uint8Array { - if (nodeCrypto && nodeCrypto.pbkdf2Sync) { - return new Uint8Array(nodeCrypto.pbkdf2Sync(password, salt, iterations, keylen, digest)) - } else { - throw new Error(`pbkdf2Sync not available. For browser environments, frameworks should provide crypto polyfills or use Web Crypto API directly.`) - } -} - -/** - * Scrypt synchronous - * Framework-friendly: Relies on Node.js crypto or framework-provided implementations - */ -export function scryptSync(password: string | Uint8Array, salt: string | Uint8Array, keylen: number, options?: any): Uint8Array { - if (nodeCrypto && nodeCrypto.scryptSync) { - return new Uint8Array(nodeCrypto.scryptSync(password, salt, keylen, options)) - } else { - throw new Error(`scryptSync not available. For browser environments, frameworks should provide crypto polyfills or use Web Crypto API directly.`) - } -} - -/** - * Create cipher - * Framework-friendly: Relies on Node.js crypto or framework-provided implementations - */ -export function createCipheriv(algorithm: string, key: Uint8Array, iv: Uint8Array): { - update: (data: string, inputEncoding?: string, outputEncoding?: string) => string - final: (outputEncoding?: string) => string -} { - if (nodeCrypto && nodeCrypto.createCipheriv) { - return nodeCrypto.createCipheriv(algorithm, key, iv) - } else { - throw new Error(`createCipheriv not available. For browser environments, frameworks should provide crypto polyfills or use Web Crypto API directly.`) - } -} - -/** - * Create decipher - * Framework-friendly: Relies on Node.js crypto or framework-provided implementations - */ -export function createDecipheriv(algorithm: string, key: Uint8Array, iv: Uint8Array): { - update: (data: string, inputEncoding?: string, outputEncoding?: string) => string - final: (outputEncoding?: string) => string -} { - if (nodeCrypto && nodeCrypto.createDecipheriv) { - return nodeCrypto.createDecipheriv(algorithm, key, iv) - } else { - throw new Error(`createDecipheriv not available. For browser environments, frameworks should provide crypto polyfills or use Web Crypto API directly.`) - } -} - -/** - * Timing safe equal - * Framework-friendly: Relies on Node.js crypto or framework-provided implementations - */ -export function timingSafeEqual(a: Uint8Array, b: Uint8Array): boolean { - if (nodeCrypto && nodeCrypto.timingSafeEqual) { - return nodeCrypto.timingSafeEqual(a, b) - } else { - throw new Error(`timingSafeEqual not available. For browser environments, frameworks should provide crypto polyfills or use Web Crypto API directly.`) - } -} - -export default { - randomBytes, - randomUUID, - createHash, - createHmac, - pbkdf2Sync, - scryptSync, - createCipheriv, - createDecipheriv, - timingSafeEqual -} \ No newline at end of file diff --git a/src/utils/BoundedRegistry.ts b/src/utils/BoundedRegistry.ts deleted file mode 100644 index f8761546..00000000 --- a/src/utils/BoundedRegistry.ts +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Bounded Registry with LRU eviction - * Prevents unbounded memory growth in production - */ -export class BoundedRegistry { - private items = new Map() // item -> last accessed timestamp - private readonly maxSize: number - - constructor(maxSize: number = 10000) { - this.maxSize = maxSize - } - - /** - * Add item to registry, evicting oldest if at capacity - */ - add(item: T): void { - // Update timestamp if already exists - if (this.items.has(item)) { - this.items.delete(item) // Remove to re-add at end - this.items.set(item, Date.now()) - return - } - - // Evict oldest if at capacity - if (this.items.size >= this.maxSize) { - const oldest = this.items.entries().next().value - if (oldest) { - this.items.delete(oldest[0]) - } - } - - this.items.set(item, Date.now()) - } - - /** - * Check if item exists - */ - has(item: T): boolean { - return this.items.has(item) - } - - /** - * Get all items - */ - getAll(): T[] { - return Array.from(this.items.keys()) - } - - /** - * Get size - */ - get size(): number { - return this.items.size - } - - /** - * Clear all items - */ - clear(): void { - this.items.clear() - } -} \ No newline at end of file diff --git a/src/utils/collation.ts b/src/utils/collation.ts index f56b0940..be584114 100644 --- a/src/utils/collation.ts +++ b/src/utils/collation.ts @@ -7,12 +7,12 @@ * and is byte-for-byte identical to Rust's `str::cmp` (`as_bytes().cmp()`). This is: * - **deterministic** across environments (unlike `localeCompare`, whose default-locale * ordering varies by OS / Node / ICU build — unsafe for a persisted sorted index), and - * - **cross-language consistent** with `@soulcraft/cortex`'s native column store and + * - **cross-language consistent** with `@soulcraft/cor`'s native column store and * aggregation sort, so query results are identical with or without the native plugin. * * Use this instead of `String.localeCompare` and the `<`/`>` operators (which compare * UTF-16 code units and diverge from code-point order for supplementary-plane characters) - * wherever ordering is persisted or must match the native engine. See cortex `docs/ADR-001`. + * wherever ordering is persisted or must match the native engine. See cor `docs/ADR-001`. */ const utf8 = new TextEncoder() diff --git a/src/utils/crc32c.ts b/src/utils/crc32c.ts new file mode 100644 index 00000000..0c2a1c14 --- /dev/null +++ b/src/utils/crc32c.ts @@ -0,0 +1,43 @@ +/** + * @module utils/crc32c + * @description CRC-32C (Castagnoli, polynomial 0x1EDC6F41, reflected 0x82F63B78) + * — the storage-industry frame checksum (ext4, iSCSI, SCTP, LSM segment files). + * Used to frame generation-fact segments: every appended record carries the + * CRC-32C of its payload, so a torn tail (crash mid-append) or bit rot is + * DETECTED at scan time and never silently read as data. + * + * Table-driven, dependency-free reference implementation. Native providers may + * substitute a hardware-accelerated (SSE4.2 / ARMv8 CRC) implementation — the + * polynomial is the contract, byte-identical results required. + */ + +/** The 256-entry lookup table for the reflected CRC-32C polynomial. */ +const TABLE: Uint32Array = (() => { + const table = new Uint32Array(256) + for (let n = 0; n < 256; n++) { + let c = n + for (let k = 0; k < 8; k++) { + c = c & 1 ? 0x82f63b78 ^ (c >>> 1) : c >>> 1 + } + table[n] = c >>> 0 + } + return table +})() + +/** + * Compute the CRC-32C checksum of a byte buffer. + * + * Known-answer vectors (RFC 3720 appendix / the standard test suite): + * - ASCII "123456789" → 0xE3069283 + * - 32 zero bytes → 0x8A9136AA + * + * @param bytes - The payload to checksum. + * @returns The CRC-32C as an unsigned 32-bit integer. + */ +export function crc32c(bytes: Uint8Array): number { + let crc = 0xffffffff + for (let i = 0; i < bytes.length; i++) { + crc = TABLE[(crc ^ bytes[i]) & 0xff] ^ (crc >>> 8) + } + return (crc ^ 0xffffffff) >>> 0 +} diff --git a/src/utils/crypto.ts b/src/utils/crypto.ts deleted file mode 100644 index f74c9fc5..00000000 --- a/src/utils/crypto.ts +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Cross-platform crypto utilities - * Provides hashing functions that work in both Node.js and browser environments - */ - -/** - * Simple string hash function that works in all environments - * Uses djb2 algorithm - fast and good distribution - * @param str - String to hash - * @returns Positive integer hash - */ -export function hashString(str: string): number { - let hash = 5381 - for (let i = 0; i < str.length; i++) { - const char = str.charCodeAt(i) - hash = ((hash << 5) + hash) + char // hash * 33 + char - } - // Ensure positive number - return Math.abs(hash) -} - -/** - * Alternative: FNV-1a hash algorithm - * Good distribution and fast - * @param str - String to hash - * @returns Positive integer hash - */ -export function fnv1aHash(str: string): number { - let hash = 2166136261 - for (let i = 0; i < str.length; i++) { - hash ^= str.charCodeAt(i) - hash = (hash * 16777619) >>> 0 - } - return hash -} - -/** - * Generate a deterministic hash for partitioning - * Uses the most appropriate algorithm for the environment - * @param input - Input string to hash - * @returns Positive integer hash suitable for modulo operations - */ -export function getPartitionHash(input: string): number { - // Use djb2 by default as it's fast and has good distribution - // This ensures consistent partitioning across all environments - return hashString(input) -} \ No newline at end of file diff --git a/src/utils/deletedItemsIndex.ts b/src/utils/deletedItemsIndex.ts deleted file mode 100644 index 3f497f40..00000000 --- a/src/utils/deletedItemsIndex.ts +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Dedicated index for tracking soft-deleted items - * This is MUCH more efficient than checking every item in the database - * - * Performance characteristics: - * - Add deleted item: O(1) - * - Remove deleted item: O(1) - * - Check if deleted: O(1) - * - Get all deleted: O(d) where d = number of deleted items << total items - */ - -export class DeletedItemsIndex { - private deletedIds: Set = new Set() - private deletedCount: number = 0 - - /** - * Mark an item as deleted - */ - markDeleted(id: string): void { - if (!this.deletedIds.has(id)) { - this.deletedIds.add(id) - this.deletedCount++ - } - } - - /** - * Mark an item as not deleted (restored) - */ - markRestored(id: string): void { - if (this.deletedIds.delete(id)) { - this.deletedCount-- - } - } - - /** - * Check if an item is deleted - O(1) - */ - isDeleted(id: string): boolean { - return this.deletedIds.has(id) - } - - /** - * Get all deleted item IDs - O(d) - */ - getAllDeleted(): string[] { - return Array.from(this.deletedIds) - } - - /** - * Filter out deleted items from results - O(k) where k = result count - */ - filterDeleted(items: T[]): T[] { - if (this.deletedCount === 0) { - // Fast path - no deleted items - return items - } - - return items.filter(item => { - const id = item.id - return id ? !this.deletedIds.has(id) : true - }) - } - - /** - * Get statistics - */ - getStats() { - return { - deletedCount: this.deletedCount, - memoryUsage: this.deletedCount * 100 // Rough estimate: 100 bytes per ID - } - } - - /** - * Clear all deleted items (for testing) - */ - clear(): void { - this.deletedIds.clear() - this.deletedCount = 0 - } - - /** - * Serialize for persistence - */ - serialize(): string { - return JSON.stringify(Array.from(this.deletedIds)) - } - - /** - * Deserialize from persistence - */ - deserialize(data: string): void { - try { - const ids = JSON.parse(data) - this.deletedIds = new Set(ids) - this.deletedCount = this.deletedIds.size - } catch (e) { - console.warn('Failed to deserialize deleted items index') - } - } -} - -/** - * Global singleton for deleted items tracking - */ -export const deletedItemsIndex = new DeletedItemsIndex() \ No newline at end of file diff --git a/src/utils/distance.ts b/src/utils/distance.ts index 524f2837..36e9e8e5 100644 --- a/src/utils/distance.ts +++ b/src/utils/distance.ts @@ -1,58 +1,60 @@ /** - * Distance functions for vector similarity calculations - * Optimized pure JavaScript implementations using enhanced array methods - * Faster than GPU for small vectors (384 dims) due to no transfer overhead + * Distance functions for vector similarity calculations. + * + * Pure-JavaScript implementations using allocation-free indexed loops: a single + * pass over the two vectors with scalar accumulators and no per-element closures + * or intermediate objects. This is the open-core distance path (the native + * provider owns the SIMD/quantized billion-scale path); for the small/medium + * vectors it serves (e.g. 384-dim sentence embeddings) a tight loop keeps the + * whole computation in registers with zero GC pressure. + * + * MEASURED (tests/benchmarks/distance-microbench.mjs, dim=384, N=20000, median + * of 41): rewriting cosine from an object-accumulating `reduce` to this loop is + * ~6x on `number[]`; euclidean ~1.4x. (`number[]` is also measurably faster than + * `Float32Array` here — V8 widens f32→f64 on every element read — so the + * resident representation stays `number[]`.) */ import { DistanceFunction, Vector } from '../coreTypes.js' /** - * Calculates the Euclidean distance between two vectors - * Lower values indicate higher similarity - * Optimized using array methods for Node.js 23.11+ + * Calculates the Euclidean (L2) distance between two vectors. + * Lower values indicate higher similarity. */ -export const euclideanDistance: DistanceFunction = ( - a: Vector, - b: Vector -): number => { +export const euclideanDistance: DistanceFunction = (a: Vector, b: Vector): number => { if (a.length !== b.length) { throw new Error('Vectors must have the same dimensions') } - // Use array.reduce for better performance in Node.js 23.11+ - const sum = a.reduce((acc, val, i) => { - const diff = val - b[i] - return acc + diff * diff - }, 0) - + let sum = 0 + const len = a.length + for (let i = 0; i < len; i++) { + const diff = a[i] - b[i] + sum += diff * diff + } return Math.sqrt(sum) } /** - * Calculates the cosine distance between two vectors - * Lower values indicate higher similarity - * Range: 0 (identical) to 2 (opposite) - * Optimized using array methods for Node.js 23.11+ + * Calculates the cosine distance between two vectors. + * Lower values indicate higher similarity. Range: 0 (identical) to 2 (opposite). */ -export const cosineDistance: DistanceFunction = ( - a: Vector, - b: Vector -): number => { +export const cosineDistance: DistanceFunction = (a: Vector, b: Vector): number => { if (a.length !== b.length) { throw new Error('Vectors must have the same dimensions') } - // Use array.reduce to calculate all values in a single pass - const { dotProduct, normA, normB } = a.reduce( - (acc, val, i) => { - return { - dotProduct: acc.dotProduct + val * b[i], - normA: acc.normA + val * val, - normB: acc.normB + b[i] * b[i] - } - }, - { dotProduct: 0, normA: 0, normB: 0 } - ) + let dotProduct = 0 + let normA = 0 + let normB = 0 + const len = a.length + for (let i = 0; i < len; i++) { + const av = a[i] + const bv = b[i] + dotProduct += av * bv + normA += av * av + normB += bv * bv + } if (normA === 0 || normB === 0) { return 2 // Maximum distance for zero vectors @@ -64,150 +66,60 @@ export const cosineDistance: DistanceFunction = ( } /** - * Calculates the Manhattan (L1) distance between two vectors - * Lower values indicate higher similarity - * Optimized using array methods for Node.js 23.11+ + * Calculates the Manhattan (L1) distance between two vectors. + * Lower values indicate higher similarity. */ -export const manhattanDistance: DistanceFunction = ( - a: Vector, - b: Vector -): number => { +export const manhattanDistance: DistanceFunction = (a: Vector, b: Vector): number => { if (a.length !== b.length) { throw new Error('Vectors must have the same dimensions') } - // Use array.reduce for better performance in Node.js 23.11+ - return a.reduce((sum, val, i) => sum + Math.abs(val - b[i]), 0) + let sum = 0 + const len = a.length + for (let i = 0; i < len; i++) { + sum += Math.abs(a[i] - b[i]) + } + return sum } /** - * Calculates the dot product similarity between two vectors - * Higher values indicate higher similarity - * Converted to a distance metric (lower is better) - * Optimized using array methods for Node.js 23.11+ + * Calculates the dot-product similarity between two vectors, negated to a + * distance metric (lower is better). */ -export const dotProductDistance: DistanceFunction = ( - a: Vector, - b: Vector -): number => { +export const dotProductDistance: DistanceFunction = (a: Vector, b: Vector): number => { if (a.length !== b.length) { throw new Error('Vectors must have the same dimensions') } - // Use array.reduce for better performance in Node.js 23.11+ - const dotProduct = a.reduce((sum, val, i) => sum + val * b[i], 0) - - // Convert to a distance metric (lower is better) + let dotProduct = 0 + const len = a.length + for (let i = 0; i < len; i++) { + dotProduct += a[i] * b[i] + } return -dotProduct } /** - * Batch distance calculation using optimized JavaScript - * More efficient than GPU for small vectors due to no memory transfer overhead + * Batch distance calculation: the query vector against each candidate. * - * @param queryVector The query vector to compare against all vectors - * @param vectors Array of vectors to compare against - * @param distanceFunction The distance function to use - * @returns Promise resolving to array of distances + * With the distance functions now allocation-free indexed loops, this is a thin + * map over the (monomorphic, JIT-inlined) `distanceFunction` — no worker, no + * stringify/`new Function` reconstruction. Kept `async` for call-site + * compatibility with the HNSW search path. + * + * @param queryVector The query vector to compare against all candidates. + * @param vectors The candidate vectors. + * @param distanceFunction The distance function to use (default: Euclidean). + * @returns The distances, index-aligned with `vectors`. */ export async function calculateDistancesBatch( queryVector: Vector, vectors: Vector[], distanceFunction: DistanceFunction = euclideanDistance ): Promise { - // For small batches, use the standard distance function - if (vectors.length < 10) { - return vectors.map((vector) => distanceFunction(queryVector, vector)) - } - - try { - // Function for optimized batch distance calculation - const distanceCalculator = (args: { - queryVector: Vector - vectors: Vector[] - distanceFnString: string - }) => { - const { queryVector, vectors, distanceFnString } = args - - // Optimized JavaScript implementations for different distance functions - let distances: number[] - - if (distanceFnString.includes('euclideanDistance')) { - // Euclidean distance: sqrt(sum((a - b)^2)) - distances = vectors.map((vector) => { - let sum = 0 - for (let i = 0; i < queryVector.length; i++) { - const diff = queryVector[i] - vector[i] - sum += diff * diff - } - return Math.sqrt(sum) - }) - } else if (distanceFnString.includes('cosineDistance')) { - // Cosine distance: 1 - (a·b / (||a|| * ||b||)) - distances = vectors.map((vector) => { - let dotProduct = 0 - let queryNorm = 0 - let vectorNorm = 0 - - for (let i = 0; i < queryVector.length; i++) { - dotProduct += queryVector[i] * vector[i] - queryNorm += queryVector[i] * queryVector[i] - vectorNorm += vector[i] * vector[i] - } - - queryNorm = Math.sqrt(queryNorm) - vectorNorm = Math.sqrt(vectorNorm) - - if (queryNorm === 0 || vectorNorm === 0) { - return 1 // Maximum distance for zero vectors - } - - const cosineSimilarity = dotProduct / (queryNorm * vectorNorm) - return 1 - cosineSimilarity - }) - } else if (distanceFnString.includes('manhattanDistance')) { - // Manhattan distance: sum(|a - b|) - distances = vectors.map((vector) => { - let sum = 0 - for (let i = 0; i < queryVector.length; i++) { - sum += Math.abs(queryVector[i] - vector[i]) - } - return sum - }) - } else if (distanceFnString.includes('dotProductDistance')) { - // Dot product distance: -sum(a * b) - distances = vectors.map((vector) => { - let dotProduct = 0 - for (let i = 0; i < queryVector.length; i++) { - dotProduct += queryVector[i] * vector[i] - } - return -dotProduct - }) - } else { - // For unknown distance functions, use the provided function - const distanceFunction = new Function( - 'return ' + distanceFnString - )() as DistanceFunction - - distances = vectors.map((vector) => - distanceFunction(queryVector, vector) - ) - } - - return { distances } - } - - // Use the optimized distance calculator - const result = distanceCalculator({ - queryVector, - vectors, - distanceFnString: distanceFunction.toString() - }) - - return result.distances - } catch (error) { - // If anything fails, fall back to the standard distance function - console.error('Batch distance calculation failed:', error) - return vectors.map((vector) => distanceFunction(queryVector, vector)) + const out = new Array(vectors.length) + for (let i = 0; i < vectors.length; i++) { + out[i] = distanceFunction(queryVector, vectors[i]) } + return out } diff --git a/src/utils/ensureDeleted.ts b/src/utils/ensureDeleted.ts deleted file mode 100644 index 6d9ddce7..00000000 --- a/src/utils/ensureDeleted.ts +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Utility to ensure all metadata has the deleted field set properly - * This is CRITICAL for O(1) soft delete filtering performance - * - * Uses _brainy namespace to avoid conflicts with user metadata - */ - -const BRAINY_NAMESPACE = '_brainy' - -/** - * Ensure metadata has internal Brainy fields set - * @param metadata The metadata object (could be null/undefined) - * @param preserveExisting If true, preserve existing deleted value - * @returns Metadata with internal fields guaranteed - */ -export function ensureDeletedField(metadata: any, preserveExisting: boolean = true): any { - // Handle null/undefined metadata - if (!metadata) { - return { - [BRAINY_NAMESPACE]: { - deleted: false, - version: 1 - } - } - } - - // Clone to avoid mutation - const result = { ...metadata } - - // Ensure _brainy namespace exists - if (!result[BRAINY_NAMESPACE]) { - result[BRAINY_NAMESPACE] = {} - } - - // Set deleted field if not present - if (!('deleted' in result[BRAINY_NAMESPACE])) { - result[BRAINY_NAMESPACE].deleted = false - } else if (!preserveExisting) { - // Force to false if not preserving - result[BRAINY_NAMESPACE].deleted = false - } - - return result -} - -/** - * Mark an item as soft deleted - * @param metadata The metadata object - * @returns Metadata with _brainy.deleted=true - */ -export function markAsDeleted(metadata: any): any { - const result = ensureDeletedField(metadata) - result[BRAINY_NAMESPACE].deleted = true - return result -} - -/** - * Mark an item as restored (not deleted) - * @param metadata The metadata object - * @returns Metadata with _brainy.deleted=false - */ -export function markAsRestored(metadata: any): any { - const result = ensureDeletedField(metadata) - result[BRAINY_NAMESPACE].deleted = false - return result -} - -/** - * Check if an item is deleted - * @param metadata The metadata object - * @returns true if deleted, false otherwise (including if field missing) - */ -export function isDeleted(metadata: any): boolean { - return metadata?.[BRAINY_NAMESPACE]?.deleted === true -} - -/** - * Check if an item is active (not deleted) - * @param metadata The metadata object - * @returns true if not deleted (default), false if deleted - */ -export function isActive(metadata: any): boolean { - // If no deleted field or deleted=false, item is active - return !isDeleted(metadata) -} - -// Export the namespace constant for use in queries -export const BRAINY_DELETED_FIELD = `${BRAINY_NAMESPACE}.deleted` \ No newline at end of file diff --git a/src/utils/entityIdMapper.ts b/src/utils/entityIdMapper.ts index 7a54846f..5b5afb5e 100644 --- a/src/utils/entityIdMapper.ts +++ b/src/utils/entityIdMapper.ts @@ -40,7 +40,7 @@ import type { EntityIdMapperProvider } from '../plugin.js' * The largest entity int the JS fallback `EntityIdMapper` will allocate. * The metadata index's roaring bitmaps are 32-bit-keyed (`Roaring32`), so * allowing the JS counter past this point would silently corrupt every - * downstream bitmap. The cortex 3.0 binary mapper supports a U64 IdSpace + * downstream bitmap. The cor 3.0 binary mapper supports a U64 IdSpace * for brains above this ceiling — see the * [`EntityIdSpaceExceeded`](#EntityIdSpaceExceeded) message for the * migration pointer. @@ -50,8 +50,8 @@ export const U32_ENTITY_ID_MAX = 0xffff_ffff /** * Thrown by the JS fallback `EntityIdMapper` when `nextId` would exceed * [`U32_ENTITY_ID_MAX`](#U32_ENTITY_ID_MAX). The JS path is the - * cortex-free fallback; once a brain has more than ~4.29 B entities, - * callers MUST install the cortex 3.0 `NativeBinaryEntityIdMapper` with + * cor-free fallback; once a brain has more than ~4.29 B entities, + * callers MUST install the cor 3.0 `NativeBinaryEntityIdMapper` with * `idSpace: 'u64'` (mmap-backed extendible-hash KV; persists the full * u64 range losslessly). * @@ -96,7 +96,7 @@ export interface EntityIdMapperData { * Maps entity UUIDs to integer IDs for use with Roaring Bitmaps. * * Implements {@link EntityIdMapperProvider}: the surface a registered - * `'entityIdMapper'` provider (e.g. Cortex's native mapper) must also satisfy. + * `'entityIdMapper'` provider (e.g. Cor's native mapper) must also satisfy. */ export class EntityIdMapper implements EntityIdMapperProvider { private storage: StorageAdapter @@ -162,7 +162,7 @@ export class EntityIdMapper implements EntityIdMapperProvider { * The JS fallback mapper caps at [`U32_ENTITY_ID_MAX`](#U32_ENTITY_ID_MAX) * to match the metadata index's Roaring32 bitmap width — once `nextId` * would exceed that, throws {@link EntityIdSpaceExceeded} so the caller - * loudly migrates to cortex's binary mapper with `idSpace: 'u64'` + * loudly migrates to cor's binary mapper with `idSpace: 'u64'` * rather than silently truncating entity ids. */ getOrAssign(uuid: string): number { @@ -291,6 +291,27 @@ export class EntityIdMapper implements EntityIdMapperProvider { return result } + /** + * @description Batch reverse-resolve u64 entity ints (a `BigInt64Array`) → UUID + * strings — the bigint counterpart of {@link EntityIdMapper.intsIterableToUuids}, + * for the native graph engine whose `Subgraph.nodes` is a `BigInt64Array`. The JS + * mapper is keyed by `number` (u32-era); each int is narrowed via `Number()` — safe + * for the JS fallback's id range. Order-preserving (one entry per input): an int the + * mapper has not assigned yields `''` (does not occur for graph-engine results, which + * reference assigned ints only). + * @param nodeInts - Entity ints as returned in a graph `Subgraph`. + * @returns One UUID per input, in order (`''` for a never-assigned int). + * @example + * const uuids = mapper.entityIntsToUuids(subgraph.nodes) + */ + entityIntsToUuids(nodeInts: BigInt64Array): string[] { + const result: string[] = new Array(nodeInts.length) + for (let i = 0; i < nodeInts.length; i++) { + result[i] = this.intToUuid.get(Number(nodeInts[i])) ?? '' + } + return result + } + /** * Flush mappings to storage */ @@ -323,6 +344,17 @@ export class EntityIdMapper implements EntityIdMapperProvider { await this.flush() } + // No `rebuild()` on the JS mapper by design (CTX-BR-RESTORE-REBUILD): after a + // `brain.restore()`, `MetadataIndex.rebuild()` re-derives this mapper from the + // restored entities via append-only `getOrAssign` (the ints it picks are + // internally consistent with the bitmaps it builds), so the JS path needs no + // explicit post-restore reload — and forcing one (blanking + reloading) would + // drop a still-referenced mapping when the snapshot omits the mapper file. The + // `rebuild()` reload is the NATIVE mapper's concern: cor's binary KV holds the + // authoritative int↔uuid the native adjacency is keyed on, so it must reload + // from the restored KV before the native graph rebuild. `restore()` therefore + // calls `rebuild()` only when the provider implements it (see brainy.ts). + /** * Get statistics about the mapper */ diff --git a/src/utils/errorClassification.ts b/src/utils/errorClassification.ts new file mode 100644 index 00000000..a3e7dad8 --- /dev/null +++ b/src/utils/errorClassification.ts @@ -0,0 +1,37 @@ +/** + * @module utils/errorClassification + * @description Shared classification of caught errors into "genuine absence" vs + * "real fault" — the antidote to blind `catch { return null }` handlers that + * cannot tell ENOENT (the object is legitimately not on disk) from EIO / EACCES + * / EMFILE / … (a transient or permission fault on data that IS on disk). + * Masking a fault as absence yields wrong results (a present record read as + * "not found") or a needless rebuild. Mandate: loud errors, never quiet losses. + */ + +/** + * The error `code`s that denote GENUINE absence of a file/object. Only `ENOENT` + * ("no such file or directory") qualifies — on every platform Node maps a + * missing file/directory to ENOENT, and no other errno means "simply not + * there". Every other errno (EIO, EACCES, EPERM, EMFILE, ENFILE, EBUSY, + * ENOTDIR, EISDIR, ELOOP) is a real fault and must propagate, as must any error + * without an errno `code` (parse/decompress failures, generic Errors). + * `ENOTDIR`/`EISDIR` are deliberately faults: a path component of the wrong + * type is corruption, not benign absence. Named constant so a future + * genuine-absence code can be added in one reviewed place. + */ +const ABSENCE_CODES: ReadonlySet = new Set(['ENOENT']) + +/** + * @description True IFF `e` represents genuine absence (an ENOENT-class errno), + * for which returning `null`/`[]`/`undefined` is the correct answer. Returns + * `false` for every real fault, so the canonical call site is: + * `catch (e) { if (isAbsentError(e)) return null; throw e }`. + * + * @param e - The caught value (typed `unknown`; non-objects are never absence). + * @returns Whether the error means "the thing is simply not there". + */ +export function isAbsentError(e: unknown): boolean { + if (e === null || typeof e !== 'object') return false + const code = (e as { code?: unknown }).code + return typeof code === 'string' && ABSENCE_CODES.has(code) +} diff --git a/src/utils/indexReadiness.ts b/src/utils/indexReadiness.ts new file mode 100644 index 00000000..16266bec --- /dev/null +++ b/src/utils/indexReadiness.ts @@ -0,0 +1,38 @@ +/** + * @module indexReadiness + * @description The single honest-readiness classifier shared by the vector, + * graph and metadata index sites. It exists to kill "Pattern A" — the dishonest + * readiness proxy where `size() > 0` / `isInitialized` is treated as "this index + * actually serves queries." A cold native index that loaded its COUNT but not its + * SERVING structure passes those proxies and silently returns `[]`. + * + * This classifier reads ONLY the provider's OPTIONAL, honest `isReady()` signal + * (see {@link import('../plugin.js').VectorIndexProvider.isReady}, + * {@link import('../plugin.js').GraphIndexProvider.isReady}, + * {@link import('../plugin.js').MetadataIndexProvider.isReady}). It NEVER inspects + * `size()` or `isInitialized`. When `isReady()` is absent, callers must fall back + * to a KNOWN-ITEM PROBE (a real search/lookup that must return a known-present + * datum) before trusting an empty result — never a `size()` proxy. + */ + +/** A provider that MAY expose the honest cold-load readiness signal. */ +export interface MaybeReadyProvider { + isReady?: () => boolean +} + +/** Three-valued honest-readiness verdict. */ +export type IndexReadiness = 'ready' | 'not-ready' | 'unknown' + +/** + * @description Classify an index provider's honest readiness. + * @param provider - Any index provider (vector / graph / metadata) or `null`. + * @returns + * - `'ready'` when `isReady() === true` (serving structure loaded — trust it); + * - `'not-ready'` when `isReady() === false` (count/manifest loaded, NOT serving — rebuild); + * - `'unknown'` when the provider exposes no `isReady()` (caller must probe / keep the JS heuristic). + */ +export function assessIndexReadiness(provider: unknown): IndexReadiness { + const p = provider as MaybeReadyProvider | null | undefined + if (p == null || typeof p.isReady !== 'function') return 'unknown' + return p.isReady() ? 'ready' : 'not-ready' +} diff --git a/src/utils/metadataFilter.ts b/src/utils/metadataFilter.ts index f38df4f5..b763dcd5 100644 --- a/src/utils/metadataFilter.ts +++ b/src/utils/metadataFilter.ts @@ -5,27 +5,33 @@ */ import { SearchResult, HNSWNoun, HNSWNounWithMetadata } from '../coreTypes.js' +import { BrainyError } from '../errors/brainyError.js' /** * Brainy Field Operators (BFO) - Our own field query system * Designed for performance, clarity, and patent independence */ export interface BrainyFieldOperators { - // Equality operators + // Equality operators (canonical + long-form aliases) + eq?: any equals?: any + ne?: any notEquals?: any - is?: any - isNot?: any - - // Comparison operators + + // Comparison operators (canonical + long-form aliases) greaterThan?: any - greaterEqual?: any + gt?: any + greaterThanOrEqual?: any + gte?: any lessThan?: any - lessEqual?: any + lt?: any + lessThanOrEqual?: any + lte?: any between?: [any, any] // Array/Set operators oneOf?: any[] + in?: any[] // documented alias for oneOf noneOf?: any[] contains?: any excludes?: any @@ -45,14 +51,6 @@ export interface BrainyFieldOperators { allOf?: MetadataFilter[] anyOf?: MetadataFilter[] not?: MetadataFilter - - // Short aliases for common operations - eq?: any - ne?: any - gt?: any - gte?: any - lt?: any - lte?: any } /** @@ -74,6 +72,65 @@ export interface MetadataFilterOptions { } } +/** + * Value-level operators (the keys inside `{ field: { : operand } }`) — the + * complete documented set, kept in lockstep with the `matchesQuery` switch below + * and the metadata-index path. `in` is the documented alias for `oneOf`. + */ +const VALUE_OPERATORS = new Set([ + 'equals', 'eq', 'notEquals', 'ne', + 'greaterThan', 'gt', 'greaterThanOrEqual', 'gte', + 'lessThan', 'lt', 'lessThanOrEqual', 'lte', + 'between', 'oneOf', 'in', 'noneOf', + 'contains', 'excludes', 'hasAll', 'length', + 'exists', 'missing', 'matches', 'startsWith', 'endsWith' +]) + +/** Filter-level logical operators (siblings of field names). */ +const LOGICAL_OPERATORS = new Set(['allOf', 'anyOf', 'not']) + +/** + * Validate a `where` filter's operators up front, throwing a typed + * `BrainyError('INVALID_QUERY')` on the first unrecognized operator — so a typo + * like `{ subtype: { notIn: [...] } }` fails LOUD instead of silently matching + * nothing (the invisible-degrade class). Called by `find()` before either the + * index path or the in-memory matcher runs, so the throw fires even when the + * result set is empty (the index path would otherwise return `[]` without ever + * invoking the matcher). Nested fields use dot notation (`{ 'a.b': v }`), so a + * field's object value carries operators, never sub-field names. + * + * @param filter - the user-supplied `where` clause (validated raw, before any + * internal field injection like visibility or `type`→`noun`). + * @throws BrainyError('INVALID_QUERY') naming the bad operator + the valid set. + */ +export function validateWhereFilter(filter: unknown): void { + if (!filter || typeof filter !== 'object' || Array.isArray(filter)) return + for (const [key, value] of Object.entries(filter as Record)) { + if (LOGICAL_OPERATORS.has(key)) { + if (key === 'not') { + validateWhereFilter(value) + } else if (Array.isArray(value)) { + for (const sub of value) validateWhereFilter(sub) + } + continue + } + // A field key. Its value is a scalar/array (equality) — nothing to validate — + // or an object of value-operators, every key of which must be recognized. + if (value && typeof value === 'object' && !Array.isArray(value)) { + for (const op of Object.keys(value as Record)) { + if (!VALUE_OPERATORS.has(op)) { + throw new BrainyError( + `Unknown filter operator "${op}" on field "${key}". Valid operators: ` + + `${[...VALUE_OPERATORS].sort().join(', ')}. For nested fields use dot ` + + `notation, e.g. { '${key}.subfield': value }.`, + 'INVALID_QUERY' + ) + } + } + } + } +} + /** * Check if a value matches a query with operators */ @@ -88,12 +145,10 @@ function matchesQuery(value: any, query: any): boolean { switch (op) { // Equality operators case 'equals': - case 'is': case 'eq': if (value !== operand) return false break case 'notEquals': - case 'isNot': case 'ne': // Special handling: if value is undefined and operand is not undefined, // they are not equal (so the condition passes) @@ -108,16 +163,16 @@ function matchesQuery(value: any, query: any): boolean { case 'gt': if (typeof value !== 'number' || typeof operand !== 'number' || !(value > operand)) return false break - case 'greaterEqual': case 'gte': + case 'greaterThanOrEqual': if (typeof value !== 'number' || typeof operand !== 'number' || !(value >= operand)) return false break case 'lessThan': case 'lt': if (typeof value !== 'number' || typeof operand !== 'number' || !(value < operand)) return false break - case 'lessEqual': case 'lte': + case 'lessThanOrEqual': if (typeof value !== 'number' || typeof operand !== 'number' || !(value <= operand)) return false break case 'between': @@ -127,6 +182,7 @@ function matchesQuery(value: any, query: any): boolean { // Array/Set operators case 'oneOf': + case 'in': // documented alias for oneOf if (!Array.isArray(operand) || !operand.includes(value)) return false break case 'noneOf': @@ -169,22 +225,26 @@ function matchesQuery(value: any, query: any): boolean { break default: - // Unknown operator, treat as field name - if (!matchesFieldQuery(value, op, operand)) return false + // Unknown operator. The old behavior treated any unknown key as a + // nested-object field name (an UNDOCUMENTED fallback — dot notation + // `{ 'a.b': v }` is the supported nested form), which silently swallowed + // operator typos: `{ x: { notIn: [...] } }` matched nothing instead of + // erroring. Fail loud. find() validates the whole where clause up front + // (validateWhereFilter), so a typo throws even on an empty result set; + // this is the belt-and-suspenders for any matcher caller that bypasses + // that path. + throw new BrainyError( + `Unknown filter operator "${op}". Valid operators: ` + + `${[...VALUE_OPERATORS].sort().join(', ')}. For nested fields use dot ` + + `notation, e.g. { 'address.city': 'NYC' }.`, + 'INVALID_QUERY' + ) } } return true } -/** - * Check if a field matches a query - */ -function matchesFieldQuery(obj: any, field: string, query: any): boolean { - const value = getNestedValue(obj, field) - return matchesQuery(value, query) -} - /** * Get nested value from object using dot notation */ diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 94fa4906..c772bce4 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -77,7 +77,7 @@ export interface MetadataIndexConfig { } export interface MetadataIndexOptions { - entityIdMapper?: EntityIdMapper // Optional pre-configured EntityIdMapper (e.g., native from cortex) + entityIdMapper?: EntityIdMapper // Optional pre-configured EntityIdMapper (e.g., native from cor) } /** @@ -107,7 +107,7 @@ interface FieldStats { /** * Implements {@link MetadataIndexProvider}: the metadata-index surface Brainy * calls on whatever the `'metadataIndex'` provider resolves to (its own - * manager, or Cortex's native Rust engine). + * manager, or Cor's native Rust engine). */ export class MetadataIndexManager implements MetadataIndexProvider { private storage: StorageAdapter @@ -221,7 +221,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { // Get global unified cache for coordinated memory management this.unifiedCache = getGlobalCache() - // Use injected EntityIdMapper (e.g., native from cortex) or create JS fallback + // Use injected EntityIdMapper (e.g., native from cor) or create JS fallback this.idMapper = options.entityIdMapper ?? new EntityIdMapper({ storage, storageKey: 'brainy:entityIdMapper' @@ -1166,8 +1166,17 @@ export class MetadataIndexManager implements MetadataIndexProvider { private extractIndexableFields(data: any): Array<{ field: string, value: any }> { const fields: Array<{ field: string, value: any }> = [] - // Fields that should NEVER be indexed (vectors, embeddings, large arrays, HNSW internals) - const NEVER_INDEX = new Set(['vector', 'embedding', 'embeddings', 'connections', 'level', 'id']) + // Fields that should NEVER be indexed: bulk structural payloads that would + // blow up the index (the 384-dim vector, embeddings, the adjacency list). + // These are also caught by the array-size guard below, but naming them is + // belt-and-suspenders. NOTE: `level` was previously here (an HNSW node's + // layer) but it never actually reaches this path — every caller passes a + // metadata bag or Entity record, neither of which carries the node's + // `level` — so its only effect was to silently drop a legitimate USER + // metadata field named `level` (log level, skill level, access level…), + // making `where: { level: … }` return nothing. Removed. (`id` stays: it is + // the reserved entity-identity field, resolved specially by find().) + const NEVER_INDEX = new Set(['vector', 'embedding', 'embeddings', 'connections', 'id']) const extract = (obj: any, prefix = ''): void => { for (const [key, value] of Object.entries(obj)) { @@ -1278,7 +1287,10 @@ export class MetadataIndexManager implements MetadataIndexProvider { return data.map(d => this.extractTextContent(d)).filter(Boolean).join(' ') } if (typeof data === 'object') { - const skipKeys = new Set(['vector', 'embedding', 'embeddings', 'connections', 'level', 'id']) + // Mirror of NEVER_INDEX for the text-extraction path: bulk structural + // payloads only. `level` removed for the same reason (it silently dropped + // a real user field from hybrid text search too). + const skipKeys = new Set(['vector', 'embedding', 'embeddings', 'connections', 'id']) const texts: string[] = [] for (const [key, value] of Object.entries(data)) { // Skip internal fields and numeric keys (array indices) @@ -1760,7 +1772,6 @@ export class MetadataIndexManager implements MetadataIndexProvider { } break case 'equals': - case 'is': case 'eq': criteria.push({ field: key, values: [operand] }) break @@ -1770,8 +1781,6 @@ export class MetadataIndexManager implements MetadataIndexProvider { break case 'greaterThan': case 'lessThan': - case 'greaterEqual': - case 'lessEqual': case 'between': // Range queries will be handled separately // Sorted index will be created/loaded when needed in getIdsForRange @@ -1791,9 +1800,12 @@ export class MetadataIndexManager implements MetadataIndexProvider { } /** - * Get IDs matching Brainy Field Operator metadata filter using indexes where possible + * Get IDs matching a Brainy Field Operator metadata filter using indexes where possible. + * The optional `_opts` page bound is part of the provider contract for the native + * index (early-stop at `offset+limit`); the JS index returns ALL matches and lets + * the caller window them, so `_opts` is intentionally ignored here. */ - async getIdsForFilter(filter: any): Promise { + async getIdsForFilter(filter: any, _opts?: { limit?: number; offset?: number }): Promise { if (!filter || Object.keys(filter).length === 0) { return [] } @@ -1855,9 +1867,26 @@ export class MetadataIndexManager implements MetadataIndexProvider { // not once per AND-clause inside it. const unindexedFields: string[] = [] - for (const [field, condition] of Object.entries(filter)) { + for (const [rawField, condition] of Object.entries(filter)) { // Skip logical operators - if (field === 'allOf' || field === 'anyOf' || field === 'not') continue + if (rawField === 'allOf' || rawField === 'anyOf' || rawField === 'not') continue + + // Metadata is FLATTENED at index time (metadata.entry.title indexes as + // entry.title), so a `metadata.`-prefixed where key is almost always + // the caller spelling the STORAGE shape rather than the index shape. + // Accept both spellings: when the key as spelled is unindexed but its + // stripped spelling is, query the stripped one. A literal nested + // custom key named `metadata` still wins when indexed as spelled + // (checked first), so that rare shape keeps working. + let field = rawField + if ( + rawField.startsWith('metadata.') && + this.columnStore && + !this.columnStore.hasField(rawField) && + this.columnStore.hasField(rawField.slice('metadata.'.length)) + ) { + field = rawField.slice('metadata.'.length) + } let fieldResults: string[] = [] @@ -1881,31 +1910,27 @@ export class MetadataIndexManager implements MetadataIndexProvider { fieldResults = [] switch (op) { // ===== EQUALITY OPERATORS ===== - // Canonical: 'eq' | Alias: 'equals' | Deprecated: 'is' - case 'is': // DEPRECATED: Use 'eq' instead + // Canonical: 'eq' | Alias: 'equals' case 'equals': // Alias for 'eq' case 'eq': fieldResults = await this.getIds(field, operand) break // ===== NEGATION OPERATORS ===== - // Canonical: 'ne' | Alias: 'notEquals' | Deprecated: 'isNot' - case 'isNot': // DEPRECATED: Use 'ne' instead + // Canonical: 'ne' | Alias: 'notEquals' case 'notEquals': // Alias for 'ne' case 'ne': { - // For notEquals, we need all IDs EXCEPT those matching the value - // This is especially important for soft delete: deleted !== true - // should include items without a deleted field - - // Use EntityIdMapper universe (in-memory) instead of getAllIds() storage scan - const allKnownIds = this.idMapper.intsIterableToUuids(this.idMapper.getAllIntIds()) - - // Then get IDs that match the value we want to exclude - const excludeIds = await this.getIds(field, operand) - const excludeSet = new Set(excludeIds) - - // Return all IDs except those to exclude - fieldResults = allKnownIds.filter(id => !excludeSet.has(id)) + // All ids EXCEPT those matching the value. Important for soft delete: + // `deleted !== true` must include items WITHOUT a deleted field. The + // excluded set is typically small (the matching value); compute the + // complement as a bitmap difference over the int-id universe rather + // than materializing the whole corpus as UUID strings to filter it. + const excludeInts: number[] = [] + for (const uuid of await this.getIds(field, operand)) { + const intId = this.idMapper.getInt(uuid) + if (intId !== undefined) excludeInts.push(intId) + } + fieldResults = this.complementIds(excludeInts) break } @@ -1931,8 +1956,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { break // ===== GREATER THAN OR EQUAL OPERATORS ===== - // Canonical: 'gte' | Alias: 'greaterThanOrEqual' | Deprecated: 'greaterEqual' - case 'greaterEqual': // DEPRECATED: Use 'gte' instead + // Canonical: 'gte' | Alias: 'greaterThanOrEqual' case 'greaterThanOrEqual': // Alias for 'gte' case 'gte': fieldResults = await this.getIdsForRange(field, operand, undefined, true, true) @@ -1946,8 +1970,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { break // ===== LESS THAN OR EQUAL OPERATORS ===== - // Canonical: 'lte' | Alias: 'lessThanOrEqual' | Deprecated: 'lessEqual' - case 'lessEqual': // DEPRECATED: Use 'lte' instead + // Canonical: 'lte' | Alias: 'lessThanOrEqual' case 'lessThanOrEqual': // Alias for 'lte' case 'lte': fieldResults = await this.getIdsForRange(field, undefined, operand, true, true) @@ -1979,11 +2002,8 @@ export class MetadataIndexManager implements MetadataIndexProvider { // exists: true — entities that HAVE this field fieldResults = this.idMapper.intsIterableToUuids(existsBitmap) } else { - // exists: false — entities that DON'T have this field - const allKnownIds = this.idMapper.intsIterableToUuids(this.idMapper.getAllIntIds()) - const existsUuids = this.idMapper.intsIterableToUuids(existsBitmap) - const existsSet = new Set(existsUuids) - fieldResults = allKnownIds.filter(id => !existsSet.has(id)) + // exists: false — entities that DON'T have this field (universe \ has-field) + fieldResults = this.complementIds(existsBitmap) } break } @@ -1996,11 +2016,8 @@ export class MetadataIndexManager implements MetadataIndexProvider { : await this.getExistsBitmapLegacy(field) if (operand) { - // missing: true — entities that DON'T have this field (same as exists: false) - const allKnownIds = this.idMapper.intsIterableToUuids(this.idMapper.getAllIntIds()) - const existsUuids = this.idMapper.intsIterableToUuids(missingBitmap) - const existsSet = new Set(existsUuids) - fieldResults = allKnownIds.filter(id => !existsSet.has(id)) + // missing: true — entities that DON'T have this field (universe \ has-field) + fieldResults = this.complementIds(missingBitmap) } else { // missing: false — entities that HAVE this field (same as exists: true) fieldResults = this.idMapper.intsIterableToUuids(missingBitmap) @@ -2073,6 +2090,21 @@ export class MetadataIndexManager implements MetadataIndexProvider { * @returns Roaring bitmap of entity int IDs (or iterable for compatibility) * @private */ + /** + * All ids EXCEPT the excluded int-id set, computed as a roaring-bitmap difference + * over the int-id universe. Used by the negation/absence operators (`ne`, + * `exists:false`, `missing:true`) so they don't first materialize the ENTIRE + * corpus as an array of UUID strings (plus a Set, plus an O(N) filter pass) just + * to remove a small subset — only the final result is converted back to UUIDs. + */ + private complementIds(excludeInts: Iterable): string[] { + const universe = new RoaringBitmap32() + for (const intId of this.idMapper.getAllIntIds()) universe.add(intId) + const exclude = new RoaringBitmap32() + for (const intId of excludeInts) exclude.add(intId) + return this.idMapper.intsIterableToUuids(RoaringBitmap32.andNot(universe, exclude)) + } + private async getExistsBitmapLegacy(field: string): Promise> { const allIntIds = new Set() const sparseIndex = await this.loadSparseIndex(field) @@ -2121,13 +2153,19 @@ export class MetadataIndexManager implements MetadataIndexProvider { * @param filter - Metadata filter criteria (uses roaring bitmaps) * @param orderBy - Field name to sort by (e.g., 'createdAt', 'title') * @param order - Sort direction: 'asc' (default) or 'desc' + * @param topK - Optional page bound: produce only the top `K` sorted ids + * (`offset + limit`) instead of the full sorted match set. A broad filter + + * orderBy that returns one page no longer materializes hundreds of millions of + * sorted ids at billion scale — the column store's top-K heap produces only the + * page. Omit for the full sorted set. * @returns Promise - Entity IDs sorted by specified field * */ async getSortedIdsForFilter( filter: any, orderBy: string, - order: 'asc' | 'desc' = 'asc' + order: 'asc' | 'desc' = 'asc', + topK?: number ): Promise { // Column store path: O(K log S) sort via k-way merge across segments. // No per-entity storage reads, no precision loss from bucketing. @@ -2146,13 +2184,17 @@ export class MetadataIndexManager implements MetadataIndexProvider { const intId = this.idMapper.getInt(id) if (intId !== undefined) filterBitmap.add(intId) } + // Page-bounded: produce only the top `topK` (offset+limit), not every + // match, so a broad filter + orderBy returning one page stays O(matches + // log K) heap, not a full sort materialization. + const k = topK !== undefined ? Math.min(topK, filteredIds.length) : filteredIds.length sortedIntIds = await this.columnStore.filteredSortTopK( - filterBitmap, orderBy, order, filteredIds.length + filterBitmap, orderBy, order, k ) } else { // Unfiltered sort — column store handles the full entity set efficiently sortedIntIds = await this.columnStore.sortTopK( - orderBy, order, this.idMapper.size + orderBy, order, topK !== undefined ? Math.min(topK, this.idMapper.size) : this.idMapper.size ) } @@ -2183,7 +2225,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { if (b.value == null) return order === 'asc' ? -1 : 1 if (a.value === b.value) return 0 // Numbers compare numerically; everything else by code-point (UTF-8 byte) order. - // This makes the JS fallback sort match cortex's native column store exactly + // This makes the JS fallback sort match cor's native column store exactly // (numeric i64/f64 vs code-point strings) and stay deterministic across // environments, unlike the `<` operator's UTF-16 ordering for strings. let comparison: number @@ -2195,7 +2237,8 @@ export class MetadataIndexManager implements MetadataIndexProvider { return order === 'asc' ? comparison : -comparison }) - return idValuePairs.map(p => p.id) + const sorted = idValuePairs.map(p => p.id) + return topK !== undefined ? sorted.slice(0, topK) : sorted } /** @@ -2300,81 +2343,6 @@ export class MetadataIndexManager implements MetadataIndexProvider { return normalized } - /** - * DEPRECATED - Old implementation for backward compatibility - */ - private async getIdsForFilterOld(filter: any): Promise { - if (!filter || Object.keys(filter).length === 0) { - return [] - } - - // Handle logical operators - if (filter.allOf && Array.isArray(filter.allOf)) { - // For allOf, we need intersection of all sub-filters - const allIds: string[][] = [] - for (const subFilter of filter.allOf) { - const subIds = await this.getIdsForFilter(subFilter) - allIds.push(subIds) - } - - if (allIds.length === 0) return [] - if (allIds.length === 1) return allIds[0] - - // Intersection of all sets - return allIds.reduce((intersection, currentSet) => - intersection.filter(id => currentSet.includes(id)) - ) - } - - if (filter.anyOf && Array.isArray(filter.anyOf)) { - // For anyOf, we need union of all sub-filters - const unionIds = new Set() - for (const subFilter of filter.anyOf) { - const subIds = await this.getIdsForFilter(subFilter) - subIds.forEach(id => unionIds.add(id)) - } - return Array.from(unionIds) - } - - // Handle regular field filters - const criteria = this.convertFilterToCriteria(filter) - const idSets: string[][] = [] - const unindexedFields: string[] = [] - - for (const { field, values } of criteria) { - const unionIds = new Set() - try { - for (const value of values) { - const ids = await this.getIds(field, value) - ids.forEach(id => unionIds.add(id)) - } - } catch (err) { - if (err instanceof BrainyError && err.type === 'FIELD_NOT_INDEXED') { - unindexedFields.push(field) - // Treat as no matches and continue — same semantics as the main - // getIdsForFilter() path. - } else { - throw err - } - } - idSets.push(Array.from(unionIds)) - } - - if (unindexedFields.length > 0) { - prodLog.warn( - `[brainy] find() where-clause referenced unindexed field(s) ` + - `${unindexedFields.join(', ')}; their clauses contributed no rows.` - ) - } - if (idSets.length === 0) return [] - if (idSets.length === 1) return idSets[0] - - // Intersection of all field criteria (implicit $and) - return idSets.reduce((intersection, currentSet) => - intersection.filter(id => currentSet.includes(id)) - ) - } - /** * Flush dirty entries to storage (non-blocking version) * NOTE: Sparse indices are flushed immediately in add/remove operations diff --git a/src/utils/metadataNamespace.ts b/src/utils/metadataNamespace.ts deleted file mode 100644 index e14af1df..00000000 --- a/src/utils/metadataNamespace.ts +++ /dev/null @@ -1,255 +0,0 @@ -/** - * Clean Metadata Architecture for Brainy 2.2 - * No backward compatibility - doing it RIGHT from the start! - */ - -// Namespace constants -export const BRAINY_NS = '_brainy' as const -export const AUG_NS = '_augmentations' as const -export const AUDIT_NS = '_audit' as const - -// Field paths for O(1) indexing -export const DELETED_FIELD = `${BRAINY_NS}.deleted` as const -export const INDEXED_FIELD = `${BRAINY_NS}.indexed` as const -export const VERSION_FIELD = `${BRAINY_NS}.version` as const - -/** - * Internal Brainy metadata structure - * These fields are ALWAYS present and indexed for O(1) access - */ -export interface BrainyInternalMetadata { - deleted: boolean // ALWAYS boolean, enables O(1) soft delete - indexed: boolean // Whether in search index - version: number // Schema version - created: number // Unix timestamp - updated: number // Unix timestamp - - // Optional internal fields - domain?: string // Domain classification - priority?: number // Query priority hint - ttl?: number // Time to live -} - -/** - * Complete metadata structure with namespaces - */ -export interface NamespacedMetadata { - // User metadata - any fields they want - [key: string]: any - - // Internal metadata - our fields - [BRAINY_NS]: BrainyInternalMetadata - - // Augmentation metadata - isolated per augmentation - [AUG_NS]?: { - [augmentationName: string]: any - } - - // Audit trail - optional - [AUDIT_NS]?: Array<{ - timestamp: number - augmentation: string - field: string - oldValue: any - newValue: any - }> -} - -/** - * Create properly namespaced metadata - * This is called for EVERY noun/verb creation - */ -export function createNamespacedMetadata( - userMetadata?: T -): NamespacedMetadata { - const now = Date.now() - - // Start with user metadata or empty object - const result: any = userMetadata ? { ...userMetadata } : {} - - // ALWAYS add internal namespace with required fields - result[BRAINY_NS] = { - deleted: false, // CRITICAL: Always false for new items - indexed: true, // New items are indexed - version: 1, // Current schema version - created: now, - updated: now - } - - return result -} - -/** - * Update metadata while preserving namespaces - */ -export function updateNamespacedMetadata( - existing: NamespacedMetadata, - updates: Partial -): NamespacedMetadata { - const now = Date.now() - - // Merge user fields - const result: any = { - ...existing, - ...updates - } - - // Preserve internal namespace but update timestamp - result[BRAINY_NS] = { - ...existing[BRAINY_NS], - updated: now - } - - // Preserve augmentation namespace - if (existing[AUG_NS]) { - result[AUG_NS] = existing[AUG_NS] - } - - // Preserve audit trail - if (existing[AUDIT_NS]) { - result[AUDIT_NS] = existing[AUDIT_NS] - } - - return result -} - -/** - * Soft delete a noun (O(1) operation) - */ -export function markDeleted( - metadata: NamespacedMetadata -): NamespacedMetadata { - return { - ...metadata, - [BRAINY_NS]: { - ...metadata[BRAINY_NS], - deleted: true, - updated: Date.now() - } - } -} - -/** - * Restore a soft-deleted noun (O(1) operation) - */ -export function markRestored( - metadata: NamespacedMetadata -): NamespacedMetadata { - return { - ...metadata, - [BRAINY_NS]: { - ...metadata[BRAINY_NS], - deleted: false, - updated: Date.now() - } - } -} - -/** - * Check if a noun is deleted (O(1) check) - */ -export function isDeleted( - metadata: NamespacedMetadata -): boolean { - return metadata[BRAINY_NS]?.deleted === true -} - -/** - * Get user metadata without internal fields - * Used by augmentations to get clean user data - */ -export function getUserMetadata( - metadata: NamespacedMetadata -): T { - const { [BRAINY_NS]: _, [AUG_NS]: __, [AUDIT_NS]: ___, ...userMeta } = metadata - return userMeta as T -} - -/** - * Set augmentation data in isolated namespace - */ -export function setAugmentationData( - metadata: NamespacedMetadata, - augmentationName: string, - data: any -): NamespacedMetadata { - const result = { ...metadata } - - if (!result[AUG_NS]) { - result[AUG_NS] = {} - } - - result[AUG_NS][augmentationName] = data - - return result -} - -/** - * Add audit entry for tracking - */ -export function addAuditEntry( - metadata: NamespacedMetadata, - entry: { - augmentation: string - field: string - oldValue: any - newValue: any - } -): NamespacedMetadata { - const result = { ...metadata } - - if (!result[AUDIT_NS]) { - result[AUDIT_NS] = [] - } - - result[AUDIT_NS].push({ - ...entry, - timestamp: Date.now() - }) - - return result -} - -/** - * INDEXING EXPLANATION: - * - * The MetadataIndex flattens nested objects into dot-notation keys: - * - * Input metadata: - * { - * name: "Django", - * _brainy: { - * deleted: false, - * indexed: true - * } - * } - * - * Creates index entries: - * - "name" -> "django" -> Set([id1, id2...]) - * - "_brainy.deleted" -> "false" -> Set([id1, id2...]) // O(1) lookup! - * - "_brainy.indexed" -> "true" -> Set([id1, id2...]) - * - * Query: { "_brainy.deleted": false } - * Lookup: index["_brainy.deleted"]["false"] -> Set of IDs in O(1) - * - * This is why namespacing doesn't hurt performance - it's all flattened! - */ - -/** - * Fields that should ALWAYS be indexed for O(1) access - */ -export const ALWAYS_INDEXED_FIELDS = [ - DELETED_FIELD, // For soft delete filtering - INDEXED_FIELD, // For index management - VERSION_FIELD // For schema versioning -] - -/** - * Fields that should use sorted index for O(log n) range queries - */ -export const SORTED_INDEX_FIELDS = [ - `${BRAINY_NS}.created`, - `${BRAINY_NS}.updated`, - `${BRAINY_NS}.priority`, - `${BRAINY_NS}.ttl` -] \ No newline at end of file diff --git a/src/utils/metadataWriteBuffer.ts b/src/utils/metadataWriteBuffer.ts index a0c0df50..4a2c7e2e 100644 --- a/src/utils/metadataWriteBuffer.ts +++ b/src/utils/metadataWriteBuffer.ts @@ -104,6 +104,11 @@ export class MetadataWriteBuffer { }) } }, this.flushIntervalMs) + // Best-effort background flush — must not keep the process alive + // (close() drains the buffer for durability). + if (this.flushTimer && typeof this.flushTimer.unref === 'function') { + this.flushTimer.unref() + } // Prevent timer from keeping the process alive if (this.flushTimer && typeof this.flushTimer === 'object' && 'unref' in this.flushTimer) { diff --git a/src/utils/osLimits.ts b/src/utils/osLimits.ts new file mode 100644 index 00000000..57e1ccf1 --- /dev/null +++ b/src/utils/osLimits.ts @@ -0,0 +1,141 @@ +/** + * @module utils/osLimits + * @description Detect-and-warn for OS resource limits that bite at POOL scale. + * + * A single brain rarely notices them, but a pool of brains — especially with a + * native accelerator memory-mapping many index files per brain — consumes file + * descriptors and memory mappings multiplicatively. On stock Linux defaults + * (RLIMIT_NOFILE soft 1024, vm.max_map_count 65530) the failure arrives as + * EMFILE or a failed mmap deep inside an index open, long after the real cause + * (the limit) stopped being visible. This module reads the limits at open and + * WARNS ONCE per process with the exact raise commands, so the operator learns + * the fix before the incident instead of from it. + * + * Read-only and Linux-only by construction: both sources are `/proc` files. + * On platforms where they are absent the check reports nulls and stays silent — + * no limit read means no claim made, never a guessed warning. + */ + +import * as fs from 'node:fs' +import { prodLog } from './logger.js' + +/** Soft-NOFILE floor below which pool-scale use is at EMFILE risk. */ +export const NOFILE_POOL_FLOOR = 65536 + +/** vm.max_map_count floor below which mmap-heavy native indexes are at risk. */ +export const MAX_MAP_COUNT_POOL_FLOOR = 262144 + +export interface OsLimitsReport { + /** RLIMIT_NOFILE soft limit (null when unreadable; Infinity for 'unlimited'). */ + nofileSoft: number | null + /** RLIMIT_NOFILE hard limit (null when unreadable; Infinity for 'unlimited'). */ + nofileHard: number | null + /** vm.max_map_count (null when unreadable). */ + maxMapCount: number | null + /** Human-actionable warnings for limits below the pool floors. Empty = fine. */ + warnings: string[] +} + +/** + * Parse the `Max open files` row of a `/proc//limits` document into + * soft/hard values. Returns nulls when the row is absent or malformed. + */ +export function parseProcLimits(content: string): { soft: number | null; hard: number | null } { + const line = content.split('\n').find((l) => l.startsWith('Max open files')) + if (!line) return { soft: null, hard: null } + const m = line.match(/^Max open files\s+(\S+)\s+(\S+)/) + if (!m) return { soft: null, hard: null } + const parse = (v: string): number | null => { + if (v === 'unlimited') return Infinity + const n = Number.parseInt(v, 10) + return Number.isNaN(n) ? null : n + } + return { soft: parse(m[1]), hard: parse(m[2]) } +} + +/** + * Assess readable limits against the pool floors. Pure — feed it any values. + * A null (unreadable) limit produces NO warning: no measurement, no claim. + */ +export function assessOsLimits(limits: { + nofileSoft: number | null + nofileHard: number | null + maxMapCount: number | null +}): string[] { + const warnings: string[] = [] + + if (limits.nofileSoft !== null && limits.nofileSoft < NOFILE_POOL_FLOOR) { + const hardNote = + limits.nofileHard !== null && limits.nofileHard >= NOFILE_POOL_FLOOR + ? ` (the hard limit ${limits.nofileHard === Infinity ? 'unlimited' : limits.nofileHard} already allows it — raise the soft limit only)` + : '' + warnings.push( + `RLIMIT_NOFILE soft limit is ${limits.nofileSoft} — below the ${NOFILE_POOL_FLOOR} recommended ` + + `for pool-scale use (a pool of brains with a native accelerator opens many index files per brain; ` + + `the failure mode is EMFILE deep inside an index open). Raise with \`ulimit -n ${NOFILE_POOL_FLOOR}\` ` + + `or LimitNOFILE=${NOFILE_POOL_FLOOR} in the service unit${hardNote}.` + ) + } + + if (limits.maxMapCount !== null && limits.maxMapCount < MAX_MAP_COUNT_POOL_FLOOR) { + warnings.push( + `vm.max_map_count is ${limits.maxMapCount} — below the ${MAX_MAP_COUNT_POOL_FLOOR} recommended ` + + `for mmap-heavy native indexes at pool scale (each mapped index segment consumes map entries; ` + + `the failure mode is a failed mmap mid-heal). Raise with ` + + `\`sysctl -w vm.max_map_count=${MAX_MAP_COUNT_POOL_FLOOR}\` (persist in /etc/sysctl.d/).` + ) + } + + return warnings +} + +/** + * Read the limits from /proc and assess them. `readFile` is injectable for + * tests; absent/unreadable sources yield nulls (and therefore no warnings). + */ +export async function checkOsLimits( + readFile: (path: string) => Promise = async (p) => fs.promises.readFile(p, 'utf-8') +): Promise { + let nofileSoft: number | null = null + let nofileHard: number | null = null + let maxMapCount: number | null = null + + try { + const parsed = parseProcLimits(await readFile('/proc/self/limits')) + nofileSoft = parsed.soft + nofileHard = parsed.hard + } catch { + // Not Linux (or /proc unavailable) — no measurement, no claim. + } + + try { + const raw = (await readFile('/proc/sys/vm/max_map_count')).trim() + const n = Number.parseInt(raw, 10) + maxMapCount = Number.isNaN(n) ? null : n + } catch { + // Not Linux — same rule. + } + + const warnings = assessOsLimits({ nofileSoft, nofileHard, maxMapCount }) + return { nofileSoft, nofileHard, maxMapCount, warnings } +} + +/** Once-per-process latch so a brain pool warns once, not once per brain. */ +let osLimitsWarned = false + +/** + * Run the check and warn (once per process) about limits below the pool + * floors. Called from brain open; safe everywhere (silent off-Linux). + */ +export async function warnOnLowOsLimits(): Promise { + if (osLimitsWarned) return + osLimitsWarned = true + try { + const report = await checkOsLimits() + for (const warning of report.warnings) { + prodLog.warn(`[Brainy] OS limit check: ${warning}`) + } + } catch { + // The check must never affect open — measurement-only. + } +} diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index 4b7e3fd6..ca439524 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -10,15 +10,13 @@ import { NounType, VerbType } from '../types/graphTypes.js' import { prodLog } from './logger.js' import { findCallerLocation } from './callerLocation.js' -// Dynamic import for Node.js os and fs modules -let os: any = null -let fs: any = null -try { - os = await import('node:os') - fs = await import('node:fs') -} catch (e) { - // OS/FS modules not available -} +// 8.0 is Node/Bun/Deno-only (no browser path), so `node:os` / `node:fs` are +// always present — static ESM imports instead of a top-level `await import()`. +// The TLA form poisoned the module graph with a top-level await, which `bun +// build --compile` refuses; the static form also drops the browser/edge +// fallback branches that no supported runtime can reach. +import * as os from 'node:os' +import * as fs from 'node:fs' const getSystemMemory = (): number => { if (os) { @@ -211,8 +209,10 @@ export interface ValidationConfigOptions { } /** - * Auto-configured limits based on system resources - * These adapt to available memory and observed performance + * Auto-configured limits based on system resources. + * Derived from memory (explicit overrides > reserved memory > container limit > + * free memory). Query timing is recorded for diagnostics only — it never + * changes the cap (see `recordQuery`). */ export class ValidationConfig { private static instance: ValidationConfig | null = null @@ -325,24 +325,23 @@ export class ValidationConfig { } /** - * Learn from actual usage to adjust limits + * Record query timing for diagnostics. Telemetry ONLY — never mutates the cap. + * + * `maxLimit` is a MEMORY-protection bound; query duration says nothing about + * memory-per-result, so duration must never drive it. An earlier version + * "learned" here: while the lifetime-average query time exceeded 1s it shrank + * `maxLimit` by 20% per recorded query down to a floor of 1000 — below the + * documented `MIN_AUTO_QUERY_LIMIT` (10 000) and with no recovery once slow + * samples poisoned the cumulative average. On a production host a burst of + * slow aggregate queries silently strangled every consumer's `find()` to + * 1000 while the error message blamed "available free memory" — exactly the + * silent throttling this module's own contract forbids. The cap now comes + * from its construction-time basis (or explicit overrides) alone. */ recordQuery(duration: number, resultCount: number) { + void resultCount this.queryCount++ this.avgQueryTime = (this.avgQueryTime * (this.queryCount - 1) + duration) / this.queryCount - - // Only auto-adjust if not using explicit overrides - if (this.limitBasis !== 'override') { - // If queries are consistently fast with large results, increase limits - if (this.avgQueryTime < 100 && resultCount > this.maxLimit * 0.8) { - this.maxLimit = Math.min(this.maxLimit * 1.5, 100000) - } - - // If queries are slow, reduce limits - if (this.avgQueryTime > 1000) { - this.maxLimit = Math.max(this.maxLimit * 0.8, 1000) - } - } } } diff --git a/src/utils/recallPreset.ts b/src/utils/recallPreset.ts index 3e9af152..2ae2f78d 100644 --- a/src/utils/recallPreset.ts +++ b/src/utils/recallPreset.ts @@ -7,7 +7,7 @@ * provider (DiskANN-style) has taken over the `'vector'` provider key. * * **Public contract** — locked in handoff thread BRAINY-8.0-RENAME-COORDINATION - * § A.2 (cortex-confirmed 2026-06-09): + * § A.2 (cor-confirmed 2026-06-09): * - `'fast'` — minimum-latency search, accepts lower recall * - `'balanced'` — Brainy 7.x defaults, the right pick for almost everyone * - `'accurate'` — maximum recall, accepts higher latency @@ -16,7 +16,7 @@ * * **Knob mapping** — the JS HNSW preset values below match what Brainy 7.x * shipped as defaults for `'balanced'`. Native acceleration providers (e.g. - * cortex's DiskANN wrapper) read the same `recall` value off the index + * cor's DiskANN wrapper) read the same `recall` value off the index * config and translate it to their own internal knobs. */ diff --git a/src/utils/resultRanking.ts b/src/utils/resultRanking.ts index 06afb3cd..8079bf94 100644 --- a/src/utils/resultRanking.ts +++ b/src/utils/resultRanking.ts @@ -8,7 +8,7 @@ * This module exposes a swappable `sort:topK` seam: * - {@link rankIndicesByScore} returns the indices of `scores` ordered exactly as a * descending stable sort would order them, then truncated to `k`. - * - {@link setSortTopKImplementation} lets a native plugin (e.g. cortex's Rust + * - {@link setSortTopKImplementation} lets a native plugin (e.g. cor's Rust * partial-sort / heap-select) replace the JS implementation. The native provider * only has to return the same *index ordering* the JS path produces. * @@ -147,7 +147,7 @@ export function reorderByIndices(items: T[], order: number[]): T[] { } /** - * Replace the `sort:topK` implementation at runtime (e.g. cortex's native partial sort). + * Replace the `sort:topK` implementation at runtime (e.g. cor's native partial sort). * Called by `brainy.ts` when a `sort:topK` provider is registered. Pass * {@link sortTopKIndicesJs} to restore the JS default. * diff --git a/src/utils/roaring/index.ts b/src/utils/roaring/index.ts index f1bf1020..f4343f14 100644 --- a/src/utils/roaring/index.ts +++ b/src/utils/roaring/index.ts @@ -31,7 +31,7 @@ let _impl: typeof WasmRoaringBitmap32 = WasmRoaringBitmap32 /** * Replace the RoaringBitmap32 implementation at runtime. - * Called by brainy.ts when a cortex 'roaring' provider is registered. + * Called by brainy.ts when a cor 'roaring' provider is registered. */ export function setRoaringImplementation(impl: typeof WasmRoaringBitmap32) { _impl = impl diff --git a/src/utils/structuredLogger.ts b/src/utils/structuredLogger.ts deleted file mode 100644 index a5baa039..00000000 --- a/src/utils/structuredLogger.ts +++ /dev/null @@ -1,545 +0,0 @@ -/** - * Enhanced Structured Logging System for Brainy - * Provides production-ready logging with structured output, context preservation, - * performance tracking, and multiple transport support - */ - -import { performance } from 'perf_hooks' -import { randomUUID } from '../universal/crypto.js' - -export enum LogLevel { - SILENT = -1, - FATAL = 0, - ERROR = 1, - WARN = 2, - INFO = 3, - DEBUG = 4, - TRACE = 5 -} - -export interface LogContext { - requestId?: string - userId?: string - operation?: string - entityId?: string - entityType?: string - [key: string]: any -} - -export interface LogEntry { - timestamp: string - level: string - levelNumeric: number - module: string - message: string - context?: LogContext - data?: any - error?: { - name: string - message: string - stack?: string - code?: string - } - performance?: { - duration?: number - memory?: { - used: number - total: number - } - } - host?: string - pid: number - version?: string -} - -export interface LogTransport { - name: string - log(entry: LogEntry): void | Promise - flush?(): Promise -} - -export interface StructuredLoggerConfig { - level: LogLevel - modules?: Record - format: 'json' | 'pretty' | 'simple' - transports: LogTransport[] - context?: LogContext - includeHost?: boolean - includeMemory?: boolean - bufferSize?: number - flushInterval?: number - version?: string -} - -class ConsoleTransport implements LogTransport { - name = 'console' - private format: 'json' | 'pretty' | 'simple' - - constructor(format: 'json' | 'pretty' | 'simple' = 'json') { - this.format = format - } - - log(entry: LogEntry): void { - const method = this.getConsoleMethod(entry.levelNumeric) - - if (this.format === 'json') { - method(JSON.stringify(entry)) - } else if (this.format === 'pretty') { - const color = this.getColor(entry.levelNumeric) - const prefix = `${entry.timestamp} ${color}[${entry.level}]\\x1b[0m [${entry.module}]` - const message = entry.message - - if (entry.error) { - method(`${prefix} ${message}`, entry.error) - } else if (entry.data) { - method(`${prefix} ${message}`, entry.data) - } else { - method(`${prefix} ${message}`) - } - } else { - // Simple format - method(`[${entry.level}] ${entry.message}`) - } - } - - private getConsoleMethod(level: number): (...args: any[]) => void { - switch (level) { - case LogLevel.FATAL: - case LogLevel.ERROR: - return console.error - case LogLevel.WARN: - return console.warn - case LogLevel.INFO: - return console.info - default: - return console.log - } - } - - private getColor(level: number): string { - switch (level) { - case LogLevel.FATAL: - return '\\x1b[35m' // Magenta - case LogLevel.ERROR: - return '\\x1b[31m' // Red - case LogLevel.WARN: - return '\\x1b[33m' // Yellow - case LogLevel.INFO: - return '\\x1b[36m' // Cyan - case LogLevel.DEBUG: - return '\\x1b[32m' // Green - case LogLevel.TRACE: - return '\\x1b[90m' // Gray - default: - return '\\x1b[0m' // Reset - } - } -} - -class BufferedTransport implements LogTransport { - name = 'buffered' - private buffer: LogEntry[] = [] - private innerTransport: LogTransport - private bufferSize: number - private flushTimer?: NodeJS.Timeout - - constructor(innerTransport: LogTransport, bufferSize: number = 100, flushInterval: number = 5000) { - this.innerTransport = innerTransport - this.bufferSize = bufferSize - - if (flushInterval > 0) { - this.flushTimer = setInterval(() => this.flush(), flushInterval) - } - } - - log(entry: LogEntry): void { - this.buffer.push(entry) - - if (this.buffer.length >= this.bufferSize) { - this.flush() - } - } - - async flush(): Promise { - const entries = this.buffer.splice(0) - for (const entry of entries) { - await this.innerTransport.log(entry) - } - - if (this.innerTransport.flush) { - await this.innerTransport.flush() - } - } - - destroy(): void { - if (this.flushTimer) { - clearInterval(this.flushTimer) - } - this.flush() - } -} - -export class StructuredLogger { - private static instance: StructuredLogger - private config: StructuredLoggerConfig - private defaultContext: LogContext = {} - private performanceMarks = new Map() - - private constructor() { - const isDevelopment = process.env.NODE_ENV !== 'production' - const format = isDevelopment ? 'pretty' : 'json' - - this.config = { - level: isDevelopment ? LogLevel.DEBUG : LogLevel.INFO, - format, - transports: [new ConsoleTransport(format)], - includeHost: !isDevelopment, - includeMemory: false, - bufferSize: 100, - flushInterval: 5000, - version: process.env.npm_package_version - } - - // Load from environment - this.loadEnvironmentConfig() - } - - private loadEnvironmentConfig(): void { - const envLevel = process.env.BRAINY_LOG_LEVEL - if (envLevel) { - const level = LogLevel[envLevel.toUpperCase() as keyof typeof LogLevel] - if (level !== undefined) { - this.config.level = level - } - } - - const envFormat = process.env.BRAINY_LOG_FORMAT - if (envFormat && ['json', 'pretty', 'simple'].includes(envFormat)) { - this.config.format = envFormat as 'json' | 'pretty' | 'simple' - } - - const moduleConfig = process.env.BRAINY_MODULE_LOG_LEVELS - if (moduleConfig) { - try { - this.config.modules = JSON.parse(moduleConfig) - } catch { - // Ignore parse errors - } - } - } - - static getInstance(): StructuredLogger { - if (!StructuredLogger.instance) { - StructuredLogger.instance = new StructuredLogger() - } - return StructuredLogger.instance - } - - configure(config: Partial): void { - this.config = { ...this.config, ...config } - } - - setContext(context: LogContext): void { - this.defaultContext = { ...this.defaultContext, ...context } - } - - clearContext(): void { - this.defaultContext = {} - } - - withContext(context: LogContext): StructuredLogger { - const contextualLogger = Object.create(this) - contextualLogger.defaultContext = { ...this.defaultContext, ...context } - return contextualLogger - } - - startTimer(label: string): void { - this.performanceMarks.set(label, performance.now()) - } - - endTimer(label: string): number | undefined { - const start = this.performanceMarks.get(label) - if (start === undefined) return undefined - - const duration = performance.now() - start - this.performanceMarks.delete(label) - return duration - } - - private shouldLog(level: LogLevel, module: string): boolean { - if (this.config.modules?.[module] !== undefined) { - return level <= this.config.modules[module] - } - return level <= this.config.level - } - - private createLogEntry( - level: LogLevel, - module: string, - message: string, - context?: LogContext, - data?: any, - error?: Error - ): LogEntry { - const entry: LogEntry = { - timestamp: new Date().toISOString(), - level: LogLevel[level], - levelNumeric: level, - module, - message, - pid: process.pid, - version: this.config.version - } - - // Merge contexts - const mergedContext = { ...this.defaultContext, ...context } - if (Object.keys(mergedContext).length > 0) { - entry.context = mergedContext - } - - if (data !== undefined) { - entry.data = data - } - - if (error) { - entry.error = { - name: error.name, - message: error.message, - stack: error.stack, - code: (error as Error & { code?: string }).code - } - } - - if (this.config.includeHost) { - try { - const os = require('node:os') - entry.host = os.hostname() - } catch { - entry.host = 'unknown' - } - } - - if (this.config.includeMemory && typeof process !== 'undefined' && process.memoryUsage) { - const mem = process.memoryUsage() - entry.performance = { - memory: { - used: Math.round(mem.heapUsed / 1024 / 1024), - total: Math.round(mem.heapTotal / 1024 / 1024) - } - } - } - - return entry - } - - private log( - level: LogLevel, - module: string, - message: string, - contextOrData?: LogContext | any, - data?: any - ): void { - if (!this.shouldLog(level, module)) { - return - } - - // Handle overloaded parameters - let context: LogContext | undefined - let logData: any - - if (contextOrData && typeof contextOrData === 'object' && !Array.isArray(contextOrData)) { - // Check if it looks like a context object - const hasContextKeys = ['requestId', 'userId', 'operation', 'entityId', 'entityType'] - .some(key => key in contextOrData) - - if (hasContextKeys) { - context = contextOrData - logData = data - } else { - logData = contextOrData - } - } else { - logData = contextOrData - } - - // Extract error if present - let error: Error | undefined - if (logData instanceof Error) { - error = logData - logData = undefined - } else if (logData?.error instanceof Error) { - error = logData.error - delete logData.error - } - - const entry = this.createLogEntry(level, module, message, context, logData, error) - - // Send to all transports - for (const transport of this.config.transports) { - try { - transport.log(entry) - } catch (err) { - // Fallback to console.error if transport fails - console.error('Logger transport error:', err) - } - } - } - - fatal(module: string, message: string, contextOrData?: LogContext | any, data?: any): void { - this.log(LogLevel.FATAL, module, message, contextOrData, data) - } - - error(module: string, message: string, contextOrData?: LogContext | any, data?: any): void { - this.log(LogLevel.ERROR, module, message, contextOrData, data) - } - - warn(module: string, message: string, contextOrData?: LogContext | any, data?: any): void { - this.log(LogLevel.WARN, module, message, contextOrData, data) - } - - info(module: string, message: string, contextOrData?: LogContext | any, data?: any): void { - this.log(LogLevel.INFO, module, message, contextOrData, data) - } - - debug(module: string, message: string, contextOrData?: LogContext | any, data?: any): void { - this.log(LogLevel.DEBUG, module, message, contextOrData, data) - } - - trace(module: string, message: string, contextOrData?: LogContext | any, data?: any): void { - this.log(LogLevel.TRACE, module, message, contextOrData, data) - } - - createModuleLogger(module: string) { - const self = this - return { - fatal: (message: string, contextOrData?: LogContext | any, data?: any) => - self.fatal(module, message, contextOrData, data), - error: (message: string, contextOrData?: LogContext | any, data?: any) => - self.error(module, message, contextOrData, data), - warn: (message: string, contextOrData?: LogContext | any, data?: any) => - self.warn(module, message, contextOrData, data), - info: (message: string, contextOrData?: LogContext | any, data?: any) => - self.info(module, message, contextOrData, data), - debug: (message: string, contextOrData?: LogContext | any, data?: any) => - self.debug(module, message, contextOrData, data), - trace: (message: string, contextOrData?: LogContext | any, data?: any) => - self.trace(module, message, contextOrData, data), - withContext: (context: LogContext) => { - const contextual = self.withContext(context) - return contextual.createModuleLogger(module) - }, - startTimer: (label: string) => self.startTimer(`${module}:${label}`), - endTimer: (label: string) => self.endTimer(`${module}:${label}`) - } - } - - async flush(): Promise { - const flushPromises = this.config.transports - .filter(t => t.flush) - .map(t => t.flush!()) - - await Promise.all(flushPromises) - } - - addTransport(transport: LogTransport): void { - this.config.transports.push(transport) - } - - removeTransport(name: string): void { - this.config.transports = this.config.transports.filter(t => t.name !== name) - } - - child(context: LogContext): StructuredLogger { - return this.withContext(context) - } -} - -// Singleton instance -export const structuredLogger = StructuredLogger.getInstance() - -// Convenience functions -export function createModuleLogger(module: string) { - return structuredLogger.createModuleLogger(module) -} - -export function setLogContext(context: LogContext) { - structuredLogger.setContext(context) -} - -export function withLogContext(context: LogContext) { - return structuredLogger.withContext(context) -} - -// Correlation ID middleware helper -export function createCorrelationId(): string { - return randomUUID() -} - -// Performance logging helper -export function logPerformance( - logger: ReturnType, - operation: string, - fn: () => any -): any { - logger.startTimer(operation) - try { - const result = fn() - if (result && typeof result.then === 'function') { - return result.finally(() => { - const duration = logger.endTimer(operation) - logger.debug(`${operation} completed`, { duration }) - }) - } - const duration = logger.endTimer(operation) - logger.debug(`${operation} completed`, { duration }) - return result - } catch (error) { - const duration = logger.endTimer(operation) - logger.error(`${operation} failed`, { duration, error }) - throw error - } -} - -// Backward compatibility wrapper for existing logger -export class LoggerCompatibilityWrapper { - private moduleLogger: ReturnType - - constructor(module: string = 'legacy') { - this.moduleLogger = createModuleLogger(module) - } - - error(module: string, message: string, ...args: any[]): void { - this.moduleLogger.error(message, { module, data: args }) - } - - warn(module: string, message: string, ...args: any[]): void { - this.moduleLogger.warn(message, { module, data: args }) - } - - info(module: string, message: string, ...args: any[]): void { - this.moduleLogger.info(message, { module, data: args }) - } - - debug(module: string, message: string, ...args: any[]): void { - this.moduleLogger.debug(message, { module, data: args }) - } - - trace(module: string, message: string, ...args: any[]): void { - this.moduleLogger.trace(message, { module, data: args }) - } - - createModuleLogger(module: string) { - const logger = createModuleLogger(module) - return { - error: (message: string, ...args: any[]) => logger.error(message, { data: args }), - warn: (message: string, ...args: any[]) => logger.warn(message, { data: args }), - info: (message: string, ...args: any[]) => logger.info(message, { data: args }), - debug: (message: string, ...args: any[]) => logger.debug(message, { data: args }), - trace: (message: string, ...args: any[]) => logger.trace(message, { data: args }) - } - } -} - -// Export types for external use -export type ModuleLogger = ReturnType -// Types are already exported above, no need to re-export \ No newline at end of file diff --git a/src/utils/typeValidation.ts b/src/utils/typeValidation.ts index 1ec95392..478fe643 100644 --- a/src/utils/typeValidation.ts +++ b/src/utils/typeValidation.ts @@ -1,3 +1,11 @@ +/** + * @module utils/typeValidation + * @description O(1) runtime validation for the {@link NounType} / {@link VerbType} + * enums. Precomputes a `Set` of each enum's values once at module load, then + * exposes type-guard predicates (`isValidNounType` / `isValidVerbType`) and their + * throwing counterparts used by the write-path parameter validators. Pure and + * side-effect-free apart from the two module-level lookup sets. + */ import { NounType, VerbType } from '../types/graphTypes.js' // Type sets for O(1) validation diff --git a/src/utils/unifiedCache.ts b/src/utils/unifiedCache.ts index 67fe386a..e63ba4b9 100644 --- a/src/utils/unifiedCache.ts +++ b/src/utils/unifiedCache.ts @@ -63,7 +63,7 @@ export interface UnifiedCacheConfig { * * Implements {@link CacheProvider}: the surface Brainy calls on whatever cache * is installed as the global cache (its own instance, or a registered - * `'cache'` provider such as Cortex's native eviction engine). + * `'cache'` provider such as Cor's native eviction engine). */ export class UnifiedCache implements CacheProvider { private cache = new Map() @@ -79,6 +79,7 @@ export class UnifiedCache implements CacheProvider { private readonly memoryInfo: MemoryInfo private readonly allocationStrategy: CacheAllocationStrategy private memoryPressureCheckTimer: NodeJS.Timeout | null = null + private fairnessMonitorTimer: NodeJS.Timeout | null = null private lastMemoryWarning = 0 constructor(config: UnifiedCacheConfig = {}) { @@ -313,12 +314,19 @@ export class UnifiedCache implements CacheProvider { } /** - * Fairness monitoring - prevent one type from hogging cache + * Fairness monitoring - prevent one type from hogging cache. + * + * The interval is unref'd: a background cache monitor must never keep the + * host process alive (this exact timer made bare scripts hang after + * `brain.close()` — the loop could not drain while it stayed ref'd). */ private startFairnessMonitor(): void { - setInterval(() => { + this.fairnessMonitorTimer = setInterval(() => { this.checkFairness() }, this.config.fairnessCheckInterval!) + if (this.fairnessMonitorTimer.unref) { + this.fairnessMonitorTimer.unref() + } } private checkFairness(): void { @@ -444,7 +452,7 @@ export class UnifiedCache implements CacheProvider { /** * Dynamically resize the cache maximum. If the new size is smaller than * the current usage, evicts lowest-value items until the cache fits within - * the new budget. Used by Cortex's ResourceManager to rebalance cache vs + * the new budget. Used by Cor's ResourceManager to rebalance cache vs * instance memory as brainy instances are created and evicted. * * @param newMaxSize - New maximum cache size in bytes diff --git a/src/utils/version.ts b/src/utils/version.ts index 718d44fe..d616cee3 100644 --- a/src/utils/version.ts +++ b/src/utils/version.ts @@ -1,100 +1,85 @@ /** - * Version utilities for Brainy + * @module utils/version + * @description Resolves the running `@soulcraft/brainy` package version. Brainy 8.0 + * targets Node-like runtimes only (Node.js, Bun, Deno — all expose `node:fs`), so the + * version is read **synchronously** from `package.json` on first call and cached. + * + * The synchronous read is load-bearing: `loadPlugins()` is the first step of `init()` + * and reads the version to drive the brainy↔provider version-coupling guard + * (`plugin.ts` → `pluginRangeSatisfies`). A deferred/async value would return a stale + * default on that first synchronous call and spuriously reject a correctly-matched + * native provider (e.g. cor 3.x declaring `>=8.0.0`). Reading synchronously removes + * that window entirely. */ +import { readFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' import { isNode } from './environment.js' -// Default version - this should be replaced at build time -const DEFAULT_VERSION = '3.14.0' +/** + * Last-resort sentinel, returned only if `package.json` cannot be read at all + * (a non-Node runtime, or a genuinely broken install). It is intentionally an + * "unknown" `0.0.0` rather than a plausible-looking release, so a real read + * failure can never masquerade as a valid version and silently satisfy a + * coupling range — it will fail loud instead. + */ +const UNKNOWN_VERSION = '0.0.0' let cachedVersion: string | null = null -let versionPromise: Promise | null = null /** - * Load version from package.json in Node.js environment + * Synchronously read `version` from the package's own `package.json`. The path + * `../../package.json` resolves to the package root from both `src/utils/` (dev) + * and `dist/utils/` (published). */ -async function loadVersionFromPackageJson(): Promise { - if (!isNode()) { - return DEFAULT_VERSION - } - +function readVersionSync(): string { + if (!isNode()) return UNKNOWN_VERSION try { - // Dynamic imports for Node.js modules - modern approach - const [{ readFileSync }, { join, dirname }, { fileURLToPath }] = await Promise.all([ - import('node:fs'), - import('node:path'), - import('node:url') - ]) - - const __filename = fileURLToPath(import.meta.url) - const __dirname = dirname(__filename) - const packageJsonPath = join(__dirname, '../../package.json') - - const packageJson = JSON.parse(readFileSync(packageJsonPath, 'utf8')) - return packageJson.version || DEFAULT_VERSION - } catch (error) { - // Silently fall back to default version - return DEFAULT_VERSION + const here = dirname(fileURLToPath(import.meta.url)) + const pkg = JSON.parse(readFileSync(join(here, '../../package.json'), 'utf8')) + return typeof pkg.version === 'string' && pkg.version.length > 0 + ? pkg.version + : UNKNOWN_VERSION + } catch { + return UNKNOWN_VERSION } } /** - * Get the current Brainy package version - * In Node.js, attempts to read from package.json - * In browser, returns the default version - * @returns The current version string + * @description The running Brainy package version, read synchronously from + * `package.json` and cached. Correct on the **first** call — including the + * version-coupling check during `init()` — with no async warm-up. + * @returns The semver version string (e.g. `"8.0.0"`). + * @example + * const v = getBrainyVersion() // "8.0.0" */ export function getBrainyVersion(): string { - if (cachedVersion) { - return cachedVersion - } - - // In browser or if we need immediate response, return default - if (!isNode()) { - cachedVersion = DEFAULT_VERSION - return cachedVersion - } - - // For Node.js, try to load synchronously first time - // This is a compromise for backward compatibility - if (!versionPromise) { - versionPromise = loadVersionFromPackageJson() - versionPromise.then(version => { - cachedVersion = version - }) - } - - // Return default while loading - return cachedVersion || DEFAULT_VERSION + if (cachedVersion === null) cachedVersion = readVersionSync() + return cachedVersion } /** - * Get the current Brainy package version asynchronously - * Guaranteed to attempt loading from package.json in Node.js - * @returns Promise resolving to the current version string + * @description Async accessor retained for API compatibility. The version is + * resolved synchronously, so this returns the same value as + * {@link getBrainyVersion} — there is no longer any deferred state. + * @returns A promise resolving to the version string. */ export async function getBrainyVersionAsync(): Promise { - if (cachedVersion) { - return cachedVersion - } - - if (!versionPromise) { - versionPromise = loadVersionFromPackageJson() - } - - const version = await versionPromise - cachedVersion = version - return version + return getBrainyVersion() } /** - * Get version information for augmentation metadata - * @param service The service/augmentation name - * @returns Version metadata object + * @description Build the version-metadata object stamped onto augmentation / + * provenance records. + * @param service - The augmentation or service name to record. + * @returns `{ augmentation, version }` carrying the current package version. + * @example + * getAugmentationVersion('aggregation') // { augmentation: 'aggregation', version: '8.0.0' } */ export function getAugmentationVersion(service: string): { augmentation: string; version: string } { return { augmentation: service, version: getBrainyVersion() } -} \ No newline at end of file +} diff --git a/src/utils/writeBuffer.ts b/src/utils/writeBuffer.ts deleted file mode 100644 index b62f98d8..00000000 --- a/src/utils/writeBuffer.ts +++ /dev/null @@ -1,411 +0,0 @@ -/** - * Write Buffer - * Accumulates writes and flushes them in bulk to reduce S3 operations - * Implements intelligent deduplication and compression - */ - -import { HNSWNoun, HNSWVerb } from '../coreTypes.js' -import { createModuleLogger } from './logger.js' -import { getGlobalBackpressure } from './adaptiveBackpressure.js' - -interface BufferedWrite { - id: string - data: T - timestamp: number - type: 'noun' | 'verb' | 'metadata' - retryCount: number -} - -interface FlushResult { - successful: number - failed: number - duration: number -} - -/** - * High-performance write buffer for bulk operations - */ -export class WriteBuffer { - private logger = createModuleLogger('WriteBuffer') - - // Buffer storage - private buffer = new Map>() - - // Configuration - More aggressive for high volume - private maxBufferSize = 2000 // Allow larger buffers - private flushInterval = 500 // Flush more frequently (0.5 seconds) - private minFlushSize = 50 // Lower minimum to flush sooner - private maxRetries = 3 // Maximum retry attempts - - // State - private flushTimer: NodeJS.Timeout | null = null - private isFlushing = false - private lastFlush = Date.now() - private pendingFlush: Promise | null = null - - // Statistics - private totalWrites = 0 - private totalFlushes = 0 - private failedWrites = 0 - private duplicatesRemoved = 0 - - // Write function - private writeFunction: (items: Map) => Promise - private type: 'noun' | 'verb' | 'metadata' - - // Backpressure integration - private backpressure = getGlobalBackpressure() - - constructor( - type: 'noun' | 'verb' | 'metadata', - writeFunction: (items: Map) => Promise, - options?: { - maxBufferSize?: number - flushInterval?: number - minFlushSize?: number - } - ) { - this.type = type - this.writeFunction = writeFunction - - if (options) { - this.maxBufferSize = options.maxBufferSize || this.maxBufferSize - this.flushInterval = options.flushInterval || this.flushInterval - this.minFlushSize = options.minFlushSize || this.minFlushSize - } - - // Start periodic flush - this.startPeriodicFlush() - } - - /** - * Add item to buffer - */ - public async add(id: string, data: T): Promise { - // Check if we're already at capacity - if (this.buffer.size >= this.maxBufferSize) { - // Wait for current flush to complete - if (this.pendingFlush) { - await this.pendingFlush - } - - // Force flush if still at capacity - if (this.buffer.size >= this.maxBufferSize) { - await this.flush('capacity') - } - } - - // Check for duplicate and update if newer - const existing = this.buffer.get(id) - if (existing) { - // Update with newer data - existing.data = data - existing.timestamp = Date.now() - this.duplicatesRemoved++ - } else { - // Add new item - this.buffer.set(id, { - id, - data, - timestamp: Date.now(), - type: this.type, - retryCount: 0 - }) - } - - this.totalWrites++ - - // Log buffer growth periodically - if (this.totalWrites % 100 === 0) { - this.logger.info(`📈 BUFFER GROWTH: ${this.buffer.size} ${this.type} items buffered (${this.totalWrites} total writes, ${this.duplicatesRemoved} deduplicated)`) - } - - // Check if we should flush - this.checkFlush() - } - - /** - * Check if we should flush - */ - private checkFlush(): void { - const bufferSize = this.buffer.size - const timeSinceFlush = Date.now() - this.lastFlush - - // Immediate flush conditions - if (bufferSize >= this.maxBufferSize) { - this.flush('size') - return - } - - // Time-based flush with minimum size - if (timeSinceFlush >= this.flushInterval && bufferSize >= this.minFlushSize) { - this.flush('time') - return - } - - // Adaptive flush based on system load - const backpressureStatus = this.backpressure.getStatus() - if (backpressureStatus.queueLength > 1000 && bufferSize > 10) { - // System under pressure - flush smaller batches more frequently - this.flush('pressure') - } - } - - /** - * Flush buffer to storage - */ - public async flush(reason: string = 'manual'): Promise { - // Prevent concurrent flushes - if (this.isFlushing) { - if (this.pendingFlush) { - return this.pendingFlush - } - return { successful: 0, failed: 0, duration: 0 } - } - - // Nothing to flush - if (this.buffer.size === 0) { - return { successful: 0, failed: 0, duration: 0 } - } - - this.isFlushing = true - const startTime = Date.now() - - // Create flush promise - this.pendingFlush = this.doFlush(reason, startTime) - - try { - const result = await this.pendingFlush - return result - } finally { - this.isFlushing = false - this.pendingFlush = null - } - } - - /** - * Perform the actual flush - */ - private async doFlush(reason: string, startTime: number): Promise { - const itemsToFlush = new Map() - const flushingItems = new Map>() - - // Take items from buffer - let count = 0 - for (const [id, item] of this.buffer.entries()) { - itemsToFlush.set(id, item.data) - flushingItems.set(id, item) - count++ - - // Limit batch size for better performance - if (count >= 500) { - break - } - } - - // Remove from buffer - for (const id of itemsToFlush.keys()) { - this.buffer.delete(id) - } - - this.logger.warn(`🔄 BUFFERING: Flushing ${itemsToFlush.size} ${this.type} items (buffer had ${this.buffer.size + itemsToFlush.size}) - reason: ${reason}`) - - try { - // Request permission from backpressure system - const opId = `flush-${Date.now()}` - await this.backpressure.requestPermission(opId, 2) // Higher priority - - try { - // Perform bulk write - await this.writeFunction(itemsToFlush) - - // Success - this.backpressure.releasePermission(opId, true) - this.totalFlushes++ - this.lastFlush = Date.now() - - const duration = Date.now() - startTime - this.logger.warn(`🚀 BATCH FLUSH: ${itemsToFlush.size} ${this.type} items → 1 bulk S3 operation (${duration}ms, reason: ${reason})`) - - return { - successful: itemsToFlush.size, - failed: 0, - duration - } - } catch (error) { - // Release with error - this.backpressure.releasePermission(opId, false) - throw error - } - } catch (error) { - this.logger.error(`Flush failed: ${error}`) - - // Put items back with retry count - for (const [id, item] of flushingItems.entries()) { - item.retryCount++ - - if (item.retryCount < this.maxRetries) { - // Put back for retry - this.buffer.set(id, item) - } else { - // Max retries exceeded - this.failedWrites++ - this.logger.error(`Max retries exceeded for ${this.type} ${id}`) - } - } - - const duration = Date.now() - startTime - - return { - successful: 0, - failed: itemsToFlush.size, - duration - } - } - } - - /** - * Start periodic flush timer - */ - private startPeriodicFlush(): void { - if (this.flushTimer) { - return - } - - this.flushTimer = setInterval(() => { - if (this.buffer.size > 0) { - const timeSinceFlush = Date.now() - this.lastFlush - - // Flush if we have items and enough time has passed - if (timeSinceFlush >= this.flushInterval) { - this.flush('periodic').catch(error => { - this.logger.error('Periodic flush failed:', error) - }) - } - } - }, Math.min(100, this.flushInterval / 2)) - } - - /** - * Stop periodic flush timer - */ - public stop(): void { - if (this.flushTimer) { - clearInterval(this.flushTimer) - this.flushTimer = null - } - } - - /** - * Force flush all pending writes - */ - public async forceFlush(): Promise { - // Flush everything regardless of size - const oldMinSize = this.minFlushSize - this.minFlushSize = 0 - - try { - const result = await this.flush('force') - - // Flush any remaining items - while (this.buffer.size > 0) { - const additionalResult = await this.flush('force-remaining') - result.successful += additionalResult.successful - result.failed += additionalResult.failed - result.duration += additionalResult.duration - } - - return result - } finally { - this.minFlushSize = oldMinSize - } - } - - /** - * Get buffer statistics - */ - public getStats(): { - bufferSize: number - totalWrites: number - totalFlushes: number - failedWrites: number - duplicatesRemoved: number - avgFlushSize: number - } { - return { - bufferSize: this.buffer.size, - totalWrites: this.totalWrites, - totalFlushes: this.totalFlushes, - failedWrites: this.failedWrites, - duplicatesRemoved: this.duplicatesRemoved, - avgFlushSize: this.totalFlushes > 0 ? this.totalWrites / this.totalFlushes : 0 - } - } - - /** - * Adjust parameters based on load - */ - public adjustForLoad(pendingRequests: number): void { - if (pendingRequests > 10000) { - // Extreme load - buffer more aggressively - this.maxBufferSize = 5000 - this.flushInterval = 500 - this.minFlushSize = 500 - } else if (pendingRequests > 1000) { - // High load - this.maxBufferSize = 2000 - this.flushInterval = 1000 - this.minFlushSize = 200 - } else if (pendingRequests > 100) { - // Moderate load - this.maxBufferSize = 1000 - this.flushInterval = 2000 - this.minFlushSize = 100 - } else { - // Low load - optimize for latency - this.maxBufferSize = 500 - this.flushInterval = 5000 - this.minFlushSize = 50 - } - } -} - -// Global write buffers -const writeBuffers = new Map>() - -/** - * Get or create a write buffer - */ -export function getWriteBuffer( - id: string, - type: 'noun' | 'verb' | 'metadata', - writeFunction: (items: Map) => Promise -): WriteBuffer { - if (!writeBuffers.has(id)) { - writeBuffers.set(id, new WriteBuffer(type, writeFunction)) - } - return writeBuffers.get(id)! -} - -/** - * Flush all write buffers - */ -export async function flushAllBuffers(): Promise { - const promises: Promise[] = [] - - for (const buffer of writeBuffers.values()) { - promises.push(buffer.forceFlush()) - } - - await Promise.all(promises) -} - -/** - * Clear all write buffers - */ -export function clearWriteBuffers(): void { - for (const buffer of writeBuffers.values()) { - buffer.stop() - } - writeBuffers.clear() -} \ No newline at end of file diff --git a/src/vfs/FSCompat.ts b/src/vfs/FSCompat.ts deleted file mode 100644 index 4715aa79..00000000 --- a/src/vfs/FSCompat.ts +++ /dev/null @@ -1,333 +0,0 @@ -/** - * fs-Compatible Interface for VFS - * - * Provides a drop-in replacement for Node's fs module - * that uses VFS for storage instead of the real filesystem. - * - * Usage: - * import { FSCompat } from '@soulcraft/brainy/vfs' - * const fs = new FSCompat(brain.vfs) - * - * // Now use like Node's fs - * await fs.promises.readFile('/path') - * fs.createReadStream('/path').pipe(output) - */ - -import { VirtualFileSystem } from './VirtualFileSystem.js' -import type { - ReadOptions, - WriteOptions, - MkdirOptions, - VFSStats, - VFSDirent -} from './types.js' - -export class FSCompat { - /** - * Promise-based API (like fs.promises) - */ - public readonly promises: FSPromises - - constructor(private vfs: VirtualFileSystem) { - this.promises = new FSPromises(vfs) - } - - // ============= Callback-style methods (for compatibility) ============= - - readFile(path: string, callback: (err: Error | null, data?: Buffer) => void): void - readFile(path: string, encoding: BufferEncoding, callback: (err: Error | null, data?: string) => void): void - readFile(path: string, options: any, callback?: any): void { - if (typeof options === 'function') { - callback = options - options = {} - } - - this.vfs.readFile(path, options) - .then(data => { - if (options?.encoding) { - callback(null, data.toString(options.encoding)) - } else { - callback(null, data) - } - }) - .catch(err => callback(err)) - } - - writeFile(path: string, data: Buffer | string, callback: (err: Error | null) => void): void - writeFile(path: string, data: Buffer | string, options: any, callback?: any): void { - if (typeof options === 'function') { - callback = options - options = {} - } - - const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, options?.encoding || 'utf8') - - this.vfs.writeFile(path, buffer, options) - .then(() => callback(null)) - .catch(err => callback(err)) - } - - mkdir(path: string, callback: (err: Error | null) => void): void - mkdir(path: string, options: any, callback?: any): void { - if (typeof options === 'function') { - callback = options - options = {} - } - - this.vfs.mkdir(path, options) - .then(() => callback(null)) - .catch(err => callback(err)) - } - - rmdir(path: string, callback: (err: Error | null) => void): void - rmdir(path: string, options: any, callback?: any): void { - if (typeof options === 'function') { - callback = options - options = {} - } - - this.vfs.rmdir(path, options) - .then(() => callback(null)) - .catch(err => callback(err)) - } - - readdir(path: string, callback: (err: Error | null, files?: string[]) => void): void - readdir(path: string, options: any, callback?: any): void { - if (typeof options === 'function') { - callback = options - options = {} - } - - this.vfs.readdir(path, options) - .then(files => callback(null, files)) - .catch(err => callback(err)) - } - - stat(path: string, callback: (err: Error | null, stats?: VFSStats) => void): void { - this.vfs.stat(path) - .then(stats => callback(null, stats)) - .catch(err => callback(err)) - } - - lstat(path: string, callback: (err: Error | null, stats?: VFSStats) => void): void { - this.vfs.lstat(path) - .then(stats => callback(null, stats)) - .catch(err => callback(err)) - } - - unlink(path: string, callback: (err: Error | null) => void): void { - this.vfs.unlink(path) - .then(() => callback(null)) - .catch(err => callback(err)) - } - - rename(oldPath: string, newPath: string, callback: (err: Error | null) => void): void { - this.vfs.rename(oldPath, newPath) - .then(() => callback(null)) - .catch(err => callback(err)) - } - - copyFile(src: string, dest: string, callback: (err: Error | null) => void): void - copyFile(src: string, dest: string, flags: number, callback: (err: Error | null) => void): void - copyFile(src: string, dest: string, flagsOrCallback: any, callback?: any): void { - if (typeof flagsOrCallback === 'function') { - callback = flagsOrCallback - } else { - // flags provided but not used - } - - this.vfs.copy(src, dest) - .then(() => callback(null)) - .catch(err => callback(err)) - } - - exists(path: string, callback: (exists: boolean) => void): void { - this.vfs.exists(path) - .then(exists => callback(exists)) - .catch(() => callback(false)) - } - - access(path: string, callback: (err: Error | null) => void): void - access(path: string, mode: number, callback: (err: Error | null) => void): void - access(path: string, modeOrCallback: any, callback?: any): void { - if (typeof modeOrCallback === 'function') { - callback = modeOrCallback - } else { - // mode provided but not used - } - - this.vfs.exists(path) - .then(exists => { - if (exists) { - callback(null) - } else { - const err: any = new Error('ENOENT: no such file or directory') - err.code = 'ENOENT' - callback(err) - } - }) - .catch(err => callback(err)) - } - - appendFile(path: string, data: Buffer | string, callback: (err: Error | null) => void): void - appendFile(path: string, data: Buffer | string, options: any, callback?: any): void { - if (typeof options === 'function') { - callback = options - options = {} - } - - const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, options?.encoding || 'utf8') - - this.vfs.appendFile(path, buffer, options) - .then(() => callback(null)) - .catch(err => callback(err)) - } - - // ============= Stream methods ============= - - createReadStream(path: string, options?: any) { - return this.vfs.createReadStream(path, options) - } - - createWriteStream(path: string, options?: any) { - return this.vfs.createWriteStream(path, options) - } - - // ============= Watch methods ============= - - watch(path: string, listener?: any) { - return this.vfs.watch(path, listener) - } - - watchFile(path: string, listener: any) { - return this.vfs.watchFile(path, listener) - } - - unwatchFile(path: string) { - return this.vfs.unwatchFile(path) - } - - // ============= Additional methods ============= - - /** - * Import a directory from real filesystem (VFS extension) - */ - async importDirectory(sourcePath: string, options?: any) { - return this.vfs.importDirectory(sourcePath, options) - } - - /** - * Search files semantically (VFS extension) - */ - async search(query: string, options?: any) { - return this.vfs.search(query, options) - } -} - -/** - * Promise-based fs API (like fs.promises) - */ -class FSPromises { - constructor(private vfs: VirtualFileSystem) {} - - async readFile(path: string, options?: any): Promise { - const buffer = await this.vfs.readFile(path, options) - if (options?.encoding) { - return buffer.toString(options.encoding) - } - return buffer - } - - async writeFile(path: string, data: Buffer | string, options?: any): Promise { - const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, options?.encoding || 'utf8') - return this.vfs.writeFile(path, buffer, options) - } - - async mkdir(path: string, options?: any): Promise { - return this.vfs.mkdir(path, options) - } - - async rmdir(path: string, options?: any): Promise { - return this.vfs.rmdir(path, options) - } - - async readdir(path: string, options?: any): Promise { - return this.vfs.readdir(path, options) - } - - async stat(path: string): Promise { - return this.vfs.stat(path) - } - - async lstat(path: string): Promise { - return this.vfs.lstat(path) - } - - async unlink(path: string): Promise { - return this.vfs.unlink(path) - } - - async rename(oldPath: string, newPath: string): Promise { - return this.vfs.rename(oldPath, newPath) - } - - async copyFile(src: string, dest: string, flags?: number): Promise { - return this.vfs.copy(src, dest) - } - - async access(path: string, mode?: number): Promise { - const exists = await this.vfs.exists(path) - if (!exists) { - const err: any = new Error('ENOENT: no such file or directory') - err.code = 'ENOENT' - throw err - } - } - - async appendFile(path: string, data: Buffer | string, options?: any): Promise { - const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, options?.encoding || 'utf8') - return this.vfs.appendFile(path, buffer, options) - } - - async realpath(path: string): Promise { - return this.vfs.realpath(path) - } - - async chmod(path: string, mode: number): Promise { - return this.vfs.chmod(path, mode) - } - - async chown(path: string, uid: number, gid: number): Promise { - return this.vfs.chown(path, uid, gid) - } - - async utimes(path: string, atime: Date, mtime: Date): Promise { - return this.vfs.utimes(path, atime, mtime) - } - - async symlink(target: string, path: string): Promise { - return this.vfs.symlink(target, path) - } - - async readlink(path: string): Promise { - return this.vfs.readlink(path) - } - - // VFS Extensions - async search(query: string, options?: any) { - return this.vfs.search(query, options) - } - - async findSimilar(path: string, options?: any) { - return this.vfs.findSimilar(path, options) - } - - async importDirectory(sourcePath: string, options?: any) { - return this.vfs.importDirectory(sourcePath, options) - } -} - -// Export a convenience function to create fs replacement -export function createFS(vfs: VirtualFileSystem): FSCompat { - return new FSCompat(vfs) -} \ No newline at end of file diff --git a/src/vfs/PathResolver.ts b/src/vfs/PathResolver.ts index 6babd922..e496c834 100644 --- a/src/vfs/PathResolver.ts +++ b/src/vfs/PathResolver.ts @@ -525,6 +525,10 @@ export class PathResolver { console.log(`[PathResolver] Cache stats: ${Math.round(hitRate * 100)}% hit rate, ${this.pathCache.size} entries, ${this.hotPaths.size} hot paths`) } }, 60000) // Every minute + // Cache maintenance must never keep the host process alive. + if (this.maintenanceTimer && typeof this.maintenanceTimer.unref === 'function') { + this.maintenanceTimer.unref() + } } /** diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts index 0cf975dd..00bddefb 100644 --- a/src/vfs/VirtualFileSystem.ts +++ b/src/vfs/VirtualFileSystem.ts @@ -36,6 +36,7 @@ import { VFSErrorCode, WriteOptions, ReadOptions, + FileVersion, MkdirOptions, ReaddirOptions, CopyOptions, @@ -88,6 +89,17 @@ export class VirtualFileSystem implements IVirtualFileSystem { // Uses deterministic UUID format for storage compatibility private static readonly VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000' + /** + * Construct a VFS bound to a Brainy instance. + * + * The VFS stores every file and directory as a Brainy entity and models the + * directory tree with graph relationships. No I/O happens here — call + * {@link init} (or rely on `brain.init()`, which auto-initializes the VFS) + * before using any filesystem operation. + * + * @param brain - The Brainy instance backing this filesystem. When omitted, a + * fresh `new Brainy()` is created and owned by this VFS. + */ constructor(brain?: Brainy) { this.brain = brain || new Brainy() this.contentCache = new Map() @@ -341,6 +353,52 @@ export class VirtualFileSystem implements IVirtualFileSystem { } } + /** + * @description Recover VFS content blobs orphaned by a 7→8 upgrade. 7.x stored + * VFS blobs under the copy-on-write area (`_cow/`) of the branch/versioning + * system that 8.0 removed; a brain upgraded before the migration adopted them + * reads a stranded blob and throws "Blob metadata not found" (every page + * backed by it 500s). This adopts every orphaned `_cow/` blob into the 8.0 + * content-addressed store (`_cas/`) IN PLACE — no snapshot restore — then + * clears the VFS caches so the recovered content reads live immediately. + * + * Idempotent and non-destructive: blobs already adopted are skipped and the + * `_cow/` originals are never deleted (a re-run or rollback stays possible). + * Safe to run on a healthy or native-8.0 brain (returns zeros). Delegates to + * {@link BaseStorage.adoptLegacyCowBlobs}. + * + * @returns `{ cowBlobs, adopted, alreadyPresent, incomplete }` counts. + * @example + * const brain = new Brainy({ storage: { type: 'filesystem', path: '/data/brain' } }) + * await brain.init() + * const r = await brain.vfs.adoptOrphanedBlobs() + * console.log(`Adopted ${r.adopted} stranded VFS blobs; ${r.alreadyPresent} already present.`) + */ + async adoptOrphanedBlobs(): Promise<{ + cowBlobs: number + adopted: number + alreadyPresent: number + incomplete: number + }> { + // The recovery scan works on raw storage objects, so it only needs the + // brain's storage wired up — not the VFS's own init. brain.init() is + // idempotent (a no-op on an already-open brain) and, on a cold brain, + // sets up storage AND runs the on-open auto-adoption itself; the explicit + // scan below is then the idempotent "force re-scan" equivalent. + await this.brain.init() + const storage = this.brain['storage'] as unknown as + | { adoptLegacyCowBlobs?: () => Promise<{ cowBlobs: number; adopted: number; alreadyPresent: number; incomplete: number }> } + | undefined + if (!storage || typeof storage.adoptLegacyCowBlobs !== 'function') { + return { cowBlobs: 0, adopted: 0, alreadyPresent: 0, incomplete: 0 } + } + const result = await storage.adoptLegacyCowBlobs() + // Drop any cached content/stat so a re-read hits the freshly adopted blobs. + this.contentCache.clear() + this.statCache.clear() + return result + } + // ============= File Operations ============= /** @@ -349,6 +407,14 @@ export class VirtualFileSystem implements IVirtualFileSystem { async readFile(path: string, options?: ReadOptions): Promise { await this.ensureInitialized() + // Temporal read: the file's exact bytes as of a past generation or + // instant. Materializes the entity from the generation history — content + // blobs referenced by any in-window generation are retention-protected, + // so the bytes are guaranteed present. Bypasses the content cache. + if (options?.asOf !== undefined) { + return this.readFileAt(path, options.asOf, options) + } + // Check cache first if (options?.cache !== false && this.contentCache.has(path)) { const cached = this.contentCache.get(path)! @@ -415,6 +481,122 @@ export class VirtualFileSystem implements IVirtualFileSystem { } } + /** + * @description The temporal read behind `readFile(path, { asOf })`: resolve + * the path's CURRENT entity, materialize its state at the target + * generation/instant from the Model-B history, and read that version's + * content blob (retention-protected, so present for every in-window + * generation). The pinned view is always released so compaction is never + * blocked by a read. + * @param path - The file path (resolved against the live tree). + * @param asOf - A generation number or wall-clock `Date`. + * @param options - `encoding` applies; caching does not (never cached). + * @returns The exact bytes the file held at that generation. + * @throws VFSError ENOENT when the file did not exist at that generation; + * the generation store's compacted-generation error when `asOf` is past + * the retention window's horizon. + */ + private async readFileAt( + path: string, + asOf: number | Date, + options?: ReadOptions + ): Promise { + const entityId = await this.pathResolver.resolve(path) + const db = await this.brain.asOf(asOf) + try { + const entity = await db.get(entityId) + if (!entity) { + throw new VFSError( + VFSErrorCode.ENOENT, + `File did not exist as of ${String(asOf)}: ${path}`, + path, + 'readFile' + ) + } + const storage = (entity.metadata as Record | undefined)?.storage + if (storage?.type !== 'blob' || typeof storage.hash !== 'string') { + throw new VFSError( + VFSErrorCode.EIO, + `File had no blob storage as of ${String(asOf)}: ${path}`, + path, + 'readFile' + ) + } + const content = await this.blobStorage.read(storage.hash) + if (options?.encoding) { + return Buffer.from(content.toString(options.encoding)) + } + return content + } finally { + await db.release() + } + } + + /** + * @description A file's version history within the retention window: one + * entry per generation that wrote the file, ascending — the newest entry is + * the current state. Pair with `readFile(path, { asOf: entry.generation })` + * to fetch any version's exact bytes, and restore by writing those bytes + * back (a NEW write — history is never rewritten). Bounded by the retention + * window: versions whose generations were compacted are not listed. + * @param path - The file path (resolved against the live tree). + * @returns The versions, oldest first. + * @example + * const versions = await brain.vfs.history('/pages/home.json') + * const before = await brain.vfs.readFile('/pages/home.json', { + * asOf: versions[versions.length - 2].generation + * }) + */ + async history(path: string): Promise { + await this.ensureInitialized() + const entityId = await this.pathResolver.resolve(path) + + // Boundary note: reaches Brainy's internal generation store (the same + // bracket-access precedent as the blobStorage getter) — the per-id + // generation chain and horizon are not on the public Brainy surface. + const generationStore = (this.brain as unknown as { + generationStore: { + horizon(): number + generation(): number + generationsTouching( + kind: 'noun' | 'verb', + id: string, + fromGen: number, + toGen: number + ): Promise + } + }).generationStore + + const gens = await generationStore.generationsTouching( + 'noun', + entityId, + generationStore.horizon(), + generationStore.generation() + ) + + const versions: FileVersion[] = [] + for (const gen of gens) { + const db = await this.brain.asOf(gen) + try { + const entity = await db.get(entityId) + const meta = entity?.metadata as Record | undefined + const storage = meta?.storage + if (storage?.type === 'blob' && typeof storage.hash === 'string') { + versions.push({ + generation: gen, + timestamp: db.timestamp, + hash: storage.hash, + size: typeof meta?.size === 'number' ? meta.size : (storage.size ?? 0), + ...(typeof meta?.mimeType === 'string' && { mimeType: meta.mimeType }) + }) + } + } finally { + await db.release() + } + } + return versions + } + /** * Write a file */ @@ -436,14 +618,18 @@ export class VirtualFileSystem implements IVirtualFileSystem { // Ensure parent directory exists const parentId = await this.ensureDirectory(parentPath) - // Check if file already exists + // Check if file already exists (and capture its current content hash so + // the overwrite can release the superseded live reference afterwards). let existingId: string | null = null + let previousHash: string | undefined try { existingId = await this.pathResolver.resolve(path, { cache: false }) // Verify the entity still exists in the brain const existing = await this.brain.get(existingId) if (!existing) { existingId = null // Entity was deleted but cache wasn't cleared + } else if (existing.metadata?.storage?.type === 'blob') { + previousHash = existing.metadata.storage.hash as string | undefined } } catch (err) { // File doesn't exist, which is fine @@ -494,14 +680,32 @@ export class VirtualFileSystem implements IVirtualFileSystem { Object.assign(metadata, await this.extractMetadata(buffer, mimeType)) } + // For embedding: use text content, for storage: use raw data. Computed + // for BOTH branches — an overwrite must refresh the entity's `data` (and + // therefore its embedding), or semantic search and any `data` read keep + // serving the FIRST version's text forever. + const embeddingData = mimeDetector.isTextFile(mimeType) + ? buffer.toString('utf-8') + : `File: ${name} (${mimeType}, ${buffer.length} bytes)` + if (existingId) { - // Update existing file - // No entity.data - content is in BlobStorage + // Update existing file — content bytes live in BlobStorage; `data` + // carries the embedding text and MUST track the new content. await this.brain.update({ id: existingId, + data: embeddingData, metadata }) + // The update committed: drop the superseded live reference. When the + // content is unchanged this cancels the dedup increment the write() + // above just made (net: one live reference per referencing file); when + // it changed, the old bytes stay retention-protected for asOf reads + // via the update's own generation record. + if (previousHash) { + await this.blobStorage.release(previousHash) + } + // Ensure Contains relationship exists (fix for missing relationships) const existingRelations = await this.brain.related({ from: parentId, @@ -521,9 +725,6 @@ export class VirtualFileSystem implements IVirtualFileSystem { } } else { // Create new file entity - // For embedding: use text content, for storage: use raw data - const embeddingData = mimeDetector.isTextFile(mimeType) ? buffer.toString('utf-8') : `File: ${name} (${mimeType}, ${buffer.length} bytes)` - const entity = await this.brain.add({ data: embeddingData, // Always provide string for embeddings type: this.getFileNounType(mimeType), @@ -592,14 +793,21 @@ export class VirtualFileSystem implements IVirtualFileSystem { throw new VFSError(VFSErrorCode.EISDIR, `Is a directory: ${path}`, path, 'unlink') } - // Delete blob from BlobStorage (decrements ref count) - if (entity.metadata.storage?.type === 'blob') { - await this.blobStorage.delete(entity.metadata.storage.hash) - } - - // Delete the entity + // Delete the entity FIRST, then drop its live blob reference — never + // release a reference while the referencing entity might survive (a + // failed remove must not leave a live file whose bytes compaction could + // later reclaim). await this.brain.remove(entityId) + // Drop the file's LIVE reference to its content blob. The bytes are NOT + // deleted — the remove's own generation record references this hash, so + // `readFile(path, { asOf })` keeps serving it inside the retention + // window; history compaction reclaims the bytes when the last referencing + // generation is reclaimed. + if (entity.metadata.storage?.type === 'blob') { + await this.blobStorage.release(entity.metadata.storage.hash) + } + // Invalidate caches this.pathResolver.invalidatePath(path) this.invalidateCaches(path) @@ -975,23 +1183,26 @@ export class VirtualFileSystem implements IVirtualFileSystem { // Phase 1: Gather all descendants in ONE batch fetch const descendants = await this.gatherDescendants(entityId, Infinity) - // Phase 2: Parallel blob cleanup (chunked to avoid overwhelming storage) - // Blob deletion is reference-counted, so safe to call for all files + // Phase 2: Batch delete all entities (including root directory) FIRST — + // never release a blob reference while its referencing entity might + // survive a failed delete (see unlink's ordering note). + const allIds = [...descendants.map(d => d.id), entityId] + await this.brain.removeMany({ ids: allIds, continueOnError: false }) + + // Phase 3: Drop the deleted files' LIVE blob references (chunked). + // Live-reference drops only — the bytes stay for in-window asOf reads + // and are reclaimed by history compaction (see unlink's note). const blobFiles = descendants.filter(d => d.metadata.vfsType === 'file' && d.metadata.storage?.type === 'blob' ) - const BLOB_CHUNK_SIZE = 20 // Parallel delete 20 blobs at a time + const BLOB_CHUNK_SIZE = 20 // Parallel release 20 blob references at a time for (let i = 0; i < blobFiles.length; i += BLOB_CHUNK_SIZE) { const chunk = blobFiles.slice(i, i + BLOB_CHUNK_SIZE) await Promise.all(chunk.map(f => - this.blobStorage.delete(f.metadata.storage!.hash) + this.blobStorage.release(f.metadata.storage!.hash) )) } - - // Phase 3: Batch delete all entities (including root directory) - const allIds = [...descendants.map(d => d.id), entityId] - await this.brain.removeMany({ ids: allIds, continueOnError: false }) } else { // No children or not recursive - just delete the directory entity await this.brain.remove(entityId) @@ -1135,7 +1346,6 @@ export class VirtualFileSystem implements IVirtualFileSystem { type: [NounType.File, NounType.Document, NounType.Media], limit: options?.limit || 10, offset: options?.offset, - explain: options?.explain, where: { vfsType: 'file' // Search VFS files } @@ -1250,6 +1460,17 @@ export class VirtualFileSystem implements IVirtualFileSystem { } } + /** + * Fetch a Brainy entity by id and normalize it into a {@link VFSEntity}. + * + * Backfills VFS metadata for legacy entities that stored fields at the top + * level (pre-nested-metadata layout) and guarantees the root directory always + * carries valid directory metadata. + * + * @param id - The Brainy entity id to fetch. + * @returns The entity with a populated `metadata.vfsType` and related VFS fields. + * @throws {VFSError} ENOENT when no entity exists for the given id. + */ async getEntityById(id: string): Promise { const entity = await this.brain.get(id) @@ -1525,6 +1746,10 @@ export class VirtualFileSystem implements IVirtualFileSystem { } } }, 60000) // Every minute + // Cache maintenance must never keep the host process alive. + if (this.backgroundTimer && typeof this.backgroundTimer.unref === 'function') { + this.backgroundTimer.unref() + } } private getDefaultConfig(): Required> & { rootEntityId?: string } { @@ -1572,8 +1797,17 @@ export class VirtualFileSystem implements IVirtualFileSystem { } } - // ============= Not Yet Implemented ============= + // ============= Lifecycle, POSIX & Extended Operations ============= + /** + * Release all resources held by the VFS. + * + * Stops background cache eviction, tears down the path resolver, and clears the + * content cache and watcher registry. After close the VFS is marked + * uninitialized; call {@link init} again before reusing it. + * + * @returns A promise that resolves once cleanup is complete. + */ async close(): Promise { // Cleanup PathResolver resources if (this.pathResolver) { @@ -1595,6 +1829,13 @@ export class VirtualFileSystem implements IVirtualFileSystem { this.initialized = false } + /** + * Change the permission bits of a file or directory (POSIX `chmod`). + * + * @param path - The VFS path whose permissions should change. + * @param mode - The new permission bits (e.g. `0o644`), stored in entity metadata. + * @returns A promise that resolves once the permissions are persisted. + */ async chmod(path: string, mode: number): Promise { await this.ensureInitialized() @@ -1616,6 +1857,14 @@ export class VirtualFileSystem implements IVirtualFileSystem { this.invalidateCaches(path) } + /** + * Change the owner and group of a file or directory (POSIX `chown`). + * + * @param path - The VFS path whose ownership should change. + * @param uid - The new owner user id, stored in entity metadata. + * @param gid - The new owning group id, stored in entity metadata. + * @returns A promise that resolves once the ownership is persisted. + */ async chown(path: string, uid: number, gid: number): Promise { await this.ensureInitialized() @@ -1638,6 +1887,14 @@ export class VirtualFileSystem implements IVirtualFileSystem { this.invalidateCaches(path) } + /** + * Set the access and modification timestamps of a file or directory (POSIX `utimes`). + * + * @param path - The VFS path to update. + * @param atime - The new access time. + * @param mtime - The new modification time. + * @returns A promise that resolves once the timestamps are persisted. + */ async utimes(path: string, atime: Date, mtime: Date): Promise { await this.ensureInitialized() @@ -1659,6 +1916,20 @@ export class VirtualFileSystem implements IVirtualFileSystem { this.invalidateCaches(path) } + /** + * Rename or move a file or directory in place. + * + * Updates the entity's path metadata without rewriting content (preserving the + * underlying blob and entity id). When the parent directory changes, a new + * `Contains` edge is added to the destination parent. Renaming a directory + * cascades the path change to every descendant. + * + * @param oldPath - The existing VFS path. + * @param newPath - The destination VFS path; must not already exist. + * @returns A promise that resolves once the rename is complete. + * @throws {VFSError} ENOENT when `oldPath` does not exist. + * @throws {VFSError} EEXIST when `newPath` already exists. + */ async rename(oldPath: string, newPath: string): Promise { await this.ensureInitialized() @@ -1683,15 +1954,27 @@ export class VirtualFileSystem implements IVirtualFileSystem { const newParentPath = this.getParentPath(newPath) if (oldParentPath !== newParentPath) { - // Remove from old parent + // Remove the OLD parent's containment edge(s) — by edge id, resolved from + // the graph's own adjacency (the removal law: a removal never requires + // reading the thing being removed). This step used to be skipped as "not + // critical", which left the moved entity a child of BOTH directories: + // readdir(oldDir) kept listing it, re-creating the old path showed the + // name twice, and tree-walking consumers saw the file in two places. if (oldParentPath) { const oldParentId = await this.pathResolver.resolve(oldParentPath) - // unrelate takes the relation ID, not params - need to find and remove relation - // For now, skip unrelate as it's not critical for rename + const staleEdges = await this.brain.related({ + from: oldParentId, + to: entityId, + type: VerbType.Contains + }) + for (const edge of staleEdges) { + await this.brain.unrelate(edge.id) + } } - // Add to new parent - if (newParentPath && newParentPath !== '/') { + // Add to the new parent. The root ('/') is a REAL parent — skipping it + // orphaned a move-to-root out of readdir('/') entirely. + if (newParentPath) { const newParentId = await this.pathResolver.resolve(newParentPath) await this.brain.relate({ from: newParentId, @@ -1732,6 +2015,110 @@ export class VirtualFileSystem implements IVirtualFileSystem { this.triggerWatchers(newPath, 'rename') } + /** + * Reconcile every VFS entity's containment edges against its canonical + * `metadata.path` — the path is the truth (maintained by write/rename); the + * `Contains` edges are a projection of it. Heals the "cosmetic ghost" class + * left by pre-fix renames that added the new parent's edge without removing + * the old one (an entity listed in TWO directories; a re-created old path + * showing its name twice), plus duplicate edges from the same parent left by + * concurrent writers. + * + * CONSERVATIVE by design: only VFS containment edges (subtype + * `'vfs-contains'` or `metadata.isVFS`) are ever touched — a user's own + * knowledge-graph `Contains` edge between the same entities is never + * removed. An entity whose expected parent path has no entity is logged + * loudly and left alone (never orphaned further). Operator-invoked via + * `brain.repairIndex()`. + * + * The canonical pagination walk (not an index query) is deliberate: this is + * a repair op — the projections are the thing under suspicion, so the walk + * reads the source of truth. + * + * @returns Counts of stale edges removed and missing expected edges restored. + */ + async repairContainment(): Promise<{ removed: number; restored: number }> { + await this.ensureInitialized() + + // Pass 1: canonical walk → every VFS entity's id + path. + const idByPath = new Map() + const vfsEntities: Array<{ id: string; path: string }> = [] + let cursor: string | undefined + for (;;) { + const page = await (this.brain as any).storage.getNounsWithPagination({ limit: 500, cursor }) + for (const noun of page.items) { + const meta = (noun as any).metadata ?? noun + const p = meta?.path + if (meta?.vfsType && typeof p === 'string') { + idByPath.set(p, noun.id) + vfsEntities.push({ id: noun.id, path: p }) + } + } + if (!page.hasMore) break + cursor = page.nextCursor + } + + let removed = 0 + let restored = 0 + for (const { id, path } of vfsEntities) { + if (path === '/') continue // the root has no parent + const expectedParentId = idByPath.get(this.getParentPath(path)) + if (!expectedParentId) { + console.warn( + `[VFS] repairContainment: no entity found for parent of ${path} — leaving its edges untouched.` + ) + continue + } + + const incoming = await this.brain.related({ to: id, type: VerbType.Contains }) + let expectedSeen = false + for (const edge of incoming) { + const isVfsEdge = edge.subtype === 'vfs-contains' || (edge.metadata as any)?.isVFS === true + if (!isVfsEdge) continue // never touch user knowledge edges + if (edge.from === expectedParentId && !expectedSeen) { + expectedSeen = true // keep exactly one correct edge + continue + } + // Stale parent (a pre-fix rename ghost) or a duplicate of the correct + // edge (concurrent-writer artifact) — remove it, loudly. + await this.brain.unrelate(edge.id) + removed++ + console.warn( + `[VFS] repairContainment: removed ${edge.from === expectedParentId ? 'duplicate' : 'stale'} ` + + `containment edge ${edge.from} -> ${id} (${path})` + ) + } + if (!expectedSeen) { + await this.brain.relate({ + from: expectedParentId, + to: id, + type: VerbType.Contains, + subtype: 'vfs-contains', + metadata: { isVFS: true } + }) + restored++ + console.warn(`[VFS] repairContainment: restored missing containment edge for ${path}`) + } + } + + return { removed, restored } + } + + /** + * Copy a file or directory to a new path. + * + * Files are duplicated as new entities (sharing content via the content-addressed + * blob store); directories are copied recursively. Unless `overwrite` is set, an + * existing destination is rejected. + * + * @param src - The source VFS path to copy from. + * @param dest - The destination VFS path to copy to. + * @param options - Copy options such as `overwrite`, `deepCopy`, `preserveVector`, + * and `preserveRelationships`. + * @returns A promise that resolves once the copy is complete. + * @throws {VFSError} ENOENT when `src` does not exist. + * @throws {VFSError} EEXIST when `dest` exists and `overwrite` is not set. + */ async copy(src: string, dest: string, options?: CopyOptions): Promise { await this.ensureInitialized() @@ -1919,6 +2306,20 @@ export class VirtualFileSystem implements IVirtualFileSystem { } } + /** + * Move a file or directory to a new path. + * + * Implemented as an in-place {@link rename} rather than copy-then-delete: on the + * content-addressed blob store a copy would share the source content hash, so + * deleting the source would orphan the destination. Renaming preserves the blob + * and entity id and handles directory descendants. + * + * @param src - The source VFS path to move from. + * @param dest - The destination VFS path to move to; must not already exist. + * @returns A promise that resolves once the move is complete. + * @throws {VFSError} ENOENT when `src` does not exist. + * @throws {VFSError} EEXIST when `dest` already exists. + */ async move(src: string, dest: string): Promise { await this.ensureInitialized() @@ -1932,6 +2333,18 @@ export class VirtualFileSystem implements IVirtualFileSystem { await this.rename(src, dest) } + /** + * Create a symbolic link at `path` that points to `target` (POSIX `symlink`). + * + * The link is stored as a dedicated `vfs-symlink` entity whose + * `metadata.symlinkTarget` records the target path; it is linked into its parent + * directory with a `Contains` edge. + * + * @param target - The path the symlink should resolve to. + * @param path - The VFS path at which to create the symlink; must not already exist. + * @returns A promise that resolves once the symlink is created. + * @throws {VFSError} EEXIST when `path` already exists. + */ async symlink(target: string, path: string): Promise { await this.ensureInitialized() @@ -1987,6 +2400,13 @@ export class VirtualFileSystem implements IVirtualFileSystem { await this.pathResolver.createPath(path, entity) } + /** + * Read the target of a symbolic link (POSIX `readlink`). + * + * @param path - The VFS path of the symlink. + * @returns The stored target path, or an empty string when no target is recorded. + * @throws {VFSError} EINVAL when `path` is not a symbolic link. + */ async readlink(path: string): Promise { await this.ensureInitialized() @@ -2001,6 +2421,17 @@ export class VirtualFileSystem implements IVirtualFileSystem { return entity.metadata.symlinkTarget || '' } + /** + * Resolve a path to its canonical form, following symbolic links (POSIX `realpath`). + * + * Symlinks are followed iteratively until a non-symlink target is reached, up to a + * fixed depth limit that guards against link cycles. + * + * @param path - The VFS path to resolve. + * @returns The fully resolved (non-symlink) path. + * @throws {VFSError} ENOENT when the path (or a link in the chain) does not exist. + * @throws {VFSError} ELOOP when too many symbolic links are encountered. + */ async realpath(path: string): Promise { await this.ensureInitialized() @@ -2030,12 +2461,27 @@ export class VirtualFileSystem implements IVirtualFileSystem { throw new VFSError(VFSErrorCode.ELOOP, `Too many symbolic links: ${path}`, path, 'realpath') } + /** + * Read a single extended attribute of a file or directory (POSIX `getxattr`). + * + * @param path - The VFS path to read. + * @param name - The extended-attribute key to fetch. + * @returns The attribute value, or `undefined` when the attribute is not set. + */ async getxattr(path: string, name: string): Promise { const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) return entity.metadata.attributes?.[name] } + /** + * Set a single extended attribute on a file or directory (POSIX `setxattr`). + * + * @param path - The VFS path to update. + * @param name - The extended-attribute key to set. + * @param value - The value to store under `name`. + * @returns A promise that resolves once the attribute is persisted. + */ async setxattr(path: string, name: string, value: any): Promise { await this.ensureInitialized() @@ -2059,12 +2505,25 @@ export class VirtualFileSystem implements IVirtualFileSystem { this.invalidateCaches(path) } + /** + * List the extended-attribute names set on a file or directory (POSIX `listxattr`). + * + * @param path - The VFS path to inspect. + * @returns The list of extended-attribute keys (empty when none are set). + */ async listxattr(path: string): Promise { const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) return Object.keys(entity.metadata.attributes || {}) } + /** + * Remove a single extended attribute from a file or directory (POSIX `removexattr`). + * + * @param path - The VFS path to update. + * @param name - The extended-attribute key to remove. + * @returns A promise that resolves once the attribute is removed. + */ async removexattr(path: string, name: string): Promise { await this.ensureInitialized() @@ -2089,6 +2548,17 @@ export class VirtualFileSystem implements IVirtualFileSystem { this.invalidateCaches(path) } + /** + * List the paths related to a file or directory via graph edges (both directions). + * + * Walks outgoing and incoming relationships and maps each connected entity back to + * its VFS path, recording the relationship type and direction. + * + * @param path - The VFS path whose relationships to list. + * @param options - Reserved relationship-filtering options. + * @returns Related entries, each with the related `path`, `relationship` type, and + * `direction` (`'from'` for outgoing edges, `'to'` for incoming). + */ async getRelated(path: string, options?: RelatedOptions): Promise { await this.ensureInitialized() @@ -2179,6 +2658,14 @@ export class VirtualFileSystem implements IVirtualFileSystem { return relationships } + /** + * Create a graph relationship between two VFS paths. + * + * @param from - The source VFS path. + * @param to - The target VFS path. + * @param type - The relationship (verb) type to create; coerced to a {@link VerbType}. + * @returns A promise that resolves once the relationship is created. + */ async addRelationship(from: string, to: string, type: string): Promise { await this.ensureInitialized() @@ -2198,6 +2685,18 @@ export class VirtualFileSystem implements IVirtualFileSystem { this.invalidateCaches(to) } + /** + * Remove a graph relationship between two VFS paths. + * + * Deletes outgoing edges from `from` to `to`; when `type` is given, only edges of + * that relationship type are removed. + * + * @param from - The source VFS path. + * @param to - The target VFS path. + * @param type - Optional relationship type to match; when omitted, all matching + * edges to `to` are removed. + * @returns A promise that resolves once the relationship(s) are removed. + */ async removeRelationship(from: string, to: string, type?: string): Promise { await this.ensureInitialized() @@ -2220,12 +2719,25 @@ export class VirtualFileSystem implements IVirtualFileSystem { this.invalidateCaches(to) } + /** + * Get the todo items attached to a file or directory. + * + * @param path - The VFS path whose todos to read. + * @returns The stored todo list, or `undefined` when none are attached. + */ async getTodos(path: string): Promise { const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) return entity.metadata.todos } + /** + * Replace the entire todo list attached to a file or directory. + * + * @param path - The VFS path to update. + * @param todos - The full set of todo items to store (overwrites any existing list). + * @returns A promise that resolves once the todos are persisted. + */ async setTodos(path: string, todos: VFSTodo[]): Promise { await this.ensureInitialized() @@ -2247,6 +2759,16 @@ export class VirtualFileSystem implements IVirtualFileSystem { this.invalidateCaches(path) } + /** + * Append a single todo item to a file or directory. + * + * Generates an id when one is not supplied and applies default `priority` + * (`'medium'`) and `status` (`'pending'`). + * + * @param path - The VFS path to attach the todo to. + * @param todo - The todo to add; `id`, `priority`, and `status` are optional. + * @returns A promise that resolves once the todo is persisted. + */ async addTodo(path: string, todo: VFSTodo): Promise { await this.ensureInitialized() @@ -2724,18 +3246,39 @@ export class VirtualFileSystem implements IVirtualFileSystem { return results } + /** + * Open a file as a Node-compatible readable stream. + * + * @param path - The VFS path of the file to read. + * @param options - Stream options (e.g. byte range, chunk size). + * @returns A readable stream that yields the file's bytes. + */ createReadStream(path: string, options?: ReadStreamOptions): NodeJS.ReadableStream { // Lazy import to avoid circular dependencies const { VFSReadStream } = require('./streams/VFSReadStream.js') return new VFSReadStream(this, path, options) } + /** + * Open a file as a Node-compatible writable stream. + * + * @param path - The VFS path of the file to write. + * @param options - Stream options controlling how buffered data is flushed. + * @returns A writable stream whose contents are persisted to the file on finish. + */ createWriteStream(path: string, options?: WriteStreamOptions): NodeJS.WritableStream { // Lazy import to avoid circular dependencies const { VFSWriteStream } = require('./streams/VFSWriteStream.js') return new VFSWriteStream(this, path, options) } + /** + * Watch a path for changes and invoke a listener on filesystem events. + * + * @param path - The VFS path to watch. + * @param listener - Called with the event type (`'rename'` or `'change'`) and the path. + * @returns A handle whose `close()` method deregisters the listener. + */ watch(path: string, listener: WatchListener): { close(): void } { if (!this.watchers.has(path)) { this.watchers.set(path, new Set()) @@ -2808,14 +3351,35 @@ export class VirtualFileSystem implements IVirtualFileSystem { yield* importer.importStream(sourcePath, options) } + /** + * Register a change listener for a path (Node `fs.watchFile`-style convenience). + * + * Delegates to {@link watch} without returning the watcher handle; remove the + * listener with {@link unwatchFile}. + * + * @param path - The VFS path to watch. + * @param listener - Called with the event type and path on each change. + */ watchFile(path: string, listener: WatchListener): void { this.watch(path, listener) } + /** + * Stop watching a path, removing all listeners registered for it. + * + * @param path - The VFS path to stop watching. + */ unwatchFile(path: string): void { this.watchers.delete(path) } + /** + * Resolve a path and return its backing {@link VFSEntity}. + * + * @param path - The VFS path to resolve. + * @returns The entity (with normalized VFS metadata) for the path. + * @throws {VFSError} ENOENT when the path cannot be resolved. + */ async getEntity(path: string): Promise { const entityId = await this.pathResolver.resolve(path) return this.getEntityById(entityId) diff --git a/src/vfs/index.ts b/src/vfs/index.ts deleted file mode 100644 index 16684214..00000000 --- a/src/vfs/index.ts +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Brainy Virtual Filesystem - * - * A simplified fs-compatible filesystem that stores data in Brainy - * Works across all storage adapters and scales to millions of files - */ - -// Core VFS -export { VirtualFileSystem } from './VirtualFileSystem.js' -export { PathResolver } from './PathResolver.js' -export * from './types.js' - -// MIME Type Detection -export { MimeTypeDetector, mimeDetector } from './MimeTypeDetector.js' - -// fs compatibility layer -export { FSCompat, createFS } from './FSCompat.js' - -// Directory import -export { DirectoryImporter } from './importers/DirectoryImporter.js' - -// Streaming -export { VFSReadStream } from './streams/VFSReadStream.js' -export { VFSWriteStream } from './streams/VFSWriteStream.js' - -// Convenience alias -export { VirtualFileSystem as VFS } from './VirtualFileSystem.js' \ No newline at end of file diff --git a/src/vfs/semantic/projections/TemporalProjection.ts b/src/vfs/semantic/projections/TemporalProjection.ts index 41b29b68..f002a07e 100644 --- a/src/vfs/semantic/projections/TemporalProjection.ts +++ b/src/vfs/semantic/projections/TemporalProjection.ts @@ -37,8 +37,8 @@ export class TemporalProjection extends BaseProjectionStrategy { where: { vfsType: 'file', modified: { - greaterEqual: startOfDay.getTime(), // BFO operator - lessEqual: endOfDay.getTime() // BFO operator + gte: startOfDay.getTime(), // BFO operator + lte: endOfDay.getTime() // BFO operator } }, limit: 1000 @@ -74,8 +74,8 @@ export class TemporalProjection extends BaseProjectionStrategy { where: { vfsType: 'file', modified: { - greaterEqual: startOfDay.getTime(), - lessEqual: endOfDay.getTime() + gte: startOfDay.getTime(), + lte: endOfDay.getTime() } }, limit: 1000 @@ -93,7 +93,7 @@ export class TemporalProjection extends BaseProjectionStrategy { const results = await brain.find({ where: { vfsType: 'file', - modified: { greaterEqual: oneDayAgo } + modified: { gte: oneDayAgo } }, limit }) diff --git a/src/vfs/streams/VFSReadStream.ts b/src/vfs/streams/VFSReadStream.ts index a2f3f7f2..f63697b1 100644 --- a/src/vfs/streams/VFSReadStream.ts +++ b/src/vfs/streams/VFSReadStream.ts @@ -25,7 +25,7 @@ export class VFSReadStream extends Readable { this.position = options.start || 0 } - async _read(size: number): Promise { + override async _read(size: number): Promise { try { // Lazy load entity if (!this.entity) { @@ -58,7 +58,7 @@ export class VFSReadStream extends Readable { } } - _destroy(error: Error | null, callback: (error?: Error | null) => void): void { + override _destroy(error: Error | null, callback: (error?: Error | null) => void): void { // Clean up resources this.entity = null this.data = null diff --git a/src/vfs/streams/VFSWriteStream.ts b/src/vfs/streams/VFSWriteStream.ts index 8402f81c..f9438121 100644 --- a/src/vfs/streams/VFSWriteStream.ts +++ b/src/vfs/streams/VFSWriteStream.ts @@ -28,7 +28,7 @@ export class VFSWriteStream extends Writable { } } - async _write( + override async _write( chunk: any, encoding: BufferEncoding, callback: (error?: Error | null) => void @@ -52,7 +52,7 @@ export class VFSWriteStream extends Writable { } } - async _final(callback: (error?: Error | null) => void): Promise { + override async _final(callback: (error?: Error | null) => void): Promise { try { await this._flush() callback() @@ -78,7 +78,7 @@ export class VFSWriteStream extends Writable { this.chunks = [] } - _destroy(error: Error | null, callback: (error?: Error | null) => void): void { + override _destroy(error: Error | null, callback: (error?: Error | null) => void): void { // Clean up resources this.chunks = [] this._closed = true diff --git a/src/vfs/types.ts b/src/vfs/types.ts index 54e9902b..9188476b 100644 --- a/src/vfs/types.ts +++ b/src/vfs/types.ts @@ -195,6 +195,36 @@ export interface ReadOptions { // VFS-specific options cache?: boolean // Use cache if available (default: true) decompress?: boolean // Auto-decompress (default: true) + + /** + * Read the file's content as it stood at a past generation (a number) or + * wall-clock instant (a Date) — the temporal read. Serves the exact bytes + * the file held then: the historical entity is materialized from the + * generation history, and its content blob is retention-protected (bytes + * referenced by any in-window generation are never reclaimed). Bounded by + * the retention window: reading past the compaction horizon throws. + * Bypasses the content cache. + */ + asOf?: number | Date +} + +/** + * One entry of a file's version history (see `vfs.history(path)`): the state + * the file held immediately after `generation` committed. + * `readFile(path, { asOf: generation })` returns these exact bytes while the + * generation remains inside the retention window. + */ +export interface FileVersion { + /** The generation whose write produced this version. */ + generation: number + /** Commit timestamp of that generation (ms since epoch). */ + timestamp: number + /** Content hash of this version's bytes in the content-addressed store. */ + hash: string + /** Content size in bytes. */ + size: number + /** Detected MIME type at that version. */ + mimeType?: string } export interface MkdirOptions { diff --git a/tests/api/batch-operations.test.ts b/tests/api/batch-operations.test.ts deleted file mode 100644 index be6673fc..00000000 --- a/tests/api/batch-operations.test.ts +++ /dev/null @@ -1,660 +0,0 @@ -/** - * Batch Operations Test Suite for Brainy v3.0 - * - * Comprehensive testing of batch operations including: - * - addMany: Bulk entity creation - * - removeMany: Bulk deletion - * - Performance validation at scale - * - Memory efficiency testing - * - Error recovery scenarios - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../src/brainy' -import { performance } from 'perf_hooks' - -interface BatchMetrics { - operation: string - totalItems: number - successCount: number - failureCount: number - duration: number - throughput: number - memoryUsed: number - errors: string[] -} - -class BatchMetricsCollector { - private metrics: BatchMetrics[] = [] - - recordBatch( - operation: string, - totalItems: number, - results: any[], - duration: number, - memoryUsed: number - ): BatchMetrics { - const successCount = results.filter(r => - r?.status === 'fulfilled' || r === true || (r && !r.error) - ).length - const failureCount = totalItems - successCount - const errors = results - .filter(r => r?.status === 'rejected' || r?.error) - .map(r => r?.reason?.message || r?.error || 'Unknown error') - .slice(0, 5) // Limit to first 5 errors - - const metric: BatchMetrics = { - operation, - totalItems, - successCount, - failureCount, - duration, - throughput: (totalItems / duration) * 1000, - memoryUsed, - errors - } - - this.metrics.push(metric) - return metric - } - - getMetrics(): BatchMetrics[] { - return this.metrics - } - - generateReport(): void { - console.log('\n=== Batch Operations Performance Report ===\n') - console.table(this.metrics.map(m => ({ - Operation: m.operation, - 'Total Items': m.totalItems, - 'Success': `${m.successCount}/${m.totalItems} (${((m.successCount/m.totalItems)*100).toFixed(1)}%)`, - 'Duration (ms)': m.duration.toFixed(2), - 'Throughput (items/s)': m.throughput.toFixed(0), - 'Memory (MB)': (m.memoryUsed / 1024 / 1024).toFixed(2), - 'Errors': m.errors.length > 0 ? m.errors[0] : 'None' - }))) - - // Summary statistics - const totalOps = this.metrics.reduce((sum, m) => sum + m.totalItems, 0) - const totalSuccess = this.metrics.reduce((sum, m) => sum + m.successCount, 0) - const avgThroughput = this.metrics.reduce((sum, m) => sum + m.throughput, 0) / this.metrics.length - - console.log('\nSummary:') - console.log(`Total Operations: ${totalOps}`) - console.log(`Success Rate: ${((totalSuccess/totalOps)*100).toFixed(2)}%`) - console.log(`Average Throughput: ${avgThroughput.toFixed(0)} items/sec`) - } -} - -describe('Batch Operations - Public API Testing', () => { - let brainy: Brainy - let metricsCollector: BatchMetricsCollector - - beforeEach(async () => { - brainy = new Brainy({ requireSubtype: false, - storage: { type: 'memory' } - }) - await brainy.init() - metricsCollector = new BatchMetricsCollector() - }) - - afterEach(async () => { - await brainy.close() - if (global.gc) global.gc() - }) - - describe('addMany - Bulk Entity Creation', () => { - it('should add 100 entities efficiently', async () => { - const entities = Array(100).fill(null).map((_, i) => ({ - data: `Batch entity ${i}: Test content for bulk operations`, - type: 'document' as const, - metadata: { - batchIndex: i, - batchId: 'batch-100', - timestamp: Date.now() - } - })) - - const startMemory = process.memoryUsage().heapUsed - const startTime = performance.now() - - const result = await brainy.addMany({ items: entities }) - - const duration = performance.now() - startTime - const memoryUsed = process.memoryUsage().heapUsed - startMemory - - const metric = metricsCollector.recordBatch( - 'addMany-100', - entities.length, - result.successful, - duration, - memoryUsed - ) - - expect(result.successful).toHaveLength(100) - expect(result.failed).toHaveLength(0) - expect(metric.throughput).toBeGreaterThan(100) // > 100 items/sec - }) - - it('should handle 1,000 entities with proper batching', async () => { - const entities = Array(1000).fill(null).map((_, i) => ({ - data: `Entity ${i}: ${' '.repeat(100)}`, // ~100 bytes each - type: 'document' as const, - metadata: { - index: i, - category: `cat-${i % 10}`, - tags: [`tag-${i % 5}`, `tag-${i % 7}`] - } - })) - - const startMemory = process.memoryUsage().heapUsed - const startTime = performance.now() - - const result = await brainy.addMany({ - items: entities, - chunkSize: 100, // Process in batches - parallel: true - }) - - const duration = performance.now() - startTime - const memoryUsed = process.memoryUsage().heapUsed - startMemory - - metricsCollector.recordBatch( - 'addMany-1k', - entities.length, - result.successful, - duration, - memoryUsed - ) - - expect(result.successful.length).toBe(1000) - expect(result.failed.length).toBe(0) - expect(duration).toBeLessThan(10000) // < 10 seconds - - // Verify data integrity - const sampleIds = result.successful.slice(0, 10) - for (const id of sampleIds) { - const entity = await brainy.get(id) - expect(entity).toBeDefined() - } - }) - - it('should handle 10,000 entities with memory efficiency', async () => { - const chunkSize = 1000 - const totalEntities = 10000 - let allSuccessful: string[] = [] - let allFailed: any[] = [] - - const startMemory = process.memoryUsage().heapUsed - const startTime = performance.now() - - // Process in chunks to avoid memory issues - for (let chunk = 0; chunk < totalEntities; chunk += chunkSize) { - const entities = Array(Math.min(chunkSize, totalEntities - chunk)) - .fill(null) - .map((_, i) => ({ - data: `Chunk entity ${chunk + i}`, - type: 'document' as const, - metadata: { globalIndex: chunk + i } - })) - - const result = await brainy.addMany({ items: entities }) - allSuccessful.push(...result.successful) - allFailed.push(...result.failed) - - // Check memory usage - const currentMemory = process.memoryUsage().heapUsed - const memoryGrowth = currentMemory - startMemory - expect(memoryGrowth).toBeLessThan(500 * 1024 * 1024) // < 500MB total - } - - const duration = performance.now() - startTime - const finalMemory = process.memoryUsage().heapUsed - startMemory - - metricsCollector.recordBatch( - 'addMany-10k', - totalEntities, - allSuccessful, - duration, - finalMemory - ) - - expect(allSuccessful.length).toBe(10000) - expect(allFailed.length).toBe(0) - }) - - it('should handle partial failures gracefully', async () => { - const entities = [ - { data: 'Valid entity 1', type: 'document' as const }, - { data: '', type: 'document' as const }, // Invalid: empty data - { data: 'Valid entity 2', type: 'document' as const }, - { data: null as any, type: 'document' as const }, // Invalid: null data - { data: 'Valid entity 3', type: 'document' as const }, - ] - - const result = await brainy.addMany({ - items: entities, - continueOnError: true - }) - - // Should process valid entities even if some fail - expect(result.successful.length).toBeGreaterThanOrEqual(3) - expect(result.failed.length).toBeGreaterThanOrEqual(0) // May or may not validate - - // Verify valid entities were added - for (const id of result.successful) { - const entity = await brainy.get(id) - expect(entity).toBeDefined() - } - }) - - it('should support concurrent batch operations', async () => { - const batches = Array(5).fill(null).map((_, batchIdx) => - Array(100).fill(null).map((_, i) => ({ - data: `Batch ${batchIdx} Entity ${i}`, - type: 'document' as const, - metadata: { batchId: batchIdx, index: i } - })) - ) - - const startTime = performance.now() - - // Execute batches concurrently - const results = await Promise.all( - batches.map(entities => brainy.addMany({ items: entities })) - ) - - const duration = performance.now() - startTime - - expect(results).toHaveLength(5) - const totalSuccess = results.reduce((sum, r) => sum + r.successful.length, 0) - expect(totalSuccess).toBe(500) - expect(duration).toBeLessThan(5000) // Concurrent should be faster - }) - }) - - let testIds: string[] = [] - - beforeEach(async () => { - // Create test entities - const entities = Array(100).fill(null).map((_, i) => ({ - data: `Original content ${i}`, - type: 'document' as const, - metadata: { version: 1, index: i } - })) - - const result = await brainy.addMany({ items: entities }) - testIds = result.successful - }) - - it('should update multiple entities by IDs', async () => { - const updates = testIds.slice(0, 50).map(id => ({ - id, - updates: { - metadata: { - version: 2, - updatedAt: Date.now() - } - } - })) - - const startTime = performance.now() - - const results = await Promise.all( - updates.map(u => brainy.update({ id: u.id, ...u.updates })) - ) - - const duration = performance.now() - startTime - - metricsCollector.recordBatch( - updates.length, - results, - duration, - 0 - ) - - // Verify updates - const sample = await brainy.get(testIds[0]) - expect(sample?.metadata?.version).toBe(2) - }) - - it('should update entities matching criteria', async () => { - // Update all entities with index < 20 - const targetIds = testIds.slice(0, 20) - - const startTime = performance.now() - - const results = await Promise.all( - targetIds.map(id => - brainy.update({ - id, - metadata: { - status: 'updated', - processedAt: Date.now() - } - }) - ) - ) - - const duration = performance.now() - startTime - - metricsCollector.recordBatch( - targetIds.length, - results, - duration, - 0 - ) - - // Verify updates - for (const id of targetIds.slice(0, 5)) { - const entity = await brainy.get(id) - expect(entity?.metadata?.status).toBe('updated') - } - }) - }) - - describe('removeMany - Bulk Deletion', () => { - let testIds: string[] = [] - - beforeEach(async () => { - // Create test entities - const entities = Array(200).fill(null).map((_, i) => ({ - data: `Delete test ${i}`, - type: 'document' as const, - metadata: { deleteGroup: i % 4 } - })) - - const result = await brainy.addMany({ items: entities }) - testIds = result.successful - }) - - it('should delete multiple entities by IDs', async () => { - const toDelete = testIds.slice(0, 100) - - const startMemory = process.memoryUsage().heapUsed - const startTime = performance.now() - - // Use removeMany if available, otherwise individual deletes - let results: any[] - if ('removeMany' in brainy && typeof brainy.removeMany === 'function') { - const result = await brainy.removeMany({ ids: toDelete }) - results = result.successful - } else { - results = await Promise.all( - toDelete.map(id => brainy.remove(id)) - ) - } - - const duration = performance.now() - startTime - const memoryUsed = process.memoryUsage().heapUsed - startMemory - - metricsCollector.recordBatch( - 'removeMany-100', - toDelete.length, - results, - duration, - memoryUsed - ) - - // Verify deletion - for (const id of toDelete.slice(0, 10)) { - const entity = await brainy.get(id) - expect(entity).toBeNull() - } - - // Remaining entities should still exist - const remaining = testIds.slice(100, 110) - for (const id of remaining) { - const entity = await brainy.get(id) - expect(entity).toBeDefined() - } - }) - - it('should handle large-scale deletion efficiently', async () => { - // Create a large dataset - const largeDataset = Array(1000).fill(null).map((_, i) => ({ - data: `Large scale delete test ${i}`, - type: 'document' as const - })) - - const addResult = await brainy.addMany({ items: largeDataset }) - const idsToDelete = addResult.successful - - const startTime = performance.now() - - // Delete in batches - const batchSize = 100 - for (let i = 0; i < idsToDelete.length; i += batchSize) { - const batch = idsToDelete.slice(i, i + batchSize) - await Promise.all(batch.map(id => brainy.remove(id))) - } - - const duration = performance.now() - startTime - - metricsCollector.recordBatch( - 'removeMany-1k', - idsToDelete.length, - idsToDelete.map(() => true), - duration, - 0 - ) - - expect(duration).toBeLessThan(10000) // < 10 seconds - }) - }) - - let nodeIds: string[] = [] - - beforeEach(async () => { - // Create nodes for relationships - const nodes = Array(50).fill(null).map((_, i) => ({ - data: `Node ${i}`, - type: 'thing' as const, - metadata: { nodeIndex: i } - })) - - const result = await brainy.addMany({ items: nodes }) - nodeIds = result.successful - }) - - it('should create multiple relationships efficiently', async () => { - const relationships: Array<{from: string, to: string, type: string}> = [] - - // Create a connected graph - for (let i = 0; i < nodeIds.length - 1; i++) { - relationships.push({ - from: nodeIds[i], - to: nodeIds[i + 1], - type: 'connected_to' - }) - } - - const startTime = performance.now() - - const results = await Promise.all( - relationships.map(r => - brainy.relate({ from: r.from, to: r.to, type: r.type as any }) - ) - ) - - const duration = performance.now() - startTime - - metricsCollector.recordBatch( - relationships.length, - results, - duration, - 0 - ) - - expect(results.every(r => typeof r === 'string')).toBe(true) // relate returns ID - expect(duration).toBeLessThan(5000) - }) - - it('should create complex graph structures', async () => { - // Create a fully connected subgraph (first 10 nodes) - const subgraph = nodeIds.slice(0, 10) - const relationships: Array<{from: string, to: string}> = [] - - for (let i = 0; i < subgraph.length; i++) { - for (let j = i + 1; j < subgraph.length; j++) { - relationships.push({ - from: subgraph[i], - to: subgraph[j] - }) - } - } - - const startTime = performance.now() - - const results = await Promise.all( - relationships.map(r => - brainy.relate({ from: r.from, to: r.to, type: 'relatedTo' }) - ) - ) - - const duration = performance.now() - startTime - - metricsCollector.recordBatch( - relationships.length, - results, - duration, - 0 - ) - - expect(relationships.length).toBe(45) // 10 choose 2 - expect(results.every(r => typeof r === 'string')).toBe(true) // relate returns ID - }) - }) - - describe('Mixed Batch Operations', () => { - it('should handle mixed operation types concurrently', async () => { - // Prepare different operation types - const addOps = Array(100).fill(null).map((_, i) => ({ - data: `Mixed add ${i}`, - type: 'document' as const - })) - - // Add initial entities for update/delete - const setupResult = await brainy.addMany({ - items: Array(100).fill(null).map((_, i) => ({ - data: `Setup ${i}`, - type: 'document' as const - })) - }) - - const updateIds = setupResult.successful.slice(0, 50) - const deleteIds = setupResult.successful.slice(50, 100) - - const startTime = performance.now() - - // Execute all operations concurrently - const [addResults, updateResults, deleteResults] = await Promise.all([ - brainy.addMany({ items: addOps }), - Promise.all(updateIds.map(id => - brainy.update({ id, metadata: { updated: true } }) - )), - Promise.all(deleteIds.map(id => brainy.remove(id))) - ]) - - const duration = performance.now() - startTime - - console.log(`Mixed operations completed in ${duration.toFixed(2)}ms`) - - expect(addResults.successful.length).toBe(100) - expect(updateResults.length).toBe(50) - // Delete returns void, so just check length - expect(deleteResults).toHaveLength(50) - }) - }) - - describe('Error Recovery', () => { - it('should recover from transient failures', async () => { - let attemptCount = 0 - const entities = Array(10).fill(null).map((_, i) => ({ - data: `Retry test ${i}`, - type: 'document' as const - })) - - // Simulate transient failures - const originalAdd = brainy.add.bind(brainy) - brainy.add = async function(params: any) { - attemptCount++ - if (attemptCount % 3 === 0) { - throw new Error('Transient failure') - } - return originalAdd(params) - } - - const results: any[] = [] - for (const entity of entities) { - let retries = 0 - while (retries < 3) { - try { - const id = await brainy.add(entity) - results.push({ status: 'fulfilled', value: id }) - break - } catch (error) { - retries++ - if (retries === 3) { - results.push({ status: 'rejected', reason: error }) - } - } - } - } - - const successful = results.filter(r => r.status === 'fulfilled') - expect(successful.length).toBeGreaterThanOrEqual(6) // Most should succeed with retry - }) - - it('should handle memory pressure gracefully', async () => { - const largeEntities = Array(100).fill(null).map((_, i) => ({ - data: 'x'.repeat(100000), // 100KB each = 10MB total - type: 'document' as const, - metadata: { index: i } - })) - - const startMemory = process.memoryUsage().heapUsed - - // Process with memory monitoring - const batchSize = 10 - const results: any[] = [] - - for (let i = 0; i < largeEntities.length; i += batchSize) { - const batch = largeEntities.slice(i, i + batchSize) - - // Check memory before processing - const currentMemory = process.memoryUsage().heapUsed - const memoryUsed = currentMemory - startMemory - - if (memoryUsed > 100 * 1024 * 1024) { // If > 100MB - if (global.gc) global.gc() // Force GC if available - } - - const batchResult = await brainy.addMany({ items: batch }) - results.push(...batchResult.successful) - } - - expect(results.length).toBe(100) - - const finalMemory = process.memoryUsage().heapUsed - startMemory - expect(finalMemory).toBeLessThan(200 * 1024 * 1024) // Should stay under 200MB - }) - }) - - describe('Performance Report', () => { - it('should generate comprehensive batch operations report', async () => { - metricsCollector.generateReport() - - const metrics = metricsCollector.getMetrics() - expect(metrics.length).toBeGreaterThan(0) - - // Validate performance targets - for (const metric of metrics) { - expect(metric.successCount).toBeGreaterThan(0) - if (metric.operation.includes('100')) { - expect(metric.throughput).toBeGreaterThan(50) // At least 50 items/sec - } - } - }) - }) -}) \ No newline at end of file diff --git a/tests/api/crud-operations-enhanced.test.ts b/tests/api/crud-operations-enhanced.test.ts deleted file mode 100644 index b6070d15..00000000 --- a/tests/api/crud-operations-enhanced.test.ts +++ /dev/null @@ -1,764 +0,0 @@ -/** - * Enhanced CRUD Operations Test Suite for Brainy v3.0 - * - * Comprehensive validation of all public API CRUD operations with: - * - Exhaustive edge case testing - * - Performance benchmarking - * - Memory leak detection - * - Concurrent operation validation - * - Large-scale data handling - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../src/brainy' -import { performance } from 'perf_hooks' - -interface PerformanceMetrics { - operation: string - itemCount: number - duration: number - memoryUsed: number - itemsPerSecond: number - percentiles?: { - p50: number - p95: number - p99: number - } -} - -class TestMetricsCollector { - private metrics: PerformanceMetrics[] = [] - private initialMemory: number = 0 - private latencies: Map = new Map() - - startOperation(operation: string, itemCount: number = 1): number { - this.initialMemory = process.memoryUsage().heapUsed - return performance.now() - } - - endOperation(operation: string, itemCount: number, startTime: number): PerformanceMetrics { - const duration = performance.now() - startTime - const memoryUsed = process.memoryUsage().heapUsed - this.initialMemory - const itemsPerSecond = (itemCount / duration) * 1000 - - // Track latencies for percentile calculations - if (!this.latencies.has(operation)) { - this.latencies.set(operation, []) - } - this.latencies.get(operation)!.push(duration / itemCount) - - const metric: PerformanceMetrics = { - operation, - itemCount, - duration, - memoryUsed, - itemsPerSecond - } - - this.metrics.push(metric) - return metric - } - - calculatePercentiles(operation: string): { p50: number; p95: number; p99: number } | undefined { - const latencies = this.latencies.get(operation) - if (!latencies || latencies.length === 0) return undefined - - const sorted = [...latencies].sort((a, b) => a - b) - return { - p50: sorted[Math.floor(sorted.length * 0.5)], - p95: sorted[Math.floor(sorted.length * 0.95)], - p99: sorted[Math.floor(sorted.length * 0.99)] - } - } - - getMetrics(): PerformanceMetrics[] { - return this.metrics - } - - async reportMetrics(): Promise { - console.log('\n=== Performance Metrics Report ===\n') - - // Add percentiles to metrics - for (const metric of this.metrics) { - metric.percentiles = this.calculatePercentiles(metric.operation) - } - - console.table(this.metrics.map(m => ({ - Operation: m.operation, - 'Items': m.itemCount, - 'Duration (ms)': m.duration.toFixed(2), - 'Items/sec': m.itemsPerSecond.toFixed(0), - 'Memory (MB)': (m.memoryUsed / 1024 / 1024).toFixed(2), - 'P50 (ms)': m.percentiles?.p50.toFixed(2) || 'N/A', - 'P95 (ms)': m.percentiles?.p95.toFixed(2) || 'N/A', - 'P99 (ms)': m.percentiles?.p99.toFixed(2) || 'N/A' - }))) - } -} - -describe('Enhanced CRUD Operations - Public API Validation', () => { - let brainy: Brainy - let metricsCollector: TestMetricsCollector - - beforeEach(async () => { - brainy = new Brainy({ requireSubtype: false, - storage: { type: 'memory' } - }) - await brainy.init() - metricsCollector = new TestMetricsCollector() - }) - - afterEach(async () => { - await brainy.close() - // Force garbage collection if available - if (global.gc) { - global.gc() - } - }) - - describe('ADD Operations - Comprehensive Testing', () => { - describe('Basic Add Functionality', () => { - it('should add entity with all supported data types', async () => { - const testData = { - text: 'This is a comprehensive test document about AI and machine learning', - metadata: { - title: 'Test Document', - author: 'Test Suite', - version: '1.0.0', - tags: ['test', 'validation', 'comprehensive'], - stats: { - words: 10, - characters: 68, - readingTime: 2 - }, - nullField: null, - booleanField: true, - numberField: 42.5, - dateField: new Date().toISOString() - } - } - - const startTime = metricsCollector.startOperation('add-single') - const id = await brainy.add({ - data: testData.text, - type: 'document' as const, - metadata: testData.metadata - }) - const metric = metricsCollector.endOperation('add-single', 1, startTime) - - expect(id).toBeDefined() - expect(typeof id).toBe('string') - expect(id.length).toBeGreaterThan(0) - expect(metric.duration).toBeLessThan(50) // 50ms SLA for single add - - // Verify stored data - const retrieved = await brainy.get(id) - expect(retrieved).toBeDefined() - expect(retrieved?.metadata?.title).toBe('Test Document') - expect(retrieved?.type).toBe('document') - expect(retrieved?.metadata?.title).toBe('Test Document') - }) - - it('should handle Unicode and special characters correctly', async () => { - const specialCases = [ - { data: '😀🎉🚀 Emoji test document', desc: 'Emoji support' }, - { data: '中文测试文档 Chinese text', desc: 'Chinese characters' }, - { data: 'العربية اختبار Arabic text', desc: 'Arabic script' }, - { data: 'Русский текст тестирование', desc: 'Cyrillic script' }, - { data: '日本語のテストドキュメント', desc: 'Japanese characters' }, - { data: 'Line\nBreak\rReturn\tTab test', desc: 'Control characters' }, - { data: '', desc: 'HTML injection attempt' }, - { data: '../../etc/passwd traversal', desc: 'Path traversal attempt' }, - { data: 'NULL\x00byte test', desc: 'Null byte injection' }, - { data: '𝓣𝓱𝓮 𝓺𝓾𝓲𝓬𝓴 𝓫𝓻𝓸𝔀𝓷 𝓯𝓸𝔁', desc: 'Mathematical alphanumeric symbols' } - ] - - for (const testCase of specialCases) { - const id = await brainy.add({ - data: testCase.data, - type: 'document' as const, - metadata: { description: testCase.desc } - }) - - const retrieved = await brainy.get(id) - expect(retrieved?.data).toBe(testCase.data) - expect(retrieved?.metadata?.description).toBe(testCase.desc) - } - }) - - it('should auto-generate unique IDs when not provided', async () => { - const ids = new Set() - const count = 100 - - for (let i = 0; i < count; i++) { - const id = await brainy.add({ - data: `Auto-generated ID test ${i}`, - type: 'test' - }) - ids.add(id) - } - - expect(ids.size).toBe(count) // All IDs must be unique - - // Verify ID format - for (const id of ids) { - expect(id).toMatch(/^[a-zA-Z0-9_-]+$/) // Valid ID characters - expect(id.length).toBeGreaterThanOrEqual(8) // Reasonable length - expect(id.length).toBeLessThanOrEqual(64) // Not too long - } - }) - }) - - describe('Bulk Add Operations', () => { - it('should efficiently handle 1,000 entities', async () => { - const entities = Array(1000).fill(null).map((_, i) => ({ - data: `Test document ${i}: This is content for testing bulk operations at scale`, - type: 'document', - metadata: { - index: i, - category: `category-${i % 10}`, - score: Math.random() * 100, - tags: [`tag-${i % 5}`, `tag-${i % 7}`] - } - })) - - const startTime = metricsCollector.startOperation('add-bulk-1k', 1000) - const ids: string[] = [] - - // Add in batches for better performance - const batchSize = 100 - for (let i = 0; i < entities.length; i += batchSize) { - const batch = entities.slice(i, i + batchSize) - const batchIds = await Promise.all( - batch.map(e => brainy.add(e)) - ) - ids.push(...batchIds) - } - - const metric = metricsCollector.endOperation('add-bulk-1k', 1000, startTime) - - expect(ids).toHaveLength(1000) - expect(new Set(ids).size).toBe(1000) // All unique - expect(metric.itemsPerSecond).toBeGreaterThan(100) // At least 100 items/sec - expect(metric.memoryUsed).toBeLessThan(100 * 1024 * 1024) // Less than 100MB - - // Verify data integrity with random sampling - for (let i = 0; i < 10; i++) { - const randomIdx = Math.floor(Math.random() * 1000) - const entity = await brainy.get(ids[randomIdx]) - expect(entity?.metadata?.index).toBe(randomIdx) - } - }) - - it('should handle 10,000 entities with memory efficiency', async () => { - const startMemory = process.memoryUsage().heapUsed - const count = 10000 - const dataSize = 500 // bytes per entity - - const startTime = metricsCollector.startOperation('add-bulk-10k', count) - const ids: string[] = [] - - // Stream-like processing - const batchSize = 500 - for (let i = 0; i < count; i += batchSize) { - const batch = Array(Math.min(batchSize, count - i)).fill(null).map((_, j) => ({ - data: 'x'.repeat(dataSize) + ` Entity ${i + j}`, - type: 'bulk-test', - metadata: { index: i + j } - })) - - const batchIds = await Promise.all( - batch.map(e => brainy.add(e)) - ) - ids.push(...batchIds) - - // Check memory periodically - if (i % 2000 === 0 && i > 0) { - const currentMemory = process.memoryUsage().heapUsed - const memoryGrowth = currentMemory - startMemory - expect(memoryGrowth).toBeLessThan(200 * 1024 * 1024) // Less than 200MB - } - } - - const metric = metricsCollector.endOperation('add-bulk-10k', count, startTime) - - expect(ids).toHaveLength(count) - expect(metric.duration).toBeLessThan(60000) // Complete within 60 seconds - expect(metric.itemsPerSecond).toBeGreaterThan(150) // At least 150 items/sec - }) - - it('should detect and prevent memory leaks', async () => { - const iterations = 5 - const memoryDeltas: number[] = [] - - for (let iter = 0; iter < iterations; iter++) { - if (global.gc) global.gc() // Force GC if available - - const beforeMemory = process.memoryUsage().heapUsed - - // Add and delete 500 entities - const ids: string[] = [] - for (let i = 0; i < 500; i++) { - const id = await brainy.add({ - data: 'x'.repeat(1000), // 1KB per entity - type: 'leak-test', - metadata: { iteration: iter, index: i } - }) - ids.push(id) - } - - // Delete all entities - for (const id of ids) { - await brainy.remove(id) - } - - if (global.gc) global.gc() - const afterMemory = process.memoryUsage().heapUsed - memoryDeltas.push(afterMemory - beforeMemory) - } - - // Memory should stabilize, not continuously grow - const avgFirstTwo = (memoryDeltas[0] + memoryDeltas[1]) / 2 - const avgLastTwo = (memoryDeltas[3] + memoryDeltas[4]) / 2 - const growth = avgLastTwo - avgFirstTwo - - expect(growth).toBeLessThan(10 * 1024 * 1024) // Less than 10MB growth - }) - }) - - describe('Concurrent Operations', () => { - it('should handle 100 concurrent adds without race conditions', async () => { - const concurrentCount = 100 - const operations = Array(concurrentCount).fill(null).map((_, i) => ({ - data: `Concurrent test document ${i}`, - type: 'concurrent-test', - metadata: { - index: i, - timestamp: Date.now(), - random: Math.random() - } - })) - - const startTime = performance.now() - const ids = await Promise.all( - operations.map(op => brainy.add(op)) - ) - const duration = performance.now() - startTime - - expect(ids).toHaveLength(concurrentCount) - expect(new Set(ids).size).toBe(concurrentCount) // All unique - expect(duration).toBeLessThan(2000) // Complete within 2 seconds - - // Verify data integrity - for (let i = 0; i < 10; i++) { - const randomIdx = Math.floor(Math.random() * concurrentCount) - const entity = await brainy.get(ids[randomIdx]) - expect(entity?.metadata?.index).toBe(randomIdx) - } - }) - - it('should handle write conflicts gracefully', async () => { - // Test optimistic concurrency control - const id = await brainy.add({ - data: 'Original content', - type: 'conflict-test', - metadata: { version: 1 } - }) - - // Simulate concurrent updates - const updates = Array(10).fill(null).map((_, i) => - brainy.update(id, { - metadata: { version: i + 2, updatedBy: `process-${i}` } - }) - ) - - const results = await Promise.allSettled(updates) - - // At least one should succeed - const succeeded = results.filter(r => r.status === 'fulfilled') - expect(succeeded.length).toBeGreaterThanOrEqual(1) - - // Final state should be consistent - const final = await brainy.get(id) - expect(final?.metadata?.version).toBeGreaterThanOrEqual(2) - }) - }) - - describe('Input Validation', () => { - it('should reject invalid input types', async () => { - const invalidCases = [ - { data: null, desc: 'null data' }, - { data: undefined, desc: 'undefined data' }, - { data: '', desc: 'empty string' }, - { data: 42, desc: 'number instead of string' }, - { data: true, desc: 'boolean instead of string' }, - { data: [], desc: 'array instead of string' }, - { data: {}, desc: 'object instead of string' } - ] - - for (const testCase of invalidCases) { - await expect( - brainy.add({ - data: testCase.data as any, - type: 'test' - }) - ).rejects.toThrow() - } - }) - - it('should handle extremely large entities', async () => { - const largeData = 'x'.repeat(5 * 1024 * 1024) // 5MB - - const startTime = performance.now() - const id = await brainy.add({ - data: largeData, - type: 'large-entity', - metadata: { size: largeData.length } - }) - const duration = performance.now() - startTime - - expect(id).toBeDefined() - expect(duration).toBeLessThan(5000) // Handle within 5 seconds - - const retrieved = await brainy.get(id) - expect(retrieved?.data.length).toBe(largeData.length) - }) - - it('should sanitize and validate metadata fields', async () => { - const metadata = { - 'normal-field': 'value1', - '$special.field': 'value2', - '__proto__': 'attempt-pollution', - 'constructor': 'attempt-override', - '../../../etc/passwd': 'path-traversal', - '': 'xss-attempt' - } - - const id = await brainy.add({ - data: 'Test document with suspicious metadata', - type: 'security-test', - metadata - }) - - const retrieved = await brainy.get(id) - expect(retrieved).toBeDefined() - - // Verify prototype pollution prevention - expect(Object.prototype).not.toHaveProperty('attempt-pollution') - expect(Object.constructor).not.toBe('attempt-override') - }) - - it('should handle circular references gracefully', async () => { - const metadata: any = { - name: 'Circular test', - level: 1 - } - metadata.self = metadata // Create circular reference - - // Should either serialize correctly or throw meaningful error - try { - const id = await brainy.add({ - data: 'Document with circular metadata', - type: 'circular-test', - metadata - }) - - const retrieved = await brainy.get(id) - expect(retrieved).toBeDefined() - // Circular reference should be handled (removed or converted) - expect(retrieved?.metadata?.self).not.toBe(retrieved?.metadata) - } catch (error: any) { - expect(error.message).toMatch(/circular|cyclic/i) - } - }) - }) - - describe('Performance SLAs', () => { - it('should meet latency SLAs for single operations', async () => { - const operations = 100 - const latencies: number[] = [] - - for (let i = 0; i < operations; i++) { - const start = performance.now() - await brainy.add({ - data: `Performance test document ${i}`, - type: 'perf-test', - metadata: { index: i } - }) - latencies.push(performance.now() - start) - } - - // Calculate percentiles - latencies.sort((a, b) => a - b) - const p50 = latencies[Math.floor(latencies.length * 0.5)] - const p95 = latencies[Math.floor(latencies.length * 0.95)] - const p99 = latencies[Math.floor(latencies.length * 0.99)] - - console.log(`Add Latencies - P50: ${p50.toFixed(2)}ms, P95: ${p95.toFixed(2)}ms, P99: ${p99.toFixed(2)}ms`) - - expect(p50).toBeLessThan(10) // P50 < 10ms - expect(p95).toBeLessThan(50) // P95 < 50ms - expect(p99).toBeLessThan(100) // P99 < 100ms - }) - - it('should maintain throughput under sustained load', async () => { - const duration = 5000 // 5 seconds - const startTime = performance.now() - let operationCount = 0 - - while (performance.now() - startTime < duration) { - await brainy.add({ - data: `Sustained load test ${operationCount}`, - type: 'load-test', - metadata: { timestamp: Date.now() } - }) - operationCount++ - } - - const actualDuration = performance.now() - startTime - const throughput = (operationCount / actualDuration) * 1000 - - console.log(`Sustained Load - Ops: ${operationCount}, Throughput: ${throughput.toFixed(2)} ops/sec`) - - expect(throughput).toBeGreaterThan(50) // At least 50 ops/sec sustained - }) - }) - }) - - describe('GET Operations - Comprehensive Testing', () => { - let testIds: string[] = [] - - beforeEach(async () => { - // Seed test data - testIds = [] - for (let i = 0; i < 10; i++) { - const id = await brainy.add({ - data: `Test document ${i}`, - type: 'test', - metadata: { index: i, category: `cat-${i % 3}` } - }) - testIds.push(id) - } - }) - - it('should retrieve existing entities', async () => { - const startTime = metricsCollector.startOperation('get-single') - const entity = await brainy.get(testIds[0]) - metricsCollector.endOperation('get-single', 1, startTime) - - expect(entity).toBeDefined() - expect(entity?.data).toBe('Test document 0') - expect(entity?.metadata?.index).toBe(0) - }) - - it('should return null for non-existent IDs', async () => { - const result = await brainy.get('non-existent-id-12345') - expect(result).toBeNull() - }) - - it('should handle batch retrieval efficiently', async () => { - const startTime = metricsCollector.startOperation('get-batch', testIds.length) - const entities = await Promise.all( - testIds.map(id => brainy.get(id)) - ) - const metric = metricsCollector.endOperation('get-batch', testIds.length, startTime) - - expect(entities.every(e => e !== null)).toBe(true) - expect(metric.itemsPerSecond).toBeGreaterThan(100) - }) - }) - - describe('UPDATE Operations - Comprehensive Testing', () => { - let testId: string - - beforeEach(async () => { - testId = await brainy.add({ - data: 'Original content', - type: 'update-test', - metadata: { version: 1, status: 'draft' } - }) - }) - - it('should update entity metadata', async () => { - const startTime = metricsCollector.startOperation('update-single') - await brainy.update(testId, { - metadata: { version: 2, status: 'published' } - }) - metricsCollector.endOperation('update-single', 1, startTime) - - const updated = await brainy.get(testId) - expect(updated?.metadata?.version).toBe(2) - expect(updated?.metadata?.status).toBe('published') - }) - - it('should handle concurrent updates', async () => { - const updates = Array(10).fill(null).map((_, i) => - brainy.update(testId, { - metadata: { lastUpdate: i } - }) - ) - - const results = await Promise.allSettled(updates) - const succeeded = results.filter(r => r.status === 'fulfilled') - expect(succeeded.length).toBeGreaterThanOrEqual(1) - }) - }) - - describe('DELETE Operations - Comprehensive Testing', () => { - it('should delete existing entities', async () => { - const id = await brainy.add({ - data: 'To be deleted', - type: 'delete-test' - }) - - const startTime = metricsCollector.startOperation('delete-single') - const result = await brainy.remove(id) - metricsCollector.endOperation('delete-single', 1, startTime) - - expect(result).toBe(true) - - const retrieved = await brainy.get(id) - expect(retrieved).toBeNull() - }) - - it('should handle bulk deletions', async () => { - const ids: string[] = [] - for (let i = 0; i < 100; i++) { - const id = await brainy.add({ - data: `Bulk delete test ${i}`, - type: 'bulk-delete' - }) - ids.push(id) - } - - const startTime = metricsCollector.startOperation('delete-bulk', ids.length) - const results = await Promise.all( - ids.map(id => brainy.remove(id)) - ) - metricsCollector.endOperation('delete-bulk', ids.length, startTime) - - expect(results.every(r => r === true)).toBe(true) - }) - }) - - describe('RELATE Operations - Comprehensive Testing', () => { - it('should create relationships between entities', async () => { - const id1 = await brainy.add({ - data: 'Entity 1', - type: 'node' - }) - const id2 = await brainy.add({ - data: 'Entity 2', - type: 'node' - }) - - const startTime = metricsCollector.startOperation('relate-single') - const result = await brainy.relate(id1, id2, 'connects_to') - metricsCollector.endOperation('relate-single', 1, startTime) - - expect(result).toBe(true) - }) - - it('should handle complex graph structures', async () => { - const nodeIds: string[] = [] - - // Create nodes - for (let i = 0; i < 20; i++) { - const id = await brainy.add({ - data: `Node ${i}`, - type: 'graph-node', - metadata: { index: i } - }) - nodeIds.push(id) - } - - // Create relationships (each node connects to next 3) - const startTime = metricsCollector.startOperation('relate-graph', 60) - for (let i = 0; i < nodeIds.length; i++) { - for (let j = 1; j <= 3; j++) { - const targetIdx = (i + j) % nodeIds.length - await brainy.relate(nodeIds[i], nodeIds[targetIdx], 'links_to') - } - } - metricsCollector.endOperation('relate-graph', 60, startTime) - - // Graph should be created successfully - expect(nodeIds).toHaveLength(20) - }) - }) - - describe('FIND Operations - Comprehensive Testing', () => { - beforeEach(async () => { - // Create diverse test dataset - for (let i = 0; i < 50; i++) { - await brainy.add({ - data: `Search test document ${i}: Lorem ipsum dolor sit amet`, - type: 'searchable', - metadata: { - index: i, - category: `cat-${i % 5}`, - score: Math.random() * 100, - active: i % 2 === 0 - } - }) - } - }) - - it('should find entities by metadata criteria', async () => { - const startTime = metricsCollector.startOperation('find-simple') - const results = await brainy.find({ - where: { - 'metadata.category': 'cat-0' - } - }) - metricsCollector.endOperation('find-simple', 1, startTime) - - expect(results.length).toBe(10) // 50 / 5 = 10 - }) - - it('should support pagination', async () => { - const page1 = await brainy.find({ - limit: 10, - offset: 0 - }) - - const page2 = await brainy.find({ - limit: 10, - offset: 10 - }) - - expect(page1).toHaveLength(10) - expect(page2).toHaveLength(10) - expect(page1[0].id).not.toBe(page2[0].id) - }) - - it('should handle complex queries', async () => { - const results = await brainy.find({ - where: { - 'metadata.active': true, - 'metadata.score': { $gte: 50 } - }, - limit: 100 - }) - - expect(results.every(r => r.metadata?.active === true)).toBe(true) - expect(results.every(r => (r.metadata?.score ?? 0) >= 50)).toBe(true) - }) - }) - - describe('Performance Summary', () => { - it('should generate comprehensive performance report', async () => { - await metricsCollector.reportMetrics() - - const metrics = metricsCollector.getMetrics() - expect(metrics.length).toBeGreaterThan(0) - - // Validate all operations met SLAs - for (const metric of metrics) { - if (metric.operation.includes('single')) { - expect(metric.duration).toBeLessThan(100) // Single ops < 100ms - } - } - }) - }) -}) \ No newline at end of file diff --git a/tests/api/error-handling.test.ts b/tests/api/error-handling.test.ts deleted file mode 100644 index b8fd91f9..00000000 --- a/tests/api/error-handling.test.ts +++ /dev/null @@ -1,638 +0,0 @@ -/** - * Error Handling and Recovery Test Suite for Brainy v3.0 - * - * Comprehensive error scenario testing including: - * - Invalid input validation - * - Network failure recovery - * - Resource exhaustion handling - * - Concurrent error scenarios - * - Graceful degradation - * - Error message consistency - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../src/brainy' - -describe('Error Handling and Recovery', () => { - let brainy: Brainy - - beforeEach(async () => { - brainy = new Brainy({ requireSubtype: false, - storage: { type: 'memory' } - }) - await brainy.init() - }) - - afterEach(async () => { - await brainy.close() - }) - - describe('Input Validation Errors', () => { - describe('ADD operation validation', () => { - it('should reject null data', async () => { - await expect( - brainy.add({ - data: null as any, - type: 'document' - }) - ).rejects.toThrow() - }) - - it('should reject undefined data', async () => { - await expect( - brainy.add({ - data: undefined as any, - type: 'document' - }) - ).rejects.toThrow() - }) - - it('should reject empty string data', async () => { - await expect( - brainy.add({ - data: '', - type: 'document' - }) - ).rejects.toThrow(/data.*required|empty/i) - }) - - it('should reject invalid type values', async () => { - await expect( - brainy.add({ - data: 'Valid data', - type: 'invalid-type' as any - }) - ).rejects.toThrow(/invalid.*type|noun.*type/i) - }) - - it('should reject non-string data types', async () => { - const invalidData = [ - 42, - true, - [], - { text: 'object' }, - () => 'function' - ] - - for (const data of invalidData) { - await expect( - brainy.add({ - data: data as any, - type: 'document' - }) - ).rejects.toThrow() - } - }) - - it('should handle extremely large metadata gracefully', async () => { - const hugeMetadata: any = {} - for (let i = 0; i < 10000; i++) { - hugeMetadata[`field${i}`] = `value${i}`.repeat(100) - } - - // Should either succeed or throw meaningful error - try { - const id = await brainy.add({ - data: 'Test with huge metadata', - type: 'document', - metadata: hugeMetadata - }) - expect(id).toBeDefined() - } catch (error: any) { - expect(error.message).toMatch(/size|memory|large/i) - } - }) - }) - - describe('GET operation validation', () => { - it('should return null for non-existent IDs', async () => { - const result = await brainy.get('non-existent-id-12345') - expect(result).toBeNull() - }) - - it('should handle invalid ID formats gracefully', async () => { - const invalidIds = [ - null, - undefined, - '', - ' ', - '../etc/passwd', - '' - ] - - for (const id of invalidIds) { - const result = await brainy.get(id as any) - expect(result).toBeNull() - } - }) - }) - - describe('UPDATE operation validation', () => { - it('should reject updates without ID', async () => { - await expect( - brainy.update({ - id: undefined as any, - metadata: { updated: true } - }) - ).rejects.toThrow(/id.*required/i) - }) - - it('should reject updates to non-existent entities', async () => { - await expect( - brainy.update({ - id: 'non-existent-id', - metadata: { updated: true } - }) - ).rejects.toThrow(/not found|doesn't exist/i) - }) - - it('should handle circular references in metadata', async () => { - const id = await brainy.add({ - data: 'Test entity', - type: 'document' - }) - - const metadata: any = { level: 1 } - metadata.self = metadata // Circular reference - - // Should either handle or throw meaningful error - try { - await brainy.update({ - id, - metadata - }) - // If successful, verify it was handled - const updated = await brainy.get(id) - expect(updated).toBeDefined() - } catch (error: any) { - expect(error.message).toMatch(/circular|cyclic/i) - } - }) - }) - - describe('DELETE operation validation', () => { - it('should handle deletion of non-existent entities', async () => { - // Should not throw, just return void - await expect( - brainy.remove('non-existent-id') - ).resolves.toBeUndefined() - }) - - it('should handle double deletion gracefully', async () => { - const id = await brainy.add({ - data: 'To be deleted', - type: 'document' - }) - - await brainy.remove(id) - // Second delete should not throw - await expect(brainy.remove(id)).resolves.toBeUndefined() - }) - }) - - describe('RELATE operation validation', () => { - it('should reject relationships without source', async () => { - await expect( - brainy.relate({ - from: undefined as any, - to: 'some-id', - type: 'relatedTo' - }) - ).rejects.toThrow(/from.*required/i) - }) - - it('should reject relationships without target', async () => { - await expect( - brainy.relate({ - from: 'some-id', - to: undefined as any, - type: 'relatedTo' - }) - ).rejects.toThrow(/to.*required/i) - }) - - it('should reject invalid relationship types', async () => { - const id1 = await brainy.add({ - data: 'Entity 1', - type: 'thing' - }) - const id2 = await brainy.add({ - data: 'Entity 2', - type: 'thing' - }) - - await expect( - brainy.relate({ - from: id1, - to: id2, - type: 'invalid-verb' as any - }) - ).rejects.toThrow(/invalid.*type|verb.*type/i) - }) - - it('should reject self-relationships by default', async () => { - const id = await brainy.add({ - data: 'Self entity', - type: 'thing' - }) - - // Some systems reject self-relationships - try { - await brainy.relate({ - from: id, - to: id, - type: 'relatedTo' - }) - } catch (error: any) { - expect(error.message).toMatch(/self|same.*entity/i) - } - }) - }) - - describe('FIND operation validation', () => { - it('should handle empty queries gracefully', async () => { - const results = await brainy.find({}) - expect(Array.isArray(results)).toBe(true) - }) - - it('should handle invalid where clauses', async () => { - const results = await brainy.find({ - where: { - 'metadata.field': { $invalidOp: 'value' } as any - } - }) - // Should return empty array or handle gracefully - expect(Array.isArray(results)).toBe(true) - }) - - it('should handle negative limits', async () => { - const results = await brainy.find({ - limit: -10 - }) - expect(Array.isArray(results)).toBe(true) - expect(results.length).toBeGreaterThanOrEqual(0) - }) - - it('should handle huge limits reasonably', async () => { - const results = await brainy.find({ - limit: Number.MAX_SAFE_INTEGER - }) - expect(Array.isArray(results)).toBe(true) - // Should cap at reasonable limit - expect(results.length).toBeLessThanOrEqual(10000) - }) - }) - }) - - describe('Batch Operation Error Handling', () => { - it('should handle partial failures in addMany', async () => { - const items = [ - { data: 'Valid 1', type: 'document' as const }, - { data: '', type: 'document' as const }, // Invalid - { data: 'Valid 2', type: 'document' as const }, - { data: null as any, type: 'document' as const }, // Invalid - { data: 'Valid 3', type: 'document' as const } - ] - - const result = await brainy.addMany({ - items, - continueOnError: true - }) - - expect(result.successful.length).toBeGreaterThanOrEqual(3) - expect(result.failed.length).toBeGreaterThanOrEqual(0) - expect(result.total).toBe(5) - }) - - it('should rollback batch on critical failure', async () => { - const items = Array(100).fill(null).map((_, i) => ({ - data: `Item ${i}`, - type: 'document' as const - })) - - // Inject a failure midway - items[50] = { - data: null as any, - type: 'document' as const - } - - const result = await brainy.addMany({ - items, - continueOnError: false - }) - - // Should either complete all or rollback - if (result.successful.length > 0) { - expect(result.successful.length).toBe(100) - } else { - expect(result.failed.length).toBeGreaterThan(0) - } - }) - }) - - describe('Concurrent Operation Errors', () => { - it('should handle concurrent updates to same entity', async () => { - const id = await brainy.add({ - data: 'Concurrent test', - type: 'document', - metadata: { version: 1 } - }) - - // Attempt concurrent updates - const updates = Array(10).fill(null).map((_, i) => - brainy.update({ - id, - metadata: { version: i + 2 } - }) - ) - - const results = await Promise.allSettled(updates) - - // At least one should succeed - const succeeded = results.filter(r => r.status === 'fulfilled') - expect(succeeded.length).toBeGreaterThanOrEqual(1) - - // Final state should be consistent - const final = await brainy.get(id) - expect(final?.metadata?.version).toBeGreaterThanOrEqual(2) - }) - - it('should handle concurrent deletes gracefully', async () => { - const id = await brainy.add({ - data: 'To be deleted concurrently', - type: 'document' - }) - - // Attempt concurrent deletes - const deletes = Array(5).fill(null).map(() => brainy.remove(id)) - - // Should not throw - await expect(Promise.all(deletes)).resolves.toBeDefined() - - // Entity should be deleted - const result = await brainy.get(id) - expect(result).toBeNull() - }) - }) - - describe('Resource Exhaustion', () => { - it('should handle memory pressure gracefully', async () => { - const largeData = 'x'.repeat(1024 * 1024) // 1MB - let successCount = 0 - let errorCount = 0 - - // Try to add many large entities - for (let i = 0; i < 100; i++) { - try { - await brainy.add({ - data: largeData + ` Entity ${i}`, - type: 'document', - metadata: { index: i, size: largeData.length } - }) - successCount++ - } catch (error) { - errorCount++ - // Should get meaningful error - expect(error).toBeDefined() - } - } - - // Should handle at least some operations - expect(successCount).toBeGreaterThan(0) - }) - - it('should handle rapid-fire operations', async () => { - const operations = 1000 - const promises: Promise[] = [] - - for (let i = 0; i < operations; i++) { - const op = i % 4 - switch (op) { - case 0: - promises.push( - brainy.add({ - data: `Rapid ${i}`, - type: 'document' - }).catch(() => null) - ) - break - case 1: - promises.push(brainy.get(`rapid-${i}`).catch(() => null)) - break - case 2: - promises.push( - brainy.update({ - id: `rapid-${i}`, - metadata: { updated: true } - }).catch(() => null) - ) - break - case 3: - promises.push(brainy.remove(`rapid-${i}`).catch(() => null)) - break - } - } - - const results = await Promise.allSettled(promises) - - // Most should complete - const completed = results.filter(r => r.status === 'fulfilled') - expect(completed.length).toBeGreaterThan(operations * 0.5) - }) - }) - - describe('Error Message Quality', () => { - it('should provide clear error messages for common mistakes', async () => { - // Missing required field - try { - await brainy.add({} as any) - } catch (error: any) { - expect(error.message).toBeDefined() - expect(error.message.length).toBeGreaterThan(10) - // Should mention what's missing - expect(error.message).toMatch(/data|required/i) - } - - // Invalid type - try { - await brainy.add({ - data: 'Valid data', - type: 'not-a-valid-type' as any - }) - } catch (error: any) { - expect(error.message).toMatch(/type|invalid|noun/i) - } - - // Entity not found - try { - await brainy.update({ - id: 'definitely-does-not-exist', - metadata: { test: true } - }) - } catch (error: any) { - expect(error.message).toMatch(/not found|doesn't exist|does not exist/i) - } - }) - - it('should not leak sensitive information in errors', async () => { - try { - await brainy.add({ - data: 'Test', - type: 'document', - metadata: { - password: 'secret123', - apiKey: 'sk-1234567890' - } - }) - - await brainy.update({ - id: 'non-existent', - metadata: { password: 'should-not-appear' } - }) - } catch (error: any) { - // Error message should not contain sensitive data - expect(error.message).not.toMatch(/secret123|sk-1234567890|should-not-appear/i) - } - }) - }) - - describe('Recovery Mechanisms', () => { - it('should recover from transient failures', async () => { - let attemptCount = 0 - const maxAttempts = 3 - - async function retryableAdd(data: string): Promise { - attemptCount++ - if (attemptCount < maxAttempts) { - throw new Error('Transient failure') - } - return brainy.add({ - data, - type: 'document' - }) - } - - // Should eventually succeed with retry - let result: string | null = null - for (let i = 0; i < maxAttempts; i++) { - try { - result = await retryableAdd('Test with retry') - break - } catch (error) { - if (i === maxAttempts - 1) throw error - } - } - - expect(result).toBeDefined() - expect(attemptCount).toBe(maxAttempts) - }) - - it('should maintain consistency after errors', async () => { - // Add some valid data - const validIds: string[] = [] - for (let i = 0; i < 5; i++) { - const id = await brainy.add({ - data: `Valid entity ${i}`, - type: 'document' - }) - validIds.push(id) - } - - // Cause some errors - try { - await brainy.add({ data: null as any, type: 'document' }) - } catch {} - - try { - await brainy.update({ id: 'non-existent', metadata: {} }) - } catch {} - - try { - await brainy.relate({ - from: 'non-existent-1', - to: 'non-existent-2', - type: 'relatedTo' - }) - } catch {} - - // Valid data should still be intact - for (const id of validIds) { - const entity = await brainy.get(id) - expect(entity).toBeDefined() - expect(entity?.type).toBe('document') - } - - // Should still be able to add new data - const newId = await brainy.add({ - data: 'Added after errors', - type: 'document' - }) - expect(newId).toBeDefined() - }) - }) - - describe('Edge Cases', () => { - it('should handle operations on just-deleted entities', async () => { - const id = await brainy.add({ - data: 'About to be deleted', - type: 'document' - }) - - await brainy.remove(id) - - // Operations on deleted entity should handle gracefully - const getResult = await brainy.get(id) - expect(getResult).toBeNull() - - await expect( - brainy.update({ - id, - metadata: { attempt: 'update-after-delete' } - }) - ).rejects.toThrow(/not found/i) - - // Should not throw for delete - await expect(brainy.remove(id)).resolves.toBeUndefined() - }) - - it('should handle rapid create-delete cycles', async () => { - const cycles = 50 - - for (let i = 0; i < cycles; i++) { - const id = await brainy.add({ - data: `Cycle ${i}`, - type: 'document' - }) - - // Immediately delete - await brainy.remove(id) - - // Verify it's gone - const result = await brainy.get(id) - expect(result).toBeNull() - } - }) - - it('should handle maximum field lengths', async () => { - const maxFieldLength = 1024 * 1024 // 1MB - const longString = 'x'.repeat(maxFieldLength) - - try { - const id = await brainy.add({ - data: longString, - type: 'document', - metadata: { - description: longString.substring(0, 1000) - } - }) - - // If successful, should be retrievable - const entity = await brainy.get(id) - expect(entity).toBeDefined() - } catch (error: any) { - // If it fails, should have meaningful error - expect(error.message).toMatch(/size|large|limit/i) - } - }) - }) -}) \ No newline at end of file diff --git a/tests/api/performance-benchmarks.test.ts b/tests/api/performance-benchmarks.test.ts index d26281d3..e83cebf2 100644 --- a/tests/api/performance-benchmarks.test.ts +++ b/tests/api/performance-benchmarks.test.ts @@ -201,7 +201,8 @@ describe('Performance Benchmarks - SLA Validation', () => { const start = performance.now() await brainy.update({ id, - metadata: { version: 2, updatedAt: Date.now() } + // `updatedAt` is system-managed — Brainy sets it on every write. + metadata: { version: 2 } }) const latency = performance.now() - start benchmark.recordOperation(latency) diff --git a/tests/benchmarks/distance-microbench.mjs b/tests/benchmarks/distance-microbench.mjs new file mode 100644 index 00000000..a3119a90 --- /dev/null +++ b/tests/benchmarks/distance-microbench.mjs @@ -0,0 +1,106 @@ +#!/usr/bin/env node +/** + * Distance microbenchmark — measures the vector-distance hot path in isolation. + * + * Compares the reduce-based implementations (current `src/utils/distance.ts`) + * against allocation-free indexed for-loops, on both `number[]` and + * `Float32Array`, to establish MEASURED evidence for the Float32Array Fork X + * change. Per the evidence-based-claims rule, no % is published without this. + * + * Run: node tests/benchmarks/distance-microbench.mjs + */ + +const DIM = 384 +const N = 20000 // candidate vectors compared per pass +const ITERS = 41 // repeat passes; report the median (robust to GC blips) + +// --- reduce-based (current distance.ts) --- +const cosineReduce = (a, b) => { + const { dotProduct, normA, normB } = a.reduce( + (acc, val, i) => ({ + dotProduct: acc.dotProduct + val * b[i], + normA: acc.normA + val * val, + normB: acc.normB + b[i] * b[i] + }), + { dotProduct: 0, normA: 0, normB: 0 } + ) + if (normA === 0 || normB === 0) return 2 + return 1 - dotProduct / (Math.sqrt(normA) * Math.sqrt(normB)) +} +const euclideanReduce = (a, b) => { + const sum = a.reduce((acc, val, i) => { + const d = val - b[i] + return acc + d * d + }, 0) + return Math.sqrt(sum) +} + +// --- allocation-free indexed for-loop (proposed) --- +const cosineLoop = (a, b) => { + let dot = 0, + na = 0, + nb = 0 + const len = a.length + for (let i = 0; i < len; i++) { + const av = a[i] + const bv = b[i] + dot += av * bv + na += av * av + nb += bv * bv + } + if (na === 0 || nb === 0) return 2 + return 1 - dot / (Math.sqrt(na) * Math.sqrt(nb)) +} +const euclideanLoop = (a, b) => { + let sum = 0 + const len = a.length + for (let i = 0; i < len; i++) { + const d = a[i] - b[i] + sum += d * d + } + return Math.sqrt(sum) +} + +function makeVectors(Ctor) { + const alloc = () => (Ctor === Array ? new Array(DIM) : new Ctor(DIM)) + const q = alloc() + for (let i = 0; i < DIM; i++) q[i] = Math.sin(i * 0.1) * 0.5 + Math.cos(i * 0.03) * 0.5 + const vs = [] + for (let n = 0; n < N; n++) { + const v = alloc() + for (let i = 0; i < DIM; i++) v[i] = Math.sin((n + i) * 0.07) + vs.push(v) + } + return { q, vs } +} + +let sink = 0 +function bench(name, fn, q, vs) { + for (let w = 0; w < 3; w++) for (let i = 0; i < vs.length; i++) sink += fn(q, vs[i]) + const times = [] + for (let it = 0; it < ITERS; it++) { + const t0 = process.hrtime.bigint() + for (let i = 0; i < vs.length; i++) sink += fn(q, vs[i]) + times.push(Number(process.hrtime.bigint() - t0) / 1e6) + } + times.sort((a, b) => a - b) + const median = times[Math.floor(times.length / 2)] + const opsPerSec = (vs.length / median) * 1000 + console.log(` ${name.padEnd(28)} ${median.toFixed(2).padStart(8)} ms ${(opsPerSec / 1e6).toFixed(2).padStart(6)} M ops/s`) + return median +} + +console.log(`Distance microbench — dim=${DIM}, N=${N}, iters=${ITERS} (median)\n`) +for (const [label, Ctor] of [ + ['number[]', Array], + ['Float32Array', Float32Array] +]) { + const { q, vs } = makeVectors(Ctor) + console.log(`--- ${label} ---`) + const cr = bench('cosine reduce (current)', cosineReduce, q, vs) + const cl = bench('cosine for-loop', cosineLoop, q, vs) + const er = bench('euclid reduce (current)', euclideanReduce, q, vs) + const el = bench('euclid for-loop', euclideanLoop, q, vs) + console.log(` → cosine for-loop is ${(cr / cl).toFixed(2)}x, euclid for-loop is ${(er / el).toFixed(2)}x\n`) +} +if (sink === Infinity) console.log('(unreachable guard)') diff --git a/tests/benchmarks/model-b-scalability.spike.ts b/tests/benchmarks/model-b-scalability.spike.ts new file mode 100644 index 00000000..5433d9f6 --- /dev/null +++ b/tests/benchmarks/model-b-scalability.spike.ts @@ -0,0 +1,196 @@ +/** + * @module tests/benchmarks/model-b-scalability.spike + * @description SPIKE (throwaway, spike/model-b-write-perf branch) — the DECISION-CRITICAL + * half of the Model B evaluation. The write-perf spike showed writes are cheap at + * durability parity; the adversarial review showed the REAL risk is structural and + * grows with the number of retained generations N: + * + * 1. READ-vs-depth — resolveAt/changedBetween LINEARLY scan the global committedGens + * array → historical reads (asOf/get, diff, since) become O(N) = O(database-age). + * 2. REOPEN-vs-depth — GenerationStore.open enumerates+sorts EVERY generation dir → O(N) cold start. + * 3. RAM-vs-depth — deltaCache holds a Set per committed generation → O(N) resident heap. + * 4. COMPACTION cost — the only thing that bounds 1-3; time it + the post-compaction footprint. + * + * Each is measured at two depths (small, large). A ~linear slope CONFIRMS that brainy-TS's + * current global-committedGens design does NOT scale under Model B (every write = a generation), + * and that the canonical Model-B history must be a PER-ID INDEX (mirroring cor's delta_history + * BTreeMap, O(log n)) rather than a global scan. Memory backend is used where the scan/RAM cost + * is backend-independent (committedGens is in-memory); filesystem only where reopen/compaction + * touch disk. + * + * Run (force GC available): node --expose-gc node_modules/.bin/tsx tests/benchmarks/model-b-scalability.spike.ts + * small scale: SPIKE_SCALE=small ... + */ + +import { rmSync, mkdirSync, existsSync } from 'node:fs' +import { execSync } from 'node:child_process' +import { Brainy } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' +import type { BrainyConfig } from '../../src/types/brainy.types.js' + +// Silence brainy's verbose per-brain logging (floods stdout, slows the run). +const _log = console.log.bind(console) +const say = (...a: any[]) => _log(...a) +for (const m of ['log', 'info', 'debug', 'warn'] as const) (console as any)[m] = () => {} + +const SMALL = process.env.SPIKE_SCALE === 'small' +const DIM = 384 +const FS_ROOT = '/media/dpsifr/storage/home/Projects/brainy/.spike-data' +// Two depths to read the slope. Memory legs can go deep cheaply; fs legs (fsync per gen) stay smaller. +const DEPTHS_MEM = SMALL ? [500, 5_000] : [2_000, 20_000] +const DEPTHS_FS = SMALL ? [300, 1_500] : [1_000, 6_000] +const READ_REPEATS = SMALL ? 20 : 50 + +function vec(i: number): number[] { + const v = new Array(DIM) + let mag = 0 + for (let d = 0; d < DIM; d++) { const x = Math.sin((i + 1) * 0.001 + d * 0.1); v[d] = x; mag += x * x } + mag = Math.sqrt(mag) || 1 + for (let d = 0; d < DIM; d++) v[d] /= mag + return v +} + +async function mkBrain(backend: 'memory' | 'filesystem', dir?: string): Promise { + const storage: BrainyConfig['storage'] = + backend === 'memory' ? { type: 'memory' } : { type: 'filesystem', path: dir! } + const brain = new Brainy({ + requireSubtype: false, + storage, + eagerEmbeddings: false, // never load WASM — we pass precomputed vectors + // Keep history fully retained for the measurement (no mid-run compaction skewing depth). + retention: { autoCompact: false } as any, + index: { m: 16, efConstruction: 200, efSearch: 50 }, + cache: { maxSize: 1000, ttl: 3600 } + } as BrainyConfig) + brain.use({ name: 'const-embedder', activate: async (ctx: any) => { ctx.registerProvider('embeddings', async () => vec(0)); return true } } as any) + await brain.init() + return brain +} + +function payload(i: number) { + return { type: NounType.Document, subtype: 'note', data: `doc ${i}`, vector: vec(i), metadata: { i, batch: 'spike' } } +} + +function freshDir(name: string): string { + const dir = `${FS_ROOT}/${name}` + if (existsSync(dir)) rmSync(dir, { recursive: true, force: true }) + mkdirSync(dir, { recursive: true }) + return dir +} +function allocBytes(dir: string): number { try { return parseInt(execSync(`du -s --block-size=1 ${dir}`, { encoding: 'utf8' }).split(/\s+/)[0], 10) } catch { return -1 } } + +const SEED = 32 // small working set; X (id[0]) is the never-updated probe entity + +/** Build N committed generations via transact([1]). X = ids[0] is seeded then NEVER updated, so a + * read of X at the oldest pin must scan all N generations to confirm it is unchanged (worst case). */ +async function buildGenerations(b: Brainy, nGens: number): Promise<{ ids: string[]; g0: number; gN: number }> { + const ids: string[] = [] + // Seed via ONE transact so g0 is a committed generation (asOf needs a committed gen). + const seedOps = Array.from({ length: SEED }, (_, i) => ({ op: 'add' as const, ...payload(i) })) + const seedDb = await b.transact(seedOps as any) + for (const id of (await b.find({ type: NounType.Document, limit: SEED }))) ids.push(id.id) + await seedDb.release?.() + const g0 = b.generation() + // N generations, each a single-op-equivalent transact([1]) update on the CHURN set (ids[1..]), + // never touching X=ids[0]. + for (let k = 0; k < nGens; k++) { + const id = ids[1 + (k % (ids.length - 1))] + const d = await b.transact([{ op: 'update', id, metadata: { k } }] as any) + await d.release?.() + } + return { ids, g0, gN: b.generation() } +} + +function median(xs: number[]): number { const s = [...xs].sort((a, b) => a - b); return s[Math.floor(s.length / 2)] } + +async function timeMs(fn: () => Promise, repeats: number): Promise { + const ts: number[] = [] + for (let r = 0; r < repeats; r++) { const t = process.hrtime.bigint(); await fn(); ts.push(Number(process.hrtime.bigint() - t) / 1e6) } + return median(ts) +} + +async function main() { + say(`[scale] start (scale=${SMALL ? 'small' : 'full'}); gc=${typeof (global as any).gc === 'function' ? 'on' : 'OFF (run with node --expose-gc)'}`) + if (existsSync(FS_ROOT)) rmSync(FS_ROOT, { recursive: true, force: true }) + mkdirSync(FS_ROOT, { recursive: true }) + const results: any = { readVsDepth: [], reopenVsDepth: [], ramVsDepth: [], compaction: [] } + + // ---- 1. READ latency vs history depth (memory; scan is in-memory) ---- + say(`\n========== READ latency vs history depth (the O(database-age) test) ==========`) + for (const N of DEPTHS_MEM) { + const b = await mkBrain('memory') + const { ids, g0, gN } = await buildGenerations(b, N) + const X = ids[0] + // asOf(oldest).get(X): resolveAt(X,g0) scans all N committedGens (X never touched) → expect O(N). + const tAsOfGet = await timeMs(async () => { const db = await b.asOf(g0); await db.get(X); await db.release?.() }, READ_REPEATS) + // diff(g0,gN): changedBetween scans the full (g0,gN] range → expect O(N). + const tDiff = await timeMs(async () => { await b.diff(g0, gN) }, Math.max(5, READ_REPEATS / 5)) + await b.close() + say(`N=${N.toLocaleString()} gens: asOf(oldest).get(X) = ${tAsOfGet.toFixed(3)} ms | diff(g0,gN) = ${tDiff.toFixed(2)} ms`) + results.readVsDepth.push({ N, asOfGetMs: tAsOfGet, diffMs: tDiff }) + } + { + const [a, c] = results.readVsDepth + if (a && c) say(` → asOf-get scaling ${a.N}→${c.N} (${(c.N / a.N).toFixed(0)}x depth): ${(c.asOfGetMs / a.asOfGetMs).toFixed(1)}x slower (linear≈${(c.N / a.N).toFixed(0)}x → confirms O(N) scan)`) + } + + // ---- 2. RAM resident vs history depth (memory; deltaCache holds a Set per gen) ---- + say(`\n========== RAM (heapUsed) vs history depth (deltaCache O(N) Sets) ==========`) + for (const N of DEPTHS_MEM) { + if ((global as any).gc) (global as any).gc() + const before = process.memoryUsage().heapUsed + const b = await mkBrain('memory') + await buildGenerations(b, N) + if ((global as any).gc) (global as any).gc() + const after = process.memoryUsage().heapUsed + const perGen = (after - before) / N + await b.close() + say(`N=${N.toLocaleString()} gens: heapUsed +${((after - before) / 1024 / 1024).toFixed(1)} MiB (${perGen.toFixed(0)} bytes/generation resident)`) + results.ramVsDepth.push({ N, deltaBytes: after - before, bytesPerGen: perGen }) + } + + // ---- 3. COLD-REOPEN vs history depth (filesystem; open() walks every gen dir) ---- + say(`\n========== COLD-REOPEN time vs history depth (O(generations) dir walk) ==========`) + for (const N of DEPTHS_FS) { + const dir = freshDir(`reopen-${N}`) + let b = await mkBrain('filesystem', dir) + await buildGenerations(b, N) + await b.close() + const tOpen = await timeMs(async () => { const rb = await mkBrain('filesystem', dir); await rb.close() }, SMALL ? 3 : 5) + rmSync(dir, { recursive: true, force: true }) + say(`N=${N.toLocaleString()} gens: cold reopen = ${tOpen.toFixed(0)} ms`) + results.reopenVsDepth.push({ N, reopenMs: tOpen }) + } + { + const [a, c] = results.reopenVsDepth + if (a && c) say(` → reopen scaling ${a.N}→${c.N} (${(c.N / a.N).toFixed(0)}x depth): ${(c.reopenMs / a.reopenMs).toFixed(1)}x slower`) + } + + // ---- 4. COMPACTION cost + footprint (filesystem) ---- + say(`\n========== COMPACTION cost (the bound on 1-3) ==========`) + { + const N = DEPTHS_FS[DEPTHS_FS.length - 1] + const dir = freshDir(`compact-${N}`) + const b = await mkBrain('filesystem', dir) + await buildGenerations(b, N) + await b.flush() + const beforeBytes = allocBytes(`${dir}/_generations`) + const t = process.hrtime.bigint() + const res = await b.compactHistory({ maxGenerations: 100 } as any) + const compMs = Number(process.hrtime.bigint() - t) / 1e6 + await b.flush() + const afterBytes = allocBytes(`${dir}/_generations`) + await b.close() + rmSync(dir, { recursive: true, force: true }) + say(`N=${N.toLocaleString()} gens → compactHistory({maxGenerations:100}): ${compMs.toFixed(0)} ms, removed ${(res as any)?.removedGenerations ?? '?'} gens`) + say(` _generations footprint: ${(beforeBytes / 1024 / 1024).toFixed(1)} MiB → ${(afterBytes / 1024 / 1024).toFixed(1)} MiB allocated`) + results.compaction.push({ N, compactMs: compMs, removed: (res as any)?.removedGenerations, beforeBytes, afterBytes }) + } + + const fs = await import('node:fs') + fs.writeFileSync('/media/dpsifr/storage/home/Projects/brainy/tests/benchmarks/_scalability-results.json', JSON.stringify(results, null, 2)) + say(`\nRaw results → tests/benchmarks/_scalability-results.json`) + if (existsSync(FS_ROOT)) rmSync(FS_ROOT, { recursive: true, force: true }) +} + +main().then(() => { say('[scale] done.'); process.exit(0) }).catch((e) => { console.error(e); process.exit(1) }) diff --git a/tests/benchmarks/model-b-write-perf.spike.ts b/tests/benchmarks/model-b-write-perf.spike.ts new file mode 100644 index 00000000..f283284f --- /dev/null +++ b/tests/benchmarks/model-b-write-perf.spike.ts @@ -0,0 +1,370 @@ +/** + * @module tests/benchmarks/model-b-write-perf.spike + * @description SPIKE (throwaway, spike/model-b-write-perf branch) — measures the + * cost of Model B (per-write immutable history). We do NOT build Model B to measure + * it: `transact([1 op])` already does exactly what Model B would make every single-op + * write do (read+save before-image → write delta → write manifest → sync). So this + * benchmarks today's single-op path (`add()`/`update()`) against `transact([1])` + * (Model B per-write, un-optimized upper bound) and `transact([K])` (group-commit + * amortization), on memory (isolates CPU/retention overhead, sync is a no-op) and + * filesystem-on-NVMe (adds real fsync), with embedding neutralized (precomputed + * vector + a constant embeddings provider) so the throughput delta is pure + * commit-path overhead. Also measures on-disk history growth (bytes/write retained). + * + * Run: node_modules/.bin/tsx tests/benchmarks/model-b-write-perf.spike.ts + * Optional smaller run: SPIKE_SCALE=small node_modules/.bin/tsx tests/benchmarks/model-b-write-perf.spike.ts + */ + +import { rmSync, mkdirSync, existsSync } from 'node:fs' +import { execSync } from 'node:child_process' +import { Brainy } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' +import type { BrainyConfig } from '../../src/types/brainy.types.js' + +// Silence brainy's verbose per-brain init logging — it floods stdout (≈1 MB at +// small scale across dozens of fresh brains) and the I/O measurably slows the run. +// Capture the real console.log first, then no-op the noisy channels; `say()` is the +// benchmark's own output channel and is unaffected. +const _log = console.log.bind(console) +const say = (...a: any[]) => _log(...a) +for (const m of ['log', 'info', 'debug', 'warn'] as const) { + ;(console as any)[m] = () => {} +} + +// ---- config --------------------------------------------------------------- + +const SMALL = process.env.SPIKE_SCALE === 'small' +const DIM = 384 +// NVMe ext4 (NOT /tmp — that is tmpfs/RAM and would understate fsync). +const FS_ROOT = '/media/dpsifr/storage/home/Projects/brainy/.spike-data' + +// Scales: memory is cheap (CPU only); filesystem pays fsync per transact so it is smaller. +const N_MEM = SMALL ? 2_000 : 20_000 +const N_FS = SMALL ? 500 : 4_000 +const K = 100 // group-commit batch size for the transact([K]) amortization run +const WARMUP = SMALL ? 100 : 500 +const REPEATS = 3 // median of N timed repeats + +// ---- helpers -------------------------------------------------------------- + +/** Deterministic, cheap, VARIED unit vector per index (avoids embedding AND a + * degenerate all-identical-vector HNSW graph). No model, ~µs. */ +function vec(i: number): number[] { + const v = new Array(DIM) + let mag = 0 + for (let d = 0; d < DIM; d++) { + const x = Math.sin((i + 1) * 0.001 + d * 0.1) * 0.5 + Math.cos(d * 0.05) * 0.3 + v[d] = x + mag += x * x + } + mag = Math.sqrt(mag) || 1 + for (let d = 0; d < DIM; d++) v[d] /= mag + return v +} + +const CONST_VEC = vec(0) + +/** Build a brain with embedding fully neutralized: + * - a constant 'embeddings' provider → skips the WASM eager-init (the provider + * "owns" embeddings) and makes any embed call O(1); + * - every write also passes a precomputed `vector`, so embed is never even called. */ +async function mkBrain(backend: 'memory' | 'filesystem', dir?: string): Promise { + const storage: BrainyConfig['storage'] = + backend === 'memory' ? { type: 'memory' } : { type: 'filesystem', path: dir! } + const brain = new Brainy({ + requireSubtype: false, + storage, + // CRITICAL: skip the WASM embedder eager-init (90-140s compile per brain) — + // we never embed (precomputed vectors), so the model must not load at all. + eagerEmbeddings: false, + index: { m: 16, efConstruction: 200, efSearch: 50 }, + cache: { maxSize: 1000, ttl: 3600 } + } as BrainyConfig) + brain.use({ + name: 'const-embedder', + activate: async (ctx: any) => { + ctx.registerProvider('embeddings', async () => CONST_VEC) + return true + } + } as any) + await brain.init() + return brain +} + +function freshDir(name: string): string { + const dir = `${FS_ROOT}/${name}` + if (existsSync(dir)) rmSync(dir, { recursive: true, force: true }) + mkdirSync(dir, { recursive: true }) + return dir +} + +function dirBytes(dir: string): number { + try { + const out = execSync(`du -sb ${dir}`, { encoding: 'utf8' }) + return parseInt(out.split(/\s+/)[0], 10) + } catch { + return -1 + } +} + +interface Timing { + perSec: number + msPerOp: number +} + +/** Median-of-REPEATS timing of one full N-write workload built by `make`. + * `make` returns an async fn that performs ALL N writes on a FRESH brain. */ +async function timeWorkload( + label: string, + build: () => Promise<{ run: () => Promise; n: number; cleanup: () => Promise }> +): Promise { + const rates: number[] = [] + for (let r = 0; r < REPEATS; r++) { + const { run, n, cleanup } = await build() + const t0 = process.hrtime.bigint() + await run() + const t1 = process.hrtime.bigint() + await cleanup() + const sec = Number(t1 - t0) / 1e9 + rates.push(n / sec) + } + rates.sort((a, b) => a - b) + const perSec = rates[Math.floor(rates.length / 2)] + return { perSec, msPerOp: 1000 / perSec } +} + +// ---- write workloads ------------------------------------------------------ +// Every workload produces N entities and is identical except for the write path, +// so index-growth + payload costs cancel in the A-vs-B delta. + +function payload(i: number) { + return { + type: NounType.Document, + subtype: 'note', + data: `doc ${i}`, + vector: vec(i), + metadata: { i, batch: 'spike', tag: i % 7 } + } +} + +/** CREATE via single-op add() (today's path). */ +function createSingleOp(backend: 'memory' | 'filesystem', n: number, dirName: string) { + return async () => { + const dir = backend === 'filesystem' ? freshDir(dirName) : undefined + const brain = await mkBrain(backend, dir) + return { + n, + run: async () => { + for (let i = 0; i < n; i++) await brain.add(payload(i)) + }, + cleanup: async () => { + await brain.close() + if (dir) rmSync(dir, { recursive: true, force: true }) + } + } + } +} + +/** CREATE via transact([batch]) — batch=1 is Model B per-write upper bound. */ +function createTransact(backend: 'memory' | 'filesystem', n: number, batch: number, dirName: string) { + return async () => { + const dir = backend === 'filesystem' ? freshDir(dirName) : undefined + const brain = await mkBrain(backend, dir) + return { + n, + run: async () => { + for (let i = 0; i < n; i += batch) { + const ops = [] + for (let j = i; j < Math.min(i + batch, n); j++) { + ops.push({ op: 'add' as const, ...payload(j) }) + } + await brain.transact(ops as any) + } + }, + cleanup: async () => { + await brain.close() + if (dir) rmSync(dir, { recursive: true, force: true }) + } + } + } +} + +/** UPDATE churn (the case where Model B's before-image = a FULL old record). + * Seeds n entities via fast single-op add(), then times n updates via the path. */ +function updateSingleOp(backend: 'memory' | 'filesystem', n: number, dirName: string) { + return async () => { + const dir = backend === 'filesystem' ? freshDir(dirName) : undefined + const brain = await mkBrain(backend, dir) + const ids: string[] = [] + for (let i = 0; i < n; i++) ids.push(await brain.add(payload(i))) + return { + n, + run: async () => { + for (let i = 0; i < n; i++) await brain.update({ id: ids[i], metadata: { i, rev: 1 } }) + }, + cleanup: async () => { + await brain.close() + if (dir) rmSync(dir, { recursive: true, force: true }) + } + } + } +} + +function updateTransact(backend: 'memory' | 'filesystem', n: number, batch: number, dirName: string) { + return async () => { + const dir = backend === 'filesystem' ? freshDir(dirName) : undefined + const brain = await mkBrain(backend, dir) + const ids: string[] = [] + for (let i = 0; i < n; i++) ids.push(await brain.add(payload(i))) + return { + n, + run: async () => { + for (let i = 0; i < n; i += batch) { + const ops = [] + for (let j = i; j < Math.min(i + batch, n); j++) { + ops.push({ op: 'update' as const, id: ids[j], metadata: { i: j, rev: 1 } }) + } + await brain.transact(ops as any) + } + }, + cleanup: async () => { + await brain.close() + if (dir) rmSync(dir, { recursive: true, force: true }) + } + } + } +} + +// ---- history growth (filesystem only) ------------------------------------- + +/** Disk bytes for N adds via single-op vs via transact([1]); delta = retained history. */ +async function historyGrowthAdd(n: number) { + const dSingle = freshDir('hist-add-single') + let b = await mkBrain('filesystem', dSingle) + for (let i = 0; i < n; i++) await b.add(payload(i)) + await b.close() + const singleBytes = dirBytes(dSingle) + + const dTx = freshDir('hist-add-tx') + b = await mkBrain('filesystem', dTx) + for (let i = 0; i < n; i++) await b.transact([{ op: 'add', ...payload(i) }] as any) + await b.close() + const txBytes = dirBytes(dTx) + + rmSync(dSingle, { recursive: true, force: true }) + rmSync(dTx, { recursive: true, force: true }) + return { n, singleBytes, txBytes, retainedTotal: txBytes - singleBytes, retainedPerWrite: (txBytes - singleBytes) / n } +} + +/** Update-churn: n entities, each updated M times via transact([1]); disk delta vs the + * same entities updated via single-op update() (no history). Captures full-record before-images. */ +async function historyGrowthUpdateChurn(n: number, churn: number) { + const seed = (b: Brainy) => (async () => { + const ids: string[] = [] + for (let i = 0; i < n; i++) ids.push(await b.add(payload(i))) + return ids + })() + + const dSingle = freshDir('hist-upd-single') + let b = await mkBrain('filesystem', dSingle) + let ids = await seed(b) + for (let c = 0; c < churn; c++) for (let i = 0; i < n; i++) await b.update({ id: ids[i], metadata: { i, rev: c } }) + await b.close() + const singleBytes = dirBytes(dSingle) + + const dTx = freshDir('hist-upd-tx') + b = await mkBrain('filesystem', dTx) + ids = await seed(b) + for (let c = 0; c < churn; c++) for (let i = 0; i < n; i++) await b.transact([{ op: 'update', id: ids[i], metadata: { i, rev: c } }] as any) + await b.close() + const txBytes = dirBytes(dTx) + + rmSync(dSingle, { recursive: true, force: true }) + rmSync(dTx, { recursive: true, force: true }) + const writes = n * churn + return { writes, singleBytes, txBytes, retainedTotal: txBytes - singleBytes, retainedPerWrite: (txBytes - singleBytes) / writes } +} + +// ---- driver --------------------------------------------------------------- + +async function main() { + say(`[spike] start (scale=${SMALL ? 'small' : 'full'}); verifying no-WASM embedding...`) + // Sanity: a brain must construct + init fast (no WASM compile). + { + const t = process.hrtime.bigint() + const b = await mkBrain('memory') + await b.add(payload(0)) + await b.close() + say(`[spike] brain init + 1 write OK in ${(Number(process.hrtime.bigint() - t) / 1e6).toFixed(0)}ms (must be <2000ms or WASM is loading)`) + } + if (existsSync(FS_ROOT)) rmSync(FS_ROOT, { recursive: true, force: true }) + mkdirSync(FS_ROOT, { recursive: true }) + + const results: any = { meta: { N_MEM, N_FS, K, WARMUP, REPEATS, DIM, dim: DIM, small: SMALL }, throughput: {}, history: {} } + + // Warmup (JIT/alloc) on a memory brain. + { + const b = await mkBrain('memory') + for (let i = 0; i < WARMUP; i++) await b.add(payload(i)) + for (let i = 0; i < WARMUP; i += K) await b.transact([{ op: 'add', ...payload(i) }] as any) + await b.close() + } + + const fmt = (t: Timing) => `${Math.round(t.perSec).toLocaleString()}/s (${t.msPerOp.toFixed(3)} ms)` + + for (const backend of ['memory', 'filesystem'] as const) { + const n = backend === 'memory' ? N_MEM : N_FS + say(`\n========== ${backend.toUpperCase()} backend (N=${n.toLocaleString()}) ==========`) + + // CREATE + const addSingle = await timeWorkload('add-single', createSingleOp(backend, n, 'add-single')) + const addTx1 = await timeWorkload('add-tx1', createTransact(backend, n, 1, 'add-tx1')) + const addTxK = await timeWorkload('add-txK', createTransact(backend, n, K, 'add-txK')) + say(`CREATE add() ${fmt(addSingle)}`) + say(`CREATE transact([1]) ${fmt(addTx1)} <- Model B per-write (slowdown ${(addSingle.perSec / addTx1.perSec).toFixed(2)}x)`) + say(`CREATE transact([${K}]) ${fmt(addTxK)} <- group-commit (slowdown vs add() ${(addSingle.perSec / addTxK.perSec).toFixed(2)}x)`) + + // UPDATE + const updSingle = await timeWorkload('upd-single', updateSingleOp(backend, n, 'upd-single')) + const updTx1 = await timeWorkload('upd-tx1', updateTransact(backend, n, 1, 'upd-tx1')) + const updTxK = await timeWorkload('upd-txK', updateTransact(backend, n, K, 'upd-txK')) + say(`UPDATE update() ${fmt(updSingle)}`) + say(`UPDATE transact([1]) ${fmt(updTx1)} <- Model B per-write (slowdown ${(updSingle.perSec / updTx1.perSec).toFixed(2)}x)`) + say(`UPDATE transact([${K}]) ${fmt(updTxK)} <- group-commit (slowdown vs update() ${(updSingle.perSec / updTxK.perSec).toFixed(2)}x)`) + + results.throughput[backend] = { + n, + create: { add: addSingle, tx1: addTx1, txK: addTxK, slowdownTx1: addSingle.perSec / addTx1.perSec, slowdownTxK: addSingle.perSec / addTxK.perSec }, + update: { single: updSingle, tx1: updTx1, txK: updTxK, slowdownTx1: updSingle.perSec / updTx1.perSec, slowdownTxK: updSingle.perSec / updTxK.perSec } + } + } + + // HISTORY GROWTH (filesystem, NVMe) + say(`\n========== HISTORY GROWTH (filesystem NVMe) ==========`) + const hN = SMALL ? 500 : 3_000 + const addGrowth = await historyGrowthAdd(hN) + say(`ADD-only (${hN.toLocaleString()} adds): retained history = ${(addGrowth.retainedTotal / 1024).toFixed(0)} KiB total, ${addGrowth.retainedPerWrite.toFixed(0)} bytes/write`) + const churn = SMALL ? 3 : 5 + const updGrowth = await historyGrowthUpdateChurn(SMALL ? 200 : 1_000, churn) + say(`UPDATE-churn (${updGrowth.writes.toLocaleString()} updates): retained history = ${(updGrowth.retainedTotal / 1024 / 1024).toFixed(1)} MiB total, ${updGrowth.retainedPerWrite.toFixed(0)} bytes/write`) + results.history = { addGrowth, updGrowth } + + // persist raw JSON for the verification workflow + const outPath = `${FS_ROOT}/../tests/benchmarks/_spike-results.json` + const fs = await import('node:fs') + fs.writeFileSync(`/media/dpsifr/storage/home/Projects/brainy/tests/benchmarks/_spike-results.json`, JSON.stringify(results, null, 2)) + say(`\nRaw results → tests/benchmarks/_spike-results.json`) + + // cleanup data root + if (existsSync(FS_ROOT)) rmSync(FS_ROOT, { recursive: true, force: true }) +} + +main() + .then(() => { + say('[spike] done.') + process.exit(0) // force exit — brain caches/timers can keep the loop alive + }) + .catch((e) => { + console.error(e) + process.exit(1) + }) diff --git a/tests/brainy-3.test.ts b/tests/brainy-3.test.ts deleted file mode 100644 index 8db458ce..00000000 --- a/tests/brainy-3.test.ts +++ /dev/null @@ -1,431 +0,0 @@ -/** - * Brainy 3.0 API Tests - * Comprehensive tests for the new beautiful, consistent API - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy, NounType, VerbType } from '../src/brainy.js' - -describe('Brainy 3.0 API', () => { - let brain: Brainy - - beforeEach(async () => { - brain = new Brainy({ requireSubtype: false, - storage: { type: 'memory' } - }) - await brain.init() - }) - - afterEach(async () => { - await brain.close() - }) - - describe('Entity Operations', () => { - it('should add entities with beautiful API', async () => { - const id = await brain.add({ - data: 'Test document about machine learning', - type: NounType.Document, - metadata: { - title: 'ML Guide', - author: 'Test Author' - }, - service: 'test-service' - }) - - expect(id).toBeTypeOf('string') - expect(id).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/) - }) - - it('should retrieve entities', async () => { - const id = await brain.add({ - data: 'Sample data', - type: NounType.Document, - metadata: { title: 'Sample Title' } - }) - - // v5.11.1: Need includeVectors to check vectors - const entity = await brain.get(id, { includeVectors: true }) - - expect(entity).toBeDefined() - expect(entity!.id).toBe(id) - expect(entity!.type).toBe(NounType.Document) - expect(entity!.metadata.title).toBe('Sample Title') - expect(entity!.vector).toBeDefined() - expect(entity!.vector.length).toBeGreaterThan(0) - }) - - it('should update entities', async () => { - const id = await brain.add({ - data: 'Original data', - type: NounType.Document, - metadata: { title: 'Original Title' } - }) - - await brain.update({ - id, - metadata: { title: 'Updated Title', version: 2 } - }) - - const entity = await brain.get(id) - expect(entity!.metadata.title).toBe('Updated Title') - expect(entity!.metadata.version).toBe(2) - }) - - it('should delete entities', async () => { - const id = await brain.add({ - data: 'To be deleted', - type: NounType.Document - }) - - await brain.remove(id) - - const entity = await brain.get(id) - expect(entity).toBeNull() - }) - }) - - describe('Relationship Operations', () => { - it('should create relationships', async () => { - const doc1 = await brain.add({ - data: 'First document', - type: NounType.Document - }) - - const doc2 = await brain.add({ - data: 'Second document', - type: NounType.Document - }) - - const relationId = await brain.relate({ - from: doc1, - to: doc2, - type: VerbType.References, - weight: 0.8, - metadata: { context: 'test' } - }) - - expect(relationId).toBeTypeOf('string') - }) - - it('should get relationships', async () => { - const doc1 = await brain.add({ - data: 'Document one', - type: NounType.Document - }) - - const doc2 = await brain.add({ - data: 'Document two', - type: NounType.Document - }) - - await brain.relate({ - from: doc1, - to: doc2, - type: VerbType.References - }) - - const relations = await brain.related({ from: doc1 }) - - expect(relations).toHaveLength(1) - expect(relations[0].from).toBe(doc1) - expect(relations[0].to).toBe(doc2) - expect(relations[0].type).toBe(VerbType.References) - }) - - it('should create bidirectional relationships', async () => { - const person = await brain.add({ - data: 'John Smith', - type: NounType.Person - }) - - const org = await brain.add({ - data: 'Tech Corp', - type: NounType.Organization - }) - - await brain.relate({ - from: person, - to: org, - type: VerbType.WorksWith, - bidirectional: true - }) - - const fromPerson = await brain.related({ from: person }) - const toPerson = await brain.related({ to: person }) - - expect(fromPerson).toHaveLength(1) - expect(toPerson).toHaveLength(1) - }) - - it('should delete relationships', async () => { - const doc1 = await brain.add({ - data: 'Doc 1', - type: NounType.Document - }) - - const doc2 = await brain.add({ - data: 'Doc 2', - type: NounType.Document - }) - - const relationId = await brain.relate({ - from: doc1, - to: doc2, - type: VerbType.References - }) - - await brain.unrelate(relationId) - - const relations = await brain.related({ from: doc1 }) - expect(relations).toHaveLength(0) - }) - }) - - describe('Search Operations', () => { - beforeEach(async () => { - // Add test data - await brain.add({ - data: 'Machine learning is a subset of artificial intelligence', - type: NounType.Document, - metadata: { category: 'AI', importance: 5 } - }) - - await brain.add({ - data: 'Neural networks are used in deep learning', - type: NounType.Document, - metadata: { category: 'AI', importance: 4 } - }) - - await brain.add({ - data: 'JavaScript is a programming language', - type: NounType.Document, - metadata: { category: 'Programming', importance: 3 } - }) - }) - - it('should find entities by text query', async () => { - const results = await brain.find({ - query: 'machine learning artificial intelligence', - limit: 2 - }) - - expect(results).toHaveLength(2) - expect(results[0].score).toBeGreaterThan(0) - expect(results[0].entity.type).toBe(NounType.Document) - }) - - it('should find similar entities', async () => { - const aiDocId = await brain.add({ - data: 'Deep learning algorithms', - type: NounType.Document - }) - - const results = await brain.similar({ - to: aiDocId, - limit: 2 - }) - - expect(results.length).toBeGreaterThan(0) - expect(results[0].entity.id).not.toBe(aiDocId) // Shouldn't include self - }) - - it('should filter by metadata', async () => { - const results = await brain.find({ - query: 'learning', - where: { category: 'AI' }, - limit: 5 - }) - - results.forEach(result => { - expect(result.entity.metadata.category).toBe('AI') - }) - }) - - it('should filter by entity type', async () => { - await brain.add({ - data: 'John Doe', - type: NounType.Person - }) - - const results = await brain.find({ - query: 'learning', - type: NounType.Document - }) - - results.forEach(result => { - expect(result.entity.type).toBe(NounType.Document) - }) - }) - - it('should support pagination', async () => { - const page1 = await brain.find({ - query: 'learning', - limit: 1, - offset: 0 - }) - - const page2 = await brain.find({ - query: 'learning', - limit: 1, - offset: 1 - }) - - expect(page1).toHaveLength(1) - expect(page2).toHaveLength(1) - expect(page1[0].entity.id).not.toBe(page2[0].entity.id) - }) - }) - - describe('Batch Operations', () => { - it('should add multiple entities', async () => { - const result = await brain.addMany({ - items: [ - { - data: 'Document 1', - type: NounType.Document, - metadata: { index: 1 } - }, - { - data: 'Document 2', - type: NounType.Document, - metadata: { index: 2 } - }, - { - data: 'Document 3', - type: NounType.Document, - metadata: { index: 3 } - } - ] - }) - - expect(result.successful).toHaveLength(3) - expect(result.failed).toHaveLength(0) - expect(result.total).toBe(3) - expect(result.duration).toBeGreaterThan(0) - }) - - it('should handle batch errors gracefully', async () => { - const result = await brain.addMany({ - items: [ - { - data: 'Valid document', - type: NounType.Document - }, - { - data: null, // Invalid data to trigger error - type: NounType.Document - } - ], - continueOnError: true - }) - - expect(result.successful).toHaveLength(1) - expect(result.failed).toHaveLength(1) - }) - - it('should delete multiple entities', async () => { - // Add test entities - const ids = await Promise.all([ - brain.add({ data: 'Delete me 1', type: NounType.Document, metadata: { delete: true } }), - brain.add({ data: 'Delete me 2', type: NounType.Document, metadata: { delete: true } }), - brain.add({ data: 'Keep me', type: NounType.Document, metadata: { delete: false } }) - ]) - - const result = await brain.removeMany({ - where: { delete: true } - }) - - expect(result.successful).toHaveLength(2) - expect(result.failed).toHaveLength(0) - - // Verify entities were deleted - const remaining = await brain.get(ids[2]) - expect(remaining).toBeDefined() - }) - }) - - describe('Type Safety', () => { - it('should enforce NounType enum', async () => { - const id = await brain.add({ - data: 'Test', - type: NounType.Document // Must use enum - }) - - const entity = await brain.get(id) - expect(entity!.type).toBe(NounType.Document) - }) - - it('should enforce VerbType enum', async () => { - const doc1 = await brain.add({ data: 'Doc 1', type: NounType.Document }) - const doc2 = await brain.add({ data: 'Doc 2', type: NounType.Document }) - - await brain.relate({ - from: doc1, - to: doc2, - type: VerbType.References // Must use enum - }) - - const relations = await brain.related({ from: doc1 }) - expect(relations[0].type).toBe(VerbType.References) - }) - }) - - describe('Error Handling', () => { - it('should handle missing entities gracefully', async () => { - const nonExistentId = '00000000-0000-0000-0000-000000000000' - const entity = await brain.get(nonExistentId) - expect(entity).toBeNull() - }) - - it('should require initialization before operations', async () => { - const uninitializedBrain = new Brainy({ requireSubtype: false }) - - await expect(uninitializedBrain.add({ - data: 'Test', - type: NounType.Document - })).rejects.toThrow('not initialized') - }) - - it('should validate required parameters', async () => { - // @ts-expect-error - Testing validation - await expect(brain.add({})).rejects.toThrow() - }) - }) - - describe('Configuration', () => { - it('should support custom configuration', async () => { - const customBrain = new Brainy({ requireSubtype: false, - storage: { type: 'memory' }, - model: { type: 'fast' }, - cache: true, - warmup: false - }) - - await customBrain.init() - - const id = await customBrain.add({ - data: 'Test with custom config', - type: NounType.Document - }) - - expect(id).toBeTypeOf('string') - await customBrain.close() - }) - }) -}) - -describe('Brainy 3.0 Neural API', () => { - let brain: Brainy - - beforeEach(async () => { - brain = new Brainy({ requireSubtype: false, - storage: { type: 'memory' }, - }) - await brain.init() - }) - - afterEach(async () => { - await brain.close() - }) - -}) \ No newline at end of file diff --git a/tests/comprehensive/core-api.test.ts b/tests/comprehensive/core-api.test.ts deleted file mode 100644 index 3970efee..00000000 --- a/tests/comprehensive/core-api.test.ts +++ /dev/null @@ -1,394 +0,0 @@ -/** - * Brainy v3.0 Core API Test Suite - * Testing all core CRUD operations and basic functionality - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../src/brainy.js' -import { NounType, VerbType } from '../../src/types/graphTypes.js' - -describe('Brainy v3.0 Core API Tests', () => { - let brain: Brainy - - beforeEach(async () => { - brain = new Brainy({ requireSubtype: false, - storage: { type: 'memory' } - }) - await brain.init() - }) - - afterEach(async () => { - // Note: Brainy doesn't have shutdown() in v3, just let GC handle it - brain = null as any - }) - - describe('1. Add Operations', () => { - it('should add a simple text item', async () => { - const id = await brain.add({ - data: 'Hello world', - type: NounType.Document - }) - - expect(id).toBeDefined() - expect(typeof id).toBe('string') - expect(id.length).toBeGreaterThan(0) - }) - - it('should add item with custom ID', async () => { - const customId = 'custom-123' - const id = await brain.add({ - id: customId, - data: 'Custom ID test', - type: NounType.Document - }) - - expect(id).toBe(customId) - }) - - it('should add item with metadata', async () => { - const metadata = { - title: 'Test Doc', - category: 'testing', - score: 95.5, - tags: ['test', 'validation'] - } - - const id = await brain.add({ - data: 'Document with metadata', - metadata, - type: NounType.Document - }) - - const retrieved = await brain.get(id) - expect(retrieved?.metadata).toEqual(metadata) - }) - - it('should add item with pre-computed vector', async () => { - const vector = new Array(384).fill(0).map(() => Math.random()) - const id = await brain.add({ - data: 'Pre-vectorized content', - vector, - type: NounType.Thing - }) - - const retrieved = await brain.get(id) - expect(retrieved).toBeDefined() - expect(retrieved?.vector).toHaveLength(384) - }) - - it('should handle concurrent adds', async () => { - const promises = Array(10).fill(0).map((_, i) => - brain.add({ - data: `Concurrent item ${i}`, - type: NounType.Document - }) - ) - - const ids = await Promise.all(promises) - expect(ids).toHaveLength(10) - expect(new Set(ids).size).toBe(10) // All unique - }) - - it('should validate noun types', async () => { - // Valid type should work - const id = await brain.add({ - data: 'Valid type', - type: NounType.Person - }) - expect(id).toBeDefined() - - // Invalid type should throw - await expect(brain.add({ - data: 'Invalid type test', - type: 'InvalidType' as any - })).rejects.toThrow() - }) - }) - - describe('2. Get Operations', () => { - let testId: string - - beforeEach(async () => { - testId = await brain.add({ - data: 'Test document for retrieval', - metadata: { test: true }, - type: NounType.Document - }) - }) - - it('should retrieve existing item by ID', async () => { - const item = await brain.get(testId) - - expect(item).toBeDefined() - expect(item?.id).toBe(testId) - expect(item?.metadata?.test).toBe(true) - expect(item?.type).toBe(NounType.Document) - }) - - it('should return null for non-existent ID', async () => { - const item = await brain.get('non-existent-id') - expect(item).toBeNull() - }) - - it('should retrieve with vector included', async () => { - const item = await brain.get(testId) - expect(item?.vector).toBeDefined() - expect(item?.vector?.length).toBeGreaterThanOrEqual(384) // Default dimension - }) - }) - - describe('3. Update Operations', () => { - let testId: string - - beforeEach(async () => { - testId = await brain.add({ - data: 'Original content', - metadata: { version: 1 }, - type: NounType.Document - }) - }) - - it('should update existing item data', async () => { - const success = await brain.update({ - id: testId, - data: 'Updated content' - }) - - expect(success).toBe(true) - - const updated = await brain.get(testId) - expect(updated).toBeDefined() - // Vector should be recalculated after content update - }) - - it('should update only metadata', async () => { - const success = await brain.update({ - id: testId, - metadata: { version: 2, updated: true } - }) - - expect(success).toBe(true) - - const updated = await brain.get(testId) - expect(updated?.metadata).toEqual({ version: 2, updated: true }) - }) - - it('should update both data and metadata', async () => { - const success = await brain.update({ - id: testId, - data: 'New content', - metadata: { version: 3 } - }) - - expect(success).toBe(true) - - const updated = await brain.get(testId) - expect(updated?.metadata?.version).toBe(3) - }) - - it('should return false for non-existent ID', async () => { - const success = await brain.update({ - id: 'non-existent', - data: 'Will fail' - }) - - expect(success).toBe(false) - }) - }) - - describe('4. Delete Operations', () => { - it('should delete existing item', async () => { - const id = await brain.add({ - data: 'To be deleted', - type: NounType.Document - }) - - const success = await brain.remove(id) - expect(success).toBe(true) - - const item = await brain.get(id) - expect(item).toBeNull() - }) - - it('should return false for non-existent ID', async () => { - const success = await brain.remove('non-existent') - expect(success).toBe(false) - }) - - it('should handle concurrent deletes', async () => { - const ids = await Promise.all( - Array(5).fill(0).map(() => - brain.add({ data: 'Concurrent delete test', type: NounType.Document }) - ) - ) - - await Promise.all(ids.map(id => brain.remove(id))) - // delete returns void, not boolean - - // Verify all deleted - const items = await Promise.all(ids.map(id => brain.get(id))) - expect(items.every(item => item === null)).toBe(true) - }) - }) - - describe('5. Relationship Operations', () => { - let entityA: string - let entityB: string - - beforeEach(async () => { - entityA = await brain.add({ data: 'Entity A', type: NounType.Person }) - entityB = await brain.add({ data: 'Entity B', type: NounType.Organization }) - }) - - it('should create relationships between entities', async () => { - const verbId = await brain.relate({ - from: entityA, - to: entityB, - type: VerbType.MemberOf, - metadata: { since: '2023' } - }) - - expect(verbId).toBeDefined() - expect(typeof verbId).toBe('string') - }) - - it('should retrieve relationships', async () => { - await brain.relate({ - from: entityA, - to: entityB, - type: VerbType.Creates - }) - - const relations = await brain.related({ from: entityA }) - - expect(relations.length).toBeGreaterThanOrEqual(1) - expect(relations[0].from).toBe(entityA) - }) - - it('should delete relationships', async () => { - const verbId = await brain.relate({ - from: entityA, - to: entityB, - type: VerbType.Owns - }) - - const success = await brain.unrelate(verbId) - expect(success).toBe(true) - - const relations = await brain.related({ from: entityA }) - const found = relations.find(r => r.id === verbId) - expect(found).toBeUndefined() - }) - }) - - describe('6. Batch Operations', () => { - it('should add multiple items in batch', async () => { - const items = Array(10).fill(0).map((_, i) => ({ - data: `Batch item ${i}`, - metadata: { index: i }, - type: NounType.Document - })) - - const result = await brain.addMany({ items }) - - expect(result.successful).toHaveLength(10) - expect(result.failed).toHaveLength(0) - expect(result.total).toBe(10) - }) - - it('should delete multiple items in batch', async () => { - // First add some items - const ids = await Promise.all( - Array(5).fill(0).map((_, i) => - brain.add({ data: `Delete batch ${i}`, type: NounType.Document }) - ) - ) - - // Then delete them in batch - const result = await brain.removeMany({ ids }) - - expect(result.successful).toHaveLength(5) - expect(result.failed).toHaveLength(0) - }) - - it('should handle partial batch failures', async () => { - const items = [ - { data: 'Valid item', type: NounType.Document }, - { data: 'Invalid item', type: 'InvalidType' as any }, - { data: 'Another valid', type: NounType.Document } - ] - - const result = await brain.addMany({ - items, - continueOnError: true - }) - - expect(result.successful.length).toBeGreaterThanOrEqual(2) - expect(result.failed.length).toBeGreaterThanOrEqual(1) - }) - }) - - describe('7. Import/Export Operations', () => { - it('should export all data', async () => { - // Add some test data - const id1 = await brain.add({ data: 'Export test 1', type: NounType.Document }) - const id2 = await brain.add({ data: 'Export test 2', type: NounType.Document }) - await brain.relate({ from: id1, to: id2, type: VerbType.References }) - - const exported = await brain.export() - - expect(exported).toBeDefined() - expect(exported.entities).toBeDefined() - expect(exported.relationships).toBeDefined() - expect(exported.metadata).toBeDefined() - }) - - it('should import data', async () => { - // Create export data - const exportData = { - entities: [ - { id: 'imp-1', data: 'Imported 1', type: NounType.Document, metadata: {} }, - { id: 'imp-2', data: 'Imported 2', type: NounType.Document, metadata: {} } - ], - relationships: [], - metadata: { version: '3.0.0' } - } - - await brain.import(exportData) - - // Verify imported data - const item1 = await brain.get('imp-1') - const item2 = await brain.get('imp-2') - - expect(item1).toBeDefined() - expect(item2).toBeDefined() - }) - }) - - describe('8. Statistics and Insights', () => { - beforeEach(async () => { - // Add some test data - await brain.add({ data: 'Doc 1', type: NounType.Document }) - await brain.add({ data: 'Person 1', type: NounType.Person }) - await brain.add({ data: 'Org 1', type: NounType.Organization }) - }) - - it('should provide insights', async () => { - const insights = await brain.insights() - - expect(insights).toBeDefined() - expect(insights.entities).toBeGreaterThanOrEqual(3) - expect(insights.types).toBeDefined() - expect(Object.keys(insights.types).length).toBeGreaterThan(0) - }) - - it('should suggest relevant queries', async () => { - const suggestions = await brain.suggest({ limit: 3 }) - - expect(suggestions).toBeDefined() - expect(suggestions.queries).toBeDefined() - expect(Array.isArray(suggestions.queries)).toBe(true) - expect(suggestions.queries.length).toBeLessThanOrEqual(3) - }) - }) -}) \ No newline at end of file diff --git a/tests/comprehensive/find-triple-intelligence.test.ts b/tests/comprehensive/find-triple-intelligence.test.ts deleted file mode 100644 index e385fd25..00000000 --- a/tests/comprehensive/find-triple-intelligence.test.ts +++ /dev/null @@ -1,367 +0,0 @@ -/** - * Brainy v3.0 Find & Triple Intelligence Test Suite - * Testing vector search, metadata filtering, and fusion strategies - */ - -import { describe, it, expect, beforeAll, afterAll } from 'vitest' -import { Brainy } from '../../src/brainy.js' -import { NounType, VerbType } from '../../src/types/graphTypes.js' - -describe('Find and Triple Intelligence', () => { - let brain: Brainy - let testData: any[] - - beforeAll(async () => { - brain = new Brainy({ requireSubtype: false, - storage: { type: 'memory' } - }) - await brain.init() - - // Add diverse test data - testData = [ - // Technology items - { data: 'JavaScript is a programming language for web development', metadata: { category: 'tech', language: 'javascript', difficulty: 'medium' }, type: NounType.Document }, - { data: 'TypeScript adds static typing to JavaScript', metadata: { category: 'tech', language: 'typescript', difficulty: 'advanced' }, type: NounType.Document }, - { data: 'Python is great for data science and AI', metadata: { category: 'tech', language: 'python', difficulty: 'easy' }, type: NounType.Document }, - { data: 'React is a JavaScript library for building UIs', metadata: { category: 'tech', framework: 'react', language: 'javascript' }, type: NounType.Document }, - - // Science items - { data: 'Quantum computing uses quantum mechanics principles', metadata: { category: 'science', field: 'physics', complexity: 'high' }, type: NounType.Concept }, - { data: 'Machine learning enables computers to learn from data', metadata: { category: 'science', field: 'ai', complexity: 'medium' }, type: NounType.Concept }, - { data: 'Neural networks are inspired by biological brains', metadata: { category: 'science', field: 'ai', complexity: 'high' }, type: NounType.Concept }, - - // Business items - { data: 'Market analysis helps understand consumer behavior', metadata: { category: 'business', domain: 'marketing', importance: 'high' }, type: NounType.Process }, - { data: 'Project management ensures successful delivery', metadata: { category: 'business', domain: 'management', importance: 'critical' }, type: NounType.Process }, - { data: 'Financial planning is essential for growth', metadata: { category: 'business', domain: 'finance', importance: 'critical' }, type: NounType.Process } - ] - - // Add all test data - for (const item of testData) { - await brain.add(item) - } - }) - - afterAll(() => { - brain = null as any - }) - - describe('Vector Search', () => { - it('should find similar items by text query', async () => { - const results = await brain.find({ - query: 'JavaScript programming', - limit: 3 - }) - - expect(results).toHaveLength(3) - expect(results[0].score).toBeGreaterThan(0) - expect(results[0].score).toBeLessThanOrEqual(1) - - // Should find JavaScript-related items first - const topResult = results[0].entity - expect(topResult.metadata?.language).toBeDefined() - }) - - it('should respect limit parameter', async () => { - const results = await brain.find({ - query: 'technology', - limit: 5 - }) - - expect(results.length).toBeLessThanOrEqual(5) - }) - - it('should find by vector directly', async () => { - // Get vector from an existing item - const jsItem = await brain.find({ query: 'JavaScript', limit: 1 }) - const vector = jsItem[0].entity.vector - - const results = await brain.find({ - vector, - limit: 3 - }) - - expect(results).toHaveLength(3) - // First result should be very similar (same or nearly same vector) - expect(results[0].score).toBeGreaterThan(0.9) - }) - - it('should return all items when no query provided', async () => { - const results = await brain.find({}) - - expect(results.length).toBeGreaterThanOrEqual(testData.length) - }) - }) - - describe('Metadata Filtering', () => { - it('should filter by single metadata field', async () => { - const results = await brain.find({ - where: { category: 'tech' } - }) - - expect(results.length).toBeGreaterThan(0) - results.forEach(r => { - expect(r.entity.metadata?.category).toBe('tech') - }) - }) - - it('should filter by multiple metadata fields', async () => { - const results = await brain.find({ - where: { - category: 'tech', - language: 'javascript' - } - }) - - results.forEach(r => { - expect(r.entity.metadata?.category).toBe('tech') - expect(r.entity.metadata?.language).toBe('javascript') - }) - }) - - it('should support $gte operator', async () => { - const results = await brain.find({ - where: { - importance: { $gte: 'high' } - } - }) - - results.forEach(r => { - const importance = r.entity.metadata?.importance - if (importance) { - expect(['high', 'critical']).toContain(importance) - } - }) - }) - - it('should support $contains operator for arrays', async () => { - // Add item with array metadata - await brain.add({ - data: 'Test with tags', - metadata: { tags: ['test', 'validation', 'qa'] }, - type: NounType.Document - }) - - const results = await brain.find({ - where: { - tags: { $contains: 'test' } - } - }) - - const found = results.find(r => - Array.isArray(r.entity.metadata?.tags) && - r.entity.metadata.tags.includes('test') - ) - expect(found).toBeDefined() - }) - - it('should handle complex nested filters', async () => { - const results = await brain.find({ - where: { - $or: [ - { category: 'tech', language: 'javascript' }, - { category: 'science', field: 'ai' } - ] - } - }) - - results.forEach(r => { - const meta = r.entity.metadata - const matchesTech = meta?.category === 'tech' && meta?.language === 'javascript' - const matchesScience = meta?.category === 'science' && meta?.field === 'ai' - expect(matchesTech || matchesScience).toBe(true) - }) - }) - }) - - describe('Type Filtering', () => { - it('should filter by single noun type', async () => { - const results = await brain.find({ - type: NounType.Document - }) - - results.forEach(r => { - expect(r.entity.type).toBe(NounType.Document) - }) - }) - - it('should filter by multiple noun types', async () => { - const results = await brain.find({ - type: [NounType.Document, NounType.Concept] - }) - - results.forEach(r => { - expect([NounType.Document, NounType.Concept]).toContain(r.entity.type) - }) - }) - }) - - describe('Combined Search (Triple Intelligence)', () => { - it('should combine vector search with metadata filtering', async () => { - const results = await brain.find({ - query: 'programming', - where: { category: 'tech' }, - limit: 5 - }) - - expect(results.length).toBeGreaterThan(0) - results.forEach(r => { - expect(r.entity.metadata?.category).toBe('tech') - }) - }) - - it('should combine vector, metadata, and type filtering', async () => { - const results = await brain.find({ - query: 'artificial intelligence', - where: { category: 'science' }, - type: NounType.Concept, - limit: 5 - }) - - results.forEach(r => { - expect(r.entity.metadata?.category).toBe('science') - expect(r.entity.type).toBe(NounType.Concept) - }) - }) - - it('should support fusion strategies', async () => { - const results = await brain.find({ - query: 'JavaScript', - where: { category: 'tech' }, - fusion: { - strategy: 'adaptive', - weights: { vector: 0.7, field: 0.3 } - } - }) - - expect(results).toBeDefined() - expect(results.length).toBeGreaterThan(0) - }) - - it('should handle fusion with graph intelligence', async () => { - // Create some relationships - const items = await brain.find({ limit: 3 }) - if (items.length >= 2) { - await brain.relate({ - from: items[0].id, - to: items[1].id, - type: VerbType.References - }) - } - - const results = await brain.find({ - query: 'technology', - connected: { to: items[0]?.id }, - fusion: { - strategy: 'adaptive', - weights: { vector: 0.5, graph: 0.3, field: 0.2 } - } - }) - - expect(results).toBeDefined() - }) - }) - - describe('Performance', () => { - it('should handle large result sets efficiently', async () => { - const start = Date.now() - const results = await brain.find({ limit: 100 }) - const elapsed = Date.now() - start - - expect(elapsed).toBeLessThan(1000) // Should complete in under 1 second - expect(results.length).toBeLessThanOrEqual(100) - }) - - it('should cache repeated searches', async () => { - const query = { query: 'caching test', limit: 5 } - - // First search - const results1 = await brain.find(query) - - // Second search (should hit cache) - const results2 = await brain.find(query) - - expect(results1).toEqual(results2) - // Note: Cache might not always be faster in tests due to overhead - // But results should be identical - }) - }) - - describe('Edge Cases', () => { - it('should handle empty query gracefully', async () => { - const results = await brain.find({ query: '' }) - expect(results).toBeDefined() - expect(Array.isArray(results)).toBe(true) - }) - - it('should handle non-matching filters', async () => { - const results = await brain.find({ - where: { nonExistentField: 'value' } - }) - - expect(results).toBeDefined() - expect(Array.isArray(results)).toBe(true) - }) - - it('should handle very long queries', async () => { - const longQuery = 'test '.repeat(1000) // 5000 characters - const results = await brain.find({ query: longQuery, limit: 1 }) - - expect(results).toBeDefined() - expect(Array.isArray(results)).toBe(true) - }) - - it('should handle special characters in queries', async () => { - const specialQuery = '!@#$%^&*()_+-=[]{}|;\':",./<>?' - const results = await brain.find({ query: specialQuery, limit: 1 }) - - expect(results).toBeDefined() - expect(Array.isArray(results)).toBe(true) - }) - - it('should handle unicode in queries', async () => { - const unicodeQuery = '你好世界 🌍 مرحبا بالعالم' - const results = await brain.find({ query: unicodeQuery, limit: 1 }) - - expect(results).toBeDefined() - expect(Array.isArray(results)).toBe(true) - }) - }) - - describe('Similarity Search', () => { - it('should find similar items to a given ID', async () => { - const items = await brain.find({ limit: 1 }) - if (items.length > 0) { - const results = await brain.similar({ - to: items[0].id, - limit: 3 - }) - - expect(results).toHaveLength(3) - // First result might be the item itself - results.forEach(r => { - expect(r.score).toBeGreaterThanOrEqual(0) - expect(r.score).toBeLessThanOrEqual(1) - }) - } - }) - - it('should exclude the source item from similar results', async () => { - const items = await brain.find({ limit: 1 }) - if (items.length > 0) { - const results = await brain.similar({ - to: items[0].id, - limit: 5 - }) - - // Check if source is excluded (implementation dependent) - const selfMatch = results.find(r => r.id === items[0].id) - if (selfMatch) { - // If included, should be first with score ~1 - expect(results[0].id).toBe(items[0].id) - expect(results[0].score).toBeGreaterThan(0.99) - } - } - }) - }) -}) \ No newline at end of file diff --git a/tests/comprehensive/public-api-complete.test.ts b/tests/comprehensive/public-api-complete.unit.test.ts similarity index 100% rename from tests/comprehensive/public-api-complete.test.ts rename to tests/comprehensive/public-api-complete.unit.test.ts diff --git a/tests/critical-error-handling.test.ts b/tests/critical-error-handling.test.ts deleted file mode 100644 index d19ae3c6..00000000 --- a/tests/critical-error-handling.test.ts +++ /dev/null @@ -1,441 +0,0 @@ -import { describe, it, expect, beforeAll, afterAll } from 'vitest' -import { Brainy } from '../src/brainy' - -describe('CRITICAL: Error Handling and Edge Cases', () => { - let brainy: Brainy - - beforeAll(async () => { - brainy = new Brainy({ requireSubtype: false, - storage: { type: 'memory' } - }) - await brainy.init() - }) - - afterAll(async () => { - await brainy.close() - }) - - describe('Invalid Input Handling', () => { - it('should handle null and undefined gracefully', async () => { - await expect(brainy.add({ - data: null as any, - type: 'document' - })).rejects.toThrow() - - await expect(brainy.add({ - data: undefined as any, - type: 'document' - })).rejects.toThrow() - - const result = await brainy.get(null as any) - expect(result).toBeNull() - }) - - it('should validate noun types', async () => { - await expect(brainy.add({ - data: { content: 'test' }, - type: 'invalid-type' as any - })).rejects.toThrow() - }) - - it('should validate verb types in relationships', async () => { - await brainy.add({ id: 'node1', data: { name: 'Node 1' }, type: 'entity' }) - await brainy.add({ id: 'node2', data: { name: 'Node 2' }, type: 'entity' }) - - await expect(brainy.relate({ - from: 'node1', - to: 'node2', - type: 'invalid-verb' as any - })).rejects.toThrow() - }) - - it('should handle extremely long strings', async () => { - const longString = 'x'.repeat(1000000) - - const id = await brainy.add({ - data: { content: longString }, - type: 'document' - }) - - expect(id).toBeDefined() - - const retrieved = await brainy.get(id) - expect(retrieved).toBeDefined() - }) - - it('should handle circular references in data', async () => { - const circular: any = { name: 'test' } - circular.self = circular - - await expect(brainy.add({ - data: circular, - type: 'entity' - })).rejects.toThrow() - }) - - it('should handle special characters in IDs', async () => { - const specialIds = [ - 'id with spaces', - 'id/with/slashes', - 'id\\with\\backslashes', - 'id:with:colons', - 'id|with|pipes', - 'id"with"quotes', - 'idbrackets' - ] - - for (const id of specialIds) { - const result = await brainy.add({ - id, - data: { content: `Test ${id}` }, - type: 'document' - }) - - expect(result).toBe(id) - - const retrieved = await brainy.get(id) - expect(retrieved).toBeDefined() - } - }) - }) - - describe('Concurrent Operation Safety', () => { - it('should handle concurrent writes to same ID', async () => { - const id = 'concurrent-test' - - const promises = Array.from({ length: 10 }, (_, i) => - brainy.add({ - id, - data: { content: `Version ${i}` }, - type: 'document' - }) - ) - - await Promise.all(promises) - - const final = await brainy.get(id) - expect(final).toBeDefined() - }) - - it('should handle concurrent updates', async () => { - const id = await brainy.add({ - data: { counter: 0 }, - type: 'entity' - }) - - const updates = Array.from({ length: 10 }, (_, i) => - brainy.update({ - id, - data: { counter: i } - }) - ) - - await Promise.all(updates) - - const final = await brainy.get(id) - expect(final).toBeDefined() - }) - - it('should handle concurrent deletes', async () => { - const ids = await Promise.all( - Array.from({ length: 10 }, (_, i) => - brainy.add({ - data: { index: i }, - type: 'entity' - }) - ) - ) - - const deletes = ids.map(id => brainy.remove(id)) - await Promise.all(deletes) - - for (const id of ids) { - const result = await brainy.get(id) - expect(result).toBeNull() - } - }) - }) - - describe('Memory and Resource Management', () => { - it('should not leak memory on failed operations', async () => { - const memBefore = process.memoryUsage().heapUsed - - for (let i = 0; i < 1000; i++) { - try { - await brainy.add({ - data: null as any, - type: 'invalid' as any - }) - } catch { - // Expected to fail - } - } - - global.gc?.() - const memAfter = process.memoryUsage().heapUsed - const leak = memAfter - memBefore - - expect(leak).toBeLessThan(10 * 1024 * 1024) // Less than 10MB - }) - - it('should handle out of memory scenarios gracefully', async () => { - const hugeArray = Array.from({ length: 100000 }, (_, i) => ({ - id: `huge-${i}`, - data: { - content: 'x'.repeat(10000), - metadata: Array.from({ length: 100 }, () => Math.random()) - }, - type: 'document' as const - })) - - try { - await brainy.addMany({ items: hugeArray }) - } catch (error: any) { - expect(error).toBeDefined() - } - }) - }) - - describe('Query Edge Cases', () => { - it('should handle empty search queries', async () => { - const results = await brainy.find('') - expect(Array.isArray(results)).toBe(true) - }) - - it('should handle queries with only special characters', async () => { - const specialQueries = [ - '!!!', - '???', - '...', - '---', - '___', - '@#$%^&*()', - '{}[]|\\', - '<>?/~`' - ] - - for (const query of specialQueries) { - const results = await brainy.find(query) - expect(Array.isArray(results)).toBe(true) - } - }) - - it('should handle metadata filters with undefined values', async () => { - await brainy.add({ - data: { name: 'Test', value: undefined }, - type: 'entity' - }) - - const results = await brainy.find({ - where: { value: undefined as any } - }) - - expect(Array.isArray(results)).toBe(true) - }) - - it('should handle complex nested metadata queries', async () => { - await brainy.add({ - data: { - nested: { - deep: { - deeper: { - value: 42 - } - } - } - }, - type: 'entity' - }) - - const results = await brainy.find({ - where: { - 'nested.deep.deeper.value': 42 - } - }) - - expect(results.length).toBeGreaterThan(0) - }) - }) - - describe('Relationship Error Cases', () => { - it('should handle relationships to non-existent nodes', async () => { - await expect(brainy.relate({ - from: 'non-existent-1', - to: 'non-existent-2', - type: 'relatedTo' - })).rejects.toThrow() - }) - - it('should handle self-referential relationships', async () => { - const id = await brainy.add({ - data: { name: 'Self' }, - type: 'entity' - }) - - const relId = await brainy.relate({ - from: id, - to: id, - type: 'relatedTo' - }) - - expect(relId).toBeDefined() - - const relations = await brainy.related({ from: id }) - expect(relations.length).toBeGreaterThan(0) - }) - - it('should handle duplicate relationships', async () => { - const id1 = await brainy.add({ data: { name: 'A' }, type: 'entity' }) - const id2 = await brainy.add({ data: { name: 'B' }, type: 'entity' }) - - const rel1 = await brainy.relate({ - from: id1, - to: id2, - type: 'relatedTo' - }) - - const rel2 = await brainy.relate({ - from: id1, - to: id2, - type: 'relatedTo' - }) - - expect(rel1).not.toBe(rel2) - }) - }) - - describe('Data Validation', () => { - it('should validate vector dimensions', async () => { - const wrongDimVector = new Array(100).fill(0) - - await brainy.add({ - data: { test: 'first' }, - type: 'entity' - }) - - await expect(brainy.add({ - vector: wrongDimVector, - type: 'entity' - })).rejects.toThrow() - }) - - it('should handle NaN and Infinity in vectors', async () => { - const badVector = [NaN, Infinity, -Infinity, 0.5] - - await expect(brainy.add({ - vector: badVector as any, - type: 'entity' - })).rejects.toThrow() - }) - - it('should validate metadata field names', async () => { - const invalidMetadata = { - '$invalid': 'value', - 'also.invalid': 'value', - 'normal': 'value' - } - - const id = await brainy.add({ - data: invalidMetadata, - type: 'entity' - }) - - const retrieved = await brainy.get(id) - expect(retrieved).toBeDefined() - }) - }) - - describe('Recovery and Cleanup', () => { - it('should recover from partial batch failures', async () => { - const items = [ - { id: 'valid1', data: { content: 'Valid 1' }, type: 'document' as const }, - { id: 'invalid', data: null as any, type: 'invalid' as any }, - { id: 'valid2', data: { content: 'Valid 2' }, type: 'document' as const } - ] - - const result = await brainy.addMany({ items }) - - expect(result.successful).toBeGreaterThan(0) - expect(result.failed).toBeGreaterThan(0) - expect(result.errors.length).toBeGreaterThan(0) - }) - - it('should clean up after failed operations', async () => { - const memBefore = process.memoryUsage().heapUsed - - for (let i = 0; i < 100; i++) { - try { - await brainy.add({ - id: `cleanup-${i}`, - data: { content: 'test' }, - type: 'entity' - }) - - await brainy.remove(`cleanup-${i}`) - } catch { - // Ignore errors - } - } - - global.gc?.() - const memAfter = process.memoryUsage().heapUsed - const diff = memAfter - memBefore - - expect(diff).toBeLessThan(5 * 1024 * 1024) // Less than 5MB growth - }) - }) - - describe('Boundary Conditions', () => { - it('should handle zero-length arrays and strings', async () => { - const id = await brainy.add({ - data: { - emptyString: '', - emptyArray: [], - zero: 0, - false: false, - null: null - }, - type: 'entity' - }) - - const retrieved = await brainy.get(id) - expect(retrieved?.data.emptyString).toBe('') - expect(retrieved?.data.emptyArray).toEqual([]) - }) - - it('should handle maximum safe integers', async () => { - const id = await brainy.add({ - data: { - maxSafe: Number.MAX_SAFE_INTEGER, - minSafe: Number.MIN_SAFE_INTEGER, - maxValue: Number.MAX_VALUE, - minValue: Number.MIN_VALUE - }, - type: 'entity' - }) - - const retrieved = await brainy.get(id) - expect(retrieved?.data.maxSafe).toBe(Number.MAX_SAFE_INTEGER) - }) - - it('should handle Unicode and emoji correctly', async () => { - const unicodeData = { - emoji: '😀🎉🚀❤️', - chinese: '你好世界', - arabic: 'مرحبا بالعالم', - hebrew: 'שלום עולם', - special: '™®©℠', - zalgo: 'ẗ̸̢̼͉͈́̏͊̈ë̵̺́͊s̴̱̈́ẗ̸͎̆' - } - - const id = await brainy.add({ - data: unicodeData, - type: 'entity' - }) - - const retrieved = await brainy.get(id) - expect(retrieved?.data.emoji).toBe(unicodeData.emoji) - expect(retrieved?.data.chinese).toBe(unicodeData.chinese) - }) - }) -}) \ No newline at end of file diff --git a/tests/helpers/test-factory.ts b/tests/helpers/test-factory.ts index 40bbe325..62dd53e6 100644 --- a/tests/helpers/test-factory.ts +++ b/tests/helpers/test-factory.ts @@ -39,17 +39,21 @@ export function generateTestVector(dimension = 384): Vector { return vector.map(val => val / magnitude) } -// Generate realistic metadata +// Generate realistic CUSTOM metadata. +// +// NOTE: this deliberately contains only custom (consumer-owned) fields. Reserved +// Brainy fields (`createdAt`, `updatedAt`, `confidence`, `weight`, `subtype`, +// `visibility`, `service`, `createdBy`, `data`, `_rev`) must NOT live in the +// metadata bag — Brainy 8.0's default `reservedFieldPolicy: 'throw'` rejects any +// reserved key passed through `metadata`. (Reserved values belong in their +// dedicated top-level params, set by the add/relate factories below.) export function generateTestMetadata(overrides: any = {}): any { return { name: `Test Item ${generateTestId()}`, description: 'Test description', tags: ['test', 'automated'], - createdAt: Date.now(), - updatedAt: Date.now(), version: 1, source: 'test-factory', - confidence: 0.95, ...overrides, } } diff --git a/tests/integration/aggregate-reserved-fields.test.ts b/tests/integration/aggregate-reserved-fields.test.ts new file mode 100644 index 00000000..e81692d6 --- /dev/null +++ b/tests/integration/aggregate-reserved-fields.test.ts @@ -0,0 +1,169 @@ +/** + * @module tests/integration/aggregate-reserved-fields + * @description One field-resolution law across the whole aggregation + query + * surface (SELF-AGGREGATE-DELETE-DRIFT). Laws: + * (1) aggregates grouped by a RESERVED field (subtype) decrement on delete — + * the delete-side entity view carries every reserved field, so the + * decrement finds its group (counts must never drift from ground truth); + * (2) same for update: moving an entity between reserved-field groups + * decrements the old group and increments the new one (no double-count); + * (3) aggregation source.where on a reserved field (subtype) FILTERS instead + * of silently matching nothing; + * (4) removeMany refuses empty/invalid selectors loudly (bare array, empty + * object, ids: []) instead of resolving as a silent no-op; + * (5) find() accepts both where spellings: flattened (entry.title) and + * storage-shaped (metadata.entry.title) resolve to the same rows. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +const stubEmbedding = async (text: string): Promise => { + const hash = text.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0) + return new Array(384).fill(0).map((_, i) => Math.sin(hash + i)) +} + +describe('aggregation + query field-resolution law', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' as const }, + embeddingFunction: stubEmbedding + }) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + it('reserved-field groupBy decrements on delete (the drift bug)', async () => { + brain.defineAggregate({ + name: 'by_subtype', + source: { type: NounType.Document }, + groupBy: ['subtype'], + metrics: { count: { op: 'count' } } + }) + + const ids: string[] = [] + for (let i = 0; i < 5; i++) { + ids.push( + await brain.add({ + data: `doc-${i}`, + type: NounType.Document, + subtype: 'note', + metadata: { team: 'alpha' } + }) + ) + } + let groups = await brain.queryAggregate('by_subtype') + expect(groups).toHaveLength(1) + expect(groups[0].groupKey).toEqual({ subtype: 'note' }) + expect(groups[0].metrics.count).toBe(5) + + await brain.remove(ids[0]) + await brain.flush() + + groups = await brain.queryAggregate('by_subtype') + expect(groups[0].metrics.count).toBe(4) + const live = await brain.find({ type: NounType.Document, limit: 100 }) + expect(groups[0].metrics.count).toBe(live.length) + }) + + it('reserved-field groupBy moves between groups on update (no double-count)', async () => { + brain.defineAggregate({ + name: 'by_subtype', + source: { type: NounType.Document }, + groupBy: ['subtype'], + metrics: { count: { op: 'count' } } + }) + const id = await brain.add({ + data: 'doc-move', + type: NounType.Document, + subtype: 'draft' + }) + await brain.update({ id, subtype: 'published' }) + + const groups = await brain.queryAggregate('by_subtype') + const byKey = Object.fromEntries( + groups.map((g) => [String(g.groupKey.subtype), g.metrics.count]) + ) + expect(byKey['published']).toBe(1) + // The old group must be gone or zero — never still counting the entity. + expect(byKey['draft'] ?? 0).toBe(0) + }) + + it('source.where on a reserved field filters instead of matching nothing', async () => { + brain.defineAggregate({ + name: 'notes_only', + source: { type: NounType.Document, where: { subtype: 'note' } }, + groupBy: ['team'], + metrics: { count: { op: 'count' } } + }) + await brain.add({ + data: 'n1', + type: NounType.Document, + subtype: 'note', + metadata: { team: 'alpha' } + }) + await brain.add({ + data: 'd1', + type: NounType.Document, + subtype: 'draft', + metadata: { team: 'alpha' } + }) + + const groups = await brain.queryAggregate('notes_only') + expect(groups).toHaveLength(1) + expect(groups[0].metrics.count).toBe(1) // the note, never the draft + }) + + it('removeMany refuses empty/invalid selectors loudly', async () => { + const id = await brain.add({ data: 'keep-me', type: NounType.Document }) + + // Bare array passed positionally — the classic silent no-op. + await expect( + brain.removeMany([id] as unknown as Parameters[0]) + ).rejects.toThrow(/bare array/) + // Empty selector object. + await expect( + brain.removeMany({} as Parameters[0]) + ).rejects.toThrow(/requires a selector/) + // Explicit empty id list. + await expect(brain.removeMany({ ids: [] })).rejects.toThrow(/ids: \[\]/) + + // Nothing was deleted by any of the refused calls. + expect(await brain.get(id)).toBeTruthy() + }) + + it('find() accepts both flattened and metadata.-prefixed where spellings', async () => { + await brain.add({ + data: 'nested-doc', + type: NounType.Document, + metadata: { entry: { title: 'T1' }, classifier: { contextHints: { vfsPath: '/n/a.md' } } } + }) + await brain.flush() + + const flat = await brain.find({ + type: NounType.Document, + where: { 'entry.title': 'T1' }, + limit: 10 + }) + const prefixed = await brain.find({ + type: NounType.Document, + where: { 'metadata.entry.title': 'T1' }, + limit: 10 + }) + const deepPrefixed = await brain.find({ + type: NounType.Document, + where: { 'metadata.classifier.contextHints.vfsPath': '/n/a.md' }, + limit: 10 + }) + expect(flat).toHaveLength(1) + expect(prefixed).toHaveLength(1) + expect(prefixed[0].id).toBe(flat[0].id) + expect(deepPrefixed).toHaveLength(1) + }) +}) diff --git a/tests/integration/aggregation-state-persistence.test.ts b/tests/integration/aggregation-state-persistence.test.ts new file mode 100644 index 00000000..db8fc088 --- /dev/null +++ b/tests/integration/aggregation-state-persistence.test.ts @@ -0,0 +1,311 @@ +/** + * @module tests/integration/aggregation-state-persistence + * @description The boot-order contract for the aggregation engine. Five laws: + * (1) STATE ADOPTION — a reopen with an unchanged defineAggregate() adopts the + * persisted state and performs NO store walk. (The pre-fix behavior: the + * synchronous define always beat the async init, flagged a backfill, and + * the first query wiped the just-loaded state and re-walked the whole + * store — every restart, forever.) + * (2) SINGLE-FLIGHT + BATCH — concurrent cold queries across multiple pending + * aggregates share exactly ONE store walk; a query never wipes another's + * partial progress and M pending aggregates cost one enumeration, not M. + * (3) CHANGED DEFINITION — a real definition change still backfills, exactly. + * (4) PERSISTED-ONLY DEFINITIONS — an app that does not re-define at boot can + * query a persisted aggregate without racing a spurious "not defined". + * (5) QUIET KEYS — the engine's persistence keys (__aggregation_*) are + * recognized system keys: no "Unknown key format" warning at boot. + */ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' +import type { AggregateDefinition } from '../../src/types/brainy.types.js' +import { prodLog } from '../../src/utils/logger.js' + +const SPENDING: AggregateDefinition = { + name: 'spending', + source: { type: NounType.Event, where: { domain: 'financial' } }, + groupBy: ['category'], + metrics: { + total: { op: 'sum', field: 'amount' }, + count: { op: 'count' } + } +} + +/** Same name, different metrics — a REAL definition change (hash differs). */ +const SPENDING_CHANGED: AggregateDefinition = { + ...SPENDING, + metrics: { + total: { op: 'sum', field: 'amount' }, + count: { op: 'count' }, + average: { op: 'avg', field: 'amount' } + } +} + +describe('aggregation state persistence — boot-order contract', () => { + let dir: string + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-agg-persist-')) + }) + + afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }) + }) + + const open = async (): Promise => { + const b: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true + }) + await b.init() + return b + } + + /** Count store walks by intercepting the storage adapter's getNouns. */ + const countWalks = (brain: any): { count: () => number } => { + const storage = brain.storage + const orig = storage.getNouns.bind(storage) + let calls = 0 + storage.getNouns = async (opts: unknown) => { + calls++ + return orig(opts) + } + return { count: () => calls } + } + + const seed = async (brain: any): Promise => { + for (let i = 0; i < 12; i++) { + await brain.add({ + data: `tx ${i}`, + type: NounType.Event, + metadata: { + domain: 'financial', + category: i % 2 === 0 ? 'food' : 'transport', + amount: 10 + i + } + }) + } + } + + it('adopts persisted state on reopen with an unchanged definition — zero walks', async () => { + const brain1 = await open() + brain1.defineAggregate(SPENDING) + await seed(brain1) + const before = await brain1.queryAggregate('spending') + expect(before.length).toBe(2) + await brain1.close() + + const brain2 = await open() + brain2.defineAggregate(SPENDING) // the standard declarative boot pattern + await brain2.getNounCount() // settle init paths before counting walks + const walks = countWalks(brain2) + + const after = await brain2.queryAggregate('spending') + + expect(walks.count()).toBe(0) + const key = (r: any) => r.groupKey.category + expect( + after.map((r: any) => [key(r), r.metrics.total, r.metrics.count]).sort() + ).toEqual( + before.map((r: any) => [key(r), r.metrics.total, r.metrics.count]).sort() + ) + await brain2.close() + }) + + it('adopted state keeps accumulating: post-reopen writes land on top of it', async () => { + const brain1 = await open() + brain1.defineAggregate(SPENDING) + await seed(brain1) + await brain1.queryAggregate('spending') + await brain1.close() + + const brain2 = await open() + brain2.defineAggregate(SPENDING) + // A write BEFORE the first query: if it lands before state adoption the + // engine must choose an exact rescan over adoption — either way the + // result must include all 13 entities. + await brain2.add({ + data: 'late tx', + type: NounType.Event, + metadata: { domain: 'financial', category: 'food', amount: 100 } + }) + const rows = await brain2.queryAggregate('spending') + const food = rows.find((r: any) => r.groupKey.category === 'food') + expect(food.metrics.count).toBe(7) // 6 seeded + 1 late + await brain2.close() + }) + + it('a changed definition still backfills — exactly once, with correct results', async () => { + const brain1 = await open() + brain1.defineAggregate(SPENDING) + await seed(brain1) + await brain1.queryAggregate('spending') + await brain1.close() + + const brain2 = await open() + brain2.defineAggregate(SPENDING_CHANGED) + await brain2.getNounCount() + const walks = countWalks(brain2) + + const rows = await brain2.queryAggregate('spending') + + expect(walks.count()).toBe(1) // 12 entities = one page = one getNouns call + const food = rows.find((r: any) => r.groupKey.category === 'food') + expect(food.metrics.count).toBe(6) + expect(food.metrics.average).toBeCloseTo(food.metrics.total / 6) + await brain2.close() + }) + + it('concurrent cold queries across two aggregates share exactly ONE walk', async () => { + const brain = await open() + brain.defineAggregate(SPENDING) + brain.defineAggregate({ + ...SPENDING, + name: 'by_category_count', + metrics: { count: { op: 'count' } } + }) + await seed(brain) + await brain.getNounCount() + const walks = countWalks(brain) + + const results = await Promise.all([ + brain.queryAggregate('spending'), + brain.queryAggregate('by_category_count'), + brain.queryAggregate('spending'), + brain.queryAggregate('by_category_count'), + brain.queryAggregate('spending'), + brain.queryAggregate('by_category_count') + ]) + + expect(walks.count()).toBe(1) // 12 entities = one page; one walk fills both + for (const rows of results) { + const total = rows.reduce((s: number, r: any) => s + r.metrics.count, 0) + expect(total).toBe(12) + } + + // Warm re-query: converged, no further walks. + await brain.queryAggregate('spending') + expect(walks.count()).toBe(1) + await brain.close() + }) + + it('persisted-only definitions are queryable without re-defining at boot', async () => { + const brain1 = await open() + brain1.defineAggregate(SPENDING) + await seed(brain1) + await brain1.queryAggregate('spending') + await brain1.close() + + const brain2 = await open() + // NO defineAggregate — the app relies on the persisted definition. + const walks = countWalks(brain2) + const rows = await brain2.queryAggregate('spending') + expect(walks.count()).toBe(0) // persisted state adopted here too + expect(rows.length).toBe(2) + await brain2.close() + }) + + it('generation-mismatched persisted state is rescanned once, loudly — never adopted', async () => { + const brain1 = await open() + brain1.defineAggregate(SPENDING) + await seed(brain1) + await brain1.queryAggregate('spending') + await brain1.close() + + // Simulate the copied-store incident class: a fact-log truncation (or an + // unclean shutdown) leaves the committed watermark different from the + // generation the flushed state was stamped with. + const tamper: any = await open() + const key = '__aggregation_state_spending__' + const stored = await tamper.storage.getMetadata(key) + expect(typeof stored.sourceGeneration).toBe('number') // the stamp is really persisted + await tamper.storage.saveMetadata(key, { + ...stored, + sourceGeneration: stored.sourceGeneration + 5 + }) + await tamper.close() + + const warnSpy = vi.spyOn(prodLog, 'warn') + const brain2 = await open() + brain2.defineAggregate(SPENDING) + await brain2.getNounCount() + const walks = countWalks(brain2) + + const rows = await brain2.queryAggregate('spending') + + expect(walks.count()).toBe(1) // exactly ONE rescan — no silent adopt, no spin + const food = rows.find((r: any) => r.groupKey.category === 'food') + expect(food.metrics.count).toBe(6) // rescan produced exact results + expect( + warnSpy.mock.calls.some(args => String(args[0]).includes('rescanning instead of adopting')) + ).toBe(true) // and it said so out loud + warnSpy.mockRestore() + await brain2.close() + }) + + it('a failing walk is loud, non-destructive, and latched — never a silent retry loop', async () => { + // Fresh define + seeded writes: the write hooks have populated LIVE state, + // and the first-query rescan is still pending. The incident shape + // (wipe-before-scan + no try/catch + per-query re-walk) would have wiped + // that live state and silently re-walked on every query. + const brain: any = await open() + brain.defineAggregate(SPENDING) + await seed(brain) + expect(brain._aggregationIndex.queryAggregate({ name: 'spending' }).length).toBe(2) + + const storage = brain.storage + const origGetNouns = storage.getNouns.bind(storage) + let walkAttempts = 0 + storage.getNouns = async () => { + walkAttempts++ + throw new Error('injected storage failure') + } + + // First query: the walk fails LOUDLY with the storage error. + await expect(brain.queryAggregate('spending')).rejects.toThrow('injected storage failure') + expect(walkAttempts).toBe(1) + + // Live state was NOT destroyed by the failed walk (staging was dropped). + expect(brain._aggregationIndex.queryAggregate({ name: 'spending' }).length).toBe(2) + + // Second query inside the cooldown: instant loud failure, NO new walk. + await expect(brain.queryAggregate('spending')).rejects.toThrow('failure cooldown') + expect(walkAttempts).toBe(1) + + // Heal the storage + expire the cooldown: one fresh walk succeeds exactly. + storage.getNouns = origGetNouns + brain._aggregationBackfillFailure.at = Date.now() - 60_000 + const rows = await brain.queryAggregate('spending') + const food = rows.find((r: any) => r.groupKey.category === 'food') + expect(food.metrics.count).toBe(6) + await brain.close() + }) + + it('aggregation persistence keys never log "Unknown key format"', async () => { + const warnSpy = vi.spyOn(prodLog, 'warn') + const brain1 = await open() + brain1.defineAggregate(SPENDING) + await seed(brain1) + await brain1.queryAggregate('spending') + await brain1.close() + + const brain2 = await open() + brain2.defineAggregate(SPENDING) + await brain2.queryAggregate('spending') + await brain2.close() + + const offenders = warnSpy.mock.calls + .map(args => String(args[0])) + .filter( + msg => + msg.includes('Unknown key format') && + (msg.includes('__aggregation_') || msg.includes('brainy:entityIdMapper')) + ) + expect(offenders).toEqual([]) + warnSpy.mockRestore() + }) +}) diff --git a/tests/integration/background-dedup-lifecycle.test.ts b/tests/integration/background-dedup-lifecycle.test.ts new file mode 100644 index 00000000..48c7a4a7 --- /dev/null +++ b/tests/integration/background-dedup-lifecycle.test.ts @@ -0,0 +1,85 @@ +/** + * @module tests/integration/background-dedup-lifecycle + * @description The post-import background deduplication pass (a merge-DELETE + * writer) obeys the same contract as the inline pass. Laws: + * (1) enableDeduplication:false schedules NO background pass — the brain-owned + * deduplicator is never even constructed; + * (2) by default the pass IS scheduled, brain-owned, with an unref'd timer + * (a pending pass never holds the process open); + * (3) repeated imports debounce into ONE pending batch on ONE instance + * (per-coordinator instances used to arm one timer per import); + * (4) close() cancels pending work — no delete pass can fire after close. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../src/brainy.js' + +const ROWS = [ + { name: 'Alice Zephyr', role: 'engineer' }, + { name: 'Bob Quill', role: 'writer' } +] + +// Keep imports fast and deterministic — dedup scheduling is what's under test. +const FAST = { + enableNeuralExtraction: false, + enableRelationshipInference: false, + enableConceptExtraction: false +} as const + +// Deterministic stub embedder (hnsw-rebuild.test.ts pattern) — dedup +// scheduling never inspects vector CONTENT, so skip the WASM model load. +const stubEmbedding = async (text: string): Promise => { + const hash = text.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0) + const vector = new Array(384).fill(0).map((_, i) => Math.sin(hash + i)) + return vector +} + +describe('background dedup lifecycle', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' as const }, + embeddingFunction: stubEmbedding + }) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + it('enableDeduplication:false schedules no background pass at all', async () => { + await brain.import(ROWS, { ...FAST, enableDeduplication: false }) + expect((brain as any)._backgroundDedup).toBeUndefined() + }) + + it('default schedules a brain-owned pass with an unref-ed timer', async () => { + await brain.import(ROWS, { ...FAST }) + const dedup = (brain as any)._backgroundDedup + expect(dedup).toBeDefined() + expect(dedup.pendingImports.size).toBe(1) + const timer = dedup.debounceTimer + expect(timer).toBeDefined() + // Node timers expose hasRef(); an unref'd timer must not hold the process. + expect(typeof timer.hasRef).toBe('function') + expect(timer.hasRef()).toBe(false) + }) + + it('imports debounce into one pending batch on one brain-owned instance', async () => { + await brain.import(ROWS, { ...FAST }) + const first = (brain as any)._backgroundDedup + await brain.import([{ name: 'Cara Vex', role: 'analyst' }], { ...FAST }) + expect((brain as any)._backgroundDedup).toBe(first) + expect(first.pendingImports.size).toBe(2) + }) + + it('close() cancels pending background dedup', async () => { + await brain.import(ROWS, { ...FAST }) + const dedup = (brain as any)._backgroundDedup + expect(dedup.debounceTimer).toBeDefined() + await brain.close() + expect(dedup.debounceTimer).toBeUndefined() + expect(dedup.pendingImports.size).toBe(0) + }) +}) diff --git a/tests/integration/batchImportWithRelations.test.ts b/tests/integration/batchImportWithRelations.test.ts index 1594db88..7fe8e511 100644 --- a/tests/integration/batchImportWithRelations.test.ts +++ b/tests/integration/batchImportWithRelations.test.ts @@ -61,10 +61,12 @@ describe('Batch Import with Immediate Relations (v5.7.3 Fix)', () => { const entityParams = Array(372).fill(null).map((_, i) => ({ data: `Extracted Entity ${i}`, type: NounType.Thing, + // `confidence` is a reserved field — pass it as the dedicated param, not + // inside the metadata bag (8.0 reservedFieldPolicy defaults to 'throw'). + confidence: 0.9, metadata: { extractedFrom: 'TfT~Sapient Species.pdf', - entityNumber: i, - confidence: 0.9 + entityNumber: i } })) diff --git a/tests/integration/bun-compile-test.ts b/tests/integration/bun-runtime-test.ts similarity index 92% rename from tests/integration/bun-compile-test.ts rename to tests/integration/bun-runtime-test.ts index a8d49ca7..5333ced1 100644 --- a/tests/integration/bun-compile-test.ts +++ b/tests/integration/bun-runtime-test.ts @@ -1,25 +1,24 @@ /** - * Bun Compile Test + * Bun Runtime Test * - * Tests that Brainy works when compiled with `bun build --compile`. - * This verifies: + * Tests that Brainy runs correctly under the Bun runtime (`bun add` / `bun run`) — + * the supported Bun story. Verifies: * 1. No native binaries required (pure WASM) * 2. Model is properly bundled * 3. Embeddings work without network access * - * To test manually: - * bun build tests/integration/bun-compile-test.ts --compile --outfile /tmp/brainy-bun-test - * /tmp/brainy-bun-test + * To run manually: + * bun tests/integration/bun-runtime-test.ts */ import { Brainy } from '../../dist/index.js' import { embeddingManager } from '../../dist/embeddings/EmbeddingManager.js' import { WASMEmbeddingEngine } from '../../dist/embeddings/wasm/index.js' -async function testBunCompile() { +async function testBunRuntime() { const results: { test: string; passed: boolean; error?: string }[] = [] - console.log('🚀 Brainy Bun Compile Test\n') + console.log('🚀 Brainy Bun Runtime Test\n') console.log('Runtime:', typeof Bun !== 'undefined' ? `Bun ${Bun.version}` : `Node.js ${process.version}`) console.log('') @@ -153,13 +152,13 @@ async function testBunCompile() { console.log('\n❌ Some tests failed!') process.exit(1) } else { - console.log('\n✅ All tests passed! Brainy works with Bun compile.') + console.log('\n✅ All tests passed! Brainy runs under Bun.') process.exit(0) } } // Run tests -testBunCompile().catch(error => { +testBunRuntime().catch(error => { console.error('Fatal error:', error) process.exit(1) }) diff --git a/tests/integration/cold-graph-connected-8.0.test.ts b/tests/integration/cold-graph-connected-8.0.test.ts new file mode 100644 index 00000000..71ae345d --- /dev/null +++ b/tests/integration/cold-graph-connected-8.0.test.ts @@ -0,0 +1,208 @@ +/** + * @module tests/integration/cold-graph-connected-8.0 + * @description BRAINY-COLD-GRAPH-CONNECTED (8.0) — regression coverage for the silent-empty + * graph-traversal bug, gated on the converged 8.0 contract: a sync `graphIndex.isReady()` that + * is true ONLY when the source→target EDGES are loaded (NOT the membership/manifest count). + * + * On the FIRST `find({ connected })` after a cold process start of a LARGE brain (≥10k nouns, + * which skips the eager index rebuild), a native graph adjacency can reload its relationship + * COUNT (so `size() > 0`) but NOT its edges — so `getNeighbors()` returns `[]` for EVERY source + * and brainy would serve that `[]` as if the anchor were genuinely edgeless. + * + * The 8.0 guard (`verifyGraphAdjacencyLive`) prefers the honest `isReady()` signal: + * - `isReady() === false` → hydrate the id-mapper, rebuild from storage, re-check; a still-false + * `isReady()` throws {@link GraphIndexNotReadyError} instead of returning `[]` ('rebuilt' when + * the rebuild heals it); + * - a genuinely edgeless anchor with `isReady() === true` verifies 'live' and the empty result + * stands — no spurious rebuild, no throw; + * - a provider WITHOUT `isReady()` falls back to the shipped 7.x known-edge-sample probe. + * + * These exercise REAL `find({ connected })` against an in-memory brain whose graph index is + * instrumented with a test-double `isReady()` (and, for the fallback case, an empty-then-healed + * `getNeighbors`). Only the readiness/edge surface is wrapped; the underlying real adjacency + * (built by `relate()`) is unmasked once a rebuild "heals" it. + */ + +import { describe, it, expect, afterEach } from 'vitest' +import { Brainy } from '../../src/index.js' +import { NounType, VerbType } from '../../src/types/graphTypes.js' +import { GraphIndexNotReadyError } from '../../src/errors/brainyError.js' +import { createTestConfig } from '../helpers/test-factory.js' + +/** + * Build a real in-memory brain. Always seeds an UNRELATED edge (`E -> F`) so a GLOBAL known-edge + * sample exists for the fallback probe even when the queried anchor is genuinely edgeless. When + * `anchorEdges` is set, the anchor links out to three target nouns via `Knows`. Ids are + * brainy-generated and returned for assertions. + */ +async function buildBrain( + opts: { anchorEdges: boolean } +): Promise<{ brain: any; anchorId: string; targetIds: string[] }> { + const brain: any = new Brainy(createTestConfig({ silent: true })) + await brain.init() + + // Unrelated edge — guarantees storage.getVerbs() always yields a known-edge source. + const eId = await brain.add({ data: 'node E', type: NounType.Person }) + const fId = await brain.add({ data: 'node F', type: NounType.Person }) + await brain.relate({ from: eId, to: fId, type: VerbType.Knows }) + + const anchorId = await brain.add({ data: 'anchor', type: NounType.Person }) + + const targetIds: string[] = [] + if (opts.anchorEdges) { + for (const label of ['B', 'C', 'D']) { + const tId = await brain.add({ data: `node ${label}`, type: NounType.Person }) + await brain.relate({ from: anchorId, to: tId, type: VerbType.Knows }) + targetIds.push(tId) + } + } + + // Fresh cold-start state: nothing verified yet. + brain._graphAdjacencyVerified = false + return { brain, anchorId, targetIds } +} + +/** + * Instrument the brain's real graph index with a test-double `isReady()` (the 8.0 contract) plus + * an edge surface that goes empty while NOT ready. `getNeighbors` returns `[]` while `!ready` + * (modelling the cold-unloaded adjacency) and delegates to the REAL index once a rebuild flips + * `ready` on. `rebuild` is counted; it heals (`ready = true`) only when `healsOnRebuild` is set. + * Pass `failFirstRebuild` to make the FIRST rebuild throw a transient error (without healing) so + * the empty-result re-collect path in executeGraphSearch is exercised. + */ +function instrumentIsReady( + brain: any, + opts: { ready: boolean; healsOnRebuild: boolean; failFirstRebuild?: boolean } +): { rebuildCalls: number } { + const gi = brain.graphIndex + const origGetNeighbors = gi.getNeighbors.bind(gi) + const state = { ready: opts.ready, rebuildCalls: 0 } + + gi.isReady = (): boolean => state.ready + + gi.getNeighbors = async (id: bigint, options?: any): Promise => + state.ready ? origGetNeighbors(id, options) : [] + + gi.rebuild = async (): Promise => { + state.rebuildCalls++ + if (opts.failFirstRebuild && state.rebuildCalls === 1) { + throw new Error('transient rebuild hiccup') + } + if (opts.healsOnRebuild) state.ready = true // unmask the real (already-populated) adjacency + } + + return state +} + +/** + * Fallback instrumentation — a provider WITHOUT `isReady()` (older cortex / JS baseline). Wraps + * `getNeighbors` to return `[]` while `broken` and delegates to the REAL index once a rebuild + * heals it. This is the shipped 7.x known-edge-sample probe path on 8.0. + */ +function instrumentNoIsReady( + brain: any, + opts: { broken: boolean; healsOnRebuild: boolean } +): { rebuildCalls: number } { + const gi = brain.graphIndex + // Ensure the provider does NOT expose isReady() — the default JS provider doesn't. + expect(typeof gi.isReady).toBe('undefined') + const origGetNeighbors = gi.getNeighbors.bind(gi) + const state = { broken: opts.broken, rebuildCalls: 0 } + + gi.getNeighbors = async (id: bigint, options?: any): Promise => + state.broken ? [] : origGetNeighbors(id, options) + + gi.rebuild = async (): Promise => { + state.rebuildCalls++ + if (opts.healsOnRebuild) state.broken = false + } + + return state +} + +describe('BRAINY-COLD-GRAPH-CONNECTED 8.0 — isReady()-gated, never serves a silent []', () => { + let brains: any[] = [] + afterEach(async () => { + for (const b of brains) { + try { + await b.close() + } catch { + /* best-effort cleanup */ + } + } + brains = [] + }) + + it('(a) isReady() false → rebuild heals it true → find({ connected }) returns correct N (rebuilt)', async () => { + const { brain, anchorId, targetIds } = await buildBrain({ anchorEdges: true }) + brains.push(brain) + const state = instrumentIsReady(brain, { ready: false, healsOnRebuild: true }) + + const results = await brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) + + expect(state.rebuildCalls).toBeGreaterThanOrEqual(1) // detected not-ready + healed it + const ids = results.map((r: any) => r.id).sort() + expect(ids).toEqual(targetIds.sort()) // B, C, D — the real edges, served after the heal + }) + + it('(b) isReady() stays false after rebuild → throws GraphIndexNotReadyError (NOT a silent [])', async () => { + const { brain, anchorId } = await buildBrain({ anchorEdges: true }) + brains.push(brain) + instrumentIsReady(brain, { ready: false, healsOnRebuild: false }) // rebuild never makes it ready + + await expect( + brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) + ).rejects.toBeInstanceOf(GraphIndexNotReadyError) + }) + + it('(c) edgeless anchor + isReady() true → returns [] with NO rebuild and NO throw', async () => { + // The anchor has no edges, but E -> F does — the adjacency is genuinely loaded (ready). + const { brain, anchorId } = await buildBrain({ anchorEdges: false }) + brains.push(brain) + const state = instrumentIsReady(brain, { ready: true, healsOnRebuild: false }) + + const results = await brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) + + expect(results).toEqual([]) // genuinely edgeless — empty is the truth + expect(state.rebuildCalls).toBe(0) // ready adjacency → no spurious rebuild + }) + + it('(d) healthy isReady() true → correct results, NO rebuild', async () => { + const { brain, anchorId, targetIds } = await buildBrain({ anchorEdges: true }) + brains.push(brain) + const state = instrumentIsReady(brain, { ready: true, healsOnRebuild: false }) + + const results = await brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) + + expect(state.rebuildCalls).toBe(0) + const ids = results.map((r: any) => r.id).sort() + expect(ids).toEqual(targetIds.sort()) + }) + + it('(e) provider WITHOUT isReady() → falls back to the known-edge-sample probe (self-heals)', async () => { + const { brain, anchorId, targetIds } = await buildBrain({ anchorEdges: true }) + brains.push(brain) + const state = instrumentNoIsReady(brain, { broken: true, healsOnRebuild: true }) + + const results = await brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) + + expect(state.rebuildCalls).toBeGreaterThanOrEqual(1) // detected the empty adjacency + healed it + const ids = results.map((r: any) => r.id).sort() + expect(ids).toEqual(targetIds.sort()) // B, C, D — served after the heal + }) + + it('(f) executeGraphSearch re-collect: a transient first rebuild leaves connectedIds empty; the empty-result guard then heals + re-collects', async () => { + // First verify (inside neighbors()) hits a transient rebuild failure → returns 'live' without + // healing, so getNeighbors stays empty and connectedIds is empty. The empty connectedIds set + // then drives executeGraphSearch's own verify, whose rebuild now heals → 'rebuilt' → re-collect. + const { brain, anchorId, targetIds } = await buildBrain({ anchorEdges: true }) + brains.push(brain) + const state = instrumentIsReady(brain, { ready: false, healsOnRebuild: true, failFirstRebuild: true }) + + const results = await brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) + + expect(state.rebuildCalls).toBeGreaterThanOrEqual(2) // first transient, second heals + const ids = results.map((r: any) => r.id).sort() + expect(ids).toEqual(targetIds.sort()) // re-collected after the heal + }) +}) diff --git a/tests/integration/count-invariant.test.ts b/tests/integration/count-invariant.test.ts new file mode 100644 index 00000000..3896b625 --- /dev/null +++ b/tests/integration/count-invariant.test.ts @@ -0,0 +1,109 @@ +/** + * @module tests/integration/count-invariant + * @description Pattern-C acceptance: the user-facing scalar total and the + * per-type array are ONE consistent projection — `getNounCount() === Σ + * nounCountsByType` (and the verb mirror) after any interleaving of add / + * update-visibility-flip / delete, AND across a reopen. + * + * The bug (finding 5): delete decremented the per-type array but never the + * scalar for nouns, and neither for verbs — so `getNounCount()`/`getVerbCount()` + * inflated permanently (the stale scalar wins pagination via + * `Math.max(total, collected.length)` and is persisted). The fix restores the + * symmetric decrement the code always intended (its own comments say "symmetric + * with the increments"). This test would report an inflated count before the + * fix and the exact count after. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType, VerbType } from '../../src/types/graphTypes.js' + +const sum = (a: Uint32Array): number => a.reduce((s, c) => s + c, 0) + +describe('count invariant — scalar total === Σ per-type, across delete + reopen (finding 5)', () => { + let dir: string + let brain: any + + const open = async (d: string) => { + const b = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: d }, + dimensions: 384, + silent: true + }) + await b.init() + return b + } + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-count-inv-')) + brain = await open(dir) + }) + + afterEach(async () => { + try { await brain.close() } catch { /* already closed */ } + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('noun delete decrements the scalar total, not just the per-type bucket', async () => { + const things: string[] = [] + for (let i = 0; i < 5; i++) things.push(await brain.add({ data: `thing ${i}`, type: NounType.Thing })) + for (let i = 0; i < 3; i++) await brain.add({ data: `concept ${i}`, type: NounType.Concept }) + + const storage = brain['storage'] + expect(await storage.getNounCount()).toBe(8) + expect(sum(storage.getNounCountsByType())).toBe(8) + + // Delete 2 Things — the scalar must drop with the bucket (pre-fix it stayed 8). + await brain.remove(things[0]) + await brain.remove(things[1]) + + expect(await storage.getNounCount()).toBe(6) + expect(sum(storage.getNounCountsByType())).toBe(6) + // The invariant: scalar === Σ per-type. + expect(await storage.getNounCount()).toBe(sum(storage.getNounCountsByType())) + }) + + it('verb delete decrements BOTH the scalar total AND the per-type bucket', async () => { + const ids: string[] = [] + for (let i = 0; i < 4; i++) ids.push(await brain.add({ data: `node ${i}`, type: NounType.Thing })) + const edges: string[] = [] + edges.push(await brain.relate({ from: ids[0], to: ids[1], type: VerbType.RelatedTo })) + edges.push(await brain.relate({ from: ids[1], to: ids[2], type: VerbType.RelatedTo })) + edges.push(await brain.relate({ from: ids[2], to: ids[3], type: VerbType.Contains })) + edges.push(await brain.relate({ from: ids[0], to: ids[3], type: VerbType.Contains })) + + const storage = brain['storage'] + expect(await storage.getVerbCount()).toBe(4) + expect(sum(storage.getVerbCountsByType())).toBe(4) + + // Unrelate one edge — pre-fix, verb delete touched NEITHER counter (both stayed 4). + await brain.unrelate(edges[0]) + + expect(await storage.getVerbCount()).toBe(3) + expect(sum(storage.getVerbCountsByType())).toBe(3) + expect(await storage.getVerbCount()).toBe(sum(storage.getVerbCountsByType())) + }) + + it('the corrected counts persist and survive a reopen', async () => { + const things: string[] = [] + for (let i = 0; i < 6; i++) things.push(await brain.add({ data: `t${i}`, type: NounType.Thing })) + const a = await brain.relate({ from: things[0], to: things[1], type: VerbType.RelatedTo }) + await brain.relate({ from: things[1], to: things[2], type: VerbType.RelatedTo }) + await brain.remove(things[5]) + await brain.unrelate(a) + await brain.flush() + await brain.close() + + // Reopen from disk — counts.json must carry the corrected totals. + brain = await open(dir) + const storage = brain['storage'] + expect(await storage.getNounCount()).toBe(5) + expect(await storage.getVerbCount()).toBe(1) + expect(await storage.getNounCount()).toBe(sum(storage.getNounCountsByType())) + expect(await storage.getVerbCount()).toBe(sum(storage.getVerbCountsByType())) + }) +}) diff --git a/tests/integration/counter-recount.test.ts b/tests/integration/counter-recount.test.ts new file mode 100644 index 00000000..310ea4d5 --- /dev/null +++ b/tests/integration/counter-recount.test.ts @@ -0,0 +1,122 @@ +/** + * @module tests/integration/counter-recount + * @description Counter honesty. Two laws under test: + * (1) REMOVAL NEVER REQUIRES RE-READING THE REMOVED RECORD — the count + * decrement falls back to the caller's pre-delete read when the canonical + * metadata re-read returns null (replace race / ghost), instead of being + * silently skipped. The skip minted permanent inflation: adds counted, + * paired removals not decremented, and Math.max(totalNounCount, scanned) + * pinned the inflated scalar forever. + * (2) THE SANCTIONED RECOUNT — repairIndex() unconditionally recomputes and + * PERSISTS every counter rollup (scalar totals + per-type maps + + * type-statistics) from one canonical walk, so an already-inflated brain + * is permanently corrected (survives reopen). + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/index.js' + +/** Absolute path of one entity's `` directory (or null if not present). */ +function entityDir(root: string, id: string): string | null { + const base = path.join(root, 'entities', 'nouns') + if (!fs.existsSync(base)) return null + for (const shard of fs.readdirSync(base)) { + const candidate = path.join(base, shard, id) + if (fs.existsSync(candidate)) return candidate + } + return null +} + +describe('counter honesty — removal without re-reading + the sanctioned recount', () => { + let dir: string + let brain: any + + const open = async () => { + const b: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) + await b.init() + return b + } + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-recount-')) + brain = await open() + }) + afterEach(async () => { + await brain.close?.().catch(() => {}) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('the counter returns to baseline across add→remove→re-create cycles (no drift)', async () => { + const baseline = await brain.storage.getNounCount() + for (let i = 0; i < 4; i++) { + const id = await brain.add({ data: `cycle ${i}`, type: 'document', metadata: { i } }) + expect(await brain.storage.getNounCount()).toBe(baseline + 1) + await brain.remove(id) + expect(await brain.storage.getNounCount()).toBe(baseline) + } + }) + + it('deleteNoun decrements from the provided prior record when the canonical re-read is null', async () => { + const id = await brain.add({ data: 'to be ghosted', type: 'document', metadata: { g: 1 } }) + await brain.flush() + const baseline = await brain.storage.getNounCount() + + // Capture the pre-delete read (what remove() holds), then simulate the + // replace-race / ghost shape: the metadata leg vanishes before the delete's + // internal re-read. + const prior = await brain.storage.getNounMetadata(id) + expect(prior).not.toBeNull() + const eDir = entityDir(dir, id)! + for (const f of fs.readdirSync(eDir)) { + if (f.startsWith('metadata.json')) fs.rmSync(path.join(eDir, f)) + } + + // Without the prior record this decrement used to be silently skipped. + await brain.storage.deleteNoun(id, prior) + expect(await brain.storage.getNounCount()).toBe(baseline - 1) + // Full removal still holds: nothing left on disk. + expect(entityDir(dir, id)).toBeNull() + }) + + it('repairIndex() recounts an inflated persisted scalar over CLEAN shelves — and it survives reopen', async () => { + for (let i = 0; i < 3; i++) { + await brain.add({ data: `real ${i}`, type: 'document', metadata: { i } }) + } + await brain.flush() + const honest = await brain.storage.getNounCount() + // Pagination's totalCount has its own baseline: it enumerates EVERYTHING + // (including the internal VFS root), while the user-facing scalar counts + // only public entities — so the two legitimately differ by the internals. + const pageHonest = (await brain.storage.getNounsWithPagination({ limit: 1000, offset: 0 })).totalCount + + // Simulate the historical drift: an inflated persisted scalar (deletes whose + // decrement was skipped). Persist it so a reopen rehydrates the lie. + ;(brain.storage as any).totalNounCount = honest + 52 + await (brain.storage as any).persistCounts() + await brain.close() + brain = await open() + expect(await brain.storage.getNounCount()).toBe(honest + 52) // the lie survived reopen + + // The sanctioned recount — unconditional in repairIndex (no orphans needed). + await brain.repairIndex() + expect(await brain.storage.getNounCount()).toBe(honest) + + // Permanently: the corrected counter survives another reopen. + await brain.close() + brain = await open() + expect(await brain.storage.getNounCount()).toBe(honest) + + // And the paginated totalCount (the Math.max consumer) is honest too — + // back to ITS baseline, no longer pinned high by the inflated scalar. + const page = await brain.storage.getNounsWithPagination({ limit: 1000, offset: 0 }) + expect(page.totalCount).toBe(pageHonest) + }) +}) diff --git a/tests/integration/db-asof-vector-defer.test.ts b/tests/integration/db-asof-vector-defer.test.ts new file mode 100644 index 00000000..09a3f3d9 --- /dev/null +++ b/tests/integration/db-asof-vector-defer.test.ts @@ -0,0 +1,138 @@ +/** + * @module tests/integration/db-asof-vector-defer + * @description 8.0 #35 — a FILTERED semantic/vector read at a historical generation + * defers to a native at-gen `VersionedIndexProvider` instead of the O(n@G) JS-HNSW + * materialization. Brainy resolves the at-gen metadata∩graph universe from the record + * overlay (no rebuild) and hands the provider `{ allowedIds, generation }`; the + * provider serves the vector leg, brainy composes. CI has no native provider, so this + * registers a faithful MOCK versioned vector index (it advertises visibility + records + * what it receives) to lock the routing + the seam. The JS index is NOT versioned, so + * the un-mocked path correctly falls through to materialization. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' +import { createTestConfig } from '../helpers/test-factory.js' + +/** A versioned vector provider that advertises visibility + records each search. */ +function makeVersionedVectorMock(ranking: string[]) { + const calls: Array<{ + generation?: bigint + allowedIds?: ReadonlySet + atGenerationVectors?: { ids: BigInt64Array; vectors: Float32Array; dim: number } + }> = [] + const provider = { + // VersionedIndexProvider quartet (duck-typed by isVersionedIndexProvider). + generation: () => 0n, + isGenerationVisible: (_g: bigint) => true, + pin: (_g: bigint) => {}, + release: (_g: bigint) => {}, + // The at-gen vector leg: record the seam fields, return ranked allowed ids. + search: async (_vec: number[], _k: number, _filter: unknown, opts: any) => { + calls.push({ + generation: opts?.generation, + allowedIds: opts?.allowedIds, + atGenerationVectors: opts?.atGenerationVectors + }) + const allowed = opts?.allowedIds as ReadonlySet | undefined + const out: Array<[string, number]> = [] + let distance = 0.1 + for (const id of ranking) { + if (!allowed || allowed.has(id)) { + out.push([id, distance]) + distance += 0.1 + } + } + return out + }, + // Unused VectorIndexProvider surface — no-op stubs so close()/flush() are safe. + addItem: async () => '', + removeItem: async () => true, + size: () => 0, + clear: () => {}, + rebuild: async () => {}, + flush: async () => 0, + getPersistMode: () => 'immediate' as const + } + return { provider, calls } +} + +describe('#35 asOf filtered vector read defers to the versioned provider', () => { + let brain: Brainy + let a: string, b: string, c: string + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + a = await brain.add({ type: NounType.Document, data: 'alpha machine learning', metadata: { team: 'x' } }) + b = await brain.add({ type: NounType.Document, data: 'beta machine learning', metadata: { team: 'x' } }) + c = await brain.add({ type: NounType.Document, data: 'gamma cooking', metadata: { team: 'y' } }) + }) + + afterEach(async () => { + await brain.close() + }) + + it('routes to the provider with {allowedIds, generation} (no materialization) — at-gen universe is correct', async () => { + const db = brain.now() // pin the generation after a, b, c + // A later write advances the generation → `db` is now historical, and `later` + // (team x) did NOT exist at the pinned generation. + const later = await brain.add({ type: NounType.Document, data: 'delta machine learning', metadata: { team: 'x' } }) + + // Inject a versioned vector provider that can serve the pinned generation. + const mock = makeVersionedVectorMock([b, a]) // ranks b before a + ;(brain as any).index = mock.provider + + const results = await db.find({ query: 'machine learning', where: { team: 'x' } }) + + // Served natively — the provider was hit exactly once with the seam fields. + expect(mock.calls).toHaveLength(1) + expect(typeof mock.calls[0].generation).toBe('bigint') + + // allowedIds = the at-gen `team: x` universe = {a, b}; NOT `later` (born after the pin). + const allowed = mock.calls[0].allowedIds! + expect(allowed.has(a)).toBe(true) + expect(allowed.has(b)).toBe(true) + expect(allowed.has(later)).toBe(false) + expect(allowed.has(c)).toBe(false) // team y, filtered out + + // Results = the at-gen entities, in the provider's ranking (b before a). + expect(results.map((r) => r.id)).toEqual([b, a]) + + await db.release() + }) + + it('respects the page window on the provider-served path', async () => { + const db = brain.now() + await brain.add({ type: NounType.Document, data: 'epsilon', metadata: { team: 'z' } }) // advance gen + const mock = makeVersionedVectorMock([b, a]) + ;(brain as any).index = mock.provider + + const page = await db.find({ query: 'machine learning', where: { team: 'x' }, limit: 1 }) + expect(page.map((r) => r.id)).toEqual([b]) // top-1 by the provider ranking + await db.release() + }) + + it('supplies the at-gen candidate vectors (part-3) — row-major, dim-correct, ids = the universe', async () => { + const db = brain.now() + await brain.add({ type: NounType.Document, data: 'zeta machine learning', metadata: { team: 'z' } }) // advance gen + const mock = makeVersionedVectorMock([b, a]) + ;(brain as any).index = mock.provider + + await db.find({ query: 'machine learning', where: { team: 'x' } }) + + const agv = mock.calls[0].atGenerationVectors + expect(agv).toBeDefined() + // ids = the at-gen `team: x` universe {a, b}, interned via the shared mapper. + expect(agv!.ids.length).toBe(2) + expect(agv!.dim).toBeGreaterThan(0) + // The row-major invariant cor asserts on ingest. + expect(agv!.vectors.length).toBe(agv!.ids.length * agv!.dim) + // Each interned id resolves back to a universe entity. + const mapper = (brain as any).metadataIndex.getIdMapper() + const uuids = Array.from(agv!.ids).map((i) => mapper.getUuid(Number(i))) + expect(new Set(uuids)).toEqual(new Set([a, b])) + await db.release() + }) +}) diff --git a/tests/integration/db-mvcc.test.ts b/tests/integration/db-mvcc.test.ts index 696e4537..959d0053 100644 --- a/tests/integration/db-mvcc.test.ts +++ b/tests/integration/db-mvcc.test.ts @@ -515,7 +515,15 @@ describe('8.0 Db API — generational MVCC', () => { await brain.transact([{ op: 'update', id: uid('compact-e'), metadata: { v: 4 } }]) ).release() - const recordsBefore = (await storage.listRawObjects('_generations')).length + // History record-sets only — the generation FACT LOG also lives under + // `_generations/` (at `facts/`) and is deliberately NOT reclaimed by + // history compaction (facts are the future canonical, not undo history). + const historyRecords = async (): Promise => + (await storage.listRawObjects('_generations')).filter( + (p: string) => !p.startsWith('_generations/facts/') + ).length + + const recordsBefore = await historyRecords() expect(recordsBefore).toBeGreaterThan(0) // Compact while pinned: record-sets above the pin survive, pinned reads stay correct. @@ -528,7 +536,7 @@ describe('8.0 Db API — generational MVCC', () => { const second = await brain.compactHistory() expect(first.removedGenerations + second.removedGenerations).toBeGreaterThan(0) - const recordsAfter = (await storage.listRawObjects('_generations')).length + const recordsAfter = await historyRecords() expect(recordsAfter).toBeLessThan(recordsBefore) expect(recordsAfter).toBe(0) @@ -1096,6 +1104,38 @@ describe('8.0 Db API — generational MVCC', () => { await after.release() }) + it('restore() rebuilds graph adjacency — relationships survive a snapshot round-trip (id-mapper reloaded before graph)', async () => { + const { brain } = await openFsBrain() + const a = uid('rr-a') + const b = uid('rr-b') + const c = uid('rr-c') + await brain.transact([ + { op: 'add', id: a, type: NounType.Person, data: 'a', vector: vec(1), metadata: {} }, + { op: 'add', id: b, type: NounType.Person, data: 'b', vector: vec(2), metadata: {} }, + { op: 'add', id: c, type: NounType.Document, data: 'c', vector: vec(3), metadata: {} }, + { op: 'relate', from: a, to: b, type: VerbType.RelatedTo }, + { op: 'relate', from: a, to: c, type: VerbType.References } + ]) + expect((await brain.related(a)).map((r) => r.to).sort()).toEqual([b, c].sort()) + + const snapDir = path.join(makeTempDir(), 'rel-snapshot') + const db = brain.now() + await db.persist(snapDir) + await db.release() + + // Drop a + its edges live so the restore must genuinely RECONSTRUCT them. + await brain.remove(a) + expect(await brain.related(a)).toEqual([]) + + await brain.restore(snapDir, { confirm: true }) + + // a's relationships are back: the graph adjacency rebuilt by resolving each + // verb's endpoints (sourceId/targetId → ints) through the id-mapper that + // restore() reloaded from the snapshot BEFORE graphIndex.rebuild(). Without + // that reload, a native adjacency keyed on those ints loses the edges. + expect((await brain.related(a)).map((r) => r.to).sort()).toEqual([b, c].sort()) + }) + it('Db values are first-class: Brainy.open() + chained with() overlays compose', async () => { const brain = await Brainy.open({ requireSubtype: false, storage: { type: 'memory' } }) brains.push(brain) @@ -1118,12 +1158,17 @@ describe('8.0 Db API — generational MVCC', () => { await db.release() }) - it('transactionLog() returns committed entries newest first, with meta and limit', async () => { + it('transactionLog() includes single-op AND transact generations, newest first, with meta and limit', async () => { const brain = await openMemoryBrain() - // Nothing committed yet — the log is empty (single-op writes do not log). + // Model-B: a single-op write is its OWN generation and IS logged (no meta — + // tx metadata is a transact()-only concept). It is generation 1 on a fresh + // brain (init-time infrastructure writes are the un-versioned gen-0 baseline). await brain.add({ id: uid('txlog-solo'), type: NounType.Document, data: 'solo', vector: vec(99), subtype: 'note' }) - expect(await brain.transactionLog()).toEqual([]) + const soloLog = await brain.transactionLog() + expect(soloLog.map((entry) => entry.generation)).toEqual([1]) + expect(soloLog[0].meta).toBeUndefined() + const soloGen = 1 const first = await brain.transact( [{ op: 'add', id: uid('txlog-a'), type: NounType.Document, data: 'a', vector: vec(100), metadata: {} }], @@ -1136,14 +1181,17 @@ describe('8.0 Db API — generational MVCC', () => { const third = await brain.transact([{ op: 'update', id: uid('txlog-a'), metadata: { v: 3 } }]) const entries = await brain.transactionLog() + // Newest first: the three transacts, then the single-op solo write (gen 1). expect(entries.map((entry) => entry.generation)).toEqual([ third.generation, second.generation, - first.generation + first.generation, + soloGen ]) expect(entries[1].meta).toEqual({ author: 'job-2' }) expect(entries[2].meta).toEqual({ author: 'job-1' }) - expect(entries[0].meta).toBeUndefined() + expect(entries[0].meta).toBeUndefined() // third() had no meta + expect(entries[3].meta).toBeUndefined() // the single-op solo write carries no meta for (const entry of entries) { expect(entry.timestamp).toBeGreaterThan(0) } @@ -1156,6 +1204,110 @@ describe('8.0 Db API — generational MVCC', () => { await third.release() }) + // ========================================================================== + // 8b. Model-B single-op generation-stamping (every write versioned) + // ========================================================================== + it('Model-B — a now() pin freezes against single-op add/update/remove (the Model-A hole is closed)', async () => { + const brain = await openMemoryBrain() + const a = uid('mb-a') + const b = uid('mb-b') + // Seed `a` with a single-op add, then pin the current generation. + await brain.add({ id: a, type: NounType.Document, data: 'a', vector: vec(1), metadata: { v: 1 } }) + const pin = brain.now() + + // Single-op mutations AFTER the pin: update `a`, add `b`. + await brain.update({ id: a, metadata: { v: 2 } }) + await brain.add({ id: b, type: NounType.Document, data: 'b', vector: vec(2), metadata: { v: 1 } }) + + // The pin is frozen at its generation — single-op writes do not leak through + // it (pre-Model-B, the pin tracked the live single-op mutation). + expect((await pin.get(a))?.metadata?.v).toBe(1) + expect(await pin.get(b)).toBeNull() + expect((await brain.get(a))?.metadata?.v).toBe(2) + expect((await brain.get(b))?.metadata?.v).toBe(1) + expect(pin.isHistorical()).toBe(true) + + // A single-op REMOVE also leaves the pin untouched. + await brain.remove(a) + expect((await pin.get(a))?.metadata?.v).toBe(1) + expect(await brain.get(a)).toBeNull() + await pin.release() + }) + + it('Model-B — historical find() overlays an un-flushed single-op write (overlay bound is generation, not committed)', async () => { + const brain = await openMemoryBrain() + const a = uid('ov-a') + const b = uid('ov-b') + await ( + await brain.transact([ + { op: 'add', id: a, type: NounType.Document, data: 'a', vector: vec(1), metadata: { v: 1 } }, + { op: 'add', id: b, type: NounType.Document, data: 'b', vector: vec(2), metadata: { v: 1 } } + ]) + ).release() + const at1 = await brain.asOf(1) + + // A single-op REMOVE of `b` lands AFTER the pin and is NOT flushed (pending). + await brain.remove(b) + + const liveIds = (await brain.find({})).map((r) => r.id) + const pastIds = (await at1.find({})).map((r) => r.id) + // Live: `b` is gone. Historical (pinned at gen 1): the un-flushed removal is + // overlaid out, so `b` is still present at its pinned state. + expect(liveIds).toContain(a) + expect(liveIds).not.toContain(b) + expect(pastIds).toContain(a) + expect(pastIds).toContain(b) + await at1.release() + }) + + it('Model-B retention — explicit caps reclaim single-op history; committed history survives reopen', async () => { + const { brain, dir } = await openFsBrain() + const a = uid('ret-a') + await brain.add({ id: a, type: NounType.Document, data: 'a', vector: vec(1), metadata: { v: 1 } }) + for (let v = 2; v <= 6; v++) await brain.update({ id: a, metadata: { v } }) + await brain.flush() // persist the per-write generations to disk + expect(brain.generation()).toBe(6) + + // Cap to the 2 most recent generations — older single-op history is reclaimed. + const res = await brain.compactHistory({ maxGenerations: 2 }) + expect(res.removedGenerations).toBeGreaterThan(0) + expect(res.horizon).toBeGreaterThan(0) + await brain.close() + + // Reopen: the survivors + live value are intact; below-horizon asOf throws. + const { brain: reopened } = await openFsBrain(dir) + expect((await reopened.get(a))?.metadata?.v).toBe(6) + await expect(reopened.asOf(1)).rejects.toBeInstanceOf(GenerationCompactedError) + }) + + it('Model-B retention — flush() NEVER compacts (8.9.0); adaptive reclaim runs at close()', async () => { + // Default brain → ADAPTIVE retention with a driven byte budget far below + // the accumulated history (~13 generations of full-vector before-images). + // The 8.9.0 law: flush() is durability-only — it must not reclaim even + // when the budget is exceeded (reclaim-on-flush blocked production writes + // for 25-191s). Maintenance runs at close(), time-bounded. + const { brain, dir } = await openFsBrain() + const a = uid('ret-budget') + await brain.add({ id: a, type: NounType.Document, data: 'v0', vector: vec(1), metadata: { v: 0 } }) + for (let v = 1; v <= 12; v++) await brain.update({ id: a, metadata: { v } }) + + brain.setRetentionBudget(6000) // ~6 KB — well below the accumulated history + await brain.flush() + + // flush() paid durability only: nothing reclaimed, all history readable. + expect(generationStoreOf(brain).horizon()).toBe(0) + const probe = await brain.asOf(1) // readable proves nothing was reclaimed… + await probe.release() // …and MUST be released: a held pin would (correctly) + // protect every newer generation through the close() compaction below. + await brain.close() // ← THE auto-compaction site now + + // close() reclaimed under the budget; live record intact; horizon durable. + const { brain: reopened } = await openFsBrain(dir) + expect(generationStoreOf(reopened).horizon()).toBeGreaterThan(0) + await expect(reopened.asOf(1)).rejects.toBeInstanceOf(GenerationCompactedError) + expect((await reopened.get(a))?.metadata?.v).toBe(12) + }) + // ========================================================================== // 9. asOf() / released-Db error paths (Y.15 spot-checks) // ========================================================================== diff --git a/tests/integration/db-temporal.test.ts b/tests/integration/db-temporal.test.ts index 137f24c0..d17f5c16 100644 --- a/tests/integration/db-temporal.test.ts +++ b/tests/integration/db-temporal.test.ts @@ -17,8 +17,9 @@ * asOf(version.generation).get(id), a removal is a null value, order is oldest→newest. * 6. Composition: "type T changed in (g1, g2]" computed two ways (diff ids filtered * to T, vs asOf(g2).find({type:T}) ∩ changed-ids) AGREE — including a where filter. - * 7. Granularity: single-operation writes (outside transact()) are invisible to the - * temporal verbs (transact commits only). + * 7. Granularity (Model-B): single-operation writes (outside transact()) are their + * own immutable generation — logged, diffable, and time-travelable, exactly like + * a transact() of one op (the test body below proves this). * 8. Compaction policy contrast: diff/since THROW below the horizon; history TRUNCATES * to it. (Locked so a refactor can't unify them incorrectly.) * @@ -330,8 +331,8 @@ describe('8.0 Db API — temporal range verbs', () => { await dbAtG2.release() }) - // 7. Granularity ------------------------------------------------------------- - it('granularity: single-operation writes (outside transact) are invisible to the temporal verbs', async () => { + // 7. Granularity (Model-B) --------------------------------------------------- + it('granularity: single-operation writes ARE versioned and visible to the temporal verbs', async () => { const brain = await openMemoryBrain() const a = uid('gran-a') const r1 = await brain.transact([ @@ -340,13 +341,26 @@ describe('8.0 Db API — temporal range verbs', () => { await r1.release() expect((await brain.transactionLog()).length).toBe(1) - // A single-op write bumps the counter but writes no generation record/log entry. + // Model-B: a single-op write is its OWN immutable generation — logged, + // diffable, and time-travelable, exactly like a transact() of one op. await brain.update({ id: a, metadata: { v: 2 } }) - expect((await brain.transactionLog()).length).toBe(1) // unchanged — not logged + // The single-op update appended a generation/log entry. + expect((await brain.transactionLog()).length).toBe(2) + expect(brain.generation()).toBe(r1.generation + 1) + + // diff sees the single-op update as a modification of `a`. const d = await brain.diff(r1.generation, brain.generation()) - expect(d.added.nouns).toEqual([]) // the single-op update is not a recorded change - expect(d.modified.nouns).toEqual([]) + expect(d.added.nouns).toEqual([]) + expect(d.modified.nouns).toEqual([a]) + + // And asOf() resolves both states: v=1 at the transact gen, v=2 at the single-op gen. + const atTransact = await brain.asOf(r1.generation) + const atSingleOp = await brain.asOf(brain.generation()) + expect((await atTransact.get(a))?.metadata?.v).toBe(1) + expect((await atSingleOp.get(a))?.metadata?.v).toBe(2) + await atTransact.release() + await atSingleOp.release() }) // 8. Compaction policy contrast --------------------------------------------- @@ -363,7 +377,7 @@ describe('8.0 Db API — temporal range verbs', () => { ).release() } // gens 1..6, all pins released - await brain.compactHistory({ retainGenerations: 2 }) // raise the horizon + await brain.compactHistory({ maxGenerations: 2 }) // raise the horizon const horizon = generationStoreOf(brain).horizon() expect(horizon).toBeGreaterThan(0) diff --git a/tests/integration/delete-full-removal.test.ts b/tests/integration/delete-full-removal.test.ts new file mode 100644 index 00000000..83675a13 --- /dev/null +++ b/tests/integration/delete-full-removal.test.ts @@ -0,0 +1,161 @@ +/** + * @module tests/integration/delete-full-removal + * @description Canonical noun/verb deletes are FULL removals: both legs + * (metadata + vectors) AND the entity's `/` container are removed, so a + * delete leaves NOTHING behind. Regression for the ghost/scar defect where + * remove() deleted the vector INDEX entry + the canonical metadata leg but never + * the canonical vectors.json leg or the directory — leaving an orphan that reads + * as absent (getNoun needs both legs) yet inflated the enumerated count and + * confused locator resolution. Also covers the operator repair sweep + * (brain.repairIndex()) that prunes orphans left by the pre-fix behavior. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/index.js' + +/** All entity-directory names (the `` dirs) under entities/. */ +function entityDirs(root: string, kind: 'nouns' | 'verbs'): string[] { + const base = path.join(root, 'entities', kind) + if (!fs.existsSync(base)) return [] + const ids: string[] = [] + for (const shard of fs.readdirSync(base)) { + const shardDir = path.join(base, shard) + if (!fs.statSync(shardDir).isDirectory()) continue + for (const id of fs.readdirSync(shardDir)) { + if (fs.statSync(path.join(shardDir, id)).isDirectory()) ids.push(id) + } + } + return ids +} + +/** Absolute path of one entity's `` directory (or null if not present). */ +function entityDir(root: string, kind: 'nouns' | 'verbs', id: string): string | null { + const base = path.join(root, 'entities', kind) + if (!fs.existsSync(base)) return null + for (const shard of fs.readdirSync(base)) { + const candidate = path.join(base, shard, id) + if (fs.existsSync(candidate)) return candidate + } + return null +} + +describe('canonical delete is a full removal (no ghost/scar directory)', () => { + let dir: string + let brain: any + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-del-')) + brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 }) + await brain.init() + }) + afterEach(async () => { + await brain.close?.().catch(() => {}) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('remove() deletes BOTH legs and the container — nothing left on disk', async () => { + const ids: string[] = [] + for (let i = 0; i < 3; i++) ids.push(await brain.add({ data: `doc ${i}`, type: 'document', metadata: { i } })) + await brain.flush() + + // (A filesystem brain also has its VFS-root noun, so don't assume an exact set.) + const before = entityDirs(dir, 'nouns') + expect(before).toEqual(expect.arrayContaining(ids)) + + await brain.remove(ids[1]) + await brain.flush() + + // getNoun is null AND the on-disk container is fully gone (no orphan), and + // EXACTLY one directory disappeared (the removed entity's). + expect(await brain.get(ids[1])).toBeNull() + expect(entityDir(dir, 'nouns', ids[1])).toBeNull() + const after = entityDirs(dir, 'nouns') + expect(after).toHaveLength(before.length - 1) + expect(after).toEqual(expect.arrayContaining([ids[0], ids[2]])) + expect(after).not.toContain(ids[1]) + }) + + it('the enumerated count is honest after a delete (no monotonic inflation)', async () => { + const ids: string[] = [] + for (let i = 0; i < 4; i++) ids.push(await brain.add({ data: `n${i}`, type: 'document', metadata: { i } })) + await brain.flush() + + const before = await brain.storage.getNounsWithPagination({ limit: 100, offset: 0 }) + + await brain.remove(ids[0]) + await brain.remove(ids[1]) + await brain.flush() + + // The count drops by EXACTLY the two removed entities — no ghost lingers to + // hold the total up (the pre-fix defect left it monotonic). + const after = await brain.storage.getNounsWithPagination({ limit: 100, offset: 0 }) + expect(after.totalCount).toBe(before.totalCount - 2) + const remaining = after.items.map((n: any) => n.id) + expect(remaining).toEqual(expect.arrayContaining([ids[2], ids[3]])) + expect(remaining).not.toContain(ids[0]) + expect(remaining).not.toContain(ids[1]) + }) +}) + +describe('repairIndex() prunes orphan containers left by the pre-fix partial delete', () => { + let dir: string + let brain: any + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-orphan-')) + brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 }) + await brain.init() + }) + afterEach(async () => { + await brain.close?.().catch(() => {}) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('a vector-only "ghost" (metadata leg deleted out-of-band) is pruned', async () => { + const ids: string[] = [] + for (let i = 0; i < 3; i++) ids.push(await brain.add({ data: `g${i}`, type: 'document', metadata: { i } })) + await brain.flush() + + // Simulate the pre-fix defect on ids[0]: remove ONLY its metadata leg, + // leaving vectors.json + the directory (the exact ghost shape). + const ghostDir = entityDir(dir, 'nouns', ids[0])! + for (const f of fs.readdirSync(ghostDir)) { + if (f.startsWith('metadata.json')) fs.rmSync(path.join(ghostDir, f)) + } + // The ghost dir (vectors.json, no metadata content leg) still sits on disk. + expect(entityDir(dir, 'nouns', ids[0])).not.toBeNull() + + await brain.repairIndex() + + // The sweep removes the ghost container; the two healthy entities survive. + expect(entityDir(dir, 'nouns', ids[0])).toBeNull() + expect(entityDir(dir, 'nouns', ids[1])).not.toBeNull() + expect(entityDir(dir, 'nouns', ids[2])).not.toBeNull() + }) + + it('an empty "scar" directory is pruned', async () => { + const id = await brain.add({ data: 'lonely', type: 'document', metadata: {} }) + await brain.flush() + const scarDir = entityDir(dir, 'nouns', id)! + for (const f of fs.readdirSync(scarDir)) fs.rmSync(path.join(scarDir, f)) // empty the dir, keep it + expect(fs.existsSync(scarDir)).toBe(true) + + await brain.repairIndex() + + expect(fs.existsSync(scarDir)).toBe(false) + }) + + it('a healthy entity (both legs present) is NEVER pruned', async () => { + const id = await brain.add({ data: 'keep me', type: 'document', metadata: { keep: true } }) + await brain.flush() + + await brain.repairIndex() + + expect(entityDir(dir, 'nouns', id)).not.toBeNull() + expect(await brain.get(id)).not.toBeNull() + }) +}) diff --git a/tests/integration/entity-tree-stamp.test.ts b/tests/integration/entity-tree-stamp.test.ts new file mode 100644 index 00000000..deefc5e6 --- /dev/null +++ b/tests/integration/entity-tree-stamp.test.ts @@ -0,0 +1,163 @@ +/** + * @module tests/integration/entity-tree-stamp + * @description The entity tree's FAMILY STAMP: written at flush/close with + * `sourceGeneration` (the committed generation the canonical tree reflects) + * plus rollup invariants (entity/relationship counts); verified at open by + * comparison — coherent on a clean close, LOUD on genuine incoherence + * (tampered counters), healed by repairIndex()'s recount + re-stamp. One + * verifier reads both member modes. + */ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { + Brainy, + readFamilyStamp, + verifyFamilyStamp, + ENTITY_TREE_STAMP_PATH, + type FamilyStamp +} from '../../src/index.js' +import { prodLog } from '../../src/utils/logger.js' + +describe('entity-tree family stamp', () => { + let dir: string + let brain: any + + const open = async () => { + const b: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) + await b.init() + return b + } + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-stamp-')) + brain = await open() + }) + afterEach(async () => { + vi.restoreAllMocks() + await brain.close?.().catch(() => {}) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('flush() writes the stamp: rollup mode, counts + sourceGeneration match live state', async () => { + for (let i = 0; i < 3; i++) await brain.add({ data: `s${i}`, type: 'document', metadata: { i } }) + await brain.flush() + + const stamp = (await readFamilyStamp(brain.storage, ENTITY_TREE_STAMP_PATH)) as FamilyStamp + expect(stamp).not.toBeNull() + expect(stamp.family).toBe('entity-tree') + expect(stamp.members.mode).toBe('rollup') + const invariants = (stamp.members as any).invariants + expect(invariants.nounCount).toBe(await brain.storage.getNounCount()) + expect(invariants.verbCount).toBe(await brain.storage.getVerbCount()) + expect(stamp.sourceGeneration).toBe(brain.generation()) + expect(stamp.generation).toBeGreaterThanOrEqual(1) + }) + + it('a cleanly-closed store reopens COHERENT (no incoherence warning)', async () => { + await brain.add({ data: 'clean', type: 'document', metadata: {} }) + await brain.close() + + const warn = vi.spyOn(prodLog, 'warn') + brain = await open() + const stampWarnings = warn.mock.calls.filter((c) => String(c[0]).includes('entity-tree stamp')) + expect(stampWarnings).toEqual([]) + }) + + it('tampered counters surface as INCOHERENT at open; repairIndex() heals + re-stamps', async () => { + for (let i = 0; i < 3; i++) await brain.add({ data: `t${i}`, type: 'document', metadata: { i } }) + await brain.close() + + // Simulate counter drift AFTER the stamp was written: inflate the + // persisted scalar the way the historical decrement-skip did. + brain = await open() + ;(brain.storage as any).totalNounCount += 52 + await (brain.storage as any).persistCounts() + await brain.close() + // The close boundary re-stamps with the inflated counter — so tamper the + // STAMP instead for a deterministic mismatch: stamped counts differ from + // the (inflated) live ones at the NEXT open only if the stamp is older. + // Rewrite the stamp with the honest counts + current sourceGeneration. + const raw = JSON.parse( + require('node:zlib') + .gunzipSync(fs.readFileSync(path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`))) + .toString('utf-8') + ) as FamilyStamp + const honest = { ...raw } + ;(honest.members as any).invariants.nounCount -= 52 + fs.writeFileSync( + path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`), + require('node:zlib').gzipSync(JSON.stringify(honest)) + ) + + const warn = vi.spyOn(prodLog, 'warn') + brain = await open() + const incoherent = warn.mock.calls.filter((c) => String(c[0]).includes('INCOHERENT')) + expect(incoherent.length).toBeGreaterThanOrEqual(1) + expect(String(incoherent[0][0])).toMatch(/nounCount/) + + // The heal: recount from canonical + re-stamp → next open is quiet. + await brain.repairIndex() + await brain.close() + const warn2 = vi.spyOn(prodLog, 'warn') + brain = await open() + const stillIncoherent = warn2.mock.calls.filter((c) => String(c[0]).includes('INCOHERENT')) + expect(stillIncoherent).toEqual([]) + }) + + it('the one verifier handles both member modes', () => { + const rollup: FamilyStamp = { + family: 'x', + generation: 1, + committedAt: new Date().toISOString(), + sourceGeneration: 5, + members: { mode: 'rollup', invariants: { nounCount: 10 } } + } + expect(verifyFamilyStamp(rollup, 5, { nounCount: 10 })).toEqual({ state: 'coherent' }) + expect(verifyFamilyStamp(rollup, 5, { nounCount: 11 }).state).toBe('incoherent') + expect(verifyFamilyStamp(rollup, 9, { nounCount: 10 })).toEqual({ + state: 'behind', + stampSource: 5, + head: 9 + }) + expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 }).state).toBe('incoherent') // ahead of head + expect(verifyFamilyStamp(null, 5, {})).toEqual({ state: 'absent' }) + + const enumerated: FamilyStamp = { + family: 'y', + generation: 1, + committedAt: new Date().toISOString(), + sourceGeneration: 2, + members: { mode: 'enumerated', files: [{ path: 'a.bin', bytes: 128 }] } + } + expect(verifyFamilyStamp(enumerated, 2, { 'a.bin': 128 })).toEqual({ state: 'coherent' }) + expect(verifyFamilyStamp(enumerated, 2, { 'a.bin': 64 }).state).toBe('incoherent') + expect(verifyFamilyStamp(enumerated, 2, {}).state).toBe('incoherent') // missing member + + // Rollup invariants may be STRING fingerprints (e.g. a per-tree SHA-256): + // strict equality either way; a type mismatch reads as incoherence. + const fingerprinted: FamilyStamp = { + family: 'z', + generation: 1, + committedAt: new Date().toISOString(), + sourceGeneration: 3, + members: { mode: 'rollup', invariants: { treeDigest: 'abc123', rows: 42 } } + } + expect(verifyFamilyStamp(fingerprinted, 3, { treeDigest: 'abc123', rows: 42 })).toEqual({ + state: 'coherent' + }) + expect(verifyFamilyStamp(fingerprinted, 3, { treeDigest: 'deadbeef', rows: 42 }).state).toBe( + 'incoherent' + ) + expect(verifyFamilyStamp(fingerprinted, 3, { treeDigest: 'abc123', rows: '42' }).state).toBe( + 'incoherent' // type mismatch never passes + ) + }) +}) diff --git a/tests/integration/fact-log-contracts.test.ts b/tests/integration/fact-log-contracts.test.ts new file mode 100644 index 00000000..eb579da9 --- /dev/null +++ b/tests/integration/fact-log-contracts.test.ts @@ -0,0 +1,125 @@ +/** + * @module tests/integration/fact-log-contracts + * @description Pinned durability + stability contracts for the fact log. + * + * (1) FSYNC-BEFORE-ACK: an acknowledged write's fact survives an abrupt + * process end (no flush, no close — reopen from disk). + * - transact(): HOLDS TODAY — the fact is fsync'd before transact returns. + * - single-op: PINNED AS `it.fails` — today's group-commit batches + * DURABILITY (ack precedes the group fsync; a hard kill loses the fact + * AND the generation together, coherently — the documented Model-B + * contract, fine while the tree is authoritative). The destination + * (ack-at-log) requires group commit to become LATENCY batching: the + * ack waits for the shared fsync. When that lands, this pin flips red — + * remove `.fails` and the contract is permanent. No cliff to discover. + * + * (2) SCAN STABILITY UNDER ROTATION: a scan handle opened before segment + * rotation yields exactly its snapshot — byte-identical facts, no gaps, + * no duplicates, and no bleed-in of facts appended after the snapshot. + * (The reclaim-during-scan variant lands with fact-log compaction, which + * does not exist yet — segments only rotate today, never reclaim.) + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy, type CommitFact } from '../../src/index.js' +import { MemoryStorage } from '../../src/storage/adapters/memoryStorage.js' +import { FactLog, type FactLogStorage } from '../../src/db/factLog.js' + +describe('fsync-before-ack contract (fact durability at the ack boundary)', () => { + let dir: string + let brain: any + + const open = async () => { + const b: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) + await b.init() + return b + } + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-factack-')) + brain = await open() + }) + afterEach(async () => { + await brain.close?.().catch(() => {}) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('transact(): the fact is durable the moment the ack returns (kill-after-ack safe)', async () => { + const receipt = await brain.transact([ + { op: 'add', type: 'document', metadata: { durable: 1 }, data: 'ack-at-commit' } + ]) + // Abrupt end: no flush(), no close() — a new instance reads only disk. + brain = await open() + const facts: CommitFact[] = [] + for await (const b of brain.scanFacts()!.batches()) facts.push(...b.facts) + expect(facts.some((f) => f.generation === receipt.generation)).toBe(true) + }) + + // PINNED (flips red when group commit becomes latency batching — then + // remove `.fails` and the ack-at-log contract is permanent on every path). + it.fails('single-op: the fact is durable the moment the ack returns (the ack-at-log target)', async () => { + await brain.add({ data: 'acked single-op', type: 'document', metadata: { n: 1 } }) + const ackedHead = brain.scanFacts()!.headGeneration + // Abrupt end immediately after the ack — before any flush window. + brain = await open() + const facts: CommitFact[] = [] + for await (const b of brain.scanFacts()!.batches()) facts.push(...b.facts) + expect(facts.some((f) => f.generation === ackedHead)).toBe(true) + }) +}) + +describe('scan stability under rotation (the snapshot contract)', () => { + const UUID = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, '0')}` + const fact = (generation: number): CommitFact => ({ + generation, + timestamp: 1_700_000_000_000 + generation, + ops: [ + { + kind: 'noun', + id: UUID(generation), + // Padding makes each frame ~1KB so a small rotateBytes forces rotations. + record: { metadata: { noun: 'document', pad: 'x'.repeat(900), g: generation }, vector: null } + } + ] + }) + + it('a scan opened before rotations yields its exact snapshot — no gaps, dups, or bleed-in', async () => { + const mem: any = new MemoryStorage() + await mem.init() + const log = new FactLog(mem as FactLogStorage, { rotateBytes: 4096 }) // ~4 facts per segment + await log.open(0) + for (let g = 1; g <= 10; g++) await log.append(fact(g)) + await log.sync() + + // Open the snapshot, THEN keep appending — forcing further rotations. + const scan = log.scanFacts() + expect(scan.headGeneration).toBe(10) + for (let g = 11; g <= 25; g++) await log.append(fact(g)) + await log.sync() + expect(log.headGeneration()).toBe(25) + + const seen: number[] = [] + for await (const batch of scan.batches()) { + for (const f of batch.facts) seen.push(f.generation) + } + // Exactly the snapshot: 1..10 in order, nothing appended-after bleeds in. + expect(seen).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + expect(scan.summary().factsYielded).toBe(10) + + // And a fresh scan sees everything, across all rotated segments. + const all: number[] = [] + for await (const batch of log.scanFacts().batches()) { + for (const f of batch.facts) all.push(f.generation) + } + expect(all).toEqual(Array.from({ length: 25 }, (_, i) => i + 1)) + expect(log.segmentPaths().length).toBeGreaterThanOrEqual(2) // rotations actually happened + }) +}) diff --git a/tests/integration/fact-log-dual-write.test.ts b/tests/integration/fact-log-dual-write.test.ts new file mode 100644 index 00000000..5ec66273 --- /dev/null +++ b/tests/integration/fact-log-dual-write.test.ts @@ -0,0 +1,219 @@ +/** + * @module tests/integration/fact-log-dual-write + * @description The generation fact log end-to-end through real commits: every + * committed generation (single-op AND transact) appends its AFTER-IMAGE fact + * at the commit point; removals append body-less tombstones; an aborted + * transaction leaves no fact; facts survive reopen and continue monotonically; + * the scan surface (brain.scanFacts) carries the frozen telemetry shape; and + * the fact-log namespace is protected against prefix-nuking. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy, ProtectedArtifactError, type CommitFact } from '../../src/index.js' + +async function allFacts(brain: any): Promise { + const scan = brain.scanFacts() + expect(scan).not.toBeNull() + const facts: CommitFact[] = [] + for await (const batch of scan!.batches()) facts.push(...batch.facts) + return facts +} + +describe('fact log dual-write (memory adapter)', () => { + let brain: any + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true, dimensions: 384 }) + await brain.init() + }) + afterEach(async () => { + await brain.close?.().catch(() => {}) + }) + + it('every single-op write appends its after-image fact; a remove appends a tombstone', async () => { + const id = await brain.add({ data: 'first', type: 'document', metadata: { rev: 1 } }) + await brain.update({ id, metadata: { rev: 2 } }) + await brain.remove(id) + + const facts = await allFacts(brain) + // add + update + remove each committed a generation (the remove may span + // cascade ops but is ONE generation). Facts are monotonic. + const gens = facts.map((f) => f.generation) + expect([...gens].sort((a, b) => a - b)).toEqual(gens) + expect(facts.length).toBeGreaterThanOrEqual(3) + + // The add fact carries the after-image of the new entity. + const addFact = facts.find((f) => f.ops.some((op) => op.id === id && op.record !== null)) + expect(addFact).toBeDefined() + + // The remove fact carries a body-less tombstone for the id. + const removeFact = facts[facts.length - 1] + const tombstone = removeFact.ops.find((op) => op.id === id) + expect(tombstone).toBeDefined() + expect(tombstone!.record).toBeNull() + expect(tombstone!.kind).toBe('noun') + }) + + it('the update fact holds the NEW state (after-image, not before)', async () => { + const id = await brain.add({ data: 'versioned', type: 'document', metadata: { v: 'old' } }) + await brain.update({ id, metadata: { v: 'new' } }) + + const facts = await allFacts(brain) + const updateFact = facts[facts.length - 1] + const op = updateFact.ops.find((o) => o.id === id)! + expect(op.record).not.toBeNull() + expect((op.record!.metadata as any).v).toBe('new') + }) + + it('a transact commits ONE fact carrying all its ops, with meta', async () => { + const receipt = await brain.transact( + [ + { op: 'add', type: 'document', metadata: { part: 1 }, data: 'a' }, + { op: 'add', type: 'document', metadata: { part: 2 }, data: 'b' } + ], + { meta: { source: 'batch-import' } } + ) + + const facts = await allFacts(brain) + const txFact = facts.find((f) => f.generation === receipt.generation) + expect(txFact).toBeDefined() + expect(txFact!.ops.filter((op) => op.kind === 'noun').length).toBeGreaterThanOrEqual(2) + expect(txFact!.meta).toEqual({ source: 'batch-import' }) + }) + + it('an aborted transact leaves NO fact (absent = never committed)', async () => { + const id = await brain.add({ data: 'cas target', type: 'document', metadata: { n: 1 } }) + const before = (await allFacts(brain)).length + + await expect( + brain.transact([{ op: 'update', id, ifRev: 999, metadata: { n: 2 } }]) + ).rejects.toThrow() + + const after = await allFacts(brain) + expect(after.length).toBe(before) + }) + + it('fact generations line up with the transaction log', async () => { + await brain.add({ data: 'x', type: 'document', metadata: {} }) + await brain.add({ data: 'y', type: 'document', metadata: {} }) + await brain.flush() + + const facts = await allFacts(brain) + const logGens = new Set((await brain.transactionLog()).map((e: any) => e.generation)) + for (const f of facts) { + expect(logGens.has(f.generation)).toBe(true) + } + }) + + it('the storage fact-scan capability serves a provider holding only `storage`', async () => { + // An index provider receives `storage` — never the brain — and reaches the + // fact log through the host-wired capability (it must never construct its + // own fact-log reader: the log's open path is writer-side). + const id = await brain.add({ data: 'via storage', type: 'document', metadata: { s: 1 } }) + await brain.remove(id) + + const storage = brain.storage + expect(typeof storage.scanFacts).toBe('function') + expect(storage.factLogHeadGeneration()).toBe(brain.scanFacts()!.headGeneration) + + const viaStorage: CommitFact[] = [] + for await (const b of storage.scanFacts()!.batches()) viaStorage.push(...b.facts) + const viaBrain: CommitFact[] = [] + for await (const b of brain.scanFacts()!.batches()) viaBrain.push(...b.facts) + expect(viaStorage.map((f) => f.generation)).toEqual(viaBrain.map((f) => f.generation)) + expect(storage.factSegmentPaths()).toEqual(brain.factSegmentPaths()) + }) + + it('scan telemetry carries the frozen shape end-to-end', async () => { + for (let i = 0; i < 5; i++) await brain.add({ data: `t${i}`, type: 'document', metadata: { i } }) + + const scan = brain.scanFacts({ batchSize: 2 })! + expect(scan.headGeneration).toBeGreaterThanOrEqual(5) + expect(scan.approxFactCount).toBeGreaterThanOrEqual(5) + let batches = 0 + for await (const b of scan.batches()) { + batches++ + expect(b.factCount).toBe(b.facts.length) + expect(b.firstGeneration).toBe(b.facts[0].generation) + expect(b.lastGeneration).toBe(b.facts[b.facts.length - 1].generation) + expect(b.byteSize).toBeGreaterThan(0) + expect(typeof b.segmentId).toBe('string') + } + expect(batches).toBeGreaterThan(1) + expect(scan.summary().factsYielded).toBe(scan.approxFactCount) + }) +}) + +describe('fact log dual-write (filesystem adapter — durability + protection)', () => { + let dir: string + let brain: any + + const open = async () => { + const b: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) + await b.init() + return b + } + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-factlog-')) + brain = await open() + }) + afterEach(async () => { + await brain.close?.().catch(() => {}) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('facts survive close + reopen and appends continue monotonically', async () => { + const id = await brain.add({ data: 'persist me', type: 'document', metadata: { k: 1 } }) + await brain.remove(id) + await brain.close() + + brain = await open() + const facts = await allFacts(brain) + expect(facts.length).toBeGreaterThanOrEqual(2) + const headBefore = facts[facts.length - 1].generation + + await brain.add({ data: 'after reopen', type: 'document', metadata: { k: 2 } }) + const facts2 = await allFacts(brain) + expect(facts2[facts2.length - 1].generation).toBeGreaterThan(headBefore) + }) + + it('the fact segments exist on disk under _generations/facts/ with zero-padded names', async () => { + await brain.add({ data: 'on disk', type: 'document', metadata: {} }) + await brain.flush() + const factsDir = path.join(dir, '_generations', 'facts') + const files = fs.readdirSync(factsDir) + // The manifest rides the store's JSON object discipline (gzip on disk). + expect(files.some((f) => f.startsWith('manifest.json'))).toBe(true) + const segs = files.filter((f) => /^seg-\d{20}\.bfl$/.test(f)) + expect(segs.length).toBeGreaterThanOrEqual(1) + }) + + it('the fact-log namespace is PROTECTED: a prefix-nuke is refused', async () => { + await brain.add({ data: 'protected', type: 'document', metadata: {} }) + await expect(brain.storage.removeRawPrefix('_generations/facts')).rejects.toBeInstanceOf( + ProtectedArtifactError + ) + // Per-generation history cleanup remains unaffected (no false intersect). + await expect(brain.storage.removeRawPrefix('_generations/999999')).resolves.toBeUndefined() + }) + + it('transact facts are durable-on-return (no flush needed before reopen)', async () => { + const receipt = await brain.transact([ + { op: 'add', type: 'document', metadata: { durable: true }, data: 'tx' } + ]) + // Simulate an abrupt end: no flush(), no close() — reopen from disk. + brain = await open() + const facts = await allFacts(brain) + expect(facts.some((f) => f.generation === receipt.generation)).toBe(true) + }) +}) diff --git a/tests/integration/graph-audit.test.ts b/tests/integration/graph-audit.test.ts new file mode 100644 index 00000000..74fcc96a --- /dev/null +++ b/tests/integration/graph-audit.test.ts @@ -0,0 +1,164 @@ +/** + * @module tests/integration/graph-audit + * @description brain.auditGraph() — the read-only graph-truth instrument. + * Laws under test: + * (1) a healthy brain audits COHERENT: every canonical verb is returned by the + * read path of its source, endpoints exist, no ghosts; + * (2) design-hidden edges (internal/system visibility) are counted separately + * and never misclassified as index loss; + * (3) a verb whose endpoint entity was destroyed at the storage layer (the + * scar class) is flagged as a dangling endpoint, loudly; + * (4) the classification core flags present-but-invisible and ghost edges + * exactly (exercised via injected seams — manufacturing a genuinely stale + * adjacency index end-to-end would require corrupting internals the + * public API rightly refuses to corrupt). + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/index.js' +import { NounType, VerbType } from '../../src/types/graphTypes.js' +import { runGraphAudit, type AuditVerbRecord } from '../../src/graph/graphAudit.js' + +describe('brain.auditGraph() — graph-truth audit', () => { + let dir: string + let brain: any + + beforeEach(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-graph-audit-')) + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true + }) + await brain.init() + }) + + afterEach(async () => { + await brain.close().catch(() => {}) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + async function seedGraph(): Promise<{ ids: string[]; verbIds: string[] }> { + const ids: string[] = [] + for (let i = 0; i < 5; i++) { + ids.push( + await brain.add({ + data: `entity ${i}`, + type: NounType.Concept, + metadata: { n: i } + }) + ) + } + const verbIds: string[] = [] + verbIds.push(await brain.relate({ from: ids[0], to: ids[1], type: VerbType.RelatedTo })) + verbIds.push(await brain.relate({ from: ids[0], to: ids[2], type: VerbType.Contains })) + verbIds.push(await brain.relate({ from: ids[1], to: ids[3], type: VerbType.DependsOn })) + verbIds.push(await brain.relate({ from: ids[3], to: ids[4], type: VerbType.RelatedTo })) + return { ids, verbIds } + } + + it('audits a healthy brain as coherent, with exact counts', async () => { + const { ids } = await seedGraph() + const report = await brain.auditGraph() + + expect(report.coherent).toBe(true) + expect(report.verbsInCanonical).toBe(4) + expect(report.entitiesInCanonical).toBeGreaterThanOrEqual(ids.length) // VFS root etc. may add system nouns + expect(report.sourcesChecked).toBe(3) // ids[0], ids[1], ids[3] + expect(report.missingFromReadsCount).toBe(0) + expect(report.danglingEndpointsCount).toBe(0) + expect(report.readOnlyCount).toBe(0) + expect(report.truncatedExamples).toBe(false) + }) + + it('counts design-hidden edges separately and stays coherent', async () => { + const { ids } = await seedGraph() + await brain.relate({ + from: ids[2], + to: ids[4], + type: VerbType.RelatedTo, + visibility: 'internal' + }) + + const report = await brain.auditGraph() + expect(report.coherent).toBe(true) // hidden-by-design is NOT a discrepancy + expect(report.verbsInCanonical).toBe(5) + expect(report.visibilityHiddenCount).toBeGreaterThanOrEqual(1) + }) + + it('flags a destroyed endpoint as a dangling verb (the scar class)', async () => { + const { ids } = await seedGraph() + // Destroy ids[4] at the STORAGE layer (bypassing remove(), which would + // also delete its verbs) — the historical partial-delete scar shape. + await brain.storage.deleteNoun(ids[4]) + + const report = await brain.auditGraph() + expect(report.coherent).toBe(false) + expect(report.danglingEndpointsCount).toBe(1) + expect(report.danglingEndpoints[0].to).toBe(ids[4]) + expect(report.danglingEndpoints[0].missingEnd).toBe('to') + }) +}) + +describe('runGraphAudit classification core (injected seams)', () => { + const verb = (id: string, from: string, to: string): AuditVerbRecord => ({ + id, + type: 'relatedTo', + sourceId: from, + targetId: to + }) + + const deps = (opts: { + nouns: string[] + verbs: AuditVerbRecord[] + reads: Record // sourceId -> verb ids the read path returns + }) => ({ + eachNounId: async (consume: (id: string) => void) => { + for (const id of opts.nouns) consume(id) + }, + eachVerb: async (consume: (v: AuditVerbRecord) => void) => { + for (const v of opts.verbs) consume(v) + }, + readRelationsFrom: async (sourceId: string) => + (opts.reads[sourceId] ?? []).map((id) => ({ id })) + }) + + it('flags a canonical verb the read path omits — present but invisible', async () => { + const report = await runGraphAudit( + deps({ + nouns: ['A', 'B', 'C'], + verbs: [verb('v1', 'A', 'B'), verb('v2', 'A', 'C')], + reads: { A: ['v1'] } // v2 exists canonically but reads miss it + }) + ) + expect(report.coherent).toBe(false) + expect(report.missingFromReadsCount).toBe(1) + expect(report.missingFromReads[0].verbId).toBe('v2') + }) + + it('flags a read-path edge with no canonical record — a ghost', async () => { + const report = await runGraphAudit( + deps({ + nouns: ['A', 'B'], + verbs: [verb('v1', 'A', 'B')], + reads: { A: ['v1', 'ghost-9'] } + }) + ) + expect(report.coherent).toBe(false) + expect(report.readOnlyCount).toBe(1) + expect(report.readOnlyVerbIds).toEqual(['ghost-9']) + }) + + it('caps example lists but keeps counts exact, and says so', async () => { + const verbs = Array.from({ length: 10 }, (_, i) => verb(`v${i}`, 'A', 'B')) + const report = await runGraphAudit( + deps({ nouns: ['A', 'B'], verbs, reads: { A: [] } }), + { maxExamples: 3 } + ) + expect(report.missingFromReadsCount).toBe(10) + expect(report.missingFromReads.length).toBe(3) + expect(report.truncatedExamples).toBe(true) + }) +}) diff --git a/tests/integration/history-repacking.test.ts b/tests/integration/history-repacking.test.ts new file mode 100644 index 00000000..2bcee038 --- /dev/null +++ b/tests/integration/history-repacking.test.ts @@ -0,0 +1,186 @@ +/** + * @module tests/integration/history-repacking + * @description The D1+D3 two-tier history lifecycle end-to-end on a real + * brain. Laws: (1) repacking is RE-REPRESENTATION — after folding, every + * asOf() read below the fold boundary answers exactly as before, across a + * cold reopen; (2) folded per-generation directories are physically gone + * (the file-count cure is real, not cosmetic); (3) repack + reclaim compose: + * bounded retention after repacking drops whole segments and asOf below the + * horizon throws GenerationCompactedError; (4) repackHistory is explicit + * API and time-bounded (spent budget = consistent no-op). + * + * Uses a tiny REPACK_LIVE_WINDOW override so a small history has a cold + * tier at all (the production window is 1024). + */ +import { describe, it, expect, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as path from 'node:path' +import * as os from 'node:os' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import { GenerationStore } from '../../src/db/generationStore.js' +import { GenerationCompactedError } from '../../src/db/errors.js' +import { SEGMENTS_PREFIX } from '../../src/db/generationSegments.js' + +const stub = async (text: string): Promise => { + const h = text.split('').reduce((a, c) => a + c.charCodeAt(0), 0) + return new Array(384).fill(0).map((_, i) => Math.sin(h + i)) +} + +const openBrain = async (dir: string): Promise => { + const brain = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + embeddingFunction: stub + }) + await brain.init() + return brain +} + +describe('history repacking — the two-tier lifecycle', () => { + const dirs: string[] = [] + const tempDir = (): string => { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-repack-')) + dirs.push(d) + return d + } + const originalWindow = GenerationStore.REPACK_LIVE_WINDOW + + afterEach(() => { + ;(GenerationStore as any).REPACK_LIVE_WINDOW = originalWindow + for (const d of dirs.splice(0)) { + try { + fs.rmSync(d, { recursive: true, force: true }) + } catch { + /* best effort */ + } + } + }) + + it('repack preserves every historical read across cold reopen; folded dirs are gone', async () => { + ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3 + const dir = tempDir() + const brain = await openBrain(dir) + + const id = await brain.add({ + data: 'versioned-entity', + type: NounType.Document, + metadata: { v: 0 } + }) + for (let v = 1; v <= 10; v++) await brain.update({ id, metadata: { v } }) + await brain.flush() + + // Ground truth BEFORE repacking: capture asOf views for early generations. + const before: Record = {} + for (const g of [2, 4, 6]) { + const db = await brain.asOf(g) + before[g] = (await db.get(id))?.metadata?.v as number + await db.release() + } + + const result = await brain.repackHistory() + expect(result.foldedGenerations).toBeGreaterThan(0) + expect(result.segmentsCreated).toBeGreaterThan(0) + + // The folded per-generation directories are PHYSICALLY gone… + const genDirs = fs + .readdirSync(path.join(dir, '_generations'), { withFileTypes: true }) + .filter((e) => e.isDirectory() && /^\d+$/.test(e.name)).length + expect(genDirs).toBeLessThanOrEqual(4) // live window (3) + at most the newest + // …and the segment tier exists (the filesystem adapter stores objects + // gzipped, so the manifest may live at either spelling). + const segDir = path.join(dir, SEGMENTS_PREFIX) + expect( + fs.existsSync(path.join(segDir, 'manifest.json')) || + fs.existsSync(path.join(segDir, 'manifest.json.gz')) + ).toBe(true) + expect(fs.readdirSync(segDir).some((f) => f.endsWith('.bgs'))).toBe(true) + + // Same asOf answers from the packed tier, same process… + for (const g of [2, 4, 6]) { + const db = await brain.asOf(g) + expect((await db.get(id))?.metadata?.v).toBe(before[g]) + await db.release() + } + await brain.close() + + // …and across a COLD REOPEN (manifest discovery, no live dirs to list). + const reopened = await openBrain(dir) + for (const g of [2, 4, 6]) { + const db = await reopened.asOf(g) + expect((await db.get(id))?.metadata?.v).toBe(before[g]) + await db.release() + } + expect((await reopened.get(id))?.metadata?.v).toBe(10) // live state untouched + await reopened.close() + }) + + it('repack + bounded reclaim compose: whole segments drop, horizon is loud', async () => { + ;(GenerationStore as any).REPACK_LIVE_WINDOW = 2 + const dir = tempDir() + const brain = await openBrain(dir) + const id = await brain.add({ data: 'reclaim-probe', type: NounType.Document, metadata: { v: 0 } }) + for (let v = 1; v <= 8; v++) await brain.update({ id, metadata: { v } }) + await brain.flush() + await brain.repackHistory() + + // Reclaim down to the 3 newest generations — packed segments below the + // horizon drop whole; asOf below throws loudly. + const res = await brain.compactHistory({ maxGenerations: 3 }) + expect(res.removedGenerations).toBeGreaterThan(0) + await expect(brain.asOf(1)).rejects.toBeInstanceOf(GenerationCompactedError) + expect((await brain.get(id))?.metadata?.v).toBe(8) + await brain.close() + }) + + it('generationDigest: reopen-stable, divergence-sensitive, loud below the horizon', async () => { + ;(GenerationStore as any).REPACK_LIVE_WINDOW = 2 + const dir = tempDir() + const brain = await openBrain(dir) + const id = await brain.add({ data: 'digest-probe', type: NounType.Document, metadata: { v: 0 } }) + for (let v = 1; v <= 6; v++) await brain.update({ id, metadata: { v } }) + await brain.flush() + await brain.repackHistory() + + const gen = brain.generation() + const atHead = await brain.generationDigest(gen) + const atMid = await brain.generationDigest(3) + expect(atHead).toMatch(/^[0-9a-f]{8}$/) + expect(atMid).not.toBe(atHead) // more history ⇒ different digest + await brain.close() + + // Reopen-stable: same history, same digests (packed prefix stability). + const reopened = await openBrain(dir) + expect(await reopened.generationDigest(gen)).toBe(atHead) + expect(await reopened.generationDigest(3)).toBe(atMid) + + // New history diverges the head digest. + await reopened.update({ id, metadata: { v: 7 } }) + await reopened.flush() + expect(await reopened.generationDigest(reopened.generation())).not.toBe(atHead) + + // Below the horizon: LOUD, never a silent pin of reclaimed history. + await reopened.compactHistory({ maxGenerations: 2 }) + await expect(reopened.generationDigest(1)).rejects.toBeInstanceOf(GenerationCompactedError) + await reopened.close() + }) + + it('a spent time budget is a consistent no-op; the next pass resumes', async () => { + ;(GenerationStore as any).REPACK_LIVE_WINDOW = 2 + const dir = tempDir() + const brain = await openBrain(dir) + const id = await brain.add({ data: 'budget-probe', type: NounType.Document, metadata: { v: 0 } }) + for (let v = 1; v <= 6; v++) await brain.update({ id, metadata: { v } }) + await brain.flush() + + const bounded = await brain.repackHistory({ timeBudgetMs: 0 }) + expect(bounded).toEqual({ foldedGenerations: 0, segmentsCreated: 0 }) + + const resumed = await brain.repackHistory() + expect(resumed.foldedGenerations).toBeGreaterThan(0) + const db = await brain.asOf(3) + expect((await db.get(id))?.metadata?.v).toBeDefined() + await db.release() + await brain.close() + }) +}) diff --git a/tests/integration/ifabsent-upsert-blob-concurrency.test.ts b/tests/integration/ifabsent-upsert-blob-concurrency.test.ts new file mode 100644 index 00000000..119a5a97 --- /dev/null +++ b/tests/integration/ifabsent-upsert-blob-concurrency.test.ts @@ -0,0 +1,212 @@ +/** + * @module tests/integration/ifabsent-upsert-blob-concurrency + * @description The 8.0.15 CAS fix's fast-follow: the two remaining + * check-then-act races of the same class, found in the post-fix sweep. + * + * 1. `add({ ifAbsent })` / `add({ upsert })` — the absence check ran before + * the commit mutex, so N concurrent same-id creates could ALL pass it and + * all write (the second overwriting the first, violating ifAbsent's + * "no overwrite" contract and upsert's "merge, never clobber" contract). + * Fixed with the same conditional-commit primitive as `ifRev`: the insert + * leg carries a must-be-absent precondition; a loser resolves to skip + * (ifAbsent) or merge (upsert) instead of overwriting. + * + * 2. `BlobStorage` reference counts — `write()`'s dedup decision and + * `delete()`'s decrement-then-remove were unserialized read-modify-writes + * over `blob-meta:`; concurrent same-content writes could lose + * references, turning a later delete into premature removal of bytes + * another file still needs. Fixed with a per-hash mutex. + * + * Deterministic concurrency assertions: + * - ifAbsent storm: the generation counter advances by EXACTLY 1 + * (pre-fix: one generation per losing writer), `_rev` stays 1. + * - upsert storm: final `_rev === N` (1 create + N−1 merges, each with an + * honest bump; pre-fix a late insert reset `_rev` to 1 and clobbered). + * - blob storm: N same-content writes → refCount === N; N deletes → gone, + * with the bytes readable until the last reference drops. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import { BlobStorage } from '../../src/storage/blobStorage.js' + +describe('ifAbsent/upsert insert race + blob refCount (conditional commit fast-follow)', () => { + let dir: string + let brain: any + let seq = 0 + const freshId = (): string => + `00000000-0000-4000-8000-${(++seq).toString(16).padStart(12, '0')}` + + beforeAll(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-upsert-race-')) + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + dimensions: 384, + silent: true + }) + await brain.init() + }) + + afterAll(async () => { + await brain.close() + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('N concurrent add({ifAbsent}) → exactly ONE write (generation +1), no overwrite', async () => { + const id = freshId() + const genBefore = brain.generationStore.generation() + + const ids = await Promise.all( + Array.from({ length: 8 }, (_, i) => + brain.add({ + id, + ifAbsent: true, + data: `contender ${i}`, + type: NounType.Thing, + metadata: { writer: i } + }) + ) + ) + + // Every caller resolves to the same canonical id. + expect(new Set(ids).size).toBe(1) + + // THE contract: exactly one write landed. Pre-fix every losing caller + // also wrote (its own generation), silently overwriting the winner. + expect(brain.generationStore.generation()).toBe(genBefore + 1) + + const after = await brain.get(id) + expect(after._rev).toBe(1) + expect(typeof after.metadata.writer).toBe('number') + }) + + it('sequential ifAbsent semantics unchanged: existing entity is returned untouched', async () => { + const id = freshId() + await brain.add({ id, data: 'original', type: NounType.Thing, metadata: { keep: true } }) + const returned = await brain.add({ + id, + ifAbsent: true, + data: 'impostor', + type: NounType.Thing, + metadata: { keep: false } + }) + expect(returned).toBe(id) + const after = await brain.get(id) + expect(after.metadata.keep).toBe(true) + expect(after.data).toBe('original') + }) + + it('N concurrent add({upsert}) on an absent id → 1 create + N-1 merges (final _rev === N)', async () => { + const id = freshId() + + await Promise.all( + Array.from({ length: 8 }, (_, i) => + brain.add({ + id, + upsert: true, + data: 'shared upsert target', + type: NounType.Thing, + metadata: { [`k${i}`]: true } + }) + ) + ) + + const after = await brain.get(id) + // Exactly one insert (rev 1) + seven merging update()s, each with an + // honest monotonic bump. Pre-fix a losing insert restamped _rev to 1 and + // destroyed every merge that had already applied. + expect(after._rev).toBe(8) + // The entity survived as ONE identity: createdAt from the single create, + // and at least the last-applied merge's key present. + expect(Object.keys(after.metadata).filter((k) => k.startsWith('k')).length) + .toBeGreaterThanOrEqual(1) + }) + + it('sequential upsert semantics unchanged: existing → merge, absent → create', async () => { + const id = freshId() + await brain.add({ id, upsert: true, data: 'v1', type: NounType.Thing, metadata: { a: 1 } }) + expect((await brain.get(id))._rev).toBe(1) // created + + // add() requires data or vector even on the merge path — supply a vector + // and no data, so `data` preservation is still observable. + const vec = Array.from({ length: 384 }, (_, i) => (i % 7) / 7 - 0.5) + await brain.add({ id, upsert: true, vector: vec, type: NounType.Thing, metadata: { b: 2 } }) + const after = await brain.get(id) + expect(after._rev).toBe(2) // merged, not overwritten + expect(after.metadata.a).toBe(1) + expect(after.metadata.b).toBe(2) + expect(after.data).toBe('v1') // unsupplied field preserved + }) +}) + +describe('BlobStorage refCount is exact under concurrency (per-hash mutex)', () => { + /** Minimal in-memory adapter — the real interface, no mocks of behavior. */ + function memAdapter() { + const kv = new Map() + return { + get: async (k: string) => kv.get(k), + put: async (k: string, v: Buffer) => void kv.set(k, v), + delete: async (k: string) => void kv.delete(k), + list: async (prefix: string) => + [...kv.keys()].filter((k) => k.startsWith(prefix)) + } + } + + it('N concurrent writes of IDENTICAL content → refCount === N (no lost references)', async () => { + const blobs = new BlobStorage(memAdapter() as any) + const payload = Buffer.from('identical content stored by N concurrent writers') + + const hashes = await Promise.all( + Array.from({ length: 10 }, () => blobs.write(payload)) + ) + expect(new Set(hashes).size).toBe(1) + const meta = await blobs.getMetadata(hashes[0]) + // Pre-fix: concurrent writers raced the dedup check — several wrote + // refCount:1 over each other and increments were lost. + expect(meta?.refCount).toBe(10) + }) + + it('references drop exactly; bytes survive live-zero (temporal immutability) until reclaim', async () => { + const blobs = new BlobStorage(memAdapter() as any) + const payload = Buffer.from('shared bytes, two referencing files') + + const hash = await blobs.write(payload) + await blobs.write(payload) // second reference (concurrent-equivalent path) + + await blobs.release(hash) // drop one reference + expect((await blobs.read(hash)).toString()).toBe(payload.toString()) // still readable + expect((await blobs.getMetadata(hash))?.refCount).toBe(1) + + await blobs.release(hash) // last LIVE reference — bytes still exist (history may need them) + expect(await blobs.has(hash)).toBe(true) + expect((await blobs.getMetadata(hash))?.refCount).toBe(0) + + // Reclamation is compaction's job: zero-zero → physically removed. + expect(await blobs.reclaimIfUnreferenced(hash)).toBe(true) + expect(await blobs.has(hash)).toBe(false) + await expect(blobs.read(hash)).rejects.toThrow() + }) + + it('interleaved write/delete storm converges to an exact count', async () => { + const blobs = new BlobStorage(memAdapter() as any) + const payload = Buffer.from('storm payload') + const hash = BlobStorage.hash(payload) + + // 12 writes and 5 releases racing: net 7 references, blob alive. + await Promise.all([ + ...Array.from({ length: 12 }, () => blobs.write(payload)), + ...Array.from({ length: 5 }, () => blobs.release(hash)) + ]) + const meta = await blobs.getMetadata(hash) + // Releases against a not-yet-written hash floor at zero, so the net can + // only be >= 12 - 5. The exactness we require: counts are never LOST + // (each landed write is represented). + expect(meta?.refCount).toBeGreaterThanOrEqual(7) + expect(await blobs.has(hash)).toBe(true) + }) +}) diff --git a/tests/integration/ifrev-concurrent-cas.test.ts b/tests/integration/ifrev-concurrent-cas.test.ts new file mode 100644 index 00000000..4b04c58f --- /dev/null +++ b/tests/integration/ifrev-concurrent-cas.test.ts @@ -0,0 +1,187 @@ +/** + * @module tests/integration/ifrev-concurrent-cas + * @description Regression for the P0 report that `update({ ifRev })` CAS was + * not atomic under concurrent in-process calls: N concurrent updates all + * carrying the same `ifRev` ALL fulfilled (0 conflicts, last-writer-wins) — + * the check ran before the commit mutex, so interleaved callers all passed it + * before any apply landed. In production that silently lost 7 of 8 + * ifRev-guarded ledger writes and let two workers both "acquire" an advisory + * lock built on exactly-one-winner semantics. + * + * The fix is a conditional commit: the generation store's `precommit` hook + * re-verifies `ifRev` against the authoritative before-image UNDER the commit + * mutex, atomically with the apply (the per-record analogue of + * `ifAtGeneration`, which always ran there). These tests pin: + * - exactly-one-winner for N concurrent same-rev `update({ ifRev })` + * - the same for `transact()` per-op ifRev (whole batch rejected) + * - `_rev` monotonicity under concurrent plain updates (no ifRev) + * - the CAS retry loop converging exactly (the ledger/credit-consume shape) + * - sequential conflict behavior unchanged + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +const isRevConflict = (e: unknown): boolean => + (e as Error)?.name === 'RevisionConflictError' || + String(e).includes('RevisionConflict') + +describe('ifRev CAS is atomic under concurrency (conditional commit)', () => { + let dir: string + let brain: any + let seq = 0 + const freshId = (): string => + `00000000-0000-4000-8000-${(++seq).toString(16).padStart(12, '0')}` + + beforeAll(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-cas-')) + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + dimensions: 384, + silent: true + }) + await brain.init() + }) + + afterAll(async () => { + await brain.close() + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('N concurrent update({ifRev}) → exactly 1 winner, N-1 RevisionConflictError', async () => { + const id = await brain.add({ + id: freshId(), + data: 'contended entity', + type: NounType.Thing, + metadata: { credits: 8 } + }) + const rev = (await brain.get(id))._rev + + const results = await Promise.allSettled( + Array.from({ length: 8 }, (_, i) => + brain.update({ id, metadata: { writer: i }, merge: false, ifRev: rev }) + ) + ) + + const wins = results.filter((r) => r.status === 'fulfilled') + const conflicts = results.filter( + (r) => r.status === 'rejected' && isRevConflict(r.reason) + ) + expect(wins.length).toBe(1) + expect(conflicts.length).toBe(7) + + // Exactly one bump; the surviving metadata belongs to the single winner. + const after = await brain.get(id) + expect(after._rev).toBe(rev + 1) + expect(typeof after.metadata.writer).toBe('number') + }) + + it('transact() per-op ifRev → exactly 1 batch wins, others rejected whole', async () => { + const id = await brain.add({ + id: freshId(), + data: 'transact contended', + type: NounType.Thing, + metadata: { state: 'initial' } + }) + const rev = (await brain.get(id))._rev + + const results = await Promise.allSettled( + Array.from({ length: 6 }, (_, i) => + brain.transact([ + { op: 'update', id, metadata: { batch: i }, merge: false, ifRev: rev } + ]) + ) + ) + expect(results.filter((r) => r.status === 'fulfilled').length).toBe(1) + expect( + results.filter((r) => r.status === 'rejected' && isRevConflict(r.reason)) + .length + ).toBe(5) + expect((await brain.get(id))._rev).toBe(rev + 1) + }) + + it('_rev is monotonic under concurrent plain updates (no ifRev): N updates → +N', async () => { + const id = await brain.add({ + id: freshId(), + data: 'plain concurrent updates', + type: NounType.Thing, + metadata: { v: 0 } + }) + const rev = (await brain.get(id))._rev + + await Promise.all( + Array.from({ length: 8 }, (_, i) => brain.update({ id, metadata: { v: i } })) + ) + // Every applied update gets its own honest bump (previously all stamped + // the same stale rev+1 they computed before the commit). + expect((await brain.get(id))._rev).toBe(rev + 8) + }) + + it('the CAS retry loop converges exactly (the ledger shape: 8 workers, 8 consumes)', async () => { + const id = await brain.add({ + id: freshId(), + data: 'credit budget', + type: NounType.Thing, + metadata: { credits: 8 } + }) + + // Each worker: read → CAS-decrement → retry on conflict. The docs' documented + // pattern; with atomic CAS this MUST land on exactly 0 with zero lost updates. + const consumeOne = async (): Promise => { + for (let attempt = 0; attempt < 50; attempt++) { + const cur = await brain.get(id) + if (cur.metadata.credits <= 0) throw new Error('budget exhausted') + try { + await brain.update({ + id, + metadata: { ...cur.metadata, credits: cur.metadata.credits - 1 }, + merge: false, + ifRev: cur._rev + }) + return + } catch (e) { + if (!isRevConflict(e)) throw e + } + } + throw new Error('no convergence after 50 attempts') + } + + await Promise.all(Array.from({ length: 8 }, () => consumeOne())) + expect((await brain.get(id)).metadata.credits).toBe(0) + }) + + it('sequential conflict behavior is unchanged (stale ifRev throws, fresh succeeds)', async () => { + const id = await brain.add({ + id: freshId(), + data: 'sequential control', + type: NounType.Thing, + metadata: { n: 1 } + }) + const rev = (await brain.get(id))._rev + + await brain.update({ id, metadata: { n: 2 }, ifRev: rev }) + await expect( + brain.update({ id, metadata: { n: 3 }, ifRev: rev }) + ).rejects.toMatchObject({ name: 'RevisionConflictError' }) + // Fresh rev succeeds. + const fresh = (await brain.get(id))._rev + await brain.update({ id, metadata: { n: 3 }, ifRev: fresh }) + expect((await brain.get(id)).metadata.n).toBe(3) + }) + + it('forward-ref add+update of the same entity in one transact() batch still works', async () => { + const id = freshId() + await brain.transact([ + { op: 'add', id, data: 'created in batch', type: NounType.Thing, metadata: { step: 1 } }, + { op: 'update', id, metadata: { step: 2 } } + ]) + const after = await brain.get(id) + expect(after.metadata.step).toBe(2) + expect(after._rev).toBe(2) // add stamps 1, in-batch update sequences to 2 + }) +}) diff --git a/tests/integration/lens-consistency.test.ts b/tests/integration/lens-consistency.test.ts new file mode 100644 index 00000000..64484a38 --- /dev/null +++ b/tests/integration/lens-consistency.test.ts @@ -0,0 +1,138 @@ +/** + * @module tests/integration/lens-consistency + * @description The three metadata "lenses" over one corpus must agree with + * canonical ground truth id-for-id, warm AND after a cold reopen: + * - combined: find({ type: T, where: { subtype: S } }) + * - subtype-only: find({ where: { subtype: S } }) + * - type-only: find({ type: T }) + * Ported from the fresh-brain probe that closed the type+subtype lens-drop + * investigation (a restored pre-8.2.2 torn capture had entities visible to the + * subtype-only lens but dropped by the combined lens — "0 of 2 migrated, all + * gates green"). The corpus is seeded through the REAL write API — never + * restored bytes — which is what made the original datapoint decisive. The + * invariants: every lens matches an unfiltered canonical scan exactly (no + * missing ids, no extras) and combined ⊆ subtype-only always holds. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/index.js' + +/** The corpus: 7 (type, subtype) pairs, uneven counts, incl. the incident's 2-of-a-pair shape. */ +const CORPUS: Array<{ type: string; subtype: string; count: number }> = [ + { type: 'proposition', subtype: 'decision', count: 2 }, // the incident shape: "0 of 2" + { type: 'concept', subtype: 'decision', count: 3 }, + { type: 'task', subtype: 'decision', count: 2 }, + { type: 'concept', subtype: 'action', count: 4 }, + { type: 'message', subtype: 'note', count: 5 }, + { type: 'message', subtype: 'ship', count: 3 }, + { type: 'document', subtype: 'guide', count: 4 } +] + +/** Canonical ground truth: unfiltered enumeration, post-filtered IN THE TEST. */ +async function groundTruth( + brain: any, + match: { type?: string; subtype?: string } +): Promise> { + const ids = new Set() + let cursor: string | undefined + for (;;) { + const page = await brain.storage.getNounsWithPagination({ limit: 500, cursor }) + for (const noun of page.items) { + // Hydrated shape: `type`/`subtype` are TOP-LEVEL; `metadata` holds only + // custom user fields (vfsType is one — the VFS plumbing marker). + const n = noun as any + if (n.metadata?.vfsType) continue // VFS plumbing is not corpus + if (!n.type || !n.subtype) continue + if (match.type && n.type !== match.type) continue + if (match.subtype && n.subtype !== match.subtype) continue + ids.add(n.id) + } + if (!page.hasMore) break + cursor = page.nextCursor + } + return ids +} + +const idSet = (results: Array<{ id: string }>): Set => new Set(results.map((r) => r.id)) + +/** Every lens vs ground truth, id-for-id, for every pair in the corpus. */ +async function assertAllLenses(brain: any): Promise { + const types = [...new Set(CORPUS.map((c) => c.type))] + const subtypes = [...new Set(CORPUS.map((c) => c.subtype))] + + for (const { type, subtype } of CORPUS) { + const combined = idSet(await brain.find({ type, where: { subtype }, limit: 1000 })) + const subtypeOnly = idSet(await brain.find({ where: { subtype }, limit: 1000 })) + const truthPair = await groundTruth(brain, { type, subtype }) + const truthSubtype = await groundTruth(brain, { subtype }) + + expect([...combined].sort()).toEqual([...truthPair].sort()) // no drops, no extras + expect([...subtypeOnly].sort()).toEqual([...truthSubtype].sort()) + for (const id of combined) expect(subtypeOnly.has(id)).toBe(true) // combined ⊆ subtype-only + } + + for (const type of types) { + const typeOnly = idSet(await brain.find({ type, limit: 1000 })) + const truthType = await groundTruth(brain, { type }) + expect([...typeOnly].sort()).toEqual([...truthType].sort()) + } + + // Count cross-check against the corpus definition itself. + for (const subtype of subtypes) { + const expected = CORPUS.filter((c) => c.subtype === subtype).reduce((s, c) => s + c.count, 0) + const got = (await brain.find({ where: { subtype }, limit: 1000 })).length + expect(got).toBe(expected) + } +} + +describe('lens consistency — combined vs subtype-only vs canonical ground truth', () => { + let dir: string + let brain: any + + beforeAll(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-lens-')) + brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 }) + await brain.init() + // Seed through the REAL write API — never restored bytes. + let i = 0 + for (const { type, subtype, count } of CORPUS) { + for (let k = 0; k < count; k++) { + await brain.add({ data: `${type} ${subtype} ${i++}`, type, subtype, metadata: { k } }) + } + } + await brain.flush() + }) + afterAll(async () => { + await brain.close?.().catch(() => {}) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('WARM: all lenses agree with ground truth id-for-id', async () => { + await assertAllLenses(brain) + }) + + it('COLD REOPEN: all lenses still agree after close + reopen from disk', async () => { + await brain.close() + brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 }) + await brain.init() + await assertAllLenses(brain) + }) + + it('after an update() flips type AND subtype, every lens tracks the move exactly', async () => { + // The historical cross-bucket-staleness path: change (concept, action) -> (task, review). + const victims = await brain.find({ type: 'concept', where: { subtype: 'action' }, limit: 1 }) + expect(victims.length).toBe(1) + const id = victims[0].id + await brain.update({ id, type: 'task', subtype: 'review' }) + + const oldCombined = idSet(await brain.find({ type: 'concept', where: { subtype: 'action' }, limit: 1000 })) + expect(oldCombined.has(id)).toBe(false) // unposted from the old buckets + const newCombined = idSet(await brain.find({ type: 'task', where: { subtype: 'review' }, limit: 1000 })) + expect(newCombined.has(id)).toBe(true) // posted to the new buckets + const subtypeOnly = idSet(await brain.find({ where: { subtype: 'review' }, limit: 1000 })) + expect(subtypeOnly.has(id)).toBe(true) + }) +}) diff --git a/tests/integration/migration-7x-to-8x.test.ts b/tests/integration/migration-7x-to-8x.test.ts index dccf8f88..6071d659 100644 --- a/tests/integration/migration-7x-to-8x.test.ts +++ b/tests/integration/migration-7x-to-8x.test.ts @@ -191,4 +191,39 @@ describe('7.x → 8.0 layout migration', () => { expect(await captureReference(reopened)).toEqual(ref) expect(fs.existsSync(path.join(dir, 'branches'))).toBe(false) }) + + it('PARITY GUARD: branch-scoped non-entity state survives migration (not silently drained)', async () => { + const dir = freshLegacyDir() + + // Simulate a 7.x engine that wrote durable state under the HEAD branch + // OUTSIDE entities/ (a branch-scoped index/blob/registry). The migration + // rescues entities/ only; the guard must PRESERVE the rest rather than let + // removeRawPrefix silently delete it — the VFS `_cow/` stranding lesson. + const raw: any = new FileSystemStorage(dir) + await raw.init() + await raw.writeRawObject('branches/main/_legacy_engine/state', { keep: 'me', n: 42 }) + await raw.flush?.() + raw.stopFlushRequestWatcher?.() + await raw.releaseWriterLock?.() + + // Migrate (entities collapse to the root as usual). + const brain = await openBrain(dir) + brains.push(brain) + await brain.init() + expect(await captureReference(brain)).toEqual(reference) + await brain.close() + brains.splice(brains.indexOf(brain), 1) + + // The non-entity branch-scoped state is PRESERVED (branch NOT drained) — + // recoverable, not lost. + const check: any = new FileSystemStorage(dir) + await check.init() + expect(await check.readRawObject('branches/main/_legacy_engine/state')).toMatchObject({ + keep: 'me', + n: 42 + }) + await check.flush?.() + check.stopFlushRequestWatcher?.() + await check.releaseWriterLock?.() + }) }) diff --git a/tests/integration/migration-backup.test.ts b/tests/integration/migration-backup.test.ts new file mode 100644 index 00000000..61b76b96 --- /dev/null +++ b/tests/integration/migration-backup.test.ts @@ -0,0 +1,189 @@ +/** + * @module tests/integration/migration-backup + * @description Proof suite for the pre-upgrade backup (#42): before a one-time + * 7.x → 8.0 rebuild mutates the derived indexes, brainy hard-link-snapshots the + * brain directory (default-on, opt-out via `migrationBackup: false`), removes it + * once the upgrade verifies + stamps, and retains it on failure for rollback. + * + * Covers: the FileSystemStorage machinery (hard-link snapshot, reuse, remove + * without touching live data, empty→null); the brainy lifecycle on a stale-epoch + * reopen (backup created then removed on success); the opt-out; and the memory + * feature-detect (no machinery → graceful no-op). + * + * Staleness is forced by stamping a marker with a mismatched `indexEpoch` (what a + * pre-8.0 brain looks like), via the same `writeRawObject` surface the marker is + * stored through — not by deleting an on-disk file (the marker path is encoded). + */ +import { describe, it, expect, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js' +import { NounType } from '../../src/types/graphTypes.js' + +const tmpDirs: string[] = [] +function mkTmp(): string { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-mig-backup-')) + tmpDirs.push(d) + return d +} +afterEach(() => { + for (const d of tmpDirs.splice(0)) { + fs.rmSync(d, { recursive: true, force: true }) + fs.rmSync(`${d}.migration-backup`, { recursive: true, force: true }) + } +}) + +/** Build a persisted brain with one entity. `staleAfter` overwrites the marker + * with a mismatched epoch so the NEXT open detects a 7.x→8.0 upgrade. */ +async function buildBrainWithData(dir: string, staleAfter = false): Promise { + const brain: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true + }) + await brain.init() + const id = await brain.add({ data: 'protect me during the upgrade', type: NounType.Concept }) + await brain.flush() + if (staleAfter) { + await brain.storage.writeRawObject('_system/brain-format.json', { dataFormat: '7.0', indexEpoch: 0 }) + } + await brain.close() + return id +} + +describe('Pre-migration backup (#42)', () => { + it('byte-copies _id_mapper/* (mmap, mutated in place) but hard-links the rest', async () => { + // cor confirmed the native _id_mapper mmap is msync/truncate-in-place, so a + // hard-linked snapshot of it would be corrupted by a live mutation reaching + // through the shared inode — it must be byte-copied. Everything else (written + // tmp+rename) hard-links safely. + const dir = mkTmp() + await buildBrainWithData(dir) + fs.mkdirSync(path.join(dir, '_id_mapper'), { recursive: true }) + fs.writeFileSync(path.join(dir, '_id_mapper', 'map.bin'), Buffer.from([1, 2, 3, 4])) + fs.writeFileSync(path.join(dir, 'regular.bin'), Buffer.from([9, 9, 9, 9])) + + const storage: any = new FileSystemStorage(dir) + await storage.init() + const backup = await storage.createMigrationBackup() + + // _id_mapper file → byte-copied: DIFFERENT inode (a live msync can't reach it) + faithful content + const mapperLive = fs.statSync(path.join(dir, '_id_mapper', 'map.bin')).ino + const mapperBackup = fs.statSync(path.join(backup, '_id_mapper', 'map.bin')).ino + expect(mapperBackup).not.toBe(mapperLive) + expect(fs.readFileSync(path.join(backup, '_id_mapper', 'map.bin'))).toEqual(Buffer.from([1, 2, 3, 4])) + + // a regular (tmp+rename) file → hard-linked: SAME inode (zero-copy) + const regLive = fs.statSync(path.join(dir, 'regular.bin')).ino + const regBackup = fs.statSync(path.join(backup, 'regular.bin')).ino + expect(regBackup).toBe(regLive) + }) + + it('FileSystemStorage.createMigrationBackup hard-links the store; removeMigrationBackup deletes it without touching live data', async () => { + const dir = mkTmp() + const id = await buildBrainWithData(dir) + + const storage: any = new FileSystemStorage(dir) + await storage.init() + + const backupPath = await storage.createMigrationBackup() + expect(backupPath).toBe(`${dir}.migration-backup`) + expect(fs.existsSync(backupPath)).toBe(true) + expect(fs.readdirSync(backupPath).length).toBeGreaterThan(0) // snapshot has content + + // Reuse: a second call returns the same path (a prior failed upgrade's + // pre-state is preserved, not overwritten) and does not throw. + expect(await storage.createMigrationBackup()).toBe(backupPath) + + // Remove: gone from disk, and the LIVE store is untouched (shared inodes). + await storage.removeMigrationBackup(backupPath) + expect(fs.existsSync(backupPath)).toBe(false) + + const reopened: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true + }) + await reopened.init() + expect(await reopened.get(id)).not.toBeNull() // live data survived the backup+remove + await reopened.close() + }) + + it('returns null for an empty store — nothing to protect', async () => { + const dir = mkTmp() + const storage: any = new FileSystemStorage(dir) // raw store, no brain → no VFS root + await storage.init() + expect(await storage.createMigrationBackup()).toBeNull() + }) + + it('takes a backup on a stale-epoch reopen (default-on) and removes it once the upgrade verifies', async () => { + const dir = mkTmp() + await buildBrainWithData(dir, /* staleAfter */ true) + + const created: string[] = [] + const removed: string[] = [] + const proto = FileSystemStorage.prototype as any + const origCreate = proto.createMigrationBackup + const origRemove = proto.removeMigrationBackup + proto.createMigrationBackup = async function () { + const p = await origCreate.call(this) + if (p) created.push(p) + return p + } + proto.removeMigrationBackup = async function (p: string) { + removed.push(p) + return origRemove.call(this, p) + } + try { + const brain: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true + }) + await brain.init() // epochStale → backup created; inline rebuild stamps → backup removed + await brain.close() + + expect(created.length).toBe(1) // a backup WAS taken before the rebuild + expect(removed).toContain(created[0]) // and removed once the upgrade stamped + expect(fs.existsSync(created[0])).toBe(false) // gone from disk + } finally { + proto.createMigrationBackup = origCreate + proto.removeMigrationBackup = origRemove + } + }) + + it('opt-out (migrationBackup:false) takes no backup; memory storage is a graceful no-op', async () => { + const dir = mkTmp() + await buildBrainWithData(dir, /* staleAfter */ true) + + const proto = FileSystemStorage.prototype as any + const origCreate = proto.createMigrationBackup + let calls = 0 + proto.createMigrationBackup = async function () { + calls++ + return origCreate.call(this) + } + try { + const brain: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + migrationBackup: false, + silent: true + }) + await brain.init() + await brain.close() + expect(calls).toBe(0) + } finally { + proto.createMigrationBackup = origCreate + } + + // Memory storage has no backup machinery — default-on must not crash. + const mem: any = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await mem.init() + expect(mem.isInitialized).toBe(true) + expect(mem.config.migrationBackup).toBe(true) // default resolved + await mem.close() + }) +}) diff --git a/tests/integration/multi-process-safety.test.ts b/tests/integration/multi-process-safety.test.ts index 6ac5265d..592d7969 100644 --- a/tests/integration/multi-process-safety.test.ts +++ b/tests/integration/multi-process-safety.test.ts @@ -110,6 +110,89 @@ describe('Multi-process safety + read-only mode', () => { // Don't track `blocked` for afterEach cleanup since init failed. }) + it('takes over a STALE foreign lock (dead PID + old heartbeat) and claims atomically', async () => { + const { mkdirSync, writeFileSync, readFileSync } = await import('node:fs') + const { join } = await import('node:path') + const os = await import('node:os') + mkdirSync(join(dir, 'locks'), { recursive: true }) + const tenMinutesAgo = new Date(Date.now() - 10 * 60 * 1000).toISOString() + writeFileSync(join(dir, 'locks', '_writer.lock'), JSON.stringify({ + pid: 999999999, // no such process — provably dead + hostname: os.hostname(), + startedAt: tenMinutesAgo, + lastHeartbeat: tenMinutesAgo, + version: '8.0.0', + rootDir: dir + })) + + writer = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await writer.init() // stale takeover must succeed + + const lock = JSON.parse(readFileSync(join(dir, 'locks', '_writer.lock'), 'utf-8')) + expect(lock.pid).toBe(process.pid) // the atomic wx claim installed OUR lock + }) + + it('the writer-locked error carries the machine-readable contract (code + lockInfo)', async () => { + const { mkdirSync, writeFileSync } = await import('node:fs') + const { join } = await import('node:path') + const os = await import('node:os') + mkdirSync(join(dir, 'locks'), { recursive: true }) + const otherPid = (process as any).ppid || 1 + writeFileSync(join(dir, 'locks', '_writer.lock'), JSON.stringify({ + pid: otherPid, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + lastHeartbeat: new Date().toISOString(), + version: '8.7.0', + rootDir: dir + })) + + const blocked = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + const err: any = await blocked.init().catch((e) => e) + expect(err.code).toBe('BRAINY_WRITER_LOCKED') + expect(err.lockInfo?.pid).toBe(otherPid) + }) + + it('release drains an in-flight heartbeat — no phantom lock re-created after unlink', async () => { + // The race (8.9.0): clearInterval stops FUTURE heartbeat ticks, but a + // tick already in flight could land its lock rewrite AFTER release's + // unlink — re-creating the lock as a phantom that blocks the next + // writer until the stale TTL. Simulate the in-flight tick explicitly + // and prove release waits for it. + writer = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await writer.init() + const storage: any = (writer as any).storage + + // An in-flight refresh that is ALREADY PAST its ownership guards + // (captured the lock info before release ran) and lands its atomic + // rewrite slowly — the exact straggler shape; absent the drain it + // writes after the unlink. + const { join: joinPath } = await import('node:path') + const capturedInfo = { ...storage.writerLockInfo } + const lockPath = joinPath(dir, 'locks', '_writer.lock') + const slowTick = (async () => { + await new Promise((r) => setTimeout(r, 100)) + await storage.writeFileAtomic( + lockPath, + JSON.stringify({ ...capturedInfo, lastHeartbeat: new Date().toISOString() }) + ) + })() + storage.writerHeartbeatInFlight = slowTick.catch(() => {}) + + await writer.close() // → releaseWriterLock must drain slowTick first + await slowTick.catch(() => {}) // both paths fully settled either way + writer = null + + const { existsSync } = await import('node:fs') + const { join } = await import('node:path') + expect(existsSync(join(dir, 'locks', '_writer.lock'))).toBe(false) + + // And the directory is immediately claimable — no stale-TTL wait. + const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await expect(next.init()).resolves.toBeUndefined() + await next.close() + }) + it('allows a second in-process writer with a warning (same PID)', async () => { // Two Brainy instances in the same Node process: not the dangerous // cross-process case. Should succeed (with a console warning). diff --git a/tests/integration/onchange-feed.test.ts b/tests/integration/onchange-feed.test.ts new file mode 100644 index 00000000..8470b59c --- /dev/null +++ b/tests/integration/onchange-feed.test.ts @@ -0,0 +1,228 @@ +/** + * @module tests/integration/onchange-feed + * @description The `brain.onChange` in-process change feed — the authoritative + * post-commit signal for every canonical mutation, regardless of origin. + * Pins the delivery contract: + * - every operation fires exactly once per affected record with the + * post-commit payload (deletes carry the record's LAST state); + * - batch operations emit one event per item; transact items share one + * generation; + * - VFS writes (a Tier-1 blind spot for router-synthesized events) emit; + * - events arrive in commit order with monotonic generations; + * - a losing `ifRev` CAS emits NOTHING (aborted commits are invisible); + * - a throwing listener is isolated; unsubscribe stops delivery; + * - clear()/restore() emit one store-level "refetch everything" event. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/brainy.js' +import type { BrainyChangeEvent } from '../../src/events/changeFeed.js' +import { NounType, VerbType } from '../../src/types/graphTypes.js' + +let seq = 0 +const freshId = (): string => + `00000000-0000-4000-8000-${(++seq).toString(16).padStart(12, '0')}` + +/** Let the microtask-dispatched feed drain. */ +const drained = (): Promise => new Promise((r) => setTimeout(r, 0)) + +describe('brain.onChange — the in-process change feed', () => { + let dir: string + let brain: any + let events: BrainyChangeEvent[] + let off: () => void + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-onchange-')) + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + dimensions: 384, + silent: true + }) + await brain.init() + events = [] + off = brain.onChange((e: BrainyChangeEvent) => events.push(e)) + }) + + afterEach(async () => { + off() + await brain.close() + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('add → update → remove: one fully-described event each; remove carries the last state', async () => { + const id = await brain.add({ + id: freshId(), + data: 'a document', + type: NounType.Document, + metadata: { status: 'draft' } + }) + await brain.update({ id, metadata: { status: 'published' } }) + await brain.remove(id) + await drained() + + expect(events.map((e) => e.op)).toEqual(['add', 'update', 'remove']) + expect(events.every((e) => e.kind === 'entity' && e.id === id)).toBe(true) + + expect(events[0].entity).toMatchObject({ id, type: 'document', metadata: { status: 'draft' } }) + expect(events[1].entity).toMatchObject({ id, metadata: { status: 'published' } }) + // The delete payload is the record's LAST committed state. + expect(events[2].entity).toMatchObject({ id, type: 'document', metadata: { status: 'published' } }) + + // Post-commit ordering: generations are monotonic. + const gens = events.map((e) => e.generation!) + expect([...gens].sort((a, b) => a - b)).toEqual(gens) + expect(new Set(gens).size).toBe(3) + }) + + it('relate → updateRelation → unrelate: relation events with endpoints + type', async () => { + const a = await brain.add({ id: freshId(), data: 'a', type: NounType.Thing }) + const b = await brain.add({ id: freshId(), data: 'b', type: NounType.Thing }) + events.length = 0 + + const rel = await brain.relate({ from: a, to: b, type: VerbType.RelatedTo, metadata: { w: 1 } }) + await brain.updateRelation({ id: rel, metadata: { w: 2 } }) + await brain.unrelate(rel) + await drained() + + expect(events.map((e) => e.op)).toEqual(['relate', 'updateRelation', 'unrelate']) + expect(events.every((e) => e.kind === 'relation' && e.id === rel)).toBe(true) + for (const e of events) { + expect(e.relation).toMatchObject({ id: rel, from: a, to: b }) + } + }) + + it('remove() cascades: deleting an entity emits unrelate for its relationships too', async () => { + const a = await brain.add({ id: freshId(), data: 'src', type: NounType.Thing }) + const b = await brain.add({ id: freshId(), data: 'tgt', type: NounType.Thing }) + const rel = await brain.relate({ from: a, to: b, type: VerbType.Contains }) + events.length = 0 + + await brain.remove(a) + await drained() + + const removeEvent = events.find((e) => e.op === 'remove') + const unrelateEvent = events.find((e) => e.op === 'unrelate') + expect(removeEvent).toMatchObject({ kind: 'entity', id: a }) + expect(unrelateEvent).toMatchObject({ + kind: 'relation', + id: rel, + relation: { from: a, to: b } + }) + // Entity + cascaded verb share the one remove() generation. + expect(removeEvent!.generation).toBe(unrelateEvent!.generation) + }) + + it('batches emit one event per item; removeMany deletes carry payloads', async () => { + const ids = [freshId(), freshId(), freshId()] + await brain.addMany({ + items: ids.map((id, i) => ({ + id, + data: `item ${i}`, + type: NounType.Thing, + metadata: { n: i } + })) + }) + await drained() + expect(events.filter((e) => e.op === 'add').length).toBe(3) + + events.length = 0 + await brain.removeMany({ ids }) + await drained() + const removes = events.filter((e) => e.op === 'remove') + expect(removes.length).toBe(3) + // Every delete is fully described (enriched, not id-only). + for (const e of removes) { + expect(e.entity).toBeDefined() + expect(e.entity!.type).toBe('thing') + expect(typeof e.entity!.metadata.n).toBe('number') + } + }) + + it('transact(): per-item events sharing ONE generation; a rejected batch emits nothing', async () => { + const x = freshId() + const y = freshId() + await brain.transact([ + { op: 'add', id: x, data: 'x', type: NounType.Thing, metadata: { k: 1 } }, + { op: 'add', id: y, data: 'y', type: NounType.Thing }, + { op: 'relate', from: x, to: y, type: VerbType.RelatedTo } + ]) + await drained() + + expect(events.map((e) => e.op)).toEqual(['add', 'add', 'relate']) + expect(new Set(events.map((e) => e.generation)).size).toBe(1) + + // Rejected batch (stale per-op CAS) → zero events. + events.length = 0 + await expect( + brain.transact([{ op: 'update', id: x, metadata: { k: 2 }, ifRev: 999 }]) + ).rejects.toMatchObject({ name: 'RevisionConflictError' }) + await drained() + expect(events.length).toBe(0) + }) + + it('a losing ifRev CAS update emits NOTHING; the winner emits once', async () => { + const id = await brain.add({ id: freshId(), data: 'contended', type: NounType.Thing }) + const rev = (await brain.get(id))._rev + events.length = 0 + + const results = await Promise.allSettled( + Array.from({ length: 4 }, (_, i) => + brain.update({ id, metadata: { w: i }, merge: false, ifRev: rev }) + ) + ) + await drained() + + expect(results.filter((r) => r.status === 'fulfilled').length).toBe(1) + expect(events.filter((e) => e.op === 'update').length).toBe(1) // exactly the winner + }) + + it('VFS writes emit (the router-synthesis blind spot)', async () => { + await brain.vfs.writeFile('/notes/hello.txt', 'hello feed') + await drained() + // A VFS write is entities + containment relations under the hood — the + // feed sees it because VFS delegates to canonical brain methods. + expect(events.length).toBeGreaterThan(0) + expect(events.some((e) => e.kind === 'entity' && e.op === 'add')).toBe(true) + }) + + it('listener errors are isolated; unsubscribe stops delivery', async () => { + const good: BrainyChangeEvent[] = [] + const offBad = brain.onChange(() => { + throw new Error('subscriber bug') + }) + const offGood = brain.onChange((e: BrainyChangeEvent) => good.push(e)) + + const id = await brain.add({ id: freshId(), data: 'p', type: NounType.Thing }) + await drained() + expect(good.length).toBeGreaterThan(0) // sibling unaffected by the throwing listener + + offBad() + offGood() + good.length = 0 + events.length = 0 + off() // unsubscribe the outer listener too + await brain.update({ id, metadata: { after: true } }) + await drained() + expect(events.length).toBe(0) + expect(good.length).toBe(0) + + // Re-subscribe for afterEach symmetry. + off = brain.onChange((e: BrainyChangeEvent) => events.push(e)) + }) + + it('clear() emits one store-level event meaning "refetch everything"', async () => { + await brain.add({ id: freshId(), data: 'doomed', type: NounType.Thing }) + events.length = 0 + await brain.clear() + await drained() + const store = events.filter((e) => e.kind === 'store') + expect(store).toHaveLength(1) + expect(store[0].op).toBe('clear') + expect(store[0].generation).toBeUndefined() + }) +}) diff --git a/tests/integration/read-your-writes-contract.test.ts b/tests/integration/read-your-writes-contract.test.ts new file mode 100644 index 00000000..aa0c93e5 --- /dev/null +++ b/tests/integration/read-your-writes-contract.test.ts @@ -0,0 +1,133 @@ +/** + * @module tests/integration/read-your-writes-contract + * @description The read-your-writes contract — brainy's spine guarantee that a + * write which has RETURNED is immediately readable, under the single writer, by + * BOTH id and every index (metadata filter, vector search, graph traversal), + * with NO await, delay, or retry between the write returning and the read. + * + * Why it holds by construction on the canonical + JS-index path: every mutation + * runs its canonical writes AND all derived-index projections (HNSW, metadata + * index, graph index) as operations inside ONE transaction that commits before + * the method returns — there is no post-return "eventual" indexing window. get() + * additionally rides the storage write-through cache (read-after-write within + * the process). The generation counter is the {seq} a caller can pin: a write + * returns generation N and every subsequent live read observes ≥ N. + * + * This pins the guarantee so no future change can quietly move index maintenance + * off the commit path (which would turn a returned write into a not-yet-queryable + * one — the "a 200 is still not durability/queryability" failure the spine + * program exists to make impossible). The native accelerator must honor the same + * in-commit contract; a background consolidation/optimization must never gate + * queryability of an acknowledged write. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType, VerbType } from '../../src/types/graphTypes.js' + +let seq = 0 +const freshId = (): string => + `00000000-0000-4000-8000-${(++seq).toString(16).padStart(12, '0')}` + +describe('read-your-writes contract (single-writer, no await between write and read)', () => { + let dir: string + let brain: any + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-ryw-')) + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + dimensions: 384, + silent: true + }) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('add(): the entity is immediately readable by id, metadata filter, AND vector search', async () => { + const id = freshId() + const marker = `ryw-${id}` + await brain.add({ id, data: 'read your writes probe alpha', type: NounType.Concept, metadata: { marker } }) + + // id read (write-through cache / canonical) + expect(await brain.get(id)).not.toBeNull() + // metadata-filter read (metadata index committed in-transaction) + const byMeta = await brain.find({ where: { marker }, limit: 10 }) + expect(byMeta.map((r: any) => r.id)).toContain(id) + // vector read (HNSW committed in-transaction) + const byVector = await brain.find({ query: 'read your writes probe alpha', limit: 10 }) + expect(byVector.map((r: any) => r.id)).toContain(id) + // type-filter read + const byType = await brain.find({ type: NounType.Concept, limit: 100 }) + expect(byType.map((r: any) => r.id)).toContain(id) + }) + + it('update(): the new metadata is immediately queryable and the old value is immediately gone', async () => { + const id = freshId() + await brain.add({ id, data: 'updatable', type: NounType.Thing, metadata: { phase: 'before' } }) + await brain.update({ id, metadata: { phase: 'after' } }) + + const after = await brain.find({ where: { phase: 'after' }, limit: 10 }) + expect(after.map((r: any) => r.id)).toContain(id) + const before = await brain.find({ where: { phase: 'before' }, limit: 10 }) + expect(before.map((r: any) => r.id)).not.toContain(id) + }) + + it('relate(): the edge is immediately traversable', async () => { + const a = await brain.add({ id: freshId(), data: 'A', type: NounType.Thing }) + const b = await brain.add({ id: freshId(), data: 'B', type: NounType.Thing }) + await brain.relate({ from: a, to: b, type: VerbType.RelatedTo }) + expect((await brain.related(a)).map((r: any) => r.to)).toContain(b) + }) + + it('remove(): the entity is immediately gone from id, metadata, and vector reads', async () => { + const id = freshId() + const marker = `gone-${id}` + await brain.add({ id, data: 'to be removed promptly', type: NounType.Thing, metadata: { marker } }) + expect(await brain.get(id)).not.toBeNull() + + await brain.remove(id) + expect(await brain.get(id)).toBeNull() + expect((await brain.find({ where: { marker }, limit: 10 })).map((r: any) => r.id)).not.toContain(id) + expect((await brain.find({ query: 'to be removed promptly', limit: 10 })).map((r: any) => r.id)).not.toContain(id) + }) + + it('transact(): every item of a committed batch is immediately readable by id, index, and traversal', async () => { + const a = freshId() + const b = freshId() + const marker = `batch-${a}` + const db = await brain.transact([ + { op: 'add', id: a, data: 'batch node A', type: NounType.Thing, metadata: { marker } }, + { op: 'add', id: b, data: 'batch node B', type: NounType.Thing, metadata: { marker } }, + { op: 'relate', from: a, to: b, type: VerbType.Contains } + ]) + expect(db.generation).toBeGreaterThan(0) + + expect(await brain.get(a)).not.toBeNull() + expect(await brain.get(b)).not.toBeNull() + const byMeta = (await brain.find({ where: { marker }, limit: 10 })).map((r: any) => r.id) + expect(byMeta).toContain(a) + expect(byMeta).toContain(b) + expect((await brain.related(a)).map((r: any) => r.to)).toContain(b) + }) + + it('the write returns a generation ({seq}) and a live read observes it — the awaitable-read primitive', async () => { + const id = freshId() + const before = brain.generationStore.generation() + await brain.add({ id, data: 'generation stamped', type: NounType.Thing }) + const after = brain.generationStore.generation() + // The write advanced the generation… + expect(after).toBeGreaterThan(before) + // …and a read pinned to that generation observes the write (asOf ≥ N sees it). + const pinned = await brain.asOf(after) + expect(await pinned.get(id)).not.toBeNull() + }) +}) diff --git a/tests/integration/readdir-no-duplicate-replace.test.ts b/tests/integration/readdir-no-duplicate-replace.test.ts new file mode 100644 index 00000000..8043bd40 --- /dev/null +++ b/tests/integration/readdir-no-duplicate-replace.test.ts @@ -0,0 +1,56 @@ +/** + * @module tests/integration/readdir-no-duplicate-replace + * @description A re-created path must NOT show up twice in readdir. The reported + * defect (memory-vm) was readdir serving DUPLICATE entries for re-created paths, + * rooted in a delete that left a ghost (partial removal) whose next delete read + * null metadata and skipped unposting the stale Contains edge. With full-removal + * deletes there is no ghost, so the edge is always unposted and the path appears + * exactly once after any number of delete→recreate cycles. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/index.js' + +describe('readdir shows a re-created path exactly once (no duplicate on replace)', () => { + let dir: string + let brain: any + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-readdir-')) + brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 }) + await brain.init() + }) + afterEach(async () => { + await brain.close?.().catch(() => {}) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('delete then re-create the same file path — readdir lists it once', async () => { + await brain.vfs.mkdir('/d', { recursive: true }) + await brain.vfs.writeFile('/d/x.txt', 'v1') + expect(await brain.vfs.readdir('/d')).toEqual(['x.txt']) + + await brain.vfs.unlink('/d/x.txt') + expect(await brain.vfs.readdir('/d')).toEqual([]) + + await brain.vfs.writeFile('/d/x.txt', 'v2') + const listing = (await brain.vfs.readdir('/d')) as string[] + expect(listing).toEqual(['x.txt']) // exactly once — no ghost duplicate + expect(listing.filter((n) => n === 'x.txt')).toHaveLength(1) + }) + + it('survives several delete→recreate cycles without accumulating duplicates', async () => { + await brain.vfs.mkdir('/c', { recursive: true }) + for (let i = 0; i < 4; i++) { + await brain.vfs.writeFile('/c/f.txt', `gen ${i}`) + await brain.vfs.unlink('/c/f.txt') + } + await brain.vfs.writeFile('/c/f.txt', 'final') + const listing = (await brain.vfs.readdir('/c')) as string[] + expect(listing).toEqual(['f.txt']) + expect(await brain.vfs.readFile('/c/f.txt')).toBeDefined() + }) +}) diff --git a/tests/integration/remaining-apis.test.ts b/tests/integration/remaining-apis.test.ts index 5514bf17..12d60983 100644 --- a/tests/integration/remaining-apis.test.ts +++ b/tests/integration/remaining-apis.test.ts @@ -61,7 +61,9 @@ describe('Remaining APIs Comprehensive Test', () => { await brain.updateMany({ items: ids.successful.map(id => ({ id, - metadata: { status: 'published', updatedAt: Date.now() } + // `updatedAt` is system-managed (set automatically on every write) — + // only the custom `status` field belongs in the metadata bag. + metadata: { status: 'published' } })) }) diff --git a/tests/integration/restore-nondestructive.test.ts b/tests/integration/restore-nondestructive.test.ts new file mode 100644 index 00000000..b36d2620 --- /dev/null +++ b/tests/integration/restore-nondestructive.test.ts @@ -0,0 +1,168 @@ +/** + * @module tests/integration/restore-nondestructive + * @description Regression for a consumer-reported recovery hazard: `restore()` + * removed the entire live brain directory and THEN `fs.cp`'d the snapshot in, so + * a copy failure (most dangerously ENOSPC — `fs.cp` also materializes the holes + * of sparse mmap blobs, ballooning a snapshot that would otherwise fit) left the + * store destroyed with only a partial copy: the recovery tool could destroy the + * brain it was meant to recover. + * + * Fix: the snapshot is copied into a staging area (sparse-aware) BEFORE any live + * data is touched; only after the copy succeeds and a marker is fsync'd does an + * atomic per-entry swap move it into place. A copy failure leaves the live store + * exactly as it was; a crash mid-swap is resumed forward on the next open. + * + * These tests exercise the property directly: a forced copy failure leaves live + * data intact, a normal restore round-trips correctly, an interrupted-but- + * committed staging area is completed on reopen, and the sparse copy preserves + * holes byte-for-byte. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +const STAGING = '_restore_staging' +const MARKER = '.restore-manifest.json' + +describe('non-destructive restore (BRAINY-RESTORE-DESTRUCTIVE)', () => { + let root: string + let snap: string + let brain: any + + const openBrain = async (dir: string) => { + const b = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + dimensions: 384, + silent: true + }) + await b.init() + return b + } + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-restore-')) + root = path.join(base, 'store') + snap = path.join(base, 'snapshot') + brain = await openBrain(root) + }) + + afterEach(async () => { + try { await brain.close() } catch { /* already closed in a test */ } + fs.rmSync(path.dirname(root), { recursive: true, force: true }) + }) + + it('a copy failure leaves the LIVE store completely intact (the core property)', async () => { + const keep = await brain.add({ id: '00000000-0000-4000-8000-0000000000c1', data: 'in snapshot', type: NounType.Thing }) + await brain.flush() + await brain['storage'].snapshotToDirectory(snap) + // Mutate live AFTER the snapshot — this state must survive a failed restore. + const extra = await brain.add({ id: '00000000-0000-4000-8000-0000000000c2', data: 'live only', type: NounType.Thing }) + + // Force the staging copy to fail on its second entry (partial staging). + const storage = brain['storage'] + const realCopy = storage.copyTreeSparse.bind(storage) + let calls = 0 + storage.copyTreeSparse = async (src: string, dest: string) => { + if (++calls === 2) throw new Error('simulated ENOSPC') + return realCopy(src, dest) + } + + await expect(brain.restore(snap, { confirm: true })).rejects.toThrow(/untouched/) + + // Live data — BOTH the snapshot entity and the live-only mutation — survives. + expect(await brain.get(keep)).not.toBeNull() + expect(await brain.get(extra)).not.toBeNull() + // The half-written staging area was cleaned up. + expect(fs.existsSync(path.join(root, STAGING))).toBe(false) + }) + + it('a normal restore round-trips to the snapshot state', async () => { + const a = await brain.add({ id: '00000000-0000-4000-8000-0000000000c3', data: 'A', type: NounType.Thing }) + await brain.flush() + await brain['storage'].snapshotToDirectory(snap) + // Diverge from the snapshot: add B, remove A. + const b = await brain.add({ id: '00000000-0000-4000-8000-0000000000c4', data: 'B', type: NounType.Thing }) + await brain.remove(a) + expect(await brain.get(a)).toBeNull() + expect(await brain.get(b)).not.toBeNull() + + await brain.restore(snap, { confirm: true }) + + // Back to exactly the snapshot: A present, B gone. + expect(await brain.get(a)).not.toBeNull() + expect(await brain.get(b)).toBeNull() + expect(fs.existsSync(path.join(root, STAGING))).toBe(false) + }) + + it('an interrupted-but-committed restore is completed on the next open (resume forward)', async () => { + const a = await brain.add({ id: '00000000-0000-4000-8000-0000000000c5', data: 'A', type: NounType.Thing }) + await brain.flush() + await brain['storage'].snapshotToDirectory(snap) + const b = await brain.add({ id: '00000000-0000-4000-8000-0000000000c6', data: 'B', type: NounType.Thing }) + await brain.flush() + await brain.close() + + // Simulate the state right after the staging copy committed but before the + // swap ran: a _restore_staging holding the snapshot's entries + the marker. + const staging = path.join(root, STAGING) + fs.mkdirSync(staging, { recursive: true }) + const entries: string[] = [] + for (const entry of fs.readdirSync(snap)) { + if (entry === 'locks' || entry === STAGING) continue + fs.cpSync(path.join(snap, entry), path.join(staging, entry), { recursive: true }) + entries.push(entry) + } + fs.writeFileSync(path.join(staging, MARKER), JSON.stringify({ entries })) + + // Reopen → init resumes the swap → the store is the snapshot (A, no B). + brain = await openBrain(root) + expect(await brain.get(a)).not.toBeNull() + expect(await brain.get(b)).toBeNull() + expect(fs.existsSync(staging)).toBe(false) + }) + + it('an uncommitted staging area (no marker) is discarded on open, live untouched', async () => { + const a = await brain.add({ id: '00000000-0000-4000-8000-0000000000c7', data: 'A live', type: NounType.Thing }) + await brain.flush() + await brain.close() + + // A staging dir with NO marker = a copy that never committed → debris. + const staging = path.join(root, STAGING) + fs.mkdirSync(staging, { recursive: true }) + fs.writeFileSync(path.join(staging, 'entities'), 'garbage partial copy') + + brain = await openBrain(root) + expect(await brain.get(a)).not.toBeNull() // live authoritative + expect(fs.existsSync(staging)).toBe(false) // debris removed + }) + + it('the sparse copy preserves holes: content byte-identical, allocation far below apparent size', async () => { + const base = path.dirname(root) + const src = path.join(base, 'sparse-src.bin') + const dest = path.join(base, 'sparse-dest.bin') + const SIZE = 32 * 1024 * 1024 // 32 MiB apparent + const DATA = Buffer.from('hello sparse world') + + // Create a sparse source: 32 MiB of holes with a little real data at the start. + const fh = fs.openSync(src, 'w') + fs.ftruncateSync(fh, SIZE) + fs.writeSync(fh, DATA, 0, DATA.length, 0) + fs.closeSync(fh) + + const storage = brain['storage'] + const stat = fs.statSync(src) + await storage.copyFileSparse(src, dest, stat.size, stat.mode) + + // Same apparent size and byte-identical content… + const destStat = fs.statSync(dest) + expect(destStat.size).toBe(SIZE) + expect(fs.readFileSync(dest).equals(fs.readFileSync(src))).toBe(true) + // …but the destination is actually sparse (allocated bytes far below size). + expect(destStat.blocks * 512).toBeLessThan(SIZE / 2) + }) +}) diff --git a/tests/integration/rollback-trapdoor.test.ts b/tests/integration/rollback-trapdoor.test.ts new file mode 100644 index 00000000..cf2245b8 --- /dev/null +++ b/tests/integration/rollback-trapdoor.test.ts @@ -0,0 +1,156 @@ +/** + * @module tests/integration/rollback-trapdoor + * @description Regression for a consumer-reported data-integrity bug: a failed + * CANONICAL rollback undo left the record durable while the request errored (a + * post-commit response lie). Reproduced by a failure-injection coherence harness: + * a late index op throws → rollback runs → an earlier op's canonical undo + * (storage.deleteNounMetadata) fails persistently → the record stays on disk + * while Transaction.rollback threw TransactionRollbackError AND lied with + * state='rolled_back'. + * + * Fix (David's ruling — adopt-forward when safe, else fail loud): + * - SINGLE-op add whose only damage is a durably-present orphan → adopt it + * forward: commit the generation (the record IS what add() wanted), return + * success + a loud warn; the derived index heals on rebuild/repairIndex. + * - MULTI-op batch, OR any restorative LOSS → StoreInconsistentError naming the + * unreconciled ids; the brain enters write-quarantine (reads OK, writes + * refused) until repairIndex() reconciles and lifts it. Transaction state is + * 'inconsistent', never the 'rolled_back' lie. + * + * The injection mirrors that harness: fault index.addToIndex (trigger rollback) and + * storage.deleteNounMetadata (fail the canonical undo). + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import { StoreInconsistentError } from '../../src/db/errors.js' + +let seq = 0 +const freshId = (): string => + `00000000-0000-4000-8000-${(++seq).toString(16).padStart(12, '0')}` + +describe('rollback trapdoor — failed canonical undo (data integrity)', () => { + let dir: string + let brain: any + let restore: Array<() => void> + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-trapdoor-')) + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + dimensions: 384, + silent: true + }) + await brain.init() + restore = [] + }) + + afterEach(async () => { + for (const r of restore) r() + try { await brain.close() } catch { /* ignore */ } + fs.rmSync(dir, { recursive: true, force: true }) + }) + + /** Fault the metadata-index add so a write's rollback is triggered. */ + const faultIndexAdd = () => { + const idx = brain['metadataIndex'] + const orig = idx.addToIndex.bind(idx) + idx.addToIndex = async () => { throw new Error('injected: index add failed') } + restore.push(() => { idx.addToIndex = orig }) + } + /** Fault the canonical additive undo (delete of a just-created record). */ + const faultCanonicalDelete = () => { + const storage = brain['storage'] + const orig = storage.deleteNounMetadata.bind(storage) + storage.deleteNounMetadata = async () => { throw new Error('injected: canonical undo failed') } + restore.push(() => { storage.deleteNounMetadata = orig }) + } + + it('ADOPT-FORWARD: a single-op add whose canonical undo fails commits as a durable, get-able record (no throw, no quarantine)', async () => { + const id = freshId() + const genBefore = brain.generationStore.generation() + + faultIndexAdd() // triggers rollback + faultCanonicalDelete() // the trapdoor: the metadata record can't be un-written + + // add() must NOT throw — the record it wanted is durable, so it is adopted. + const returned = await brain.add({ id, data: 'orphan adopted', type: NounType.Thing }) + expect(returned).toBe(id) + + // Un-fault so reads/subsequent writes use the real methods. + for (const r of restore) r() + restore = [] + + // The record is durable and get-able, and the generation advanced. + expect(await brain.get(id)).not.toBeNull() + expect(brain.generationStore.generation()).toBe(genBefore + 1) + + // The store is NOT quarantined — a subsequent write succeeds. + const ok = await brain.add({ id: freshId(), data: 'still writable', type: NounType.Thing }) + expect(await brain.get(ok)).not.toBeNull() + }) + + it('FAIL-LOUD: a multi-op transact whose canonical undo fails throws StoreInconsistentError and write-quarantines the brain', async () => { + // A pre-existing record to prove reads keep working under quarantine. + const anchor = await brain.add({ id: freshId(), data: 'anchor', type: NounType.Thing }) + await brain.flush() + + faultIndexAdd() + faultCanonicalDelete() + + const a = freshId() + const b = freshId() + await expect( + brain.transact([ + { op: 'add', id: a, data: 'batch A', type: NounType.Thing }, + { op: 'add', id: b, data: 'batch B', type: NounType.Thing } + ]) + ).rejects.toBeInstanceOf(StoreInconsistentError) + + // Writes are refused (quarantine); reads still work. + await expect( + brain.add({ id: freshId(), data: 'blocked', type: NounType.Thing }) + ).rejects.toThrow(/WRITE-QUARANTINED/) + expect(await brain.get(anchor)).not.toBeNull() + + // repairIndex() reconciles and lifts the quarantine → writes resume. + for (const r of restore) r() + restore = [] + await brain.repairIndex() + + const ok = await brain.add({ id: freshId(), data: 'writable again', type: NounType.Thing }) + expect(await brain.get(ok)).not.toBeNull() + }) + + it('the StoreInconsistentError names the unreconciled records with dispositions', async () => { + faultIndexAdd() + faultCanonicalDelete() + + const a = freshId() + const b = freshId() + let caught: unknown + await brain + .transact([ + { op: 'add', id: a, data: 'A', type: NounType.Thing }, + { op: 'add', id: b, data: 'B', type: NounType.Thing } + ]) + .catch((e: unknown) => (caught = e)) + + expect(caught).toBeInstanceOf(StoreInconsistentError) + const err = caught as StoreInconsistentError + expect(err.records.length).toBeGreaterThan(0) + // Every unreconciled record here is a durably-present orphan. + expect(err.records.every((r) => r.disposition === 'orphan')).toBe(true) + expect(err.message).toMatch(/WRITE-QUARANTINED/) + + // Recover so afterEach can close cleanly. + for (const r of restore) r() + restore = [] + await brain.repairIndex() + }) +}) diff --git a/tests/integration/temporal-vfs.test.ts b/tests/integration/temporal-vfs.test.ts new file mode 100644 index 00000000..77206ed8 --- /dev/null +++ b/tests/integration/temporal-vfs.test.ts @@ -0,0 +1,200 @@ +/** + * @module tests/integration/temporal-vfs + * @description VFS content joins the Model-B immutability model. Previously + * the temporal model had a hole exactly where files were concerned: entity + * records were versioned per write (before-images), but the content BYTES + * lived under an eager refCount GC — `rm` could physically delete bytes that + * in-window history still referenced, and overwrite never released the old + * hash at all (an unbounded silent leak that only accidentally preserved + * history). Now blob bytes are retention-protected: a content blob referenced + * by any generation inside the retention window (or live) survives, and the + * ONE reclamation point is history compaction. + * + * Pins: + * - `readFile(path, { asOf })` returns each version's exact bytes. + * - `history(path)` lists versions ascending with generation/hash/size. + * - overwrite releases the superseded live reference (leak fixed) while + * history protects the bytes; rm keeps bytes readable via asOf. + * - compaction past the last referencing generation physically reclaims + * bytes (zero live + zero history), and never reclaims in-window content — + * including the cross-file dedup case where an old file's history and a + * newer file's history share one hash. + * - overwrite refreshes the entity's `data` (embedding text) — the stale + * `.data` defect. + * - the backfill/scrub recounts exactly from the generation records. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/brainy.js' + +describe('temporal VFS — blob immutability under Model B', () => { + let dir: string + let brain: any + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-temporal-vfs-')) + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + dimensions: 384, + silent: true + }) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + fs.rmSync(dir, { recursive: true, force: true }) + }) + + /** The file's blob metadata via the storage layer (test introspection). */ + async function blobMeta(hash: string) { + return brain['storage'].blobStorage.getMetadata(hash) + } + + it('readFile(path, {asOf}) returns each version‘s exact bytes; history(path) lists them', async () => { + const p = '/pages/home.json' + await brain.vfs.writeFile(p, 'v1 — original') + const g1 = brain.generationStore.generation() + await brain.vfs.writeFile(p, 'v2 — edited') + await brain.vfs.writeFile(p, 'v3 — final') + + // Live read = newest. + expect((await brain.vfs.readFile(p)).toString()).toBe('v3 — final') + + // History lists the file's write-generations ascending, distinct hashes. + const versions = await brain.vfs.history(p) + expect(versions.length).toBeGreaterThanOrEqual(3) + const gens = versions.map((v: any) => v.generation) + expect([...gens].sort((a: number, b: number) => a - b)).toEqual(gens) + + // Every listed version's exact bytes are readable. + const texts: string[] = [] + for (const v of versions) { + texts.push((await brain.vfs.readFile(p, { asOf: v.generation })).toString()) + } + expect(texts).toContain('v1 — original') + expect(texts).toContain('v2 — edited') + expect(texts).toContain('v3 — final') + + // Direct generation read too (the incident shape: recover the past bytes). + expect((await brain.vfs.readFile(p, { asOf: g1 })).toString()).toBe('v1 — original') + }) + + it('overwrite releases the superseded LIVE reference (leak fixed) while history protects the bytes', async () => { + const p = '/notes/leak.txt' + await brain.vfs.writeFile(p, 'old content A') + const oldHash = (await brain.vfs.history(p))[0].hash + const g1 = brain.generationStore.generation() + + await brain.vfs.writeFile(p, 'new content B') + + // The old hash's LIVE reference is released (previously it leaked forever)… + expect((await blobMeta(oldHash))?.refCount).toBe(0) + // …but the bytes survive (history-protected) and asOf still reads them. + expect((await brain.vfs.readFile(p, { asOf: g1 })).toString()).toBe('old content A') + }) + + it('rm keeps the bytes readable via asOf inside the window (no premature deletion)', async () => { + const p = '/docs/doomed.txt' + await brain.vfs.writeFile(p, 'still recoverable after rm') + const g = brain.generationStore.generation() + const hash = (await brain.vfs.history(p))[0].hash + + await brain.vfs.unlink(p) + + // Live path is gone… + await expect(brain.vfs.readFile(p)).rejects.toThrow() + // …the bytes are not: zero live refs, history-protected. + expect((await blobMeta(hash))?.refCount).toBe(0) + expect((await brain['storage'].blobStorage.read(hash)).toString()).toBe( + 'still recoverable after rm' + ) + // (Path-level asOf resolution of a DELETED path is future work — the + // bytes + entity history are intact; recovery reads go through the hash + // or a pre-delete generation view.) + void g + }) + + it('compaction is the ONE reclamation point: past-window bytes are reclaimed, in-window preserved', async () => { + const p = '/pages/compact.json' + await brain.vfs.writeFile(p, 'version ONE') + const v1Hash = (await brain.vfs.history(p))[0].hash + await brain.vfs.writeFile(p, 'version TWO') + await brain.vfs.writeFile(p, 'version THREE') + + // Sanity: v1's bytes exist (history-protected, live-released). + expect(await blobMeta(v1Hash)).toBeDefined() + expect((await blobMeta(v1Hash))?.refCount).toBe(0) + + // Compact everything reclaimable (retain nothing beyond the live state). + await brain.compactHistory({ maxGenerations: 0 }) + + // v1's bytes are physically GONE — zero live, zero history. + expect(await blobMeta(v1Hash)).toBeUndefined() + // The live version is untouched. + expect((await brain.vfs.readFile(p)).toString()).toBe('version THREE') + }) + + it('cross-file dedup: a shared hash survives while ANY in-window generation references it', async () => { + const shared = 'identical bytes shared across files and time' + const pA = '/a.txt' + const pB = '/b.txt' + + // A holds the shared content, then moves off it (A's history references it). + await brain.vfs.writeFile(pA, shared) + await brain.vfs.writeFile(pA, 'A moved on') + // B now holds the shared content live, then is removed (B's remove-gen + // before-image references it too). + await brain.vfs.writeFile(pB, shared) + const gBLive = brain.generationStore.generation() + const sharedHash = (await brain.vfs.history(pA))[0].hash + await brain.vfs.unlink(pB) + + // Live refs are zero (A moved off; B removed) — bytes survive on history. + expect((await blobMeta(sharedHash))?.refCount).toBe(0) + expect((await brain['storage'].blobStorage.read(sharedHash)).toString()).toBe(shared) + + // B's content at its live generation is still exactly readable… via asOf + // on A's early version too (same bytes, same blob). + const versionsA = await brain.vfs.history(pA) + const aV1 = versionsA[0] + expect((await brain.vfs.readFile(pA, { asOf: aV1.generation })).toString()).toBe(shared) + void gBLive + + // Full compaction drops the last history references → bytes reclaimed. + await brain.compactHistory({ maxGenerations: 0 }) + expect(await blobMeta(sharedHash)).toBeUndefined() + }) + + it('overwrite refreshes the entity data/embedding text (the stale-.data defect)', async () => { + const p = '/pages/data-fresh.txt' + await brain.vfs.writeFile(p, 'first words') + await brain.vfs.writeFile(p, 'second words entirely') + + const entityId = await brain.vfs['pathResolver'].resolve(p) + const entity = await brain.get(entityId) + expect(entity.data).toBe('second words entirely') + }) + + it('the scrub recounts history references exactly from the generation records', async () => { + const p = '/pages/scrubbed.md' + await brain.vfs.writeFile(p, 'scrub v1') + await brain.vfs.writeFile(p, 'scrub v2') + const v1Hash = (await brain.vfs.history(p))[0].hash + await brain.flush() + + const storage = brain['storage'] + const before = (await blobMeta(v1Hash))?.historyRefCount ?? 0 + expect(before).toBeGreaterThan(0) + + // Corrupt the counter (simulates a legacy store / crash drift), then scrub. + await storage.blobStorage.setHistoryRefCount(v1Hash, 0) + const result = await storage.scrubBlobHistoryRefCounts() + expect(result.records).toBeGreaterThan(0) + expect((await blobMeta(v1Hash))?.historyRefCount).toBe(before) + }) +}) diff --git a/tests/integration/transact-durability-barrier.test.ts b/tests/integration/transact-durability-barrier.test.ts new file mode 100644 index 00000000..9311ce67 --- /dev/null +++ b/tests/integration/transact-durability-barrier.test.ts @@ -0,0 +1,151 @@ +/** + * @module tests/integration/transact-durability-barrier + * @description Regression for a consumer-reported durability gap: a committed + * `transact()` reported success while its canonical entity writes were still + * only in the page cache (tmp+rename, not yet fsync'd), whereas the generation + * counter/manifest WERE fsync'd — so a hard kill could leave the counter ahead + * of the persisted entity bytes ("phantom progress" for any generation-based + * consumer resuming from the counter). + * + * Fix: `commitTransaction` opens a write barrier before running the planned + * operations and flushes it (fsync of every canonical write + the parent dir of + * every canonical delete) BEFORE advancing the counter and writing the manifest. + * These tests assert the ordering property directly by recording the storage's + * `syncRawObjects` calls: the entity writes are fsync'd in an earlier call than + * the manifest, and a precommit-rejected batch opens no barrier and advances + * nothing. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType, VerbType } from '../../src/types/graphTypes.js' +import { MemoryStorage } from '../../src/storage/adapters/memoryStorage.js' + +const MANIFEST_REL = '_system/manifest.json' + +describe('transact durability barrier — entity writes fsync before the counter advances', () => { + let dir: string + let brain: any + let syncCalls: string[][] + let beginCount: number + let flushCount: number + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-durability-')) + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + dimensions: 384, + silent: true + }) + await brain.init() + + // Instrument the real filesystem storage: record every fsync batch in order, + // and count barrier open/flush, delegating to the originals. + syncCalls = [] + beginCount = 0 + flushCount = 0 + const storage = brain['storage'] + const origSync = storage.syncRawObjects.bind(storage) + storage.syncRawObjects = async (paths: string[]) => { + syncCalls.push([...paths]) + return origSync(paths) + } + const origBegin = storage.beginWriteBarrier.bind(storage) + storage.beginWriteBarrier = () => { + beginCount++ + return origBegin() + } + const origFlush = storage.flushWriteBarrier.bind(storage) + storage.flushWriteBarrier = async () => { + flushCount++ + return origFlush() + } + }) + + afterEach(async () => { + await brain.close() + fs.rmSync(dir, { recursive: true, force: true }) + }) + + /** Index of the first sync call whose paths satisfy `pred` (or -1). */ + const findSync = (pred: (p: string) => boolean): number => + syncCalls.findIndex((paths) => paths.some(pred)) + + it('a committed transact fsyncs the entity write BEFORE the manifest', async () => { + const id = '00000000-0000-4000-8000-00000000ab01' + await brain.transact([{ op: 'add', id, data: 'durable entity', type: NounType.Thing }]) + + // The barrier opened and flushed exactly once. + expect(beginCount).toBe(1) + expect(flushCount).toBe(1) + + // The entity's canonical metadata write was fsync'd… + const entitySyncIdx = findSync((p) => p.includes(`/${id}/`) && p.includes('entities/nouns')) + expect(entitySyncIdx).toBeGreaterThanOrEqual(0) + // …and the manifest (the commit point) was fsync'd in a LATER call. + const manifestSyncIdx = findSync((p) => p === MANIFEST_REL) + expect(manifestSyncIdx).toBeGreaterThanOrEqual(0) + expect(entitySyncIdx).toBeLessThan(manifestSyncIdx) + }) + + it('a multi-op transact (add + relate) fsyncs both endpoints and the edge before the manifest', async () => { + const a = '00000000-0000-4000-8000-00000000ab02' + const b = '00000000-0000-4000-8000-00000000ab03' + await brain.transact([ + { op: 'add', id: a, data: 'A', type: NounType.Thing }, + { op: 'add', id: b, data: 'B', type: NounType.Thing }, + { op: 'relate', from: a, to: b, type: VerbType.Contains } + ]) + + expect(flushCount).toBe(1) + const aIdx = findSync((p) => p.includes(`/${a}/`)) + const bIdx = findSync((p) => p.includes(`/${b}/`)) + const manifestIdx = findSync((p) => p === MANIFEST_REL) + expect(aIdx).toBeGreaterThanOrEqual(0) + expect(bIdx).toBeGreaterThanOrEqual(0) + expect(manifestIdx).toBeGreaterThanOrEqual(0) + expect(Math.max(aIdx, bIdx)).toBeLessThan(manifestIdx) + + // Sanity: the data is actually there and traverses. + expect((await brain.related(a)).map((r: any) => r.to)).toContain(b) + }) + + it('a precommit-rejected batch opens no barrier and advances nothing', async () => { + const anchor = await brain.add({ id: '00000000-0000-4000-8000-00000000ab04', data: 'anchor', type: NounType.Thing }) + const genBefore = brain.generationStore.generation() + syncCalls.length = 0 + beginCount = 0 + flushCount = 0 + + const never = '00000000-0000-4000-8000-00000000ab05' + await expect( + brain.transact([ + { op: 'add', id: never, data: 'never', type: NounType.Thing }, + { op: 'update', id: anchor, metadata: { poke: 1 }, ifRev: 999 } + ]) + ).rejects.toMatchObject({ name: 'RevisionConflictError' }) + + // Precommit throws before the barrier opens (it lives just before execute): + // the barrier never opens, nothing is flushed, and the generation counter is + // unchanged. (A manifest sync CAN appear here from transact() flushing the + // anchor's previously-buffered single-op — that is the anchor's deferred + // durability, not this rejected batch committing; beginCount is the clean + // signal that this batch's commit path never reached execute.) + expect(beginCount).toBe(0) + expect(flushCount).toBe(0) + expect(brain.generationStore.generation()).toBe(genBefore) + expect(await brain.get(never)).toBeNull() + }) + + it('MemoryStorage does not implement the barrier (optional-chaining no-op)', () => { + const mem = new MemoryStorage() as any + // In-memory has no durability seam; the generation store treats the absent + // barrier methods as no-ops via optional chaining. + expect(mem.beginWriteBarrier).toBeUndefined() + expect(mem.flushWriteBarrier).toBeUndefined() + }) +}) diff --git a/tests/integration/transact-forward-ref-graph.test.ts b/tests/integration/transact-forward-ref-graph.test.ts new file mode 100644 index 00000000..a9619424 --- /dev/null +++ b/tests/integration/transact-forward-ref-graph.test.ts @@ -0,0 +1,140 @@ +/** + * @module tests/integration/transact-forward-ref-graph + * @description Regression for a consumer-reported native/JS parity bug: + * `transact([{op:'add', id:X}, {op:'relate', to:X}])` — the platform's first + * atomic add+relate consumer — threw in `EntityIdMapper.getOrAssign` at PLAN + * time on the native id mapper (an entity added in the same batch does not + * exist when the planner runs, so a strict mapper rightly refuses to assign; + * the permissive JS mapper masked the bug — and silently leaked an int + * assignment whenever the batch was later rejected at precommit). + * + * Fix: graph-index operations resolve endpoint ints at EXECUTE time (a lazy + * thunk, mirroring the existing lazy `generationFn`), after the batch's add + * operations have applied. These tests pin every forward-ref shape on the JS + * path, including the JS-observable deferral proof: a REJECTED batch leaves + * the id mapper untouched (pre-fix, plan-time resolution assigned ints for + * entities that never came to exist). The native side is locked in by the + * accelerator's own gate running this same shape. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType, VerbType } from '../../src/types/graphTypes.js' + +let seq = 0 +const freshId = (): string => + `00000000-0000-4000-8000-${(++seq).toString(16).padStart(12, '0')}` + +describe('transact() forward references into the graph index (native/JS parity)', () => { + let dir: string + let brain: any + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-fwd-ref-')) + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + dimensions: 384, + silent: true + }) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('CASE 1 (the exact repro): add + relate-to-that-add in ONE transact', async () => { + const agent = await brain.add({ id: freshId(), data: 'agent probe', type: NounType.Person }) + const thread = freshId() + + const db = await brain.transact([ + { op: 'add', id: thread, data: 'thread PROBE-ONE', type: NounType.Thing }, + { op: 'relate', from: agent, to: thread, type: VerbType.RelatedTo } + ]) + + // The batch committed as one generation and the edge traverses. + expect(db.generation).toBeGreaterThan(0) + const related = await brain.related(agent) + expect(related.map((r: any) => r.to)).toContain(thread) + }) + + it('both endpoints created in-batch: add A + add B + relate A→B', async () => { + const a = freshId() + const b = freshId() + await brain.transact([ + { op: 'add', id: a, data: 'node A', type: NounType.Thing }, + { op: 'add', id: b, data: 'node B', type: NounType.Thing }, + { op: 'relate', from: a, to: b, type: VerbType.Contains } + ]) + expect((await brain.related(a)).map((r: any) => r.to)).toContain(b) + }) + + it('bidirectional relate to an in-batch add: both edges traverse', async () => { + const hub = await brain.add({ id: freshId(), data: 'hub', type: NounType.Thing }) + const spoke = freshId() + await brain.transact([ + { op: 'add', id: spoke, data: 'spoke', type: NounType.Thing }, + { op: 'relate', from: hub, to: spoke, type: VerbType.RelatedTo, bidirectional: true } + ]) + expect((await brain.related(hub)).map((r: any) => r.to)).toContain(spoke) + expect((await brain.related(spoke)).map((r: any) => r.to)).toContain(hub) + }) + + it('add + relate + remove in ONE transact: the cascade covers the in-batch verb', async () => { + const keeper = await brain.add({ id: freshId(), data: 'keeper', type: NounType.Thing }) + const doomed = freshId() + await brain.transact([ + { op: 'add', id: doomed, data: 'doomed', type: NounType.Thing }, + { op: 'relate', from: keeper, to: doomed, type: VerbType.Contains }, + { op: 'remove', id: doomed } + ]) + expect(await brain.get(doomed)).toBeNull() + expect((await brain.related(keeper)).length).toBe(0) + }) + + it('CASE 2 control: the same two ops split across two transacts still work', async () => { + const agent = await brain.add({ id: freshId(), data: 'agent two', type: NounType.Person }) + const thread = freshId() + await brain.transact([{ op: 'add', id: thread, data: 'thread two', type: NounType.Thing }]) + await brain.transact([{ op: 'relate', from: agent, to: thread, type: VerbType.RelatedTo }]) + expect((await brain.related(agent)).map((r: any) => r.to)).toContain(thread) + }) + + it('a REJECTED forward-ref batch leaves the id mapper untouched (the deferral proof)', async () => { + const existing = await brain.add({ + id: freshId(), + data: 'cas anchor', + type: NounType.Thing + }) + const never = freshId() + + // Reject the batch via a stale per-op CAS — planning completes, the + // commit precondition throws, nothing applies. + await expect( + brain.transact([ + { op: 'add', id: never, data: 'never exists', type: NounType.Thing }, + { op: 'relate', from: existing, to: never, type: VerbType.RelatedTo }, + { op: 'update', id: existing, metadata: { poke: 1 }, ifRev: 999 } + ]) + ).rejects.toMatchObject({ name: 'RevisionConflictError' }) + + // Nothing applied… + expect(await brain.get(never)).toBeNull() + // …and the id mapper was never asked to assign for the phantom entity — + // pre-fix, plan-time resolution leaked an int here on every rejected batch. + const idMapper = brain['metadataIndex'].getIdMapper() + expect(idMapper.getInt(never)).toBeUndefined() + + // The same shape then succeeds cleanly on retry. + await brain.transact([ + { op: 'add', id: never, data: 'exists now', type: NounType.Thing }, + { op: 'relate', from: existing, to: never, type: VerbType.RelatedTo } + ]) + expect((await brain.related(existing)).map((r: any) => r.to)).toContain(never) + }) +}) diff --git a/tests/integration/vfs-cow-blob-adoption.test.ts b/tests/integration/vfs-cow-blob-adoption.test.ts new file mode 100644 index 00000000..b7dec300 --- /dev/null +++ b/tests/integration/vfs-cow-blob-adoption.test.ts @@ -0,0 +1,149 @@ +/** + * @module tests/integration/vfs-cow-blob-adoption + * @description P0 recovery: a 7→8 upgrade left VFS content blobs in the removed + * branch system's copy-on-write area (`_cow/`) instead of the 8.0 + * content-addressed store (`_cas/`), so a VFS read of a stranded blob throws + * "Blob metadata not found" and every page backed by it 500s. + * + * Proves the cure end to end: + * - `storage.adoptLegacyCowBlobs()` copies each orphaned `_cow/` blob (bytes + + * metadata) into `_cas/` verbatim, idempotently, non-destructively, and + * reports an `incomplete` blob (bytes without metadata) rather than + * half-adopting it. + * - a real brain that upgraded into this state SELF-HEALS on its next open + * (auto-adoption) — the stranded VFS file reads correctly again with no + * operator action. + * - `brain.vfs.adoptOrphanedBlobs()` is the explicit equivalent. + */ +import { describe, it, expect, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js' + +const tmpDirs: string[] = [] +function mkTmp(): string { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-cow-adopt-')) + tmpDirs.push(d) + return d +} + +/** Release a raw FileSystemStorage: flush, stop its timer, drop the writer lock + * so a subsequent open of the same directory is not blocked. */ +async function teardownRawStorage(s: any): Promise { + try { await s.flush?.() } catch { /* best effort */ } + try { s.stopFlushRequestWatcher?.() } catch { /* best effort */ } + try { await s.releaseWriterLock?.() } catch { /* best effort */ } +} +afterEach(() => { + for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }) +}) + +/** Move every `_cas/blob*` object into `_cow/` and delete the `_cas/` copy — + * reproducing the exact on-disk state a 7.x brain leaves after the 8.0 layout + * migration collapses entities but does not adopt the copy-on-write blobs. */ +async function strandCasBlobsIntoCow(dir: string): Promise { + const s: any = new FileSystemStorage(dir) + await s.init() + const casKeys = (await s.listRawObjects('_cas/')).filter((k: string) => + /(^|\/)(blob:|blob-meta:)/.test(k) + ) + let moved = 0 + for (const full of casKeys) { + const key = full.replace(/^_cas\//, '') + const obj = await s.readRawObject(`_cas/${key}`) + if (obj === null) continue + await s.writeRawObject(`_cow/${key}`, obj) + await s.deleteRawObject(`_cas/${key}`) + moved++ + } + await teardownRawStorage(s) + return moved +} + +describe('VFS _cow/ blob adoption (P0 7→8 recovery)', () => { + it('adoptLegacyCowBlobs copies bytes + metadata into _cas/, verbatim, idempotent, non-destructive', async () => { + const dir = mkTmp() + const storage: any = new FileSystemStorage(dir) + await storage.init() + + const hash = 'a'.repeat(64) + const bytes = Buffer.from('the homepage payload') + // 7.x layout: the pair lives ONLY under _cow/ (blob-meta carries the refCount). + await storage.writeRawObject(`_cow/blob:${hash}`, { _binary: true, data: bytes.toString('base64') }) + await storage.writeRawObject(`_cow/blob-meta:${hash}`, { hash, size: bytes.length, refCount: 3, mimeType: 'application/json' }) + + // Pre-adoption: _cas/ has neither → this is the "Blob metadata not found" state. + expect(await storage.readRawObject(`_cas/blob-meta:${hash}`)).toBeNull() + + const r = await storage.adoptLegacyCowBlobs() + expect(r).toMatchObject({ cowBlobs: 1, adopted: 1, alreadyPresent: 0, incomplete: 0 }) + + // Post: _cas/ has both, metadata verbatim (refCount preserved), bytes intact. + const casMeta = await storage.readRawObject(`_cas/blob-meta:${hash}`) + expect(casMeta).toMatchObject({ hash, refCount: 3, size: bytes.length }) + const casBlob = await storage.readRawObject(`_cas/blob:${hash}`) + expect(Buffer.from(casBlob.data, 'base64').toString()).toBe('the homepage payload') + + // Idempotent: a second run adopts nothing, counts the pair present. + expect(await storage.adoptLegacyCowBlobs()).toMatchObject({ adopted: 0, alreadyPresent: 1 }) + // Non-destructive: the _cow/ original is still there (rollback stays possible). + expect(await storage.readRawObject(`_cow/blob:${hash}`)).not.toBeNull() + + // Incomplete: bytes without metadata are reported, not half-adopted. + await storage.writeRawObject(`_cow/blob:${'b'.repeat(64)}`, { _binary: true, data: 'x' }) + const r3 = await storage.adoptLegacyCowBlobs() + expect(r3.incomplete).toBe(1) + expect(await storage.readRawObject(`_cas/blob:${'b'.repeat(64)}`)).toBeNull() // not adopted + await teardownRawStorage(storage) + }) + + it('a brain that upgraded into the stranded state self-heals on next open — the VFS page reads again', async () => { + const dir = mkTmp() + + // 1. Author a VFS page (its content blob lands in _cas/). + let brain: any = new Brainy({ storage: { type: 'filesystem', path: dir }, plugins: [], silent: true }) + await brain.init() + await brain.vfs.writeFile('/pages/homepage/page.json', '{"title":"Homepage"}') + expect((await brain.vfs.readFile('/pages/homepage/page.json')).toString()).toContain('Homepage') + await brain.close() + + // 2. Reproduce the post-7→8 stranding: the blob is now only under _cow/. + const moved = await strandCasBlobsIntoCow(dir) + expect(moved).toBeGreaterThan(0) + + // 3. Reopen: the on-open auto-adoption heals it with no operator action. + brain = new Brainy({ storage: { type: 'filesystem', path: dir }, plugins: [], silent: true }) + await brain.init() + const healed = await brain.vfs.readFile('/pages/homepage/page.json') + expect(healed.toString()).toContain('Homepage') + + // A subsequent open records the adoption marker and does not re-scan-heal. + await brain.close() + brain = new Brainy({ storage: { type: 'filesystem', path: dir }, plugins: [], silent: true }) + await brain.init() + expect((await brain.vfs.readFile('/pages/homepage/page.json')).toString()).toContain('Homepage') + await brain.close() + }) + + it('brain.vfs.adoptOrphanedBlobs() is the explicit recovery equivalent', async () => { + const dir = mkTmp() + const body = JSON.stringify({ page: 'about', body: 'x'.repeat(400) }) + let brain: any = new Brainy({ storage: { type: 'filesystem', path: dir }, plugins: [], silent: true }) + await brain.init() + await brain.vfs.writeFile('/about.json', body) + await brain.close() + await strandCasBlobsIntoCow(dir) + + // The explicit API self-initializes (like other VFS methods); init's + // auto-heal adopts, so the explicit call sees the pair present. Either way + // it reports the _cow/ blobs it accounted for, and the read is healed. + brain = new Brainy({ storage: { type: 'filesystem', path: dir }, plugins: [], silent: true }) + const r = await brain.vfs.adoptOrphanedBlobs() + expect(r.cowBlobs).toBeGreaterThan(0) + expect(r.adopted + r.alreadyPresent).toBe(r.cowBlobs) + expect((await brain.vfs.readFile('/about.json')).toString()).toContain('about') + await brain.close() + }) +}) diff --git a/tests/integration/vfs-rename-containment.test.ts b/tests/integration/vfs-rename-containment.test.ts new file mode 100644 index 00000000..0169bf01 --- /dev/null +++ b/tests/integration/vfs-rename-containment.test.ts @@ -0,0 +1,157 @@ +/** + * @module tests/integration/vfs-rename-containment + * @description A cross-directory rename must MOVE the containment edge, not + * accumulate one per parent. The pre-fix rename added the new parent's + * Contains edge but skipped removing the old one ("not critical") — leaving + * the entity a child of BOTH directories: readdir(oldDir) kept listing it, + * re-creating the old path showed the name twice (the duplicate-readdir / + * "cosmetic ghost" field report), and tree-walking consumers saw the file in + * two places. Also covers the repair sweep (repairIndex → vfs.repairContainment) + * that heals ghosts left by earlier versions, and move-to-root (whose edge + * used to be skipped entirely). + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy, VerbType } from '../../src/index.js' + +describe('VFS rename moves the containment edge (no ghost in the old directory)', () => { + let brain: any + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true, dimensions: 384 }) + await brain.init() + await brain.vfs.mkdir('/a', { recursive: true }) + await brain.vfs.mkdir('/b', { recursive: true }) + }) + afterEach(async () => { + await brain.close?.().catch(() => {}) + }) + + it('cross-directory move: old dir stops listing it, new dir lists it exactly once', async () => { + await brain.vfs.writeFile('/a/x.txt', 'v1') + await brain.vfs.rename('/a/x.txt', '/b/x.txt') + + expect(await brain.vfs.readdir('/a')).toEqual([]) + expect(await brain.vfs.readdir('/b')).toEqual(['x.txt']) + expect(String(await brain.vfs.readFile('/b/x.txt'))).toBe('v1') + }) + + it('re-creating the old path lists each name exactly once in each directory', async () => { + await brain.vfs.writeFile('/a/x.txt', 'moved away') + await brain.vfs.rename('/a/x.txt', '/b/x.txt') + await brain.vfs.writeFile('/a/x.txt', 'new file at old path') + + const a = (await brain.vfs.readdir('/a')) as string[] + const b = (await brain.vfs.readdir('/b')) as string[] + expect(a).toEqual(['x.txt']) // exactly once — the pre-fix ghost made this list the moved entity too + expect(b).toEqual(['x.txt']) + expect(String(await brain.vfs.readFile('/a/x.txt'))).toBe('new file at old path') + expect(String(await brain.vfs.readFile('/b/x.txt'))).toBe('moved away') + }) + + it('repeated moves never accumulate containment edges', async () => { + await brain.vfs.writeFile('/a/f.txt', 'wanderer') + for (let i = 0; i < 3; i++) { + await brain.vfs.rename('/a/f.txt', '/b/f.txt') + await brain.vfs.rename('/b/f.txt', '/a/f.txt') + } + expect(await brain.vfs.readdir('/a')).toEqual(['f.txt']) + expect(await brain.vfs.readdir('/b')).toEqual([]) + + // Exactly ONE containment edge exists on the entity. + const stat = await brain.vfs.stat('/a/f.txt') + const edges = await brain.related({ to: stat.entityId, type: VerbType.Contains }) + expect(edges).toHaveLength(1) + }) + + it('move to the root gets a containment edge (used to be skipped → orphan)', async () => { + await brain.vfs.writeFile('/a/up.txt', 'to the top') + await brain.vfs.rename('/a/up.txt', '/up.txt') + + const rootListing = (await brain.vfs.readdir('/')) as string[] + expect(rootListing).toContain('up.txt') + expect(await brain.vfs.readdir('/a')).toEqual([]) + expect(String(await brain.vfs.readFile('/up.txt'))).toBe('to the top') + }) +}) + +describe('repairIndex() heals pre-fix containment ghosts (vfs.repairContainment)', () => { + let brain: any + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true, dimensions: 384 }) + await brain.init() + await brain.vfs.mkdir('/a', { recursive: true }) + await brain.vfs.mkdir('/b', { recursive: true }) + }) + afterEach(async () => { + await brain.close?.().catch(() => {}) + }) + + /** Reproduce the PRE-FIX defect state: file lives at /b/g.txt but a stale + * vfs-contains edge from /a lingers (what old renames left behind). */ + const synthesizeGhost = async () => { + await brain.vfs.writeFile('/b/g.txt', 'ghost target') + const fileId = (await brain.vfs.stat('/b/g.txt')).entityId + const aId = (await brain.vfs.stat('/a')).entityId + await brain.relate({ + from: aId, + to: fileId, + type: VerbType.Contains, + subtype: 'vfs-contains', + metadata: { isVFS: true } + }) + return { fileId, aId } + } + + it('a stale old-parent edge is removed; listings become honest', async () => { + const { fileId } = await synthesizeGhost() + // The defect state is visible: /a lists a file whose path says /b. + expect((await brain.vfs.readdir('/a')) as string[]).toContain('g.txt') + + await brain.repairIndex() + + expect(await brain.vfs.readdir('/a')).toEqual([]) + expect(await brain.vfs.readdir('/b')).toEqual(['g.txt']) + const edges = await brain.related({ to: fileId, type: VerbType.Contains }) + expect(edges).toHaveLength(1) + }) + + it("a user's own knowledge Contains edge onto the file survives the repair", async () => { + const { fileId } = await synthesizeGhost() + // A knowledge-graph containment from a NON-directory entity (a collection + // curating the file) — same verb TYPE, not a vfs-contains edge. The repair + // must remove only VFS containment ghosts, never user knowledge edges. + // (Edges dedupe by (from,to,type), so the user edge needs its own source.) + const collectionId = await brain.add({ + data: 'reading list', + type: 'collection', + metadata: { kind: 'curation' } + }) + await brain.relate({ from: collectionId, to: fileId, type: VerbType.Contains, subtype: 'curates' }) + + await brain.repairIndex() + + const edges = await brain.related({ to: fileId, type: VerbType.Contains }) + const subtypes = edges.map((e: any) => e.subtype).sort() + // The stale vfs ghost edge is gone; the correct vfs edge + the user's + // curation edge remain. + expect(subtypes).toEqual(['curates', 'vfs-contains']) + expect(edges.find((e: any) => e.subtype === 'curates')?.from).toBe(collectionId) + }) + + it('a missing expected edge is restored (entity unreachable from its own directory)', async () => { + await brain.vfs.writeFile('/b/lost.txt', 'find me') + const fileId = (await brain.vfs.stat('/b/lost.txt')).entityId + // Simulate total edge loss (an older damage shape). + for (const e of await brain.related({ to: fileId, type: VerbType.Contains })) { + await brain.unrelate(e.id) + } + expect((await brain.vfs.readdir('/b')) as string[]).not.toContain('lost.txt') + + await brain.repairIndex() + + expect((await brain.vfs.readdir('/b')) as string[]).toContain('lost.txt') + }) +}) diff --git a/tests/integrations/core.test.ts b/tests/integrations/core.unit.test.ts similarity index 100% rename from tests/integrations/core.test.ts rename to tests/integrations/core.unit.test.ts diff --git a/tests/integrations/odata.test.ts b/tests/integrations/odata.unit.test.ts similarity index 100% rename from tests/integrations/odata.test.ts rename to tests/integrations/odata.unit.test.ts diff --git a/tests/regression/metadata-index-cleanup.test.ts b/tests/regression/metadata-index-cleanup.unit.test.ts similarity index 97% rename from tests/regression/metadata-index-cleanup.test.ts rename to tests/regression/metadata-index-cleanup.unit.test.ts index 266b9a4d..0984d727 100644 --- a/tests/regression/metadata-index-cleanup.test.ts +++ b/tests/regression/metadata-index-cleanup.unit.test.ts @@ -205,10 +205,10 @@ describe('Metadata index cleanup after remove / removeMany', () => { } }) - it('handles empty ids array gracefully', async () => { - const result = await brain.removeMany({ ids: [] }) - expect(result.successful).toHaveLength(0) - expect(result.failed).toHaveLength(0) + it('refuses an empty ids array loudly (a silent no-op is not "graceful")', async () => { + // 8.8.2: an empty selector used to resolve successfully having deleted + // NOTHING — the caller believed the delete happened. Now it throws. + await expect(brain.removeMany({ ids: [] })).rejects.toThrow(/ids: \[\]/) }) it('handles large batch (> 1 chunk) without leaving stale index entries', async () => { diff --git a/tests/regression/v5.7.0-deadlock.test.ts b/tests/regression/v5.7.0-deadlock.unit.test.ts similarity index 100% rename from tests/regression/v5.7.0-deadlock.test.ts rename to tests/regression/v5.7.0-deadlock.unit.test.ts diff --git a/tests/streaming-pipeline.test.ts b/tests/streaming-pipeline.test.ts deleted file mode 100644 index 24e51521..00000000 --- a/tests/streaming-pipeline.test.ts +++ /dev/null @@ -1,330 +0,0 @@ -/** - * Streaming Pipeline Tests - * Tests for the new streaming data pipeline system - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Pipeline, createPipeline } from '../src/streaming/pipeline.js' -import { Brainy } from '../src/brainy.js' -import { NounType } from '../src/types/graphTypes.js' - -describe('Streaming Pipeline', () => { - let brain: Brainy - - beforeEach(async () => { - brain = new Brainy({ requireSubtype: false, - storage: { type: 'memory' }, - warmup: false - }) - await brain.init() - }) - - afterEach(async () => { - await brain.close() - }) - - describe('Core Operations', () => { - it('should process data through pipeline', async () => { - const results: number[] = [] - - await new Pipeline() - .source(async function* () { - for (let i = 1; i <= 5; i++) { - yield i - } - }) - .map(x => x * 2) - .filter(x => x > 4) - .sink(x => { results.push(x) }) - .run() - - expect(results).toEqual([6, 8, 10]) - }) - - it('should handle async transformations', async () => { - const results: string[] = [] - - await new Pipeline() - .source(async function* () { - yield 'hello' - yield 'world' - }) - .map(async (text) => { - await new Promise(resolve => setTimeout(resolve, 10)) - return text.toUpperCase() - }) - .sink(x => { results.push(x) }) - .run() - - expect(results).toEqual(['HELLO', 'WORLD']) - }) - - it('should batch items', async () => { - const batches: number[][] = [] - - await new Pipeline() - .source(async function* () { - for (let i = 1; i <= 10; i++) { - yield i - } - }) - .batch(3) - .sink(batch => { batches.push(batch) }) - .run() - - expect(batches).toEqual([ - [1, 2, 3], - [4, 5, 6], - [7, 8, 9], - [10] - ]) - }) - - it('should collect all results', async () => { - const pipeline = new Pipeline() - .source(async function* () { - for (let i = 1; i <= 5; i++) { - yield i - } - }) - .map(x => x * x) - - const results = await pipeline.collect() - expect(results).toEqual([1, 4, 9, 16, 25]) - }) - }) - - describe('Window Operations', () => { - it('should support tumbling windows', async () => { - const windows: number[][] = [] - - await new Pipeline() - .source(async function* () { - for (let i = 1; i <= 10; i++) { - yield i - } - }) - .window(3, 'tumbling') - .sink(window => { windows.push(window) }) - .run() - - expect(windows).toEqual([ - [1, 2, 3], - [4, 5, 6], - [7, 8, 9], - [10] - ]) - }) - - it('should support sliding windows', async () => { - const windows: number[][] = [] - - await new Pipeline() - .source(async function* () { - for (let i = 1; i <= 5; i++) { - yield i - } - }) - .window(3, 'sliding') - .sink(window => { windows.push([...window]) }) - .run() - - expect(windows).toEqual([ - [1, 2, 3], - [2, 3, 4], - [3, 4, 5] - ]) - }) - }) - - describe('Reduce Operations', () => { - it('should reduce values', async () => { - let result = 0 - - await new Pipeline() - .source(async function* () { - for (let i = 1; i <= 5; i++) { - yield i - } - }) - .reduce((acc, val) => acc + val, 0) - .sink(sum => { result = sum }) - .run() - - expect(result).toBe(15) - }) - - it('should work with complex reducers', async () => { - interface Stats { - count: number - sum: number - max: number - } - - let stats: Stats = { count: 0, sum: 0, max: 0 } - - await new Pipeline() - .source(async function* () { - for (let i = 1; i <= 10; i++) { - yield i - } - }) - .reduce((acc, val) => ({ - count: acc.count + 1, - sum: acc.sum + val, - max: Math.max(acc.max, val) - }), { count: 0, sum: 0, max: 0 }) - .sink(s => { stats = s }) - .run() - - expect(stats).toEqual({ - count: 10, - sum: 55, - max: 10 - }) - }) - }) - - describe('Brainy Integration', () => { - it('should sink data to Brainy', async () => { - const pipeline = new Pipeline(brain) - .source(async function* () { - yield { content: 'Document 1' } - yield { content: 'Document 2' } - yield { content: 'Document 3' } - }) - .toBrainy({ - type: NounType.Document, - metadata: { source: 'pipeline' }, - batchSize: 2 - }) - - await pipeline.run() - - // Verify data was added - const results = await brain.find({ - where: { 'metadata.source': 'pipeline' } - }) - - expect(results.length).toBe(3) - }) - - it('should process and transform before storing', async () => { - const pipeline = new Pipeline(brain) - .source(async function* () { - yield 'hello world' - yield 'goodbye world' - }) - .map(text => ({ - content: text, - processed: text.toUpperCase() - })) - .toBrainy({ - type: NounType.Document, - metadata: { pipeline: true } - }) - - await pipeline.run() - - const results = await brain.find({ - where: { 'metadata.pipeline': true } - }) - - expect(results.length).toBe(2) - }) - }) - - describe('Error Handling', () => { - it('should handle errors with handler', async () => { - const errors: Error[] = [] - - await new Pipeline() - .source(async function* () { - yield 1 - yield 2 - yield 3 - }) - .map(x => { - if (x === 2) throw new Error('Test error') - return x - }) - .sink(() => {}) - .run({ - errorHandler: (error) => errors.push(error) - }) - - expect(errors.length).toBe(1) - expect(errors[0].message).toBe('Test error') - }) - - it('should stop on abort signal', async () => { - let count = 0 - - const pipeline = new Pipeline() - .source(async function* () { - for (let i = 1; i <= 100; i++) { - yield i - } - }) - .sink(() => { count++ }) - - // Start pipeline - const runPromise = pipeline.run() - - // Stop after a short delay - setTimeout(() => pipeline.stop(), 50) - - await runPromise - - // Should have processed some but not all items - expect(count).toBeGreaterThan(0) - expect(count).toBeLessThan(100) - }) - }) - - describe('Performance Features', () => { - it('should throttle sink operations', async () => { - const times: number[] = [] - const startTime = Date.now() - - await new Pipeline() - .source(async function* () { - for (let i = 1; i <= 3; i++) { - yield i - } - }) - .throttledSink( - () => { times.push(Date.now() - startTime) }, - 10 // 10 ops/sec max - ) - .run() - - // Check that operations were throttled - expect(times.length).toBe(3) - expect(times[2] - times[0]).toBeGreaterThanOrEqual(200) // At least 200ms for 3 items at 10/sec - }) - - it('should run monitoring', async () => { - const logs: string[] = [] - const originalLog = console.log - console.log = (msg: string) => logs.push(msg) - - try { - await new Pipeline() - .source(async function* () { - for (let i = 1; i <= 10; i++) { - yield i - } - }) - .sink(() => {}) - .run({ monitoring: true }) - - // Should have logged completion metrics - expect(logs.some(log => log.includes('Pipeline completed'))).toBe(true) - expect(logs.some(log => log.includes('Throughput'))).toBe(true) - } finally { - console.log = originalLog - } - }) - }) -}) \ No newline at end of file diff --git a/tests/transaction/Transaction.test.ts b/tests/transaction/Transaction.unit.test.ts similarity index 100% rename from tests/transaction/Transaction.test.ts rename to tests/transaction/Transaction.unit.test.ts diff --git a/tests/transaction/TransactionManager.test.ts b/tests/transaction/TransactionManager.unit.test.ts similarity index 81% rename from tests/transaction/TransactionManager.test.ts rename to tests/transaction/TransactionManager.unit.test.ts index 5ffee866..86b1692c 100644 --- a/tests/transaction/TransactionManager.test.ts +++ b/tests/transaction/TransactionManager.unit.test.ts @@ -10,6 +10,7 @@ import { describe, it, expect, beforeEach } from 'vitest' import { TransactionManager } from '../../src/transaction/TransactionManager.js' +import { transactTimeoutBudget } from '../../src/transaction/Transaction.js' import { TransactionError } from '../../src/transaction/errors.js' describe('TransactionManager', () => { @@ -105,14 +106,17 @@ describe('TransactionManager', () => { const result = await manager.executeTransactionWithResult(async (tx) => { tx.addOperation({ execute: async () => { - await new Promise(resolve => setTimeout(resolve, 10)) + await new Promise(resolve => setTimeout(resolve, 25)) return async () => {} } }) return 'done' }) - expect(result.executionTimeMs).toBeGreaterThanOrEqual(10) + // Timer coalescing can fire a setTimeout up to a few ms EARLY under + // load, so assert well below the sleep — this tests that time is + // MEASURED, not the OS timer's precision. + expect(result.executionTimeMs).toBeGreaterThanOrEqual(20) }) }) @@ -325,4 +329,45 @@ describe('TransactionManager', () => { expect(stats1).toEqual(stats2) // Same values }) }) + + describe('Timeout budget + telemetry', () => { + it('transactTimeoutBudget: explicit override wins; default scales with batch size', () => { + expect(transactTimeoutBudget(1)).toBe(30_000) // small batches keep the 30s floor + expect(transactTimeoutBudget(15)).toBe(30_000) // the old flat cap's break-even point + expect(transactTimeoutBudget(100)).toBe(200_000) // 100 ops × 2s — bulk gets an honest budget + expect(transactTimeoutBudget(1000, 5_000)).toBe(5_000) // caller override is untouched + }) + + it('a tripped budget rolls back and names the operation, progress, and budget', async () => { + const rolledBack: string[] = [] + + const failing = manager.executeTransaction( + async (tx) => { + tx.addOperation({ + name: 'slow-first-op', + execute: async () => { + await new Promise((r) => setTimeout(r, 30)) + return async () => { + rolledBack.push('slow-first-op') + } + } + }) + tx.addOperation({ + name: 'never-reached', + execute: async () => undefined + }) + }, + { timeout: 5 } // the first op's 30ms sleep guarantees the pre-op-2 check trips + ) + + await expect(failing).rejects.toThrow(TransactionError) + const err = await failing.catch((e) => e) + expect(err.name).toBe('TransactionTimeoutError') + expect(err.message).toContain('operation 1/2') // which op, of how many + expect(err.message).toContain("('never-reached')") // its name + expect(err.message).toContain('budget 5ms') // the budget that tripped + expect(err.message).toContain('rolled back') // the retryability statement + expect(rolledBack).toEqual(['slow-first-op']) // the applied op was undone + }) + }) }) diff --git a/tests/transaction/integration/typeaware-transactions.test.ts b/tests/transaction/integration/typeaware-transactions.test.ts deleted file mode 100644 index 51a659c6..00000000 --- a/tests/transaction/integration/typeaware-transactions.test.ts +++ /dev/null @@ -1,368 +0,0 @@ -/** - * TypeAware + Transactions Integration Tests - * - * Verifies that transactions work correctly with type-aware storage: - * - Type-specific routing - * - Type cache updates - * - Type changes during updates - * - Per-type performance optimization - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../../src/brainy.js' -import { NounType, VerbType } from '../../../src/types/graphTypes.js' -import { tmpdir } from 'os' -import { join } from 'path' -import { mkdirSync, rmSync } from 'fs' - -describe('Transactions + TypeAware Storage Integration', () => { - let brain: Brainy - let testDir: string - - beforeEach(async () => { - testDir = join(tmpdir(), `brainy-typeaware-test-${Date.now()}`) - mkdirSync(testDir, { recursive: true }) - - brain = new Brainy({ requireSubtype: false, - storage: { - type: 'filesystem', - path: testDir - } - }) - - await brain.init() - }) - - afterEach(async () => { - if (brain) { - await brain.shutdown() - } - if (testDir) { - rmSync(testDir, { recursive: true, force: true }) - } - }) - - describe('Type-Specific Routing', () => { - it('should route entities to type-specific storage atomically', async () => { - // Add entities of different types - const personId = await brain.add({ - data: { name: 'John Doe', role: 'Developer' }, - type: NounType.Person - }) - - const orgId = await brain.add({ - data: { name: 'Acme Corp', industry: 'Tech' }, - type: NounType.Organization - }) - - const placeId = await brain.add({ - data: { name: 'San Francisco', country: 'USA' }, - type: NounType.Place - }) - - // Verify all entities stored with correct types - const person = await brain.get(personId) - const org = await brain.get(orgId) - const place = await brain.get(placeId) - - expect(person?.type).toBe(NounType.Person) - expect(org?.type).toBe(NounType.Organization) - expect(place?.type).toBe(NounType.Place) - }) - - it('should handle type-specific queries atomically', async () => { - // Add multiple entities of same type - await brain.add({ - data: { name: 'Alice', role: 'Engineer' }, - type: NounType.Person - }) - - await brain.add({ - data: { name: 'Bob', role: 'Designer' }, - type: NounType.Person - }) - - await brain.add({ - data: { name: 'Acme', industry: 'Tech' }, - type: NounType.Organization - }) - - // Query by type - const results = await brain.find({ - filter: { type: NounType.Person } - }) - - expect(results).toHaveLength(2) - expect(results.every(r => r.type === NounType.Person)).toBe(true) - }) - }) - - describe('Type Changes During Updates', () => { - it('should handle type changes atomically in update operations', async () => { - // Add entity as Person - const id = await brain.add({ - data: { name: 'John Smith', category: 'individual' }, - type: NounType.Person - }) - - // Verify initial type - let entity = await brain.get(id) - expect(entity?.type).toBe(NounType.Person) - - // Update to Organization (type change) - await brain.update({ - id, - type: NounType.Organization, - data: { name: 'Smith Corp', category: 'business' } - }) - - // Verify type changed - entity = await brain.get(id) - expect(entity?.type).toBe(NounType.Organization) - expect(entity?.data.category).toBe('business') - }) - - it('should rollback type changes on failed update', async () => { - // Add entity as Person - const id = await brain.add({ - data: { name: 'Jane Doe' }, - type: NounType.Person - }) - - // Attempt to update with invalid data (will fail) - let failed = false - try { - await brain.update({ - id, - type: NounType.Organization, - data: null as any // Invalid - }) - } catch (e) { - failed = true - } - - expect(failed).toBe(true) - - // Type should remain Person (rollback) - const entity = await brain.get(id) - expect(entity?.type).toBe(NounType.Person) - expect(entity?.data.name).toBe('Jane Doe') - }) - }) - - describe('Type Cache Consistency', () => { - it('should maintain type cache consistency during transactions', async () => { - // Add multiple entities - const ids: string[] = [] - const types = [ - NounType.Person, - NounType.Organization, - NounType.Place, - NounType.Event, - NounType.Concept - ] - - for (let i = 0; i < types.length; i++) { - const id = await brain.add({ - data: { name: `Entity ${i}`, index: i }, - type: types[i] - }) - ids.push(id) - } - - // Verify all types cached correctly - for (let i = 0; i < ids.length; i++) { - const entity = await brain.get(ids[i]) - expect(entity?.type).toBe(types[i]) - } - }) - - it('should update type cache on type changes', async () => { - const id = await brain.add({ - data: { name: 'Initial' }, - type: NounType.Thing - }) - - // Sequence of type changes - const typeSequence = [ - NounType.Concept, - NounType.Event, - NounType.Organization, - NounType.Person - ] - - for (const newType of typeSequence) { - await brain.update({ - id, - type: newType, - data: { name: `As ${newType}` } - }) - - const entity = await brain.get(id) - expect(entity?.type).toBe(newType) - } - }) - }) - - describe('Per-Type Performance', () => { - it('should handle large numbers of same-type entities efficiently', async () => { - const startTime = Date.now() - - // Add 50 entities of same type (type-aware storage optimization) - const ids: string[] = [] - for (let i = 0; i < 50; i++) { - const id = await brain.add({ - data: { name: `Person ${i}`, index: i }, - type: NounType.Person - }) - ids.push(id) - } - - const addTime = Date.now() - startTime - - // Verify all entities stored - const retrieveStart = Date.now() - for (const id of ids) { - const entity = await brain.get(id) - expect(entity).toBeTruthy() - } - const retrieveTime = Date.now() - retrieveStart - - // Type-aware storage should be reasonably fast - expect(addTime).toBeLessThan(5000) // 5 seconds for 50 adds - expect(retrieveTime).toBeLessThan(2000) // 2 seconds for 50 retrieves - }) - }) - - describe('Mixed Type Operations', () => { - it('should handle operations across multiple types atomically', async () => { - // Create entities of different types - const personId = await brain.add({ - data: { name: 'Alice' }, - type: NounType.Person - }) - - const orgId = await brain.add({ - data: { name: 'TechCorp' }, - type: NounType.Organization - }) - - const eventId = await brain.add({ - data: { name: 'Conference 2024' }, - type: NounType.Event - }) - - // Create relationships across types (atomic) - await brain.relate({ - from: personId, - to: orgId, - type: VerbType.WorksFor - }) - - await brain.relate({ - from: personId, - to: eventId, - type: VerbType.Attends - }) - - await brain.relate({ - from: orgId, - to: eventId, - type: VerbType.Sponsors - }) - - // Verify relationships exist - const personRelations = await brain.related({ from: personId }) - const orgRelations = await brain.related({ from: orgId }) - - expect(personRelations).toHaveLength(2) - expect(orgRelations).toHaveLength(1) - }) - - it('should handle delete with cascade across types', async () => { - // Create multi-type graph - const personId = await brain.add({ - data: { name: 'Bob' }, - type: NounType.Person - }) - - const projectId = await brain.add({ - data: { name: 'Project X' }, - type: NounType.Thing - }) - - const taskId = await brain.add({ - data: { name: 'Task 1' }, - type: NounType.Thing - }) - - // Create relationships - await brain.relate({ - from: personId, - to: projectId, - type: VerbType.WorksOn - }) - - await brain.relate({ - from: projectId, - to: taskId, - type: VerbType.Contains - }) - - // Delete person (should cascade delete relationships) - await brain.remove(personId) - - // Verify person deleted - const person = await brain.get(personId) - expect(person).toBeNull() - - // Verify project and task still exist (different types) - const project = await brain.get(projectId) - const task = await brain.get(taskId) - expect(project).toBeTruthy() - expect(task).toBeTruthy() - - // Verify relationships from person are deleted - const relations = await brain.related({ from: personId }) - expect(relations).toHaveLength(0) - }) - }) - - describe('Type Validation', () => { - it('should validate types during atomic operations', async () => { - // Valid type - should succeed - const id = await brain.add({ - data: { name: 'Test' }, - type: NounType.Person - }) - - expect(id).toBeTruthy() - - // Verify type stored correctly - const entity = await brain.get(id) - expect(entity?.type).toBe(NounType.Person) - }) - - it('should handle type-specific metadata atomically', async () => { - // Add with type-specific metadata - const personId = await brain.add({ - data: { name: 'Charlie', age: 30, occupation: 'Engineer' }, - type: NounType.Person, - metadata: { verified: true, source: 'HR' } - }) - - const orgId = await brain.add({ - data: { name: 'StartupCo', employees: 50, founded: 2020 }, - type: NounType.Organization, - metadata: { verified: false, source: 'Registration' } - }) - - // Verify metadata with type context - const person = await brain.get(personId) - const org = await brain.get(orgId) - - expect(person?.metadata?.verified).toBe(true) - expect(org?.metadata?.verified).toBe(false) - }) - }) -}) diff --git a/tests/type-utils.test.ts b/tests/type-utils.unit.test.ts similarity index 100% rename from tests/type-utils.test.ts rename to tests/type-utils.unit.test.ts diff --git a/tests/unit/aggregation/AggregationIndex.test.ts b/tests/unit/aggregation/AggregationIndex.test.ts index 05f4a62a..a0c10e62 100644 --- a/tests/unit/aggregation/AggregationIndex.test.ts +++ b/tests/unit/aggregation/AggregationIndex.test.ts @@ -675,4 +675,104 @@ describe('AggregationIndex', () => { await reloaded.close() }) }) + + // ============= Boot-order reconciliation ============= + // + // The production boot pattern: defineAggregate() is synchronous and always + // beats the async init() that loads persisted state. The old code flagged a + // backfill at define time and init never cleared it — so the loaded state + // was wiped and the whole store re-walked on EVERY restart. + + describe('boot-order reconciliation (define-before-init)', () => { + const DEF: AggregateDefinition = { + name: 'boot_agg', + source: { type: NounType.Event }, + groupBy: ['category'], + metrics: { count: { op: 'count' } } + } + + const entity = (id: string, category: string): Record => ({ + id, + noun: NounType.Event, + metadata: { category }, + createdAt: Date.now(), + updatedAt: Date.now() + }) + + /** Simulate the previous session: define, contribute, flush. */ + async function seedAndFlush(store: MemoryStorage): Promise { + const first = new AggregationIndex(store) + await first.init() + first.defineAggregate(DEF) + first.onEntityAdded('e1', entity('e1', 'food')) + first.onEntityAdded('e2', entity('e2', 'food')) + first.onEntityAdded('e3', entity('e3', 'transport')) + await first.flush() + } + + it('adopts persisted state when define beats init with an unchanged definition', async () => { + const store = new MemoryStorage() + await store.init() + await seedAndFlush(store) + + const second = new AggregationIndex(store) + second.defineAggregate(DEF) // synchronous define FIRST — the real boot order + await second.init() + await second.ready() + + expect(second.getPendingBackfills()).toEqual([]) + const rows = second.queryAggregate({ name: 'boot_agg' }) + const food = rows.find(r => r.groupKey.category === 'food')! + expect(food.metrics.count).toBe(2) + }) + + it('a write landing before adoption forces an exact rescan instead', async () => { + const store = new MemoryStorage() + await store.init() + await seedAndFlush(store) + + const second = new AggregationIndex(store) + second.defineAggregate(DEF) + second.onEntityAdded('e4', entity('e4', 'food')) // lands before init settles + await second.init() + await second.ready() + + // Adoption would lose e4's contribution — the engine must rescan. + expect(second.getPendingBackfills()).toEqual(['boot_agg']) + }) + + it('init never clobbers a changed app definition registered before it', async () => { + const store = new MemoryStorage() + await store.init() + await seedAndFlush(store) + + const CHANGED: AggregateDefinition = { + ...DEF, + metrics: { count: { op: 'count' }, total: { op: 'sum', field: 'amount' } } + } + const second = new AggregationIndex(store) + second.defineAggregate(CHANGED) + await second.init() + await second.ready() + + const def = second.getDefinitions().find(d => d.name === 'boot_agg')! + expect(Object.keys(def.metrics).sort()).toEqual(['count', 'total']) + expect(second.getPendingBackfills()).toEqual(['boot_agg']) + }) + + it('init alone restores persisted definitions with adopted state, no backfill', async () => { + const store = new MemoryStorage() + await store.init() + await seedAndFlush(store) + + const second = new AggregationIndex(store) + await second.init() + await second.ready() + + expect(second.hasAggregate('boot_agg')).toBe(true) + expect(second.getPendingBackfills()).toEqual([]) + const rows = second.queryAggregate({ name: 'boot_agg' }) + expect(rows.reduce((s, r) => s + (r.metrics.count as number), 0)).toBe(3) + }) + }) }) diff --git a/tests/unit/aggregation/materialize-failure-loud.test.ts b/tests/unit/aggregation/materialize-failure-loud.test.ts new file mode 100644 index 00000000..9aa2d1cb --- /dev/null +++ b/tests/unit/aggregation/materialize-failure-loud.test.ts @@ -0,0 +1,51 @@ +/** + * @module tests/unit/aggregation/materialize-failure-loud + * @description Aggregation (Pattern C): a failed materialization must not be + * swallowed. `scheduleMaterialize`'s debounced `materializeOne(key).catch(() => {})` + * silently discarded the error, leaving the materialized Measurement entity stale + * with no signal. It now emits a loud warning (the value is still rebuildable via + * backfill-on-query, so it stays non-fatal — but visible). Load-bearing assertion: + * prodLog.warn fires instead of an empty catch. + */ +import { describe, it, expect, vi, afterEach } from 'vitest' +import { AggregateMaterializer } from '../../../src/aggregation/materializer.js' +import { prodLog } from '../../../src/utils/logger.js' +import { NounType } from '../../../src/types/graphTypes.js' + +describe('Aggregation — swallowed materialize failure is surfaced', () => { + afterEach(() => { + vi.restoreAllMocks() + vi.useRealTimers() + }) + + it('logs loudly when materializeOne rejects', async () => { + vi.useFakeTimers() + const warn = vi.spyOn(prodLog, 'warn').mockImplementation(() => {}) + + // Brain access whose add()/update() always reject → doMaterialize throws. + const brain = { + add: vi.fn().mockRejectedValue(new Error('storage down')), + update: vi.fn().mockRejectedValue(new Error('storage down')) + } + const m = new AggregateMaterializer(brain as any, 10) + const def: any = { + name: 'sales', + source: { type: NounType.Document }, + groupBy: ['region'], + metrics: { total: { op: 'count' } }, + materialize: { debounceMs: 10 } + } + m.scheduleMaterialize( + 'sales', + def, + { region: 'us' }, + { groupKey: { region: 'us' }, metrics: { total: { count: 1 } } } as any + ) + + await vi.advanceTimersByTimeAsync(20) // fire the debounce timer + await Promise.resolve() + + expect(warn).toHaveBeenCalled() + expect(String(warn.mock.calls[0][0])).toMatch(/Failed to materialize aggregate group/) + }) +}) diff --git a/tests/unit/boundary-no-native.test.ts b/tests/unit/boundary-no-native.test.ts index 20acc6e0..7a6dce39 100644 --- a/tests/unit/boundary-no-native.test.ts +++ b/tests/unit/boundary-no-native.test.ts @@ -46,9 +46,31 @@ function sourceFiles(dir: string): string[] { // `@soulcraft/cortex` both match; `@soulcraft/core` (a different name) does not, // because the package must be followed by a subpath separator or the closing quote. const NATIVE_MODULE = - /(?:import\s[^'"]*from\s*|import\s*\(\s*|require\s*\(\s*)['"]@soulcraft\/cor(?:tex)?(?:\/[^'"]*)?['"]/ + /(?:import\s[^'"]*from\s*|export\s[^'"]*from\s*|import\s*\(\s*|require\s*\(\s*|import\s*)['"]@soulcraft\/cor(?:tex)?(?:\/[^'"]*)?['"]/ describe('open/proprietary boundary — public repo must not depend on @soulcraft/cor', () => { + it('the detector catches every static-dependency form (incl. side-effect imports + re-exports)', () => { + // Each of these creates a hard dependency on the proprietary package and MUST + // be caught (the prior regex missed the bare side-effect import and re-export forms). + for (const form of [ + `import cor from '@soulcraft/cor'`, + `import { x } from '@soulcraft/cor'`, + `import '@soulcraft/cor'`, + `import '@soulcraft/cor/native'`, + `export { x } from '@soulcraft/cor'`, + `export * from '@soulcraft/cor'`, + `const x = require('@soulcraft/cor')`, + `const x = await import('@soulcraft/cor')`, + `import x from '@soulcraft/cortex'` + ]) { + expect(NATIVE_MODULE.test(form), `should detect: ${form}`).toBe(true) + } + // A different package whose name merely starts with the same prefix is NOT a match. + for (const ok of [`import x from '@soulcraft/core'`, `// see @soulcraft/cor docs`]) { + expect(NATIVE_MODULE.test(ok), `should NOT match: ${ok}`).toBe(false) + } + }) + it('no src/ or tests/ file imports or requires the native package', () => { const offenders: string[] = [] for (const dir of ['src', 'tests']) { diff --git a/tests/unit/brainy/batch-operations.test.ts b/tests/unit/brainy/batch-operations.test.ts index 9bbef3d7..58b25744 100644 --- a/tests/unit/brainy/batch-operations.test.ts +++ b/tests/unit/brainy/batch-operations.test.ts @@ -180,6 +180,49 @@ describe('Brainy Batch Operations', () => { } }) + it('removes more than the legacy 10-item chunk in a single call (adaptive chunking)', async () => { + // Seed 25 entities — more than the old hardcoded chunk of 10 — to prove the + // storage-adaptive chunk size processes every chunk and nothing is capped. + const seed = await brain.addMany({ + items: Array.from({ length: 25 }, (_, i) => ({ + data: `Adaptive Chunk ${i}`, + type: NounType.Thing, + metadata: { batch: 'adaptive' } + })) + }) + expect(seed.successful).toHaveLength(25) + + const result = await brain.removeMany({ ids: seed.successful }) + expect(result.successful).toHaveLength(25) + expect(result.failed).toHaveLength(0) + expect(result.total).toBe(25) + + // Every entity is actually gone. + for (const id of seed.successful) { + expect(await brain.get(id)).toBeNull() + } + }) + + it('removes all entities across multiple chunks when chunkSize is overridden', async () => { + const seed = await brain.addMany({ + items: Array.from({ length: 12 }, (_, i) => ({ + data: `Override Chunk ${i}`, + type: NounType.Thing, + metadata: { batch: 'override' } + })) + }) + expect(seed.successful).toHaveLength(12) + + // chunkSize 5 → 3 chunks (5 + 5 + 2); all must be removed. + const result = await brain.removeMany({ ids: seed.successful, chunkSize: 5 }) + expect(result.successful).toHaveLength(12) + expect(result.failed).toHaveLength(0) + + for (const id of seed.successful) { + expect(await brain.get(id)).toBeNull() + } + }) + it('should handle selective deletion', async () => { // Delete only some const toDelete = [testIds[0], testIds[2], testIds[4]] @@ -362,21 +405,24 @@ describe('Brainy Batch Operations', () => { expect(companyRelations.length).toBeGreaterThanOrEqual(50) }) - it('should skip invalid relationships', async () => { + it('a batch with a forward-reference endpoint still creates the fully-valid relationships', async () => { + // The middle item references a source id that does not (yet) exist — a + // forward reference. Whatever its fate, the two fully-valid edges MUST be + // created (a bad item must not sink the good ones in a batch). const relationships = [ { from: entities[0], to: entities[1], type: VerbType.FriendOf }, - { from: 'invalid-id', to: entities[2], type: VerbType.FriendOf }, + { from: 'no-such-entity', to: entities[2], type: VerbType.FriendOf }, { from: entities[1], to: entities[2], type: VerbType.FriendOf } ] - - try { - // Should skip invalid and continue - const relationIds = await brain.relateMany({ items: relationships }) - expect(relationIds.length).toBeLessThanOrEqual(3) - } catch (error) { - // Or might throw - that's ok too - expect(error).toBeDefined() - } + + const relationIds = await brain.relateMany({ items: relationships }) + expect(Array.isArray(relationIds)).toBe(true) + + // Both fully-valid edges are queryable, regardless of the forward-ref item. + const from0 = await brain.related({ from: entities[0] }) + expect(from0.some((r) => r.to === entities[1])).toBe(true) + const from1 = await brain.related({ from: entities[1] }) + expect(from1.some((r) => r.to === entities[2])).toBe(true) }) }) @@ -487,8 +533,10 @@ describe('Brainy Batch Operations', () => { expect(result.successful).toHaveLength(0) await brain.updateMany({ items: [] }) - await brain.removeMany({ ids: [] }) - // Should not throw + // removeMany is the exception (8.8.2): an empty id list is a refused + // selector, not an empty batch — deleting "nothing" silently was the + // bug class (a positional/bare-array call looked identical). + await expect(brain.removeMany({ ids: [] })).rejects.toThrow(/ids: \[\]/) }) it('should validate batch size limits', async () => { diff --git a/tests/unit/brainy/brain-format-handshake.test.ts b/tests/unit/brainy/brain-format-handshake.test.ts new file mode 100644 index 00000000..e5fce73d --- /dev/null +++ b/tests/unit/brainy/brain-format-handshake.test.ts @@ -0,0 +1,243 @@ +/** + * @module brain-format-handshake.test + * @description Tests for the 7.x → 8.0 version-handshake marker (GA #30): the + * `_system/brain-format.json` surface cor reads to drive whole-brain + * auto-upgrade, and Brainy's own JS-index rebuild-on-format-drift trigger. + * + * The contract (converged with the cor team — see `src/storage/brainFormat.ts`): + * + * - `formatInfo()` is a SYNC accessor returning the running build's + * `{ dataFormat, indexEpoch }` (the compiled constants). + * - On open, an on-disk `indexEpoch` that differs from `EXPECTED_INDEX_EPOCH`, + * or an absent marker, means the derived JS indexes are stale and must be + * rebuilt from the canonical records, then the marker is re-stamped — but + * only AFTER the rebuild verifies (non-destructive: a crash mid-rebuild + * leaves the old / absent marker, so the next open idempotently re-rebuilds). + * + * Persistence note: the derived JS indexes are in-memory and cold-load via + * `rebuild()` on every reopen, so "did a rebuild happen" is not, on its own, a + * signal that the EPOCH triggered it. The epoch-force (rebuild past the warm + * `size()>0` fast path) is therefore proven by a focused white-box test; the + * reopen tests assert the observable handshake effects — the marker content, + * whether it was re-stamped, and the rebuild-before-stamp ordering. + */ + +import { describe, it, expect, afterEach, vi } from 'vitest' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { Brainy } from '../../../src/index.js' +import { NounType } from '../../../src/types/graphTypes.js' +import { BaseStorage } from '../../../src/storage/baseStorage.js' +import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js' +import { MetadataIndexManager } from '../../../src/utils/metadataIndex.js' +import { + BRAIN_FORMAT_PATH, + CURRENT_DATA_FORMAT, + EXPECTED_INDEX_EPOCH +} from '../../../src/storage/brainFormat.js' + +const CURRENT_MARKER = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH } + +const tempDirs: string[] = [] +const brains: Brainy[] = [] + +function makeTempDir(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-format-handshake-')) + tempDirs.push(dir) + return dir +} + +/** Open (and track) a filesystem brain rooted at `dir`. */ +async function openFsBrain(dir: string): Promise { + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await brain.init() + brains.push(brain) + return brain +} + +/** Read the on-disk marker through a fresh raw adapter (after the brain closed). */ +async function readMarkerFromDisk(dir: string): Promise { + const adapter = new FileSystemStorage(dir) + await adapter.init() + return adapter.readRawObject(BRAIN_FORMAT_PATH) +} + +/** Overwrite (or delete, when `marker` is null) the on-disk marker between reopens. */ +async function seedMarkerOnDisk(dir: string, marker: unknown | null): Promise { + const adapter = new FileSystemStorage(dir) + await adapter.init() + if (marker === null) { + await adapter.deleteRawObject(BRAIN_FORMAT_PATH) + } else { + await adapter.writeRawObject(BRAIN_FORMAT_PATH, marker) + } +} + +afterEach(async () => { + for (const brain of brains.splice(0)) { + try { + await brain.close() + } catch { + // already closed by the test + } + } + vi.restoreAllMocks() + for (const dir of tempDirs.splice(0)) { + await fs.promises.rm(dir, { recursive: true, force: true }) + } +}) + +describe('8.0 ⇄ cor version-handshake marker (_system/brain-format.json)', () => { + // (a) Fresh brain → formatInfo() is the current format; the marker is written. + it('(a) fresh brain: formatInfo() reports the current format and writes the marker', async () => { + const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + brains.push(brain) + + expect(brain.formatInfo()).toEqual(CURRENT_MARKER) + + const onDisk = await (brain as unknown as { + storage: { readRawObject(p: string): Promise } + }).storage.readRawObject(BRAIN_FORMAT_PATH) + expect(onDisk).toEqual(CURRENT_MARKER) + }) + + // (b) Reopen a current-epoch brain → no re-stamp; formatInfo() consistent. + it('(b) reopen at the current epoch: no re-stamp, formatInfo() stays consistent', async () => { + const dir = makeTempDir() + const first = await openFsBrain(dir) + await first.add({ data: 'x', type: NounType.Document, metadata: { k: 1 } }) + await first.close() + brains.splice(brains.indexOf(first), 1) + + // The on-disk marker is present and current; reopening must NOT rewrite it. + const writeSpy = vi.spyOn(BaseStorage.prototype, 'writeRawObject') + const second = await openFsBrain(dir) + + const stampWrites = writeSpy.mock.calls.filter(([p]) => p === BRAIN_FORMAT_PATH) + expect(stampWrites.length).toBe(0) + expect((second as unknown as { _indexEpochStale: boolean })._indexEpochStale).toBe(false) + expect(second.formatInfo()).toEqual(CURRENT_MARKER) + }) + + // (c) Stale on-disk epoch → derived-index rebuild, THEN re-stamp (rebuild before stamp). + it('(c) stale on-disk epoch: rebuilds the derived indexes, then re-stamps (rebuild BEFORE stamp)', async () => { + const dir = makeTempDir() + const first = await openFsBrain(dir) + await first.add({ data: 'x', type: NounType.Document, metadata: { k: 1 } }) + await first.close() + brains.splice(brains.indexOf(first), 1) + + // Simulate a brain written by an OLDER build: drift the epoch back to 0. + await seedMarkerOnDisk(dir, { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH - 1 }) + + // Spy AFTER the seed write so only the reopen's calls are observed. Both + // spies call through; vitest's `invocationCallOrder` is a single global + // counter, so cross-spy ordering is comparable. + const rebuildSpy = vi.spyOn(MetadataIndexManager.prototype, 'rebuild') + const writeSpy = vi.spyOn(BaseStorage.prototype, 'writeRawObject') + + const second = await openFsBrain(dir) + + // The marker was re-stamped exactly once, to the EXPECTED epoch. + const stampOrders = writeSpy.mock.calls + .map((call, i) => ({ path: call[0] as string, order: writeSpy.mock.invocationCallOrder[i] })) + .filter((c) => c.path === BRAIN_FORMAT_PATH) + expect(stampOrders.length).toBe(1) + expect((second as unknown as { _indexEpochStale: boolean })._indexEpochStale).toBe(false) + + // The derived index rebuilt, and the rebuild happened BEFORE the stamp + // (non-destructive ordering: the marker only advances once the rebuild that + // it certifies has run). + expect(rebuildSpy).toHaveBeenCalled() + const firstRebuildOrder = rebuildSpy.mock.invocationCallOrder[0] + expect(firstRebuildOrder).toBeLessThan(stampOrders[0].order) + + await second.close() + brains.splice(brains.indexOf(second), 1) + + // And the new marker is durable on disk at the EXPECTED epoch. + expect(await readMarkerFromDisk(dir)).toEqual(CURRENT_MARKER) + }) + + // (d) No marker (pre-handshake brain) → treated as stale → rebuild + stamp. + it('(d) no marker (pre-handshake brain): treated as stale → rebuilds and stamps the marker', async () => { + const dir = makeTempDir() + const first = await openFsBrain(dir) + await first.add({ data: 'x', type: NounType.Document, metadata: { k: 1 } }) + await first.close() + brains.splice(brains.indexOf(first), 1) + + // A brain written before the handshake existed has NO marker on disk. + await seedMarkerOnDisk(dir, null) + expect(await readMarkerFromDisk(dir)).toBeNull() + + const rebuildSpy = vi.spyOn(MetadataIndexManager.prototype, 'rebuild') + const writeSpy = vi.spyOn(BaseStorage.prototype, 'writeRawObject') + + const second = await openFsBrain(dir) + + expect(rebuildSpy).toHaveBeenCalled() + const stampWrites = writeSpy.mock.calls.filter(([p]) => p === BRAIN_FORMAT_PATH) + expect(stampWrites.length).toBe(1) + expect((second as unknown as { _indexEpochStale: boolean })._indexEpochStale).toBe(false) + + await second.close() + brains.splice(brains.indexOf(second), 1) + expect(await readMarkerFromDisk(dir)).toEqual(CURRENT_MARKER) + }) + + // The epoch-drift trigger itself: a format change forces a rebuild past the + // warm-index fast path (the gap this feature closes); a current epoch does not. + it('epoch drift forces a rebuild of all three indexes past the warm fast path; current epoch does not', async () => { + const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + brains.push(brain) + + // Warm the indexes (size() > 0) so the fast-path early return is live. + await brain.add({ data: 'a', type: NounType.Document, metadata: { k: 1 } }) + await brain.add({ data: 'b', type: NounType.Document, metadata: { k: 2 } }) + const internals = brain as unknown as { + index: { size(): number; rebuild(...a: unknown[]): Promise } + metadataIndex: { rebuild(...a: unknown[]): Promise } + graphIndex: { rebuild(...a: unknown[]): Promise } + _indexEpochStale: boolean + rebuildIndexesIfNeeded(force?: boolean): Promise + } + expect(internals.index.size()).toBeGreaterThan(0) + + const idxSpy = vi.spyOn(internals.index, 'rebuild').mockResolvedValue(undefined) + const miSpy = vi.spyOn(internals.metadataIndex, 'rebuild').mockResolvedValue(undefined) + const giSpy = vi.spyOn(internals.graphIndex, 'rebuild').mockResolvedValue(undefined) + + // Current epoch: the warm fast path skips every rebuild. + internals._indexEpochStale = false + await internals.rebuildIndexesIfNeeded() + expect(idxSpy).not.toHaveBeenCalled() + expect(miSpy).not.toHaveBeenCalled() + expect(giSpy).not.toHaveBeenCalled() + + // Drifted epoch: all three rebuild even though the indexes are non-empty. + internals._indexEpochStale = true + await internals.rebuildIndexesIfNeeded() + expect(idxSpy).toHaveBeenCalledTimes(1) + expect(miSpy).toHaveBeenCalledTimes(1) + expect(giSpy).toHaveBeenCalledTimes(1) + // The stamp cleared the stale flag after the verified rebuild. + expect(internals._indexEpochStale).toBe(false) + }) + + // (e) formatInfo() is synchronously available immediately after init (no await). + it('(e) formatInfo() is synchronously available immediately after init', async () => { + const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + brains.push(brain) + + // No await on the call itself — it is a pure in-memory constant read. + const info = brain.formatInfo() + expect(info).toEqual(CURRENT_MARKER) + expect(typeof info.indexEpoch).toBe('number') + expect(typeof info.dataFormat).toBe('string') + }) +}) diff --git a/tests/unit/brainy/degraded-reads-surfaced.test.ts b/tests/unit/brainy/degraded-reads-surfaced.test.ts new file mode 100644 index 00000000..29a8a77c --- /dev/null +++ b/tests/unit/brainy/degraded-reads-surfaced.test.ts @@ -0,0 +1,79 @@ +/** + * @module tests/unit/brainy/degraded-reads-surfaced + * @description Finding 10: a known-degraded derived index must be SURFACED, not + * silently served as authoritative. Two degraded sources: + * (a) a non-fatal index rebuild failure at init() (`_indexRebuildFailed`), and + * (b) an adopt-forward failed-rollback recovery (`commitSingleOp`'s `degraded` + * ids, previously dropped on the floor by persistSingleOp). + * Both now fold into checkHealth()/validateIndexConsistency() and emit ONE loud + * read-path warning per degraded window; repairIndex() reconciles + clears them. + * + * The private fields are the observable contract of the fix, so the test drives + * them directly (a real failed rollback is exercised elsewhere). + */ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/brainy.js' +import { NounType } from '../../../src/types/graphTypes.js' +import { prodLog } from '../../../src/utils/logger.js' + +const UUID = (suffix: string): string => `00000000-0000-4000-8000-0000000000${suffix}` + +describe('Finding 10 — degraded derived-index state is surfaced on reads', () => { + beforeEach(() => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + }) + afterEach(() => vi.restoreAllMocks()) + + it('checkHealth() reports adopt-forward degraded ids as unhealthy', async () => { + const brain = new Brainy({ storage: { type: 'memory' }, dimensions: 384, requireSubtype: false }) + await brain.init() + ;(brain as any)._indexDegradedIds.add(UUID('de')) + + const health = await brain.checkHealth() + expect(health.healthy).toBe(false) + expect(health.recommendation).toMatch(/repairIndex\(\)/) + }) + + it('find()/get() warn loudly while degraded, ONCE, then repairIndex() clears it', async () => { + const warn = vi.spyOn(prodLog, 'warn').mockImplementation(() => {}) + const brain = new Brainy({ storage: { type: 'memory' }, dimensions: 384, requireSubtype: false }) + await brain.init() + await brain.add({ id: UUID('a1'), data: 'x', type: NounType.Document }) + ;(brain as any)._indexRebuildFailed = new Error('rebuild boom') + + await brain.find({ type: NounType.Document }) + await brain.get(UUID('a1')) // second read: must NOT double-warn + const degradedWarns = warn.mock.calls.filter((c) => + String(c[0]).includes('derived index is INCOMPLETE') + ) + expect(degradedWarns.length).toBe(1) + + await brain.repairIndex() + expect((brain as any)._indexRebuildFailed).toBeNull() + expect((brain as any)._indexDegradedIds.size).toBe(0) + + warn.mockClear() + await brain.find({ type: NounType.Document }) + expect(warn.mock.calls.filter((c) => String(c[0]).includes('INCOMPLETE')).length).toBe(0) + }) + + it('persistSingleOp records receipt.degraded (widened return type, not dropped)', async () => { + const brain = new Brainy({ storage: { type: 'memory' }, dimensions: 384, requireSubtype: false }) + await brain.init() + // Simulate a degraded receipt by wrapping the generation store's commitSingleOp. + const gs: any = (brain as any).generationStore + const realCommit = gs.commitSingleOp.bind(gs) + vi.spyOn(gs, 'commitSingleOp').mockImplementation(async (args: any) => { + const r = await realCommit(args) + return { ...r, degraded: [...(args.touched.nouns ?? [])] } + }) + + await brain.add({ id: UUID('b2'), data: 'y', type: NounType.Document }) + expect((brain as any)._indexDegradedIds.size).toBeGreaterThan(0) + expect((brain as any)._indexDegradedIds.has(UUID('b2'))).toBe(true) + + const health = await brain.checkHealth() + expect(health.healthy).toBe(false) + expect(health.recommendation).toMatch(/repairIndex\(\)/) + }) +}) diff --git a/tests/unit/brainy/find-complement-operators.test.ts b/tests/unit/brainy/find-complement-operators.test.ts new file mode 100644 index 00000000..76fbb017 --- /dev/null +++ b/tests/unit/brainy/find-complement-operators.test.ts @@ -0,0 +1,60 @@ +/** + * @module tests/unit/brainy/find-complement-operators + * @description The negation/absence where-operators (`ne`, `exists:false`, + * `missing:true`) are served as a roaring-bitmap difference over the int-id + * universe (rather than materializing the whole corpus as UUID strings to + * filter). These tests pin the exact result set — including the load-bearing + * soft-delete semantic: `field !== value` MUST include entities that have no + * such field at all. + */ +import { describe, it, expect, beforeEach } from 'vitest' +import { Brainy } from '../../../src/brainy' +import { NounType } from '../../../src/types/graphTypes' + +describe('find() complement operators (ne / exists:false / missing:true)', () => { + let brain: Brainy + // Three entities WITH a `status` field, two WITHOUT it. + const ids: Record = {} + + beforeEach(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + ids.active1 = await brain.add({ data: 'a1', type: NounType.Thing, metadata: { status: 'active' } }) + ids.active2 = await brain.add({ data: 'a2', type: NounType.Thing, metadata: { status: 'active' } }) + ids.closed = await brain.add({ data: 'c', type: NounType.Thing, metadata: { status: 'closed' } }) + ids.noField1 = await brain.add({ data: 'n1', type: NounType.Thing, metadata: { other: 1 } }) + ids.noField2 = await brain.add({ data: 'n2', type: NounType.Thing, metadata: { other: 2 } }) + }) + + it('ne returns everything except the matching value — INCLUDING entities without the field', async () => { + const rows = await brain.find({ where: { status: { ne: 'active' } }, limit: 100 }) + const got = new Set(rows.map((r) => r.id)) + // closed (status≠active) + the two without a status field at all. + expect(got).toEqual(new Set([ids.closed, ids.noField1, ids.noField2])) + expect(got.has(ids.active1)).toBe(false) + expect(got.has(ids.active2)).toBe(false) + }) + + it('exists:false returns exactly the entities without the field', async () => { + const rows = await brain.find({ where: { status: { exists: false } }, limit: 100 }) + expect(new Set(rows.map((r) => r.id))).toEqual(new Set([ids.noField1, ids.noField2])) + }) + + it('missing:true matches exists:false (entities without the field)', async () => { + const rows = await brain.find({ where: { status: { missing: true } }, limit: 100 }) + expect(new Set(rows.map((r) => r.id))).toEqual(new Set([ids.noField1, ids.noField2])) + }) + + it('exists:true returns exactly the entities with the field', async () => { + const rows = await brain.find({ where: { status: { exists: true } }, limit: 100 }) + expect(new Set(rows.map((r) => r.id))).toEqual(new Set([ids.active1, ids.active2, ids.closed])) + }) + + it('the complement reflects deletes (a removed entity drops out of ne / exists:false)', async () => { + await brain.remove(ids.noField1) + const ne = await brain.find({ where: { status: { ne: 'active' } }, limit: 100 }) + expect(new Set(ne.map((r) => r.id))).toEqual(new Set([ids.closed, ids.noField2])) + const absent = await brain.find({ where: { status: { exists: false } }, limit: 100 }) + expect(new Set(absent.map((r) => r.id))).toEqual(new Set([ids.noField2])) + }) +}) diff --git a/tests/unit/brainy/find-include-vectors.test.ts b/tests/unit/brainy/find-include-vectors.test.ts new file mode 100644 index 00000000..38216692 --- /dev/null +++ b/tests/unit/brainy/find-include-vectors.test.ts @@ -0,0 +1,94 @@ +/** + * Unit tests for FindParams.includeVectors — opt-in vector hydration on find(). + * + * Verifies that find({ includeVectors: true }) returns each result's stored + * embedding under entity.vector (matching get({ includeVectors: true })), that + * the default keeps the perf contract of an empty vector, and that the + * metadata-only find({ where }) path honors the flag too. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/brainy' +import { createTestConfig } from '../../helpers/test-factory' + +describe('FindParams.includeVectors', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + it('returns the stored vector for query-based find when includeVectors is true', async () => { + const id = await brain.add({ + type: 'document', + data: 'machine learning and neural networks', + metadata: { topic: 'ai' } + }) + + // Stored vector, as get() surfaces it — the reference for the find() result. + const stored = await brain.get(id, { includeVectors: true }) + expect(stored!.vector.length).toBeGreaterThan(0) + + const results = await brain.find({ + query: 'machine learning and neural networks', + includeVectors: true, + limit: 10 + }) + + const hit = results.find((r) => r.entity.id === id) + expect(hit).toBeDefined() + expect(hit!.entity.vector.length).toBeGreaterThan(0) + expect(hit!.entity.vector).toEqual(stored!.vector) + }) + + it('returns an empty vector by default (perf contract preserved)', async () => { + const id = await brain.add({ + type: 'document', + data: 'default path should not hydrate vectors', + metadata: { topic: 'perf' } + }) + + const results = await brain.find({ + query: 'default path should not hydrate vectors', + limit: 10 + }) + + const hit = results.find((r) => r.entity.id === id) + expect(hit).toBeDefined() + expect(hit!.entity.vector.length).toBe(0) + }) + + it('honors includeVectors on the metadata-only where path', async () => { + const id = await brain.add({ + type: 'thing', + data: 'metadata only filter target', + metadata: { category: 'filtered' } + }) + + const stored = await brain.get(id, { includeVectors: true }) + + const withVectors = await brain.find({ + where: { category: 'filtered' }, + includeVectors: true, + limit: 10 + }) + const hit = withVectors.find((r) => r.entity.id === id) + expect(hit).toBeDefined() + expect(hit!.entity.vector.length).toBeGreaterThan(0) + expect(hit!.entity.vector).toEqual(stored!.vector) + + // And the same filter without the flag stays empty. + const withoutVectors = await brain.find({ + where: { category: 'filtered' }, + limit: 10 + }) + const plainHit = withoutVectors.find((r) => r.entity.id === id) + expect(plainHit).toBeDefined() + expect(plainHit!.entity.vector.length).toBe(0) + }) +}) diff --git a/tests/unit/brainy/find-index-integrity-guard.test.ts b/tests/unit/brainy/find-index-integrity-guard.test.ts new file mode 100644 index 00000000..30cfdf1b --- /dev/null +++ b/tests/unit/brainy/find-index-integrity-guard.test.ts @@ -0,0 +1,101 @@ +/** + * @module find-index-integrity-guard.test + * @description Regression for the "phantom row" class (Venue 2026-06-24): a + * corrupt/stale metadata-index posting must never surface an entity that does + * not actually match the find() predicate. + * + * The metadata index is an acceleration structure; the loaded entity is ground + * truth. find() re-validates every result against `type`/`subtype`/`where`/ + * `service` at a single egress chokepoint (the live-path mirror of the + * historical path's `entityMatchesFind`). These tests inject a poisoned + * `getIdsForFilter` (the exact failure mode seen in production: a native index + * returns an id whose record matches NEITHER the type nor the where filter) and + * assert the phantom is dropped while the genuine matches survive. + */ +import { describe, it, expect, beforeEach } from 'vitest' +import { Brainy } from '../../../src/brainy' +import { NounType } from '../../../src/types/graphTypes' + +describe('find() index-integrity guard (phantom row class)', () => { + let brain: Brainy + let staffId: string + let timeslotId: string + let customerId: string + + beforeEach(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + + // A real staff Person (the only legitimate match for the roster query). + staffId = await brain.add({ + data: 'Kristine Perry', + type: NounType.Person, + metadata: { entityType: 'staff', firstName: 'Kristine', lastName: 'Perry' } + }) + // A timeslot Event — matches NEITHER type:Person NOR where.entityType:'staff' + // (the production phantom was exactly this: a NounType.Event timeslot). + timeslotId = await brain.add({ + data: 'kitten-experience 2026-08-19 12:00', + type: NounType.Event, + metadata: { entityType: 'timeslot', date: '2026-08-19', startTime: '12:00' } + }) + // A Person that matches type:Person but NOT where.entityType:'staff' — proves + // the WHERE leg of the guard, not just the structural type leg. + customerId = await brain.add({ + data: 'Walk-in Customer', + type: NounType.Person, + metadata: { entityType: 'customer', firstName: 'Walk-in' } + }) + }) + + it('healthy index: the discriminant query returns only the staff Person', async () => { + const rows = await brain.find({ type: NounType.Person, where: { entityType: 'staff' }, limit: 100 }) + expect(rows.map((r) => r.id)).toEqual([staffId]) + }) + + it('drops a phantom of the WRONG TYPE that a corrupt index leaks into the where bucket', async () => { + // Poison the metadata index so it returns the timeslot's id for the staff + // query — the precise production corruption (a cross-bucket/stale posting). + const mi = (brain as any).metadataIndex + const original = mi.getIdsForFilter.bind(mi) + mi.getIdsForFilter = async (filter: any): Promise => { + const ids = await original(filter) + return ids.includes(timeslotId) ? ids : [...ids, timeslotId] + } + + const rows = await brain.find({ type: NounType.Person, where: { entityType: 'staff' }, limit: 100 }) + + // The timeslot (NounType.Event, entityType:'timeslot') must NOT appear. + expect(rows.map((r) => r.id)).not.toContain(timeslotId) + // The genuine staff row survives. + expect(rows.map((r) => r.id)).toEqual([staffId]) + + mi.getIdsForFilter = original + }) + + it('drops a phantom of the RIGHT TYPE but WRONG where-value (exercises the where leg)', async () => { + // The customer is a Person (passes type) but entityType:'customer' (fails + // where). A corrupt index leaking it into the 'staff' bucket must still be + // rejected by the where re-validation. + const mi = (brain as any).metadataIndex + const original = mi.getIdsForFilter.bind(mi) + mi.getIdsForFilter = async (filter: any): Promise => { + const ids = await original(filter) + return ids.includes(customerId) ? ids : [...ids, customerId] + } + + const rows = await brain.find({ type: NounType.Person, where: { entityType: 'staff' }, limit: 100 }) + + expect(rows.map((r) => r.id)).not.toContain(customerId) + expect(rows.map((r) => r.id)).toEqual([staffId]) + + mi.getIdsForFilter = original + }) + + it('a healthy query for the phantom\'s own bucket still returns it (guard is not over-eager)', async () => { + // Sanity: the guard must only drop genuine non-matches. Querying the + // timeslot's real bucket returns it normally. + const rows = await brain.find({ type: NounType.Event, where: { entityType: 'timeslot' }, limit: 100 }) + expect(rows.map((r) => r.id)).toEqual([timeslotId]) + }) +}) diff --git a/tests/unit/brainy/find-orderby-pagek.test.ts b/tests/unit/brainy/find-orderby-pagek.test.ts new file mode 100644 index 00000000..9a453f8d --- /dev/null +++ b/tests/unit/brainy/find-orderby-pagek.test.ts @@ -0,0 +1,60 @@ +/** + * @module tests/unit/brainy/find-orderby-pagek + * @description CTX-BR-FIND-ORDERBY item (1) — `find({ where, orderBy, limit })` must + * produce only the requested PAGE of sorted ids, not the full sorted match set. At + * billion scale a broad filter + orderBy returning 20 rows previously materialized + * every matching sorted id (O(matches) heap); the page bound (`offset+limit`) is now + * threaded into `getSortedIdsForFilter` → the column store's top-K heap. Ordering + * correctness is covered by orderby-sort-bug.test.ts; this locks the page bound. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/index.js' +import { NounType } from '../../../src/types/graphTypes.js' +import { createTestConfig } from '../../helpers/test-factory.js' + +describe('find({ where, orderBy }) bounds the sort to the page (CTX-BR-FIND-ORDERBY #1)', () => { + let brain: Brainy + const MATCHES = 50 + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + for (let i = 0; i < MATCHES; i++) { + await brain.add({ type: NounType.Document, data: `doc ${i}`, metadata: { bucket: 'x', seq: i } }) + } + }) + + afterEach(async () => { + await brain.close() + }) + + it('threads the page bound (offset+limit) into getSortedIdsForFilter, not the full match count', async () => { + let capturedTopK: number | undefined + const real = (brain as any).metadataIndex.getSortedIdsForFilter.bind((brain as any).metadataIndex) + ;(brain as any).metadataIndex.getSortedIdsForFilter = async ( + f: any, + ob: string, + o: 'asc' | 'desc', + topK?: number + ) => { + capturedTopK = topK + return real(f, ob, o, topK) + } + + const results = await brain.find({ where: { bucket: 'x' }, orderBy: 'createdAt', order: 'desc', limit: 5 }) + + expect(results).toHaveLength(5) + // Page-bounded: ~ limit (5) + a small hidden-tier over-fetch — NOT all 50 matches. + expect(capturedTopK).toBeDefined() + expect(capturedTopK!).toBeGreaterThanOrEqual(5) + expect(capturedTopK!).toBeLessThan(MATCHES) + }) + + it('still returns the correct page with offset (sort + window unchanged)', async () => { + // seq is a numeric metadata field; orderBy seq asc → 0,1,2,… ; page [10,15). + const page = await brain.find({ where: { bucket: 'x' }, orderBy: 'seq', order: 'asc', limit: 5, offset: 10 }) + expect(page).toHaveLength(5) + expect(page.map((r) => (r.metadata as { seq: number }).seq)).toEqual([10, 11, 12, 13, 14]) + }) +}) diff --git a/tests/unit/brainy/graph-analytics.test.ts b/tests/unit/brainy/graph-analytics.test.ts new file mode 100644 index 00000000..bfec3347 --- /dev/null +++ b/tests/unit/brainy/graph-analytics.test.ts @@ -0,0 +1,211 @@ +/** + * @module tests/unit/brainy/graph-analytics + * @description Graph analytics surface — `brain.graph.rank()` (PageRank importance), + * `brain.graph.communities()` (connected-component grouping), and + * `brain.graph.path()` (best route). These exercise the pure-TS fallbacks (no native + * GraphAccelerationProvider is registered in CI); a native provider answers the same + * INTENT (which nodes matter most / which things group / best route) and returns the + * same public shapes, cross-layer-tested against the provider. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/index.js' +import { NounType, VerbType } from '../../../src/types/graphTypes.js' +import { createTestConfig } from '../../helpers/test-factory.js' + +describe('brain.graph.communities()', () => { + let brain: Brainy + let a: string, b: string, c: string, d: string, e: string, f: string + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + // Two disjoint clusters + one isolated node: + // cluster 1: a → b → c + // cluster 2: d → e + // isolated: f (no edges) + a = await brain.add({ type: NounType.Person, data: 'A' }) + b = await brain.add({ type: NounType.Person, data: 'B' }) + c = await brain.add({ type: NounType.Person, data: 'C' }) + d = await brain.add({ type: NounType.Person, data: 'D' }) + e = await brain.add({ type: NounType.Person, data: 'E' }) + f = await brain.add({ type: NounType.Person, data: 'F' }) + await brain.relate({ from: a, to: b, type: VerbType.RelatedTo }) + await brain.relate({ from: b, to: c, type: VerbType.RelatedTo }) + await brain.relate({ from: d, to: e, type: VerbType.RelatedTo }) + }) + + afterEach(async () => { + await brain.close() + }) + + it('partitions the graph into connected groups (isolated nodes are singletons)', async () => { + const { groups, count } = await brain.graph.communities() + expect(count).toBe(3) + const asSets = groups.map((g) => new Set(g)) + expect(asSets).toContainEqual(new Set([a, b, c])) + expect(asSets).toContainEqual(new Set([d, e])) + expect(asSets).toContainEqual(new Set([f])) + }) + + it('orders groups largest-first', async () => { + const { groups } = await brain.graph.communities() + for (let i = 1; i < groups.length; i++) { + expect(groups[i - 1].length).toBeGreaterThanOrEqual(groups[i].length) + } + expect(groups[0]).toHaveLength(3) // the a-b-c cluster + }) + + it('directed mode groups by strong connectivity (a→b→c is NOT mutually reachable)', async () => { + // Undirected: {a,b,c} is one group. Directed: no cycle, so each is its own SCC. + const undirected = await brain.graph.communities() + expect(undirected.groups).toContainEqual(expect.arrayContaining([a, b, c])) + + const directed = await brain.graph.communities({ directed: true }) + // a,b,c split into singletons; d,e split too; f stays singleton → 6 groups. + expect(directed.count).toBe(6) + expect(directed.groups.every((g) => g.length === 1)).toBe(true) + }) + + it('directed mode keeps a cycle together as one strongly-connected community', async () => { + // x → y → z → x is a cycle: mutually reachable → one SCC even when directed. + const x = await brain.add({ type: NounType.Concept, data: 'X' }) + const y = await brain.add({ type: NounType.Concept, data: 'Y' }) + const z = await brain.add({ type: NounType.Concept, data: 'Z' }) + await brain.relate({ from: x, to: y, type: VerbType.RelatedTo }) + await brain.relate({ from: y, to: z, type: VerbType.RelatedTo }) + await brain.relate({ from: z, to: x, type: VerbType.RelatedTo }) + + const directed = await brain.graph.communities({ directed: true }) + const asSets = directed.groups.map((g) => new Set(g)) + expect(asSets).toContainEqual(new Set([x, y, z])) + }) + + it('excludes internal-visibility edges by default; includeInternal re-links', async () => { + // A hidden edge bridging the two clusters is invisible by default. + await brain.relate({ from: c, to: d, type: VerbType.RelatedTo, visibility: 'internal' }) + + const hidden = await brain.graph.communities() + expect(hidden.count).toBe(3) // bridge hidden → still 3 groups + + const surfaced = await brain.graph.communities({ includeInternal: true }) + const asSets = surfaced.groups.map((g) => new Set(g)) + expect(asSets).toContainEqual(new Set([a, b, c, d, e])) // bridge merges the clusters + }) +}) + +describe('brain.graph.rank()', () => { + let brain: Brainy + let hub: string, a: string, b: string, c: string + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + // a, b, c all point at hub → hub is the most "important" node. + hub = await brain.add({ type: NounType.Person, data: 'HUB' }) + a = await brain.add({ type: NounType.Person, data: 'A' }) + b = await brain.add({ type: NounType.Person, data: 'B' }) + c = await brain.add({ type: NounType.Person, data: 'C' }) + await brain.relate({ from: a, to: hub, type: VerbType.RelatedTo }) + await brain.relate({ from: b, to: hub, type: VerbType.RelatedTo }) + await brain.relate({ from: c, to: hub, type: VerbType.RelatedTo }) + }) + + afterEach(async () => { + await brain.close() + }) + + it('ranks the most-pointed-to node highest, descending', async () => { + const ranked = await brain.graph.rank() + expect(ranked).toHaveLength(4) + expect(ranked[0].id).toBe(hub) + for (let i = 1; i < ranked.length; i++) { + expect(ranked[i - 1].score).toBeGreaterThanOrEqual(ranked[i].score) + } + }) + + it('scores form a probability distribution (PageRank sums to ~1)', async () => { + const ranked = await brain.graph.rank() + const total = ranked.reduce((sum, r) => sum + r.score, 0) + expect(total).toBeCloseTo(1, 5) + }) + + it('topK returns only the K highest', async () => { + const top1 = await brain.graph.rank({ topK: 1 }) + expect(top1).toHaveLength(1) + expect(top1[0].id).toBe(hub) + }) + + it('returns [] on an empty graph', async () => { + const empty = new Brainy(createTestConfig()) + await empty.init() + expect(await empty.graph.rank()).toEqual([]) + await empty.close() + }) +}) + +describe('brain.graph.path()', () => { + let brain: Brainy + let a: string, b: string, c: string, d: string, isolated: string + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + // Chain a → b → c → d (light edges), plus a heavy direct shortcut a → d. + // weight is a 0–1 connection strength; `by:'weight'` minimizes summed weight. + a = await brain.add({ type: NounType.Person, data: 'A' }) + b = await brain.add({ type: NounType.Person, data: 'B' }) + c = await brain.add({ type: NounType.Person, data: 'C' }) + d = await brain.add({ type: NounType.Person, data: 'D' }) + isolated = await brain.add({ type: NounType.Person, data: 'ISO' }) + await brain.relate({ from: a, to: b, type: VerbType.RelatedTo, weight: 0.1 }) + await brain.relate({ from: b, to: c, type: VerbType.RelatedTo, weight: 0.1 }) + await brain.relate({ from: c, to: d, type: VerbType.RelatedTo, weight: 0.1 }) + await brain.relate({ from: a, to: d, type: VerbType.RelatedTo, weight: 0.9 }) // direct but heavy + }) + + afterEach(async () => { + await brain.close() + }) + + it('finds the fewest-hops route by default', async () => { + const route = await brain.graph.path(a, d) + expect(route).not.toBeNull() + // a → d direct edge is 1 hop, the shortest. + expect(route?.nodes).toEqual([a, d]) + expect(route?.relationships).toHaveLength(1) + expect(route?.cost).toBe(1) + }) + + it('by:"weight" prefers the lighter multi-hop route over the heavy shortcut', async () => { + const route = await brain.graph.path(a, d, { by: 'weight' }) + expect(route?.nodes).toEqual([a, b, c, d]) // 0.1+0.1+0.1 = 0.3 < 0.9 + expect(route?.relationships).toHaveLength(3) + expect(route?.cost).toBeCloseTo(0.3, 6) + }) + + it('returns a zero-length route from a node to itself', async () => { + const route = await brain.graph.path(a, a) + expect(route).toEqual({ nodes: [a], relationships: [], cost: 0 }) + }) + + it('returns null when the target is unreachable', async () => { + expect(await brain.graph.path(a, isolated)).toBeNull() + }) + + it('direction:"out" cannot walk backwards up the chain', async () => { + // d has no outgoing edges, so d → a is unreachable following only out-edges. + expect(await brain.graph.path(d, a, { direction: 'out' })).toBeNull() + // …but 'both' finds it. + const both = await brain.graph.path(d, a, { direction: 'both' }) + expect(both?.nodes[0]).toBe(d) + expect(both?.nodes[both.nodes.length - 1]).toBe(a) + }) + + it('maxDepth abandons routes that are too long', async () => { + // The light route is 3 hops; cap at 1 hop → only the heavy direct shortcut. + const capped = await brain.graph.path(a, d, { by: 'weight', maxDepth: 1 }) + expect(capped?.nodes).toEqual([a, d]) + expect(capped?.cost).toBeCloseTo(0.9, 6) + }) +}) diff --git a/tests/unit/brainy/graph-export.test.ts b/tests/unit/brainy/graph-export.test.ts new file mode 100644 index 00000000..73d66c6a --- /dev/null +++ b/tests/unit/brainy/graph-export.test.ts @@ -0,0 +1,97 @@ +/** + * @module tests/unit/brainy/graph-export + * @description Graph engine #22: `brain.graph.export()` — stream the whole graph + * in one O(N+E) pass (the right primitive for visualizing all data, vs. paging + * per node). Exercises the pure-TS fallback (no native provider in CI), which + * streams all nouns then all verbs via cursor pagination. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/index.js' +import { NounType, VerbType } from '../../../src/types/graphTypes.js' +import { createTestConfig } from '../../helpers/test-factory.js' +import type { GraphView } from '../../../src/index.js' + +async function collect(stream: AsyncIterable): Promise<{ + nodes: Map + edges: Map + chunks: number +}> { + const nodes = new Map() + const edges = new Map() + let chunks = 0 + for await (const chunk of stream) { + chunks++ + for (const n of chunk.nodes) nodes.set(n.id, n) + for (const e of chunk.edges) edges.set(e.id, e) + } + return { nodes, edges, chunks } +} + +describe('brain.graph.export() (graph engine #22)', () => { + let brain: Brainy + let a: string, b: string, c: string, iso: string + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + a = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'A' }) + b = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'B' }) + c = await brain.add({ type: NounType.Project, subtype: 'milestone', data: 'C' }) + iso = await brain.add({ type: NounType.Concept, subtype: 'general', data: 'isolated' }) + await brain.relate({ from: a, to: b, type: VerbType.RelatedTo, subtype: 'colleague' }) + await brain.relate({ from: b, to: c, type: VerbType.ParticipatesIn, subtype: 'assignment' }) + }) + + afterEach(async () => { + await brain.close() + }) + + it('streams the WHOLE graph — every node (incl. isolated) and every edge', async () => { + const { nodes, edges } = await collect(brain.graph.export()) + // All four nouns, including the isolated one (the node stream catches it). + expect(new Set(nodes.keys())).toEqual(new Set([a, b, c, iso])) + // Node refs carry type/subtype straight from the noun records. + expect(nodes.get(c)?.type).toBe(NounType.Project) + expect(nodes.get(c)?.subtype).toBe('milestone') + // Both edges. + const pairs = [...edges.values()].map((e) => `${e.from}->${e.to}`).sort() + expect(pairs).toEqual([`${a}->${b}`, `${b}->${c}`].sort()) + }) + + it('excludes internal-visibility nodes/edges by default; includeInternal surfaces them', async () => { + const hidden = await brain.add({ + type: NounType.Person, + subtype: 'employee', + data: 'hidden', + visibility: 'internal' + }) + await brain.relate({ from: a, to: hidden, type: VerbType.RelatedTo, subtype: 'colleague', visibility: 'internal' }) + + const visible = await collect(brain.graph.export()) + expect(visible.nodes.has(hidden)).toBe(false) + expect([...visible.edges.values()].some((e) => e.to === hidden)).toBe(false) + + const all = await collect(brain.graph.export({ includeInternal: true })) + expect(all.nodes.has(hidden)).toBe(true) + expect([...all.edges.values()].some((e) => e.to === hidden)).toBe(true) + }) + + it('includeNodes:false streams only edges; includeEdges:false streams only nodes', async () => { + const edgesOnly = await collect(brain.graph.export({ includeNodes: false })) + expect(edgesOnly.nodes.size).toBe(0) + expect(edgesOnly.edges.size).toBe(2) + + const nodesOnly = await collect(brain.graph.export({ includeEdges: false })) + expect(nodesOnly.edges.size).toBe(0) + expect(nodesOnly.nodes.size).toBe(4) + }) + + it('chunks the stream by chunkSize (multiple passes, same total)', async () => { + const { nodes, edges, chunks } = await collect(brain.graph.export({ chunkSize: 1 })) + // 4 nodes + 2 edges at 1 per chunk → several chunks. + expect(chunks).toBeGreaterThan(1) + expect(nodes.size).toBe(4) + expect(edges.size).toBe(2) + }) +}) diff --git a/tests/unit/brainy/graph-native-routing.test.ts b/tests/unit/brainy/graph-native-routing.test.ts new file mode 100644 index 00000000..dfa4912f --- /dev/null +++ b/tests/unit/brainy/graph-native-routing.test.ts @@ -0,0 +1,334 @@ +/** + * @module tests/unit/brainy/graph-native-routing + * @description Graph engine — NATIVE seam coverage. brain.graph.subgraph/export + * route to a registered GraphAccelerationProvider and hydrate its columnar + * `Subgraph` (node ints -> ids, depth alignment, edge verb-ints -> Relations). + * In production that provider is cor's native engine, cross-layer-tested on + * bxl9000; brainy CI never registers one, so graphSubgraphNative / + * graphExportNative / hydrateNativeSubgraph + the provider-resolution + routing + * were previously UNEXERCISED — a return-shape or hydration-alignment drift would + * pass CI silently. This registers a faithful MOCK provider (returning a columnar + * Subgraph built from the brain's REAL ints, so hydration resolves to real + * entities/relations) to lock those paths. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/index.js' +import { NounType, VerbType } from '../../../src/types/graphTypes.js' +import { createTestConfig } from '../../helpers/test-factory.js' + +/** A stateful mock GraphAccelerationProvider; the test sets the columnar payloads. */ +function makeMockAccel(opts: { isInitialized?: boolean } = {}) { + const calls = { traverse: 0, cursorOpen: 0, cursorNext: 0, cursorClose: 0, rank: 0, communities: 0, path: 0 } + const state: { + calls: typeof calls + traverseResult: any + cursorChunks: any[] + rankResult: any + communitiesResult: any + pathResult: any + lastTraverseSeeds: any + } = { + calls, + traverseResult: null, + cursorChunks: [], + rankResult: null, + communitiesResult: null, + pathResult: null, + lastTraverseSeeds: undefined + } + const empty = { nodeInts: new BigInt64Array(0), scores: new Float64Array(0) } + const emptyCommunities = { + nodeInts: new BigInt64Array(0), + communityIds: new Uint32Array(0), + communityCount: 0 + } + const provider = { + isInitialized: opts.isInitialized ?? true, + traverse: async (seeds: any) => { + calls.traverse++ + state.lastTraverseSeeds = seeds + return state.traverseResult + }, + edgesForNode: async () => state.traverseResult, + graphCursorOpen: async () => { + calls.cursorOpen++ + return 'mock-handle' + }, + graphCursorNext: async () => { + calls.cursorNext++ + const subgraph = state.cursorChunks.shift() + return { subgraph, done: state.cursorChunks.length === 0 } + }, + graphCursorClose: async () => { + calls.cursorClose++ + }, + rank: async () => { + calls.rank++ + return state.rankResult ?? empty + }, + communities: async () => { + calls.communities++ + return state.communitiesResult ?? emptyCommunities + }, + path: async () => { + calls.path++ + return state.pathResult + }, + sample: async () => state.traverseResult, + mostConnected: async () => empty + } + return { provider, state } +} + +describe('brain.graph.* native routing + columnar hydration (native seam)', () => { + let brain: Brainy + let mock: ReturnType + let a: string, b: string, c: string + + // Build a faithful columnar Subgraph from the brain's REAL ints, so brainy's + // hydration resolves the node ints to ids and the verb ints to Relations. + async function realSubgraph(withUnresolvable = false) { + const gei = (id: string): bigint => (brain as any).graphEntityInt(id) + const gi = (brain as any).graphIndex + const intA = gei(a), intB = gei(b), intC = gei(c) + const vAB = (await gi.getVerbIdsBySource(intA))[0] as bigint // verb int a->b + const vBC = (await gi.getVerbIdsBySource(intB))[0] as bigint // verb int b->c + const nodeInts = [intA, intB, intC] + const depths = [0, 1, 2] + if (withUnresolvable) { + nodeInts.push(999_999_999n) // a never-assigned int (simulates a deleted/unknown node) + depths.push(3) + } + return { + nodes: BigInt64Array.from(nodeInts), + nodeDepth: Uint8Array.from(depths), + edgeSources: BigInt64Array.from([intA, intB]), + edgeTargets: BigInt64Array.from([intB, intC]), + edgeVerbInts: BigInt64Array.from([vAB, vBC]), + edgeTypes: Uint16Array.from([0, 0]), + truncated: false + } + } + + beforeEach(async () => { + mock = makeMockAccel() + brain = new Brainy(createTestConfig()) + brain.use({ + name: 'mock-graph-accel', + activate: async (ctx: any) => { + ctx.registerProvider('graphAcceleration', mock.provider) + return true + } + } as any) + await brain.init() + a = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'A' }) + b = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'B' }) + c = await brain.add({ type: NounType.Project, subtype: 'milestone', data: 'C' }) + await brain.relate({ from: a, to: b, type: VerbType.RelatedTo, subtype: 'colleague' }) + await brain.relate({ from: b, to: c, type: VerbType.ParticipatesIn, subtype: 'assignment' }) + }) + + afterEach(async () => { + await brain.close() + }) + + it('subgraph() routes to the native provider and hydrates the columnar result', async () => { + mock.state.traverseResult = await realSubgraph() + const view = await brain.graph.subgraph(a, { depth: 2 }) + + expect(mock.state.calls.traverse).toBe(1) // native path, not the TS fallback + const byId = new Map(view.nodes.map((n) => [n.id, n])) + expect(new Set(byId.keys())).toEqual(new Set([a, b, c])) // node int -> id + expect(byId.get(a)?.depth).toBe(0) + expect(byId.get(c)?.depth).toBe(2) // depth column aligned to node column + expect(byId.get(c)?.type).toBe(NounType.Project) // node type hydrated via batchGet + const pairs = view.edges.map((e) => `${e.from}->${e.to}`).sort() + expect(pairs).toEqual([`${a}->${b}`, `${b}->${c}`].sort()) // verb int -> Relation + }) + + it('keeps node<->depth alignment when a node int does not resolve (deleted/unknown)', async () => { + mock.state.traverseResult = await realSubgraph(true) // appends an unresolvable int at depth 3 + const view = await brain.graph.subgraph(a, { depth: 3 }) + + const byId = new Map(view.nodes.map((n) => [n.id, n])) + // The 3 real nodes keep their CORRECT depths — the unresolvable int is dropped, + // not collapsed into the array (which would shift every later depth). + expect(byId.get(a)?.depth).toBe(0) + expect(byId.get(b)?.depth).toBe(1) + expect(byId.get(c)?.depth).toBe(2) + expect(view.nodes.length).toBe(3) + }) + + it('export() routes to the native graph cursor, hydrates chunks, and always closes', async () => { + mock.state.cursorChunks = [await realSubgraph()] + const chunks: any[] = [] + for await (const v of brain.graph.export()) chunks.push(v) + + expect(mock.state.calls.cursorOpen).toBe(1) + expect(mock.state.calls.cursorClose).toBe(1) // cursor released even on normal completion + const nodes = new Set(chunks.flatMap((c) => c.nodes.map((n: any) => n.id))) + expect(nodes).toEqual(new Set([a, b, c])) + const edges = chunks.flatMap((c) => c.edges.map((e: any) => `${e.from}->${e.to}`)).sort() + expect(edges).toEqual([`${a}->${b}`, `${b}->${c}`].sort()) + }) + + it('resolves a provider registered as a FACTORY (storage) => provider, not just an instance', async () => { + const m = makeMockAccel() + const fb = new Brainy(createTestConfig()) + fb.use({ + name: 'mock-graph-accel-factory', + activate: async (ctx: any) => { + ctx.registerProvider('graphAcceleration', () => m.provider) // factory form + return true + } + } as any) + await fb.init() + const x = await fb.add({ type: NounType.Person, subtype: 'employee', data: 'X' }) + const y = await fb.add({ type: NounType.Person, subtype: 'employee', data: 'Y' }) + await fb.relate({ from: x, to: y, type: VerbType.RelatedTo, subtype: 'colleague' }) + const gei = (id: string): bigint => (fb as any).graphEntityInt(id) + const gi = (fb as any).graphIndex + const ix = gei(x), iy = gei(y) + const vxy = (await gi.getVerbIdsBySource(ix))[0] as bigint + m.state.traverseResult = { + nodes: BigInt64Array.from([ix, iy]), + nodeDepth: Uint8Array.from([0, 1]), + edgeSources: BigInt64Array.from([ix]), + edgeTargets: BigInt64Array.from([iy]), + edgeVerbInts: BigInt64Array.from([vxy]), + edgeTypes: Uint16Array.from([0]), + truncated: false + } + const view = await fb.graph.subgraph(x, { depth: 1 }) + expect(m.state.calls.traverse).toBe(1) // factory was invoked + provider routed + expect(new Set(view.nodes.map((n) => n.id))).toEqual(new Set([x, y])) + await fb.close() + }) + + it('rank() routes to the native provider and hydrates node ints -> ids, order preserved', async () => { + const gei = (id: string): bigint => (brain as any).graphEntityInt(id) + // Provider returns DESCENDING scores: c, then a, then b. + mock.state.rankResult = { + nodeInts: BigInt64Array.from([gei(c), gei(a), gei(b)]), + scores: Float64Array.from([0.5, 0.3, 0.2]) + } + const ranked = await brain.graph.rank() + + expect(mock.state.calls.rank).toBe(1) // native path, not the TS PageRank fallback + expect(ranked.map((r) => r.id)).toEqual([c, a, b]) // int -> id, provider order kept + expect(ranked[0].score).toBe(0.5) + + const top2 = await brain.graph.rank({ topK: 2 }) + expect(top2.map((r) => r.id)).toEqual([c, a]) + }) + + it('communities() routes to the native provider and buckets ids by community label', async () => { + const gei = (id: string): bigint => (brain as any).graphEntityInt(id) + // a,b in community 0; c alone in community 1. + mock.state.communitiesResult = { + nodeInts: BigInt64Array.from([gei(a), gei(b), gei(c)]), + communityIds: Uint32Array.from([0, 0, 1]), + communityCount: 2 + } + const { groups, count } = await brain.graph.communities() + + expect(mock.state.calls.communities).toBe(1) // native path, not the TS fallback + expect(count).toBe(2) + const asSets = groups.map((g) => new Set(g)) + expect(asSets).toContainEqual(new Set([a, b])) + expect(asSets).toContainEqual(new Set([c])) + expect(groups[0]).toHaveLength(2) // largest group first + }) + + it('path() routes to the native provider and hydrates node + verb ints', async () => { + const gei = (id: string): bigint => (brain as any).graphEntityInt(id) + const gi = (brain as any).graphIndex + const vAB = (await gi.getVerbIdsBySource(gei(a)))[0] as bigint + const vBC = (await gi.getVerbIdsBySource(gei(b)))[0] as bigint + mock.state.pathResult = { + nodeInts: BigInt64Array.from([gei(a), gei(b), gei(c)]), + edgeVerbInts: BigInt64Array.from([vAB, vBC]), + cost: 2 + } + const route = await brain.graph.path(a, c) + + expect(mock.state.calls.path).toBe(1) // native path, not the TS BFS/Dijkstra fallback + expect(route?.nodes).toEqual([a, b, c]) // node ints -> ids + expect(route?.relationships).toHaveLength(2) // verb ints -> verb-id strings + expect(route?.cost).toBe(2) + }) + + it('path() returns null when the native provider reports unreachable', async () => { + mock.state.pathResult = null + expect(await brain.graph.path(a, c)).toBeNull() + expect(mock.state.calls.path).toBe(1) + }) + + it('subgraph(query) forwards the metadata universe to traverse as an OpaqueIdSet (query→expand #61)', async () => { + // The native metadata index would return its roaring filter result as a Buffer; + // stub that producer and assert it reaches traverse WITHOUT id materialization. + const sentinel = new Uint8Array([0x01, 0x02, 0x03]) + ;(brain as any).metadataIndex.getIdSetForFilter = async () => sentinel + mock.state.traverseResult = await realSubgraph() + + await brain.graph.subgraph({ type: NounType.Person }, { depth: 1 }) + + expect(mock.state.calls.traverse).toBe(1) + // The opaque Buffer is the traverse seed argument — passed straight through. + expect(mock.state.lastTraverseSeeds).toBe(sentinel) + }) + + it('subgraph(query) materializes seeds via find() when no opaque producer exists', async () => { + // No getIdSetForFilter on the (real) metadata index → general path: find() runs, + // its matched ids resolve to entity ints, and those seed the native traverse. + mock.state.traverseResult = await realSubgraph() + + await brain.graph.subgraph({ type: NounType.Person }, { depth: 1 }) + + expect(mock.state.calls.traverse).toBe(1) + expect(Array.isArray(mock.state.lastTraverseSeeds)).toBe(true) // bigint[], not a Buffer + expect(typeof (mock.state.lastTraverseSeeds as unknown[])[0]).toBe('bigint') + }) + + // Readiness gate (cor boundary-audit item #1): the native engine reports + // isInitialized=false during its cold-start/rebuild window. Brainy must route to the + // pure-TS path then — NOT call the not-ready provider (which would throw) — across the + // whole brain.graph.* surface, and re-engage the native path once it flips true. + it('routes to the TS fallback (never calls the provider) while accel.isInitialized is false', async () => { + const notReady = makeMockAccel({ isInitialized: false }) + const b = new Brainy(createTestConfig()) + b.use({ + name: 'mock-graph-accel-not-ready', + activate: async (ctx: any) => { + ctx.registerProvider('graphAcceleration', notReady.provider) + return true + } + } as any) + await b.init() + const p = await b.add({ type: NounType.Person, subtype: 'employee', data: 'P' }) + const q = await b.add({ type: NounType.Person, subtype: 'employee', data: 'Q' }) + await b.relate({ from: p, to: q, type: VerbType.RelatedTo }) + + // Every brain.graph.* route + the query→expand fusion must NOT throw and must serve + // from the TS fallback while the engine is cold. + const sub = await b.graph.subgraph(p, { depth: 1 }) + const ranked = await b.graph.rank() + const comm = await b.graph.communities() + const route = await b.graph.path(p, q) + const fused = await b.graph.subgraph({ type: NounType.Person }, { depth: 1 }) + + // The provider was never touched — all served by the JS fallback. + expect(notReady.state.calls.traverse).toBe(0) + expect(notReady.state.calls.rank).toBe(0) + expect(notReady.state.calls.communities).toBe(0) + expect(notReady.state.calls.path).toBe(0) + // And the TS fallback returned real answers (not throws / empties from a cold engine). + expect(sub.nodes.some((n) => n.id === p)).toBe(true) + expect(ranked.length).toBeGreaterThan(0) + expect(comm.count).toBeGreaterThan(0) + expect(route?.nodes).toEqual([p, q]) + expect(fused.nodes.some((n) => n.id === p)).toBe(true) + await b.close() + }) +}) diff --git a/tests/unit/brainy/graph-subgraph.test.ts b/tests/unit/brainy/graph-subgraph.test.ts new file mode 100644 index 00000000..c2a4620b --- /dev/null +++ b/tests/unit/brainy/graph-subgraph.test.ts @@ -0,0 +1,132 @@ +/** + * @module tests/unit/brainy/graph-subgraph + * @description Graph engine #25: `brain.graph.subgraph()` — bounded multi-hop + * neighborhood extraction. These exercise the pure-TS fallback (no native + * GraphAccelerationProvider is registered in CI); the native path returns the + * same `GraphView` shape, cross-layer-tested against the provider. + * + * Graph under test (RelatedTo unless noted): + * e → a → b → c , a →(ReportsTo) d , a →(internal) f + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/index.js' +import { NounType, VerbType } from '../../../src/types/graphTypes.js' +import { createTestConfig } from '../../helpers/test-factory.js' + +describe('brain.graph.subgraph() (graph engine #25)', () => { + let brain: Brainy + let a: string, b: string, c: string, d: string, e: string, f: string + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + a = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'A' }) + b = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'B' }) + c = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'C' }) + d = await brain.add({ type: NounType.Project, subtype: 'milestone', data: 'D' }) + e = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'E' }) + f = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'F' }) + await brain.relate({ from: a, to: b, type: VerbType.RelatedTo, subtype: 'colleague' }) + await brain.relate({ from: b, to: c, type: VerbType.RelatedTo, subtype: 'colleague' }) + await brain.relate({ from: a, to: d, type: VerbType.ReportsTo, subtype: 'direct' }) + await brain.relate({ from: e, to: a, type: VerbType.RelatedTo, subtype: 'colleague' }) + await brain.relate({ from: a, to: f, type: VerbType.RelatedTo, subtype: 'colleague', visibility: 'internal' }) + }) + + afterEach(async () => { + await brain.close() + }) + + it('depth-1 returns the seed + immediate neighbors (both directions) with depths', async () => { + const view = await brain.graph.subgraph(a, { depth: 1 }) + const byId = new Map(view.nodes.map((n) => [n.id, n])) + // a is the seed (depth 0); b, d (out) and e (in) are 1 hop. f is internal → hidden. + expect(new Set(byId.keys())).toEqual(new Set([a, b, d, e])) + expect(byId.get(a)?.depth).toBe(0) + expect(byId.get(b)?.depth).toBe(1) + expect(byId.get(e)?.depth).toBe(1) + // Edges among them. + const pairs = view.edges.map((r) => `${r.from}->${r.to}`).sort() + expect(pairs).toEqual([`${a}->${b}`, `${a}->${d}`, `${e}->${a}`].sort()) + }) + + it('depth-2 expands one more hop (c via b), tracking depth', async () => { + const view = await brain.graph.subgraph(a, { depth: 2 }) + const byId = new Map(view.nodes.map((n) => [n.id, n])) + expect(byId.has(c)).toBe(true) + expect(byId.get(c)?.depth).toBe(2) + expect(view.edges.some((r) => r.from === b && r.to === c)).toBe(true) + }) + + it("direction:'out' follows only outgoing edges", async () => { + const view = await brain.graph.subgraph(a, { depth: 1, direction: 'out' }) + const ids = new Set(view.nodes.map((n) => n.id)) + expect(ids).toEqual(new Set([a, b, d])) // e (e→a, incoming) excluded + }) + + it('composes with a verb-type filter', async () => { + const view = await brain.graph.subgraph(a, { depth: 1, type: VerbType.ReportsTo }) + expect(view.edges.map((r) => `${r.from}->${r.to}`)).toEqual([`${a}->${d}`]) + expect(new Set(view.nodes.map((n) => n.id))).toEqual(new Set([a, d])) + }) + + it('includeInternal surfaces the hidden edge/node', async () => { + const view = await brain.graph.subgraph(a, { depth: 1, includeInternal: true }) + expect(view.nodes.some((n) => n.id === f)).toBe(true) + expect(view.edges.some((r) => r.to === f)).toBe(true) + }) + + it('hydrates node type/subtype by default; skips it with hydrateNodes:false', async () => { + const hydrated = await brain.graph.subgraph(a, { depth: 1 }) + const dNode = hydrated.nodes.find((n) => n.id === d) + expect(dNode?.type).toBe(NounType.Project) + expect(dNode?.subtype).toBe('milestone') + + const bare = await brain.graph.subgraph(a, { depth: 1, hydrateNodes: false }) + expect(bare.nodes.every((n) => n.type === undefined)).toBe(true) + }) + + it('caps the result and flags truncated when maxNodes is hit', async () => { + const view = await brain.graph.subgraph(a, { depth: 2, maxNodes: 2 }) + expect(view.truncated).toBe(true) + expect(view.nodes.length).toBeLessThanOrEqual(2) + }) + + it('accepts multiple seeds', async () => { + const view = await brain.graph.subgraph([a, c], { depth: 0 }) + // depth 0 = just the seeds, no expansion. + expect(new Set(view.nodes.map((n) => n.id))).toEqual(new Set([a, c])) + }) + + // ---- Query→expand fusion (graph #61): seed from a query / find() result ---- + + it('query→expand: subgraph(query) seeds from the matches and expands their neighborhood', async () => { + // The only Project is d; 1-hop expansion brings in a (a → d ReportsTo). + const view = await brain.graph.subgraph({ type: NounType.Project }, { depth: 1 }) + const ids = new Set(view.nodes.map((n) => n.id)) + expect(ids.has(d)).toBe(true) // the query match (seed) + expect(ids.has(a)).toBe(true) // expanded neighbor + expect(view.edges.some((r) => r.from === a && r.to === d)).toBe(true) + }) + + it('query→expand: depth 0 returns just the matches, no expansion', async () => { + const view = await brain.graph.subgraph({ type: NounType.Project }, { depth: 0 }) + expect(new Set(view.nodes.map((n) => n.id))).toEqual(new Set([d])) + }) + + it('accepts a find() result array as the seed set', async () => { + const matches = await brain.find({ type: NounType.Project }) // → [d] + expect(matches.length).toBeGreaterThan(0) + const view = await brain.graph.subgraph(matches, { depth: 1 }) + const ids = new Set(view.nodes.map((n) => n.id)) + expect(ids.has(d)).toBe(true) + expect(ids.has(a)).toBe(true) + }) + + it('query→expand returns empty when the query matches nothing', async () => { + const view = await brain.graph.subgraph({ type: NounType.Event }, { depth: 1 }) + expect(view.nodes).toEqual([]) + expect(view.edges).toEqual([]) + }) +}) diff --git a/tests/unit/brainy/metadata-provider-contract.test.ts b/tests/unit/brainy/metadata-provider-contract.test.ts new file mode 100644 index 00000000..7e690978 --- /dev/null +++ b/tests/unit/brainy/metadata-provider-contract.test.ts @@ -0,0 +1,90 @@ +/** + * @module tests/unit/brainy/metadata-provider-contract + * @description Brainy-side wiring of the two metadata-provider contract additions + * confirmed with cor for the lockstep: + * + * 1. `probeConsistency()` — an OPTIONAL O(1) cold-open consistency sampler. On the + * first read, brainy calls it once; on `false` it self-heals via + * `detectAndRepairCorruption()` (the metadata counterpart of the graph cold-load + * guard). The native provider implements it; the JS index omits it (no-op). + * 2. `getIdsForFilter(filter, opts?)` — brainy passes a page bound on the UNSORTED + * `find({ type, where, limit })` path so a native provider can early-stop. The JS + * index ignores `opts`. + * + * These are unit tests of brainy's CALL behaviour (the real end-to-end honoring is + * exercised by cor's combined matrix); they inject probe/spy hooks onto the live JS + * metadata index, which has neither method by default. + */ +import { describe, it, expect, beforeEach } from 'vitest' +import { Brainy } from '../../../src/brainy' +import { NounType } from '../../../src/types/graphTypes' + +describe('metadata-provider contract wiring (probeConsistency + getIdsForFilter opts)', () => { + let brain: Brainy + let mi: any + + beforeEach(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true }) + await brain.init() + await brain.add({ data: 'a', type: NounType.Thing, metadata: { kind: 'x' } }) + await brain.add({ data: 'b', type: NounType.Thing, metadata: { kind: 'y' } }) + mi = (brain as any).metadataIndex + ;(brain as any)._metadataConsistencyProbed = false // reset the one-shot guard + }) + + it('calls probeConsistency once on cold open and self-heals via detectAndRepairCorruption on false', async () => { + let probes = 0 + let repairs = 0 + mi.probeConsistency = async () => { probes++; return false } // corrupt → must repair + const origRepair = mi.detectAndRepairCorruption.bind(mi) + mi.detectAndRepairCorruption = async () => { repairs++; return origRepair() } + + await brain.find({ where: { kind: 'x' } }) + expect(probes).toBe(1) + expect(repairs).toBe(1) + + // Second read must NOT re-probe (once per brain). + await brain.find({ where: { kind: 'y' } }) + expect(probes).toBe(1) + expect(repairs).toBe(1) + }) + + it('does NOT repair when the probe reports healthy', async () => { + let repairs = 0 + mi.probeConsistency = async () => true // clean + const origRepair = mi.detectAndRepairCorruption.bind(mi) + mi.detectAndRepairCorruption = async () => { repairs++; return origRepair() } + + await brain.find({ where: { kind: 'x' } }) + expect(repairs).toBe(0) + }) + + it('a probe failure never breaks the read (best-effort, retried next time)', async () => { + let probes = 0 + mi.probeConsistency = async () => { probes++; throw new Error('probe boom') } + + // The read still succeeds despite the throwing probe. + const rows = await brain.find({ where: { kind: 'x' } }) + expect(rows.length).toBe(1) + expect(probes).toBe(1) + // Guard reset on failure → the next read retries the probe. + await brain.find({ where: { kind: 'y' } }) + expect(probes).toBe(2) + }) + + it('passes a page bound to getIdsForFilter on the unsorted find path (offset 0, brainy re-windows)', async () => { + let seenOpts: { limit?: number; offset?: number } | undefined + const orig = mi.getIdsForFilter.bind(mi) + mi.getIdsForFilter = async (filter: any, opts?: { limit?: number; offset?: number }) => { + seenOpts = opts + return orig(filter, opts) + } + + const rows = await brain.find({ where: { kind: 'x' }, limit: 5 }) + expect(rows.length).toBe(1) // result correctness preserved (JS ignores opts) + expect(seenOpts).toBeDefined() + expect(typeof seenOpts!.limit).toBe('number') + expect(seenOpts!.limit).toBeGreaterThanOrEqual(5) + expect(seenOpts!.offset).toBe(0) // brainy applies offset itself via slice + }) +}) diff --git a/tests/unit/brainy/migration-deference.test.ts b/tests/unit/brainy/migration-deference.test.ts new file mode 100644 index 00000000..6471b4ef --- /dev/null +++ b/tests/unit/brainy/migration-deference.test.ts @@ -0,0 +1,251 @@ +/** + * @module tests/unit/brainy/migration-deference + * @description rc.8 no-freeze auto-upgrade hooks — the deference seam that lets a + * native provider run an ONLINE, background build-new→verify→swap index migration + * while brainy stays out of the way (never a minutes-long blocking + * rebuild-on-open / first-query on a large-brain epoch-drift upgrade). + * + * Three hooks, locked with the cor team: + * + * - Hook 1 (deference): an OPTIONAL sync `isMigrating(): boolean` on each index + * provider (metadata / vector / graph). While it returns true, brainy SKIPS its + * own rebuild of that index — both in {@link Brainy.rebuildIndexesIfNeeded} + * (even under epoch-drift or `size()===0`) and on the large-path first-query + * lazy force-rebuild. A NON-migrating sibling still rebuilds when it needs to. + * - Hook 2: the public `brain.stampBrainFormat()` the provider calls once its + * background migration has verified-and-swapped, authoring the shared + * `_system/brain-format.json` marker. + * - Hook 3: the marker module is re-exported at `@soulcraft/brainy/brain-format` + * so cor reads the SAME `EXPECTED_INDEX_EPOCH` / `CURRENT_DATA_FORMAT` constants + * (single source of truth, no duplicated value). + * + * These paths are never exercised by brainy CI (cor registers no provider there), + * so the deference gates and the public stamp are pinned here with a white-box + * test-double provider — the same pattern as the cold-graph / handshake tests. + */ + +import { describe, it, expect, afterEach, vi } from 'vitest' +import { Brainy } from '../../../src/index.js' +import { NounType } from '../../../src/types/graphTypes.js' +import { createTestConfig } from '../../helpers/test-factory.js' +import { BaseStorage } from '../../../src/storage/baseStorage.js' +import { + BRAIN_FORMAT_PATH, + CURRENT_DATA_FORMAT, + EXPECTED_INDEX_EPOCH +} from '../../../src/storage/brainFormat.js' + +const CURRENT_MARKER = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH } + +/** The white-box surface this suite drives on a live brain instance. */ +interface BrainInternals { + index: { size(): number; rebuild(...a: unknown[]): Promise } + metadataIndex: { rebuild(...a: unknown[]): Promise } + graphIndex: { size(): number; rebuild(...a: unknown[]): Promise } + _indexEpochStale: boolean + lazyRebuildCompleted: boolean + rebuildIndexesIfNeeded(force?: boolean): Promise + ensureIndexesLoaded(): Promise + storage: { readRawObject(p: string): Promise } +} + +const brains: Brainy[] = [] + +/** Open (and track) a warmed memory brain holding `count` document entities. */ +async function makeWarmBrain(count = 2, extraConfig: Record = {}): Promise { + const brain = new Brainy(createTestConfig(extraConfig)) + await brain.init() + brains.push(brain) + for (let i = 0; i < count; i++) { + await brain.add({ data: `doc-${i}`, type: NounType.Document, metadata: { k: i } }) + } + return brain +} + +/** Cast a brain to its white-box internals. */ +function internalsOf(brain: Brainy): BrainInternals { + return brain as unknown as BrainInternals +} + +/** Force `provider.isMigrating()` to a fixed value (the native provider's deference flag). */ +function setMigrating(provider: object, value: boolean): void { + ;(provider as { isMigrating?: () => boolean }).isMigrating = () => value +} + +afterEach(async () => { + for (const brain of brains.splice(0)) { + try { + await brain.close() + } catch { + // already closed by the test + } + } + vi.restoreAllMocks() +}) + +describe('rc.8 no-freeze migration deference (isMigrating / stampBrainFormat / brain-format export)', () => { + // --- Hook 1: per-index deference in rebuildIndexesIfNeeded ---------------- + + it('metadata provider isMigrating(): its rebuild is skipped under epoch-drift; vector + graph siblings still rebuild', async () => { + const brain = await makeWarmBrain() + const internals = internalsOf(brain) + + const miSpy = vi.spyOn(internals.metadataIndex, 'rebuild').mockResolvedValue(undefined) + const idxSpy = vi.spyOn(internals.index, 'rebuild').mockResolvedValue(undefined) + const giSpy = vi.spyOn(internals.graphIndex, 'rebuild').mockResolvedValue(undefined) + + setMigrating(internals.metadataIndex, true) + // Epoch-drift would normally force ALL three to rebuild past the warm fast path. + internals._indexEpochStale = true + + await internals.rebuildIndexesIfNeeded() + + // The migrating provider owns its index — brainy does NOT rebuild it. + expect(miSpy).toHaveBeenCalledTimes(0) + // Non-migrating siblings still rebuild under the drift. + expect(idxSpy).toHaveBeenCalledTimes(1) + expect(giSpy).toHaveBeenCalledTimes(1) + // The marker is NOT advanced while a migration is in flight — cor stamps it + // when its build-new→verify→swap completes, so the stale flag stays set. + expect(internals._indexEpochStale).toBe(true) + }) + + it('vector provider isMigrating(): its rebuild is skipped under epoch-drift; metadata + graph siblings still rebuild', async () => { + const brain = await makeWarmBrain() + const internals = internalsOf(brain) + + const miSpy = vi.spyOn(internals.metadataIndex, 'rebuild').mockResolvedValue(undefined) + const idxSpy = vi.spyOn(internals.index, 'rebuild').mockResolvedValue(undefined) + const giSpy = vi.spyOn(internals.graphIndex, 'rebuild').mockResolvedValue(undefined) + + setMigrating(internals.index, true) + internals._indexEpochStale = true + + await internals.rebuildIndexesIfNeeded() + + expect(idxSpy).toHaveBeenCalledTimes(0) + expect(miSpy).toHaveBeenCalledTimes(1) + expect(giSpy).toHaveBeenCalledTimes(1) + expect(internals._indexEpochStale).toBe(true) + }) + + it('graph provider isMigrating(): its rebuild is skipped under epoch-drift; metadata + vector siblings still rebuild', async () => { + const brain = await makeWarmBrain() + const internals = internalsOf(brain) + + const miSpy = vi.spyOn(internals.metadataIndex, 'rebuild').mockResolvedValue(undefined) + const idxSpy = vi.spyOn(internals.index, 'rebuild').mockResolvedValue(undefined) + const giSpy = vi.spyOn(internals.graphIndex, 'rebuild').mockResolvedValue(undefined) + + setMigrating(internals.graphIndex, true) + internals._indexEpochStale = true + + await internals.rebuildIndexesIfNeeded() + + expect(giSpy).toHaveBeenCalledTimes(0) + expect(miSpy).toHaveBeenCalledTimes(1) + expect(idxSpy).toHaveBeenCalledTimes(1) + expect(internals._indexEpochStale).toBe(true) + }) + + it('a migrating provider is skipped even though its empty-signal would trigger a rebuild; clearing the flag lets the empty leg rebuild', async () => { + const brain = await makeWarmBrain() + const internals = internalsOf(brain) + + const miSpy = vi.spyOn(internals.metadataIndex, 'rebuild').mockResolvedValue(undefined) + const idxSpy = vi.spyOn(internals.index, 'rebuild').mockResolvedValue(undefined) + const giSpy = vi.spyOn(internals.graphIndex, 'rebuild').mockResolvedValue(undefined) + + // No epoch drift: the only rebuild trigger is a leg's own empty signal. The + // JS vector's rebuild() IS its load path, so size()===0 is its trigger — + // the leg where "empty → rebuild" is architecturally correct — so we drive + // deference through it. (The JS graph cold-loads before this gate, so its + // size()===0 is a valid empty state, not a rebuild trigger; graph deference + // is covered by the epoch-drift case above.) + internals._indexEpochStale = false + vi.spyOn(internals.index, 'size').mockReturnValue(0) + + // Migrating: the provider's background swap owns the index, so the + // size()===0 load trigger is suppressed. + setMigrating(internals.index, true) + await internals.rebuildIndexesIfNeeded() + expect(idxSpy).toHaveBeenCalledTimes(0) + // Metadata has entries; the graph has no edges; neither drifted → no rebuild. + expect(miSpy).toHaveBeenCalledTimes(0) + expect(giSpy).toHaveBeenCalledTimes(0) + + // Clearing the flag: the same empty, non-migrating vector now rebuilds — and + // this second call re-evaluating at all confirms the gate is not latched. + setMigrating(internals.index, false) + await internals.rebuildIndexesIfNeeded() + expect(idxSpy).toHaveBeenCalledTimes(1) + }) + + // --- Hook 1: large-path first-query lazy force-rebuild deference ---------- + + it('lazy first-query force-rebuild is SKIPPED when the vector provider isMigrating()', async () => { + // disableAutoRebuild routes first queries through ensureIndexesLoaded() (the + // large-brain lazy path that would otherwise force a blocking rebuild). + const brain = await makeWarmBrain(2, { disableAutoRebuild: true }) + const internals = internalsOf(brain) + + const rebuildSpy = vi.spyOn(internals, 'rebuildIndexesIfNeeded').mockResolvedValue(undefined) + // Simulate a cold/empty live vector index (cor is mid-swap, serving canonical). + vi.spyOn(internals.index, 'size').mockReturnValue(0) + internals.lazyRebuildCompleted = false + setMigrating(internals.index, true) + + await internals.ensureIndexesLoaded() + + // A query during cor's background swap must not trigger brainy's blocking rebuild. + expect(rebuildSpy).toHaveBeenCalledTimes(0) + }) + + it('lazy first-query force-rebuild STILL fires when the vector provider is not migrating (control)', async () => { + const brain = await makeWarmBrain(2, { disableAutoRebuild: true }) + const internals = internalsOf(brain) + + const rebuildSpy = vi.spyOn(internals, 'rebuildIndexesIfNeeded').mockResolvedValue(undefined) + vi.spyOn(internals.index, 'size').mockReturnValue(0) + internals.lazyRebuildCompleted = false + // No isMigrating → not deferring. + + await internals.ensureIndexesLoaded() + + // Without deference, the cold empty index drives the lazy force-rebuild. + expect(rebuildSpy).toHaveBeenCalledTimes(1) + expect(rebuildSpy).toHaveBeenCalledWith(true) + }) + + // --- Hook 2: public stampBrainFormat() ----------------------------------- + + it('brain.stampBrainFormat() writes _system/brain-format.json with the current {dataFormat, indexEpoch}', async () => { + const brain = new Brainy(createTestConfig()) + await brain.init() + brains.push(brain) + const internals = internalsOf(brain) + + // Spy AFTER init so only the stamp call's write is observed (init already + // stamped the fresh brain). + const writeSpy = vi.spyOn(BaseStorage.prototype, 'writeRawObject') + + await brain.stampBrainFormat() + + const stampWrites = writeSpy.mock.calls.filter(([p]) => p === BRAIN_FORMAT_PATH) + expect(stampWrites.length).toBe(1) + expect(stampWrites[0][1]).toEqual(CURRENT_MARKER) + + // And the marker is durable on disk at the current epoch. + const onDisk = await internals.storage.readRawObject(BRAIN_FORMAT_PATH) + expect(onDisk).toEqual(CURRENT_MARKER) + }) + + // --- Hook 3: marker module export ---------------------------------------- + + it('the brain-format marker module exports the compiled epoch + data-format constants', () => { + // cor imports these from '@soulcraft/brainy/brain-format' (Hook 3) so both + // sides share ONE source of truth — no duplicated constant to drift. + expect(EXPECTED_INDEX_EPOCH).toBe(1) + expect(CURRENT_DATA_FORMAT).toBe('8.0') + }) +}) diff --git a/tests/unit/brainy/migration-gate-family-scoped.test.ts b/tests/unit/brainy/migration-gate-family-scoped.test.ts new file mode 100644 index 00000000..b71c3899 --- /dev/null +++ b/tests/unit/brainy/migration-gate-family-scoped.test.ts @@ -0,0 +1,94 @@ +/** + * @module tests/unit/brainy/migration-gate-family-scoped + * @description The coordinated migration LOCK is FAMILY-SCOPED: a read waits only + * on the index families it actually consults. A one-time native migration of ONE + * family (its `isMigrating()` held) must NOT block a read served entirely from + * canonical storage (get / VFS content + dir) or from a HEALTHY family — only a + * read that needs the migrating family blocks. Regression for the whole-brain + * gate that hung getStats / readdir / readFile behind an unrelated family's + * migration until the wait timed out. + */ +import { describe, it, expect, beforeEach } from 'vitest' +import { Brainy } from '../../../src/brainy.js' +import { MigrationInProgressError } from '../../../src/errors/brainyError.js' + +const seed = async () => { + const brain = new Brainy({ + storage: { type: 'memory' }, + dimensions: 384, + requireSubtype: false, + silent: true, + // Short so a genuine block resolves fast; a real block would otherwise wait + // the 30 s default — this test asserts the wait does NOT happen for scoped + // reads, and DOES (bounded) for the one that needs the migrating family. + migrationWaitTimeoutMs: 300 + }) + await brain.init() + for (let i = 0; i < 5; i++) { + await brain.add({ data: `doc ${i}`, type: 'document', metadata: { i } }) + } + await brain.vfs.writeFile('/notes/hello.txt', 'canonical bytes — no index needed') + await brain.flush() + return brain +} + +/** Force a provider to report an in-flight (stuck) migration. */ +const jam = (provider: unknown) => { + ;(provider as { isMigrating: () => boolean }).isMigrating = () => true +} + +describe('migration LOCK is family-scoped', () => { + beforeEach(() => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + }) + + it('a stuck VECTOR migration does not block canonical or graph/metadata reads', async () => { + const brain = await seed() + const childId = ( + (await brain.vfs.readdir('/notes', { withFileTypes: true })) as Array<{ entityId: string }> + )[0].entityId + + jam((brain as any).index) // vector provider migrating; graph + metadata healthy + + // None of these consult the vector family → they serve immediately. + await expect(brain.getStats()).resolves.toBeDefined() + await expect(brain.vfs.readdir('/notes')).resolves.toHaveLength(1) // graph traversal + await expect(brain.vfs.readFile('/notes/hello.txt')).resolves.toBeDefined() // canonical blob + await expect(brain.get(childId)).resolves.not.toBeNull() // canonical by id + await expect(brain.find({ where: { i: 1 } })).resolves.toBeDefined() // metadata filter + }) + + it('a stuck VECTOR migration STILL blocks a read that needs the vector family', async () => { + const brain = await seed() + jam((brain as any).index) + + // A semantic query consults the vector index — it must wait, and (bounded by + // migrationWaitTimeoutMs) surface the retryable MigrationInProgressError + // rather than serve a half-built result. + await expect(brain.find({ query: 'doc' })).rejects.toBeInstanceOf(MigrationInProgressError) + }) + + it('a stuck GRAPH migration blocks traversal but not vector/canonical reads', async () => { + const brain = await seed() + const childId = ( + (await brain.vfs.readdir('/notes', { withFileTypes: true })) as Array<{ entityId: string }> + )[0].entityId + + jam((brain as any).graphIndex) // graph provider migrating; vector + metadata healthy + + // Canonical + vector + metadata are unaffected. + await expect(brain.get(childId)).resolves.not.toBeNull() + await expect(brain.find({ query: 'doc' })).resolves.toBeDefined() + await expect(brain.find({ where: { i: 1 } })).resolves.toBeDefined() + + // A graph traversal needs the migrating family → it blocks (bounded). + await expect(brain.related({ from: childId })).rejects.toBeInstanceOf(MigrationInProgressError) + }) + + it('with no migration in flight, every read serves (the fast path is a no-op)', async () => { + const brain = await seed() + await expect(brain.getStats()).resolves.toBeDefined() + await expect(brain.find({ query: 'doc' })).resolves.toBeDefined() + await expect(brain.vfs.readdir('/notes')).resolves.toHaveLength(1) + }) +}) diff --git a/tests/unit/brainy/related-node-bidirectional.test.ts b/tests/unit/brainy/related-node-bidirectional.test.ts new file mode 100644 index 00000000..0c46232e --- /dev/null +++ b/tests/unit/brainy/related-node-bidirectional.test.ts @@ -0,0 +1,86 @@ +/** + * @module tests/unit/brainy/related-node-bidirectional + * @description Graph-perf #4 (8.0): `related({ node })` — the one-call, + * both-direction "all edges on a noun" query. + * + * `related({ from })` / `related({ to })` each return one direction; `related({ node })` + * unions both (edges where the node is source OR target), deduped, in a single + * O(degree) call — the indexed counterpart of merging two `related()` calls by hand. + * It composes with `type` / `subtype` / visibility filters and is mutually exclusive + * with `from` / `to`. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/index.js' +import { NounType, VerbType } from '../../../src/types/graphTypes.js' +import { createTestConfig } from '../../helpers/test-factory.js' + +describe('related({ node }) both-direction incident edges (graph-perf #4)', () => { + let brain: Brainy + let a: string, b: string, c: string, d: string + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + a = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'A' }) + b = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'B' }) + c = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'C' }) + d = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'D' }) + // a's incident edges: a→b (out), c→a (in), a→d (out, ReportsTo), c→a internal (in, hidden) + await brain.relate({ from: a, to: b, type: VerbType.RelatedTo, subtype: 'colleague' }) + await brain.relate({ from: c, to: a, type: VerbType.RelatedTo, subtype: 'colleague' }) + await brain.relate({ from: a, to: d, type: VerbType.ReportsTo, subtype: 'direct' }) + }) + + afterEach(async () => { + await brain.close() + }) + + it('returns edges incident in BOTH directions (source OR target)', async () => { + const edges = await brain.related({ node: a }) + expect(edges).toHaveLength(3) + // Both out-edges (a is `from`) and the in-edge (a is `to`) are present. + const pairs = edges.map((e) => `${e.from}->${e.to}`).sort() + expect(pairs).toEqual([`${a}->${b}`, `${a}->${d}`, `${c}->${a}`].sort()) + }) + + it('matches the union of related({from}) + related({to})', async () => { + const out = await brain.related({ from: a }) + const inc = await brain.related({ to: a }) + const expected = new Set([...out, ...inc].map((e) => e.id)) + const node = await brain.related({ node: a }) + expect(new Set(node.map((e) => e.id))).toEqual(expected) + }) + + it('dedupes a self-loop (an edge where the node is both source and target)', async () => { + await brain.relate({ from: a, to: a, type: VerbType.RelatedTo, subtype: 'self' }) + const edges = await brain.related({ node: a }) + const selfLoops = edges.filter((e) => e.from === a && e.to === a) + expect(selfLoops).toHaveLength(1) // appears once, not twice + }) + + it('composes with a type filter (both directions)', async () => { + const related = await brain.related({ node: a, type: VerbType.RelatedTo }) + // a→b and c→a are RelatedTo; a→d is ReportsTo (excluded). + expect(related.map((e) => `${e.from}->${e.to}`).sort()).toEqual( + [`${a}->${b}`, `${c}->${a}`].sort() + ) + }) + + it('excludes internal edges by default, includes them with includeInternal', async () => { + const e = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'E' }) + await brain.relate({ from: e, to: a, type: VerbType.RelatedTo, subtype: 'colleague', visibility: 'internal' }) + + const visible = await brain.related({ node: a }) + expect(visible.some((r) => r.from === e)).toBe(false) + + const all = await brain.related({ node: a, includeInternal: true }) + const internalEdge = all.find((r) => r.from === e) + expect(internalEdge?.visibility).toBe('internal') + }) + + it("throws when 'node' is combined with 'from' or 'to'", async () => { + await expect(brain.related({ node: a, from: b })).rejects.toThrow(/mutually exclusive/) + await expect(brain.related({ node: a, to: b })).rejects.toThrow(/mutually exclusive/) + }) +}) diff --git a/tests/unit/brainy/related-visibility-fast-path.test.ts b/tests/unit/brainy/related-visibility-fast-path.test.ts new file mode 100644 index 00000000..7b289390 --- /dev/null +++ b/tests/unit/brainy/related-visibility-fast-path.test.ts @@ -0,0 +1,81 @@ +/** + * @module tests/unit/brainy/related-visibility-fast-path + * @description Graph-perf #1 (8.0): the visibility-aware fast adjacency path. + * + * `related({ from })` / `related({ to })` route through the O(degree) + * GraphAdjacencyIndex fast paths in `storage.getVerbs`. Before this fix, the + * default visibility exclusion (`excludeVisibility: ['internal','system']`, + * which every default `related()` sets) DISQUALIFIED those fast paths and forced + * a full O(E) shard scan — the root cause of multi-second per-node edge lookups + * on large graphs. The fast paths now hydrate each candidate's metadata and + * apply the subtype / visibility filters on the small O(degree) candidate set. + * + * Two observable contracts are pinned here (the perf change itself is structural + * — these prove the fast path produces the SAME correct results the scan did): + * 1. default `related()` excludes `internal` / `system` edges; `includeInternal` + * / `includeSystem` opt them back in. + * 2. returned `Relation`s now carry `visibility` (previously dropped by + * `hydrateVerbWithMetadata`, so `related({ includeInternal })` could not even + * report which edges were internal). + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/index.js' +import { NounType, VerbType } from '../../../src/types/graphTypes.js' +import { createTestConfig } from '../../helpers/test-factory.js' + +describe('related() visibility-aware fast adjacency (graph-perf #1)', () => { + let brain: Brainy + let a: string, b: string, c: string, d: string + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + a = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'A' }) + b = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'B' }) + c = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'C' }) + d = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'D' }) + // A → B public, A → C internal, A → D public. + await brain.relate({ from: a, to: b, type: VerbType.RelatedTo, subtype: 'colleague' }) + await brain.relate({ from: a, to: c, type: VerbType.RelatedTo, subtype: 'colleague', visibility: 'internal' }) + await brain.relate({ from: a, to: d, type: VerbType.RelatedTo, subtype: 'colleague' }) + }) + + afterEach(async () => { + await brain.close() + }) + + it('default related({ from }) excludes internal edges (out-direction fast path)', async () => { + const edges = await brain.related({ from: a }) + const targets = edges.map((e) => e.to).sort() + expect(targets).toEqual([b, d].sort()) + // None of the default results carry a non-public visibility. + expect(edges.every((e) => e.visibility === undefined || e.visibility === 'public')).toBe(true) + }) + + it('related({ from, includeInternal }) surfaces the internal edge AND reports its visibility', async () => { + const edges = await brain.related({ from: a, includeInternal: true }) + expect(edges.map((e) => e.to).sort()).toEqual([b, c, d].sort()) + const internalEdge = edges.find((e) => e.to === c) + // The latent-bug fix: visibility now flows through hydration to the Relation. + expect(internalEdge?.visibility).toBe('internal') + }) + + it('default related({ to }) excludes an internal edge (in-direction fast path)', async () => { + // B has one public in-edge; C has one internal in-edge. + const intoB = await brain.related({ to: b }) + expect(intoB.map((e) => e.from)).toEqual([a]) + + const intoC = await brain.related({ to: c }) + expect(intoC).toHaveLength(0) // internal edge hidden by default + + const intoCAll = await brain.related({ to: c, includeInternal: true }) + expect(intoCAll.map((e) => e.from)).toEqual([a]) + expect(intoCAll[0]?.visibility).toBe('internal') + }) + + it('subtype filter still works on the fast path (returns only matching, excludes internal)', async () => { + const colleagues = await brain.related({ from: a, subtype: 'colleague' }) + expect(colleagues.map((e) => e.to).sort()).toEqual([b, d].sort()) + }) +}) diff --git a/tests/unit/brainy/reserved-field-policy.test.ts b/tests/unit/brainy/reserved-field-policy.test.ts new file mode 100644 index 00000000..c5f37af4 --- /dev/null +++ b/tests/unit/brainy/reserved-field-policy.test.ts @@ -0,0 +1,251 @@ +/** + * @module tests/unit/brainy/reserved-field-policy + * @description The 8.0 `reservedFieldPolicy` matrix — what happens when an + * untyped (JavaScript) caller smuggles a Brainy-reserved field INSIDE the + * `metadata` bag of a write call, past the compile-time guard. + * + * 8.0 is a clean break with no silent failures. The decided contract: + * - `'throw'` (DEFAULT): a reserved key in the bag throws a clear Error naming + * the offending key(s) and the correct write path. No remap, no data loss. + * - `'warn'`: legacy remap PLUS a one-shot (per method+field, per process) + * warning for EVERY reserved key found. + * - `'remap'`: the pre-8.0 silent remap, no warning. + * + * The deep correctness of the remap itself (top-level precedence, system-managed + * drops, transact()/with() mirrors, read-side splitting) lives in + * tests/unit/brainy/update-reserved-metadata-remap.test.ts (which now runs under + * `reservedFieldPolicy: 'remap'`). This file pins the POLICY SELECTION and the + * throw/warn behaviors. + * + * Compile-time callers can't write these shapes at all (see + * tests/unit/types/reserved-metadata-keys.test-d.ts); the `as object` widenings + * below simulate untyped callers. + */ + +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { Brainy } from '../../../src/index.js' +import { NounType, VerbType } from '../../../src/types/graphTypes.js' +import { createTestConfig } from '../../helpers/test-factory.js' +import { prodLog } from '../../../src/utils/logger.js' + +describe('reservedFieldPolicy', () => { + describe("default policy is 'throw'", () => { + let brain: Brainy + + beforeEach(async () => { + // No reservedFieldPolicy override → resolves to 'throw'. + brain = new Brainy(createTestConfig()) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + it('add() throws naming the offending key and the correct write path', async () => { + await expect( + brain.add({ + type: NounType.Concept, + subtype: 'general', + data: 'x', + metadata: { confidence: 0.8 } as object + }) + ).rejects.toThrow(/metadata\.confidence is a reserved field/) + + // The error names the right param and the reserved list for discoverability. + await expect( + brain.add({ + type: NounType.Concept, + subtype: 'general', + data: 'x', + metadata: { confidence: 0.8 } as object + }) + ).rejects.toThrow(/'confidence' param.*RESERVED_ENTITY_FIELDS/s) + }) + + it('add() lists EVERY offending key when several are present', async () => { + const err = await brain + .add({ + type: NounType.Person, + data: 'multi', + metadata: { confidence: 0.5, weight: 0.6, subtype: 'employee' } as object + }) + .catch((e) => e as Error) + expect(err).toBeInstanceOf(Error) + expect(err.message).toMatch(/confidence/) + expect(err.message).toMatch(/weight/) + expect(err.message).toMatch(/subtype/) + }) + + it('update() throws on a reserved key in the patch', async () => { + const id = await brain.add({ type: NounType.Concept, subtype: 'general', data: 'y' }) + await expect( + brain.update({ id, metadata: { confidence: 0.3 } as object }) + ).rejects.toThrow(/metadata\.confidence is a reserved field/) + }) + + it('relate() throws on a reserved key in the bag', async () => { + const a = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'A' }) + const b = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'B' }) + await expect( + brain.relate({ + from: a, + to: b, + type: VerbType.RelatedTo, + subtype: 'colleague', + metadata: { confidence: 0.4 } as object + }) + ).rejects.toThrow(/metadata\.confidence is a reserved field.*RESERVED_RELATION_FIELDS/s) + }) + + it('updateRelation() throws on a reserved key in the patch', async () => { + const a = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'A' }) + const b = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'B' }) + const relId = await brain.relate({ + from: a, + to: b, + type: VerbType.ReportsTo, + subtype: 'direct' + }) + await expect( + brain.updateRelation({ id: relId, metadata: { weight: 0.2 } as object }) + ).rejects.toThrow(/metadata\.weight is a reserved field/) + }) + + it('transact() add op throws on a reserved key in the bag', async () => { + await expect( + brain.transact([ + { + op: 'add', + type: NounType.Concept, + subtype: 'general', + data: 'tx', + metadata: { confidence: 0.7 } as object + } + ]) + ).rejects.toThrow(/metadata\.confidence is a reserved field/) + }) + + it('a custom (non-reserved) key in the bag does NOT throw', async () => { + const id = await brain.add({ + type: NounType.Concept, + subtype: 'general', + data: 'ok', + metadata: { status: 'draft', rating: 4 } + }) + const entity = await brain.get(id) + expect(entity?.metadata).toEqual({ status: 'draft', rating: 4 }) + }) + }) + + describe("'remap' policy remaps silently (no warning)", () => { + let brain: Brainy + let warnSpy: ReturnType + + beforeEach(async () => { + warnSpy = vi.spyOn(prodLog, 'warn').mockImplementation(() => {}) + brain = new Brainy(createTestConfig({ reservedFieldPolicy: 'remap' })) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + warnSpy.mockRestore() + }) + + it('lifts user-mutable reserved fields to top-level without warning', async () => { + const id = await brain.add({ + type: NounType.Person, + data: 'remap lift', + metadata: { confidence: 0.8, weight: 0.6, subtype: 'employee', dept: 'eng' } as object + }) + const entity = await brain.get(id) + expect(entity?.confidence).toBe(0.8) + expect(entity?.weight).toBe(0.6) + expect(entity?.subtype).toBe('employee') + expect(entity?.metadata).toEqual({ dept: 'eng' }) + // 'remap' is silent about reserved fields (unrelated storage logs may fire, + // so assert specifically that no reserved-field warning was emitted). + const reservedWarned = warnSpy.mock.calls.some((c) => + String(c[0]).includes('reserved field') + ) + expect(reservedWarned).toBe(false) + }) + + it('preserves _originalId on natural-key ids through the remap path', async () => { + // A speculative view applies the same normalization and maps a natural-key + // id to a stable UUID, preserving the caller's original string. + const base = await brain.now() + const speculative = await base.with([ + { + op: 'add', + id: 'remap-spec-entity', + type: NounType.Concept, + subtype: 'general', + data: 'spec', + metadata: { confidence: 0.65, custom: 'spec' } as object + } + ]) + const entity = await speculative.get('remap-spec-entity') + expect(entity?.confidence).toBe(0.65) + expect(entity?.metadata).toEqual({ custom: 'spec', _originalId: 'remap-spec-entity' }) + await speculative.release() + await base.release() + }) + }) + + describe("'warn' policy remaps AND warns once per key", () => { + let brain: Brainy + let warnSpy: ReturnType + + beforeEach(async () => { + warnSpy = vi.spyOn(prodLog, 'warn').mockImplementation(() => {}) + brain = new Brainy(createTestConfig({ reservedFieldPolicy: 'warn' })) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + warnSpy.mockRestore() + }) + + it('remaps the value (same as remap) and emits a warning naming the field', async () => { + // Use a method+field combo unique to this test so the per-process one-shot + // registry has not already consumed it. + const id = await brain.add({ + type: NounType.Person, + data: 'warn lift', + // weight is user-mutable → remapped; this is the only 'warn'-policy + // add({ weight }) in the suite, so the one-shot warning fires here. + metadata: { weight: 0.42, dept: 'eng' } as object + }) + const entity = await brain.get(id) + // Value is honored (remap still happens under 'warn'). + expect(entity?.weight).toBe(0.42) + expect(entity?.metadata).toEqual({ dept: 'eng' }) + // And a warning was emitted naming the reserved field. + expect(warnSpy).toHaveBeenCalled() + const warned = warnSpy.mock.calls.some((c) => + String(c[0]).includes("'weight'") + ) + expect(warned).toBe(true) + }) + + it('warns for system-managed keys too (closes the historical gap)', async () => { + // Pre-8.0 only system-managed fields warned; 'warn' warns for every key. + // 'createdBy' (system-managed on update) is unique to this test. + const id = await brain.add({ type: NounType.Concept, subtype: 'general', data: 'sys' }) + warnSpy.mockClear() + await brain.update({ id, metadata: { createdBy: 'nope', keep: 'me' } as object }) + const entity = await brain.get(id) + // System-managed key dropped; custom field merged. + expect((entity?.metadata as Record)?.createdBy).toBeUndefined() + expect((entity?.metadata as Record)?.keep).toBe('me') + // A warning was emitted for the dropped system-managed key. + const warned = warnSpy.mock.calls.some((c) => + String(c[0]).includes("'createdBy'") + ) + expect(warned).toBe(true) + }) + }) +}) diff --git a/tests/unit/brainy/update-reserved-metadata-remap.test.ts b/tests/unit/brainy/update-reserved-metadata-remap.test.ts index 8e1a97c3..31713f99 100644 --- a/tests/unit/brainy/update-reserved-metadata-remap.test.ts +++ b/tests/unit/brainy/update-reserved-metadata-remap.test.ts @@ -10,13 +10,19 @@ * consumer's confidence-evolution writes no-oped for weeks before being * caught by reading values back. * - * 8.0 contract under test (every write path, entities AND relationships): + * These tests pin the LEGACY REMAP behavior, which in 8.0 is opt-in via + * `reservedFieldPolicy: 'remap'` (the default is `'throw'` — see the policy + * matrix in tests/unit/brainy/reserved-field-policy.test.ts). The brain in + * every test below is constructed with `reservedFieldPolicy: 'remap'` so these + * deep correctness assertions about the remap path stay exercised. + * + * Remap contract under test (every write path, entities AND relationships): * - user-mutable reserved fields (`confidence`, `weight`, `subtype` — plus * `service`/`createdBy` at add()/relate() time) remap from the metadata * bag to their dedicated top-level param, with top-level winning when both * are present; * - system-managed reserved fields (`createdAt`, `_rev`, `noun`/`verb`, - * `data`, …) are dropped from the bag (one-shot warning); + * `data`, …) are dropped from the bag; * - the same normalization applies to `transact()` operations and `with()` * speculative views; * - reads NEVER echo a reserved field inside `metadata`. @@ -32,11 +38,12 @@ import { Brainy } from '../../../src/index.js' import { NounType, VerbType } from '../../../src/types/graphTypes.js' import { createTestConfig } from '../../helpers/test-factory.js' -describe('reserved-field metadata remap (8.0 contract)', () => { +describe('reserved-field metadata remap (8.0 legacy remap path)', () => { let brain: Brainy beforeEach(async () => { - brain = new Brainy(createTestConfig()) + // The remap path is opt-in in 8.0 (default policy is 'throw'). + brain = new Brainy(createTestConfig({ reservedFieldPolicy: 'remap' })) await brain.init() }) diff --git a/tests/unit/brainy/upsert.test.ts b/tests/unit/brainy/upsert.test.ts new file mode 100644 index 00000000..92e56b05 --- /dev/null +++ b/tests/unit/brainy/upsert.test.ts @@ -0,0 +1,194 @@ +/** + * Unit tests for AddParams.upsert — create-or-update in a single add() call. + * + * Covers the live add() path and the addMany batch flag. Verifies that upsert + * inserts on a fresh id, merges (not overwrites) into an existing id while + * preserving createdAt and bumping _rev, is mutually exclusive with ifAbsent, + * and is a plain insert when no id is supplied. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/brainy' +import { createTestConfig } from '../../helpers/test-factory' + +describe('AddParams.upsert', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + it('inserts a new entity when the id does not exist (_rev 1)', async () => { + // 8.0 normalizes a natural string key to a stable canonical UUID; the entity is + // still reachable by the natural key via get()'s id resolution. + const id = await brain.add({ + id: 'customer-1', + type: 'thing', + data: 'Acme Corp', + metadata: { tier: 'silver' }, + upsert: true + }) + + expect(typeof id).toBe('string') + + const entity = await brain.get('customer-1') + expect(entity).not.toBeNull() + expect(entity!.type).toBe('thing') + expect(entity!.metadata.tier).toBe('silver') + expect(entity!._rev).toBe(1) + }) + + it('merges into an existing entity, preserves createdAt, bumps _rev, applies changed data', async () => { + // Seed the entity. + await brain.add({ + id: 'invoice-9', + type: 'thing', + data: 'Original line items', + metadata: { status: 'draft', amount: 100 } + }) + + const before = await brain.get('invoice-9') + expect(before).not.toBeNull() + const originalCreatedAt = before!.createdAt + expect(before!._rev).toBe(1) + + // Upsert over the existing id with a partial metadata patch + new data. + await brain.add({ + id: 'invoice-9', + type: 'thing', + data: 'Revised line items', + metadata: { status: 'sent' }, + upsert: true + }) + + const after = await brain.get('invoice-9') + expect(after).not.toBeNull() + + // Metadata MERGED, not replaced — the untouched 'amount' field survives. + expect(after!.metadata.status).toBe('sent') + expect(after!.metadata.amount).toBe(100) + + // Changed data applied. + expect(after!.data).toBe('Revised line items') + + // _rev bumped, createdAt preserved. + expect(after!._rev).toBe(2) + expect(after!.createdAt).toBe(originalCreatedAt) + }) + + it('re-embeds when data changes on upsert', async () => { + await brain.add({ + id: 'doc-vec', + type: 'document', + data: 'cats and dogs and small furry animals' + }) + const before = await brain.get('doc-vec', { includeVectors: true }) + expect(before!.vector.length).toBeGreaterThan(0) + + await brain.add({ + id: 'doc-vec', + type: 'document', + data: 'quarterly financial revenue projections spreadsheet', + upsert: true + }) + const after = await brain.get('doc-vec', { includeVectors: true }) + expect(after!.vector.length).toBe(before!.vector.length) + // A genuinely different document re-embeds to a different vector. + expect(after!.vector).not.toEqual(before!.vector) + }) + + it('throws when both ifAbsent and upsert are true', async () => { + await expect( + brain.add({ + id: 'conflict-1', + type: 'thing', + data: 'x', + ifAbsent: true, + upsert: true + }) + ).rejects.toThrow(/mutually exclusive/) + }) + + it('behaves like a plain insert when no id is supplied', async () => { + const id = await brain.add({ + type: 'thing', + data: 'no id supplied', + metadata: { k: 'v' }, + upsert: true + }) + + expect(typeof id).toBe('string') + const entity = await brain.get(id) + expect(entity).not.toBeNull() + expect(entity!._rev).toBe(1) + expect(entity!.metadata.k).toBe('v') + }) + + it('merges an existing item via the addMany batch upsert flag', async () => { + // Seed an item that the batch will later upsert into. + await brain.add({ + id: 'order-1', + type: 'thing', + data: 'first', + metadata: { region: 'west', count: 1 } + }) + const seeded = await brain.get('order-1') + const seededCreatedAt = seeded!.createdAt + + const result = await brain.addMany({ + upsert: true, + items: [ + { id: 'order-1', type: 'thing', data: 'updated', metadata: { count: 2 } }, + { id: 'order-2', type: 'thing', data: 'brand new', metadata: { region: 'east' } } + ] + }) + + // Both items resolved (returned ids are the canonical UUIDs the natural keys map to). + expect(result.failed).toHaveLength(0) + expect(result.successful).toHaveLength(2) + + // Existing item merged (region survived), _rev bumped, createdAt preserved. + const merged = await brain.get('order-1') + expect(merged!.metadata.region).toBe('west') + expect(merged!.metadata.count).toBe(2) + expect(merged!.data).toBe('updated') + expect(merged!._rev).toBe(2) + expect(merged!.createdAt).toBe(seededCreatedAt) + + // New item inserted fresh. + const fresh = await brain.get('order-2') + expect(fresh!._rev).toBe(1) + expect(fresh!.metadata.region).toBe('east') + }) + + it('upserts inside a transact batch — existing id merges, new id inserts', async () => { + await brain.add({ + id: 'acct-1', + type: 'thing', + data: 'seed', + metadata: { plan: 'free', seats: 1 } + }) + const seeded = await brain.get('acct-1') + const seededCreatedAt = seeded!.createdAt + + await brain.transact([ + { op: 'add', id: 'acct-1', type: 'thing', data: 'seed', metadata: { plan: 'pro' }, upsert: true }, + { op: 'add', id: 'acct-2', type: 'thing', data: 'new account', metadata: { plan: 'free' }, upsert: true } + ]) + + const merged = await brain.get('acct-1') + expect(merged!.metadata.plan).toBe('pro') + expect(merged!.metadata.seats).toBe(1) // merged, not overwritten + expect(merged!._rev).toBe(2) + expect(merged!.createdAt).toBe(seededCreatedAt) + + const fresh = await brain.get('acct-2') + expect(fresh!._rev).toBe(1) + expect(fresh!.metadata.plan).toBe('free') + }) +}) diff --git a/tests/unit/brainy/visibility.test.ts b/tests/unit/brainy/visibility.test.ts index 8bd37ebd..a5a02422 100644 --- a/tests/unit/brainy/visibility.test.ts +++ b/tests/unit/brainy/visibility.test.ts @@ -198,36 +198,60 @@ describe('visibility (8.0 reserved field)', () => { expect(entity?.visibility).toBeUndefined() }) - it('an untyped caller passing visibility inside metadata is normalized (lifted to top-level)', async () => { + it('an untyped caller passing visibility inside metadata is normalized under reservedFieldPolicy:"remap" (lifted to top-level)', async () => { // Simulate a JavaScript caller smuggling the reserved key past the compile-time guard. - const id = await brain.add({ - type: NounType.Concept, - data: 'y', - metadata: { visibility: 'internal', tag: 't' } as object - }) - const entity = await brain.get(id) - // Lifted to the top-level field… - expect(entity?.visibility).toBe('internal') - // …and stripped from the metadata bag. - expect((entity?.metadata as Record)?.visibility).toBeUndefined() - expect((entity?.metadata as Record)?.tag).toBe('t') - // It is excluded from the default count, exactly like a top-level internal write. - expect(await brain.getNounCount()).toBe(0) + // The legacy remap behavior is now opt-in (8.0 default is 'throw'). + const remapBrain = new Brainy(createTestConfig({ reservedFieldPolicy: 'remap' })) + await remapBrain.init() + try { + const id = await remapBrain.add({ + type: NounType.Concept, + data: 'y', + metadata: { visibility: 'internal', tag: 't' } as object + }) + const entity = await remapBrain.get(id) + // Lifted to the top-level field… + expect(entity?.visibility).toBe('internal') + // …and stripped from the metadata bag. + expect((entity?.metadata as Record)?.visibility).toBeUndefined() + expect((entity?.metadata as Record)?.tag).toBe('t') + // It is excluded from the default count, exactly like a top-level internal write. + expect(await remapBrain.getNounCount()).toBe(0) + } finally { + await remapBrain.close() + } }) - it('a "system" value smuggled through metadata is dropped, not honored', async () => { + it('a "system" value smuggled through metadata is dropped under reservedFieldPolicy:"remap", not honored', async () => { // 'system' is Brainy-only; an untyped caller must not be able to set it. - const id = await brain.add({ - type: NounType.Concept, - data: 'z', - metadata: { visibility: 'system' } as object - }) - const entity = await brain.get(id) - // The smuggled 'system' was dropped → entity stays public (counted, visible). - expect(entity?.visibility).toBeUndefined() - expect(await brain.getNounCount()).toBe(1) - const found = await brain.find({ type: NounType.Concept, limit: 10 }) - expect(found.map((r) => r.id)).toContain(id) + const remapBrain = new Brainy(createTestConfig({ reservedFieldPolicy: 'remap' })) + await remapBrain.init() + try { + const id = await remapBrain.add({ + type: NounType.Concept, + data: 'z', + metadata: { visibility: 'system' } as object + }) + const entity = await remapBrain.get(id) + // The smuggled 'system' was dropped → entity stays public (counted, visible). + expect(entity?.visibility).toBeUndefined() + expect(await remapBrain.getNounCount()).toBe(1) + const found = await remapBrain.find({ type: NounType.Concept, limit: 10 }) + expect(found.map((r) => r.id)).toContain(id) + } finally { + await remapBrain.close() + } + }) + + it('an untyped caller passing visibility inside metadata throws under the default policy', async () => { + // 8.0 default: no silent remap — a reserved key in the bag is a loud error. + await expect( + brain.add({ + type: NounType.Concept, + data: 'throws', + metadata: { visibility: 'internal', tag: 't' } as object + }) + ).rejects.toThrow(/visibility.*reserved field/) }) }) }) diff --git a/tests/unit/cold-open-rebuild-gate.test.ts b/tests/unit/cold-open-rebuild-gate.test.ts new file mode 100644 index 00000000..136cfa75 --- /dev/null +++ b/tests/unit/cold-open-rebuild-gate.test.ts @@ -0,0 +1,213 @@ +/** + * Cold-open rebuild gate — the readiness contract (8.0.12). + * + * A production deployment measured 48 seconds on EVERY boot because the + * rebuild gate keyed off in-memory size()/count heuristics that are dishonest + * for durable indexes: a disk-native provider legitimately reports 0 resident + * entries while fully durable, and the JS graph loaded nothing at gate time + * (its LSM initialized lazily) — so `rebuildIndexesIfNeeded()` re-derived + * indexes from a full canonical scan on every open. + * + * The contract pinned here: + * - JS reopen: the graph must COLD-LOAD its persisted LSM (no re-derive; the + * old `_initializeGraphIndex` rebuilt from a full verb scan every boot), + * the metadata leg must not rebuild (id-mapper signal is honest), and + * queries stay correct. The JS vector leg still runs `rebuild()` — that IS + * its load path. + * - A vector provider exposing `isReady() === true` is never rebuilt, even + * at `size() === 0`; `isReady() === false` gets its rebuild. `init?()` is + * eagerly awaited before the gate. + */ +import { describe, it, expect, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy, NounType, VerbType } from '../../src/index.js' +import { GraphAdjacencyIndex } from '../../src/graph/graphAdjacencyIndex.js' +import { MetadataIndexManager } from '../../src/utils/metadataIndex.js' + +const tmpDirs: string[] = [] +function mkTmp(): string { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-cold-open-')) + tmpDirs.push(d) + return d +} +afterEach(() => { + for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }) +}) + +const V = () => Array.from({ length: 384 }, () => Math.random()) + +/** Populate a brain with nouns + edges and close it. */ +async function buildBrain(dir: string, n = 12): Promise { + const brain: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + plugins: [], + silent: true + }) + await brain.init() + const ids: string[] = [] + for (let i = 0; i < n; i++) { + ids.push( + await brain.add({ + data: `entity ${i}`, + type: NounType.Concept, + subtype: 's', + metadata: { wave: i % 3 }, + vector: V() + }) + ) + } + for (let i = 0; i + 1 < ids.length; i++) { + await brain.relate({ from: ids[i], to: ids[i + 1], type: VerbType.RelatedTo, subtype: 's' }) + } + await brain.close() + return ids +} + +describe('Cold-open rebuild gate (readiness contract)', () => { + it('JS reopen: graph cold-loads its LSM (no re-derive), metadata skips, queries correct', async () => { + const dir = mkTmp() + const ids = await buildBrain(dir) + + // Spy on the two rebuilds that must NOT run on a healthy reopen. + const graphRebuilds: number[] = [] + const metadataRebuilds: number[] = [] + const origGraphRebuild = GraphAdjacencyIndex.prototype.rebuild + const origMetaRebuild = MetadataIndexManager.prototype.rebuild + GraphAdjacencyIndex.prototype.rebuild = async function (...args: any[]) { + graphRebuilds.push(1) + return origGraphRebuild.apply(this, args as any) + } + MetadataIndexManager.prototype.rebuild = async function (...args: any[]) { + metadataRebuilds.push(1) + return origMetaRebuild.apply(this, args as any) + } + try { + const brain: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + plugins: [], + silent: true + }) + await brain.init() + + expect(graphRebuilds.length).toBe(0) // persisted LSM loaded — no O(E) re-derive + expect(metadataRebuilds.length).toBe(0) // id-mapper signal honest — no rebuild + expect(await brain.graphIndex.size()).toBeGreaterThan(0) // durable edges COLD-LOADED at boot + + // Correctness after the load — the whole point of not rebuilding. + const byWhere = await brain.find({ type: NounType.Concept, where: { wave: 1 } }) + expect(byWhere.length).toBe(4) // waves 1,4,7,10 of 12 + const connected = await brain.find({ connected: { from: ids[0], depth: 1 } }) + expect(connected.length).toBe(1) + await brain.close() + } finally { + GraphAdjacencyIndex.prototype.rebuild = origGraphRebuild + MetadataIndexManager.prototype.rebuild = origMetaRebuild + } + }) + + it('a vector provider with isReady()===true is never rebuilt, even at size()===0 (init eagerly awaited)', async () => { + const dir = mkTmp() + await buildBrain(dir, 6) + + const calls = { init: 0, rebuild: 0 } + const stubVector = { + addItem: async (item: any) => item?.id ?? 'stub-id', + removeItem: async () => true, + search: async () => [] as Array<[string, number]>, + size: () => 0, // disk-native posture: durable, zero resident + clear: () => {}, + rebuild: async () => { + calls.rebuild++ + }, + flush: async () => 0, + getPersistMode: () => 'deferred' as const, + init: async () => { + calls.init++ + }, + isReady: () => true + } + const brain: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + plugins: [], + silent: true + }) + brain.use({ name: 'fake-native-vector', activate: async (ctx: any) => { ctx.registerProvider('vector', () => stubVector); return true } }) + await brain.init() + + expect(calls.init).toBe(1) // the eager cold-load ran before the gate + expect(calls.rebuild).toBe(0) // isReady()===true replaced the size()===0 heuristic + await brain.close() + }) + + it('a vector provider with isReady()===false gets its rebuild (honest in both directions)', async () => { + const dir = mkTmp() + await buildBrain(dir, 6) + + const calls = { rebuild: 0 } + let ready = false + const stubVector = { + addItem: async (item: any) => item?.id ?? 'stub-id', + removeItem: async () => true, + search: async () => [] as Array<[string, number]>, + size: () => 999, // even a non-zero size must NOT mask a not-ready provider + clear: () => {}, + rebuild: async () => { + calls.rebuild++ + ready = true // rebuild restores readiness + }, + flush: async () => 0, + getPersistMode: () => 'deferred' as const, + isReady: () => ready + } + const brain: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + plugins: [], + silent: true + }) + brain.use({ name: 'fake-native-vector', activate: async (ctx: any) => { ctx.registerProvider('vector', () => stubVector); return true } }) + await brain.init() + + expect(calls.rebuild).toBe(1) + await brain.close() + }) + + it('self-heal survives: durable graph state deleted → one rebuild from canonical on reopen', async () => { + const dir = mkTmp() + const ids = await buildBrain(dir, 6) + + // Simulate lost durable graph state: remove the persisted LSM artifacts + // (metadata channel keys live under _system hash buckets — nuke the graph + // manifests via the storage API instead of guessing paths). + const wipe: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + plugins: [], + silent: true + }) + await wipe.init() + // Clear both LSM trees' persisted manifests through the live index, then + // close WITHOUT letting them re-flush a fresh manifest state. + const gi: any = wipe.graphIndex + await gi.lsmTreeVerbsBySource.clear?.() + await gi.lsmTreeVerbsByTarget.clear?.() + await wipe.close() + + const brain: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + plugins: [], + silent: true + }) + await brain.init() + // Canonical verbs still exist → the boot self-heal must have restored the adjacency. + const connected = await brain.find({ connected: { from: ids[0], depth: 1 } }) + expect(connected.length).toBe(1) + await brain.close() + }) +}) diff --git a/tests/unit/db/bounded-chains.test.ts b/tests/unit/db/bounded-chains.test.ts new file mode 100644 index 00000000..034bc663 --- /dev/null +++ b/tests/unit/db/bounded-chains.test.ts @@ -0,0 +1,542 @@ +/** + * @module tests/unit/db/bounded-chains + * @description Correctness + RAM-boundedness oracle for the bounded per-id + * generation history chains in `src/db/generationStore.ts` (GA #33, Approach C: + * hot-tail window + bounded cold LRU + a mutex-free bulk `resolveManyAt`). + * + * The old design kept ONE resident chain per id ever touched across retained + * history (`Map`) → O(distinct-ids) RAM, which defeats + * billion-scale time travel. The replacement bounds RAM to O(W·d̄)/O(L) — a + * resident window over the newest `W` generations plus a bounded LRU of + * reconstructed deep-pin chains — while staying EXACTLY correct. + * + * These tests are the correctness oracle for that change: + * + * 1. Oracle vs brute-force: for every id × every pin, `resolveAt` must equal a + * brute-force scan of every reserved generation's persisted `tx.json`. + * 2. Bounded RAM independent of N: the resident structures stay O(W)/O(L) no + * matter how many distinct ids the corpus touched. + * 3. Held-Db across cold eviction + a concurrent compaction (lock-light). + * 4. Un-flushed pending single-ops resolve identically before and after flush. + * 5. A deep-pin materialize is O(R) `getDelta` (not O(N·R)) and never deadlocks + * inside the mutex-held reconciliation pass. + * 6. `resolveManyAt` ≡ per-id `resolveAt` for random id-sets × random pins. + * 7. The write path is I/O-free across window slides (the inverse-index ring + * supplies slid-out ids — zero `tx.json` reads on commit). + * 8. Compaction invalidates + rebuilds the window over survivors; a pin below + * the new horizon throws `GenerationCompactedError`. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' +import { GenerationStore } from '../../../src/db/generationStore.js' +import { GenerationCompactedError } from '../../../src/db/errors.js' +import { Brainy } from '../../../src/index.js' +import { NounType } from '../../../src/types/graphTypes.js' +import { createTestConfig, generateTestVector } from '../../helpers/test-factory.js' + +/** A precomputed embedding so adds skip the (slow) embedding model — these tests + * exercise the generation layer, not semantics. */ +const VEC = generateTestVector() + +/** Deterministic UUID-shaped id (the sharded storage layout derives the shard + * from the UUID hex), unique per `n`. */ +function uid(n: number): string { + return `00000000-0000-4000-8000-${n.toString(16).padStart(12, '0')}` +} + +/** Stored-metadata fixture carrying a `version` the oracle compares on. */ +function fixture(version: number): Record { + return { + noun: NounType.Document, + subtype: 'note', + data: `payload-v${version}`, + version, + createdAt: 1000, + updatedAt: 1000 + version, + _rev: version + } +} + +/** A small deterministic PRNG so the random oracle runs are reproducible. */ +function mulberry32(seed: number): () => number { + let a = seed >>> 0 + return () => { + a |= 0 + a = (a + 0x6d2b79f5) | 0 + let t = Math.imul(a ^ (a >>> 15), 1 | a) + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t + return ((t ^ (t >>> 14)) >>> 0) / 4294967296 + } +} + +/** Reject after `ms` so a deadlock surfaces as a test failure, not a hang. */ +function withTimeout(p: Promise, ms: number, label: string): Promise { + return Promise.race([ + p, + new Promise((_, reject) => + setTimeout(() => reject(new Error(`timed out after ${ms}ms (likely deadlock): ${label}`)), ms) + ) + ]) +} + +/** Normalize a `resolveAt` result to a comparable scalar. */ +function norm(r: { source: string; metadata?: any }): string { + if (r.source === 'current') return 'current' + if (r.source === 'absent') return 'absent' + return `rec:${r.metadata?.version}` +} + +describe('db/GenerationStore — bounded per-id chains (GA #33)', () => { + let storage: MemoryStorage + let store: GenerationStore + + beforeEach(async () => { + storage = new MemoryStorage() + await storage.init() + store = new GenerationStore(storage) + await store.open() + }) + + /** Commit one single-op noun write (its own generation); returns the gen. */ + async function writeNoun(id: string, version: number): Promise { + const { generation } = await store.commitSingleOp({ + touched: { nouns: [id] }, + execute: async () => { + await storage.saveNounMetadata(id, fixture(version)) + } + }) + return generation + } + + /** Commit one single-op verb write (raw, so before-images are captured). */ + async function writeVerb(id: string, version: number): Promise { + const { generation } = await store.commitSingleOp({ + touched: { verbs: [id] }, + execute: async () => { + await storage.writeVerbRaw(id, { + metadata: { ...fixture(version), verb: 'RelatedTo' }, + vector: { sourceId: uid(900001), targetId: uid(900002), verb: 'RelatedTo' } + }) + } + }) + return generation + } + + /** Brute-force `resolveAt`: scan EVERY reserved generation's persisted tx.json + * (the gens must be flushed) for the first one after `pin` that touched `id`, + * then read its before-image. Fully independent of the resident chains. */ + async function bruteForce( + kind: 'noun' | 'verb', + id: string, + pin: number, + maxGen: number + ): Promise { + for (let g = pin + 1; g <= maxGen; g++) { + const delta = (await storage.readRawObject(`_generations/${g}/tx.json`)) as + | { nouns: string[]; verbs: string[] } + | null + if (!delta) continue + const touched = kind === 'noun' ? delta.nouns : delta.verbs + if (Array.isArray(touched) && touched.includes(id)) { + const rec = (await storage.readRawObject(`_generations/${g}/prev/${id}.json`)) as + | { metadata: any; vector: any } + | null + if (!rec || (rec.metadata === null && rec.vector === null)) return 'absent' + return `rec:${rec.metadata?.version}` + } + } + return 'current' + } + + // ========================================================================== + // 1. Oracle vs brute-force — every id × every pin, nouns + verbs. + // ========================================================================== + it('resolveAt equals a brute-force tx.json scan for every id × every pin (recent, deep, boundary)', async () => { + const s = store as any + s.recentWindowGenerations = 8 // W + s.coldChainLruMax = 4 // L + + const nounIds = Array.from({ length: 5 }, (_, i) => uid(100 + i)) + const verbIds = Array.from({ length: 5 }, (_, i) => uid(200 + i)) + const rand = mulberry32(1234) + let v = 0 + + // 60 overlapping single-op generations. Trigger a historical read at the + // halfway point so the SECOND half exercises the live slide path + // (extendChains + window advance), not just the lazy build. + for (let i = 0; i < 60; i++) { + if (i === 30) { + // Force the window to build so the rest of the writes drive slides. + await store.resolveAt('noun', nounIds[0], 1) + } + if (rand() < 0.5) { + await writeNoun(nounIds[Math.floor(rand() * nounIds.length)], ++v) + } else { + await writeVerb(verbIds[Math.floor(rand() * verbIds.length)], ++v) + } + } + // Flush so every reserved generation has a persisted tx.json for the oracle. + await store.flushPendingSingleOps() + const maxGen = store.generation() + + // Window must have actually slid below the head (proves the slide path ran). + expect(s.windowLo).toBeGreaterThan(1) + expect(s.windowLo).toBe(Math.max(1, maxGen - 8 + 1)) + + // Every id × every pin from 0..maxGen — covers recent (≥windowLo), deep + // ( { + const s = store as any + s.recentWindowGenerations = 4 + s.coldChainLruMax = 16 + + const X = uid(300) + const filler = (i: number) => uid(400 + i) + + const g1 = await writeNoun(X, 1) // X = v1 + // Push X out of the window with filler writes (X untouched). + for (let i = 0; i < 10; i++) await writeNoun(filler(i), 100 + i) + + // Deep read at g1 BEFORE any later touch: X untouched after g1 → 'current'. + expect(norm(await store.resolveAt('noun', X, g1))).toBe('current') + expect((await store.resolveAt('noun', X, g1)).source).toBe('current') // now cached cold + + // Now touch X again (v2). The stale cached cold chain would still answer + // 'current' → reading the NEW live value (v1 corrupted to v2). extendChains + // must invalidate the cold entry so the re-read resolves the g1 before-image. + await writeNoun(X, 2) + await store.flushPendingSingleOps() + + const resolved = await store.resolveAt('noun', X, g1) + expect(resolved.source).toBe('record') + if (resolved.source === 'record') expect(resolved.metadata.version).toBe(1) + }) + + // ========================================================================== + // 2. Bounded RAM independent of N. + // ========================================================================== + it('resident window/cold structures stay O(W)/O(L), independent of distinct-id count', async () => { + const s = store as any + const W = 8 + const L = 4 + s.recentWindowGenerations = W + s.coldChainLruMax = L + + const N = 1200 // distinct ids, one single-op each + for (let i = 0; i < N; i++) await writeNoun(uid(1000 + i), i) + await store.flushPendingSingleOps() + const maxGen = store.generation() + + // Build the window (first historical read). + await store.resolveAt('noun', uid(1000), maxGen) + + // The hot-tail window holds at most W generations' worth of ids; one id per + // gen here ⇒ exactly W resident recent chains + W inverse-index entries. + expect(s.recentNounChains.size).toBeLessThanOrEqual(W) + expect(s.windowNounDeltas.size).toBeLessThanOrEqual(W) + expect(s.recentNounChains.size).toBeLessThan(N) + + // Deep-read many distinct ids → the cold LRU is capped at L regardless. + for (let i = 0; i < 40; i++) await store.resolveAt('noun', uid(1000 + i), 1) + expect(s.coldNounChains.size).toBeLessThanOrEqual(L) + }) + + // ========================================================================== + // 3. Held-Db across cold eviction + a concurrent compaction (lock-light). + // ========================================================================== + it('a held pin reads correctly across cold eviction AND a concurrent compact()', async () => { + const s = store as any + s.recentWindowGenerations = 4 + s.coldChainLruMax = 4 + + const heldIds = [uid(500), uid(501), uid(502)] + const filler = (i: number) => uid(600 + i) + + // Each held id gets value 10/11/12 at g_old, then a later mutation so a + // before-image exists to resolve. + const gOldVals = new Map() + let pOld = 0 + for (let k = 0; k < heldIds.length; k++) { + const val = 10 + k + pOld = await writeNoun(heldIds[k], val) + gOldVals.set(heldIds[k], val) + } + const gOld = pOld // pin: at gOld every held id holds its value above + store.pin(gOld) + + // Mutate the held ids AFTER the pin (so resolveAt resolves a before-image), + // then slide the window well past gOld with lots of filler writes. + for (const id of heldIds) await writeNoun(id, 99) + for (let i = 0; i < 30; i++) await writeNoun(filler(i), 700 + i) + await store.flushPendingSingleOps() + + // Populate then evict the held ids' cold chains by cycling other deep ids. + for (const id of heldIds) await store.resolveAt('noun', id, gOld) + for (let i = 0; i < 12; i++) await store.resolveAt('noun', filler(i), 1) + + // Re-read the held ids WHILE a compaction (which reclaims everything ≤ the + // pin) runs concurrently. Lock-light reconstruction must skip the reclaimed + // gens (all ≤ minPinned, never an answer) and still match the at-gOld oracle. + const reads = heldIds.map((id) => + withTimeout(store.resolveAt('noun', id, gOld), 5000, `held read ${id}`) + ) + const compaction = withTimeout(store.compact(), 5000, 'concurrent compact') + const [r0, r1, r2] = await Promise.all(reads) + await compaction + + for (const [id, res] of [ + [heldIds[0], r0], + [heldIds[1], r1], + [heldIds[2], r2] + ] as const) { + expect(res.source).toBe('record') + if (res.source === 'record') expect(res.metadata.version).toBe(gOldVals.get(id)) + } + + // And a fresh read after compaction still matches (reconstruct over survivors). + for (const id of heldIds) { + const res = await store.resolveAt('noun', id, gOld) + expect(res.source).toBe('record') + if (res.source === 'record') expect(res.metadata.version).toBe(gOldVals.get(id)) + } + store.release(gOld) + }) + + // ========================================================================== + // 4. Un-flushed pending single-ops — flush-agnostic before-images. + // ========================================================================== + it('a now()-style pin reads pre-mutation before-images of un-flushed single-ops, identical after flush', async () => { + const X = uid(800) + const g1 = await writeNoun(X, 1) // flushed below to make g1 durable history + + // More UN-FLUSHED single-ops mutating the same id. + await writeNoun(X, 2) + await writeNoun(X, 3) + expect(store.committedGeneration()).toBe(0) // nothing on disk yet + + const before = await store.resolveAt('noun', X, g1) + expect(before.source).toBe('record') + if (before.source === 'record') expect(before.metadata.version).toBe(1) + + // Flush → the answer is identical (resolution is flush-agnostic). + await store.flushPendingSingleOps() + const after = await store.resolveAt('noun', X, g1) + expect(norm(after)).toBe(norm(before)) + }) + + // ========================================================================== + // 6. resolveManyAt ≡ per-id resolveAt (random id-sets × random pins). + // ========================================================================== + it('resolveManyAt equals per-id resolveAt for random id-sets across recent and deep pins', async () => { + const s = store as any + s.recentWindowGenerations = 6 + s.coldChainLruMax = 8 + + const ids = Array.from({ length: 8 }, (_, i) => uid(2000 + i)) + const rand = mulberry32(99) + let v = 0 + for (let i = 0; i < 50; i++) await writeNoun(ids[Math.floor(rand() * ids.length)], ++v) + await store.flushPendingSingleOps() + const maxGen = store.generation() + + /** Map a resolveManyAt firstAfter entry to the same scalar resolveAt yields. */ + const derived = async (id: string, firstAfter: Map): Promise => { + const g = firstAfter.get(id) + if (g === undefined) return 'current' + const rec = await store.readGenerationRecord('noun', g, id) + if (!rec || (rec.metadata === null && rec.vector === null)) return 'absent' + return `rec:${(rec.metadata as any)?.version}` + } + + for (let pin = 0; pin <= maxGen; pin++) { + // A random subset of ids for this pin. + const subset = new Set(ids.filter(() => rand() < 0.6)) + if (subset.size === 0) subset.add(ids[0]) + const many = await store.resolveManyAt('noun', subset, pin) + for (const id of subset) { + expect(await derived(id, many)).toBe(norm(await store.resolveAt('noun', id, pin))) + } + } + }) + + // ========================================================================== + // 7. Write path is I/O-free across window slides (the ring supplies slid-out + // ids — no tx.json reads on commit, even with W > deltaCacheMax). + // ========================================================================== + it('committing across window slides reads ZERO tx.json from disk (W > deltaCacheMax)', async () => { + const s = store as any + s.recentWindowGenerations = 6 // W + s.deltaCacheMax = 2 // W > deltaCacheMax: a naive slide would re-read tx.json + + // Seed + build the window so subsequent commits drive slides. + for (let i = 0; i < 8; i++) await writeNoun(uid(3000 + i), i) + await store.flushPendingSingleOps() + await store.resolveAt('noun', uid(3000), store.generation()) + + // Count tx.json reads from this point on. + let txReads = 0 + const realRead = storage.readRawObject.bind(storage) + ;(storage as any).readRawObject = async (path: string) => { + if (typeof path === 'string' && path.endsWith('/tx.json')) txReads++ + return realRead(path) + } + try { + // Many more commits, each sliding the window past evicted-from-cache gens. + for (let i = 0; i < 20; i++) await writeNoun(uid(3100 + i), 1000 + i) + } finally { + ;(storage as any).readRawObject = realRead + } + expect(txReads).toBe(0) + }) + + // ========================================================================== + // 8. Compaction invalidates + rebuilds the window over survivors; a pin below + // the new horizon throws GenerationCompactedError. + // ========================================================================== + it('rebuilds the window over survivors after compaction and rejects pins below the horizon', async () => { + const s = store as any + s.recentWindowGenerations = 4 + + const X = uid(900) + const g0 = await writeNoun(X, 0) + for (let v = 1; v <= 6; v++) await writeNoun(X, v) + await store.flushPendingSingleOps() + const maxGen = store.generation() + + // Build the window, then compact to keep only the 2 most recent gens. + await store.resolveAt('noun', X, maxGen) + const res = await store.compact({ maxGenerations: 2 }) + expect(res.removedGenerations).toBeGreaterThan(0) + expect(s.windowReady).toBe(false) // invalidated + + // A pin ABOVE the horizon rebuilds the window from survivors and resolves. + const aboveHorizon = res.horizon + 1 + const resolved = await store.resolveAt('noun', X, aboveHorizon) + expect(['record', 'current']).toContain(resolved.source) + + // A pin below the horizon is unreachable. + expect(() => store.assertReachable(g0)).toThrow(GenerationCompactedError) + }) +}) + +// ============================================================================ +// 5. Deep-pin materialize — O(R) getDelta (not O(N·R)) + no mutex re-entrancy. +// ============================================================================ +describe('materializeAtGeneration — bounded & deadlock-free (GA #33)', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + }) + afterEach(async () => { + await brain.close() + }) + + it('resolveManyAt + readGenerationRecord do NOT re-enter the commit mutex (safe inside snapshotWith)', async () => { + const store = (brain as any).generationStore + const id = await brain.add({ data: 'a', type: NounType.Document, subtype: 'note', vector: VEC }) + await brain.add({ data: 'b', type: NounType.Document, subtype: 'note', vector: VEC }) + const pin = 1 + + // snapshotWith HOLDS the commit mutex. If resolveManyAt/readGenerationRecord + // took the mutex, this section would deadlock — assert it completes promptly. + const out = await withTimeout( + store.snapshotWith(async () => { + const many = await store.resolveManyAt('noun', new Set([id]), pin) + const rec = many.has(id) ? await store.readGenerationRecord('noun', many.get(id), id) : null + return { many, rec } + }), + 3000, + 'snapshotWith + resolveManyAt/readGenerationRecord' + ) + expect(out.many).toBeInstanceOf(Map) + }) + + it('a deep-pin materialize over many ids invokes getDelta O(R) (not O(N·R)) and matches a per-id oracle', async () => { + const store = (brain as any).generationStore + + const N = 400 + for (let i = 0; i < N; i++) { + await brain.add({ data: `doc ${i}`, type: NounType.Document, subtype: 'note', metadata: { i }, vector: VEC }) + } + const R = brain.generation() // ≈ N (each add is its own generation) + expect(R).toBeGreaterThanOrEqual(N) + + const deepGen = 1 + + // Count getDelta invocations during the materialize. + const realGetDelta = store.getDelta.bind(store) + let getDeltaCalls = 0 + store.getDelta = async (g: number) => { + getDeltaCalls++ + return realGetDelta(g) + } + + let handle: any + try { + handle = await withTimeout( + (brain as any).materializeAtGeneration(deepGen), + 15000, + 'deep materialize' + ) + } finally { + store.getDelta = realGetDelta + } + + // O(R): a small constant number of ascending passes (changedBetween + + // resolveManyAt per kind), NOT a per-id rescan (which would be ~N·R). + expect(getDeltaCalls).toBeLessThan(R * 5) + expect(getDeltaCalls).toBeLessThan(N * N) // the regression guard + + // The materialized at-gen-1 brain holds exactly the one entity that existed. + const atGen1 = await handle.find({ limit: N + 10 }) + expect(atGen1.length).toBe(1) + await handle.close() + }) + + it('a materialize completes (no deadlock) under a forced concurrent commit', async () => { + const N = 120 + for (let i = 0; i < N; i++) { + await brain.add({ data: `x ${i}`, type: NounType.Document, subtype: 'note', vector: VEC }) + } + const deepGen = 1 + + // Fire the materialize and several commits together: the reconciliation pass + // (under snapshotWith's mutex) must resolve the raced ids via the mutex-free + // bulk path without deadlocking. + const matPromise = withTimeout( + (brain as any).materializeAtGeneration(deepGen), + 15000, + 'materialize under concurrent commits' + ) + const commits = Promise.all( + Array.from({ length: 5 }, (_, i) => + brain.add({ data: `concurrent ${i}`, type: NounType.Document, subtype: 'note', vector: VEC }) + ) + ) + const [handle] = await Promise.all([matPromise, commits]) + expect(handle).toBeTruthy() + await handle.close() + }) +}) diff --git a/tests/unit/db/fact-log.test.ts b/tests/unit/db/fact-log.test.ts new file mode 100644 index 00000000..f1c226cc --- /dev/null +++ b/tests/unit/db/fact-log.test.ts @@ -0,0 +1,237 @@ +/** + * @module tests/unit/db/fact-log + * @description The generation fact log in isolation: wire-format round-trip + * (positional msgpack facts, bin16 uuids, body-less tombstones), crc32c + * framing with torn-tail detection, open-time truncation to committed truth + * (the log can only ever be AHEAD after a crash; open cuts it back), rotation + * with a manifest-first flip, exactly-once scans with the frozen telemetry + * shape, and the mmap segment handoff excluding the mutable tail. + */ +import { describe, it, expect, beforeEach } from 'vitest' +import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' +import { + FactLog, + FACTS_PREFIX, + type CommitFact, + type FactLogStorage, + storageSupportsFactLog +} from '../../../src/db/factLog.js' +import { crc32c } from '../../../src/utils/crc32c.js' + +const UUID = (n: number): string => + `00000000-0000-4000-8000-${String(n).padStart(12, '0')}` + +const fact = (generation: number, overrides?: Partial): CommitFact => ({ + generation, + timestamp: 1_700_000_000_000 + generation, + ops: [ + { + kind: 'noun', + id: UUID(generation), + record: { metadata: { noun: 'document', title: `doc ${generation}` }, vector: { v: [1, 2] } } + } + ], + ...overrides +}) + +describe('crc32c known-answer vectors', () => { + it('matches the RFC 3720 test vectors', () => { + expect(crc32c(new TextEncoder().encode('123456789'))).toBe(0xe3069283) + expect(crc32c(new Uint8Array(32))).toBe(0x8a9136aa) + }) +}) + +describe('fact log — round-trip, framing, reconcile, rotation, scan', () => { + let storage: FactLogStorage + let log: FactLog + + beforeEach(async () => { + const mem: any = new MemoryStorage() + await mem.init() + expect(storageSupportsFactLog(mem)).toBe(true) + storage = mem + log = new FactLog(storage) + await log.open(0) + }) + + it('facts round-trip byte-exactly: ops, tombstones, meta, blobHashes', async () => { + await log.append(fact(1)) + await log.append( + fact(2, { + ops: [ + { kind: 'verb', id: UUID(21), record: { metadata: { verb: 'contains' }, vector: null } }, + { kind: 'noun', id: UUID(22), record: null } // TOMBSTONE + ], + meta: { source: 'test' }, + blobHashes: ['abc123', 'abc123'] // multiset — duplicates preserved + }) + ) + await log.sync() + + const scan = log.scanFacts() + expect(scan.headGeneration).toBe(2) + const all: CommitFact[] = [] + for await (const batch of scan.batches()) all.push(...batch.facts) + + expect(all).toHaveLength(2) + expect(all[0].generation).toBe(1) + expect(all[0].ops[0].id).toBe(UUID(1)) + expect(all[0].ops[0].record?.metadata).toEqual({ noun: 'document', title: 'doc 1' }) + expect(all[1].ops[0].kind).toBe('verb') + expect(all[1].ops[1].record).toBeNull() // the tombstone is body-less + expect(all[1].meta).toEqual({ source: 'test' }) + expect(all[1].blobHashes).toEqual(['abc123', 'abc123']) + expect(scan.summary().factsYielded).toBe(2) + }) + + it('appends are monotonic — a replayed/duplicate generation throws', async () => { + await log.append(fact(5)) + await expect(log.append(fact(5))).rejects.toThrow(/non-monotonic/) + await expect(log.append(fact(3))).rejects.toThrow(/non-monotonic/) + await expect(log.append(fact(6))).resolves.toBeUndefined() // gaps are fine (aborted reservations) + }) + + it('a torn tail (partial frame) is detected and ignored — intact prefix survives', async () => { + await log.append(fact(1)) + await log.append(fact(2)) + await log.sync() + + // Simulate a crash mid-append: chop bytes off the tail file. + const tailPath = `${FACTS_PREFIX}/seg-${'1'.padStart(20, '0')}.bfl` + const bytes = (await storage.readRawBytes(tailPath))! + await storage.writeRawBytes(tailPath, bytes.subarray(0, bytes.length - 7)) + + const reopened = new FactLog(storage) + await reopened.open(2) + expect(reopened.headGeneration()).toBe(1) // fact 2's frame was torn → gone + + const all: CommitFact[] = [] + for await (const b of reopened.scanFacts().batches()) all.push(...b.facts) + expect(all.map((f) => f.generation)).toEqual([1]) + }) + + it('open() truncates facts beyond committed truth (the crash-ahead shape)', async () => { + await log.append(fact(1)) + await log.append(fact(2)) + await log.append(fact(3)) + await log.sync() + + // The store's committed generation is 1 — facts 2..3 never committed. + const reopened = new FactLog(storage) + await reopened.open(1) + expect(reopened.headGeneration()).toBe(1) + + const all: CommitFact[] = [] + for await (const b of reopened.scanFacts().batches()) all.push(...b.facts) + expect(all.map((f) => f.generation)).toEqual([1]) + + // And appends continue cleanly from the truncated head. + await reopened.append(fact(2)) + expect(reopened.headGeneration()).toBe(2) + }) + + it('a cleared store (committed=0) truncates everything', async () => { + await log.append(fact(1)) + await log.append(fact(2)) + await log.sync() + const reopened = new FactLog(storage) + await reopened.open(0) + expect(reopened.headGeneration()).toBe(0) + }) + + it('scan honors fromGeneration/toGeneration inclusively and filters kinds', async () => { + for (let g = 1; g <= 6; g++) await log.append(fact(g)) + await log.sync() + + const scan = log.scanFacts({ fromGeneration: 2, toGeneration: 4 }) + const all: CommitFact[] = [] + for await (const b of scan.batches()) all.push(...b.facts) + expect(all.map((f) => f.generation)).toEqual([2, 3, 4]) + + const verbsOnly = log.scanFacts({ kinds: ['verb'] }) + for await (const b of verbsOnly.batches()) { + for (const f of b.facts) expect(f.ops.every((op) => op.kind === 'verb')).toBe(true) + } + }) + + it('batch telemetry carries the frozen shape', async () => { + for (let g = 1; g <= 5; g++) await log.append(fact(g)) + await log.sync() + + const scan = log.scanFacts({ batchSize: 2 }) + expect(scan.approxFactCount).toBe(5) + const batches = [] + for await (const b of scan.batches()) batches.push(b) + expect(batches.length).toBe(3) + expect(batches[0]).toMatchObject({ firstGeneration: 1, lastGeneration: 2, factCount: 2 }) + expect(batches[0].byteSize).toBeGreaterThan(0) + expect(typeof batches[0].segmentId).toBe('string') + expect(scan.summary()).toEqual({ factsYielded: 5, segmentsRead: 1 }) + }) + + it('survives reopen: head and content come back from disk', async () => { + for (let g = 1; g <= 3; g++) await log.append(fact(g)) + await log.sync() + + const reopened = new FactLog(storage) + await reopened.open(3) + expect(reopened.headGeneration()).toBe(3) + const all: CommitFact[] = [] + for await (const b of reopened.scanFacts().batches()) all.push(...b.facts) + expect(all.map((f) => f.generation)).toEqual([1, 2, 3]) + }) + + it('segmentPaths excludes the mutable tail (mmap handoff = sealed only)', async () => { + await log.append(fact(1)) + await log.sync() + expect(log.segmentPaths()).toEqual([]) // only a tail exists — nothing sealed + }) + + describe('scanFacts liveness contract (Stage-2 D1)', () => { + it('a wedged store fails LOUDLY within the first-batch bound — never a silent hang', async () => { + // Force a sealed segment (tiny rotateBytes) so the scan must READ from + // storage, then wedge that read: the exact production shape (a + // backlogged brain whose segment read never returned). + const mem: any = new MemoryStorage() + await mem.init() + const wedgeable = new FactLog(mem, { rotateBytes: 1 }) + await wedgeable.open(0) + await wedgeable.append(fact(1)) + await wedgeable.append(fact(2)) // second append rotates → seg 1 sealed + await wedgeable.sync() + + const realRead = mem.readRawBytes.bind(mem) + mem.readRawBytes = (p: string) => + p.includes('facts/seg-') ? new Promise(() => {}) : realRead(p) // hangs forever + + const scan = wedgeable.scanFacts({ firstBatchTimeoutMs: 200 }) + const started = Date.now() + await expect(scan.batches().next()).rejects.toThrow(/no first batch within 200ms/) + expect(Date.now() - started).toBeLessThan(5_000) // bound held, not a hang + }) + + it('a healthy scan is unaffected — first batch well inside the bound, all facts delivered', async () => { + for (let g = 1; g <= 5; g++) await log.append(fact(g)) + await log.sync() + const scan = log.scanFacts({ batchSize: 2 }) + const all: CommitFact[] = [] + for await (const b of scan.batches()) all.push(...b.facts) + expect(all.map((f) => f.generation)).toEqual([1, 2, 3, 4, 5]) + expect(scan.summary().factsYielded).toBe(5) + }) + + it('consumer think-time between pulls never counts against the producer', async () => { + for (let g = 1; g <= 4; g++) await log.append(fact(g)) + await log.sync() + // Bound tighter than the consumer's pause: only the FIRST pull is + // raced, so a slow consumer after batch 1 must not trip the deadline. + const gen = log.scanFacts({ batchSize: 2, firstBatchTimeoutMs: 150 }).batches() + const first = await gen.next() + expect(first.done).toBe(false) + await new Promise((r) => setTimeout(r, 400)) // dawdle past the bound + const second = await gen.next() + expect(second.done).toBe(false) + expect((await gen.next()).done).toBe(true) + }) + }) +}) diff --git a/tests/unit/db/generation-chain.test.ts b/tests/unit/db/generation-chain.test.ts new file mode 100644 index 00000000..d00bbe65 --- /dev/null +++ b/tests/unit/db/generation-chain.test.ts @@ -0,0 +1,112 @@ +/** + * @module tests/unit/db/generation-chain + * @description Locks in the Model-B scalability fixes in `generationStore.ts`: + * the per-id history chains that make `resolveAt` O(log) instead of an + * O(database-age) scan of the global `committedGens`, the bounded `deltaCache` + * (evicted deltas are transparently re-read from storage), and chain + * maintenance across commit + compaction. These assert the OBSERVABLE behavior + * — `asOf()` correctness — across the code paths those fixes touch, including the + * edge cases the existing temporal suites don't reach (eviction past the cap, + * and chain rebuild after compaction). + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/index.js' +import { NounType } from '../../../src/types/graphTypes.js' +import { createTestConfig } from '../../helpers/test-factory.js' + +const X = '11111111-1111-4111-8111-111111111111' +const Y = '22222222-2222-4222-8222-222222222222' + +describe('generation history chains (Model-B scalability)', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + }) + afterEach(async () => { + await brain.close() + }) + + /** Seed X (its own committed generation), then return g0 = that generation. */ + async function seedX(): Promise { + const db = await brain.transact([ + { op: 'add', id: X, type: NounType.Document, subtype: 'note', data: 'x', metadata: { v: 0 } } + ]) + await db.release() + return brain.generation() + } + + async function bumpX(v: number): Promise { + const db = await brain.transact([{ op: 'update', id: X, metadata: { v } }]) + await db.release() + return brain.generation() + } + + const vAt = async (gen: number): Promise => { + const db = await brain.asOf(gen) + const e = (await db.get(X)) as any + await db.release() + return e?.metadata?.v + } + + it('resolveAt returns the value as-of each pinned generation (chain binary search)', async () => { + const g0 = await seedX() + const g1 = await bumpX(1) + const g2 = await bumpX(2) + + expect(await vAt(g0)).toBe(0) // before-image of the g1 update = the seed value + expect(await vAt(g1)).toBe(1) // before-image of the g2 update + expect(((await brain.get(X)) as any)?.metadata?.v).toBe(2) // live head + expect(g2).toBeGreaterThan(g1) + expect(g1).toBeGreaterThan(g0) + }) + + it('stays correct when an UNCHANGED entity is read at an old pin (no chain → O(1) current)', async () => { + const g0 = await seedX() + // Churn a DIFFERENT entity many times; X is never touched again. + const db = await brain.transact([{ op: 'add', id: Y, type: NounType.Document, subtype: 'note', data: 'y', metadata: { v: 0 } }]) + await db.release() + for (let i = 1; i <= 30; i++) { + const d = await brain.transact([{ op: 'update', id: Y, metadata: { v: i } }]) + await d.release() + } + // X has no chain entry after g0 → resolveAt returns 'current' without scanning the 31 Y-generations. + expect(await vAt(g0)).toBe(0) + expect(((await brain.get(X)) as any)?.metadata?.v).toBe(0) + }) + + it('survives deltaCache eviction (cap lowered; evicted deltas re-read from storage)', async () => { + ;(brain as any).generationStore.deltaCacheMax = 4 // force eviction well before the gen count + const g0 = await seedX() + for (let v = 1; v <= 12; v++) await bumpX(v) // 12 gens > cap 4 → evictions + + // asOf still resolves correctly (the window build re-read every evicted delta)… + expect(await vAt(g0)).toBe(0) + // …and a range op (since scans committedGens via getDelta, re-reading evicted deltas) still works. + const now = await brain.now() + const changed = await now.since(g0) + await now.release() + expect(changed.nouns).toContain(X) + }) + + it('rebuilds chains after compaction and keeps asOf correct above the horizon', async () => { + const g0 = await seedX() + const g1 = await bumpX(1) + const g2 = await bumpX(2) + await bumpX(3) + await bumpX(4) + + // Reclaim everything except the 2 most recent committed generations. + const res = await brain.compactHistory({ maxGenerations: 2 }) + expect(res.removedGenerations).toBeGreaterThan(0) + + // A pin ABOVE the new horizon still resolves (chains were rebuilt from the survivors)… + const live = ((await brain.get(X)) as any)?.metadata?.v + expect(live).toBe(4) + // …and the reclaimed-depth pins are below the horizon now. + expect(res.horizon).toBeGreaterThanOrEqual(g0) + expect(g2).toBeGreaterThan(g1) + }) +}) diff --git a/tests/unit/db/generation-segments.test.ts b/tests/unit/db/generation-segments.test.ts new file mode 100644 index 00000000..27ab85cb --- /dev/null +++ b/tests/unit/db/generation-segments.test.ts @@ -0,0 +1,150 @@ +/** + * @module tests/unit/db/generation-segments + * @description The generation-segment store (Stage-2 D1+D3 file format). + * Laws: (1) fold → read round-trips deltas and records byte-faithfully via + * sidecar point-reads; (2) the manifest is the ONLY discovery path — reopen + * reads one file, never a listing; (3) a lost/corrupt sidecar rebuilds from + * its segment loudly, a damaged SEGMENT fails loudly (never silent wrong + * data); (4) D3 reclaim drops whole segments only and bumps compactedBelow; + * (5) the packed digest is deterministic across reopen; (6) immutability — + * fold refuses overlap with sealed ranges. + */ +import { describe, it, expect, beforeEach } from 'vitest' +import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' +import { + GenerationSegmentStore, + SEGMENTS_PREFIX, + type FoldGeneration +} from '../../../src/db/generationSegments.js' + +const UUID = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, '0')}` + +const gen = (g: number, recordCount = 2): FoldGeneration => ({ + generation: g, + timestamp: 1_700_000_000_000 + g, + delta: { generation: g, nouns: [UUID(g)], verbs: [], bytes: 123 + g }, + records: Array.from({ length: recordCount }, (_, i) => ({ + kind: (i % 2 === 0 ? 'noun' : 'verb') as 'noun' | 'verb', + id: UUID(g * 100 + i), + record: { metadata: { noun: 'document', v: g }, vector: { v: [g, i] } } + })) +}) + +describe('db/GenerationSegmentStore — the D1+D3 packed tier', () => { + let storage: MemoryStorage + let store: GenerationSegmentStore + + beforeEach(async () => { + storage = new MemoryStorage() + await storage.init() + store = new GenerationSegmentStore(storage as any) + await store.open() + }) + + it('fold → read round-trips deltas and records via sidecar point-reads', async () => { + const meta = await store.fold([gen(1), gen(2), gen(3)]) + expect(meta).toMatchObject({ firstGeneration: 1, lastGeneration: 3, frames: 3 }) + expect(meta.checksum).toBeGreaterThan(0) + + expect(store.hasGeneration(2)).toBe(true) + expect(store.hasGeneration(4)).toBe(false) + + const d2 = await store.readDelta(2) + expect(d2?.delta).toEqual({ generation: 2, nouns: [UUID(2)], verbs: [], bytes: 125 }) + expect(d2?.timestamp).toBe(1_700_000_000_002) + + const records = await store.readRecords(3) + expect(records).toHaveLength(2) + expect(records![0]).toEqual({ + kind: 'noun', + id: UUID(300), + record: { metadata: { noun: 'document', v: 3 }, vector: { v: [3, 0] } } + }) + // Point read by id, both kinds. + expect(await store.readRecord(3, 'verb', UUID(301))).toEqual({ + metadata: { noun: 'document', v: 3 }, + vector: { v: [3, 1] } + }) + expect(await store.readRecord(3, 'noun', UUID(999))).toBeNull() + }) + + it('reopen discovers everything from the manifest alone — no listing', async () => { + await store.fold([gen(1), gen(2)]) + await store.fold([gen(3), gen(4)]) + + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + expect(reopened.segments()).toHaveLength(2) + expect(reopened.hasGeneration(4)).toBe(true) + expect((await reopened.readDelta(1))?.timestamp).toBe(1_700_000_000_001) + }) + + it('a lost sidecar rebuilds from its segment; a damaged segment fails LOUDLY', async () => { + const meta = await store.fold([gen(1), gen(2)]) + const idxPath = `${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.idx` + await storage.deleteRawObject(idxPath) + + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + // Rebuild path: still serves correct data. + expect((await reopened.readRecords(2))!).toHaveLength(2) + + // Now damage the SEGMENT itself: flip a payload byte → CRC mismatch, loud. + const segPath = `${SEGMENTS_PREFIX}/${meta.file}` + const bytes = (await storage.readRawBytes(segPath))! + bytes[bytes.length - 3] ^= 0xff + await storage.writeRawBytes(segPath, bytes) + const damaged = new GenerationSegmentStore(storage as any) + await damaged.open() + ;(damaged as any).sidecars.clear() + await storage.deleteRawObject(idxPath) // force the sequential rebuild over damaged bytes + await expect(damaged.readRecords(2)).rejects.toThrow(/CRC mismatch|damaged/) + }) + + it('D3 reclaim drops whole segments only and bumps compactedBelow', async () => { + await store.fold([gen(1), gen(2)]) + await store.fold([gen(3), gen(4)]) + await store.fold([gen(5), gen(6)]) + + // Horizon mid-segment-2 (below 4): only segment 1 is FULLY below → drops. + const r1 = await store.dropSegmentsBelow(4) + expect(r1).toEqual({ dropped: 1, compactedBelow: 3 }) + expect(store.hasGeneration(1)).toBe(false) + expect(store.hasGeneration(3)).toBe(true) // partial segment survives whole + + // Bytes actually gone. + expect(await storage.readRawBytes(`${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.bgs`)).toBeNull() + + // Horizon past everything: the rest drop; compactedBelow is durable. + const r2 = await store.dropSegmentsBelow(7) + expect(r2.dropped).toBe(2) + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + expect(reopened.compactedBelow()).toBe(7) + expect(reopened.segments()).toHaveLength(0) + }) + + it('the packed digest is deterministic across reopen and changes with history', async () => { + await store.fold([gen(1), gen(2), gen(3)]) + const atSeal = await store.digestThroughPacked(3) + const midSegment = await store.digestThroughPacked(2) + expect(atSeal).not.toBeNull() + expect(midSegment).not.toBeNull() + expect(midSegment).not.toBe(atSeal) + + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + expect(await reopened.digestThroughPacked(3)).toBe(atSeal) + expect(await reopened.digestThroughPacked(2)).toBe(midSegment) + + await reopened.fold([gen(4)]) + expect(await reopened.digestThroughPacked(4)).not.toBe(atSeal) + }) + + it('sealed segments are immutable — fold refuses overlap, requires ascending input', async () => { + await store.fold([gen(1), gen(2)]) + await expect(store.fold([gen(2), gen(3)])).rejects.toThrow(/overlaps the packed tier/) + await expect(store.fold([gen(4), gen(4)])).rejects.toThrow(/strictly ascending/) + await expect(store.fold([])).rejects.toThrow(/at least one generation/) + }) +}) diff --git a/tests/unit/db/generationStore.test.ts b/tests/unit/db/generationStore.test.ts index f847b92a..5b667415 100644 --- a/tests/unit/db/generationStore.test.ts +++ b/tests/unit/db/generationStore.test.ts @@ -8,7 +8,7 @@ * resolution, and crash rollback of uncommitted generations. */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, vi } from 'vitest' import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' import { GenerationStore, @@ -249,16 +249,20 @@ describe('db/GenerationStore', () => { store.release(pinned) const result = await store.compact() expect(result.removedGenerations).toBeGreaterThan(0) - const remaining = await storage.listRawObjects(GENERATIONS_PREFIX) + // History record-sets only — the fact log (at `_generations/facts/`) is + // deliberately NOT reclaimed by history compaction. + const remaining = (await storage.listRawObjects(GENERATIONS_PREFIX)).filter( + (p: string) => !p.startsWith(`${GENERATIONS_PREFIX}/facts/`) + ) expect(remaining).toEqual([]) }) - it('retainGenerations keeps the most recent record-sets', async () => { + it('maxGenerations cap keeps the most recent record-sets', async () => { await commitWrite(ID_A, 1) await commitWrite(ID_A, 2) await commitWrite(ID_A, 3) - const result = await store.compact({ retainGenerations: 2 }) + const result = await store.compact({ maxGenerations: 2 }) expect(result.removedGenerations).toBe(1) expect(result.horizon).toBe(1) @@ -330,4 +334,249 @@ describe('db/GenerationStore', () => { expect(manifest.generation).toBe(2) }) }) + + // ========================================================================== + // Model-B single-op generation-stamping (per-write history + group-commit) + // ========================================================================== + describe('Model-B single-op generations', () => { + /** Apply one single-op write as its own generation (live write + buffered history). */ + async function singleOpWrite(id: string, version: number): Promise { + const { generation } = await store.commitSingleOp({ + touched: { nouns: [id] }, + execute: async () => { + await storage.saveNounMetadata(id, metadataFixture(version)) + } + }) + return generation + } + + it('each single-op write is its own generation, resolvable BEFORE any flush', async () => { + const g1 = await singleOpWrite(ID_A, 1) + const g2 = await singleOpWrite(ID_A, 2) + expect(g1).toBe(1) + expect(g2).toBe(2) + // Counter advanced per write; nothing persisted to disk yet. + expect(store.generation()).toBe(2) + expect(store.committedGeneration()).toBe(0) + // Reads resolve through the in-memory pending tier with NO forced flush: + // the state as-of g1 is v1 (the before-image buffered by g2). + const atG1 = await store.resolveAt('noun', ID_A, g1) + expect(atG1.source).toBe('record') + expect(((atG1 as { metadata: { version: number } }).metadata).version).toBe(1) + // hasCommittedAfter counts pending → a now()-style pin at g1 is historical. + expect(store.hasCommittedAfter(g1)).toBe(true) + }) + + it('range queries (changedBetween / generationsTouching) include un-flushed pending generations', async () => { + // Two single-op writes to different ids, NEITHER flushed (still in the + // pending tier). diff()/since()/history() are built on these, so they must + // see un-flushed single-ops with NO forced flush. + const g1 = await singleOpWrite(ID_A, 1) + const g2 = await singleOpWrite(ID_B, 1) + expect(store.committedGeneration()).toBe(0) // nothing on disk yet + + const changed = await store.changedBetween(0, store.generation()) + expect(changed.nouns).toEqual([ID_A, ID_B]) // resolves over committed ∪ pending + expect(await store.generationsTouching('noun', ID_A, 0, store.generation())).toEqual([g1]) + expect(await store.generationsTouching('noun', ID_B, 0, store.generation())).toEqual([g2]) + + // …and the range stays identical across the flush (now reading from disk). + await store.flushPendingSingleOps() + expect((await store.changedBetween(0, store.generation())).nouns).toEqual([ID_A, ID_B]) + }) + + it('flush persists pending single-op generations (groupCommit-marked) and advances committed', async () => { + await singleOpWrite(ID_A, 1) + await singleOpWrite(ID_A, 2) + await store.flushPendingSingleOps() + expect(store.committedGeneration()).toBe(2) + const delta = await storage.readRawObject(`${GENERATIONS_PREFIX}/2/tx.json`) + expect(delta.groupCommit).toBe(true) + expect(typeof delta.bytes).toBe('number') + // asOf resolution survives the flush (now reading before-images from disk). + const atG1 = await store.resolveAt('noun', ID_A, 1) + expect(((atG1 as { metadata: { version: number } }).metadata).version).toBe(1) + }) + + it('🛑 a crash mid group-commit flush DROPS the generation WITHOUT reverting the live write', async () => { + // gen 1: X = v1, flushed (durable). + await singleOpWrite(ID_A, 1) + await store.flushPendingSingleOps() + expect(store.committedGeneration()).toBe(1) + + // gen 2: X = v2 acknowledged to canonical storage; its history is buffered. + await singleOpWrite(ID_A, 2) + expect((await storage.readNounRaw(ID_A)).metadata).toMatchObject({ version: 2 }) + + // Crash before the manifest rename of the group-commit flush: the gen-2 + // dir (before-image v1) lands on disk; the manifest stays at gen 1. + store.setCommitFaultInjector((phase) => { + if (phase === 'before-manifest-rename') throw new Error('simulated crash mid-flush') + }) + await expect(store.flushPendingSingleOps()).rejects.toThrow('simulated crash mid-flush') + store.setCommitFaultInjector(undefined) + expect(await storage.readRawObject(`${GENERATIONS_PREFIX}/2/tx.json`)).not.toBeNull() + expect((await storage.readRawObject(MANIFEST_PATH)).generation).toBe(1) + + // Recover on a FRESH store (process restart: in-memory pending is gone). + const reopened = new GenerationStore(storage) + const result = await reopened.open() + expect(result.rolledBackGenerations).toBe(1) + expect(reopened.committedGeneration()).toBe(1) + + // THE CORRUPTION TRAP: drop-without-restore — the acknowledged live write + // (v2) is NOT reverted to the before-image (v1). Only the crashed flush + // window's HISTORY is lost; live data stays correct. + expect((await storage.readNounRaw(ID_A)).metadata).toMatchObject({ version: 2 }) + // The orphan dir is gone and the dropped generation number is never reissued. + expect(await storage.readRawObject(`${GENERATIONS_PREFIX}/2/tx.json`)).toBeNull() + expect(reopened.generation()).toBeGreaterThanOrEqual(2) + }) + + it('a clean reopen replays committed single-op history (asOf survives restart)', async () => { + await singleOpWrite(ID_A, 1) + await singleOpWrite(ID_A, 2) + await store.flushPendingSingleOps() + + const reopened = new GenerationStore(storage) + await reopened.open() + expect(reopened.committedGeneration()).toBe(2) + const atG1 = await reopened.resolveAt('noun', ID_A, 1) + expect(((atG1 as { metadata: { version: number } }).metadata).version).toBe(1) + }) + }) + + // ========================================================================== + // Retention CAPS (Model-B `retention` knob) + // ========================================================================== + describe('retention caps', () => { + async function manyGens(n: number): Promise { + for (let v = 1; v <= n; v++) await commitWrite(ID_A, v) + } + + it('maxBytes reclaims the oldest generations until total history fits the budget', async () => { + await manyGens(5) + const total = await store.historyBytes() + expect(total).toBeGreaterThan(0) + const perGen = total / 5 + const budget = Math.floor(perGen * 2.5) + const result = await store.compact({ maxBytes: budget }) + expect(result.removedGenerations).toBeGreaterThan(0) + expect(await store.historyBytes()).toBeLessThanOrEqual(budget) + }) + + it('maxAge keeps generations within the window and reclaims older ones', async () => { + await manyGens(3) + // A wide window keeps everything. + expect((await store.compact({ maxAge: 60_000 })).removedGenerations).toBe(0) + // Advance the clock 10s past the commits, then a 1s window reclaims all + // three (fake timers make the age comparison deterministic). + vi.useFakeTimers() + try { + vi.setSystemTime(Date.now() + 10_000) + expect((await store.compact({ maxAge: 1_000 })).removedGenerations).toBe(3) + } finally { + vi.useRealTimers() + } + }) + + it('no caps reclaims every unpinned generation; a pin protects newer ones', async () => { + await manyGens(3) + store.pin(2) // protects generations > 2 (i.e. gen 3) from reclamation + const result = await store.compact() + // gens 1 and 2 are ≤ the pin → reclaimable; gen 3 is pin-protected. + expect(result.removedGenerations).toBe(2) + store.release(2) + }) + + it('timeBudgetMs bounds a pass; the next pass resumes the same prefix', async () => { + await manyGens(4) + // A spent budget (0ms) stops before reclaiming anything — an early stop + // is a consistent prefix, never a partial generation. + const bounded = await store.compact({ timeBudgetMs: 0 }) + expect(bounded.removedGenerations).toBe(0) + expect(bounded.horizon).toBe(0) + // The next (unbounded) pass picks up exactly where the bounded one + // stopped and completes the same work. + const resumed = await store.compact() + expect(resumed.removedGenerations).toBe(4) + expect(resumed.horizon).toBe(4) + }) + }) + + // ========================================================================== + describe('history-bytes running total (the O(1) retention check)', () => { + /** A fresh walk with the cache dropped — ground truth for the invariant. */ + async function groundTruthBytes(): Promise { + ;(store as any).historyBytesTotal = null + return store.historyBytes() + } + + it('is seeded once, then maintained through commits WITHOUT re-walks', async () => { + await commitWrite(ID_A, 1) + await commitWrite(ID_A, 2) + const seeded = await store.historyBytes() + expect(seeded).toBe(await groundTruthBytes()) + + // From here every read must come from the running total, not a walk: + // getDelta re-reads are the walk's cost — commits must not trigger any. + const getDeltaSpy = vi.spyOn(store as any, 'getDelta') + await commitWrite(ID_B, 1) + const afterCommit = await store.historyBytes() + expect(getDeltaSpy).not.toHaveBeenCalled() + getDeltaSpy.mockRestore() + expect(afterCommit).toBe(await groundTruthBytes()) + }) + + it('stays exact through single-op group commits and compaction', async () => { + await commitWrite(ID_A, 1) + await store.historyBytes() // seed + // Single-op path: buffered generations flushed as one group commit. + await store.commitSingleOp({ + touched: { nouns: [ID_B] }, + execute: async () => { + await storage.saveNounMetadata(ID_B, metadataFixture(1)) + } + }) + await store.flushPendingSingleOps() + expect(await store.historyBytes()).toBe(await groundTruthBytes()) + + await store.historyBytes() // re-seed after ground-truth reset + await store.compact({ maxGenerations: 1 }) + expect(await store.historyBytes()).toBe(await groundTruthBytes()) + }) + + it('historyStats reports counts, bytes, range, and horizon read-only', async () => { + await commitWrite(ID_A, 1) + await commitWrite(ID_B, 1) + const stats = await store.historyStats() + expect(stats.generations).toBe(2) + expect(stats.bytes).toBe(await store.historyBytes()) + expect(stats.oldestGeneration).toBe(1) + expect(stats.newestGeneration).toBe(2) + expect(stats.oldestTimestamp).toBeLessThanOrEqual(stats.newestTimestamp!) + expect(stats.horizon).toBe(0) + // Read-only: nothing was reclaimed by asking. + expect(store.committedGeneration()).toBe(2) + + await store.compact({ maxGenerations: 1 }) + const after = await store.historyStats() + expect(after.generations).toBe(1) + expect(after.oldestGeneration).toBe(2) + expect(after.horizon).toBe(1) + }) + + it('empty history reports null range and zero bytes', async () => { + const stats = await store.historyStats() + expect(stats).toMatchObject({ + generations: 0, + bytes: 0, + oldestGeneration: null, + newestGeneration: null, + oldestTimestamp: null, + newestTimestamp: null, + horizon: 0 + }) + }) + }) }) diff --git a/tests/unit/db/pending-flush-durability.test.ts b/tests/unit/db/pending-flush-durability.test.ts new file mode 100644 index 00000000..f8501c46 --- /dev/null +++ b/tests/unit/db/pending-flush-durability.test.ts @@ -0,0 +1,106 @@ +/** + * @module tests/unit/db/pending-flush-durability + * @description Finding 8: the async group-commit flush that persists single-op + * generation HISTORY must not swallow persist failures. Before the fix a failed + * background flush was a bare `prodLog.warn` — writes kept succeeding while their + * before-images silently piled up in memory, never durable and unbounded. + * + * The contract now: + * - a single transient flush failure does NOT trip the alarm (tolerance), and + * - after PENDING_FLUSH_FAILURE_THRESHOLD consecutive failures the store LATCHES + * and REFUSES further writes with a typed PendingFlushDurabilityError rather + * than accumulate un-durable history, and + * - it self-heals: once a flush finally succeeds the latch lifts and writes resume. + * Live canonical data is never touched — only the immutable history is stuck. + * + * Fake timers keep the background retry/coalesce timers from firing, so the exact + * number of flush attempts (and thus the latch point) is deterministic — the test + * drives every flush explicitly. + */ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' +import { GenerationStore } from '../../../src/db/generationStore.js' +import { PendingFlushDurabilityError } from '../../../src/db/errors.js' +import { NounType } from '../../../src/types/graphTypes.js' + +const ID = (suffix: string): string => `00000000-0000-4000-8000-0000000000${suffix}` +const meta = (v: number): Record => ({ + noun: NounType.Document, + subtype: 'note', + data: `payload-v${v}`, + version: v, + _rev: v +}) + +describe('db/GenerationStore pending-flush durability (finding 8)', () => { + let storage: MemoryStorage + let store: GenerationStore + + beforeEach(async () => { + vi.useFakeTimers() + storage = new MemoryStorage() + await storage.init() + store = new GenerationStore(storage) + await store.open() + // High size threshold so single-ops never auto-flush — the test owns every flush. + ;(store as any).pendingFlushThreshold = 1000 + }) + + afterEach(() => { + vi.useRealTimers() + }) + + /** One single-op write that buffers a pending generation. */ + const singleOp = (id: string, v: number) => + store.commitSingleOp({ + touched: { nouns: [id], verbs: [] }, + execute: async () => { + await storage.saveNounMetadata(id, meta(v)) + } + }) + + it('latches after repeated flush failures, refuses writes, then self-heals', async () => { + // Buffer a pending generation while storage is healthy. + await singleOp(ID('aa'), 1) + + // Fault the HISTORY flush (raw generation-record writes) — NOT the live + // canonical write, which uses a different path (saveNounMetadata). + const fault = Object.assign(new Error('EIO simulated'), { code: 'EIO' }) + const spy = vi.spyOn(storage as any, 'writeRawObject').mockRejectedValue(fault) + + // Each explicit flush fails and is accounted; the third trips the latch. + for (let i = 0; i < 3; i++) { + await expect(store.flushPendingSingleOps()).rejects.toThrow('EIO simulated') + } + + // Writes are now refused LOUDLY rather than piling up un-durable history. + await expect(singleOp(ID('bb'), 1)).rejects.toBeInstanceOf(PendingFlushDurabilityError) + // The live counter did not advance for the refused write (no generation consumed). + expect(store.generation()).toBe(1) + + // Storage recovers → an explicit flush drains the retained tier and lifts the latch. + spy.mockRestore() + await expect(store.flushPendingSingleOps()).resolves.toBeUndefined() + + // Writes resume; the next single-op commits normally. + const receipt = await singleOp(ID('cc'), 1) + expect(receipt.generation).toBe(2) + }) + + it('a single transient flush failure does NOT latch — writes continue', async () => { + await singleOp(ID('aa'), 1) + + const spy = vi.spyOn(storage as any, 'writeRawObject').mockRejectedValue(new Error('blip')) + await expect(store.flushPendingSingleOps()).rejects.toThrow('blip') // failure #1, below threshold + spy.mockRestore() + + // Below the latch threshold → the store still accepts writes. + const receipt = await singleOp(ID('bb'), 1) + expect(receipt.generation).toBe(2) + + // A subsequent healthy flush drains the tier and resets the failure counter. + await expect(store.flushPendingSingleOps()).resolves.toBeUndefined() + expect((store as any).pendingFlushFailures).toBe(0) + expect((store as any).pendingFlushError).toBeNull() + }) +}) diff --git a/tests/unit/db/whereMatcher.test.ts b/tests/unit/db/whereMatcher.test.ts index 42f88a7a..2223117c 100644 --- a/tests/unit/db/whereMatcher.test.ts +++ b/tests/unit/db/whereMatcher.test.ts @@ -80,10 +80,9 @@ describe('db/whereMatcher — operators', () => { expect(whereMatches(e, { status: 'closed' })).toBe(false) }) - it('eq / equals / is aliases', () => { + it('eq / equals aliases', () => { expect(whereMatches(e, { amount: { eq: 250 } })).toBe(true) expect(whereMatches(e, { amount: { equals: 250 } })).toBe(true) - expect(whereMatches(e, { amount: { is: 250 } })).toBe(true) expect(whereMatches(e, { amount: { eq: 99 } })).toBe(false) }) @@ -93,10 +92,9 @@ describe('db/whereMatcher — operators', () => { expect(whereMatches(e, { tags: 'missing-tag' })).toBe(false) }) - it('ne / notEquals / isNot aliases', () => { + it('ne / notEquals aliases', () => { expect(whereMatches(e, { status: { ne: 'closed' } })).toBe(true) expect(whereMatches(e, { status: { notEquals: 'open' } })).toBe(false) - expect(whereMatches(e, { status: { isNot: 'open' } })).toBe(false) }) it('in / oneOf set membership', () => { @@ -109,12 +107,10 @@ describe('db/whereMatcher — operators', () => { expect(whereMatches(e, { amount: { greaterThan: 250 } })).toBe(false) expect(whereMatches(e, { amount: { gte: 250 } })).toBe(true) expect(whereMatches(e, { amount: { greaterThanOrEqual: 251 } })).toBe(false) - expect(whereMatches(e, { amount: { greaterEqual: 250 } })).toBe(true) expect(whereMatches(e, { amount: { lt: 251 } })).toBe(true) expect(whereMatches(e, { amount: { lessThan: 250 } })).toBe(false) expect(whereMatches(e, { amount: { lte: 250 } })).toBe(true) expect(whereMatches(e, { amount: { lessThanOrEqual: 249 } })).toBe(false) - expect(whereMatches(e, { amount: { lessEqual: 250 } })).toBe(true) }) it('string range comparison is lexicographic', () => { diff --git a/tests/unit/get-index-status-readiness.test.ts b/tests/unit/get-index-status-readiness.test.ts new file mode 100644 index 00000000..7f82ec5d --- /dev/null +++ b/tests/unit/get-index-status-readiness.test.ts @@ -0,0 +1,59 @@ +/** + * @module tests/unit/get-index-status-readiness + * @description Pattern-A / Finding 9: getIndexStatus reported `populated: size>0` + * and only `migrating`, so a native index that loaded its count but not its + * serving structure reported populated:true — a k8s readiness probe would 200 a + * brain that serves []. It now folds in the honest isReady() signal + the + * _indexRebuildFailed / _indexDegradedIds degraded states (mirroring + * validateIndexConsistency / checkHealth). + */ +import { describe, it, expect, beforeEach } from 'vitest' +import { Brainy, NounType } from '../../src/index.js' + +describe('getIndexStatus honest readiness (Finding 9)', () => { + let brain: any + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, dimensions: 384 }) + await brain.init() + await brain.add({ data: 'x', type: NounType.Concept }) + await brain.flush() + }) + + it('a not-ready provider makes populated honest (false) and exposes ready:false', async () => { + brain.index.isReady = () => false // count present, serving structure NOT loaded + const status = await brain.getIndexStatus() + expect(status.hnswIndex.populated).toBe(false) + expect(status.hnswIndex.ready).toBe(false) + delete brain.index.isReady + }) + + it('a ready provider reports populated:true + ready:true', async () => { + brain.index.isReady = () => true + const status = await brain.getIndexStatus() + expect(status.hnswIndex.populated).toBe(true) + expect(status.hnswIndex.ready).toBe(true) + delete brain.index.isReady + }) + + it('rebuildFailed / rebuildError surface the init-degraded state', async () => { + brain._indexRebuildFailed = new Error('boom') + const status = await brain.getIndexStatus() + expect(status.rebuildFailed).toBe(true) + expect(status.rebuildError).toBe('boom') + brain._indexRebuildFailed = null + }) + + it('degradedIds surfaces the adopt-forward degraded set', async () => { + brain._indexDegradedIds.add('00000000-0000-4000-8000-0000000000de') + const status = await brain.getIndexStatus() + expect(status.degradedIds).toBe(1) + brain._indexDegradedIds.clear() + }) + + it('a JS-baseline provider (no isReady) omits ready and falls back to size>0', async () => { + const status = await brain.getIndexStatus() + expect(status.hnswIndex.ready).toBeUndefined() + expect(status.hnswIndex.populated).toBe(brain.index.size() > 0) + }) +}) diff --git a/tests/unit/graph/analyticsFallback.test.ts b/tests/unit/graph/analyticsFallback.test.ts new file mode 100644 index 00000000..9686172e --- /dev/null +++ b/tests/unit/graph/analyticsFallback.test.ts @@ -0,0 +1,163 @@ +/** + * @module tests/unit/graph/analyticsFallback + * @description Direct unit tests for the pure-TS graph-analytics kernels + * (`connectedComponents`, `stronglyConnectedComponents`, `pageRank`, `MinHeap`). + * These cover the algorithmic edge cases — empty graphs, dangling PageRank mass, + * nested SCCs, heap ordering — without the cost of a full Brainy instance. The + * `brain.graph.*` surface is tested end-to-end in graph-analytics.test.ts. + */ + +import { describe, it, expect } from 'vitest' +import { + connectedComponents, + stronglyConnectedComponents, + pageRank, + MinHeap +} from '../../../src/graph/analyticsFallback.js' + +/** Group node indices that share a label into sorted sets for stable comparison. */ +function groupsOf(labels: number[]): number[][] { + const byLabel = new Map() + labels.forEach((label, i) => { + const arr = byLabel.get(label) + if (arr) arr.push(i) + else byLabel.set(label, [i]) + }) + return [...byLabel.values()].map((g) => g.sort((a, b) => a - b)).sort((a, b) => a[0] - b[0]) +} + +describe('connectedComponents (weakly-connected, undirected projection)', () => { + it('labels an empty graph with no components', () => { + expect(connectedComponents([])).toEqual([]) + }) + + it('treats every isolated node as its own component', () => { + expect(groupsOf(connectedComponents([[], [], []]))).toEqual([[0], [1], [2]]) + }) + + it('merges across edge direction (0→1, 2→1 ⇒ one group)', () => { + // 0 → 1, 2 → 1, 3 isolated + const labels = connectedComponents([[1], [], [1], []]) + expect(groupsOf(labels)).toEqual([[0, 1, 2], [3]]) + }) + + it('keeps disjoint clusters separate', () => { + // 0↔1 , 2↔3 + const labels = connectedComponents([[1], [0], [3], [2]]) + expect(groupsOf(labels)).toEqual([ + [0, 1], + [2, 3] + ]) + }) +}) + +describe('stronglyConnectedComponents (directed, Tarjan)', () => { + it('labels an empty graph with no components', () => { + expect(stronglyConnectedComponents([])).toEqual([]) + }) + + it('splits a directed chain into singletons (no mutual reachability)', () => { + // 0 → 1 → 2 + expect(groupsOf(stronglyConnectedComponents([[1], [2], []]))).toEqual([[0], [1], [2]]) + }) + + it('collapses a directed cycle into one SCC', () => { + // 0 → 1 → 2 → 0 + expect(groupsOf(stronglyConnectedComponents([[1], [2], [0]]))).toEqual([[0, 1, 2]]) + }) + + it('separates a cycle from the acyclic tail that feeds it', () => { + // 3 → 0 → 1 → 2 → 0 : {0,1,2} is an SCC, 3 is its own SCC + const labels = stronglyConnectedComponents([[1], [2], [0], [0]]) + expect(groupsOf(labels)).toEqual([[0, 1, 2], [3]]) + }) + + it('handles two independent cycles', () => { + // 0↔1 cycle, 2↔3 cycle + const labels = stronglyConnectedComponents([ + [1], + [0], + [3], + [2] + ]) + expect(groupsOf(labels)).toEqual([ + [0, 1], + [2, 3] + ]) + }) + + it('produces contiguous community indices', () => { + const labels = stronglyConnectedComponents([[1], [2], [0], []]) + const distinct = new Set(labels) + expect(Math.max(...labels)).toBe(distinct.size - 1) + }) +}) + +describe('pageRank', () => { + it('returns an empty vector for an empty graph', () => { + expect(pageRank([]).length).toBe(0) + }) + + it('is uniform on a symmetric cycle', () => { + // 0 → 1 → 2 → 0 — every node identical by symmetry. + const scores = pageRank([[1], [2], [0]]) + expect(scores[0]).toBeCloseTo(1 / 3, 6) + expect(scores[1]).toBeCloseTo(1 / 3, 6) + expect(scores[2]).toBeCloseTo(1 / 3, 6) + }) + + it('ranks a hub (high in-degree) above its sources', () => { + // 0 → 3, 1 → 3, 2 → 3 : node 3 is the hub. + const scores = pageRank([[3], [3], [3], []]) + expect(scores[3]).toBeGreaterThan(scores[0]) + expect(scores[3]).toBeGreaterThan(scores[1]) + expect(scores[3]).toBeGreaterThan(scores[2]) + }) + + it('redistributes dangling mass so scores sum to 1', () => { + // Node 3 is dangling (no out-edges) — its mass must not leak away. + const scores = pageRank([[3], [3], [3], []]) + const total = scores.reduce((sum, s) => sum + s, 0) + expect(total).toBeCloseTo(1, 6) + }) + + it('sums to 1 even when every node is dangling (no edges)', () => { + const scores = pageRank([[], [], []]) + const total = scores.reduce((sum, s) => sum + s, 0) + expect(total).toBeCloseTo(1, 6) + expect(scores[0]).toBeCloseTo(1 / 3, 6) + }) +}) + +describe('MinHeap', () => { + it('pops in ascending priority order', () => { + const heap = new MinHeap() + heap.push('c', 3) + heap.push('a', 1) + heap.push('b', 2) + heap.push('d', 4) + expect([heap.pop(), heap.pop(), heap.pop(), heap.pop()]).toEqual(['a', 'b', 'c', 'd']) + }) + + it('returns undefined when empty and tracks size', () => { + const heap = new MinHeap() + expect(heap.size).toBe(0) + expect(heap.pop()).toBeUndefined() + heap.push(42, 0.5) + expect(heap.size).toBe(1) + expect(heap.pop()).toBe(42) + expect(heap.size).toBe(0) + }) + + it('handles interleaved push/pop correctly', () => { + const heap = new MinHeap() + heap.push(5, 5) + heap.push(1, 1) + expect(heap.pop()).toBe(1) + heap.push(3, 3) + heap.push(2, 2) + expect(heap.pop()).toBe(2) + expect(heap.pop()).toBe(3) + expect(heap.pop()).toBe(5) + }) +}) diff --git a/tests/unit/graph/graph-fastpath-honest-readiness.test.ts b/tests/unit/graph/graph-fastpath-honest-readiness.test.ts new file mode 100644 index 00000000..46d318b4 --- /dev/null +++ b/tests/unit/graph/graph-fastpath-honest-readiness.test.ts @@ -0,0 +1,95 @@ +/** + * @module tests/unit/graph/graph-fastpath-honest-readiness + * @description Pattern-A / Finding 2: getVerbsBySource/ByTarget gated the graph + * fast path on `graphIndex.isInitialized` (a proxy that reads true once the + * manifest/count loaded even if the source→target adjacency did NOT → the fast + * path then returned a silent []). The honest gate skips the fast path when the + * provider reports isReady()===false, falling to the correct canonical shard + * scan; and a one-shot probe self-heals a no-isReady provider whose adjacency + * did not cold-load. + */ +import { describe, it, expect, beforeEach } from 'vitest' +import { Brainy, NounType, VerbType } from '../../../src/index.js' + +describe('graph fast-path honest readiness (Finding 2)', () => { + let brain: any + let storage: any + let a: string + let b: string + let c: string + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, dimensions: 384 }) + await brain.init() + a = await brain.add({ data: 'a', type: NounType.Concept }) + b = await brain.add({ data: 'b', type: NounType.Concept }) + c = await brain.add({ data: 'c', type: NounType.Concept }) + await brain.relate({ from: a, to: b, type: VerbType.Contains }) + await brain.relate({ from: a, to: c, type: VerbType.Contains }) + await brain.flush() + storage = brain.storage + // Warm + wire the storage graph index (lazy) so we can stub it. + await storage.getVerbsBySource(a) + }) + + it('not-ready provider → shard scan returns the REAL edges, not a silent []', async () => { + const gi = storage.graphIndex + // Simulate a cold native provider: count/manifest loaded (isInitialized) but + // the source→target adjacency is NOT (isReady false; fast-path lookup empty). + gi.isReady = () => false + const origBySource = gi.getVerbIdsBySource.bind(gi) + gi.getVerbIdsBySource = async () => [] // cold adjacency — old fast path trusted this + storage._graphFastPathProbed = true // isolate the GATE (probe skips isReady-capable anyway) + try { + const verbs = await storage.getVerbsBySource(a) + // Honest gate skipped the cold fast path → canonical shard scan → real edges. + expect(verbs.length).toBe(2) + } finally { + delete gi.isReady + gi.getVerbIdsBySource = origBySource + } + }) + + it('ready provider → fast path is used and an empty result is trusted', async () => { + const gi = storage.graphIndex + gi.isReady = () => true + let fastPathCalls = 0 + const origBySource = gi.getVerbIdsBySource.bind(gi) + gi.getVerbIdsBySource = async (...args: any[]) => { fastPathCalls++; return origBySource(...args) } + storage._graphFastPathProbed = true + try { + // `c` has no OUTGOING edges — a genuinely empty result via the fast path. + const verbs = await storage.getVerbsBySource(c) + expect(verbs).toEqual([]) + expect(fastPathCalls).toBeGreaterThan(0) // fast path was taken, not a shard scan + } finally { + delete gi.isReady + gi.getVerbIdsBySource = origBySource + } + }) + + it('no-isReady provider whose adjacency did not cold-load → probe rebuilds once', async () => { + const gi = storage.graphIndex + if ('isReady' in gi) delete gi.isReady // force the "unknown" (no honest signal) path + // Force the probe to see a cold adjacency: a known edge resolves to no ints. + const origBySource = gi.getVerbIdsBySource.bind(gi) + let cold = true + gi.getVerbIdsBySource = async (...args: any[]) => (cold ? [] : origBySource(...args)) + let rebuilds = 0 + const origRebuild = gi.rebuild.bind(gi) + gi.rebuild = async (...args: any[]) => { rebuilds++; cold = false; return origRebuild(...args) } + storage._graphFastPathProbed = false // re-arm the one-shot probe + try { + await storage.getVerbsBySource(a) + expect(rebuilds).toBe(1) // probe detected cold adjacency + rebuilt once + expect(storage._graphFastPathProbed).toBe(true) // latched — no second rebuild + rebuilds = 0 + await storage.getVerbsByTarget(b) + expect(rebuilds).toBe(0) // probe does not re-run + } finally { + gi.getVerbIdsBySource = origBySource + gi.rebuild = origRebuild + } + }) +}) diff --git a/tests/unit/graph/lsm-partial-load-failclosed.test.ts b/tests/unit/graph/lsm-partial-load-failclosed.test.ts new file mode 100644 index 00000000..7082c0c6 --- /dev/null +++ b/tests/unit/graph/lsm-partial-load-failclosed.test.ts @@ -0,0 +1,76 @@ +/** + * @module tests/unit/graph/lsm-partial-load-failclosed + * @description LSMTree partial-load fail-closed guard (Finding 3 of the honest + * index-readiness spine plan). `loadSSTables()` used to swallow a per-SSTable + * load failure with only a `prodLog.warn`, then `loadManifest()` still published + * the FULL persisted `totalRelationships` count — so on a partial cold load, + * `size()`/`isHealthy()` reported the full count and "healthy" while a subset of + * the tree's edges were silently unqueryable. Fixed by failing the whole load + * closed on ANY per-SSTable failure: `loadManifest()`'s existing catch then + * resets `sstables`/`totalRelationships`/`sstablesByLevel` to honest-empty, so + * `size()` reports 0 and the graph layer's existing `size()===0` self-heal + * rebuilds from canonical records — the dishonest partial state can no longer be + * observed. + */ +import { describe, it, expect } from 'vitest' +import { LSMTree } from '../../../src/graph/lsm/LSMTree.js' +import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' + +describe('LSMTree partial SSTable load fails closed (honest size())', () => { + it('one SSTable fails to load → size() reports 0, not the manifest count', async () => { + const storage = new MemoryStorage() + await storage.init() + const prefix = 'test-lsm-partial' + + const t1 = new LSMTree(storage, { storagePrefix: prefix, enableCompaction: false }) + await t1.init() + await t1.add('a', 'b'); await t1.flush() // SSTable #1 + await t1.add('c', 'd'); await t1.flush() // SSTable #2 + expect(t1.size()).toBeGreaterThanOrEqual(2) + await t1.close() + + // Find an SSTable id from the persisted manifest. + const manifest = await storage.getMetadata(`${prefix}-manifest`) + const sstableIds = Object.keys((manifest!.data as any).sstables) + expect(sstableIds.length).toBeGreaterThanOrEqual(2) + + // Reopen with ONE SSTable load forced to fail (deterministic). + const t2 = new LSMTree(storage, { storagePrefix: prefix, enableCompaction: false }) + const realGet = storage.getMetadata.bind(storage) + ;(storage as any).getMetadata = async (key: string) => { + if (key === `${prefix}-${sstableIds[1]}`) throw new Error('simulated SSTable load failure') + return realGet(key) + } + await t2.init() + + // BEFORE THE FIX: t2.size() === (full manifest count) ← LIE (partial load, full count) + // AFTER THE FIX: t2.size() === 0 ← fail-closed → self-heal rebuilds + expect(t2.size()).toBe(0) + expect(t2.isHealthy()).toBe(true) // honestly-empty-but-initialized, not "healthy with a lie" + + await t2.close() + }) + + it('a clean load with no failures still reports the full honest count', async () => { + const storage = new MemoryStorage() + await storage.init() + const prefix = 'test-lsm-clean' + + const t1 = new LSMTree(storage, { storagePrefix: prefix, enableCompaction: false }) + await t1.init() + await t1.add('a', 'b'); await t1.flush() + await t1.add('c', 'd'); await t1.flush() + const expectedSize = t1.size() + expect(expectedSize).toBeGreaterThanOrEqual(2) + await t1.close() + + // Reopen with no injected failures — a normal, fully-successful cold load. + const t2 = new LSMTree(storage, { storagePrefix: prefix, enableCompaction: false }) + await t2.init() + + expect(t2.size()).toBe(expectedSize) + expect(t2.isHealthy()).toBe(true) + + await t2.close() + }) +}) diff --git a/tests/unit/hnsw/allowed-ids-pushdown.test.ts b/tests/unit/hnsw/allowed-ids-pushdown.test.ts new file mode 100644 index 00000000..183d5bd6 --- /dev/null +++ b/tests/unit/hnsw/allowed-ids-pushdown.test.ts @@ -0,0 +1,184 @@ +/** + * @module tests/unit/hnsw/allowed-ids-pushdown + * @description 8.0 #46 (CTX-PUSHDOWN-ALLOWEDIDS) — the vector-search predicate + * pushdown. Two layers: + * + * 1. The JS HNSW index honors `allowedIds: ReadonlySet` by restricting the + * beam walk to allowed nodes INSIDE the traversal (walk-all, collect-allowed) — + * the semantics that recover the filtered recall a naive top-k-then-filter loses. + * An `allowedIds` arriving as an `OpaqueIdSet` (a native/cor roaring Buffer) is + * opaque to the JS index and ignored — only a native provider decodes it. + * + * 2. `find()` forwards the matched universe to the vector beam walk as an + * `OpaqueIdSet` (zero id materialization) when the active metadata provider + * exposes the `getIdSetForFilter` producer — the native-stack zero-crossing path. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { JsHnswVectorIndex } from '../../../src/hnsw/hnswIndex.js' +import { euclideanDistance } from '../../../src/utils/index.js' +import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' +import { Brainy } from '../../../src/index.js' +import { NounType } from '../../../src/types/graphTypes.js' +import { createTestConfig } from '../../helpers/test-factory.js' + +const DIM = 8 + +/** Deterministic PRNG (mulberry32) so the synthetic graph is identical every run. */ +function seededRand(seed: number): () => number { + let s = seed >>> 0 + return () => { + s = (s + 0x6d2b79f5) | 0 + let t = Math.imul(s ^ (s >>> 15), 1 | s) + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t + return ((t ^ (t >>> 14)) >>> 0) / 4294967296 + } +} + +/** + * A vector at the requested distance-from-origin pointing in a deterministic RANDOM + * direction (so the HNSW graph is well-connected — diverse directions, not a + * degenerate axis cluster). The query is the origin, so distance-from-query equals + * `magnitude` exactly — letting the test place nodes in clean distance bands. + */ +function vecAt(magnitude: number, idx: number): number[] { + const rand = seededRand(idx + 1) + const dir = Array.from({ length: DIM }, () => rand() * 2 - 1) + const norm = Math.hypot(...dir) || 1 + return dir.map((x) => (x / norm) * magnitude) +} + +describe('#46 JS HNSW allowedIds pushdown (recall-preserving)', () => { + let index: JsHnswVectorIndex + const query = new Array(DIM).fill(0) + // Two clean distance bands: the disallowed nodes are STRICTLY closer to the query + // than the allowed targets. A naive top-k-then-filter returns only the disallowed + // (then drops them ⇒ empty); the pushdown must reach the allowed targets. `M` ≥ N−1 + // makes the small graph complete, so reachability is guaranteed and the test asserts + // the FILTER mechanism (walk-all, collect-allowed), not emergent HNSW connectivity. + const disallowed: string[] = [] + const targets: string[] = [] + + beforeEach(async () => { + index = new JsHnswVectorIndex( + { M: 16, efConstruction: 200, efSearch: 50, ml: 16 }, + euclideanDistance, + { useParallelization: false, storage: new MemoryStorage() } + ) + let n = 0 + for (let i = 0; i < 5; i++) { + const id = `disallowed-${i}` + disallowed.push(id) + await index.addItem({ id, vector: vecAt(0.3, n++) }) // closer band + } + for (let i = 0; i < 5; i++) { + const id = `target-${i}` + targets.push(id) + await index.addItem({ id, vector: vecAt(0.8, n++) }) // farther band + } + }) + + afterEach(() => { + disallowed.length = 0 + targets.length = 0 + }) + + it('without restriction, the top-k are all disallowed (so post-filtering would lose recall)', async () => { + const results = await index.search(query, 3) + const ids = results.map(([id]) => id) + // Every nearest neighbor is in the (closer) disallowed band → naive "filter after" = 0 hits. + expect(ids.length).toBe(3) + expect(ids.every((id) => disallowed.includes(id))).toBe(true) + }) + + it('with allowedIds, the walk reaches the allowed targets the naive path would miss', async () => { + const allowed = new Set(targets) + const results = await index.search(query, 3, undefined, { allowedIds: allowed }) + const ids = results.map(([id]) => id) + + expect(ids.length).toBe(3) + expect(ids.some((id) => disallowed.includes(id))).toBe(false) // no disallowed leaks through + expect(ids.every((id) => allowed.has(id))).toBe(true) // only allowed targets + }) + + it('ANDs allowedIds with candidateIds when both are given', async () => { + const results = await index.search(query, 5, undefined, { + candidateIds: [targets[0], targets[1], disallowed[0]], + allowedIds: new Set([targets[0], disallowed[0], disallowed[1]]) + }) + const ids = results.map(([id]) => id) + // Intersection = { target-0, disallowed-0 } — nothing outside it may appear. + expect(ids.every((id) => id === targets[0] || id === disallowed[0])).toBe(true) + expect(ids).toContain(targets[0]) + }) + + it('ignores an OpaqueIdSet (Uint8Array) — only a native provider can decode it', async () => { + const opaque = new Uint8Array([0xca, 0xfe, 0x01, 0x02]) + const results = await index.search(query, 3, undefined, { allowedIds: opaque }) + const ids = results.map(([id]) => id) + // Unrestricted behavior: the opaque buffer is not interpreted as a filter. + expect(ids.every((id) => disallowed.includes(id))).toBe(true) + }) + + it('ignores the #35 `generation` field — the JS index only ever holds "now"', async () => { + // The JS index has no retained per-generation segments, so it accepts and + // ignores `generation` (the refuse/fall-back posture) rather than throwing. + const withGen = await index.search(query, 3, undefined, { generation: 42n }) + const withoutGen = await index.search(query, 3) + expect(withGen.map(([id]) => id)).toEqual(withoutGen.map(([id]) => id)) + }) +}) + +describe('#46 find() forwards the opaque universe to the vector beam walk', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + await brain.add({ type: NounType.Document, data: 'machine learning notes', metadata: { author: 'alice' } }) + await brain.add({ type: NounType.Document, data: 'unrelated cooking', metadata: { author: 'bob' } }) + }) + + afterEach(async () => { + await brain.close() + }) + + it('passes the metadata getIdSetForFilter() OpaqueIdSet straight through as allowedIds', async () => { + const sentinel = new Uint8Array([0xab, 0xcd]) // stand-in for cor's roaring Buffer + // Stub the native producer the JS metadata manager does not implement. + ;(brain as any).metadataIndex.getIdSetForFilter = async () => sentinel + + // Record what the vector index actually receives, then run the real search. + const calls: any[] = [] + const realSearch = (brain as any).index.search.bind((brain as any).index) + ;(brain as any).index.search = async (vec: any, k: any, f: any, opts: any) => { + if (opts) calls.push(opts) + return realSearch(vec, k, f, opts) + } + + await brain.find({ query: 'machine learning', where: { author: 'alice' } }) + + const withUniverse = calls.find((o) => o.allowedIds !== undefined) + expect(withUniverse).toBeDefined() + // The opaque Buffer is forwarded verbatim (same reference — never materialized/copied). + expect(withUniverse.allowedIds).toBe(sentinel) + // The materialized candidateIds path is still present for the JS index. + expect(withUniverse.candidateIds).toBeDefined() + }) + + it('omits allowedIds when the metadata provider has no opaque producer (JS path)', async () => { + // No getIdSetForFilter stub — the JS manager genuinely lacks it. + const calls: any[] = [] + const realSearch = (brain as any).index.search.bind((brain as any).index) + ;(brain as any).index.search = async (vec: any, k: any, f: any, opts: any) => { + if (opts) calls.push(opts) + return realSearch(vec, k, f, opts) + } + + await brain.find({ query: 'machine learning', where: { author: 'alice' } }) + + const withOpts = calls.find((o) => o.candidateIds !== undefined) + expect(withOpts).toBeDefined() + expect(withOpts.allowedIds).toBeUndefined() // JS path: materialized candidateIds only + }) +}) diff --git a/tests/unit/hnsw/flush-failure.test.ts b/tests/unit/hnsw/flush-failure.test.ts new file mode 100644 index 00000000..14bdff6e --- /dev/null +++ b/tests/unit/hnsw/flush-failure.test.ts @@ -0,0 +1,157 @@ +/** + * HNSW deferred-flush durability tests (Finding 6 — spine-plan Part B, + * "blind catch" audit). + * + * `JsHnswVectorIndex.flush()` used to swallow a per-node + * `persistNodeConnections` failure (and a system-record `saveHNSWSystem` + * failure) behind `console.error`, then unconditionally clear the dirty set + * and report the pre-flush node count as "success". A transient write fault + * therefore silently dropped that node's connections from durable storage + * forever, with `flush()` having lied about it. These tests pin the fix: + * a node (or the system record) that fails to persist stays in the + * dirty/retry set, and `flush()` throws {@link HnswFlushError} instead of + * returning a count. The immediate-mode first-noun `saveHNSWSystem` swallow + * (while `addItem()` still returned the id) is covered too. + */ + +import { describe, it, expect, vi } from 'vitest' +import { v4 as uuidv4 } from 'uuid' +import { JsHnswVectorIndex, HnswFlushError } from '../../../src/hnsw/hnswIndex.js' +import { euclideanDistance } from '../../../src/utils/index.js' +import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' + +// Helper: generate a random vector of given dimension (mirrors lazy-vectors.test.ts). +function randomVector(dim: number): number[] { + return Array.from({ length: dim }, () => Math.random() * 2 - 1) +} + +describe('HNSW deferred-flush durability (Finding 6)', () => { + const dim = 8 + + it('a clean flush persists every dirty node and clears the dirty set', async () => { + const storage = new MemoryStorage() + const index = new JsHnswVectorIndex( + { M: 4, efConstruction: 50, efSearch: 20 }, + euclideanDistance, + { useParallelization: false, storage, persistMode: 'deferred' } + ) + + for (let i = 0; i < 5; i++) { + await index.addItem({ id: uuidv4(), vector: randomVector(dim) }) + } + + // Sanity: inserting several items in deferred mode does dirty something. + expect((index as any).dirtyNodes.size).toBeGreaterThan(0) + + const flushed = await index.flush() + expect(typeof flushed).toBe('number') + expect((index as any).dirtyNodes.size).toBe(0) + expect((index as any).dirtySystem).toBe(false) + }) + + it('retains a node whose connections failed to persist and surfaces HnswFlushError instead of reporting success', async () => { + const storage = new MemoryStorage() + const index = new JsHnswVectorIndex( + { M: 4, efConstruction: 50, efSearch: 20 }, + euclideanDistance, + { useParallelization: false, storage, persistMode: 'deferred' } + ) + + const ids: string[] = [] + for (let i = 0; i < 5; i++) { + const id = uuidv4() + ids.push(id) + await index.addItem({ id, vector: randomVector(dim) }) + } + + // The very first node is guaranteed to become a neighbor of the second + // insert (it is the only existing node in the graph at that point), so it + // is deterministically present in the dirty set before any fault. + const failId = ids[0] + const dirtyBefore = (index as any).dirtyNodes as Set + expect(dirtyBefore.has(failId)).toBe(true) + + const originalSave = storage.saveVectorIndexData.bind(storage) + const spy = vi + .spyOn(storage, 'saveVectorIndexData') + .mockImplementation(async (nounId, hnswData) => { + if (nounId === failId) { + throw Object.assign(new Error('simulated write fault'), { code: 'EIO' }) + } + return originalSave(nounId, hnswData) + }) + + await expect(index.flush()).rejects.toBeInstanceOf(HnswFlushError) + + const dirtyAfter = (index as any).dirtyNodes as Set + // The failed node stays dirty for the next retry ... + expect(dirtyAfter.has(failId)).toBe(true) + // ... and every node that DID persist successfully leaves the dirty set — + // the failure of one node must not re-dirty (or fail to clear) the rest. + expect(dirtyAfter.size).toBe(1) + + // Recovery: once the fault clears, the retained node persists and the + // flush reports success again (the intended retry path). + spy.mockRestore() + const flushed = await index.flush() + expect(typeof flushed).toBe('number') + expect((index as any).dirtyNodes.size).toBe(0) + }) + + it('surfaces a system-record persist failure as HnswFlushError and keeps dirtySystem set for retry', async () => { + const storage = new MemoryStorage() + const index = new JsHnswVectorIndex( + { M: 4, efConstruction: 50, efSearch: 20 }, + euclideanDistance, + { useParallelization: false, storage, persistMode: 'deferred' } + ) + + // First noun in deferred mode marks dirtySystem (entryPoint/maxLevel). + await index.addItem({ id: uuidv4(), vector: randomVector(dim) }) + expect((index as any).dirtySystem).toBe(true) + + const spy = vi + .spyOn(storage, 'saveHNSWSystem') + .mockRejectedValue( + Object.assign(new Error('simulated system write fault'), { code: 'EIO' }) + ) + + let caught: unknown + try { + await index.flush() + } catch (error) { + caught = error + } + + expect(caught).toBeInstanceOf(HnswFlushError) + expect((caught as HnswFlushError).systemFailed).toBe(true) + // The system record must stay dirty — a lost entry point/maxLevel update + // must be retried, not silently dropped. + expect((index as any).dirtySystem).toBe(true) + + spy.mockRestore() + const flushed = await index.flush() + expect(typeof flushed).toBe('number') + expect((index as any).dirtySystem).toBe(false) + }) + + it('surfaces the immediate-mode first-noun system persist failure via a rejecting addItem()', async () => { + const storage = new MemoryStorage() + const index = new JsHnswVectorIndex( + { M: 4, efConstruction: 50, efSearch: 20 }, + euclideanDistance, + { useParallelization: false, storage } // default persistMode: 'immediate' + ) + + vi.spyOn(storage, 'saveHNSWSystem').mockRejectedValue( + Object.assign(new Error('simulated system write fault'), { code: 'EIO' }) + ) + + // Previously this swallowed the error (console.error) and addItem() + // still resolved with the id — stranding a brand-new index whose root + // (entry point) was never actually persisted. Now it must reject. + await expect( + index.addItem({ id: uuidv4(), vector: randomVector(dim) }) + ).rejects.toThrow('simulated system write fault') + }) +}) diff --git a/tests/unit/hnsw/remove-reverse-adjacency.test.ts b/tests/unit/hnsw/remove-reverse-adjacency.test.ts new file mode 100644 index 00000000..a3a3b7ce --- /dev/null +++ b/tests/unit/hnsw/remove-reverse-adjacency.test.ts @@ -0,0 +1,168 @@ +/** + * @module tests/unit/hnsw/remove-reverse-adjacency + * @description Correctness guard for the reverse-adjacency index that makes + * `removeItem` O(in-degree) instead of O(N) (and bulk delete O(N·degree) instead + * of O(N²)). The risk of a reverse index is staleness — a missed maintenance site + * leaves a dangling reference to a deleted node or a stale referrer. These tests + * exercise mixed add/delete workloads and assert three invariants: + * + * 1. No surviving node's connections reference a deleted id (no dangling edges). + * 2. The incrementally-maintained reverse index matches a fresh rebuild from the + * live adjacency — i.e. link/prune/remove maintenance is exactly consistent. + * 3. Search still returns the true nearest neighbours of the surviving set + * (verified exactly against a brute-force scan on a complete small graph). + */ +import { describe, it, expect, beforeEach } from 'vitest' +import { JsHnswVectorIndex } from '../../../src/hnsw/hnswIndex.js' +import { euclideanDistance } from '../../../src/utils/index.js' +import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' + +const DIM = 8 + +function seededRand(seed: number): () => number { + let s = seed >>> 0 + return () => { + s = (s + 0x6d2b79f5) | 0 + let t = Math.imul(s ^ (s >>> 15), 1 | s) + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t + return ((t ^ (t >>> 14)) >>> 0) / 4294967296 + } +} + +/** A deterministic vector pointing in a pseudo-random direction (well-connected graph). */ +function vec(idx: number): number[] { + const rand = seededRand(idx + 1) + return Array.from({ length: DIM }, () => rand() * 2 - 1) +} + +type Noun = { id: string; vector: number[]; connections: Map> } + +function nounsOf(index: JsHnswVectorIndex): Map { + return (index as unknown as { nouns: Map }).nouns +} + +/** Flatten a reverse index (or a freshly-derived one) to sorted `target|level|source` triples. */ +function triplesFromIncoming(inc: Map>>): string[] { + const out: string[] = [] + for (const [target, byLevel] of inc) { + for (const [level, sources] of byLevel) { + for (const source of sources) out.push(`${target}|${level}|${source}`) + } + } + return out.sort() +} + +/** Derive the ground-truth reverse index directly from the live forward adjacency. */ +function triplesFromAdjacency(nouns: Map): string[] { + const out: string[] = [] + for (const [nodeId, node] of nouns) { + for (const [level, targets] of node.connections) { + for (const target of targets) out.push(`${target}|${level}|${nodeId}`) + } + } + return out.sort() +} + +function assertNoDanglingRefs(nouns: Map): void { + for (const [nodeId, node] of nouns) { + for (const [level, targets] of node.connections) { + for (const target of targets) { + expect(nouns.has(target), `node ${nodeId} (L${level}) links to deleted ${target}`).toBe(true) + } + } + } +} + +function assertReverseIndexConsistent(index: JsHnswVectorIndex): void { + // ensureIncoming() returns the live (incrementally-maintained) index, building it + // only if it was invalidated. The triples it yields must equal the triples derived + // straight from the forward adjacency — empty leftover sets contribute nothing. + const live = (index as unknown as { ensureIncoming: () => Map>> }).ensureIncoming() + expect(triplesFromIncoming(live)).toEqual(triplesFromAdjacency(nounsOf(index))) +} + +function makeIndex(M: number): JsHnswVectorIndex { + return new JsHnswVectorIndex( + { M, efConstruction: 200, efSearch: 64, ml: 16 }, + euclideanDistance, + { useParallelization: false, storage: new MemoryStorage() } + ) +} + +describe('HNSW removeItem — reverse-adjacency correctness', () => { + it('leaves no dangling references and a consistent reverse index after interleaved add/delete', async () => { + const index = makeIndex(16) + const live = new Set() + + // Add 120, then interleave: delete every 3rd as we keep adding — exercises the + // incremental maintenance (link + prune on add, unlink on remove) continuously. + for (let i = 0; i < 120; i++) { + const id = `n-${i}` + await index.addItem({ id, vector: vec(i) }) + live.add(id) + if (i % 3 === 0 && i > 0) { + const victim = `n-${i - 1}` + await index.removeItem(victim) + live.delete(victim) + } + } + + // Now delete ~half of the survivors in one burst (the O(N²)-prone path). + const survivors = [...live] + for (let i = 0; i < survivors.length; i += 2) { + await index.removeItem(survivors[i]) + live.delete(survivors[i]) + } + + const nouns = nounsOf(index) + expect(nouns.size).toBe(live.size) + assertNoDanglingRefs(nouns) + assertReverseIndexConsistent(index) + }) + + it('removing the entry point repeatedly stays consistent', async () => { + const index = makeIndex(8) + for (let i = 0; i < 40; i++) await index.addItem({ id: `e-${i}`, vector: vec(i) }) + + // Delete whatever is currently the entry point, 20 times. + for (let k = 0; k < 20; k++) { + const ep = (index as unknown as { entryPointId: string | null }).entryPointId + if (!ep) break + await index.removeItem(ep) + assertNoDanglingRefs(nounsOf(index)) + } + assertReverseIndexConsistent(index) + }) + + it('search returns the true nearest neighbours of the surviving set (exact, complete graph)', async () => { + // M ≥ N-1 ⇒ the small graph is complete and never prunes, so HNSW recall is 100% + // and we can assert the survivors\' exact top-k against a brute-force scan. + const index = makeIndex(64) + const all: Array<{ id: string; v: number[] }> = [] + for (let i = 0; i < 24; i++) { + const v = vec(i + 1000) + all.push({ id: `s-${i}`, v }) + await index.addItem({ id: `s-${i}`, vector: v }) + } + + const deleted = new Set(['s-2', 's-5', 's-9', 's-14', 's-20']) + for (const id of deleted) await index.removeItem(id) + + const survivors = all.filter((x) => !deleted.has(x.id)) + const query = vec(7).map((x) => x * 0.5) // an arbitrary deterministic query point + const k = 5 + + const bruteTopK = survivors + .map((x) => ({ id: x.id, d: euclideanDistance(query, x.v) })) + .sort((a, b) => a.d - b.d) + .slice(0, k) + .map((x) => x.id) + + const got = (await index.search(query, k)).map(([id]) => id) + + expect(got).toEqual(bruteTopK) + // And none of the deleted ids can ever surface. + expect(got.some((id) => deleted.has(id))).toBe(false) + assertNoDanglingRefs(nounsOf(index)) + }) +}) diff --git a/tests/unit/import/InstancePool.test.ts b/tests/unit/import/InstancePool.test.ts deleted file mode 100644 index 740be5d7..00000000 --- a/tests/unit/import/InstancePool.test.ts +++ /dev/null @@ -1,317 +0,0 @@ -import { describe, it, expect, beforeEach } from 'vitest' -import { Brainy } from '../../../src/brainy.js' -import { InstancePool, createInstancePool } from '../../../src/import/InstancePool.js' - -describe('InstancePool', () => { - let brain: Brainy - let pool: InstancePool - - beforeEach(async () => { - brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) - await brain.init() - pool = new InstancePool(brain) - }) - - describe('lazy initialization', () => { - it('should not create instances until requested', () => { - const stats = pool.getStats() - expect(stats.nlpCreated).toBe(false) - expect(stats.extractorCreated).toBe(false) - }) - - it('should create NLP instance on first access', async () => { - const nlp = await pool.getNLP() - expect(nlp).toBeDefined() - - const stats = pool.getStats() - expect(stats.nlpCreated).toBe(true) - expect(stats.nlpReuses).toBe(1) - }) - - it('should create extractor instance on first access', () => { - const extractor = pool.getExtractor() - expect(extractor).toBeDefined() - - const stats = pool.getStats() - expect(stats.extractorCreated).toBe(true) - expect(stats.extractorReuses).toBe(1) - }) - }) - - describe('instance reuse', () => { - it('should return same NLP instance on multiple calls', async () => { - const nlp1 = await pool.getNLP() - const nlp2 = await pool.getNLP() - const nlp3 = await pool.getNLP() - - expect(nlp1).toBe(nlp2) - expect(nlp2).toBe(nlp3) - - const stats = pool.getStats() - expect(stats.nlpReuses).toBe(3) - }) - - it('should return same extractor instance on multiple calls', () => { - const extractor1 = pool.getExtractor() - const extractor2 = pool.getExtractor() - const extractor3 = pool.getExtractor() - - expect(extractor1).toBe(extractor2) - expect(extractor2).toBe(extractor3) - - const stats = pool.getStats() - expect(stats.extractorReuses).toBe(3) - }) - - it('should track reuse counts correctly', async () => { - await pool.getNLP() - await pool.getNLP() - pool.getExtractor() - pool.getExtractor() - pool.getExtractor() - - const stats = pool.getStats() - expect(stats.nlpReuses).toBe(2) - expect(stats.extractorReuses).toBe(3) - }) - }) - - describe('initialization', () => { - it('should initialize all instances with init()', async () => { - await pool.init() - - expect(pool.isInitialized()).toBe(true) - - const stats = pool.getStats() - expect(stats.nlpCreated).toBe(true) - expect(stats.extractorCreated).toBe(true) - expect(stats.initialized).toBe(true) - }) - - it('should handle concurrent init calls safely', async () => { - // Call init multiple times concurrently - const promises = [ - pool.init(), - pool.init(), - pool.init() - ] - - await Promise.all(promises) - - // Should only initialize once - expect(pool.isInitialized()).toBe(true) - }) - - it('should auto-initialize NLP when accessed', async () => { - const nlp = await pool.getNLP() - - // NLP is lazy-initialized but extractor might not be - const stats = pool.getStats() - expect(stats.nlpCreated).toBe(true) - expect(stats.initialized).toBe(true) - }) - - it('should provide sync access to NLP', () => { - const nlp = pool.getNLPSync() - expect(nlp).toBeDefined() - - const stats = pool.getStats() - expect(stats.nlpCreated).toBe(true) - }) - }) - - describe('statistics', () => { - it('should track creation time', async () => { - await pool.init() - - const stats = pool.getStats() - expect(stats.creationTime).toBeGreaterThanOrEqual(0) - }) - - it('should calculate memory saved', async () => { - // Use instances multiple times - await pool.getNLP() - await pool.getNLP() - await pool.getNLP() - pool.getExtractor() - pool.getExtractor() - - const stats = pool.getStats() - expect(stats.memorySaved).toBeGreaterThan(0) - }) - - it('should reset statistics', async () => { - await pool.getNLP() - pool.getExtractor() - - pool.resetStats() - - const stats = pool.getStats() - expect(stats.nlpReuses).toBe(0) - expect(stats.extractorReuses).toBe(0) - expect(stats.creationTime).toBe(0) - }) - - it('should provide string representation', async () => { - await pool.init() - - const str = pool.toString() - expect(str).toContain('InstancePool') - expect(str).toContain('nlp=true') - expect(str).toContain('extractor=true') - }) - }) - - describe('memory efficiency', () => { - it('should reuse instances in loop (no memory leak)', async () => { - const initialStats = pool.getStats() - - // Simulate import loop - for (let i = 0; i < 1000; i++) { - const nlp = await pool.getNLP() - const extractor = pool.getExtractor() - - // All iterations should get same instances - expect(nlp).toBeDefined() - expect(extractor).toBeDefined() - } - - const finalStats = pool.getStats() - expect(finalStats.nlpReuses).toBe(1000) - expect(finalStats.extractorReuses).toBe(1000) - - // Should have saved ~60GB of memory (1000 iterations × ~60MB) - expect(finalStats.memorySaved).toBeGreaterThan(50 * 1024 * 1024 * 1000) // > 50GB - }) - - it('should handle rapid concurrent access', async () => { - // Simulate concurrent row processing - const promises = [] - for (let i = 0; i < 100; i++) { - promises.push(pool.getNLP()) - promises.push(Promise.resolve(pool.getExtractor())) - } - - await Promise.all(promises) - - const stats = pool.getStats() - expect(stats.nlpReuses).toBe(100) - expect(stats.extractorReuses).toBe(100) - }) - }) - - describe('cleanup', () => { - it('should cleanup instances', async () => { - await pool.init() - expect(pool.isInitialized()).toBe(true) - - pool.cleanup() - - expect(pool.isInitialized()).toBe(false) - const stats = pool.getStats() - expect(stats.nlpCreated).toBe(false) - expect(stats.extractorCreated).toBe(false) - }) - - it('should allow reinitialization after cleanup', async () => { - await pool.init() - pool.cleanup() - - await pool.init() - expect(pool.isInitialized()).toBe(true) - }) - }) - - describe('factory function', () => { - it('should create pool with auto-init', async () => { - const newPool = await createInstancePool(brain, true) - - expect(newPool.isInitialized()).toBe(true) - }) - - it('should create pool without auto-init', async () => { - const newPool = await createInstancePool(brain, false) - - expect(newPool.isInitialized()).toBe(false) - }) - - it('should default to auto-init', async () => { - const newPool = await createInstancePool(brain) - - expect(newPool.isInitialized()).toBe(true) - }) - }) - - describe('error handling', () => { - it('should handle missing NLP instance gracefully', async () => { - const emptyPool = new InstancePool(brain) - - // Should create NLP on first access - const nlp = await emptyPool.getNLP() - expect(nlp).toBeDefined() - }) - - it('should handle missing extractor instance gracefully', () => { - const emptyPool = new InstancePool(brain) - - // Should create extractor on first access - const extractor = emptyPool.getExtractor() - expect(extractor).toBeDefined() - }) - }) - - describe('real-world usage', () => { - it('should work with actual NLP operations', async () => { - const nlp = await pool.getNLP() - - // Should be initialized and ready to use - expect(nlp).toBeDefined() - - // NLP should have init method - expect(typeof nlp.init).toBe('function') - }) - - it('should work with actual entity extraction', async () => { - const extractor = pool.getExtractor() - - // Should be ready to use - expect(extractor).toBeDefined() - - // Can call extractor methods - const entities = await extractor.extract('Paris is a beautiful city', { - confidence: 0.5 - }) - - expect(Array.isArray(entities)).toBe(true) - }) - - it('should handle full import workflow', async () => { - // Initialize pool - await pool.init() - - // Simulate processing multiple rows - const rows = [ - { text: 'Paris is beautiful' }, - { text: 'London is historic' }, - { text: 'Tokyo is modern' } - ] - - for (const row of rows) { - const nlp = await pool.getNLP() - const extractor = pool.getExtractor() - - // Process row - extract entities - const entities = await extractor.extract(row.text, { confidence: 0.5 }) - - expect(nlp).toBeDefined() - expect(extractor).toBeDefined() - expect(entities).toBeDefined() - } - - // Verify instances were reused - const stats = pool.getStats() - expect(stats.nlpReuses).toBe(3) - expect(stats.extractorReuses).toBe(3) - }) - }) -}) diff --git a/tests/unit/indexes/columnStore/segment-load-fault.test.ts b/tests/unit/indexes/columnStore/segment-load-fault.test.ts new file mode 100644 index 00000000..deb0868f --- /dev/null +++ b/tests/unit/indexes/columnStore/segment-load-fault.test.ts @@ -0,0 +1,128 @@ +/** + * @module tests/unit/indexes/columnStore/segment-load-fault + * @description Pattern-B acceptance for the ColumnStore (finding 4): a segment + * the manifest LISTS but that cannot be loaded must never be silently skipped — + * doing so dropped every entity in that segment out of `filter`/`rangeQuery`/ + * `sortTopK` with no error, so a corrupt index looked like a merely short result. + * + * The three failure classes and their required behaviour: + * - a real storage IO fault (EIO) PROPAGATES verbatim — a present-but-unreadable + * segment is not "absent", so it must not read as an empty result; + * - a manifest-listed segment with undecodable bytes throws `ColumnSegmentLoadError`; + * - a manifest-listed segment with NO bytes (gone on disk) throws `ColumnSegmentLoadError`. + * Only genuine absence stays benign: querying a field that has no manifest at all + * returns empty (nothing was ever written for it) — that is not a fault. + */ +import { describe, it, expect, beforeEach } from 'vitest' +import { + ColumnStore, + ColumnSegmentLoadError +} from '../../../../src/indexes/columnStore/ColumnStore.js' +import { MemoryStorage } from '../../../../src/storage/adapters/memoryStorage.js' +import { EntityIdMapper } from '../../../../src/utils/entityIdMapper.js' + +type FaultMode = 'none' | 'io' | 'corrupt' | 'missing' + +/** + * A MemoryStorage that can fault reads of persisted column SEGMENTS only + * (keys/paths containing the `L0-` segment marker). Manifest and DELETED-bitmap + * reads pass through untouched so a fresh store still initialises normally — the + * fault is isolated to the exact seam finding 4 hardened. + */ +class FaultInjectingStorage extends MemoryStorage { + public faultMode: FaultMode = 'none' + + private eio(): Error { + const e = new Error('simulated disk read fault') as Error & { code: string } + e.code = 'EIO' + return e + } + + public async loadBinaryBlob(key: string): Promise { + if (this.faultMode !== 'none' && key.includes('/L0-')) { + if (this.faultMode === 'io') throw this.eio() + // Too small to hold even a header → readSegmentFromBuffer throws → wrapped. + if (this.faultMode === 'corrupt') return Buffer.from([1, 2, 3, 4, 5]) + if (this.faultMode === 'missing') return null + } + return super.loadBinaryBlob(key) + } +} + +describe('ColumnStore segment-load faults surface loudly, absence stays benign (finding 4)', () => { + let storage: FaultInjectingStorage + let idMapper: EntityIdMapper + + beforeEach(async () => { + storage = new FaultInjectingStorage() + await storage.init() + idMapper = new EntityIdMapper({ storage, storageKey: 'test:idMapper' }) + await idMapper.init() + + // Write one persisted L0 segment for `createdAt`, then close the writer. + const writer = new ColumnStore({ flushThreshold: 10 }) + await writer.init(storage, idMapper) + for (let i = 0; i < 5; i++) { + writer.addEntity(BigInt(idMapper.getOrAssign(`e${i}`)), { + createdAt: (i + 1) * 100 + }) + } + await writer.flush() + await writer.close() + storage.faultMode = 'none' + }) + + // Fresh reader over the same storage: empty segment cache, so every query is + // forced to actually load the persisted segment (that is the seam under test). + const reopen = async (): Promise => { + const s = new ColumnStore({ flushThreshold: 10 }) + await s.init(storage, idMapper) + return s + } + + it('propagates a storage IO fault verbatim — not [] and not a ColumnSegmentLoadError', async () => { + storage.faultMode = 'io' + const store = await reopen() + await expect(store.filter('createdAt', 300)).rejects.toMatchObject({ + code: 'EIO' + }) + await store.close() + }) + + it('throws ColumnSegmentLoadError when a manifest-listed segment is undecodable', async () => { + storage.faultMode = 'corrupt' + const store = await reopen() + await expect( + store.sortTopK('createdAt', 'desc', 10) + ).rejects.toBeInstanceOf(ColumnSegmentLoadError) + await store.close() + }) + + it('throws ColumnSegmentLoadError when a manifest-listed segment has no loadable bytes', async () => { + storage.faultMode = 'missing' + const store = await reopen() + await expect( + store.rangeQuery('createdAt', 100, 500) + ).rejects.toBeInstanceOf(ColumnSegmentLoadError) + await store.close() + }) + + it('a field with no manifest is genuine absence — returns empty, never throws', async () => { + storage.faultMode = 'none' + const store = await reopen() + const bitmap = await store.filter('no_such_field', 'x') + expect(bitmap.size).toBe(0) + const sorted = await store.sortTopK('no_such_field', 'asc', 10) + expect(sorted).toEqual([]) + await store.close() + }) + + it('with no fault, the persisted segment still loads and answers queries', async () => { + storage.faultMode = 'none' + const store = await reopen() + const sorted = await store.sortTopK('createdAt', 'desc', 10) + const uuids = sorted.map((id) => idMapper.getUuid(Number(id))) + expect(uuids).toEqual(['e4', 'e3', 'e2', 'e1', 'e0']) + await store.close() + }) +}) diff --git a/tests/unit/metadata-cold-read-guard.test.ts b/tests/unit/metadata-cold-read-guard.test.ts new file mode 100644 index 00000000..40d37de6 --- /dev/null +++ b/tests/unit/metadata-cold-read-guard.test.ts @@ -0,0 +1,97 @@ +/** + * Metadata cold-read guard (verifyMetadataLive) — a downstream deployment + * reported cold `find({ where })` returning a silent `[]` on a freshly-opened + * brain (a native metadata index that reports data but has not loaded its field + * postings). This guard, the field-index counterpart of verifyGraphAdjacencyLive, + * probes a known persisted value on the first filtered find(): if the index does + * not serve it, brainy rebuilds and re-probes, and raises a loud + * MetadataIndexNotReadyError only if the rebuild still can't serve — never a + * silent empty result that misrepresents existing data. + * + * The 8.0 JS index cold-loads correctly, so we simulate the cold native failure + * mode by intercepting the provider's getIdsForFilter/rebuild. + */ +import { describe, it, expect, beforeEach } from 'vitest' +import { Brainy, NounType, MetadataIndexNotReadyError } from '../../src/index.js' + +const V = () => Array.from({ length: 384 }, (_, i) => Math.sin(i * 0.1) + 0.001) + +describe('Metadata cold-read guard (#venue silent-[])', () => { + let brain: any + + beforeEach(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + await brain.add({ vector: V(), type: NounType.Concept, metadata: { status: 'active' } }) + await brain.add({ vector: V(), type: NounType.Concept, metadata: { status: 'archived' } }) + await brain.flush() + }) + + it('warm brain: filtered find is correct and the guard does not rebuild', async () => { + const mi = brain.metadataIndex + let rebuilds = 0 + const origRebuild = mi.rebuild.bind(mi) + mi.rebuild = async () => { + rebuilds++ + return origRebuild() + } + const res = await brain.find({ where: { status: 'active' }, limit: 100 }) + expect(res.length).toBe(1) + expect(rebuilds).toBe(0) // served live — no rebuild + expect(brain._metadataVerified).toBe(true) // one-shot latched + mi.rebuild = origRebuild + }) + + it('cold index: verifyMetadataLive self-heals via rebuild — find({where}) is correct, NOT silent []', async () => { + const mi = brain.metadataIndex + const origGetIds = mi.getIdsForFilter.bind(mi) + const origRebuild = mi.rebuild.bind(mi) + let cold = true + brain._metadataVerified = false // re-arm the one-shot for this scenario + mi.getIdsForFilter = async (...a: any[]) => (cold ? [] : origGetIds(...a)) + mi.rebuild = async () => { + await origRebuild() + cold = false // the rebuild warms the postings + } + try { + const res = await brain.find({ where: { status: 'active' }, limit: 100 }) + expect(res.length).toBe(1) // self-healed — the known entity is returned + } finally { + mi.getIdsForFilter = origGetIds + mi.rebuild = origRebuild + } + }) + + it('unrecoverably cold index: find({where}) throws MetadataIndexNotReadyError — never a silent []', async () => { + const mi = brain.metadataIndex + const origGetIds = mi.getIdsForFilter.bind(mi) + const origRebuild = mi.rebuild.bind(mi) + brain._metadataVerified = false + mi.getIdsForFilter = async () => [] // always cold; rebuild can't fix it + mi.rebuild = async () => {} + try { + await expect(brain.find({ where: { status: 'active' }, limit: 100 })).rejects.toBeInstanceOf( + MetadataIndexNotReadyError + ) + } finally { + mi.getIdsForFilter = origGetIds + mi.rebuild = origRebuild + } + }) + + it('a query with no filter does not trigger the metadata probe', async () => { + const mi = brain.metadataIndex + let probes = 0 + const origGetIds = mi.getIdsForFilter.bind(mi) + mi.getIdsForFilter = async (...a: any[]) => { + probes++ + return origGetIds(...a) + } + brain._metadataVerified = false + // A pure vector query (no where/type) must not run verifyMetadataLive's probe. + await brain.find({ vector: V(), limit: 5 }) + expect(brain._metadataVerified).toBe(false) // guard never ran + mi.getIdsForFilter = origGetIds + void probes + }) +}) diff --git a/tests/unit/migration-lock.test.ts b/tests/unit/migration-lock.test.ts new file mode 100644 index 00000000..f0fbbe4c --- /dev/null +++ b/tests/unit/migration-lock.test.ts @@ -0,0 +1,179 @@ +/** + * Migration LOCK (#18) — coordinated, automatic 7.x → 8.0 auto-upgrade. + * + * Exercises the real block-and-queue behavior of `awaitMigrationLock` at the + * `ensureInitialized` choke point. The gate is FAMILY-SCOPED: a write (which + * touches every index) WAITS while ANY provider reports `isMigrating() === true`, + * and a read WAITS only when the migrating provider is one of the index families + * that read consults (so a read served from canonical storage or a healthy family + * is never blocked by an unrelated family's migration — see + * `tests/unit/brainy/migration-gate-family-scoped.test.ts`). Observability + * (`getIndexStatus`, `health`) and the lock-clearing `stampBrainFormat` stay + * exempt; a lock that outlives the configured window surfaces a retryable + * `MigrationInProgressError`. Here the graph provider holds the lock, so the + * read cases use a graph traversal (`related`) — the family that actually waits. + * + * A native (cortex) provider owns the real migration; here we simulate the lock + * by feature-detect-injecting `isMigrating()` onto a live provider, exactly as + * the production feature-detection reads it. + */ + +import { describe, it, expect, beforeEach } from 'vitest' +import { Brainy, NounType, MigrationInProgressError } from '../../src/index.js' +import { GraphAdjacencyIndex } from '../../src/graph/graphAdjacencyIndex.js' + +/** Force the graph provider to hold (or release) the migration lock. */ +function setMigrating(brain: any, migrating: boolean): void { + brain.graphIndex.isMigrating = () => migrating +} + +describe('Migration LOCK (#18) — coordinated 7.x→8.0 auto-upgrade', () => { + let brain: any + + beforeEach(async () => { + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' }, + migrationWaitTimeoutMs: 5000 // generous; timing tests clear well within it + }) + await brain.init() + }) + + it('does not gate operations when no provider is migrating (fast path)', async () => { + const id = await brain.add({ data: 'hello', type: NounType.Concept }) + expect(id).toBeTruthy() + const status = await brain.getIndexStatus() + expect(status.migrating).toBe(false) + expect(status.migration).toBeUndefined() + }) + + it('getIndexStatus() reports migrating WITHOUT blocking (lock-exempt observability)', async () => { + setMigrating(brain, true) + const status = await brain.getIndexStatus() // must resolve, never hang + expect(status.migrating).toBe(true) + expect(status.migration).toBeDefined() + expect(typeof status.migration.elapsedMs).toBe('number') + }) + + it('health() surfaces the migration as a warn check without blocking', async () => { + setMigrating(brain, true) + const h = await brain.health() // lock-exempt + expect(h.overall).toBe('warn') + expect(h.checks.some((c: any) => c.name === 'migration')).toBe(true) + }) + + it('checkHealth() answers during a migration (lock-exempt diagnostics)', async () => { + setMigrating(brain, true) + const r = await brain.checkHealth() + expect(r).toHaveProperty('healthy') + }) + + it('blocks a write while migrating (poll path), then completes when the lock clears', async () => { + setMigrating(brain, true) + let resolved = false + const p = brain + .add({ data: 'queued write', type: NounType.Concept }) + .then((id: string) => { + resolved = true + return id + }) + + // Still blocked shortly after issue. + await new Promise((r) => setTimeout(r, 40)) + expect(resolved).toBe(false) + + // Clearing the lock releases the queued write against the good indexes. + setMigrating(brain, false) + const id = await p + expect(resolved).toBe(true) + expect(id).toBeTruthy() + }) + + it('blocks a graph read while the graph provider migrates, then releases when the flag clears (poll path)', async () => { + const anchor = await brain.add({ data: 'anchor', type: NounType.Concept }) + let migrating = true + brain.graphIndex.isMigrating = () => migrating + + const p = brain.related({ from: anchor }) // a GRAPH read → gated on the graph migration + let resolved = false + p.then(() => { + resolved = true + }) + + await new Promise((r) => setTimeout(r, 40)) + expect(resolved).toBe(false) + + // Clearing the flag lets the next poll release the read against good indexes. + migrating = false + await expect(p).resolves.toEqual([]) + }) + + it('throws a retryable MigrationInProgressError when the lock outlives the timeout', async () => { + const shortBrain: any = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' }, + migrationWaitTimeoutMs: 120 + }) + await shortBrain.init() + const anchor = await shortBrain.add({ data: 'anchor', type: NounType.Concept }) + setMigrating(shortBrain, true) // graph lock never clears + + // A graph read needs the migrating family → it waits out the window and + // surfaces the retryable error rather than serving a half-built traversal. + await expect(shortBrain.related({ from: anchor })).rejects.toBeInstanceOf( + MigrationInProgressError + ) + try { + await shortBrain.add({ data: 'x', type: NounType.Concept }) + throw new Error('expected MigrationInProgressError') + } catch (e: any) { + expect(e).toBeInstanceOf(MigrationInProgressError) + expect(e.retryable).toBe(true) + expect(typeof e.elapsedMs).toBe('number') + } + }) + + it('stampBrainFormat() is NOT gated — cor clears the lock through it (no deadlock)', async () => { + setMigrating(brain, true) + // Must resolve, not hang, even while a migration is "in progress". + await expect(brain.stampBrainFormat()).resolves.toBeUndefined() + }) + + it('close() is NOT gated — an instance can shut down mid-migration', async () => { + setMigrating(brain, true) + await expect(brain.close()).resolves.toBeUndefined() + }) + + it('C1: init() waits out a migration BEFORE VFS bootstrap (no deadlock, no failure)', async () => { + // The critical regression: init()'s own VFS bootstrap (get/add/find on the + // root) routes through the gated ensureInitialized. If a provider migrates + // during init, init must WAIT for the lock to clear BEFORE bootstrapping VFS — + // otherwise it deadlocks (< timeout) or throws MigrationInProgressError past + // the timeout. We make the graph provider report migrating for a short window + // spanning init, then clear it, and assert init completes and the brain works. + const orig = (GraphAdjacencyIndex.prototype as any).isMigrating + let migrating = true + ;(GraphAdjacencyIndex.prototype as any).isMigrating = () => migrating + const clearMigration = setTimeout(() => { + migrating = false + }, 60) + try { + const b: any = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' }, + migrationWaitTimeoutMs: 5000 + }) + await b.init() // must resolve once the migration clears — not hang, not throw + expect(b.isInitialized).toBe(true) + // VFS bootstrapped post-migration → normal write/read work end-to-end. + const id = await b.add({ data: 'after upgrade', type: NounType.Concept }) + expect(id).toBeTruthy() + expect(await b.get(id)).not.toBeNull() + await b.close() + } finally { + clearTimeout(clearMigration) + if (orig) (GraphAdjacencyIndex.prototype as any).isMigrating = orig + else delete (GraphAdjacencyIndex.prototype as any).isMigrating + } + }) +}) diff --git a/tests/unit/neural/presets.test.ts b/tests/unit/neural/presets.test.ts deleted file mode 100644 index d27add8f..00000000 --- a/tests/unit/neural/presets.test.ts +++ /dev/null @@ -1,561 +0,0 @@ -import { describe, it, expect } from 'vitest' -import { - autoDetectPreset, - getPreset, - getPresetNames, - explainPresetChoice, - createCustomPreset, - validatePreset, - formatPreset, - FAST_PRESET, - BALANCED_PRESET, - ACCURATE_PRESET, - EXPLICIT_PRESET, - PATTERN_PRESET, - PRESETS, - type ImportContext, - type PresetConfig -} from '../../../src/neural/presets.js' - -describe('Presets', () => { - describe('preset definitions', () => { - it('should have all 5 presets defined', () => { - expect(PRESETS).toBeDefined() - expect(Object.keys(PRESETS)).toHaveLength(5) - expect(PRESETS.fast).toBe(FAST_PRESET) - expect(PRESETS.balanced).toBe(BALANCED_PRESET) - expect(PRESETS.accurate).toBe(ACCURATE_PRESET) - expect(PRESETS.explicit).toBe(EXPLICIT_PRESET) - expect(PRESETS.pattern).toBe(PATTERN_PRESET) - }) - - it('should have valid fast preset', () => { - expect(FAST_PRESET.name).toBe('fast') - expect(FAST_PRESET.signals.enabled).toEqual(['exact', 'pattern']) - expect(FAST_PRESET.strategies.enabled).toEqual(['explicit']) - expect(FAST_PRESET.streaming).toBe(true) - expect(FAST_PRESET.strategies.earlyTermination).toBe(true) - }) - - it('should have valid balanced preset', () => { - expect(BALANCED_PRESET.name).toBe('balanced') - expect(BALANCED_PRESET.signals.enabled).toEqual(['exact', 'embedding', 'pattern']) - expect(BALANCED_PRESET.strategies.enabled).toEqual(['explicit', 'pattern', 'embedding']) - expect(BALANCED_PRESET.streaming).toBe(false) - }) - - it('should have valid accurate preset', () => { - expect(ACCURATE_PRESET.name).toBe('accurate') - expect(ACCURATE_PRESET.signals.enabled).toEqual(['exact', 'embedding', 'pattern', 'context']) - expect(ACCURATE_PRESET.strategies.enabled).toEqual(['explicit', 'pattern', 'embedding']) - expect(ACCURATE_PRESET.strategies.earlyTermination).toBe(false) - }) - - it('should have valid explicit preset', () => { - expect(EXPLICIT_PRESET.name).toBe('explicit') - expect(EXPLICIT_PRESET.signals.enabled).toEqual(['exact', 'pattern']) - expect(EXPLICIT_PRESET.strategies.enabled).toEqual(['explicit', 'pattern']) - expect(EXPLICIT_PRESET.strategies.minConfidence).toBeGreaterThanOrEqual(0.80) - }) - - it('should have valid pattern preset', () => { - expect(PATTERN_PRESET.name).toBe('pattern') - expect(PATTERN_PRESET.signals.enabled).toEqual(['embedding', 'pattern', 'context']) - expect(PATTERN_PRESET.strategies.enabled).toEqual(['pattern', 'embedding']) - }) - }) - - describe('autoDetectPreset', () => { - it('should return fast preset for large datasets', () => { - const context: ImportContext = { - rowCount: 15000, - fileSize: 5_000_000 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('fast') - }) - - it('should return fast preset for large files', () => { - const context: ImportContext = { - fileSize: 15_000_000 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('fast') - }) - - it('should return accurate preset for small datasets', () => { - const context: ImportContext = { - rowCount: 50 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('accurate') - }) - - it('should return explicit preset for Excel with explicit columns', () => { - const context: ImportContext = { - fileType: 'excel', - hasExplicitColumns: true, - rowCount: 500 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('explicit') - }) - - it('should return explicit preset for CSV with explicit columns', () => { - const context: ImportContext = { - fileType: 'csv', - hasExplicitColumns: true - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('explicit') - }) - - it('should return pattern preset for PDF files', () => { - const context: ImportContext = { - fileType: 'pdf', - rowCount: 200 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('pattern') - }) - - it('should return pattern preset for Markdown files', () => { - const context: ImportContext = { - fileType: 'markdown' - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('pattern') - }) - - it('should return pattern preset for narrative content', () => { - const context: ImportContext = { - hasNarrativeContent: true, - rowCount: 300 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('pattern') - }) - - it('should return pattern preset for long definitions', () => { - const context: ImportContext = { - avgDefinitionLength: 800, - fileType: 'csv' - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('pattern') - }) - - it('should return balanced preset for JSON', () => { - const context: ImportContext = { - fileType: 'json', - rowCount: 500 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('balanced') - }) - - it('should return balanced preset for medium datasets', () => { - const context: ImportContext = { - fileType: 'excel', - rowCount: 2000 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('balanced') - }) - - it('should return balanced preset for empty context', () => { - const preset = autoDetectPreset() - expect(preset.name).toBe('balanced') - }) - - it('should return balanced preset for unknown file type', () => { - const context: ImportContext = { - fileType: 'unknown', - rowCount: 500 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('balanced') - }) - }) - - describe('getPreset', () => { - it('should get preset by name', () => { - expect(getPreset('fast')).toBe(FAST_PRESET) - expect(getPreset('balanced')).toBe(BALANCED_PRESET) - expect(getPreset('accurate')).toBe(ACCURATE_PRESET) - expect(getPreset('explicit')).toBe(EXPLICIT_PRESET) - expect(getPreset('pattern')).toBe(PATTERN_PRESET) - }) - - it('should be case-insensitive', () => { - expect(getPreset('FAST')).toBe(FAST_PRESET) - expect(getPreset('Balanced')).toBe(BALANCED_PRESET) - expect(getPreset('EXPLICIT')).toBe(EXPLICIT_PRESET) - }) - - it('should throw error for unknown preset', () => { - expect(() => getPreset('unknown')).toThrow('Unknown preset: unknown') - }) - }) - - describe('getPresetNames', () => { - it('should return all preset names', () => { - const names = getPresetNames() - expect(names).toHaveLength(5) - expect(names).toContain('fast') - expect(names).toContain('balanced') - expect(names).toContain('accurate') - expect(names).toContain('explicit') - expect(names).toContain('pattern') - }) - }) - - describe('explainPresetChoice', () => { - it('should explain large dataset choice', () => { - const context: ImportContext = { - rowCount: 15000, - fileSize: 12_000_000 - } - - const explanation = explainPresetChoice(context) - expect(explanation).toContain('Large dataset') - expect(explanation).toContain('15000 rows') - expect(explanation).toContain('fast preset') - }) - - it('should explain small dataset choice', () => { - const context: ImportContext = { - rowCount: 50 - } - - const explanation = explainPresetChoice(context) - expect(explanation).toContain('Small critical dataset') - expect(explanation).toContain('50 rows') - expect(explanation).toContain('accurate preset') - }) - - it('should explain explicit columns choice', () => { - const context: ImportContext = { - fileType: 'excel', - hasExplicitColumns: true - } - - const explanation = explainPresetChoice(context) - expect(explanation).toContain('EXCEL') - expect(explanation).toContain('explicit relationship columns') - expect(explanation).toContain('explicit preset') - }) - - it('should explain narrative content choice', () => { - const context: ImportContext = { - fileType: 'pdf', - hasNarrativeContent: true - } - - const explanation = explainPresetChoice(context) - expect(explanation).toContain('Narrative content') - expect(explanation).toContain('pattern preset') - }) - - it('should explain default choice', () => { - const context: ImportContext = { - rowCount: 500 - } - - const explanation = explainPresetChoice(context) - expect(explanation).toContain('balanced preset') - }) - }) - - describe('createCustomPreset', () => { - it('should create custom preset from base', () => { - const custom = createCustomPreset('balanced', { - name: 'my-custom', - batchSize: 2000 - }) - - expect(custom.name).toBe('my-custom') - expect(custom.batchSize).toBe(2000) - expect(custom.signals).toEqual(BALANCED_PRESET.signals) - expect(custom.strategies).toEqual(BALANCED_PRESET.strategies) - }) - - it('should override signals', () => { - const custom = createCustomPreset('fast', { - signals: { - enabled: ['embedding'], - weights: { embedding: 1.0, exact: 0, pattern: 0, context: 0 }, - timeout: 200 - } - }) - - expect(custom.signals.enabled).toEqual(['embedding']) - expect(custom.signals.timeout).toBe(200) - }) - - it('should override strategies', () => { - const custom = createCustomPreset('balanced', { - strategies: { - enabled: ['pattern'], - timeout: 500, - earlyTermination: false, - minConfidence: 0.75 - } - }) - - expect(custom.strategies.enabled).toEqual(['pattern']) - expect(custom.strategies.timeout).toBe(500) - expect(custom.strategies.earlyTermination).toBe(false) - }) - - it('should merge partial signal overrides', () => { - const custom = createCustomPreset('balanced', { - signals: { - timeout: 300 - } as any - }) - - expect(custom.signals.enabled).toEqual(BALANCED_PRESET.signals.enabled) - expect(custom.signals.timeout).toBe(300) - }) - }) - - describe('validatePreset', () => { - it('should validate all built-in presets', () => { - expect(() => validatePreset(FAST_PRESET)).not.toThrow() - expect(() => validatePreset(BALANCED_PRESET)).not.toThrow() - expect(() => validatePreset(ACCURATE_PRESET)).not.toThrow() - expect(() => validatePreset(EXPLICIT_PRESET)).not.toThrow() - expect(() => validatePreset(PATTERN_PRESET)).not.toThrow() - }) - - it('should reject preset with no signals', () => { - const invalid: PresetConfig = { - ...BALANCED_PRESET, - signals: { - ...BALANCED_PRESET.signals, - enabled: [] - } - } - - expect(() => validatePreset(invalid)).toThrow('at least one enabled signal') - }) - - it('should reject preset with no strategies', () => { - const invalid: PresetConfig = { - ...BALANCED_PRESET, - strategies: { - ...BALANCED_PRESET.strategies, - enabled: [] - } - } - - expect(() => validatePreset(invalid)).toThrow('at least one enabled strategy') - }) - - it('should reject preset with invalid weight sum', () => { - const invalid: PresetConfig = { - ...BALANCED_PRESET, - signals: { - enabled: ['exact', 'embedding'], - weights: { - exact: 0.3, - embedding: 0.5, - pattern: 0, - context: 0 - }, - timeout: 100 - } - } - - expect(() => validatePreset(invalid)).toThrow('weights must sum to 1.0') - }) - - it('should reject preset with negative timeout', () => { - const invalid: PresetConfig = { - ...BALANCED_PRESET, - signals: { - ...BALANCED_PRESET.signals, - timeout: -100 - } - } - - expect(() => validatePreset(invalid)).toThrow('Timeouts must be positive') - }) - - it('should reject preset with invalid batch size', () => { - const invalid: PresetConfig = { - ...BALANCED_PRESET, - batchSize: 0 - } - - expect(() => validatePreset(invalid)).toThrow('Batch size must be positive') - }) - }) - - describe('formatPreset', () => { - it('should format preset for display', () => { - const formatted = formatPreset(BALANCED_PRESET) - - expect(formatted).toContain('Preset: balanced') - expect(formatted).toContain('Description:') - expect(formatted).toContain('Signals:') - expect(formatted).toContain('exact: 40%') - expect(formatted).toContain('embedding: 35%') - expect(formatted).toContain('Strategies:') - expect(formatted).toContain('explicit') - expect(formatted).toContain('pattern') - expect(formatted).toContain('embedding') - expect(formatted).toContain('Streaming: false') - expect(formatted).toContain('Batch size: 500') - }) - - it('should format fast preset correctly', () => { - const formatted = formatPreset(FAST_PRESET) - - expect(formatted).toContain('fast') - expect(formatted).toContain('Streaming: true') - expect(formatted).toContain('Early termination: true') - }) - }) - - describe('preset priorities', () => { - it('should prioritize size over explicit columns for large datasets', () => { - const context: ImportContext = { - rowCount: 20000, - fileType: 'excel', - hasExplicitColumns: true - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('fast') // Size trumps explicit - }) - - it('should prioritize small size over other factors', () => { - const context: ImportContext = { - rowCount: 50, - fileType: 'pdf', - hasNarrativeContent: true - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('accurate') // Small size trumps pattern - }) - - it('should prioritize explicit columns over narrative for Excel', () => { - const context: ImportContext = { - rowCount: 500, - fileType: 'excel', - hasExplicitColumns: true, - hasNarrativeContent: true - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('explicit') // Explicit trumps narrative - }) - }) - - describe('edge cases', () => { - it('should handle zero row count', () => { - const context: ImportContext = { - rowCount: 0, - fileType: 'csv' - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('balanced') - }) - - it('should handle boundary row count (exactly 100)', () => { - const context: ImportContext = { - rowCount: 100 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('balanced') // Not accurate (< 100) - }) - - it('should handle boundary row count (exactly 10000)', () => { - const context: ImportContext = { - rowCount: 10000 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('balanced') // Not fast (> 10000) - }) - - it('should handle missing hasExplicitColumns flag', () => { - const context: ImportContext = { - fileType: 'excel', - rowCount: 500 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('balanced') - }) - }) - - describe('real-world scenarios', () => { - it('should handle glossary correctly', () => { - const context: ImportContext = { - fileType: 'excel', - rowCount: 567, - hasExplicitColumns: true, // Has "Related Terms" column - fileSize: 50_000 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('explicit') - - const explanation = explainPresetChoice(context) - expect(explanation).toContain('explicit') - }) - - it('should handle large CSV import', () => { - const context: ImportContext = { - fileType: 'csv', - rowCount: 50000, - fileSize: 25_000_000 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('fast') - expect(preset.streaming).toBe(true) - }) - - it('should handle PDF documentation', () => { - const context: ImportContext = { - fileType: 'pdf', - rowCount: 150, - hasNarrativeContent: true, - avgDefinitionLength: 600 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('pattern') - }) - - it('should handle JSON API import', () => { - const context: ImportContext = { - fileType: 'json', - rowCount: 1000, - fileSize: 500_000 - } - - const preset = autoDetectPreset(context) - expect(preset.name).toBe('balanced') - }) - }) -}) diff --git a/tests/unit/plugin-autodetect.test.ts b/tests/unit/plugin-autodetect.test.ts new file mode 100644 index 00000000..37c181ba --- /dev/null +++ b/tests/unit/plugin-autodetect.test.ts @@ -0,0 +1,136 @@ +/** + * Guarded plugin auto-detection (8.0.9) — the "install it and it's on" contract. + * + * With `plugins` unset, brainy probes for the first-party accelerator + * (@soulcraft/cor). The probe applies the same loud posture as the explicit + * list to everything except "not installed": + * - not installed → plain brainy, silently (the free path) + * - installed + healthy → registered and activated + * - installed + broken → init() THROWS (import failure, invalid shape, + * or failed activation — an installed + * accelerator never silently vanishes) + * - plugins: [] / false → no probe at all (explicit opt-out) + * - explicit list → unchanged: required, loud on any failure + * + * The import is isolated behind the private `importPluginPackage` seam so the + * three outcomes are simulated without the package being present. + */ +import { describe, it, expect, afterEach } from 'vitest' +import { Brainy, NounType } from '../../src/index.js' + +const proto = Brainy.prototype as any +const origImport = proto.importPluginPackage + +afterEach(() => { + proto.importPluginPackage = origImport +}) + +/** Simulate the probe outcome per package name. */ +function stubImport(fn: (pkg: string) => Promise): string[] { + const probed: string[] = [] + proto.importPluginPackage = async function (pkg: string) { + probed.push(pkg) + return fn(pkg) + } + return probed +} + +function notInstalledError(pkg: string): Error { + // Node's exact shape for a missing package in a dynamic ESM import. + const e = new Error(`Cannot find package '${pkg}' imported from /app/index.js`) + ;(e as any).code = 'ERR_MODULE_NOT_FOUND' + return e +} + +/** A minimal valid plugin that registers one well-known provider. */ +function validPlugin(name: string) { + return { + name, + activate: async (context: any) => { + context.registerProvider('distance', { euclidean: (a: number[], b: number[]) => 0 }) + return true + } + } +} + +describe('Guarded plugin auto-detection (plugins: undefined)', () => { + it('not installed → plain brainy, no plugins, init succeeds silently', async () => { + const probed = stubImport(async (pkg) => { + throw notInstalledError(pkg) + }) + const brain: any = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true }) + await brain.init() + expect(probed).toContain('@soulcraft/cor') // the probe ran… + expect(brain.pluginRegistry.hasActivePlugins()).toBe(false) // …and found nothing, quietly + const id = await brain.add({ data: 'works without any plugin', type: NounType.Concept }) + expect(id).toBeTruthy() + await brain.close() + }) + + it('installed + healthy → auto-registered and activated', async () => { + stubImport(async () => ({ default: validPlugin('@soulcraft/cor') })) + const brain: any = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true }) + await brain.init() + expect(brain.pluginRegistry.getActivePlugins()).toContain('@soulcraft/cor') + expect(brain.pluginRegistry.hasProvider('distance')).toBe(true) + await brain.close() + }) + + it('installed but import fails (broken install) → init() throws, never a silent JS fallback', async () => { + stubImport(async () => { + // A resolution failure INSIDE the package (names a sub-path, not the + // package) — present but broken, must be loud. + const e = new Error( + "Cannot find module '/app/node_modules/@soulcraft/cor/dist/native.js' imported from " + + '/app/node_modules/@soulcraft/cor/dist/index.js' + ) + ;(e as any).code = 'ERR_MODULE_NOT_FOUND' + throw e + }) + const brain: any = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true }) + await expect(brain.init()).rejects.toThrow(/installed but failed to load/) + }) + + it('installed but not a valid plugin (missing activate) → init() throws', async () => { + stubImport(async () => ({ default: { name: '@soulcraft/cor' } })) // no activate() + const brain: any = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true }) + await expect(brain.init()).rejects.toThrow(/not a valid Brainy plugin/) + }) + + it('installed but activation fails → init() throws (activateAll posture applies)', async () => { + stubImport(async () => ({ + default: { + name: '@soulcraft/cor', + activate: async () => { + throw new Error('native binary missing for platform') + } + } + })) + const brain: any = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true }) + await expect(brain.init()).rejects.toThrow(/failed to activate/) + }) + + it('plugins: [] and plugins: false → no probe at all (explicit opt-out)', async () => { + const probed = stubImport(async () => ({ default: validPlugin('@soulcraft/cor') })) + for (const plugins of [[], false] as const) { + const brain: any = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, plugins: plugins as any, silent: true }) + await brain.init() + expect(brain.pluginRegistry.hasActivePlugins()).toBe(false) + await brain.close() + } + expect(probed).toEqual([]) // the seam was never touched + }) + + it('explicit list is unchanged: a listed-but-missing plugin throws with the config-pointing message', async () => { + stubImport(async (pkg) => { + throw notInstalledError(pkg) + }) + const brain: any = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' }, + plugins: ['@soulcraft/cor'], + silent: true + }) + await expect(brain.init()).rejects.toThrow(/listed in config\.plugins but could not be loaded/) + }) +}) diff --git a/tests/unit/plugin-version-coupling.test.ts b/tests/unit/plugin-version-coupling.test.ts index 26f551c1..00b236aa 100644 --- a/tests/unit/plugin-version-coupling.test.ts +++ b/tests/unit/plugin-version-coupling.test.ts @@ -12,9 +12,17 @@ * 6. a graceful decline (activate()→false) → no throw, loud warn */ import { describe, it, expect } from 'vitest' +import { readFileSync } from 'node:fs' +import { fileURLToPath } from 'node:url' +import { dirname, join } from 'node:path' import { Brainy } from '../../src/brainy.js' +import { getBrainyVersion } from '../../src/utils/version.js' import { pluginRangeSatisfies, type BrainyPlugin, type BrainyPluginContext } from '../../src/plugin.js' +const PACKAGE_VERSION: string = JSON.parse( + readFileSync(join(dirname(fileURLToPath(import.meta.url)), '../../package.json'), 'utf8') +).version + function memBrain(): Brainy { return new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true }) } @@ -44,6 +52,21 @@ describe('pluginRangeSatisfies (dep-free range matcher)', () => { }) }) +describe('getBrainyVersion() — synchronously correct on first call', () => { + it('returns the real package version, not a stale build-time default', () => { + // Regression: the version was read async, so the FIRST (synchronous) call — + // the one loadPlugins() makes to build context.version — returned a stale + // default ('3.14.0'), which fails any realistic cor 3.x `>=8.0.0` range and + // hard-rejects the matched native pairing on cold init. The sync read makes + // the first call correct. + const v = getBrainyVersion() + expect(v).toBe(PACKAGE_VERSION) + expect(v).not.toBe('3.14.0') + expect(v).not.toBe('0.0.0') // the unknown-read sentinel must not surface in a real install + expect(v.startsWith('8.')).toBe(true) + }) +}) + /** A minimal fake plugin; `onActivate` lets a test register providers / throw. */ function fakePlugin( name: string, @@ -72,6 +95,17 @@ describe('version coupling at init() — no silent fallback', () => { await brain.close() }) + it('does NOT throw for a realistic cor 3.x range (^8.0.0) on a COLD init', async () => { + // The actual regression: loadPlugins() is the first init step and makes the + // first getBrainyVersion() call, so a stale sync default would reject a + // correctly-matched native provider declaring the real 8.x range. A fresh + // brain registering a `^8.0.0` plugin must init cleanly. + const brain = memBrain() + brain.use(fakePlugin('@fake/cor-3x', { brainyRange: '^8.0.0' })) + await expect(brain.init()).resolves.toBeUndefined() + await brain.close() + }) + it('throws (does not swallow) when activate() throws', async () => { const brain = memBrain() brain.use(fakePlugin('@fake/boom', { onActivate: () => { throw new Error('kaboom') } })) diff --git a/tests/unit/process-exit-sweep.test.ts b/tests/unit/process-exit-sweep.test.ts new file mode 100644 index 00000000..d0365d8c --- /dev/null +++ b/tests/unit/process-exit-sweep.test.ts @@ -0,0 +1,120 @@ +/** + * Process-exit sweep — no operation class may leave a ref'd timer behind. + * + * A consumer's clean-room verification of the close()-hang fix found the + * add-only path clean but `relate()` leaving ~5 uncleared Timeouts — the fix + * had covered a repro, not the bug class. This sweep turns the class off: + * for EVERY op family (add / relate / graph find / vfs), assert that after + * `close()` no NEW ref'd Timeout survives (unref'd timers are fine — they + * never keep the host process alive; that is the standard for every + * background-maintenance interval in brainy). + * + * Filesystem storage on purpose: it is the reported environment and the one + * that arms the most timers (LSM flush/compaction, write buffers, watchers). + */ +import { describe, it, expect, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy, NounType, VerbType } from '../../src/index.js' + +const tmpDirs: string[] = [] +function mkTmp(): string { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-exit-sweep-')) + tmpDirs.push(d) + return d +} +afterEach(() => { + for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }) +}) + +const V = () => Array.from({ length: 384 }, () => Math.random()) + +/** Identity set of currently-ref'd Timeout handles (the only kind that can + * keep the event loop — and a bare consumer script — alive). */ +function refdTimeouts(): Set { + const handles: any[] = (process as any)._getActiveHandles?.() ?? [] + return new Set( + handles.filter( + (h) => + h?.constructor?.name === 'Timeout' && + (typeof h.hasRef !== 'function' || h.hasRef()) + ) + ) +} + +/** Describe a leaked timer well enough to find its source. */ +function describeTimer(h: any): string { + const fn = h?._onTimeout + return `Timeout(after=${h?._idleTimeout}ms, repeat=${!!h?._repeat}, fn=${ + fn?.name || 'anon' + }: ${String(fn).slice(0, 120).replace(/\s+/g, ' ')})` +} + +async function sweep(run: (brain: any) => Promise): Promise { + const before = refdTimeouts() + const brain: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: mkTmp() }, + plugins: [], + silent: true + }) + await brain.init() + await run(brain) + await brain.close() + return [...refdTimeouts()].filter((h) => !before.has(h)).map(describeTimer) +} + +describe('Process-exit sweep — close() leaves no ref’d timers, per op class', () => { + it('add', async () => { + const leaked = await sweep(async (brain) => { + await brain.add({ data: 'solo', type: NounType.Concept, subtype: 's', vector: V() }) + }) + expect(leaked).toEqual([]) + }) + + it('add + relate (the reported hang shape)', async () => { + const leaked = await sweep(async (brain) => { + const a = await brain.add({ data: 'a', type: NounType.Concept, subtype: 's', vector: V() }) + const b = await brain.add({ data: 'b', type: NounType.Concept, subtype: 's', vector: V() }) + await brain.relate({ from: a, to: b, type: VerbType.RelatedTo, subtype: 's' }) + }) + expect(leaked).toEqual([]) + }) + + it('relate + graph find + related()', async () => { + const leaked = await sweep(async (brain) => { + const ids: string[] = [] + for (let i = 0; i < 6; i++) { + ids.push(await brain.add({ data: `n${i}`, type: NounType.Concept, subtype: 's', vector: V() })) + } + for (let i = 0; i + 1 < ids.length; i++) { + await brain.relate({ from: ids[i], to: ids[i + 1], type: VerbType.RelatedTo, subtype: 's' }) + } + await brain.find({ connected: { from: ids[0], depth: 2 } }) + await brain.related({ node: ids[1] }) + }) + expect(leaked).toEqual([]) + }) + + it('metadata find + update (write-buffer path)', async () => { + const leaked = await sweep(async (brain) => { + const id = await brain.add({ + data: 'meta', type: NounType.Concept, subtype: 's', + metadata: { wave: 1 }, vector: V() + }) + await brain.update({ id, metadata: { wave: 2 } }) + await brain.find({ where: { wave: { greaterThan: 1 } } }) + }) + expect(leaked).toEqual([]) + }) + + it('vfs write/read/tree (VFS + PathResolver timers)', async () => { + const leaked = await sweep(async (brain) => { + await brain.vfs.writeFile('/notes/a.md', 'alpha') + await brain.vfs.readFile('/notes/a.md') + await brain.vfs.getTreeStructure('/', { maxDepth: 2 }) + }) + expect(leaked).toEqual([]) + }) +}) diff --git a/tests/unit/shutdown-hooks-lifecycle.test.ts b/tests/unit/shutdown-hooks-lifecycle.test.ts new file mode 100644 index 00000000..3897daf7 --- /dev/null +++ b/tests/unit/shutdown-hooks-lifecycle.test.ts @@ -0,0 +1,79 @@ +/** + * Global shutdown-hook lifecycle — a library must never keep its host process + * alive. init() registers process-level SIGTERM/SIGINT/beforeExit listeners + * (once, globally) whose signal handles are ref'd by Node; if they outlive the + * last brain, a bare script hangs forever after close() (consumer-reported on + * the GA pair). The contract pinned here: + * + * - first init() → exactly one listener added per signal + * - more inits → no additional listeners (registered once) + * - close() → listeners survive while OTHER instances remain live + * - last close() → listeners removed, counts back to baseline + * - re-init → hooks re-register (the once-flag resets) + */ +import { describe, it, expect } from 'vitest' +import { Brainy, NounType } from '../../src/index.js' + +const SIGNALS = ['SIGTERM', 'SIGINT', 'beforeExit'] as const + +function listenerCounts(): Record { + return Object.fromEntries(SIGNALS.map((s) => [s, process.listeners(s as any).length])) +} + +function makeBrain(): any { + return new Brainy({ requireSubtype: false, storage: { type: 'memory' }, plugins: [], silent: true }) +} + +describe('Shutdown-hook lifecycle (process must exit after the last close)', () => { + it('registers each hook once on first init and removes ALL of them on the last close', async () => { + const baseline = listenerCounts() + + const brain = makeBrain() + await brain.init() + for (const s of SIGNALS) { + expect(process.listeners(s as any).length).toBe(baseline[s] + 1) // exactly one each + } + + await brain.close() + expect(listenerCounts()).toEqual(baseline) // gone — the loop can drain + }) + + it('keeps the hooks while any other instance is live; only the LAST close deregisters', async () => { + const baseline = listenerCounts() + + const first = makeBrain() + const second = makeBrain() + await first.init() + await second.init() + for (const s of SIGNALS) { + expect(process.listeners(s as any).length).toBe(baseline[s] + 1) // once globally, not per instance + } + + await first.close() + for (const s of SIGNALS) { + expect(process.listeners(s as any).length).toBe(baseline[s] + 1) // second is still live + } + + await second.close() + expect(listenerCounts()).toEqual(baseline) + }) + + it('re-registers after a full close/re-init cycle (the once-flag resets)', async () => { + const baseline = listenerCounts() + + const first = makeBrain() + await first.init() + await first.close() + expect(listenerCounts()).toEqual(baseline) + + const second = makeBrain() + await second.init() + const id = await second.add({ data: 'still fully functional', type: NounType.Concept }) + expect(id).toBeTruthy() + for (const s of SIGNALS) { + expect(process.listeners(s as any).length).toBe(baseline[s] + 1) + } + await second.close() + expect(listenerCounts()).toEqual(baseline) + }) +}) diff --git a/tests/unit/storage/blob-save-durability.test.ts b/tests/unit/storage/blob-save-durability.test.ts new file mode 100644 index 00000000..f775ebdf --- /dev/null +++ b/tests/unit/storage/blob-save-durability.test.ts @@ -0,0 +1,111 @@ +/** + * @module tests/unit/storage/blob-save-durability + * @description Finding 13 (cortex sweep): saveBinaryBlob must never acknowledge a + * durable write that stored nothing. With the UNIQUE per-writer temp suffix, a + * rename ENOENT can no longer mean "a concurrent idempotent writer already + * renamed it" — nobody else has this writer's temp — so it means our temp + * vanished before the rename and the bytes did NOT land. The old code returned + * success on that ENOENT; the native provider mmaps these blobs, so a + * phantom-acked blob is the exact silent-loss class the registered-blob work + * chases. The fix retries once with a fresh temp, then fails loud. + */ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js' + +const enoent = (): Error => Object.assign(new Error('temp removed'), { code: 'ENOENT' }) + +describe('FileSystemStorage.saveBinaryBlob durable-write honesty (finding 13)', () => { + let dir: string + let storage: any + + beforeEach(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-blob-save-')) + storage = new FileSystemStorage(dir) + await storage.init() + }) + + afterEach(() => { + vi.restoreAllMocks() + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('persists and reads back a blob under normal conditions', async () => { + await storage.saveBinaryBlob('graph-lsm/seg-1', Buffer.from([1, 2, 3, 4])) + expect(await storage.loadBinaryBlob('graph-lsm/seg-1')).toEqual(Buffer.from([1, 2, 3, 4])) + }) + + it('throws (never acks) when the temp vanishes on every rename attempt', async () => { + vi.spyOn(fs.promises, 'rename').mockRejectedValue(enoent()) + + await expect( + storage.saveBinaryBlob('k/silent-loss', Buffer.from([9])) + ).rejects.toThrow(/did NOT persist/) + + // Prove the write truly did not land — and was not acknowledged. (loadBinaryBlob + // uses readFile, not rename, so it reflects the real on-disk state.) + expect(await storage.loadBinaryBlob('k/silent-loss')).toBeNull() + }) + + it('self-heals: a single transient temp-vanish retries and persists', async () => { + const real = fs.promises.rename.bind(fs.promises) + let calls = 0 + vi.spyOn(fs.promises, 'rename').mockImplementation(async (from: any, to: any) => { + calls++ + if (calls === 1) throw enoent() + return real(from, to) + }) + + await expect( + storage.saveBinaryBlob('k/heals', Buffer.from([7, 7])) + ).resolves.toBeUndefined() + expect(calls).toBe(2) // first attempt vanished, retry landed + expect(await storage.loadBinaryBlob('k/heals')).toEqual(Buffer.from([7, 7])) + }) + + it('a non-ENOENT rename fault propagates verbatim (not swallowed)', async () => { + vi.spyOn(fs.promises, 'rename').mockRejectedValue( + Object.assign(new Error('disk fault'), { code: 'EIO' }) + ) + await expect( + storage.saveBinaryBlob('k/eio', Buffer.from([1])) + ).rejects.toMatchObject({ code: 'EIO' }) + }) +}) + +describe('FileSystemStorage.loadBinaryBlob fault-propagation (finding 11; cor 3.0.13 lockstep)', () => { + let dir: string + let storage: any + + beforeEach(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-blob-load-')) + storage = new FileSystemStorage(dir) + await storage.init() + }) + + afterEach(() => { + vi.restoreAllMocks() + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('returns null for a genuinely absent blob (ENOENT)', async () => { + expect(await storage.loadBinaryBlob('never/written')).toBeNull() + }) + + it('reads back a present blob', async () => { + await storage.saveBinaryBlob('present/blob', Buffer.from([1, 2, 3])) + expect(await storage.loadBinaryBlob('present/blob')).toEqual(Buffer.from([1, 2, 3])) + }) + + it('propagates a real IO fault instead of masking it as absent', async () => { + await storage.saveBinaryBlob('present/blob', Buffer.from([1, 2, 3])) + vi.spyOn(fs.promises, 'readFile').mockRejectedValue( + Object.assign(new Error('disk fault'), { code: 'EIO' }) + ) + // A present-but-unreadable blob must NOT read as null (that drove needless + // native rebuilds / empty reads); cor 3.0.13's column-store handles the throw. + await expect(storage.loadBinaryBlob('present/blob')).rejects.toMatchObject({ code: 'EIO' }) + }) +}) diff --git a/tests/unit/storage/blobStorage.test.ts b/tests/unit/storage/blobStorage.test.ts index 8138c64f..55bc5276 100644 --- a/tests/unit/storage/blobStorage.test.ts +++ b/tests/unit/storage/blobStorage.test.ts @@ -258,21 +258,31 @@ describe('BlobStorage', () => { expect(metadata?.refCount).toBe(2) }) - it('should only delete when refCount reaches 0', async () => { + it('release() drops live references; bytes are reclaimed ONLY by reclaimIfUnreferenced at zero-zero', async () => { const data = Buffer.from('test') - // Write twice (refCount = 2) + // Write twice (live refCount = 2) const hash = await blobStorage.write(data) await blobStorage.write(data) - // Delete once (refCount = 1, blob still exists) - await blobStorage.delete(hash) - + // Release once (refCount = 1, blob still exists) + await blobStorage.release(hash) expect(await blobStorage.has(hash)).toBe(true) - // Delete again (refCount = 0, blob deleted) - await blobStorage.delete(hash) + // Release again (refCount = 0) — bytes STILL exist: content is + // immutable under the temporal model; only history compaction reclaims. + await blobStorage.release(hash) + expect(await blobStorage.has(hash)).toBe(true) + expect((await blobStorage.getMetadata(hash))?.refCount).toBe(0) + // A history reference blocks reclamation even at live-zero. + await blobStorage.recordHistoryReference(hash) + expect(await blobStorage.reclaimIfUnreferenced(hash)).toBe(false) + expect(await blobStorage.has(hash)).toBe(true) + + // Drop the history reference → zero-zero → reclaim succeeds. + await blobStorage.releaseHistoryReference(hash) + expect(await blobStorage.reclaimIfUnreferenced(hash)).toBe(true) expect(await blobStorage.has(hash)).toBe(false) }) }) diff --git a/tests/unit/storage/clear-native-footprint.test.ts b/tests/unit/storage/clear-native-footprint.test.ts new file mode 100644 index 00000000..313ed9b8 --- /dev/null +++ b/tests/unit/storage/clear-native-footprint.test.ts @@ -0,0 +1,74 @@ +/** + * @module tests/unit/storage/clear-native-footprint + * @description clear() must remove the COMPLETE derived/native on-disk footprint + * (finding 7). Before the fix it wiped only entities/indexes/system/_cas and left + * three top-level trees behind: + * - `_blobs` raw binary blobs (HNSW/LSM segments, the native dkann index, + * and column-store segment bytes under `_blobs/_column_index/`) + * - `_id_mapper` the native shared mmap id-mapper + * - `_column_index` the column-store MANIFEST.json files + * A cleared brain therefore re-read stale native state, and — worst — an orphaned + * column manifest whose segment bytes had been removed with `_blobs`, which the + * hardened load path now refuses with ColumnSegmentLoadError. The three must fall + * together as a set. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js' + +const exists = (p: string): boolean => fs.existsSync(p) + +describe('FileSystemStorage.clear() wipes the full native/derived footprint (finding 7)', () => { + let dir: string + let storage: any + + beforeEach(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-clear-footprint-')) + storage = new FileSystemStorage(dir) + await storage.init() + }) + + afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('removes _blobs, _id_mapper and _column_index (previously left stranded)', async () => { + // A raw blob (this is how HNSW/LSM/column segments persist) → creates _blobs. + await storage.saveBinaryBlob('graph-lsm/source/sstable-1', Buffer.from([1, 2, 3, 4])) + // A column segment blob lives UNDER _blobs/_column_index/... + await storage.saveBinaryBlob('_column_index/createdAt/L0-000000', Buffer.from([5, 6, 7, 8])) + + // Simulate the native id-mapper mmap dir and the column manifest object-tree. + const idMapperDir = path.join(dir, '_id_mapper') + fs.mkdirSync(idMapperDir, { recursive: true }) + fs.writeFileSync(path.join(idMapperDir, 'main.slotmap'), Buffer.from([0, 1])) + const colDir = path.join(dir, '_column_index', 'createdAt') + fs.mkdirSync(colDir, { recursive: true }) + fs.writeFileSync( + path.join(colDir, 'MANIFEST.json'), + JSON.stringify({ segments: [{ id: 0, level: 0, count: 1 }] }) + ) + + // Sanity: everything is present before the clear. + expect(exists(path.join(dir, '_blobs'))).toBe(true) + expect(exists(idMapperDir)).toBe(true) + expect(exists(path.join(dir, '_column_index'))).toBe(true) + + await storage.clear() + + // The complete derived footprint is gone — no stale native state, and no + // orphaned column manifest pointing at removed segment bytes. + expect(exists(path.join(dir, '_blobs'))).toBe(false) + expect(exists(idMapperDir)).toBe(false) + expect(exists(path.join(dir, '_column_index'))).toBe(false) + }) + + it('clear() is a no-op-safe when the native dirs never existed', async () => { + // Fresh store, no blobs written — clear must not throw on absent dirs. + await expect(storage.clear()).resolves.toBeUndefined() + expect(exists(path.join(dir, '_blobs'))).toBe(false) + expect(exists(path.join(dir, '_column_index'))).toBe(false) + }) +}) diff --git a/tests/unit/storage/pagination-parallel-hydration.test.ts b/tests/unit/storage/pagination-parallel-hydration.test.ts new file mode 100644 index 00000000..ada324bb --- /dev/null +++ b/tests/unit/storage/pagination-parallel-hydration.test.ts @@ -0,0 +1,94 @@ +/** + * @module tests/unit/storage/pagination-parallel-hydration + * @description The canonical enumeration walk (getNounsWithPagination) hydrated + * each item's vector + metadata ONE-AT-A-TIME — N×per-op-latency serially, the + * dominant term in an index heal (cortex heal-cost decomposition). It now + * hydrates 16-way, and a new getNounIdsWithPagination returns ids WITHOUT + * hydration (zero per-entity reads when unfiltered). Both must preserve the exact + * pagination contract: same order, cursor continuation, filters, totalCount. + */ +import { describe, it, expect, beforeEach, vi } from 'vitest' +import { Brainy, NounType } from '../../../src/index.js' + +describe('paginated enumeration — parallel hydration + id-only (cortex heal-cost)', () => { + let brain: any + let storage: any + const N = 30 + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, dimensions: 384 }) + await brain.init() + for (let i = 0; i < N; i++) { + await brain.add({ + data: `n${i}`, + type: i % 3 === 0 ? NounType.Task : NounType.Concept, + metadata: { i } + }) + } + await brain.flush() + storage = brain.storage + }) + + /** Page the whole dataset through a small limit via cursor and collect ordered ids. */ + const pageAll = async (fn: (opts: any) => Promise, key: 'items' | 'ids') => { + const out: string[] = [] + let cursor: string | undefined + for (let guard = 0; guard < 1000; guard++) { + const page = await fn({ limit: 4, cursor }) + const batch = key === 'items' ? page.items.map((n: any) => n.id) : page.ids + out.push(...batch) + if (!page.hasMore) break + cursor = page.nextCursor + } + return out + } + + it('parallel hydration yields the SAME ordered pages as one big page', async () => { + const big = await storage.getNounsWithPagination({ limit: 1000, offset: 0 }) + const bigIds = big.items.map((n: any) => n.id) + // At least the N we added (a brain also has its VFS root entity). + expect(bigIds.length).toBeGreaterThanOrEqual(N) + expect(big.totalCount).toBe(bigIds.length) + + const paged = await pageAll((o) => storage.getNounsWithPagination(o), 'items') + expect(paged).toEqual(bigIds) // identical order, no dupes, no gaps across pages + }) + + it('getNounIdsWithPagination returns exactly the same ids, in the same order', async () => { + const idsPaged = await pageAll((o) => storage.getNounIdsWithPagination(o), 'ids') + const itemsPaged = await pageAll((o) => storage.getNounsWithPagination(o), 'items') + expect(idsPaged).toEqual(itemsPaged) + expect(new Set(idsPaged).size).toBe(idsPaged.length) // every id exactly once + expect(idsPaged.length).toBeGreaterThanOrEqual(N) + }) + + it('id-only enumeration does ZERO per-entity hydration reads when unfiltered', async () => { + const readSpy = vi.spyOn(storage as any, 'readCanonicalObject') + await storage.getNounIdsWithPagination({ limit: 1000, offset: 0 }) + expect(readSpy).not.toHaveBeenCalled() + readSpy.mockRestore() + + // The hydrating walk, by contrast, DOES read each entity. + const readSpy2 = vi.spyOn(storage as any, 'readCanonicalObject') + await storage.getNounsWithPagination({ limit: 1000, offset: 0 }) + expect(readSpy2.mock.calls.length).toBeGreaterThan(0) + readSpy2.mockRestore() + }) + + it('a type filter matches between the hydrating and id-only walks', async () => { + const taskItems = await storage.getNounsWithPagination({ + limit: 1000, + offset: 0, + filter: { nounType: NounType.Task } + }) + const taskIds = await storage.getNounIdsWithPagination({ + limit: 1000, + offset: 0, + filter: { nounType: NounType.Task } + }) + const expected = Math.ceil(N / 3) // every 3rd is a Task + expect(taskItems.items.length).toBe(expected) + expect(new Set(taskIds.ids)).toEqual(new Set(taskItems.items.map((n: any) => n.id))) + }) +}) diff --git a/tests/unit/storage/registered-blob-contract.test.ts b/tests/unit/storage/registered-blob-contract.test.ts new file mode 100644 index 00000000..6ad51ae7 --- /dev/null +++ b/tests/unit/storage/registered-blob-contract.test.ts @@ -0,0 +1,130 @@ +/** + * @module tests/unit/storage/registered-blob-contract + * @description Declared derived-index blob FAMILIES are + * undeletable through the storage layer — an in-process GC/sweeper cannot remove + * a load-bearing index file (the lost-main.dkann class). Covers declare → + * protected-delete-throws; unregister → delete-ok; transient passthrough; + * namespace prefix protects children; removeRawPrefix refuses a protected + * intersection; persistence across reopen; missing-on-open detection. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' +import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js' +import { ProtectedArtifactError } from '../../../src/index.js' + +describe('registered-blob family contract', () => { + let storage: any + + beforeEach(async () => { + storage = new MemoryStorage() + await storage.init() + }) + + const seedVectorFamily = async () => { + await storage.saveBinaryBlob('_system/vector-index/main.dkann', Buffer.from([1])) + await storage.saveBinaryBlob('_system/vector-index/main.slotmap', Buffer.from([2])) + await storage.saveBinaryBlob('_system/vector-index/main.slotrev', Buffer.from([3])) + await storage.registerDerivedFamily({ + name: 'vector-base', + members: [ + '_system/vector-index/main.dkann', + '_system/vector-index/main.slotmap', + '_system/vector-index/main.slotrev' + ] + }) + } + + it('a declared member is undeletable (throws ProtectedArtifactError)', async () => { + await seedVectorFamily() + await expect(storage.deleteBinaryBlob('_system/vector-index/main.dkann')).rejects.toBeInstanceOf( + ProtectedArtifactError + ) + // The blob is still there. + expect(await storage.loadBinaryBlob('_system/vector-index/main.dkann')).not.toBeNull() + }) + + it('unregistering the family makes its members deletable again', async () => { + await seedVectorFamily() + await storage.unregisterDerivedFamily('vector-base') + await expect(storage.deleteBinaryBlob('_system/vector-index/main.dkann')).resolves.toBeUndefined() + expect(await storage.loadBinaryBlob('_system/vector-index/main.dkann')).toBeNull() + }) + + it('an undeclared blob is still deletable (contract does not lock everything)', async () => { + await seedVectorFamily() + await storage.saveBinaryBlob('graph-lsm/source/sstable-1', Buffer.from([9])) + await expect(storage.deleteBinaryBlob('graph-lsm/source/sstable-1')).resolves.toBeUndefined() + }) + + it('a transient (*.tmp.*) is deletable even if it sits under a protected namespace', async () => { + await storage.registerDerivedFamily({ + name: 'vector-ns', + members: ['_system/vector-index/'], + namespace: true + }) + await storage.saveBinaryBlob('_system/vector-index/main.dkann.tmp.123', Buffer.from([1])) + await expect( + storage.deleteBinaryBlob('_system/vector-index/main.dkann.tmp.123') + ).resolves.toBeUndefined() + }) + + it('a namespace family protects a growing child key', async () => { + await storage.registerDerivedFamily({ + name: 'vector-segments', + members: ['_system/vector-index/'], + namespace: true + }) + await storage.saveBinaryBlob('_system/vector-index/seg-42', Buffer.from([7])) + await expect(storage.deleteBinaryBlob('_system/vector-index/seg-42')).rejects.toBeInstanceOf( + ProtectedArtifactError + ) + }) + + it('checkDerivedFamiliesPresent() names a member deleted outside the write path', async () => { + await seedVectorFamily() + // Simulate an EXTERNAL deleter (bypasses the in-process refusal): drop a + // member straight from the underlying store. + ;(storage as any).blobStore.delete('_system/vector-index/main.slotmap') + const incomplete = await storage.checkDerivedFamiliesPresent() + expect(incomplete).toHaveLength(1) + expect(incomplete[0].name).toBe('vector-base') + expect(incomplete[0].missing).toContain('_system/vector-index/main.slotmap') + expect(incomplete[0].rebuildable).toBe(true) + }) + + it('no families registered → deleteBinaryBlob behaves exactly as before (no enforcement)', async () => { + await storage.saveBinaryBlob('some/blob', Buffer.from([1])) + await expect(storage.deleteBinaryBlob('some/blob')).resolves.toBeUndefined() + }) +}) + +describe('registered-blob family protection survives a reopen (FileSystemStorage)', () => { + let dir: string + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-regblob-')) + }) + afterEach(() => fs.rmSync(dir, { recursive: true, force: true })) + + it('a family declared in one session protects members after reopen', async () => { + const s1: any = new FileSystemStorage(dir) + await s1.init() + await s1.saveBinaryBlob('_system/vector-index/main.dkann', Buffer.from([1])) + await s1.registerDerivedFamily({ name: 'vector-base', members: ['_system/vector-index/main.dkann'] }) + + // Reopen — the registry is loaded from _system/derived-artifacts.json. + const s2: any = new FileSystemStorage(dir) + await s2.init() + expect(await s2.listDerivedFamilies()).toHaveLength(1) + await expect(s2.deleteBinaryBlob('_system/vector-index/main.dkann')).rejects.toBeInstanceOf( + ProtectedArtifactError + ) + // removeRawPrefix nuking the vector dir is also refused. + await expect(s2.removeRawPrefix('_blobs/_system/vector-index')).rejects.toBeInstanceOf( + ProtectedArtifactError + ) + }) +}) diff --git a/tests/unit/storage/sharding-new-installation-log.test.ts b/tests/unit/storage/sharding-new-installation-log.test.ts new file mode 100644 index 00000000..c1af8427 --- /dev/null +++ b/tests/unit/storage/sharding-new-installation-log.test.ts @@ -0,0 +1,128 @@ +/** + * @module tests/unit/storage/sharding-new-installation-log + * @description Regression for BR-8-BOOT-INDEX symptom 1. Brainy 8.0 stores + * entities in the canonical `entities/nouns///vectors.json` layout, + * but the legacy sharding probe (`detectExistingShardingDepth`) inspected + * `entities/nouns/hnsw` — a 7.x directory the 8.0 write path never populates — + * so it returned null and logged "📁 New installation" for EVERY store on EVERY + * boot, including established brains holding thousands of entities. The + * new-vs-existing decision now consults the canonical layout the DB actually + * uses (`hasCanonicalEntities`), so an established brain reports its true entity + * count and only a genuinely empty store says "New installation". + */ +import { describe, it, expect, vi, afterEach } from 'vitest' +import { mkdtempSync, rmSync } from 'fs' +import { tmpdir } from 'os' +import { join } from 'path' +import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js' +import type { NounMetadata } from '../../../src/coreTypes.js' + +const dirs: string[] = [] +function mkTmp(): string { + const d = mkdtempSync(join(tmpdir(), 'brainy-newinstall-')) + dirs.push(d) + return d +} +afterEach(() => { + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) + vi.restoreAllMocks() +}) + +/** Release a raw FileSystemStorage so a subsequent open of the dir is unblocked. */ +async function teardown(s: any): Promise { + try { await s.flush?.() } catch { /* best effort */ } + try { s.stopFlushRequestWatcher?.() } catch { /* best effort */ } + try { await s.releaseWriterLock?.() } catch { /* best effort */ } +} + +/** Capture everything written to console.log during `fn()`. */ +async function captureLog(fn: () => Promise): Promise { + const lines: string[] = [] + const spy = vi + .spyOn(console, 'log') + .mockImplementation((...args: any[]) => { + lines.push(args.map(String).join(' ')) + }) + try { + await fn() + } finally { + spy.mockRestore() + } + return lines.join('\n') +} + +const VEC = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8] +async function seedOne(s: any, id: string): Promise { + await s.saveNounMetadata(id, { + noun: 'thing', + createdAt: Date.now(), + updatedAt: Date.now() + } as NounMetadata) + await s.saveNoun({ id, vector: VEC, connections: new Map(), level: 0 }) +} + +describe('FileSystemStorage boot log: an established brain is not "New installation" (BR-8-BOOT-INDEX)', () => { + it('a genuinely fresh store still logs "New installation"', async () => { + const dir = mkTmp() + const storage = new FileSystemStorage(dir) + const out = await captureLog(() => storage.init()) + expect(out).toContain('New installation') + await teardown(storage) + }) + + it('an established store (canonical entities present) does NOT log "New installation" on reopen', async () => { + const dir = mkTmp() + + // 1. Author an entity → canonical entities/nouns///vectors.json. + const first = new FileSystemStorage(dir) + await first.init() + await seedOne(first, 'a'.repeat(32)) + await teardown(first) + + // 2. Reopen: the boot log must reflect the established store, not "New installation". + const second = new FileSystemStorage(dir) + const out = await captureLog(() => second.init()) + expect(out).not.toContain('New installation') + expect(out).toContain('Using depth 1 sharding') + await teardown(second) + }) + + it('counts RECOVER from the canonical layout when counts.json is lost (container-restart case)', async () => { + const dir = mkTmp() + + // Establish a store with 3 entities (canonical layout + counts.json). + const first: any = new FileSystemStorage(dir) + await first.init() + for (const c of ['a', 'b', 'c']) await seedOne(first, c.repeat(32)) + await teardown(first) + + // Simulate the lost/corrupted counts.json a container restart can leave. + for (const f of ['counts.json', 'counts.json.gz']) { + rmSync(join(dir, '_system', f), { force: true }) + } + + // Reopen: the recovery scan must count the CANONICAL tree. The removed + // implementation scanned the vestigial `entities/*/hnsw` dirs and + // re-initialized every counter to ZERO on real data. + const second: any = new FileSystemStorage(dir) + await second.init() + expect(await second.getNounCount()).toBe(3) + await teardown(second) + }) + + it('hasCanonicalEntities() sees the canonical shard tree — independent of counts.json', async () => { + const dir = mkTmp() + const s: any = new FileSystemStorage(dir) + await s.init() + + // Fresh: only the vestigial `hnsw` dir exists → no canonical entities. + expect(await s.hasCanonicalEntities()).toBe(false) + + // After a canonical write the 2-hex shard dir exists → established, even if + // counts.json were later lost on a container restart (the fallback path). + await seedOne(s, 'c'.repeat(32)) + expect(await s.hasCanonicalEntities()).toBe(true) + + await teardown(s) + }) +}) diff --git a/tests/unit/storage/verb-cursor-pagination.test.ts b/tests/unit/storage/verb-cursor-pagination.test.ts new file mode 100644 index 00000000..cff41d02 --- /dev/null +++ b/tests/unit/storage/verb-cursor-pagination.test.ts @@ -0,0 +1,109 @@ +/** + * @module tests/unit/storage/verb-cursor-pagination + * @description Graph-perf #2 (8.0): cursor pagination over the verb shard walk. + * + * `getVerbsWithPagination` used to be offset-only — it re-scanned from shard 0 on + * every page and early-terminated at `offset+limit`, so a full edge walk was O(N²) + * (a consumer measured 19k edges → 27s paging at 900/page). It now accepts an + * opaque cursor that resumes immediately after the last returned verb, making a + * full walk O(N) at any page size. These tests pin the correctness guarantees a + * cursor MUST provide: every item exactly once, no duplicates, no skips, and the + * same set as an offset walk — plus graceful fallback for a foreign token. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/index.js' +import { NounType, VerbType } from '../../../src/types/graphTypes.js' +import { createTestConfig } from '../../helpers/test-factory.js' + +describe('verb cursor pagination (graph-perf #2)', () => { + let brain: Brainy + // The storage layer is where the cursor lives; exercise the primitive directly. + let storage: { + getVerbs(opts: { + pagination?: { offset?: number; limit?: number; cursor?: string } + }): Promise<{ items: Array<{ id: string }>; hasMore: boolean; nextCursor?: string }> + } + const EDGE_COUNT = 40 + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + const ids: string[] = [] + for (let i = 0; i <= EDGE_COUNT; i++) { + ids.push(await brain.add({ type: NounType.Person, subtype: 'employee', data: `N${i}` })) + } + // Hub-and-spoke: EDGE_COUNT edges from one node, spread across id-hash shards. + for (let i = 1; i <= EDGE_COUNT; i++) { + await brain.relate({ from: ids[0], to: ids[i], type: VerbType.RelatedTo, subtype: 'colleague' }) + } + storage = (brain as unknown as { storage: typeof storage }).storage + }) + + afterEach(async () => { + await brain.close() + }) + + it('a full cursored walk visits every edge exactly once (no dup, no skip)', async () => { + const all = await storage.getVerbs({ pagination: { limit: 10000 } }) + const refIds = new Set(all.items.map((v) => v.id)) + expect(refIds.size).toBe(EDGE_COUNT) + + const seen: string[] = [] + let cursor: string | undefined + let pages = 0 + for (;;) { + const page = await storage.getVerbs({ + pagination: cursor ? { limit: 7, cursor } : { limit: 7 } + }) + pages++ + for (const v of page.items) seen.push(v.id) + if (!page.hasMore) break + expect(page.nextCursor).toBeTruthy() + cursor = page.nextCursor + if (pages > 100) throw new Error('cursor walk failed to terminate') + } + + expect(pages).toBeGreaterThan(1) // genuinely paginated + expect(new Set(seen).size).toBe(seen.length) // no duplicates + expect(seen.length).toBe(EDGE_COUNT) // no skips + expect(new Set(seen)).toEqual(refIds) // exactly the full set + }) + + it('cursor and offset walks return the same set', async () => { + const offsetSeen: string[] = [] + let offset = 0 + for (;;) { + const page = await storage.getVerbs({ pagination: { limit: 9, offset } }) + for (const v of page.items) offsetSeen.push(v.id) + if (!page.hasMore) break + offset += page.items.length + } + + const cursorSeen: string[] = [] + let cursor: string | undefined + for (;;) { + const page = await storage.getVerbs({ + pagination: cursor ? { limit: 9, cursor } : { limit: 9 } + }) + for (const v of page.items) cursorSeen.push(v.id) + if (!page.hasMore) break + cursor = page.nextCursor + } + + expect(cursorSeen.length).toBe(EDGE_COUNT) + expect(new Set(cursorSeen)).toEqual(new Set(offsetSeen)) + }) + + it('a foreign/malformed cursor FAILS LOUDLY — never a silent restart from page 1', async () => { + // The old behavior (decode-null → silent offset-0 fallback) re-served page 1 + // forever to any while(hasMore) walker: an unbounded CPU loop with no log + // line. An undecodable resume token now refuses the walk instead. + await expect( + storage.getVerbs({ pagination: { limit: 5, cursor: 'not-a-cv1-token' } }) + ).rejects.toThrow('invalid pagination cursor') + await expect( + storage.getNouns({ pagination: { limit: 5, cursor: 'not-a-cv1-token' } }) + ).rejects.toThrow('invalid pagination cursor') + }) +}) diff --git a/tests/unit/test-suite-coverage-guard.test.ts b/tests/unit/test-suite-coverage-guard.test.ts new file mode 100644 index 00000000..93db4421 --- /dev/null +++ b/tests/unit/test-suite-coverage-guard.test.ts @@ -0,0 +1,69 @@ +/** + * @module tests/unit/test-suite-coverage-guard + * @description Prevents a test file from silently falling outside EVERY vitest + * config (so it never runs and gives false coverage confidence — the exact drift + * that left ~27 test files un-run before 8.0). Every `*.test.ts` must either match + * a gate config (`tests/unit/**`, `tests/integration/**`, `*.unit.test.ts`, + * `*.integration.test.ts`) or be explicitly listed in MANUAL_ONLY below. + */ +import { describe, it, expect } from 'vitest' +import { readdirSync } from 'node:fs' +import { join, relative, dirname } from 'node:path' +import { fileURLToPath } from 'node:url' + +const repoRoot = join(dirname(fileURLToPath(import.meta.url)), '../..') +const testsDir = join(repoRoot, 'tests') + +function allTestFiles(dir: string, out: string[] = []): string[] { + for (const e of readdirSync(dir, { withFileTypes: true })) { + const p = join(dir, e.name) + if (e.isDirectory()) allTestFiles(p, out) + else if (e.isFile() && p.endsWith('.test.ts')) out.push(relative(repoRoot, p).split('\\').join('/')) + } + return out +} + +/** + * Test files INTENTIONALLY excluded from the unit/integration gate: benchmarks, + * scale/perf measurements, package-size checks, and real-model-load checks. They + * are run manually (slow / need real resources), not in CI. Every entry is a + * conscious decision — a NEW orphan not listed here fails the guard below. + */ +const MANUAL_ONLY = new Set([ + 'tests/api/performance-benchmarks.test.ts', + 'tests/critical-neural-validation.test.ts', + 'tests/critical-performance-benchmark.test.ts', + 'tests/model-loading.test.ts', + 'tests/package-size-breakdown.test.ts', + 'tests/package-size-limit.test.ts', + 'tests/performance/graph-scale-performance.test.ts', + 'tests/performance/triple-intelligence-scale.test.ts', + 'tests/performance/typeAware.bench.test.ts' +]) + +function inGate(rel: string): boolean { + return ( + rel.startsWith('tests/unit/') || + rel.startsWith('tests/integration/') || + rel.endsWith('.unit.test.ts') || + rel.endsWith('.integration.test.ts') + ) +} + +describe('test-suite coverage guard', () => { + it('every *.test.ts runs in a gate config or is explicitly allowlisted as manual', () => { + const orphans = allTestFiles(testsDir).filter((f) => !inGate(f) && !MANUAL_ONLY.has(f)) + expect( + orphans, + 'These test files match NO vitest config and are not in MANUAL_ONLY — rename to ' + + '*.unit.test.ts / *.integration.test.ts (or move under tests/unit|integration), or add to ' + + `MANUAL_ONLY if they are benchmarks:\n${orphans.join('\n')}` + ).toEqual([]) + }) + + it('the manual allowlist has no stale entries (every listed file still exists)', () => { + const all = new Set(allTestFiles(testsDir)) + const stale = [...MANUAL_ONLY].filter((f) => !all.has(f)) + expect(stale, `MANUAL_ONLY lists files that no longer exist — remove them:\n${stale.join('\n')}`).toEqual([]) + }) +}) diff --git a/tests/unit/transaction/graphIndexOperations-generation.test.ts b/tests/unit/transaction/graphIndexOperations-generation.test.ts index 46e7dde2..9dc5c90c 100644 --- a/tests/unit/transaction/graphIndexOperations-generation.test.ts +++ b/tests/unit/transaction/graphIndexOperations-generation.test.ts @@ -55,7 +55,7 @@ describe('Graph index operations — generation threading', () => { it('AddToGraphIndexOperation stamps addVerb (and its rollback removeVerb) at the resolved generation', async () => { const { provider, calls } = makeSpyProvider() const verb = makeVerb('verb-1') - const op = new AddToGraphIndexOperation(provider, verb, 10n, 20n, () => 7n) + const op = new AddToGraphIndexOperation(provider, verb, { sourceInt: 10n, targetInt: 20n }, () => 7n) const rollback = await op.execute() expect(calls).toEqual([{ method: 'addVerb', generation: 7n, verbId: 'verb-1' }]) @@ -67,7 +67,7 @@ describe('Graph index operations — generation threading', () => { it('RemoveFromGraphIndexOperation stamps removeVerb (and its rollback addVerb) at the resolved generation', async () => { const { provider, calls } = makeSpyProvider() const verb = makeVerb('verb-2') - const op = new RemoveFromGraphIndexOperation(provider, verb, 10n, 20n, () => 12n) + const op = new RemoveFromGraphIndexOperation(provider, verb, { sourceInt: 10n, targetInt: 20n }, () => 12n) const rollback = await op.execute() expect(calls).toEqual([{ method: 'removeVerb', generation: 12n, verbId: 'verb-2' }]) @@ -83,7 +83,7 @@ describe('Graph index operations — generation threading', () => { // at its execute-time value. const { provider, calls } = makeSpyProvider() let current = 1n - const op = new AddToGraphIndexOperation(provider, makeVerb('verb-3'), 10n, 20n, () => current) + const op = new AddToGraphIndexOperation(provider, makeVerb('verb-3'), { sourceInt: 10n, targetInt: 20n }, () => current) current = 42n // store assigns the batch generation after the op is built await op.execute() @@ -97,8 +97,7 @@ describe('Graph index operations — generation threading', () => { const op = new AddToGraphIndexOperation( provider, makeVerb('verb-4'), - 10n, - 20n, + { sourceInt: 10n, targetInt: 20n }, () => 5n, (verbInt) => { seen = verbInt diff --git a/tests/unit/transaction/timeout-rollback.test.ts b/tests/unit/transaction/timeout-rollback.test.ts new file mode 100644 index 00000000..c09ff098 --- /dev/null +++ b/tests/unit/transaction/timeout-rollback.test.ts @@ -0,0 +1,177 @@ +/** + * @module tests/unit/transaction/timeout-rollback + * @description Regression for a consumer-reported P0 data-integrity bug: + * `Transaction.execute()` checked its time budget at the TOP of the operation + * loop and threw `TransactionTimeoutError` from OUTSIDE the per-operation + * try/catch — so the timeout bypassed rollback entirely (only per-operation + * failures rolled back). On a bulk transact that tripped the 30s budget + * mid-flight, the operations already applied to canonical storage were left in + * place while the generation was never stamped: torn, generation-less state. + * The generation-store commit path's abort cleanup explicitly ASSUMES a throw + * from `execute()` already restored the applied operations byte-identically + * (it only discards the uncommitted staging directory), so the missing + * rollback broke that invariant. + * + * Fix: `execute()` now has a SINGLE rollback point — any error escaping the + * operation loop (operation failure OR mid-flight timeout) rolls back every + * applied operation in reverse order, then surfaces the original error. + * + * These tests model canonical storage as an in-memory map and give each + * operation a real rollback action that restores the map byte-identically — + * exercising the exact fixed code path and asserting the reported requirement: + * a mid-flight timeout leaves storage byte-identical to its pre-transaction + * state. + */ +import { describe, it, expect } from 'vitest' +import { Transaction } from '../../../src/transaction/Transaction.js' +import type { Operation, RollbackAction } from '../../../src/transaction/types.js' +import { + TransactionTimeoutError, + TransactionExecutionError, + TransactionRollbackError +} from '../../../src/transaction/errors.js' + +/** A tiny stand-in for canonical storage. */ +type Store = Map + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) + +/** + * An operation that writes `value` at `key` in `store` and returns a rollback + * action restoring the key's PRIOR state byte-identically (delete if it was + * absent, else restore the previous value) — exactly how the real graph/metadata + * operations undo themselves. `delayMs` lets an op consume wall-clock so a small + * transaction budget trips on the NEXT loop iteration; `fail` makes it throw + * after doing its write-and-rollback capture (to exercise the shared path via an + * operation failure rather than a timeout). + */ +function writeOp( + store: Store, + key: string, + value: string, + opts: { delayMs?: number; fail?: boolean; name?: string } = {} +): Operation & { executed: boolean } { + const op = { + name: opts.name ?? `write:${key}`, + executed: false, + async execute(): Promise { + op.executed = true + if (opts.delayMs) await sleep(opts.delayMs) + // Operations are individually atomic: a failing op throws having left NO + // net change (it never gets far enough to register an undo), so only the + // PRIOR operations need rolling back. + if (opts.fail) throw new Error(`operation ${key} failed`) + const had = store.has(key) + const prev = store.get(key) + store.set(key, value) + return async () => { + if (had) store.set(key, prev as string) + else store.delete(key) + } + } + } + return op +} + +describe('Transaction — mid-flight timeout rolls back (P0 data integrity)', () => { + it('a mid-flight timeout leaves storage BYTE-IDENTICAL to its pre-transaction state', async () => { + const store: Store = new Map([['seed', 'unchanged']]) + const snapshot = new Map(store) + + // Budget 1ms; op0 sleeps 60ms so the budget is blown when the loop checks + // it before op1 — a deterministic mid-flight timeout at operation index 1. + const op0 = writeOp(store, 'A', 'applied-then-rolled-back', { delayMs: 60 }) + const op1 = writeOp(store, 'B', 'never-applied') + + const tx = new Transaction({ timeout: 1 }) + tx.addOperation(op0) + tx.addOperation(op1) + + await expect(tx.execute()).rejects.toBeInstanceOf(TransactionTimeoutError) + + // op0 ran (and was rolled back); op1 never started. + expect(op0.executed).toBe(true) + expect(op1.executed).toBe(false) + // The whole point: canonical storage is byte-identical — no torn state. + expect(store).toEqual(snapshot) + expect(store.has('A')).toBe(false) + expect(store.has('B')).toBe(false) + // And the transaction ended in the rolled-back terminal state (so the + // manager counts it correctly), not stranded in 'executing'. + expect(tx.getState()).toBe('rolled_back') + }) + + it('an operation failure mid-flight rolls back byte-identically (the shared single-rollback path)', async () => { + const store: Store = new Map([['seed', 'unchanged']]) + const snapshot = new Map(store) + + const op0 = writeOp(store, 'A', 'applied-then-rolled-back') + const op1 = writeOp(store, 'B', 'partially-applied', { fail: true }) + const op2 = writeOp(store, 'C', 'never-applied') + + const tx = new Transaction() + tx.addOperation(op0) + tx.addOperation(op1) + tx.addOperation(op2) + + await expect(tx.execute()).rejects.toBeInstanceOf(TransactionExecutionError) + + expect(op0.executed).toBe(true) + expect(op1.executed).toBe(true) + expect(op2.executed).toBe(false) + // op1's partial write is undone by its own rollback; op0's write is undone; + // op2 never wrote. Byte-identical. + expect(store).toEqual(snapshot) + expect(tx.getState()).toBe('rolled_back') + }) + + it('a rollback failure during a timeout surfaces TransactionRollbackError wrapping the timeout (loud, not silent)', async () => { + const store: Store = new Map() + + // op0 applies, but its rollback throws every time (maxRollbackRetries + // exhausted) — the manager must surface a TransactionRollbackError whose + // originalError is the timeout, never swallow it. + const op0: Operation & { executed: boolean } = { + name: 'unrollbackable', + executed: false, + async execute() { + op0.executed = true + await sleep(60) + store.set('A', 'applied') + return async () => { + throw new Error('rollback is impossible for this op') + } + } + } + const op1 = writeOp(store, 'B', 'never-applied') + + const tx = new Transaction({ timeout: 1, maxRollbackRetries: 1 }) + tx.addOperation(op0) + tx.addOperation(op1) + + let caught: unknown + await tx.execute().catch((e) => (caught = e)) + + expect(caught).toBeInstanceOf(TransactionRollbackError) + expect((caught as TransactionRollbackError).originalError).toBeInstanceOf( + TransactionTimeoutError + ) + expect(op1.executed).toBe(false) + }) + + it('a clean (non-timeout) transaction still commits and applies all writes', async () => { + const store: Store = new Map() + const op0 = writeOp(store, 'A', 'a') + const op1 = writeOp(store, 'B', 'b') + + const tx = new Transaction() + tx.addOperation(op0) + tx.addOperation(op1) + + await tx.execute() + + expect(tx.getState()).toBe('committed') + expect(store.get('A')).toBe('a') + expect(store.get('B')).toBe('b') + }) +}) diff --git a/tests/unit/utils/entity-id-mapper-u32-ceiling.test.ts b/tests/unit/utils/entity-id-mapper-u32-ceiling.test.ts index 0b92f768..2f9e8f9b 100644 --- a/tests/unit/utils/entity-id-mapper-u32-ceiling.test.ts +++ b/tests/unit/utils/entity-id-mapper-u32-ceiling.test.ts @@ -96,3 +96,31 @@ describe('EntityIdMapper U32 IdSpace ceiling (Brainy 8.0)', () => { expect(m.getOrAssign('uuid-existing')).toBe(assigned) }) }) + +describe('EntityIdMapper.entityIntsToUuids (bigint batch resolver — graph engine)', () => { + it('reverse-resolves a BigInt64Array of assigned ints to UUIDs, in order', async () => { + const m = makeMapper() + await m.init() + const a = m.getOrAssign('uuid-a') + const b = m.getOrAssign('uuid-b') + const c = m.getOrAssign('uuid-c') + // Out-of-assignment order, to prove it's positional (not sorted). + const ints = BigInt64Array.from([BigInt(c), BigInt(a), BigInt(b)]) + expect(m.entityIntsToUuids(ints)).toEqual(['uuid-c', 'uuid-a', 'uuid-b']) + }) + + it('yields "" for a never-assigned int (order-preserving, no drop)', async () => { + const m = makeMapper() + await m.init() + const a = m.getOrAssign('uuid-a') + const ints = BigInt64Array.from([BigInt(a), 999999n]) + // Position is preserved — the unknown int does not collapse the array. + expect(m.entityIntsToUuids(ints)).toEqual(['uuid-a', '']) + }) + + it('returns an empty array for empty input', async () => { + const m = makeMapper() + await m.init() + expect(m.entityIntsToUuids(new BigInt64Array(0))).toEqual([]) + }) +}) diff --git a/tests/unit/utils/osLimits.test.ts b/tests/unit/utils/osLimits.test.ts new file mode 100644 index 00000000..a56d593e --- /dev/null +++ b/tests/unit/utils/osLimits.test.ts @@ -0,0 +1,76 @@ +/** + * @module tests/unit/utils/osLimits + * @description OS-limit detection for pool-scale use. Laws: + * (1) the /proc/self/limits parser reads soft/hard NOFILE exactly, including + * 'unlimited'; (2) assessment warns ONLY below the pool floors and NEVER + * on an unreadable (null) limit — no measurement, no claim; (3) the full + * check composes both sources and survives unreadable /proc silently. + */ +import { describe, it, expect } from 'vitest' +import { + parseProcLimits, + assessOsLimits, + checkOsLimits, + NOFILE_POOL_FLOOR, + MAX_MAP_COUNT_POOL_FLOOR +} from '../../../src/utils/osLimits.js' + +const SAMPLE_LIMITS = [ + 'Limit Soft Limit Hard Limit Units', + 'Max cpu time unlimited unlimited seconds', + 'Max open files 1024 1048576 files', + 'Max locked memory 8388608 8388608 bytes' +].join('\n') + +describe('osLimits — detect + warn at pool scale', () => { + it('parses soft/hard NOFILE from /proc/self/limits, including unlimited', () => { + expect(parseProcLimits(SAMPLE_LIMITS)).toEqual({ soft: 1024, hard: 1048576 }) + expect( + parseProcLimits('Max open files unlimited unlimited files') + ).toEqual({ soft: Infinity, hard: Infinity }) + expect(parseProcLimits('no such row here')).toEqual({ soft: null, hard: null }) + }) + + it('warns below the floors, stays quiet at or above them', () => { + const low = assessOsLimits({ nofileSoft: 1024, nofileHard: 1048576, maxMapCount: 65530 }) + expect(low).toHaveLength(2) + expect(low[0]).toContain('RLIMIT_NOFILE soft limit is 1024') + expect(low[0]).toContain(`ulimit -n ${NOFILE_POOL_FLOOR}`) + expect(low[0]).toContain('raise the soft limit only') // hard already allows it + expect(low[1]).toContain('vm.max_map_count is 65530') + expect(low[1]).toContain(`vm.max_map_count=${MAX_MAP_COUNT_POOL_FLOOR}`) + + expect( + assessOsLimits({ + nofileSoft: NOFILE_POOL_FLOOR, + nofileHard: Infinity, + maxMapCount: MAX_MAP_COUNT_POOL_FLOOR + }) + ).toEqual([]) + }) + + it('an unreadable limit makes NO claim — nulls never warn', () => { + expect(assessOsLimits({ nofileSoft: null, nofileHard: null, maxMapCount: null })).toEqual([]) + }) + + it('checkOsLimits composes both sources and survives unreadable /proc silently', async () => { + const report = await checkOsLimits(async (p) => { + if (p === '/proc/self/limits') return SAMPLE_LIMITS + if (p === '/proc/sys/vm/max_map_count') return '65530\n' + throw new Error('unexpected path') + }) + expect(report.nofileSoft).toBe(1024) + expect(report.maxMapCount).toBe(65530) + expect(report.warnings).toHaveLength(2) + + const offLinux = await checkOsLimits(async () => { + throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) + }) + expect(offLinux).toEqual({ + nofileSoft: null, + nofileHard: null, + maxMapCount: null, + warnings: [] + }) + }) +}) diff --git a/tests/unit/utils/paramValidation.test.ts b/tests/unit/utils/paramValidation.test.ts index e7113506..4dc83554 100644 --- a/tests/unit/utils/paramValidation.test.ts +++ b/tests/unit/utils/paramValidation.test.ts @@ -270,27 +270,24 @@ describe('Zero-Config Parameter Validation', () => { expect(config.availableMemory).toBeGreaterThan(0) }) - it('should adapt limits based on query performance', () => { - const initialConfig = getValidationConfig() - const initialLimit = initialConfig.maxLimit - - // Simulate fast queries with large results + it('never mutates the cap from query timing (telemetry only)', () => { + const initialLimit = getValidationConfig().maxLimit + + // Fast queries with large results: no silent growth. for (let i = 0; i < 10; i++) { recordQueryPerformance(50, initialLimit * 0.9) } - - const updatedConfig = getValidationConfig() - // Limit might increase if performance is good - expect(updatedConfig.maxLimit).toBeGreaterThanOrEqual(initialLimit) - - // Simulate slow queries - for (let i = 0; i < 10; i++) { - recordQueryPerformance(2000, 100) + expect(getValidationConfig().maxLimit).toBe(initialLimit) + + // A burst of catastrophically slow queries must not strangle the cap. + // The removed "learning" ratchet shrank it 20% per recorded query down + // to a floor of 1000 — below the documented MIN_AUTO_QUERY_LIMIT — and + // the error message blamed "available free memory" (a production + // incident: every find({ limit: 5000 }) failed on an idle 23GB-free box). + for (let i = 0; i < 50; i++) { + recordQueryPerformance(90_000, 100) } - - const finalConfig = getValidationConfig() - // Limit should decrease if performance is poor - expect(finalConfig.maxLimit).toBeLessThanOrEqual(updatedConfig.maxLimit) + expect(getValidationConfig().maxLimit).toBe(initialLimit) }) }) }) \ No newline at end of file diff --git a/tests/unit/validate-invariants-delegation.test.ts b/tests/unit/validate-invariants-delegation.test.ts new file mode 100644 index 00000000..45e12ccd --- /dev/null +++ b/tests/unit/validate-invariants-delegation.test.ts @@ -0,0 +1,99 @@ +/** + * @module tests/unit/validate-invariants-delegation + * @description validateIndexConsistency() was blind to native + * providers — it only saw the JS metadata index, so a native manifest↔segments↔count + * divergence read as "healthy". It now feature-detects + aggregates each provider's + * validateInvariants(), and repairIndex() maps a failing invariant with heal:'rebuild' + * to that provider's rebuild(). "healthy-while-broken must be impossible." + */ +import { describe, it, expect, beforeEach } from 'vitest' +import { Brainy, NounType } from '../../src/index.js' +import type { ProviderInvariantReport } from '../../src/index.js' + +const healthyReport = (provider: string): ProviderInvariantReport => ({ + provider, + healthy: true, + serving: true, + invariants: [{ name: 'manifest-residency', holds: true, detail: 'ok', heal: 'none' }], + checkedAt: 1, + durationMs: 1 +}) + +const brokenReport = (provider: string): ProviderInvariantReport => ({ + provider, + healthy: false, + serving: true, + invariants: [ + { name: 'manifest-residency', holds: true, detail: 'ok', heal: 'none' }, + { + name: 'posted-count-floor', + holds: false, + detail: 'posted 2304 < canonical 2354', + expected: 2354, + actual: 2304, + heal: 'rebuild' + } + ], + checkedAt: 1, + durationMs: 2 +}) + +describe('validateIndexConsistency delegates to provider validateInvariants() (Pass 3)', () => { + let brain: any + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, dimensions: 384 }) + await brain.init() + await brain.add({ data: 'x', type: NounType.Concept }) + await brain.flush() + }) + + it('a broken provider report makes the store unhealthy and names the failing invariant', async () => { + brain.index.validateInvariants = async () => brokenReport('vector') + const v = await brain.validateIndexConsistency() + expect(v.healthy).toBe(false) + expect(v.recommendation).toMatch(/posted-count-floor/) + expect(v.recommendation).toMatch(/posted 2304 < canonical 2354/) + expect(v.recommendation).toMatch(/repairIndex\(\)/) + expect(v.providers?.some((p: ProviderInvariantReport) => p.provider === 'vector' && !p.healthy)).toBe(true) + delete brain.index.validateInvariants + }) + + it('all-healthy provider reports do not flip the store unhealthy', async () => { + brain.index.validateInvariants = async () => healthyReport('vector') + brain.graphIndex.validateInvariants = async () => healthyReport('graph') + const v = await brain.validateIndexConsistency() + expect(v.healthy).toBe(true) + expect(v.providers?.length).toBe(2) + delete brain.index.validateInvariants + delete brain.graphIndex.validateInvariants + }) + + it('a validateInvariants() that THROWS is surfaced as unhealthy, never swallowed', async () => { + brain.index.validateInvariants = async () => { throw new Error('provider blew up') } + const v = await brain.validateIndexConsistency() + expect(v.healthy).toBe(false) + expect(v.recommendation).toMatch(/validate-invariants-threw/) + delete brain.index.validateInvariants + }) + + it('providers without validateInvariants() are omitted (JS baseline unchanged)', async () => { + const v = await brain.validateIndexConsistency() + expect(v.providers).toBeUndefined() + expect(typeof v.healthy).toBe('boolean') + }) + + it('repairIndex() rebuilds a provider whose failing invariant asks for it', async () => { + let rebuilt = false + brain.index.validateInvariants = async () => (rebuilt ? healthyReport('vector') : brokenReport('vector')) + const origRebuild = brain.index.rebuild.bind(brain.index) + brain.index.rebuild = async (...a: any[]) => { rebuilt = true; return origRebuild(...a) } + await brain.repairIndex() + expect(rebuilt).toBe(true) + // After repair, the store validates healthy again. + const v = await brain.validateIndexConsistency() + expect(v.providers?.find((p: ProviderInvariantReport) => p.provider === 'vector')?.healthy).toBe(true) + brain.index.rebuild = origRebuild + delete brain.index.validateInvariants + }) +}) diff --git a/tests/unit/vector-cold-read-guard.test.ts b/tests/unit/vector-cold-read-guard.test.ts new file mode 100644 index 00000000..49ca6426 --- /dev/null +++ b/tests/unit/vector-cold-read-guard.test.ts @@ -0,0 +1,115 @@ +/** + * @module tests/unit/vector-cold-read-guard + * @description Pattern-A / Finding 1: a pure semantic find({ query }) has no + * filter, so verifyMetadataLive never fires — nothing guarded the vector index. + * A cold native vector index that loaded its COUNT but not its serving structure + * returned a silent []. verifyVectorLive() closes that: honest isReady() first, + * else a known-vector self-match probe; self-heal (rebuild) or throw + * VectorIndexNotReadyError — never a silent empty result. + */ +import { describe, it, expect, beforeEach } from 'vitest' +import { Brainy, NounType, VectorIndexNotReadyError } from '../../src/index.js' + +const V = (): number[] => Array.from({ length: 384 }, (_, i) => Math.sin(i * 0.1) + 0.001) + +describe('Vector cold-read guard (verifyVectorLive) — silent-[] on cold semantic find', () => { + let brain: any + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, dimensions: 384 }) + await brain.init() + await brain.add({ vector: V(), type: NounType.Concept, metadata: { status: 'active' } }) + await brain.add({ vector: V(), type: NounType.Concept, metadata: { status: 'archived' } }) + await brain.flush() + }) + + it('warm brain: semantic find is correct and the guard does not rebuild', async () => { + const vi = brain.index + let rebuilds = 0 + const origRebuild = vi.rebuild.bind(vi) + vi.rebuild = async (...a: any[]) => { rebuilds++; return origRebuild(...a) } + await brain.find({ query: 'anything', searchMode: 'semantic', limit: 100 }) + expect(rebuilds).toBe(0) + expect(brain._vectorVerified).toBe(true) + vi.rebuild = origRebuild + }) + + it('cold index: verifyVectorLive self-heals via rebuild — semantic find is correct, NOT silent []', async () => { + const vi = brain.index + const origSearch = vi.search.bind(vi) + const origRebuild = vi.rebuild.bind(vi) + let cold = true + brain._vectorVerified = false + // size()>0 (count present) but search returns nothing until a rebuild warms it. + vi.search = async (...a: any[]) => (cold ? [] : origSearch(...a)) + vi.rebuild = async (...a: any[]) => { await origRebuild(...a); cold = false } + try { + const res = await brain.find({ query: 'x', searchMode: 'semantic', limit: 100 }) + expect(res.length).toBeGreaterThan(0) // self-healed + } finally { + vi.search = origSearch; vi.rebuild = origRebuild + } + }) + + it('unrecoverably cold index: semantic find throws VectorIndexNotReadyError', async () => { + const vi = brain.index + const origSearch = vi.search.bind(vi) + const origRebuild = vi.rebuild.bind(vi) + brain._vectorVerified = false + vi.search = async () => [] // always cold; rebuild can't fix it + vi.rebuild = async () => {} + try { + await expect( + brain.find({ query: 'x', searchMode: 'semantic', limit: 100 }) + ).rejects.toBeInstanceOf(VectorIndexNotReadyError) + } finally { + vi.search = origSearch; vi.rebuild = origRebuild + } + }) + + it('native provider reporting isReady()===false rebuilds, then serves', async () => { + const vi = brain.index + const origRebuild = vi.rebuild.bind(vi) + let ready = false + brain._vectorVerified = false + vi.isReady = () => ready + vi.rebuild = async (...a: any[]) => { await origRebuild(...a); ready = true } + try { + const res = await brain.find({ query: 'x', searchMode: 'semantic', limit: 100 }) + expect(ready).toBe(true) // rebuild ran because isReady() was false + expect(res).toBeDefined() + } finally { + delete vi.isReady; vi.rebuild = origRebuild + } + }) + + it('a text-only query does not trigger the vector guard', async () => { + brain._vectorVerified = false + await brain.find({ query: 'active', searchMode: 'text', limit: 5 }) + expect(brain._vectorVerified).toBe(false) // executeVectorSearch never called + }) + + // Regression: the probe must check "returns ANY hit", not an exact self-match — + // HNSW is approximate and get() may re-hydrate the vector, so a healthy + // many-entity index would false-positive under an exact-self check, wrongly + // rebuild, and throw VectorIndexNotReadyError on working data. + it('a healthy many-entity brain with distinct vectors serves semantic find, never throws/rebuilds', async () => { + const many = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, dimensions: 384 }) + await many.init() + for (let i = 0; i < 25; i++) { + const v = Array.from({ length: 384 }, (_, j) => Math.sin((i * 7 + j) * 0.13) + 0.001) + await many.add({ vector: v, type: NounType.Concept, metadata: { n: i } }) + } + await many.flush() + let rebuilds = 0 + const vi = (many as any).index + const origRebuild = vi.rebuild.bind(vi) + vi.rebuild = async (...a: any[]) => { rebuilds++; return origRebuild(...a) } + const res = await many.find({ query: 'x', searchMode: 'semantic', limit: 10 }) + expect(res.length).toBeGreaterThan(0) + expect(rebuilds).toBe(0) + expect((many as any)._vectorVerified).toBe(true) + vi.rebuild = origRebuild + await many.close() + }) +}) diff --git a/tests/unit/where-operator-validation.test.ts b/tests/unit/where-operator-validation.test.ts new file mode 100644 index 00000000..34dc1d7e --- /dev/null +++ b/tests/unit/where-operator-validation.test.ts @@ -0,0 +1,82 @@ +/** + * Where-clause operator correctness (8.0.12) — the three query-layer fixes: + * + * 1. Unknown operators fail LOUD. A typo like `{ role: { notIn: [...] } }` or + * any gibberish operator now throws BrainyError('INVALID_QUERY') instead of + * silently matching nothing (the invisible-degrade class). Validated up + * front in find(), so it throws even when the result set is empty. + * 2. Documented operators the in-memory matcher was missing now work: `in` + * (alias for oneOf) and the long-form comparison aliases + * `greaterThanOrEqual` / `lessThanOrEqual`. + * 3. A user metadata field named `level` is indexable + queryable — it was + * silently dropped by an over-broad HNSW-internal skip-list, so + * `where: { level: ... }` returned nothing. + */ +import { describe, it, expect, afterEach } from 'vitest' +import { Brainy, NounType, BrainyError } from '../../src/index.js' + +const brains: any[] = [] +async function makeBrain(): Promise { + const brain: any = new Brainy({ storage: { type: 'memory' }, requireSubtype: false, silent: true, plugins: [] }) + await brain.init() + brains.push(brain) + return brain +} +afterEach(async () => { + for (const b of brains.splice(0)) await b.close().catch(() => {}) +}) + +const V = () => Array.from({ length: 384 }, () => Math.random()) + +describe('Where-operator validation + documented-operator coverage (8.0.12)', () => { + it('unknown operators throw INVALID_QUERY instead of silently returning [] (item 3)', async () => { + const brain = await makeBrain() + for (const role of ['staff', 'customer', 'vendor']) { + await brain.add({ data: role, type: NounType.Person, metadata: { role }, vector: V() }) + } + // notIn is not an operator (use ne / not+in). It must THROW, not match nothing. + await expect(brain.find({ type: NounType.Person, where: { role: { notIn: ['staff'] } } })) + .rejects.toBeInstanceOf(BrainyError) + await expect(brain.find({ type: NounType.Person, where: { role: { notIn: ['staff'] } } })) + .rejects.toMatchObject({ type: 'INVALID_QUERY' }) + // Gibberish likewise. + await expect(brain.find({ where: { role: { blorp: 1 } } })).rejects.toMatchObject({ type: 'INVALID_QUERY' }) + // Nested inside a logical operator is still validated. + await expect(brain.find({ where: { not: { role: { nope: 1 } } } })).rejects.toMatchObject({ type: 'INVALID_QUERY' }) + await expect(brain.find({ where: { anyOf: [{ role: { xx: 1 } }] } })).rejects.toMatchObject({ type: 'INVALID_QUERY' }) + }) + + it('the documented `in` alias and long-form comparison aliases work (item 2)', async () => { + const brain = await makeBrain() + for (let i = 0; i < 9; i++) { + await brain.add({ data: 'n' + i, type: NounType.Person, metadata: { role: ['staff', 'customer', 'vendor'][i % 3], score: i }, vector: V() }) + } + await brain.flush() + // `in` === oneOf + expect((await brain.find({ type: NounType.Person, where: { role: { in: ['customer', 'vendor'] } } })).length).toBe(6) + expect((await brain.find({ type: NounType.Person, where: { not: { role: { in: ['staff'] } } } })).length).toBe(6) + // long-form comparison aliases (were silently unhandled by the in-memory matcher) + expect((await brain.find({ type: NounType.Person, where: { score: { greaterThanOrEqual: 6 } } })).length).toBe(3) // 6,7,8 + expect((await brain.find({ type: NounType.Person, where: { score: { lessThanOrEqual: 2 } } })).length).toBe(3) // 0,1,2 + }) + + it('a user metadata field named "level" is indexable + queryable (item: HNSW field-name collision)', async () => { + const brain = await makeBrain() + for (let i = 0; i < 9; i++) { + await brain.add({ data: 'entity ' + i, type: NounType.Person, metadata: { level: i }, vector: V() }) + } + await brain.flush() + expect((await brain.find({ type: NounType.Person, where: { level: 4 } })).length).toBe(1) // exact + expect((await brain.find({ type: NounType.Person, where: { level: { gt: 4 } } })).length).toBe(4) // 5..8 + expect((await brain.find({ type: NounType.Person, where: { level: { exists: true } } })).length).toBe(9) + }) + + it('valid operators are unaffected — control', async () => { + const brain = await makeBrain() + for (const s of ['a', 'b', 'c']) await brain.add({ data: s, type: NounType.Concept, metadata: { tag: s }, vector: V() }) + await brain.flush() + expect((await brain.find({ type: NounType.Concept, where: { tag: { oneOf: ['a', 'b'] } } })).length).toBe(2) + expect((await brain.find({ type: NounType.Concept, where: { tag: 'c' } })).length).toBe(1) + expect((await brain.find({ type: NounType.Concept, where: { tag: { exists: true } } })).length).toBe(3) + }) +}) diff --git a/tests/vfs/tree-operations.test.ts b/tests/vfs/tree-operations.unit.test.ts similarity index 100% rename from tests/vfs/tree-operations.test.ts rename to tests/vfs/tree-operations.unit.test.ts diff --git a/tests/vfs/vfs-bulkwrite-race.test.ts b/tests/vfs/vfs-bulkwrite-race.unit.test.ts similarity index 100% rename from tests/vfs/vfs-bulkwrite-race.test.ts rename to tests/vfs/vfs-bulkwrite-race.unit.test.ts diff --git a/tests/vfs/vfs-initialization.unit.test.ts b/tests/vfs/vfs-initialization.unit.test.ts index 944e2200..e4c8d3e3 100644 --- a/tests/vfs/vfs-initialization.unit.test.ts +++ b/tests/vfs/vfs-initialization.unit.test.ts @@ -5,7 +5,7 @@ */ import { describe, it, expect } from 'vitest' -import { VirtualFileSystem } from '../../src/vfs/index.js' +import { VirtualFileSystem } from '../../src/vfs/VirtualFileSystem.js' import { Brainy } from '../../src/brainy.js' describe('VFS Initialization', () => { diff --git a/tests/vfs/vfs-relationships.test.ts b/tests/vfs/vfs-relationships.test.ts deleted file mode 100644 index c466cfcb..00000000 --- a/tests/vfs/vfs-relationships.test.ts +++ /dev/null @@ -1,344 +0,0 @@ -/** - * Test VFS Graph Relationships - * - * Verifies that VFS properly uses Brainy's graph relationships - * instead of metadata-based path queries - */ - -import { describe, it, expect, beforeEach } from 'vitest' -import { VirtualFileSystem } from '../../src/vfs/VirtualFileSystem.js' -import { Brainy } from '../../src/brainy.js' -import { VerbType } from '../../src/types/graphTypes.js' - -describe('VFS Graph Relationships', () => { - let vfs: VirtualFileSystem - let brain: Brainy - - beforeEach(async () => { - brain = new Brainy({ requireSubtype: false }) - await brain.init() - vfs = new VirtualFileSystem(brain) - await vfs.init() - }) - - it('should use proper graph relationships for directory structure', async () => { - // Create a directory structure - await vfs.mkdir('/projects') - await vfs.mkdir('/projects/brainy') - await vfs.writeFile('/projects/brainy/README.md', 'Test content') - await vfs.writeFile('/projects/brainy/package.json', '{}') - - // Get the entity IDs - const projectsId = await vfs.resolvePath('/projects') - const brainyId = await vfs.resolvePath('/projects/brainy') - const readmeId = await vfs.resolvePath('/projects/brainy/README.md') - - // Verify relationships are created properly - const projectRelations = await brain.related({ - from: projectsId, - type: VerbType.Contains - }) - - expect(projectRelations).toHaveLength(1) - expect(projectRelations[0].to).toBe(brainyId) - - // Verify brainy directory contains its files - const brainyRelations = await brain.related({ - from: brainyId, - type: VerbType.Contains - }) - - expect(brainyRelations).toHaveLength(2) - const childIds = brainyRelations.map(r => r.to) - expect(childIds).toContain(readmeId) - }) - - it('should traverse directory tree using relationships', async () => { - // Create nested structure - await vfs.mkdir('/a') - await vfs.mkdir('/a/b') - await vfs.mkdir('/a/b/c') - await vfs.writeFile('/a/b/c/file.txt', 'deep file') - - // Read directory using relationships - const contents = await vfs.readdir('/a/b/c') - expect(contents).toContain('file.txt') - - // Verify path resolution uses graph traversal - const fileId = await vfs.resolvePath('/a/b/c/file.txt') - const fileEntity = await brain.get(fileId) - expect(fileEntity?.metadata?.name).toBe('file.txt') - }) - - it('should properly handle custom relationships between files', async () => { - // Create two files - await vfs.writeFile('/doc1.md', 'Document 1') - await vfs.writeFile('/doc2.md', 'Document 2') - - // Add a custom relationship - await vfs.addRelationship('/doc1.md', '/doc2.md', VerbType.References) - - // Get relationships using proper graph API - const relationships = await vfs.getRelationships('/doc1.md') - - // Should find the reference relationship - const refRelation = relationships.find(r => r.type === VerbType.References) - expect(refRelation).toBeDefined() - - // Verify it's using actual graph relationships, not metadata - const doc1Id = await vfs.resolvePath('/doc1.md') - const doc2Id = await vfs.resolvePath('/doc2.md') - - const directRelations = await brain.related({ - from: doc1Id, - type: VerbType.References - }) - - expect(directRelations).toHaveLength(1) - expect(directRelations[0].to).toBe(doc2Id) - }) - - it('should not fall back to metadata path queries', async () => { - // Create a file - await vfs.writeFile('/test.txt', 'test') - - // Get the entity - const entity = await vfs.getEntity('/test.txt') - - // Verify the entity has proper metadata - expect(entity.metadata.path).toBe('/test.txt') - expect(entity.metadata.name).toBe('test.txt') - - // But the parent relationship should be through graph, not metadata - const rootId = await vfs.resolvePath('/') - const testId = entity.id - - // Check that root contains test.txt via relationships - const rootRelations = await brain.related({ - from: rootId, - type: VerbType.Contains - }) - - expect(rootRelations.some(r => r.to === testId)).toBe(true) - }) - - it('should efficiently query children using relationships', async () => { - // Create many files in a directory - await vfs.mkdir('/many') - for (let i = 0; i < 10; i++) { - await vfs.writeFile(`/many/file${i}.txt`, `content ${i}`) - } - - // Get directory contents - const files = await vfs.readdir('/many') - expect(files).toHaveLength(10) - - // Verify it's using relationships, not metadata queries - const manyId = await vfs.resolvePath('/many') - const relations = await brain.related({ - from: manyId, - type: VerbType.Contains - }) - - expect(relations).toHaveLength(10) - }) - - it('should handle moving files by updating relationships', async () => { - // Create source structure - await vfs.mkdir('/source') - await vfs.writeFile('/source/file.txt', 'content') - await vfs.mkdir('/dest') - - // Move file - await vfs.move('/source/file.txt', '/dest/file.txt') - - // Verify relationships are updated - const sourceId = await vfs.resolvePath('/source') - const destId = await vfs.resolvePath('/dest') - const fileId = await vfs.resolvePath('/dest/file.txt') - - // Source should not contain file anymore - const sourceRelations = await brain.related({ - from: sourceId, - type: VerbType.Contains - }) - expect(sourceRelations).toHaveLength(0) - - // Dest should contain file - const destRelations = await brain.related({ - from: destId, - type: VerbType.Contains - }) - expect(destRelations).toHaveLength(1) - expect(destRelations[0].to).toBe(fileId) - }) - - it('should support complex graph queries', async () => { - // Create interconnected structure - await vfs.mkdir('/docs') - await vfs.writeFile('/docs/main.md', 'Main doc') - await vfs.writeFile('/docs/related1.md', 'Related 1') - await vfs.writeFile('/docs/related2.md', 'Related 2') - - // Add cross-references - await vfs.addRelationship('/docs/main.md', '/docs/related1.md', VerbType.References) - await vfs.addRelationship('/docs/main.md', '/docs/related2.md', VerbType.References) - await vfs.addRelationship('/docs/related1.md', '/docs/related2.md', VerbType.References) - - // Get all related documents - const related = await vfs.getRelated('/docs/main.md') - - // Should find parent (Contains) and references - const references = related.filter(r => r.direction === 'from') - expect(references.length).toBeGreaterThanOrEqual(2) - }) - - it('should always create Contains relationship when writing files', async () => { - // This test verifies the fix for the critical bug where writeFile() - // was not creating Contains relationships for updated files - - // Test 1: New file should have Contains relationship - await vfs.writeFile('/test-new.txt', 'Hello World') - - const rootId = await vfs.resolvePath('/') - const fileEntityId = await vfs.resolvePath('/test-new.txt') - - // Check that Contains relationship exists - const relations = await brain.related({ - from: rootId, - to: fileEntityId, - type: VerbType.Contains - }) - - expect(relations).toHaveLength(1) - expect(relations[0].type).toBe(VerbType.Contains) - - // Verify readdir returns the file - const files = await vfs.readdir('/') - expect(files).toContain('test-new.txt') - - // Test 2: Updated file should maintain Contains relationship - await vfs.writeFile('/test-new.txt', 'Updated content') - - // Relationship should still exist after update - const relationsAfterUpdate = await brain.related({ - from: rootId, - to: fileEntityId, - type: VerbType.Contains - }) - - expect(relationsAfterUpdate).toHaveLength(1) - - // readdir should still work - const filesAfterUpdate = await vfs.readdir('/') - expect(filesAfterUpdate).toContain('test-new.txt') - - // Test 3: Multiple updates should not create duplicate relationships - await vfs.writeFile('/test-new.txt', 'Another update') - await vfs.writeFile('/test-new.txt', 'Yet another update') - - const finalRelations = await brain.related({ - from: rootId, - to: fileEntityId, - type: VerbType.Contains - }) - - // Should still have exactly one Contains relationship - expect(finalRelations).toHaveLength(1) - }) - - it('should repair missing Contains relationships on file update', async () => { - // This test simulates a scenario where a file exists but its Contains - // relationship is missing (could happen due to corruption or bugs) - - // Create a file normally first - await vfs.writeFile('/orphan-test.txt', 'Initial content') - - const rootId = await vfs.resolvePath('/') - const fileEntityId = await vfs.resolvePath('/orphan-test.txt') - - // Manually delete the Contains relationship to simulate the bug - const initialRelations = await brain.related({ - from: rootId, - to: fileEntityId, - type: VerbType.Contains - }) - - // Delete the relationship (simulating the corruption/bug) - for (const rel of initialRelations) { - await brain.unrelate(rel.id) - } - - // Verify the relationship is gone - const brokenRelations = await brain.related({ - from: rootId, - to: fileEntityId, - type: VerbType.Contains - }) - expect(brokenRelations).toHaveLength(0) - - // readdir should fail to find the file (the bug symptom) - const brokenList = await vfs.readdir('/') - expect(brokenList).not.toContain('orphan-test.txt') - - // Now update the file - this should repair the missing relationship - await vfs.writeFile('/orphan-test.txt', 'Fixed content') - - // Verify the relationship is restored - const fixedRelations = await brain.related({ - from: rootId, - to: fileEntityId, - type: VerbType.Contains - }) - - expect(fixedRelations).toHaveLength(1) - expect(fixedRelations[0].type).toBe(VerbType.Contains) - - // readdir should now work again - const fixedList = await vfs.readdir('/') - expect(fixedList).toContain('orphan-test.txt') - }) - - it('should create Contains relationships for files in nested directories', async () => { - // Test that the fix works for nested directory structures - - await vfs.mkdir('/level1') - await vfs.mkdir('/level1/level2') - await vfs.mkdir('/level1/level2/level3') - - // Write a file deep in the structure - await vfs.writeFile('/level1/level2/level3/deep.txt', 'Deep content') - - // Get entity IDs - const level3Id = await vfs.resolvePath('/level1/level2/level3') - const fileEntityId = await vfs.resolvePath('/level1/level2/level3/deep.txt') - - // Verify Contains relationship exists - const relations = await brain.related({ - from: level3Id, - to: fileEntityId, - type: VerbType.Contains - }) - - expect(relations).toHaveLength(1) - - // Verify readdir works at the deep level - const files = await vfs.readdir('/level1/level2/level3') - expect(files).toContain('deep.txt') - - // Update the file and verify relationship persists - await vfs.writeFile('/level1/level2/level3/deep.txt', 'Updated deep content') - - const relationsAfterUpdate = await brain.related({ - from: level3Id, - to: fileEntityId, - type: VerbType.Contains - }) - - expect(relationsAfterUpdate).toHaveLength(1) - - // readdir should still work - const filesAfterUpdate = await vfs.readdir('/level1/level2/level3') - expect(filesAfterUpdate).toContain('deep.txt') - }) -}) \ No newline at end of file diff --git a/tests/vfs/vfs.unit.test.ts b/tests/vfs/vfs.unit.test.ts index c2f49c9f..5ea79377 100644 --- a/tests/vfs/vfs.unit.test.ts +++ b/tests/vfs/vfs.unit.test.ts @@ -6,7 +6,7 @@ */ import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { VirtualFileSystem } from '../../src/vfs/index.js' +import { VirtualFileSystem } from '../../src/vfs/VirtualFileSystem.js' import { Brainy } from '../../src/brainy.js' import { VFSErrorCode } from '../../src/vfs/types.js' @@ -396,19 +396,19 @@ describe('VirtualFileSystem - Production Tests', () => { await vfs.mkdir('/cached') await vfs.writeFile(path, 'cached content') - // First read (cold cache) - const start1 = Date.now() - await vfs.readFile(path) - const time1 = Date.now() - start1 + // Prime the path cache + verify the content round-trips. + expect((await vfs.readFile(path)).toString()).toBe('cached content') - // Second read (warm cache) - const start2 = Date.now() - await vfs.readFile(path) - const time2 = Date.now() - start2 + // Cached repeated access is sub-millisecond. Average many reads so the result + // doesn't hinge on Date.now()'s millisecond rounding of a single sub-ms op — the + // old cold-vs-warm compare spuriously failed when both rounded to 0/1ms. A + // generous absolute bound still catches a regression to per-read disk/index work. + const ITERATIONS = 100 + const start = performance.now() + for (let i = 0; i < ITERATIONS; i++) await vfs.readFile(path) + const avgWarmMs = (performance.now() - start) / ITERATIONS - // Cache should make second read faster - expect(time2).toBeLessThanOrEqual(time1) - console.log(`Cold read: ${time1}ms, Warm read: ${time2}ms`) + expect(avgWarmMs).toBeLessThan(5) }) }) diff --git a/tsconfig.json b/tsconfig.json index 4c8c81f6..b1293663 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "target": "ES2020", + "target": "ES2023", "module": "NodeNext", "moduleResolution": "NodeNext", "esModuleInterop": true, @@ -10,16 +10,15 @@ "outDir": "./dist", "rootDir": "./src", "lib": [ - "DOM", - "ESNext", - "DOM.Asynciterable" + "ES2023" ], "skipLibCheck": true, "forceConsistentCasingInFileNames": true, "noEmit": false, "preserveConstEnums": true, "sourceMap": true, - "downlevelIteration": true + "isolatedModules": true, + "noImplicitOverride": true }, "include": [ "src/**/*"