feat: promote the 8.0 u64-id line to main for the 8.0.0 GA
Brings the u64-id core to main as a merge of feat/8.0-u64-ids. main's tree now matches the gated 8.0 line exactly; the 7.31.8-7.33.5 hotfix history is retained as the merge's first parent (every one of those fixes is already in the 8.0 line).
This commit is contained in:
commit
55d57f89f7
594 changed files with 39814 additions and 149709 deletions
|
|
@ -11,15 +11,12 @@
|
||||||
- **BaseStorage** (`src/storage/baseStorage.ts`): Base implementation with built-in type-aware partitioning (TypeAwareStorageAdapter was removed -- functionality merged into BaseStorage).
|
- **BaseStorage** (`src/storage/baseStorage.ts`): Base implementation with built-in type-aware partitioning (TypeAwareStorageAdapter was removed -- functionality merged into BaseStorage).
|
||||||
- **Adapters** (`src/storage/adapters/`):
|
- **Adapters** (`src/storage/adapters/`):
|
||||||
- `fileSystemStorage.ts` -- local filesystem
|
- `fileSystemStorage.ts` -- local filesystem
|
||||||
- `opfsStorage.ts` -- browser Origin Private File System (formerly browserStorage.ts)
|
|
||||||
- `memoryStorage.ts` -- in-memory
|
- `memoryStorage.ts` -- in-memory
|
||||||
- `s3CompatibleStorage.ts` -- generic S3-compatible (AWS, MinIO, etc.)
|
- `baseStorageAdapter.ts` -- shared adapter base (counts, batch ops)
|
||||||
- `r2Storage.ts` -- Cloudflare R2
|
- Cloud + OPFS adapters were removed in 8.0 (cloud backup is operator tooling)
|
||||||
- `gcsStorage.ts` -- Google Cloud Storage
|
- **Generational MVCC / Db API** (`src/db/`): immutable `Db` values over generation-stamped records
|
||||||
- `azureBlobStorage.ts` -- Azure Blob Storage
|
- `db.ts` (the `Db` value), `generationStore.ts` (record layer + commit protocol), `types.ts`, `errors.ts`, `whereMatcher.ts`
|
||||||
- Supporting: `batchS3Operations.ts`, `optimizedS3Search.ts`
|
- Design record: `docs/ADR-001-generational-mvcc.md`; replaced the pre-8.0 COW branching + versioning subsystems
|
||||||
- **COW** (`src/storage/cow/`): Copy-on-Write infrastructure for versioning and branching
|
|
||||||
- CommitLog, CommitObject, CommitBuilder, BlobStorage, RefManager, TreeObject
|
|
||||||
|
|
||||||
### Vector Search (`src/hnsw/`)
|
### Vector Search (`src/hnsw/`)
|
||||||
- `hnswIndex.ts` -- HNSW-based approximate nearest neighbor search
|
- `hnswIndex.ts` -- HNSW-based approximate nearest neighbor search
|
||||||
|
|
|
||||||
40
.github/workflows/ci.yml
vendored
Normal file
40
.github/workflows/ci.yml
vendored
Normal file
|
|
@ -0,0 +1,40 @@
|
||||||
|
name: CI
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
pull_request:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
node:
|
||||||
|
name: Node ${{ matrix.node-version }}
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
node-version: ['22', '24']
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: ${{ matrix.node-version }}
|
||||||
|
cache: npm
|
||||||
|
- run: npm ci
|
||||||
|
- run: npm run test:unit
|
||||||
|
|
||||||
|
bun:
|
||||||
|
name: Bun (latest)
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: '22'
|
||||||
|
cache: npm
|
||||||
|
- uses: oven-sh/setup-bun@v2
|
||||||
|
with:
|
||||||
|
bun-version: latest
|
||||||
|
- run: npm ci
|
||||||
|
# test:bun imports the built dist/, so build first.
|
||||||
|
- run: npm run build
|
||||||
|
# Bun as a runtime is the supported Bun story (`bun add` / `bun run`).
|
||||||
|
- run: npm run test:bun
|
||||||
9
.gitignore
vendored
9
.gitignore
vendored
|
|
@ -93,9 +93,14 @@ docs/internal/
|
||||||
# Rust/Cargo build artifacts
|
# Rust/Cargo build artifacts
|
||||||
src/embeddings/candle-wasm/target/
|
src/embeddings/candle-wasm/target/
|
||||||
src/embeddings/candle-wasm/Cargo.lock
|
src/embeddings/candle-wasm/Cargo.lock
|
||||||
src/embeddings/wasm/pkg/
|
|
||||||
|
|
||||||
# But keep the pre-built WASM (committed for users without Rust)
|
# Ignore the wasm-pack output dir's CONTENTS (note the `/*`, not `/`, so the
|
||||||
|
# re-includes below can take effect — git cannot re-include a file whose parent
|
||||||
|
# DIRECTORY is excluded). Keep the pre-built WASM committed: it ships in the npm
|
||||||
|
# package anyway, it lets consumers + CI build without a Rust/wasm-pack toolchain,
|
||||||
|
# and versioning it makes the shipped artifact reproducible (not "whatever the
|
||||||
|
# maintainer last built").
|
||||||
|
src/embeddings/wasm/pkg/*
|
||||||
!src/embeddings/wasm/pkg/*.wasm
|
!src/embeddings/wasm/pkg/*.wasm
|
||||||
!src/embeddings/wasm/pkg/*.js
|
!src/embeddings/wasm/pkg/*.js
|
||||||
!src/embeddings/wasm/pkg/*.d.ts
|
!src/embeddings/wasm/pkg/*.d.ts
|
||||||
|
|
|
||||||
|
|
@ -1,24 +0,0 @@
|
||||||
{
|
|
||||||
"types": [
|
|
||||||
{"type": "feat", "section": "✨ Features"},
|
|
||||||
{"type": "fix", "section": "🐛 Bug Fixes"},
|
|
||||||
{"type": "docs", "section": "📚 Documentation"},
|
|
||||||
{"type": "refactor", "section": "♻️ Code Refactoring"},
|
|
||||||
{"type": "perf", "section": "⚡ Performance Improvements"},
|
|
||||||
{"type": "test", "section": "✅ Tests"},
|
|
||||||
{"type": "build", "section": "🔧 Build System"},
|
|
||||||
{"type": "ci", "section": "🔄 CI/CD"},
|
|
||||||
{"type": "style", "hidden": true},
|
|
||||||
{"type": "chore", "hidden": true}
|
|
||||||
],
|
|
||||||
"compareUrlFormat": "https://github.com/soulcraftlabs/brainy/compare/{{previousTag}}...{{currentTag}}",
|
|
||||||
"commitUrlFormat": "https://github.com/soulcraftlabs/brainy/commit/{{hash}}",
|
|
||||||
"issueUrlFormat": "https://github.com/soulcraftlabs/brainy/issues/{{id}}",
|
|
||||||
"userUrlFormat": "https://github.com/{{user}}",
|
|
||||||
"releaseCommitMessageFormat": "chore(release): {{currentTag}}",
|
|
||||||
"issuePrefixes": ["#"],
|
|
||||||
"header": "# Changelog\n\nAll notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.\n",
|
|
||||||
"scripts": {
|
|
||||||
"postbump": "echo '✅ Version bumped to' $(node -p \"require('./package.json').version\")"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
188
CHANGELOG.md
188
CHANGELOG.md
|
|
@ -2,79 +2,169 @@
|
||||||
|
|
||||||
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
||||||
|
|
||||||
### [7.33.5](https://github.com/soulcraftlabs/brainy/compare/v7.33.4...v7.33.5) (2026-07-02)
|
### [8.0.0-rc.9](https://github.com/soulcraftlabs/brainy/compare/v8.0.0-rc.8...v8.0.0-rc.9) (2026-07-01)
|
||||||
|
|
||||||
- fix: metadata cold-read guard — no more silent [] on cold find({where}) (7.33.5) (9dc4c5e)
|
- docs(8.0): RELEASES.md — rc.9 (migration LOCK + 6x cosine + ES2023/Node22 floor) (3a33987)
|
||||||
|
- chore(8.0): ES2023 target + drop DOM lib + downlevelIteration (config truth-up) (cf74c25)
|
||||||
|
- perf(8.0): allocation-free distance loops (6x cosine) — evidence-revised Fork X (b5bc73f)
|
||||||
|
- feat(8.0): #18 coordinated migration LOCK — block-and-queue the 7.x→8.0 auto-upgrade (67bbf69)
|
||||||
|
- chore(8.0): modernize toolchain + position Bun as a runtime (ca9129a)
|
||||||
|
|
||||||
|
|
||||||
### [7.33.4](https://github.com/soulcraftlabs/brainy/compare/v7.33.3...v7.33.4) (2026-06-29)
|
### [8.0.0-rc.8](https://github.com/soulcraftlabs/brainy/compare/v8.0.0-rc.7...v8.0.0-rc.8) (2026-06-30)
|
||||||
|
|
||||||
- fix: never serve a silent [] from find({connected}) on a cold-loaded graph (fd699d0)
|
- docs(8.0): RELEASES.md — rc.8 (no-freeze online whole-brain auto-upgrade) (5af48a9)
|
||||||
|
- feat(8.0): no-freeze auto-upgrade hooks — isMigrating() deference + stampBrainFormat() + brain-format export (b6b9198)
|
||||||
|
|
||||||
|
|
||||||
### [7.33.3](https://github.com/soulcraftlabs/brainy/compare/v7.33.2...v7.33.3) (2026-06-24)
|
### [8.0.0-rc.7](https://github.com/soulcraftlabs/brainy/compare/v8.0.0-rc.6...v8.0.0-rc.7) (2026-06-30)
|
||||||
|
|
||||||
- fix: re-validate find() results against the predicate (index-integrity guard) (7b5db0d)
|
- docs(8.0): RELEASES.md — rc.7 (cold-graph self-heal + billion-scale RAM + version handshake) (1ddc786)
|
||||||
|
- perf(8.0): bound per-id generation history chains (O(W+L) resident, was O(N)) (a859d6e)
|
||||||
|
- feat(8.0): eager graphIndex.init() before the isReady() rebuild gate (8f4787b)
|
||||||
|
- feat(8.0): version-handshake marker (formatInfo + indexEpoch) for whole-brain auto-upgrade (fc7f110)
|
||||||
|
- fix(8.0): never serve a silent [] from find({connected}) on a cold-loaded graph (229b067)
|
||||||
|
- perf(8.0): represent the committed-generation ledger as an interval set (93f61db)
|
||||||
|
- perf(8.0): drop O(N)-resident id-keyed storage caches; source counts from the record (b6beb7f)
|
||||||
|
|
||||||
|
|
||||||
### [7.33.2](https://github.com/soulcraftlabs/brainy/compare/v7.33.1...v7.33.2) (2026-06-24)
|
### [8.0.0-rc.6](https://github.com/soulcraftlabs/brainy/compare/v8.0.0-rc.5...v8.0.0-rc.6) (2026-06-29)
|
||||||
|
|
||||||
- fix: graph adjacency cold-load consistency guard — no more silent [] on connected (1694f68)
|
- docs(8.0): RELEASES.md — rc.6 (perf + native-provider contract + test hygiene) (6daa70e)
|
||||||
|
- feat(8.0): wire the two cor-confirmed metadata-provider contract additions (8b19122)
|
||||||
|
- test(8.0): re-home orphaned test files into the gate + guard against recurrence (3f9f140)
|
||||||
|
- perf(8.0): negation/absence where-operators via roaring-bitmap difference (5f974ab)
|
||||||
|
- perf(8.0): HNSW removeItem is O(in-degree) via a reverse-adjacency index (72df557)
|
||||||
|
|
||||||
|
|
||||||
### [7.33.1](https://github.com/soulcraftlabs/brainy/compare/v7.33.0...v7.33.1) (2026-06-22)
|
### [8.0.0-rc.5](https://github.com/soulcraftlabs/brainy/compare/v8.0.0-rc.4...v8.0.0-rc.5) (2026-06-29)
|
||||||
|
|
||||||
- fix: getNouns cursor pagination re-scanned the first page forever (permanent CPU loop) (6721c52)
|
- docs(8.0): RELEASES.md — rc.5 hardening + the breaking operator removal (6c9a438)
|
||||||
|
- refactor(8.0): remove the 4 deprecated query-operator aliases (clean break) (ddcc0c7)
|
||||||
|
- refactor(8.0): remove dead/deprecated code (legacy sweep) (b9369f2)
|
||||||
|
- refactor(8.0): API-surface + quality polish from the readiness audit (a52dba2)
|
||||||
|
- fix(8.0): close GA-blocking correctness gaps from the readiness audit (47e8031)
|
||||||
|
- docs(8.0): correct public docs to the real 8.0 API + honest perf claims (40d2cd5)
|
||||||
|
- fix(8.0): re-validate find() results against the predicate (index-integrity guard) (3d11619)
|
||||||
|
|
||||||
|
|
||||||
### [7.33.0](https://github.com/soulcraftlabs/brainy/compare/v7.32.2...v7.33.0) (2026-06-19)
|
### [8.0.0-rc.4](https://github.com/soulcraftlabs/brainy/compare/v8.0.0-rc.3...v8.0.0-rc.4) (2026-06-24)
|
||||||
|
|
||||||
- feat: visibility tier (public/internal/system) on nouns + verbs (3a62445)
|
- docs(8.0): drop the DeletedItemsIndex section + pseudo-code from index-architecture (e7b50cf)
|
||||||
|
- fix(8.0): gate native graph analytics on the provider readiness flag (d321cf5)
|
||||||
|
- refactor(8.0): remove dead, unreachable, and unwired modules (bf0afe8)
|
||||||
|
- build(8.0): clean dist before every build so stale artifacts never ship (03d6540)
|
||||||
|
- feat(8.0): #35 part-3 — supply at-gen candidate vectors for the native exact-rerank (c9e2169)
|
||||||
|
|
||||||
|
|
||||||
### [7.32.2](https://github.com/soulcraftlabs/brainy/compare/v7.32.1...v7.32.2) (2026-06-19)
|
### [8.0.0-rc.3](https://github.com/soulcraftlabs/brainy/compare/v8.0.0-rc.2...v8.0.0-rc.3) (2026-06-23)
|
||||||
|
|
||||||
- refactor: rename BackupData → PortableGraph (the type is interchange, not a backup) (89036de)
|
- test(8.0): de-flake the VFS path-cache timing assertion (0e8972c)
|
||||||
|
- feat(8.0): asOf at-gen vector defer — provider-served historical semantic search (#35) (1c363e8)
|
||||||
|
- perf(8.0): bound find({ where, orderBy }) sort to the page (CTX-BR-FIND-ORDERBY) (450084b)
|
||||||
|
- feat(8.0): brain.graph.subgraph(query) query→expand fusion (#61) (82dde92)
|
||||||
|
- feat(8.0): vector allowedIds predicate-pushdown into find() (#46) (dd325f2)
|
||||||
|
- feat(8.0): graph analytics — brain.graph.rank / communities / path (632d90a)
|
||||||
|
- fix(8.0): restore() reloads a native entity-id mapper before graphIndex.rebuild() (4d0b64f)
|
||||||
|
- test(8.0): cover pending-tier range queries + setRetentionBudget adaptive reclaim (3783e61)
|
||||||
|
- feat(8.0): Model-B per-write generation-stamping + adaptive retention knob (5c3bb2c)
|
||||||
|
- test(8.0): Model-B write-perf + scalability spike harnesses (afac7f9)
|
||||||
|
- perf(8.0): per-id history chains for O(log) historical reads + bounded delta cache (ceed70d)
|
||||||
|
- refactor(8.0): graph analytics contract — intent names, not algorithm names (f3e6911)
|
||||||
|
- docs(8.0): RELEASES — native provider is @soulcraft/cor 3.0 (fix cortex 3.0 self-contradiction) (96d9c0b)
|
||||||
|
- test(8.0): cover the native graph seam + make provider resolution factory-tolerant (29410bc)
|
||||||
|
|
||||||
|
|
||||||
### [7.32.1](https://github.com/soulcraftlabs/brainy/compare/v7.32.0...v7.32.1) (2026-06-17)
|
### [8.0.0-rc.2](https://github.com/soulcraftlabs/brainy/compare/v8.0.0-rc.1...v8.0.0-rc.2) (2026-06-21)
|
||||||
|
|
||||||
- fix: getNouns().totalCount reports true total, not page size; quiet benign mmap-vector log (edff637)
|
- docs(8.0): RELEASES rc.2 additions — graph engine + additive wins + correctness fixes (18f27cb)
|
||||||
|
- feat(8.0): brain.graph.export() + noun-walk cursor + noun visibility hydration (c2a84c9)
|
||||||
|
- feat(8.0): brain.graph.subgraph() + native-provider routing (8c2b57a)
|
||||||
|
- feat(8.0): related({ node }) — one-call both-direction incident edges (d4de48d)
|
||||||
|
- feat(8.0): GraphAccelerationProvider contract — the native graph-engine seam (a3d6fdb)
|
||||||
|
- perf(8.0): cursor pagination for the verb walk — full edge pagination O(N²) → O(N) (682e786)
|
||||||
|
- perf(8.0): visibility-aware fast adjacency — related() stays O(degree) under default visibility (a914313)
|
||||||
|
- feat(8.0): upsert + FindParams.includeVectors + removeMany adaptive chunking (4cc2088)
|
||||||
|
- docs(8.0): note reserved-field default-throw in RELEASES rc additions (1bc709d)
|
||||||
|
- feat(8.0): reserved-field enforcement — reservedFieldPolicy defaults to throw (54c7c39)
|
||||||
|
- docs: mark 8.0.0-rc.1 published (npm tag rc) + note rc.1 additions (ae3fe82)
|
||||||
|
|
||||||
|
|
||||||
### [7.32.0](https://github.com/soulcraftlabs/brainy/compare/v7.31.8...v7.32.0) (2026-06-16)
|
### [8.0.0-rc.1](https://github.com/soulcraftlabs/brainy/compare/v7.31.2...v8.0.0-rc.1) (2026-06-20)
|
||||||
|
|
||||||
- feat: portable graph export()/import() (BackupData v1) on brain.data() (a408d37)
|
- feat(8.0): id-normalization (#18) + aggregation min/max delete-safety + RC-safe release (d02e522)
|
||||||
|
- feat(8.0): API simplification — remove neural()/Db.search, one storage `path` key, integration→0 (606445c)
|
||||||
|
- feat(8.0): 7.x→8.0 layout migration — fix silent total data loss on first open (0c4a51c)
|
||||||
### [7.31.8](https://github.com/soulcraftlabs/brainy/compare/v7.31.7...v7.31.8) (2026-06-16)
|
- feat(8.0): temporal range verbs — diff, history, since(gen|Date), asOf{exclusive}, transactionLog window (2c84f86)
|
||||||
|
- refactor(8.0): rename BackupData → PortableGraph (the type is interchange, not a backup) (373a481)
|
||||||
- fix: query-cap memory misread (MemAvailable + floor) + rootDirectory getter for native mmap fast-path (3f8e097)
|
- fix(8.0): VFS path-cache instance-scoping + verb totalCount page-cap (3a3aa43)
|
||||||
|
- fix(8.0): multi-valued array fields index every element (contains no longer misses) (eccf420)
|
||||||
|
- fix(8.0): column-store range queries honor exclusive bounds (lessThan/greaterThan) (009e506)
|
||||||
### [7.31.7](https://github.com/soulcraftlabs/brainy/compare/v7.31.6...v7.31.7) (2026-06-11)
|
- fix(8.0): per-type counts rehydrate after cold reopen (column store, not dead sparse index) (d918f49)
|
||||||
|
- feat(8.0): version-coupling guard — a mismatched/failed native plugin fails loud, never silent JS fallback (1264fec)
|
||||||
- fix: vfs.rename() issues a metadata-only update + rollback of fresh adds removes them (ac29b0e)
|
- test(8.0): boundary guard forbids @soulcraft/cor too (cortex→cor rename) (b198281)
|
||||||
|
- fix(8.0): stats() per-type counts no longer inflate with HNSW re-saves (21d02d3)
|
||||||
|
- fix(8.0): getNouns().totalCount reports true total, not page size (port of 7.32.1) (b2005ff)
|
||||||
### [7.31.6](https://github.com/soulcraftlabs/brainy/compare/v7.31.5...v7.31.6) (2026-06-11)
|
- fix(8.0): real bugs surfaced by integration hardening — where-intersect, related() offset, relate() updatedAt (5eaf579)
|
||||||
|
- test(8.0): integration rot pass — 77→17 failures (parallel per-file hardening) (e5997a1)
|
||||||
- fix: remap reserved fields from update() metadata patches to their canonical location (67e5fc8)
|
- test(8.0): begin integration rot pass — clear-persistence (drop COW internals) + metadata-only addRelationship→relate (c600468)
|
||||||
|
- docs(8.0): RELEASES — portable export/import (BackupData v1) + distinctCount any-type section (4741e23)
|
||||||
|
- fix(8.0): distinctCount aggregates distinct values of any type + edge-case regression tests (574a8b1)
|
||||||
### [7.31.5](https://github.com/soulcraftlabs/brainy/compare/v7.31.4...v7.31.5) (2026-06-11)
|
- feat(8.0): validateBackup() dry-run + includeContent blob round-trip test + clone test (7aad803)
|
||||||
|
- docs(8.0): export/import guide + api/README portable backup section (c2b73d4)
|
||||||
- fix: feature-detect setVectorBackend before wiring the mmap-vector backend (a537b36)
|
- feat(8.0): portable graph export()/import() (BackupData v1) — Db.export + polymorphic import (010ccf8)
|
||||||
|
- feat(8.0): visibility field (public/internal/system) on nouns + verbs (f4dea80)
|
||||||
|
- test(8.0): close brains in afterEach (count-sync, get-relations teardown) (0ca0e5c)
|
||||||
### [7.31.4](https://github.com/soulcraftlabs/brainy/compare/v7.31.3...v7.31.4) (2026-06-10)
|
- fix(8.0): neural.clusters()/similarity must request vectors from get() (cc1a431)
|
||||||
|
- test(8.0): drop dead s3/distributed/cloud scripts + 32GB→8GB integration heap (73a7d82)
|
||||||
- fix: feature-detect setConnectionsCodec before wiring the connections codec (747ab97)
|
- test(8.0): remove dead s3/distributed/cloud scripts + stale s3 suite (af1ee46)
|
||||||
|
- test(8.0): Tier-1 integration via deterministic embedder (suite runnable again) (542b52e)
|
||||||
|
- test(8.0): use valid camelCase VerbType values in test-factory (e31ba89)
|
||||||
### [7.31.3](https://github.com/soulcraftlabs/brainy/compare/v7.31.2...v7.31.3) (2026-06-10)
|
- test(8.0): get() resolves null for absent custom ids instead of throwing (dc94af3)
|
||||||
|
- fix(8.0): accept application-supplied entity ids, not just UUIDs (36b7216)
|
||||||
- fix: mmap-vector backend capacity NaN at the provider FFI boundary (eade6ff)
|
- fix(8.0): honor top-level storage.path as a rootDirectory alias (5096f90)
|
||||||
|
- feat(8.0): thread commit generation through the graph-write provider contract (0951fa1)
|
||||||
|
- fix(8.0): drive query-cap off MemAvailable + floor auto-detected caps (b26d3d4)
|
||||||
|
- test(8.0): A/B benchmark harness (open leg) — generic corpus+metrics lib, brainy-alone scaling bench, boundary guard, real-embedding recall guard (c605b34)
|
||||||
|
- docs(8.0): remove unbacked Cortex '5.2x' perf claim + dangling /docs/cortex/comparison link (33caa52)
|
||||||
|
- docs(8.0): measured find() performance at 5k/100k in SCALING.md (f986832)
|
||||||
|
- test(8.0): asOf() error-path spot-checks + find() triple-composition correctness + scale-bench harness (af96064)
|
||||||
|
- docs(8.0): RELEASES.md — record removed BrainyZeroConfig + isFullyInitialized/awaitBackgroundInit in the breaking-change inventory (f12ca68)
|
||||||
|
- refactor(8.0)!: remove orphaned zero-config subsystem + dead cloud/progressive-init storage vestige (35b9d7e)
|
||||||
|
- refactor(8.0)!: remove distributed clustering subsystem — inert/orphaned, scale is single-process + native provider (00d3203)
|
||||||
|
- feat(8.0): zero-config finalize + cut JS quantization (config.vector = recall + persistMode) (f8e0079)
|
||||||
|
- fix(8.0): vfs.rename() issues a metadata-only update (port of the 7.31.7 fix) (f4c5d97)
|
||||||
|
- chore(8.0): final pre-RC1 sweep — API consistency, named errors, orphans, zero-cast codebase (1f7e365)
|
||||||
|
- feat(8.0): reserved-field contract — one canonical location, typed prevention, unified read/write (970e08c)
|
||||||
|
- feat(8.0): brain.fillSubtypes migration helper + pre-RC1 gap closure (c446783)
|
||||||
|
- docs(8.0): RELEASES.md 8.0.0 release-candidate entry — full breaking-change inventory + upgrade guide (9b0f4ac)
|
||||||
|
- refactor(8.0): delete DataAPI — superseded by Db persist/restore + import API + stats (478fa17)
|
||||||
|
- docs(8.0): consistency-model concept + snapshots guide — Db API replaces branching docs (cc8037d)
|
||||||
|
- feat(8.0): full query surface at historical generations via ephemeral index materialization (e5feae4)
|
||||||
|
- feat(8.0)!: delete fork/branch/commit/history/versions — superseded by the Db API (8f93add)
|
||||||
|
- feat(8.0): generational MVCC storage + Datomic-style Db API (now/transact/asOf/with/persist) (431cd64)
|
||||||
|
- fix(8.0): createIndex resolves the canonical 'vector' provider key — drop diskann/hnsw key lookups + legacy migration APIs (49e4948)
|
||||||
|
- feat(8.0): u64 BigInt graph provider contract — punch list a-d,g,h (2427bb7)
|
||||||
|
- chore(8.0): delete vectorStore:mmap wiring — dead in the 8.0 provider world (62f6472)
|
||||||
|
- chore(8.0): collapse dead defensive guards + redundant polyfills (42159f2)
|
||||||
|
- chore(8.0)!: drop browser support, cloud SDKs, legacy pipeline, dead threading (266715a)
|
||||||
|
- docs(8.0): Phase F — deep clean across 21 docs (adda157)
|
||||||
|
- chore(8.0): Phase C + D + E — config simplification, TODO sweep, test race fix (2626ab8)
|
||||||
|
- chore(8.0): Phase A + B — purge all @deprecated APIs + cacheManager dead branches (cb16a39)
|
||||||
|
- chore(8.0): step-7 follow-through — collapse remaining cloud branches + docs sweep (scaffold step 13) (9f9a415)
|
||||||
|
- feat(8.0)!: flip requireSubtype default to true (BRAINY-8.0-SUBTYPE-CONTRACT § C-1) (780fb64)
|
||||||
|
- fix(8.0): implement multi-hop subtype BFS in pure JS (open-core works standalone) (221fc45)
|
||||||
|
- refactor(8.0): SubtypeRegistry hook + drop multi-hop subtype throw (scaffold steps 11-12) (ed75f25)
|
||||||
|
- docs(8.0): document subtype required-by-default deferral (scaffold step 10) (1eb0ffc)
|
||||||
|
- refactor(8.0): drop strictConfig — surface too small to justify the option (scaffold step 9) (694a31f)
|
||||||
|
- refactor(8.0): simplify config.vector to 3 knobs + fold persistMode (scaffold step 8) (8e76740)
|
||||||
|
- refactor(8.0): drop cloud + OPFS storage adapters; filesystem + memory only (scaffold step 7) (0e6263a)
|
||||||
|
- refactor(8.0): final cleanup — drop HnswProvider alias + config.hnsw + 'hnsw' surface (scaffold step 6) (b20666e)
|
||||||
|
- refactor(8.0): strictConfig + brain.stats() vector field + wireConnectionsCodec feature-detect (scaffold step 5) (3e1ef95)
|
||||||
|
- refactor(8.0): add saveVectorIndexData / getVectorIndexData storage contract (scaffold step 4) (356f044)
|
||||||
|
- refactor(8.0): rename HNSWIndex class → JsHnswVectorIndex (scaffold step 3) (f39d420)
|
||||||
|
- refactor(8.0): add config.vector path + 'vectors' cache category (scaffold step 2) (8f87b35)
|
||||||
|
- refactor(8.0): rename HnswProvider → VectorIndexProvider (8.0 scaffold) (076c26f)
|
||||||
|
|
||||||
|
|
||||||
### [7.31.2](https://github.com/soulcraftlabs/brainy/compare/v7.31.1...v7.31.2) (2026-06-09)
|
### [7.31.2](https://github.com/soulcraftlabs/brainy/compare/v7.31.1...v7.31.2) (2026-06-09)
|
||||||
|
|
@ -1613,7 +1703,7 @@ Expected behavior after upgrade:
|
||||||
|
|
||||||
### 🐛 Bug Fixes
|
### 🐛 Bug Fixes
|
||||||
|
|
||||||
* **storage**: Fix `clear()` not deleting COW version control data ([#workshop-bug-report](https://github.com/soulcraftlabs/brainy/issues))
|
* **storage**: Fix `clear()` not deleting COW version control data (consumer-reported)
|
||||||
- Fixed all storage adapters to properly delete `_cow/` directory on clear()
|
- Fixed all storage adapters to properly delete `_cow/` directory on clear()
|
||||||
- Fixed in-memory entity counters not being reset after clear()
|
- Fixed in-memory entity counters not being reset after clear()
|
||||||
- Prevents COW reinitialization after clear() by setting `cowEnabled = false`
|
- Prevents COW reinitialization after clear() by setting `cowEnabled = false`
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,7 @@ Handoff file: `/home/dpsifr/.strategy/PLATFORM-HANDOFF.md`
|
||||||
|
|
||||||
**Brainy's current open actions:** None. MIT open-source — no platform-specific actions.
|
**Brainy's current open actions:** None. MIT open-source — no platform-specific actions.
|
||||||
|
|
||||||
**Current version:** `@soulcraft/brainy@7.19.10`
|
**Current version:** `@soulcraft/brainy@7.31.5` (latest published; 8.0.0 release candidate on `feat/8.0-u64-ids`)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
|
||||||
109
README.md
109
README.md
|
|
@ -21,7 +21,8 @@ Built because we were tired of stitching together Pinecone + Neo4j + MongoDB and
|
||||||
## Install
|
## Install
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
npm install @soulcraft/brainy
|
bun add @soulcraft/brainy # fastest — recommended
|
||||||
|
npm install @soulcraft/brainy # Node.js — fully supported
|
||||||
```
|
```
|
||||||
|
|
||||||
## Quick Start
|
## Quick Start
|
||||||
|
|
@ -144,38 +145,34 @@ const results = await brain.find({
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
### Git-Style Branching
|
### Database as a Value
|
||||||
|
|
||||||
Fork your entire database in <100ms. Snowflake-style copy-on-write.
|
The whole database, pinned as an immutable value. Snapshot isolation, time travel, atomic transactions, instant hard-link snapshots.
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
const experiment = await brain.fork('test-migration')
|
const db = brain.now() // Pin current state — O(1)
|
||||||
await experiment.add({ data: 'test data', type: NounType.Concept })
|
|
||||||
await experiment.commit({ message: 'Add test data', author: 'dev@co.com' })
|
|
||||||
await brain.checkout('test-migration')
|
|
||||||
|
|
||||||
// Time-travel: query at any past commit
|
// Atomic multi-write transaction (all-or-nothing, with CAS)
|
||||||
const snapshot = await brain.asOf(commitId)
|
await brain.transact([
|
||||||
const pastResults = await snapshot.find({ query: 'historical data' })
|
{ op: 'update', id: orderId, metadata: { status: 'paid' } },
|
||||||
await snapshot.close()
|
{ op: 'relate', from: invoiceId, to: orderId, type: VerbType.References, subtype: 'billing' }
|
||||||
|
], { meta: { author: 'billing-service' }, ifAtGeneration: db.generation })
|
||||||
|
|
||||||
|
await db.get(orderId) // Still 'pending' — pinned, forever
|
||||||
|
await brain.get(orderId) // 'paid' — live
|
||||||
|
|
||||||
|
// Time travel: full query surface at any past state
|
||||||
|
const yesterday = await brain.asOf(new Date(Date.now() - 86_400_000))
|
||||||
|
const past = await yesterday.find({ query: 'unpaid orders' })
|
||||||
|
|
||||||
|
// What-if: speculative writes, nothing touches disk
|
||||||
|
const whatIf = await db.with([{ op: 'remove', id: orderId }])
|
||||||
|
|
||||||
|
// Instant backup: hard-link snapshot, opens read-only with Brainy.load()
|
||||||
|
await brain.now().persist('/backups/today')
|
||||||
```
|
```
|
||||||
|
|
||||||
**[Branching Documentation](docs/features/instant-fork.md)**
|
**[Consistency Model](docs/concepts/consistency-model.md)** | **[Snapshots & Time Travel](docs/guides/snapshots-and-time-travel.md)**
|
||||||
|
|
||||||
### Entity Versioning
|
|
||||||
|
|
||||||
Save, restore, and compare entity snapshots.
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
const userId = await brain.add({ data: 'Alice', type: NounType.Person })
|
|
||||||
await brain.versions.save(userId, { tag: 'v1.0' })
|
|
||||||
|
|
||||||
await brain.update(userId, { data: 'Alice Smith' })
|
|
||||||
await brain.versions.save(userId, { tag: 'v2.0' })
|
|
||||||
|
|
||||||
const diff = await brain.versions.compare(userId, 1, 2)
|
|
||||||
await brain.versions.restore(userId, 1)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Virtual Filesystem
|
### Virtual Filesystem
|
||||||
|
|
||||||
|
|
@ -200,9 +197,10 @@ CSV, Excel, PDF, URLs — auto-detected format, auto-classified entities.
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
await brain.import('customers.csv')
|
await brain.import('customers.csv')
|
||||||
await brain.import('sales-data.xlsx', { excelSheets: ['Q1', 'Q2'] })
|
await brain.import('sales-data.xlsx') // all sheets processed
|
||||||
await brain.import('research-paper.pdf', { pdfExtractTables: true })
|
await brain.import('research-paper.pdf') // tables extracted automatically
|
||||||
await brain.import('https://api.example.com/data.json')
|
await brain.import('https://api.example.com/data.json')
|
||||||
|
await brain.import('./handbook.md', { vfsPath: '/imports/handbook' }) // preserve in VFS
|
||||||
```
|
```
|
||||||
|
|
||||||
**[Import Guide](docs/guides/import-anything.md)**
|
**[Import Guide](docs/guides/import-anything.md)**
|
||||||
|
|
@ -269,7 +267,7 @@ await brain.relate({
|
||||||
|
|
||||||
// Filter on the fast path — column-store hit, not metadata fallback:
|
// Filter on the fast path — column-store hit, not metadata fallback:
|
||||||
const employees = await brain.find({ type: NounType.Person, subtype: 'employee' })
|
const employees = await brain.find({ type: NounType.Person, subtype: 'employee' })
|
||||||
const directReports = await brain.getRelations({ from: ceoId, subtype: 'direct' })
|
const directReports = await brain.related({ from: ceoId, subtype: 'direct' })
|
||||||
|
|
||||||
// O(1) counts via the persisted rollups:
|
// O(1) counts via the persisted rollups:
|
||||||
brain.counts.bySubtype(NounType.Person)
|
brain.counts.bySubtype(NounType.Person)
|
||||||
|
|
@ -282,11 +280,16 @@ brain.counts.byRelationshipSubtype(VerbType.ReportsTo)
|
||||||
**Enforce the pairing.** Register a vocabulary per type or turn on brain-wide strict mode to ensure every entity AND relationship has both `type` AND `subtype`:
|
**Enforce the pairing.** Register a vocabulary per type or turn on brain-wide strict mode to ensure every entity AND relationship has both `type` AND `subtype`:
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// Per-type rule with vocabulary
|
// Per-type rule with a closed vocabulary
|
||||||
brain.requireSubtype(NounType.Person, { values: ['employee', 'customer'], required: true })
|
brain.requireSubtype(NounType.Person, { values: ['employee', 'customer'], required: true })
|
||||||
|
|
||||||
// Or brain-wide strict mode
|
// 8.0 default: every write requires a subtype. Exempt genuine catch-all types…
|
||||||
const brain = new Brainy({ requireSubtype: true })
|
const brain = new Brainy({ requireSubtype: { except: [NounType.Thing] } })
|
||||||
|
|
||||||
|
// …or opt out while migrating pre-8.0 data, then audit and back-fill:
|
||||||
|
const legacy = new Brainy({ requireSubtype: false })
|
||||||
|
await legacy.audit() // gaps, grouped by type
|
||||||
|
await legacy.fillSubtypes({ [NounType.Person]: 'unspecified' }) // close them
|
||||||
```
|
```
|
||||||
|
|
||||||
For other facets you want counted (`status`, `source`, `role`), register them with `brain.trackField(name)`. Renaming an existing convention to `subtype`? Use `brain.migrateField({from, to, entityKind: 'both'})` to walk nouns AND verbs in one pass. Full guide: **[Subtypes & Facets](docs/guides/subtypes-and-facets.md)**.
|
For other facets you want counted (`status`, `source`, `role`), register them with `brain.trackField(name)`. Renaming an existing convention to `subtype`? Use `brain.migrateField({from, to, entityKind: 'both'})` to walk nouns AND verbs in one pass. Full guide: **[Subtypes & Facets](docs/guides/subtypes-and-facets.md)**.
|
||||||
|
|
@ -295,7 +298,7 @@ For other facets you want counted (`status`, `source`, `role`), register them wi
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Storage: Memory to Cloud
|
## Storage: Memory and Filesystem
|
||||||
|
|
||||||
The same API at every scale. Change one config line to go from prototype to production.
|
The same API at every scale. Change one config line to go from prototype to production.
|
||||||
|
|
||||||
|
|
@ -305,28 +308,31 @@ The same API at every scale. Change one config line to go from prototype to prod
|
||||||
const brain = new Brainy()
|
const brain = new Brainy()
|
||||||
```
|
```
|
||||||
|
|
||||||
### Production — Filesystem with Compression
|
### Production — Filesystem (gzip compression on by default)
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: { type: 'filesystem', path: './data', compression: true }
|
storage: { type: 'filesystem', path: './data' }
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
### Cloud — S3, GCS, Azure, Cloudflare R2
|
### Backups and Portability — Snapshots
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
const brain = new Brainy({
|
const db = brain.now()
|
||||||
storage: {
|
await db.persist('/backups/2026-06-11') // instant hard-link snapshot
|
||||||
type: 's3',
|
await db.release()
|
||||||
s3Storage: { bucketName: 'my-knowledge-base', region: 'us-east-1' }
|
|
||||||
}
|
const snapshot = await Brainy.load('/backups/2026-06-11') // read-only Db
|
||||||
})
|
const hits = await snapshot.find({ query: 'quarterly invoices' })
|
||||||
|
await snapshot.release()
|
||||||
```
|
```
|
||||||
|
|
||||||
|
A snapshot directory is self-contained — copy it to another machine, open it with `Brainy.load()`, or restore it wholesale with `brain.restore(path, { confirm: true })`.
|
||||||
|
|
||||||
Performance benchmarks and capacity planning in **[docs/PERFORMANCE.md](docs/PERFORMANCE.md)**.
|
Performance benchmarks and capacity planning in **[docs/PERFORMANCE.md](docs/PERFORMANCE.md)**.
|
||||||
|
|
||||||
**[Cloud Deployment Guide](docs/deployment/CLOUD_DEPLOYMENT_GUIDE.md)** | **[Capacity Planning](docs/operations/capacity-planning.md)**
|
**[Capacity Planning](docs/operations/capacity-planning.md)**
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -375,23 +381,21 @@ Performance benchmarks and capacity planning in **[docs/PERFORMANCE.md](docs/PER
|
||||||
|
|
||||||
### Operations
|
### Operations
|
||||||
|
|
||||||
- **[Cloud Deployment](docs/deployment/CLOUD_DEPLOYMENT_GUIDE.md)** — AWS, GCS, Azure
|
|
||||||
- **[Capacity Planning](docs/operations/capacity-planning.md)** — Memory, storage, and scaling
|
- **[Capacity Planning](docs/operations/capacity-planning.md)** — Memory, storage, and scaling
|
||||||
- **[Performance](docs/PERFORMANCE.md)** — Benchmarks and architecture details
|
- **[Performance](docs/PERFORMANCE.md)** — Benchmarks and architecture details
|
||||||
- Cost Optimization: **[AWS S3](docs/operations/cost-optimization-aws-s3.md)** | **[GCS](docs/operations/cost-optimization-gcs.md)** | **[Azure](docs/operations/cost-optimization-azure.md)** | **[R2](docs/operations/cost-optimization-cloudflare-r2.md)**
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Requirements
|
## Requirements
|
||||||
|
|
||||||
**Bun 1.0+** (recommended) or **Node.js 22 LTS**
|
**Bun 1.1+** (recommended) or **Node.js 22 LTS**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
bun install @soulcraft/brainy # Bun — best performance
|
bun add @soulcraft/brainy # Bun — best performance
|
||||||
npm install @soulcraft/brainy # Node.js — fully supported
|
npm install @soulcraft/brainy # Node.js — fully supported
|
||||||
```
|
```
|
||||||
|
|
||||||
> **Deprecation Notice:** Browser support (OPFS, Web Workers, WASM embeddings) is deprecated in v7.10.0 and will be removed in v8.0.0. Brainy v8+ will be server-only.
|
> Brainy 8.0 is server-only. Browser support (OPFS storage, Web Workers, in-browser WASM embeddings) was removed in 8.0 — the 7.x line remains available on npm if you need it.
|
||||||
|
|
||||||
## Single-Writer Model
|
## Single-Writer Model
|
||||||
|
|
||||||
|
|
@ -402,13 +406,13 @@ existing one.
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Live application — writer mode is the default
|
// Live application — writer mode is the default
|
||||||
const brain = new Brainy({ storage: { type: 'filesystem', rootDirectory: '/data/brain' } })
|
const brain = new Brainy({ storage: { type: 'filesystem', path: '/data/brain' } })
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
||||||
// Out-of-band diagnostics from a separate process — safe to run while the
|
// Out-of-band diagnostics from a separate process — safe to run while the
|
||||||
// writer is live
|
// writer is live
|
||||||
const reader = await Brainy.openReadOnly({
|
const reader = await Brainy.openReadOnly({
|
||||||
storage: { type: 'filesystem', rootDirectory: '/data/brain' }
|
storage: { type: 'filesystem', path: '/data/brain' }
|
||||||
})
|
})
|
||||||
await reader.requestFlush({ timeoutMs: 5000 })
|
await reader.requestFlush({ timeoutMs: 5000 })
|
||||||
const stats = await reader.stats()
|
const stats = await reader.stats()
|
||||||
|
|
@ -425,8 +429,7 @@ brainy inspect health /data/brain
|
||||||
|
|
||||||
See [the multi-process model](docs/concepts/multi-process.md) and the
|
See [the multi-process model](docs/concepts/multi-process.md) and the
|
||||||
[inspection guide](docs/guides/inspection.md) for the full story, including
|
[inspection guide](docs/guides/inspection.md) for the full story, including
|
||||||
stale-lock detection, the cross-process flush RPC, and what's not yet
|
stale-lock detection and the cross-process flush RPC.
|
||||||
enforced on cloud storage backends.
|
|
||||||
|
|
||||||
## Contributing
|
## Contributing
|
||||||
|
|
||||||
|
|
|
||||||
713
RELEASES.md
713
RELEASES.md
|
|
@ -10,194 +10,582 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## v7.33.5 — 2026-07-02
|
## v8.0.0 — release candidate (`8.0.0-rc.9` published 2026-07-01 on npm tag `rc`)
|
||||||
|
|
||||||
**Affected products:** any consumer whose brain restarts cold (every deploy) and issues
|
> **rc.9 changes (2026-07-01):**
|
||||||
`find({ where: { field } })` filtered reads against a **native metadata index** — where a cold
|
> - **Whole-brain auto-upgrade is now a coordinated, observable LOCK — supersedes rc.8's no-freeze
|
||||||
open could return `[]` for a field lookup that HAS matching data, silently blanking filtered
|
> approach.** When a large brain's derived-index format changes (7.x→8.0), the native provider
|
||||||
pages until the index warmed. Drop-in; no API or data changes.
|
> rebuilds the indexes from the canonical records **in place** while Brainy **blocks and queues**
|
||||||
|
> reads and writes — so no operation ever touches a half-built index. This is a clean *blocking
|
||||||
|
> upgrade to a known-good state* (vs rc.8's online background swap): unknown/halfway states are
|
||||||
|
> more dangerous than a bounded wait. Small brains still rebuild inline on open. New surface, all
|
||||||
|
> additive: a typed, exported, retryable **`MigrationInProgressError`** (catch it → HTTP 503 +
|
||||||
|
> `Retry-After`); `getIndexStatus()` gains **`migrating` + `migration`** progress (never gated —
|
||||||
|
> the readiness-probe signal); a **`migrationWaitTimeoutMs`** config (default 30 s — it bounds the
|
||||||
|
> *caller's wait*, NOT the rebuild, which is unbounded); `health()` / `checkHealth()` report the
|
||||||
|
> upgrade without blocking. No effect without a native provider.
|
||||||
|
> - **Faster vector search (open-core).** The JS distance functions are rewritten from `reduce` to
|
||||||
|
> allocation-free loops — **~6× cosine, ~1.4× euclidean** (MEASURED, `tests/benchmarks/distance-microbench.mjs`,
|
||||||
|
> 384-dim, median of 41). Numerically identical → recall unchanged.
|
||||||
|
> - **Runtime floor + Bun.** Engines are now **Node ≥22 / Bun ≥1.1** (fixes a Node-24 `EBADENGINE`);
|
||||||
|
> compiler target ES2023 with DOM dropped from the type lib (8.0 is Node/Bun/Deno-only, no browser
|
||||||
|
> path). **Bun is recommended as a runtime** (`bun add` / `bun run`); the single-binary
|
||||||
|
> `bun build --compile` is not a supported target (native addon can't embed + a Bun 1.3.10 codegen
|
||||||
|
> regression). `.d.ts` stays TS-5.x-parseable.
|
||||||
|
|
||||||
### Fix — filtered `find({ where })` no longer silently returns `[]` on a cold open
|
> **rc.8 additions (2026-06-30) — additive, no breaking change:**
|
||||||
|
> - **No-freeze (online) whole-brain auto-upgrade.** When a derived-index format changes, a large
|
||||||
|
> brain upgrades **without blocking** — the native provider rebuilds the new indexes in the
|
||||||
|
> background and serves correct reads from the canonical records meanwhile, then atomically swaps;
|
||||||
|
> no minutes-long freeze on the first open/query. (Small brains still auto-rebuild inline on open.)
|
||||||
|
> New surface: an optional provider `isMigrating()` deference signal, a public `brain.stampBrainFormat()`
|
||||||
|
> the provider calls once its swap verifies, and a `@soulcraft/brainy/brain-format` export so the
|
||||||
|
> provider shares the `indexEpoch` constant. No effect without a native provider.
|
||||||
|
|
||||||
The companion to 7.33.2's graph cold-read guard, now for the **metadata field index**. On a fresh
|
> **rc.7 additions (2026-06-30) — additive, no breaking change:**
|
||||||
brain open, a native metadata provider can report that data exists but not yet serve its `where`
|
> - **8.0 cold-graph self-heal.** `find({ connected })` / `neighbors()` / `related()` never
|
||||||
postings — so `find({ where: { status: 'active' } })` returned an empty array **indistinguishable
|
> silently return `[]` on a cold open where the graph adjacency loaded its membership but not
|
||||||
from "no such data"**, and callers rendered "nothing found" over real data.
|
> its source→target edges — it self-heals (rebuild from storage) or throws a loud
|
||||||
|
> `GraphIndexNotReadyError`, never empty-for-persisted-data. (The 8.0 equivalent of the 7.33.4
|
||||||
|
> fix 7.x consumers already have; gates on the native provider's honest `isReady()` edge-readiness
|
||||||
|
> signal.)
|
||||||
|
> - **Billion-scale RAM — nothing O(N)-resident on the write/time-travel path.** Eliminated the
|
||||||
|
> per-id storage caches (per-type counts now sourced from the record), made the
|
||||||
|
> committed-generation ledger an interval set, and bounded the per-id time-travel history chains
|
||||||
|
> (hot-window + LRU, with lock-light reconstruction so historical reads never stall writers).
|
||||||
|
> Resident memory is now independent of entity count.
|
||||||
|
> - **Whole-brain version handshake.** A `_system/brain-format.json` marker + `brain.formatInfo()`
|
||||||
|
> + a shared, lockstep `indexEpoch`: a future derived-index format change auto-rebuilds the
|
||||||
|
> indexes from the canonical records on open (non-destructively) — the foundation for
|
||||||
|
> whole-brain auto-upgrade with the native provider. No effect on an unchanged brain.
|
||||||
|
|
||||||
Brainy now runs a one-time serving check on the first filtered `find()`: it probes a known
|
> **rc.6 additions (2026-06-29) — additive, no breaking change:**
|
||||||
persisted field value and, if the index doesn't return the known entity, **rebuilds the index from
|
> - **Open-core perf**: HNSW delete is now O(in-degree) (was O(N²) for bulk delete) via a
|
||||||
the canonical records** and re-probes. If it still cannot serve, it throws a loud, catchable
|
> reverse-adjacency index; the negation/absence operators (`ne`/`exists:false`/`missing:true`)
|
||||||
`MetadataIndexNotReadyError` (now exported, alongside `GraphIndexNotReadyError` and `BrainyError`)
|
> are served as a roaring-bitmap difference instead of materializing the whole corpus.
|
||||||
rather than let a silent `[]` stand. On a warm brain the only cost is one O(1) probe.
|
> - **Native provider contract** (cor lockstep, optional/feature-detected — no effect without a
|
||||||
|
> native provider): a cold-open `probeConsistency()` self-heal hook, and a `getIdsForFilter`
|
||||||
|
> page bound so the native index can early-stop the unsorted `find({ type, where, limit })` path.
|
||||||
|
> - **Test hygiene**: re-homed previously-unrun test suites into CI + a guard so a test file can
|
||||||
|
> never silently fall outside every config again. No source/API change from these.
|
||||||
|
|
||||||
**What you get:** cold `where` reads are correct-by-default (self-healed) or fail loudly — never a
|
|
||||||
silent empty result. Catch `MetadataIndexNotReadyError` (or check `error.type ===
|
|
||||||
'METADATA_INDEX_NOT_READY'`) to render a "warming up / retry" state instead of a false "no results".
|
|
||||||
The durable native cold-load cure remains a cortex-side follow-up; this guard makes silent-empty
|
|
||||||
impossible regardless of provider.
|
|
||||||
|
|
||||||
---
|
**Affected products:** every consumer — this is a major release with removed
|
||||||
|
surfaces, hard renames (no aliases, no deprecation period), and one flipped
|
||||||
|
default. This entry **is** the migration guide: the find/replace pairs, the
|
||||||
|
sed snippet, and the upgrade checklist below are the complete story — there is
|
||||||
|
no separate migration doc. **`8.0.0-rc.6` is live** — install with
|
||||||
|
`npm i @soulcraft/brainy@rc` to try it; the `latest` tag stays on 7.x until GA.
|
||||||
|
|
||||||
## v7.33.2 — 2026-06-24
|
> **rc.3–rc.5 additions (2026-06-24):**
|
||||||
|
> - **Showcase-quality GA hardening (rc.5).** A full readiness audit closed a cold-init
|
||||||
|
> version-coupling bug (a matched native provider could be rejected on first open), turned
|
||||||
|
> silent storage-read failures into named `BrainyError`s (no more empty-result-on-error),
|
||||||
|
> stopped the JS graph LSM from orphaning compacted SSTables, corrected the public docs to
|
||||||
|
> the real API, and documented the two headline methods. Plus a dead/deprecated-code sweep.
|
||||||
|
> - **BREAKING — 4 deprecated query operators removed.** `is`→`eq`, `isNot`→`ne`,
|
||||||
|
> `greaterEqual`→`gte`, `lessEqual`→`lte`. The canonical operators and their clean long-form
|
||||||
|
> aliases (`equals`/`notEquals`/`greaterThan`/`greaterThanOrEqual`/`lessThan`/`lessThanOrEqual`)
|
||||||
|
> are unchanged — only the four redundant spellings are gone. Find/replace if you used them.
|
||||||
|
> - **`find()` search mode** is now the single `searchMode: SearchMode` option (the redundant,
|
||||||
|
> silently-ignored `mode` alias and the unwired `explain` flag were removed from `FindParams`).
|
||||||
|
> - The phantom index-integrity guard (find() re-validates every result against its predicate)
|
||||||
|
> shipped in rc.3; rc.4 added the `accel.isInitialized` gate + #35 at-gen candidate vectors.
|
||||||
|
|
||||||
**Affected products:** any consumer using graph traversals — `find({ connected: { from, via } })`,
|
> **rc.1 additions (this entry predates them — full writeup lands at GA):** entity-id
|
||||||
`neighbors()`, `related()` — with a **native graph index** on **`mmap-filesystem`** (or any adapter
|
> normalization — `brain.newId()` (UUID v7) + v7 default ids, and non-UUID string ids are
|
||||||
whose cold-open adjacency load can fail), where a process restart could leave graph queries returning
|
> transparently normalized to a stable UUID v5 (original key preserved under `_originalId`);
|
||||||
`[]` despite persisted edges. Drop-in; no API or data changes.
|
> `brain.neural()` clustering and `Db.search()` removed (use `find({ vector })` /
|
||||||
|
> `find()` + aggregation `GROUP BY`); storage config collapsed to one `path` key (the
|
||||||
|
> pre-8.0 aliases now throw); and `queryAggregate` no longer hangs/staled min-max after a delete.
|
||||||
|
> **Reserved fields in a `metadata` bag now throw by default** — passing a Brainy-reserved key
|
||||||
|
> (`confidence`, `weight`, `subtype`, `visibility`, `service`, `createdBy`, `noun`/`verb`, `data`,
|
||||||
|
> `createdAt`, `updatedAt`, `_rev`) inside `metadata` on `add()`/`update()`/`relate()`/`updateRelation()`
|
||||||
|
> (untyped/JS callers — TypeScript already blocks it) is rejected with an Error naming the correct
|
||||||
|
> param, instead of the old silent remap-or-drop. Pass reserved values as their dedicated top-level
|
||||||
|
> params. Opt back into the legacy behavior with `new Brainy({ reservedFieldPolicy: 'warn' | 'remap' })`.
|
||||||
|
|
||||||
### Fix — graph adjacency no longer silently returns `[]` on a cold open
|
> **rc.2 additions (2026-06-21):**
|
||||||
|
> - **Graph performance + the `brain.graph` namespace.** New `brain.graph.subgraph(seeds, opts)`
|
||||||
|
> (bounded multi-hop neighborhood → `{ nodes, edges, truncated }`) and `brain.graph.export(opts)`
|
||||||
|
> (stream the whole graph in one O(N+E) pass — the right primitive for visualizing all data
|
||||||
|
> instead of paging per node). `related({ node })` returns every edge incident to an entity in
|
||||||
|
> **both directions** in one O(degree) call. Under the hood: `related({ from/to })` now stays
|
||||||
|
> O(degree) under the default visibility filter (was a full scan), and the verb **and** noun
|
||||||
|
> pagination walks are cursor-based, so a full edge/node walk is **O(N), not O(N²)** (a consumer
|
||||||
|
> measured a 19k-edge whole-graph read drop from ~27s toward a single scan). These transparently
|
||||||
|
> use a native graph engine when present (the `@soulcraft/cor` 3.0 acceleration layer) and fall
|
||||||
|
> back to pure-TS adjacency otherwise.
|
||||||
|
> - **Additive ergonomics:** `add({ upsert: true })` (create-or-update in one call — merges into an
|
||||||
|
> existing id instead of overwriting), `find({ includeVectors: true })`, and `removeMany`'s batch
|
||||||
|
> size is now storage-adaptive (was a hardcoded 10).
|
||||||
|
> - **Correctness fixes (apply to all consumers, not just graph users):** `related()` results now
|
||||||
|
> carry `visibility`; whole-graph/`getNouns` streaming no longer leaks `system`/`internal` entities;
|
||||||
|
> and small-page cursor walks over nouns no longer loop. No API change — just correct behavior.
|
||||||
|
|
||||||
On a fresh brain open, a native graph provider can load its relationship **count** (the manifest) but
|
> **rc.3 additions (2026-06-23):**
|
||||||
fail to load the source→target **adjacency** itself — so `find({ connected })` / `neighbors()` /
|
> - **Graph analytics on the `brain.graph` namespace.** Three intent-level reads that answer
|
||||||
`related()` returned an empty array **indistinguishable from "no edges"**, and stayed empty after
|
> whole-graph questions in one call:
|
||||||
warm-up. Because it was `[]` and not an error, callers could not tell real data was missing.
|
> - **`brain.graph.rank(opts?)`** → `{ id, score }[]` descending — "which entities matter most"
|
||||||
|
> (importance / centrality). `topK` to cap.
|
||||||
|
> - **`brain.graph.communities(opts?)`** → `{ groups: string[][], count }` — "which things group
|
||||||
|
> together". Weakly-connected components by default; `{ directed: true }` returns
|
||||||
|
> strongly-connected components.
|
||||||
|
> - **`brain.graph.path(from, to, opts?)`** → `{ nodes, relationships, cost } | null` — the best
|
||||||
|
> route between two entities. Fewest hops by default; `{ by: 'weight' }` minimizes summed edge
|
||||||
|
> weight (the 0–1 connection strength); `direction`, `type`, and `maxDepth` filters apply.
|
||||||
|
> These are **intent contracts, not algorithm contracts** — the question is the promise, the
|
||||||
|
> algorithm is the engine's choice. They transparently use the native `@soulcraft/cor` 3.0 graph
|
||||||
|
> engine when present, and fall back to pure-TS kernels (PageRank, connected components / Tarjan
|
||||||
|
> SCC, BFS / Dijkstra) otherwise. Both paths return identical shapes and respect the default
|
||||||
|
> visibility filter (opt in with `includeInternal` / `includeSystem`).
|
||||||
|
> - **Filtered vector search keeps its recall (`allowedIds` pushdown).** A `find({ query, where })`
|
||||||
|
> that combines semantic search with a metadata filter now restricts the vector walk to the
|
||||||
|
> matching candidates *inside* the search (walk-all, collect-allowed) instead of filtering the
|
||||||
|
> top-k afterward — so a query whose nearest vectors are all filtered out still returns the best
|
||||||
|
> matches that DO pass the filter, rather than coming back empty. No API change. With the native
|
||||||
|
> `@soulcraft/cor` 3.0 stack the matched universe is forwarded as an opaque roaring buffer with
|
||||||
|
> zero id materialization in TypeScript; the pure-JS path restricts the beam walk with a string set.
|
||||||
|
> - **Historical (`asOf`) semantic search can skip the rebuild.** A filtered semantic query at a
|
||||||
|
> past generation — `db.asOf(g).find({ query, where })` — no longer always rebuilds an ephemeral
|
||||||
|
> in-memory vector index over every at-`g` vector (O(n@G)). When a native versioned vector engine
|
||||||
|
> is registered and can serve the pinned generation, Brainy resolves the at-`g` filter universe from
|
||||||
|
> its record layer (no rebuild) and routes the vector leg to the engine with the matched ids + the
|
||||||
|
> generation; without one it falls back to the materialization (unchanged). The `VectorIndexProvider.search`
|
||||||
|
> options gained an optional `generation?: bigint` ("omitted = now") to carry this — additive, the
|
||||||
|
> built-in index ignores it. No public API change.
|
||||||
|
> - **`find({ where, orderBy })` bounds the sort to the page.** A broad filter + `orderBy`
|
||||||
|
> returning one page no longer materializes every matching sorted id (it produced the full sorted
|
||||||
|
> match set, then sliced) — the page bound (`offset + limit`) is threaded into the column store's
|
||||||
|
> top-K sort, so returning 20 rows from a million matches stays a bounded-K heap. No API change;
|
||||||
|
> ordering and pagination are unchanged.
|
||||||
|
> - **`brain.graph.subgraph()` accepts a query (query→expand).** The seed selector now takes not
|
||||||
|
> just entity id(s) but a `find()` result or a `FindParams` query — `brain.graph.subgraph({ where:
|
||||||
|
> { team: 'platform' } }, { depth: 1 })` runs the query and expands the neighborhood of every
|
||||||
|
> match in one call. With the native engine a metadata-only query's matched universe is handed to
|
||||||
|
> the traversal as an opaque set with no id materialization in TypeScript (the query→expand
|
||||||
|
> fusion); the pure-JS path materializes the matched ids and expands from them. Existing
|
||||||
|
> id-seeded calls are unchanged.
|
||||||
|
|
||||||
Brainy now runs a one-time consistency check on the first graph read: if the index reports that
|
### Headline: Database as a Value
|
||||||
relationships exist but a known persisted edge's source resolves to **no** neighbors, it **rebuilds
|
|
||||||
the adjacency from storage**. If even a rebuild cannot read the edge, it throws a loud, catchable
|
|
||||||
`GraphIndexNotReadyError` instead of serving `[]` as truth. The probe is O(1) (one edge + one neighbor
|
|
||||||
lookup), cached per brain, and a **no-op once the adjacency is live** — so consumers can return to
|
|
||||||
clean graph queries instead of in-memory-filter workarounds.
|
|
||||||
|
|
||||||
> The underlying cold-open load is a native-provider concern (being addressed upstream); this Brainy
|
8.0 replaces Brainy's two overlapping version-control subsystems (copy-on-write
|
||||||
> change makes the failure **self-healing and loud** rather than silent.
|
branching and per-entity versioning, ~5,100 LOC combined) with **one
|
||||||
|
mechanism**: generational MVCC over immutable, generation-stamped records,
|
||||||
|
exposed through a Datomic-style immutable database value — the **`Db`**.
|
||||||
|
|
||||||
---
|
```ts
|
||||||
|
const db = brain.now() // pin the current state — O(1), no I/O
|
||||||
|
|
||||||
## v7.33.1 — 2026-06-22
|
await brain.transact([
|
||||||
|
{ op: 'update', id: invoiceId, metadata: { status: 'paid' } }
|
||||||
|
], { meta: { author: 'billing-service', reason: 'PO-7741' } })
|
||||||
|
|
||||||
**Affected products:** any consumer on `filesystem` / `mmap-filesystem` storage with **more than one
|
await db.get(invoiceId) // still 'pending' — pinned, forever
|
||||||
page (> 500) of entities** that defines an **aggregate** over an already-populated brain (the common
|
await brain.get(invoiceId) // 'paid' — live
|
||||||
reopen-then-`defineAggregate()`/`queryAggregate()` case). **Critical fix** — a permanent post-init
|
await db.release() // unpin when done
|
||||||
CPU loop. Drop-in; no API or data changes. **Upgrade from 7.32.1 / 7.33.0 immediately.**
|
|
||||||
|
|
||||||
### Fix — `getNouns()` cursor pagination no longer re-scans the first page forever (permanent CPU loop)
|
|
||||||
|
|
||||||
`storage.getNouns({ pagination: { cursor } })` paginated by **cursor** never advanced: the shard-scan
|
|
||||||
adapter is offset-based and **ignored** the cursor, so every cursor call re-returned the first page.
|
|
||||||
On its own that was harmless — until v7.32.1 correctly made `totalCount` the true dataset total, which
|
|
||||||
makes `hasMore` stay `true` until a caller has paged through everything. A caller that paginates by
|
|
||||||
cursor (`cursor = page.nextCursor`) — notably **aggregate backfill** (`defineAggregate()` /
|
|
||||||
`queryAggregate()` over a pre-populated store) — therefore looped **forever**, re-walking the entire
|
|
||||||
entity shard tree on every iteration. With more than one page of entities this pegged 1–2 CPU cores
|
|
||||||
permanently on the JS main thread — with zero queries or traffic — and never settled. (Before v7.32.1
|
|
||||||
the very same loop terminated after one page, silently backfilling only the first 500 entities — an
|
|
||||||
incomplete aggregate.)
|
|
||||||
|
|
||||||
`getNouns()` now treats the cursor as an **opaque, advancing offset token**, so cursor pagination
|
|
||||||
behaves exactly like offset pagination — it advances and terminates — and an aggregate backfill now
|
|
||||||
streams the **whole** corpus exactly once (no longer truncated at 500, no longer looping). The backfill
|
|
||||||
loop was additionally hardened to pure offset pagination as defense in depth. Regression that
|
|
||||||
reproduces the infinite loop and fails fast on any re-scan:
|
|
||||||
`tests/unit/storage/getNouns-cursor-pagination.test.ts`.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## v7.32.1 — 2026-06-17
|
|
||||||
|
|
||||||
**Affected products:** consumers on `filesystem` / `mmap-filesystem` storage whose logs showed
|
|
||||||
`🔄 Small dataset (1 items) - rebuilding all indexes…` on cold start, or noise from a
|
|
||||||
`mmap-vector backend not wired` line on every init. Two fixes. Drop-in; no API or data changes.
|
|
||||||
|
|
||||||
### Fix — `getNouns().totalCount` reports the true total, not the page size
|
|
||||||
|
|
||||||
`storage.getNouns({ pagination: { limit } })` returned `totalCount` equal to the **page size**, not
|
|
||||||
the dataset total: the type-first shard scan early-terminates at `offset + limit` for memory
|
|
||||||
efficiency, and the page-collected length was returned as the total. So
|
|
||||||
`getNouns({ pagination: { limit: 1 } })` reported `totalCount: 1` for **any** non-empty brain.
|
|
||||||
|
|
||||||
The index-rebuild gate uses exactly this call to size the corpus, so a cold start that needs a
|
|
||||||
rebuild logged `Small dataset (1 items) - rebuilding all indexes…` regardless of the real entity
|
|
||||||
count (a production deployment saw this for an ~8,800-entity brain — the rebuild then ran from
|
|
||||||
scratch instead of loading the persisted vector snapshot).
|
|
||||||
|
|
||||||
`getNounsWithPagination` now reports the authoritative O(1) noun counter (maintained on every
|
|
||||||
add/delete and rehydrated from `counts.json` on init) as the unfiltered `totalCount`, and computes
|
|
||||||
`hasMore` from it. Filtered scans are unchanged (collected length, a lower bound). Layout-independent
|
|
||||||
(applies equally to branch/COW layouts). Regression: `tests/unit/storage/getNouns-totalCount.test.ts`.
|
|
||||||
|
|
||||||
### Log — benign "mmap-vector backend not wired" downgraded to debug
|
|
||||||
|
|
||||||
When a native vector provider replaces the JS HNSW index (it owns its own vector storage and exposes
|
|
||||||
no `setVectorBackend` hook), brainy logged `mmap-vector backend not wired … per-entity reads in use`
|
|
||||||
on **every** init. This is expected and benign in the native-index model — not a fault, and not by
|
|
||||||
itself an indication of per-entity reads — but it appeared on every warm and was repeatedly mistaken
|
|
||||||
for a cold-start cause. It is now a debug-level line (surface it with `BRAINY_LOG_LEVEL=debug`).
|
|
||||||
|
|
||||||
> Note: this release fixes the misleading *count/log*. The remaining cold-start symptom on the
|
|
||||||
> native line (rebuilding instead of loading the persisted vector snapshot) is resolved when the
|
|
||||||
> native provider loads its snapshot at construction so the index reports a non-zero size before
|
|
||||||
> brainy's rebuild gate — already the model in the next major (8.0 + native 3.0).
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## v7.32.0 — 2026-06-16
|
|
||||||
|
|
||||||
**Affected products:** anyone who needs a **portable graph backup**, a partial export, or a
|
|
||||||
**7.x → 8.0 migration path**. Purely additive — no API removed, no data change. Drop-in.
|
|
||||||
|
|
||||||
### New — portable graph `export()` / `import()` on `brain.data()`
|
|
||||||
|
|
||||||
`brain.data()` gains a portable backup/restore pair that serializes part or all of a brain
|
|
||||||
to a single versioned JSON document (`BackupData`) and restores it. This is the format that
|
|
||||||
embeds inside portable artifacts (e.g. a serialized subgraph carried by a higher-level file
|
|
||||||
type) and the recommended way to move a graph between environments or **upgrade an existing
|
|
||||||
7.x brain into an 8.0 brain**.
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const data = await brain.data()
|
|
||||||
|
|
||||||
const backup = await data.export() // whole brain → BackupData
|
|
||||||
const subset = await data.export({ ids }, { includeVectors: true })
|
|
||||||
await otherBrain.data().then(d => d.import(subset, { onConflict: 'merge' }))
|
|
||||||
```
|
```
|
||||||
|
|
||||||
- **Selectors** — `export(selector?, options?)` picks the node set: `{ ids }`, `{ collection }`
|
What you get:
|
||||||
(alias `memberOf`) + transitive `Contains` members, `{ connected: { from, depth, verbs?, direction? } }`,
|
|
||||||
`{ vfsPath, recursive? }`, a predicate (`{ type, subtype, where, service }`), or the whole brain
|
|
||||||
(omit). Structural + predicate selectors **compose**.
|
|
||||||
- **Options** — `includeVectors` (default off → re-embed on import), `includeContent` (VFS file
|
|
||||||
bytes in `blobs`), `includeSystem` (include the VFS root etc.), `edges` (`'induced'` default /
|
|
||||||
`'incident'` / `'none'`).
|
|
||||||
- **Import** — `import(backup, options?)` → `{ imported, merged, skipped, reembedded, blobsWritten, errors }`.
|
|
||||||
`onConflict` defaults to `'merge'` (dedup-by-id, so you can assemble many documents into one
|
|
||||||
graph); `reembed` defaults to `'auto'` (re-embed from `data` when no vector is carried);
|
|
||||||
`remapIds` clones a subgraph under fresh ids.
|
|
||||||
- **Format** — `{ format:'brainy-backup', formatVersion:1, brainyVersion, createdAt, embedding,
|
|
||||||
entities, relations, blobs?, danglingIds?, stats }`. Entities carry `subtype` and the standard
|
|
||||||
reserved fields at the top level; `metadata` is custom-only. **Current-state** (no generation
|
|
||||||
history). `formatVersion` gates cross-version import (a 7.x document imports into 8.0).
|
|
||||||
- **Types** — `BackupData`, `BackupEntity`, `BackupRelation`, `ExportSelector`, `ExportOptions`,
|
|
||||||
`ImportOptions`, `ImportResult`, and the `DataAPI` class are now exported from the package root.
|
|
||||||
|
|
||||||
This replaces the previous `brain.data().export()` flat-entity array (which dropped relations)
|
- **`brain.now()`** — pins the current generation as an immutable `Db` view.
|
||||||
with a graph-complete document. Distinct from `brain.import(file)` (CSV/PDF/Excel/JSON ingestion),
|
True snapshot isolation: the view reads exactly its pinned state no matter
|
||||||
which is unchanged. The CLI `brainy export` now writes a `BackupData` document (json/jsonl/csv).
|
what commits afterwards, including deletes. Readers never block writers and
|
||||||
|
writers never block readers.
|
||||||
|
- **`brain.transact(ops, { meta, ifAtGeneration })`** — atomic multi-write
|
||||||
|
batches (`add` / `update` / `remove` / `relate` / `unrelate`) committed as
|
||||||
|
exactly one generation. Either every operation applies or none do.
|
||||||
|
`ifAtGeneration` is whole-store compare-and-swap (`GenerationConflictError`
|
||||||
|
on conflict) — the big sibling of the per-entity `ifRev` CAS from 7.31.
|
||||||
|
`meta` is reified Datomic-style into an append-only transaction log,
|
||||||
|
readable via `brain.transactionLog()`.
|
||||||
|
- **`brain.asOf(generation | Date | snapshotPath, { exclusive? })`** — time
|
||||||
|
travel with the **full query surface**: `get()`, `find()` in every mode,
|
||||||
|
semantic search, graph traversal, cursors, aggregation, all at the pinned past
|
||||||
|
state. `exclusive: true` pins the generation immediately before the target
|
||||||
|
(strict-before).
|
||||||
|
- **`db.with(ops)`** — speculative writes applied in memory on top of a view.
|
||||||
|
Nothing touches disk, the generation counter, or the indexes. What-if
|
||||||
|
analysis, then `transact()` the same ops for real.
|
||||||
|
- **`db.persist(path)`** — instant self-contained snapshots. On filesystem
|
||||||
|
storage they are built from hard links (no entity data is copied; later
|
||||||
|
writes to the source can never alter the snapshot because rewrites swap
|
||||||
|
inodes). `brain.restore(path, { confirm: true })` replaces the whole store
|
||||||
|
from one; `Brainy.load(path)` opens one read-only with the full query
|
||||||
|
surface.
|
||||||
|
- **Range verbs over history** — answer "what happened BETWEEN two points":
|
||||||
|
- **`db.since(Db | generation | Date)`** — the entity and relationship ids
|
||||||
|
committed transactions touched after an **exclusive** lower bound (now
|
||||||
|
accepts a generation or `Date`, not just a prior `Db`).
|
||||||
|
- **`brain.diff(a, b)`** — the touched ids CLASSIFIED as `{ added, removed,
|
||||||
|
modified }` (split by nouns/verbs) by resolving each at both endpoints; a
|
||||||
|
touched-but-reverted id lands in none of the buckets. Endpoints are a
|
||||||
|
generation, `Date`, or `Db`, in either order.
|
||||||
|
- **`brain.history(id, { from?, to? })`** — every distinct version of ONE
|
||||||
|
entity/relationship over a range, oldest first (`value: null` marks a
|
||||||
|
removal); each version equals `asOf(version.generation).get(id)`.
|
||||||
|
- **`brain.transactionLog({ from?, to?, limit? })`** — the commit log over an
|
||||||
|
**inclusive** generation/`Date` window (contrast `since`'s exclusive lower
|
||||||
|
bound); `limit` applies last.
|
||||||
|
- **Every write is versioned (Model-B).** History granularity is **per-write**:
|
||||||
|
EVERY write — `transact()` AND a single-operation `add`/`update`/`remove`/
|
||||||
|
`relate` — is its own immutable generation. A `now()` pin always freezes
|
||||||
|
against later writes, and `asOf`/`since`/`diff`/`history`/`transactionLog`
|
||||||
|
reflect single-ops exactly like transacts. `transact()` groups several
|
||||||
|
operations into ONE atomic generation. Single-op history durability is **async
|
||||||
|
group-commit** (the live write is acknowledged immediately; its before-image
|
||||||
|
is batched to disk in one fsync) — a hard crash can lose only the last
|
||||||
|
un-flushed window's *history*, never live data, and a crash mid-flush is
|
||||||
|
recovered by drop-without-restore. A freshly-initialized brain is
|
||||||
|
`generation() === 0` with an empty `transactionLog()` (init-time
|
||||||
|
infrastructure is the un-versioned baseline); the first user write is gen 1.
|
||||||
|
- **`new Brainy({ retention })`** — the retention knob governs auto-compaction on
|
||||||
|
`flush()`/`close()`: **unset → ADAPTIVE** (disk/RAM-pressure byte budget,
|
||||||
|
zero-config; a coordinator can drive it via `brain.setRetentionBudget(bytes)`)
|
||||||
|
· **`'all'`** → unbounded · **`{ maxGenerations?, maxAge?, maxBytes? }`** →
|
||||||
|
explicit caps (reclaim oldest-unpinned while ANY cap is exceeded). Live pins
|
||||||
|
are ALWAYS exempt.
|
||||||
|
- **`brain.compactHistory({ maxGenerations?, maxAge?, maxBytes? })`** — manual
|
||||||
|
reclaim on the same caps. Compaction never breaks a pinned read. `diff`/`since`
|
||||||
|
throw `GenerationCompactedError` below the horizon; `history` truncates to it.
|
||||||
|
|
||||||
Guide: `docs/guides/export-and-import.md`. Tested in `tests/unit/api/data-backup.test.ts`.
|
The precise guarantees are documented in:
|
||||||
|
|
||||||
---
|
- [docs/concepts/consistency-model.md](docs/concepts/consistency-model.md) —
|
||||||
|
the guarantees, each proven by a dedicated test in
|
||||||
|
`tests/integration/db-mvcc.test.ts`
|
||||||
|
- [docs/guides/snapshots-and-time-travel.md](docs/guides/snapshots-and-time-travel.md)
|
||||||
|
— the recipes: backup, restore, time-travel debugging, what-if, audit trails
|
||||||
|
- [docs/ADR-001-generational-mvcc.md](docs/ADR-001-generational-mvcc.md) — the
|
||||||
|
design record: persisted layout, commit protocol, crash recovery, proof table
|
||||||
|
|
||||||
## v7.31.8 — 2026-06-16
|
### Portable export & import (PortableGraph v1)
|
||||||
|
|
||||||
**Affected products:** every consumer on a bare VM or `mmap-filesystem` storage — **upgrade recommended** (Memory, Workshop on 7.31.x; Venue on 7.28.x). Two platform-wide production fixes. Drop-in; no API or data changes.
|
A portable, versioned graph format that serializes part or all of a brain to a
|
||||||
|
single JSON document and restores it — the cross-environment, cross-version
|
||||||
|
(7.x↔8.0), partial-or-whole companion to the native `persist()` snapshot.
|
||||||
|
|
||||||
### Fix 1 — auto query-limit cap no longer collapses on a healthy VM (the 500s)
|
```ts
|
||||||
|
// Export is a method on the immutable Db, so it composes with now()/asOf()/with()
|
||||||
|
const graph = await brain.export({ collection: id }, { includeVectors: true })
|
||||||
|
await otherBrain.import(graph, { onConflict: 'merge' }) // dedup-by-id
|
||||||
|
|
||||||
`new Brainy()`'s auto-configured `maxQueryLimit` could collapse to ~1000 on a perfectly healthy box, throwing `exceeds the auto-configured query limit` on legitimate queries (e.g. `find({ limit: 2400 })`) → HTTP 500s. Root cause: detection read `os.freemem()` (kernel MemFree), which **excludes reclaimable page cache** — on a long-running box using mmap-filesystem storage the page cache grows until MemFree is a thin sliver (tens of MB) even though several GB are actually available, so `floor(freemem / 25KB)·1000` floored the cap to ~1000.
|
;(await brain.asOf(gen)).export(sel) // time-travel export
|
||||||
|
brain.now().with(ops).export(sel) // what-if export
|
||||||
|
```
|
||||||
|
|
||||||
- Memory detection now reads **`/proc/meminfo` `MemAvailable`** (the figure `free -h` shows — counts reclaimable cache), falling back to `os.freemem()` only off-Linux.
|
- **`brain.export(selector?, options?)` / `db.export(...)`** → a versioned
|
||||||
- A **10,000 floor** on auto-detected caps (container / free-memory branches) so a detection miss can never silently collapse the cap. Explicit `maxQueryLimit` / `reservedQueryMemory` settings are honored as-is (never floored).
|
`PortableGraph` document. Selectors reuse `find()`'s grammar: `{ ids }`,
|
||||||
|
`{ collection }` (alias `memberOf`, transitive `Contains`),
|
||||||
|
`{ connected: { from, depth } }`, `{ vfsPath }`, predicate
|
||||||
|
(`{ type, subtype, where, service }`), or the whole brain (omit) — and they
|
||||||
|
compose. Options: `includeVectors`, `includeContent` (VFS file bytes),
|
||||||
|
`includeSystem`, `edges: 'induced' | 'incident' | 'none'`.
|
||||||
|
- **`brain.import(graph, options?)`** is polymorphic: a `PortableGraph` document is
|
||||||
|
restored as **one atomic transaction** (`onConflict: 'merge' | 'replace' | 'skip'`,
|
||||||
|
`reembed: 'auto' | 'never'`, `remapIds` for cloning); a file/buffer routes to the
|
||||||
|
existing CSV/PDF/Excel/JSON ingestion. No migration for ingestion callers.
|
||||||
|
- **`validatePortableGraph(data)`** — dry-run structural/version/endpoint check before import.
|
||||||
|
- **Format** (`format:'brainy-portable-graph'`, `formatVersion: 1`) is identical on
|
||||||
|
7.x and 8.0; standard fields (`subtype`/`visibility`/`data`/…) are top-level,
|
||||||
|
`metadata` is custom-only. Current-state (no generation history — that lives in
|
||||||
|
`persist()`). Types exported from the package root.
|
||||||
|
- **Naming:** the type is `PortableGraph` (with `PortableGraphEntity` /
|
||||||
|
`PortableGraphRelation`) — it is the portable interchange form of a graph, not a
|
||||||
|
backup (that role is `persist()`/`load()`). Renamed from the short-lived
|
||||||
|
`BackupData`/`'brainy-backup'` (introduced in 7.32.0, never adopted) with no
|
||||||
|
compatibility shim.
|
||||||
|
- Guide: [docs/guides/export-and-import.md](docs/guides/export-and-import.md).
|
||||||
|
|
||||||
No action beyond upgrading. A `MEMORY_LIMIT` override (if you set one as a workaround) is still honored.
|
### Aggregation: distinctCount over any value type
|
||||||
|
|
||||||
### Fix 2 — native mmap vector fast-path re-engages (cold-start 57s → instant)
|
`distinctCount` now counts distinct values of **any** type (strings, booleans,
|
||||||
|
numbers) — previously it silently returned `0` for non-numeric fields, missing its
|
||||||
|
primary use (distinct categories / users / tags). `percentile` (with `p`; median =
|
||||||
|
`p: 0.5`), `stddev`, and `variance` are exact and delete-safe alongside the
|
||||||
|
write-time `sum`/`count`/`avg`/`min`/`max`.
|
||||||
|
|
||||||
`FileSystemStorage` now exposes a public `rootDirectory` getter. The optional native vector provider feature-detects `storage.rootDirectory` to enable its memory-mapped graph fast path; brainy stored the path as the protected `rootDir`, so the gate silently failed and every cold start rebuilt the index via per-entity disk reads (~57s for a 283 MB brain on a measured deployment). With the getter, the mmap path engages and cold start becomes a near-instant memory-mapped load.
|
### Removed surfaces and their replacements
|
||||||
|
|
||||||
Self-heals after the first post-upgrade flush writes the mmap file — the first restart after upgrading rebuilds once, then every subsequent boot is fast. Benefits the full deployed matrix (the gate exists in the native provider versions Memory/Workshop and Venue run).
|
| Removed in 8.0 | Replacement |
|
||||||
|
|---|---|
|
||||||
|
| `brain.fork(name)` | Speculation: `db.with(ops)` (in-memory). Long-lived writable copy: `brain.restore()` a persisted snapshot into a fresh data directory. |
|
||||||
|
| `brain.checkout(branch)` | Open the snapshot you want — `Brainy.load(path)` read-only, or restore into its own directory. No in-place switching: every handle always sees one unambiguous store. |
|
||||||
|
| `brain.listBranches()` / `brain.getCurrentBranch()` / `brain.deleteBranch()` | A "branch" is now a name → snapshot-path mapping owned by your application. |
|
||||||
|
| `brain.commit({ message })` | `brain.transact(ops, { meta })` — every batch is an atomic, logged, time-travelable commit; audit fields live in the database, not in commit messages. |
|
||||||
|
| `brain.getHistory()` / `brain.streamHistory()` | `brain.transactionLog({ limit })` + `db.since(olderDb)`. |
|
||||||
|
| `brain.versions.*` (`save` / `list` / `compare` / `restore` / `prune`) | A pinned `Db` or persisted snapshot captures *every* entity at that moment; `asOf()` reads any entity's past state; `restore()` for whole-store rollback. |
|
||||||
|
| `brain.data()` (DataAPI: `clear` / `import` / `export` / `getStats`) | `brain.clear()`, `brain.import()`, `db.persist(path)` (the full-fidelity export format), `brain.restore(path, { confirm: true })`, `brain.stats()`. |
|
||||||
|
| Cloud + browser storage adapters (`s3` / `gcs` / `r2` / `opfs` storage types, Azure Blob, plus the `storage.branch` option) | `filesystem` and `memory` only. Cloud backup is now an operator concern: `db.persist()` produces a plain directory — sync it with `gsutil` / `aws s3 cp` / `rclone` / `azcopy`. Passing a removed storage type throws at construction with this exact guidance. |
|
||||||
|
| Browser support | Node.js 22 LTS or Bun ≥ 1.0 (`engines` enforces `node: 22.x`); Deno works through its Node compatibility layer. |
|
||||||
|
| `brain.migrateToDiskAnn()` / `brain.migrateToHnsw()` | Gone — there is one canonical `'vector'` provider slot. A registered native vector provider selects its own operating mode; there is nothing to call. |
|
||||||
|
| Distributed-clustering subsystem — `config.distributed`, the `DistributedRole` enum, the 13 `BRAINY_*` cluster env vars (`BRAINY_DISTRIBUTED`, `BRAINY_ROLE`, `BRAINY_HTTP_PORT`, `BRAINY_WS_PORT`, `BRAINY_DNS`, `BRAINY_SERVICE`, `BRAINY_NAMESPACE`, `BRAINY_CONSENSUS`, `BRAINY_COORDINATOR`, `BRAINY_NODES`, `BRAINY_REPLICAS`, `BRAINY_SHARDS`, `BRAINY_TRANSPORT`), the storage `setDistributedComponents` hook, and the `@soulcraft/brainy/config` preset/augmentation registry | Removed. Brainy 8.0 is a single-process library — there is no coordinator, peer discovery, or consensus to operate. Scale via: the optional native provider (`@soulcraft/cor` — on-disk DiskANN to 10B+ vectors on one machine); per-tenant pools (one instance + storage dir per tenant); and horizontal read scaling (many reader processes against one shared store, single writer). The `mode: 'reader' \| 'writer'` multi-process roles are unchanged. |
|
||||||
|
| CLI `fork` / `branch` / `checkout` / `migrate` | CLI `snapshot <path>` / `restore <path>` / `history` / `generation`. |
|
||||||
|
| `BrainyZeroConfig` type | Removed. 8.0 zero-config is automatic and internal — `new Brainy()` auto-detects storage, derives HNSW quality from `config.vector.recall`, picks `persistMode` from the adapter, and sizes caches to the detected container-memory limit. There is no config-generation type to import. |
|
||||||
|
| `brain.isFullyInitialized()` / `brain.awaitBackgroundInit()` | `await brain.ready`. The built-in filesystem/memory adapters finish initialization synchronously inside `init()`, so a single readiness promise is the whole story — these methods were no-ops once cloud adapters were removed. |
|
||||||
|
|
||||||
**Upgrade:** bump `@soulcraft/brainy` to `7.31.8`. No code or data migration.
|
**Opening a 7.x store auto-migrates it.** 8.0 stores entities at the root; 7.x
|
||||||
|
stored them branch-scoped under `branches/<branch>/`. On first open, 8.0 collapses
|
||||||
|
the **HEAD branch** (`config.storage.branch`, default `main`) to the 8.0 layout in
|
||||||
|
place and rebuilds all derived state — no action required (`autoMigrate` defaults
|
||||||
|
to `true`; set it to `false` to make 8.0 refuse a legacy layout with an explicit
|
||||||
|
error instead). Two caveats:
|
||||||
|
|
||||||
|
- **Non-HEAD branches are not imported.** 8.0 has no COW branches; only the head
|
||||||
|
branch's data is migrated. If you care about other branches, **export each one
|
||||||
|
while still on 7.x** (`fork → export`, or copy its store).
|
||||||
|
- **Back up first for rollback.** The migration mutates the directory in place and
|
||||||
|
8.0 does not keep the old layout — copy the data directory before the first 8.0
|
||||||
|
open if you need to roll back.
|
||||||
|
|
||||||
|
### Renames — find/replace pairs
|
||||||
|
|
||||||
|
Hard renames, no aliases. Every pair below is verified against the 8.0 source.
|
||||||
|
|
||||||
|
| Brainy 7.x | Brainy 8.0 |
|
||||||
|
|---|---|
|
||||||
|
| `brain.delete(id)` | `brain.remove(id)` — matches the transact op vocabulary (`add` / `update` / `remove` / `relate` / `unrelate`) |
|
||||||
|
| `brain.deleteMany(params)` | `brain.removeMany(params)` |
|
||||||
|
| `DeleteManyParams` (type) | `RemoveManyParams` |
|
||||||
|
| `brain.getRelations(paramsOrId)` | `brain.related(paramsOrId)` — the same name a pinned `Db` exposes (`db.related()`) |
|
||||||
|
| `GetRelationsParams` (type) | `RelatedParams` |
|
||||||
|
| CLI `delete <id>` | CLI `remove <id>` (JSON output `{ id, removed: true }`, matching `unrelate`) |
|
||||||
|
| `HnswProvider` (from `@soulcraft/brainy/plugin`) | `VectorIndexProvider` |
|
||||||
|
| `HNSWIndex` (exported class) | `JsHnswVectorIndex` |
|
||||||
|
| Provider keys `'hnsw'` and `'diskann'` | `'vector'` (the only key Brainy consults for the vector index) |
|
||||||
|
| `config.hnsw = { quantization, vectorStorage }` | `config.vector = { recall?, persistMode? }` (shape change — see below) |
|
||||||
|
| `config.vector.quantization` (and 7.x `config.hnsw.quantization`) | **Removed.** The JS vector path is full-precision (exact float32 distances) only — quantization at scale is the native provider's job (e.g. DiskANN PQ). |
|
||||||
|
| `hnswPersistMode: 'immediate' \| 'deferred'` (top level) | `config.vector.persistMode` (auto-selected from the storage adapter when omitted: `'immediate'` on filesystem, `'deferred'` on memory) |
|
||||||
|
| `config.storage.type: 's3' \| 'gcs' \| 'r2' \| 'opfs'` | `'filesystem'` (or `'memory'` / `'auto'`) |
|
||||||
|
| `config.storage.branch` | Removed (no COW branches) |
|
||||||
|
| `brain.stats().indexHealth.hnsw` | `brain.stats().indexHealth.vector` |
|
||||||
|
| Storage adapter contract `saveHNSWData()` / `getHNSWData()` | `saveVectorIndexData()` / `getVectorIndexData()` |
|
||||||
|
| Cache category `'hnsw'` (`CacheProvider` union) | `'vectors'` |
|
||||||
|
| `config.history = { retainGenerations, retainMs, autoCompact }` | `config.retention` — `'all'` \| `'adaptive'` \| `{ maxGenerations?, maxAge?, maxBytes?, budgetBytes?, autoCompact? }`; **unset → adaptive** (was keep-100/7-days). The fields are now **caps**, not floors. |
|
||||||
|
| `compactHistory({ retainGenerations, retainMs })` | `compactHistory({ maxGenerations, maxAge, maxBytes })` — `retainGenerations`→`maxGenerations`, `retainMs`→`maxAge` (now upper-bound CAPS), plus new `maxBytes`. |
|
||||||
|
| `CompactHistoryOptions.retainGenerations` / `.retainMs` | `.maxGenerations` / `.maxAge` (+ `.maxBytes`) |
|
||||||
|
|
||||||
|
Mechanical renames as one command (run from your repo root, review the diff):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
find . -name '*.ts' -not -path '*/node_modules/*' -print0 | xargs -0 sed -i \
|
||||||
|
-e 's/\bHnswProvider\b/VectorIndexProvider/g' \
|
||||||
|
-e 's/\bHNSWIndex\b/JsHnswVectorIndex/g' \
|
||||||
|
-e 's/\bsaveHNSWData\b/saveVectorIndexData/g' \
|
||||||
|
-e 's/\bgetHNSWData\b/getVectorIndexData/g' \
|
||||||
|
-e 's/indexHealth\.hnsw\b/indexHealth.vector/g' \
|
||||||
|
-e "s/registerProvider('hnsw'/registerProvider('vector'/g" \
|
||||||
|
-e "s/registerProvider('diskann'/registerProvider('vector'/g" \
|
||||||
|
-e "s/getProvider('hnsw'/getProvider('vector'/g" \
|
||||||
|
-e "s/getProvider('diskann'/getProvider('vector'/g" \
|
||||||
|
-e 's/\.getRelations(/.related(/g' \
|
||||||
|
-e 's/\bGetRelationsParams\b/RelatedParams/g' \
|
||||||
|
-e 's/\.deleteMany(/.removeMany(/g' \
|
||||||
|
-e 's/\bDeleteManyParams\b/RemoveManyParams/g'
|
||||||
|
```
|
||||||
|
|
||||||
|
`delete()` → `remove()` is deliberately **not** in the snippet: `.delete(` is
|
||||||
|
too common (`Map`/`Set`/storage adapters) for a blind sed. Find your
|
||||||
|
`brain.delete(...)` call sites and rename them by hand.
|
||||||
|
|
||||||
|
The config shape changes need a human (they are not 1:1 textual):
|
||||||
|
|
||||||
|
```ts
|
||||||
|
// 7.x
|
||||||
|
new Brainy({
|
||||||
|
hnswPersistMode: 'deferred',
|
||||||
|
hnsw: { quantization: { enabled: true, bits: 8, rerankMultiplier: 3 } }
|
||||||
|
})
|
||||||
|
|
||||||
|
// 8.0 — config.vector is { recall?, persistMode? }
|
||||||
|
new Brainy({
|
||||||
|
vector: {
|
||||||
|
recall: 'balanced', // 'fast' | 'balanced' | 'accurate'
|
||||||
|
persistMode: 'deferred' // optional; auto-selected from storage when omitted
|
||||||
|
}
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
`config.vector.recall` replaces the algorithm-internal HNSW knobs: `'balanced'`
|
||||||
|
(the default) maps to exactly the 7.x default parameters, so an upgrade without
|
||||||
|
explicit knobs changes nothing about search behaviour. `config.vector.quantization`
|
||||||
|
and `config.hnsw.vectorStorage` are removed: the JS vector path now computes exact
|
||||||
|
float32 distances throughout (no rerank/approximate branch), which is what made
|
||||||
|
its quantization a memory *increase* — it stored both the full and the quantized
|
||||||
|
vectors in RAM. Quantization at scale belongs to the native provider's own PQ.
|
||||||
|
The on-disk vector index file names are **unchanged** (e.g. `hnsw-system.json`) —
|
||||||
|
existing filesystem stores need no data migration for this rename; only the API
|
||||||
|
surface moved.
|
||||||
|
|
||||||
|
### Behavior changes
|
||||||
|
|
||||||
|
- **`subtype` is required by default.** 7.30's opt-in strict mode is now the
|
||||||
|
default: every `add()` / `addMany()` / `update()` / `relate()` /
|
||||||
|
`relateMany()` / `updateRelation()` rejects writes whose type carries no
|
||||||
|
non-empty `subtype` (`src/brainy.ts:9759` — `requireSubtype ?? true`).
|
||||||
|
Escape hatches: `requireSubtype: false` (last-resort opt-out for legacy
|
||||||
|
data) or `requireSubtype: { except: [NounType.Thing, ...] }` (per-type
|
||||||
|
allowlist). Per-type rules registered via `brain.requireSubtype(type, opts)`
|
||||||
|
still compose. `brain.audit()` reports entries missing a subtype and the new
|
||||||
|
`brain.fillSubtypes(rules)` migration helper backfills them — one rule per
|
||||||
|
NounType/VerbType (literal default or per-entry function), applied only to
|
||||||
|
entries still missing a subtype, returning
|
||||||
|
`{ scanned, filled, skipped, errors, byType }`. Idempotent: re-running fills
|
||||||
|
nothing, so a crashed run is resumed by running it again.
|
||||||
|
- **Verb ids are Brainy-generated UUIDs — by contract.** `relate()` now throws
|
||||||
|
a teaching error if a caller passes an `id` field
|
||||||
|
(`src/utils/paramValidation.ts:526`). In 7.x a supplied id was silently
|
||||||
|
ignored (a generated UUID was used anyway); 8.0 says so out loud, because
|
||||||
|
graph-index providers key verb-int interning on the raw UUID bytes.
|
||||||
|
- **`update({ vector })` is honored on its own.** 7.x only applied a supplied
|
||||||
|
pre-computed vector when `data` was also passed (it was silently dropped
|
||||||
|
otherwise). 8.0 applies an explicit `params.vector` with dimension
|
||||||
|
validation (`src/brainy.ts:1702-1713`; proven by
|
||||||
|
`tests/unit/brainy/update.test.ts:256`).
|
||||||
|
- **`brain.clear()` re-resolves all indexes exactly as `init()` does** —
|
||||||
|
including plugin-provided vector/metadata/entityIdMapper factories and VFS
|
||||||
|
root re-creation. In 7.x, clearing a plugin-accelerated brain could leave
|
||||||
|
the metadata index rebuilt without its native id-mapper wiring.
|
||||||
|
- **`find({ connected: { …, subtype } })` with `depth > 1` now works.** 7.30
|
||||||
|
threw `NOT_SUPPORTED` for multi-hop subtype-filtered traversal; 8.0
|
||||||
|
implements the BFS in the JS graph index and routes to a native provider
|
||||||
|
when one is registered.
|
||||||
|
- **Every write advances the generation clock; only `transact()` writes
|
||||||
|
history.** Single-operation writes (`add` / `update` / `remove` / `relate`
|
||||||
|
outside `transact()`) bump `brain.generation()` so watermarks and CAS stay
|
||||||
|
sound, but they do not stage historical records — they remain visible
|
||||||
|
through earlier pins and are not reported by `db.since()`. Writes you want
|
||||||
|
to travel back through go through `transact()`. This is the documented
|
||||||
|
contract, stated rather than papered over.
|
||||||
|
- **Reserved fields have one canonical location — enforced at every layer.**
|
||||||
|
The Brainy-owned field names (`RESERVED_ENTITY_FIELDS`:
|
||||||
|
`noun`/`subtype`/`createdAt`/`updatedAt`/`confidence`/`weight`/`service`/
|
||||||
|
`data`/`createdBy`/`_rev`; verb mirror `RESERVED_RELATION_FIELDS` with
|
||||||
|
`verb` for the type key) are now (1) a **compile error** inside any
|
||||||
|
`metadata` param — `add`/`update`/`relate`/`updateRelation` and the
|
||||||
|
matching `transact()` ops; (2) **normalized at write time** for untyped
|
||||||
|
callers — user-settable fields remap to their dedicated top-level param
|
||||||
|
(top-level wins; `update({metadata:{confidence}})` no longer silently
|
||||||
|
no-ops, closing a 7.x trap), system-managed fields drop with a one-shot
|
||||||
|
warning naming the right path; (3) **split at read time** through one
|
||||||
|
canonical helper, so `entity.metadata` / `relation.metadata` contain ONLY
|
||||||
|
custom fields on every read path — `get`, `find`, `related` (several
|
||||||
|
paginated/by-source/by-target paths previously echoed the full stored
|
||||||
|
record, including the `verb` type key, inside `metadata`), batch reads,
|
||||||
|
and historical `asOf()` reads. `related()` results now also surface
|
||||||
|
`confidence`/`updatedAt` top-level, and `updateRelation()` no longer
|
||||||
|
erases a relationship's `service`/`createdBy`. See "Reserved fields" in
|
||||||
|
`docs/concepts/consistency-model.md`.
|
||||||
|
- **Named errors for not-found contract failures.** `update()`, `relate()`,
|
||||||
|
`updateRelation()`, `similar({ to: id })`, `transact()` planning, and
|
||||||
|
speculative `db.with()` planning now throw `EntityNotFoundError` /
|
||||||
|
`RelationNotFoundError` (both exported from the package root, both carrying
|
||||||
|
the missing `id` as a field) instead of a generic `Error`. Message texts are
|
||||||
|
unchanged, so existing `/not found/` matching keeps working — `instanceof`
|
||||||
|
is now the supported way to branch.
|
||||||
|
- **Unchanged from 7.31:** per-entity `_rev`, `update({ ifRev })`
|
||||||
|
(`RevisionConflictError`), and `add({ ifAbsent })` work exactly as before,
|
||||||
|
and `ifRev` is also accepted on `transact()` update operations (a conflict
|
||||||
|
rejects the whole batch).
|
||||||
|
|
||||||
|
### For provider and plugin authors
|
||||||
|
|
||||||
|
The provider contracts in `@soulcraft/brainy/plugin` (`src/plugin.ts`) changed
|
||||||
|
shape:
|
||||||
|
|
||||||
|
- **`GraphIndexProvider` speaks BigInt at the boundary.** Reads take entity
|
||||||
|
ints and return entity/verb ints as `bigint[]`; the coordinator owns all
|
||||||
|
UUID ↔ int conversion. `addVerb(verb, sourceInt, targetInt)` receives the
|
||||||
|
resolved endpoint ints (also mirrored on the new optional
|
||||||
|
`GraphVerb.sourceInt` / `GraphVerb.targetInt` fields) and returns the
|
||||||
|
interned verb int. The batch reverse resolver
|
||||||
|
`verbIntsToIds(verbInts: bigint[]): Promise<(string | null)[]>` is
|
||||||
|
**required** — the provider owns durable verb-int interning; Brainy keeps
|
||||||
|
only a bounded in-memory warm cache. Implementations may stay u32 internally
|
||||||
|
(`Number(bigint)` narrowing is lossless under the `EntityIdSpaceExceeded`
|
||||||
|
guard) but must speak the bigint contract.
|
||||||
|
- **`VersionedIndexProvider`** is a new optional 4-method capability —
|
||||||
|
`generation()`, `isGenerationVisible(g)`, `pin(g)`, `release(g)`, BigInt
|
||||||
|
generations — feature-detected on every registered index provider. Providers
|
||||||
|
are *post-commit appliers*: the storage-record commit is the source of
|
||||||
|
truth; on open, a provider behind the committed watermark replays the gap or
|
||||||
|
requests a rebuild. Explicit pins override any time-based snapshot retention
|
||||||
|
the provider has. Speculative `db.with()` overlays never reach providers. A
|
||||||
|
provider implementing this serves historical reads with **no rebuild** —
|
||||||
|
the open-core materializer is the correctness baseline, the provider is the
|
||||||
|
accelerator.
|
||||||
|
- **The vector index registers under `'vector'`** and implements
|
||||||
|
`VectorIndexProvider`. The `'hnsw'` and `'diskann'` keys are retired and
|
||||||
|
never looked up. `CacheProvider`'s category union uses `'vectors'` in place
|
||||||
|
of `'hnsw'`.
|
||||||
|
|
||||||
|
### Scale and cost — how the mechanisms behave
|
||||||
|
|
||||||
|
Mechanism descriptions, not benchmark numbers (none are published for 8.0 yet):
|
||||||
|
|
||||||
|
- `brain.now()` pins in O(1) and adds **zero read overhead until history
|
||||||
|
actually moves** — while nothing has committed past the pin, every read
|
||||||
|
delegates to the live fast paths.
|
||||||
|
- A `transact()` commit pays O(ids touched) extra writes (before-images +
|
||||||
|
delta + manifest) — never O(store size). Single-operation writes pay an
|
||||||
|
in-memory counter bump with coalesced persistence.
|
||||||
|
- `db.persist()` on filesystem storage is a hard-link farm: snapshot creation
|
||||||
|
copies no entity data and the snapshot shares disk space with the source.
|
||||||
|
Cross-device targets fall back to per-file byte copies; persisting an
|
||||||
|
in-memory brain serializes a real, durable, loadable store.
|
||||||
|
- Historical **index-accelerated** queries (semantic search, traversal,
|
||||||
|
cursors, aggregation) on the open-core path pay a one-time O(n at the
|
||||||
|
pinned generation) in-memory index materialization per `Db`, cached until
|
||||||
|
`release()`. Record-path reads (`get`, metadata `find`, filter `related`)
|
||||||
|
at any reachable generation are served directly from the immutable record
|
||||||
|
layer. A native `VersionedIndexProvider` serves the same reads rebuild-free.
|
||||||
|
|
||||||
|
The full cost model is in
|
||||||
|
[docs/concepts/consistency-model.md](docs/concepts/consistency-model.md).
|
||||||
|
|
||||||
|
### Upgrade checklist (from 7.x)
|
||||||
|
|
||||||
|
1. **While still on 7.x:** back up your data directory (a plain file copy is
|
||||||
|
fine). If you used COW branches, materialize each branch you need — 8.0
|
||||||
|
does not read branch state. If you ran a cloud storage adapter, export your
|
||||||
|
data with 7.x tooling (`(await brain.data()).export()`) — 8.0 opens
|
||||||
|
`filesystem` and `memory` stores only.
|
||||||
|
2. **Runtime:** Node.js 22 LTS or Bun ≥ 1.0.
|
||||||
|
3. **Config:** change removed storage types to `'filesystem'`; delete
|
||||||
|
`storage.branch`; move `config.hnsw` / `hnswPersistMode` to
|
||||||
|
`config.vector` per the shape example above.
|
||||||
|
4. **Renames:** run the sed snippet, then fix the manual shape changes.
|
||||||
|
5. **Removed APIs:** replace `fork` / `checkout` / branch methods / `commit` /
|
||||||
|
`getHistory` / `streamHistory` / `versions` / `data()` per the table above.
|
||||||
|
6. **Subtypes:** if your data predates subtype discipline, start with
|
||||||
|
`requireSubtype: false`, run `brain.audit()`, backfill with
|
||||||
|
`brain.fillSubtypes(rules)` (one rule per type — a literal default or a
|
||||||
|
function deriving the subtype from each entry), re-run `audit()` until
|
||||||
|
`total === 0`, then remove the opt-out so the 8.0 default enforcement
|
||||||
|
protects you going forward.
|
||||||
|
7. **CLI scripts:** `fork` / `branch` / `checkout` / `migrate` →
|
||||||
|
`snapshot <path>` / `restore <path>` / `history` / `generation`.
|
||||||
|
8. **Plugin authors:** apply the contract renames and the BigInt graph
|
||||||
|
contract; optionally implement `VersionedIndexProvider`.
|
||||||
|
9. **Verify, then take your first snapshot:** run your test suite, then
|
||||||
|
`const db = brain.now(); await db.persist('/backups/post-8.0-upgrade');
|
||||||
|
await db.release()`.
|
||||||
|
|
||||||
|
What did **not** change: the core query surface (`find` / `search` / `get` /
|
||||||
|
`add` / `update` / `relate` and friends), the aggregation engine, the VFS,
|
||||||
|
neural extraction, and the on-disk entity/relationship layout — 8.0 creates
|
||||||
|
its MVCC bookkeeping (`_system/generation.json`, `_system/manifest.json`,
|
||||||
|
`_system/tx-log.jsonl`, `_generations/`) alongside the existing files on
|
||||||
|
first use.
|
||||||
|
|
||||||
|
### Native-provider (Cor) compatibility
|
||||||
|
|
||||||
|
Brainy 8.0 pairs with `@soulcraft/cor` 3.0 — the renamed successor to the
|
||||||
|
`@soulcraft/cortex` 2.x native provider — shipping in lockstep. The old
|
||||||
|
`@soulcraft/cortex` 2.x cannot accelerate Brainy 8.0: 8.0 never consults the
|
||||||
|
`'hnsw'` / `'diskann'` provider keys, and the graph/column provider contracts
|
||||||
|
are BigInt at the boundary. Upgrade both packages together (`@soulcraft/brainy@8`
|
||||||
|
+ `@soulcraft/cor@3`).
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -458,8 +846,7 @@ alongside every other metadata field on the existing write paths.
|
||||||
`_rev`, `ifRev`, `RevisionConflictError`, and `ifAbsent` all survive the 8.0 Db
|
`_rev`, `ifRev`, `RevisionConflictError`, and `ifAbsent` all survive the 8.0 Db
|
||||||
redesign unchanged. 8.0 layers `brain.transact(tx, { ifAtGeneration })` for
|
redesign unchanged. 8.0 layers `brain.transact(tx, { ifAtGeneration })` for
|
||||||
whole-tx CAS on top of the same per-entity mechanism — per-entity for single-record
|
whole-tx CAS on top of the same per-entity mechanism — per-entity for single-record
|
||||||
patterns, generation-based for "did the world move under me." See
|
patterns, generation-based for "did the world move under me."
|
||||||
`.strategy/BRAINY-8.0-SUBTYPE-CONTRACT.md` § C-6 (internal).
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -577,8 +964,7 @@ in `validateFindParams()`, both fire before any storage/index/Cortex call. Corte
|
||||||
|
|
||||||
The new `docs/guides/find-limits.md` calls out that 8.0's Datomic-style `Db.find()`
|
The new `docs/guides/find-limits.md` calls out that 8.0's Datomic-style `Db.find()`
|
||||||
may tighten per-call limits; that's a Brainy 8.0 / Cortex 3.0 coordination point
|
may tighten per-call limits; that's a Brainy 8.0 / Cortex 3.0 coordination point
|
||||||
documented in `.strategy/BRAINY-8.0-SUBTYPE-CONTRACT.md` (open question #4 covers
|
whose rollout staging now also covers query limits.
|
||||||
the rollout staging, which now also applies to query limits).
|
|
||||||
|
|
||||||
### What consumers should do
|
### What consumers should do
|
||||||
|
|
||||||
|
|
@ -735,8 +1121,7 @@ Brainy before any native call.
|
||||||
|
|
||||||
- **Native `audit()` proxy.** For billion-scale brains, `audit()` walks every entity (O(N)).
|
- **Native `audit()` proxy.** For billion-scale brains, `audit()` walks every entity (O(N)).
|
||||||
A native implementation reading from a "null-subtype" bitmap in the column store would be
|
A native implementation reading from a "null-subtype" bitmap in the column store would be
|
||||||
O(buckets). Listed as the 6th open question in the
|
O(buckets).
|
||||||
[Brainy 8.0 spec doc](`/media/dpsifr/storage/home/Projects/brainy/.strategy/BRAINY-8.0-SUBTYPE-CONTRACT.md`).
|
|
||||||
- **Strict-mode parity test.** Cortex should mirror Brainy's new
|
- **Strict-mode parity test.** Cortex should mirror Brainy's new
|
||||||
`tests/integration/strict-mode-self-test.test.ts` against their native paths to catch any
|
`tests/integration/strict-mode-self-test.test.ts` against their native paths to catch any
|
||||||
latent bug where native writes bypass JS validation.
|
latent bug where native writes bypass JS validation.
|
||||||
|
|
|
||||||
295
docs/ADR-001-generational-mvcc.md
Normal file
295
docs/ADR-001-generational-mvcc.md
Normal file
|
|
@ -0,0 +1,295 @@
|
||||||
|
# ADR-001: Generational MVCC storage and the immutable Db API
|
||||||
|
|
||||||
|
**Status:** Accepted (ships in 8.0)
|
||||||
|
**Date:** 2026-06-10
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
Before 8.0, Brainy carried two overlapping version-control subsystems: a
|
||||||
|
copy-on-write branching layer (`fork`/`checkout`/`commit`/`branches`) and a
|
||||||
|
separate versioning subsystem (`versions.save/list/compare/restore/prune`),
|
||||||
|
plus a read-only historical adapter for commit-based time travel. Together
|
||||||
|
they were ~5,100 LOC of mechanism for one product need: *read a consistent
|
||||||
|
past state while the store keeps moving, and snapshot/restore cheaply.*
|
||||||
|
|
||||||
|
Neither subsystem gave a precise isolation guarantee. Reads raced in-place
|
||||||
|
JSON overwrites, so a "snapshot" was only as immutable as the bytes it
|
||||||
|
happened to share with the live store.
|
||||||
|
|
||||||
|
8.0 replaces both with **one mechanism**: generational MVCC over immutable,
|
||||||
|
generation-stamped records, exposed through a Datomic-style immutable
|
||||||
|
database value (`Db`). The same model is implemented natively by versioned
|
||||||
|
index providers (LSM snapshots), so semantics are identical on the pure-JS
|
||||||
|
path and the native path.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
### The model
|
||||||
|
|
||||||
|
- A **monotonic u64 generation counter** is the store's logical clock. It
|
||||||
|
advances once per committed `transact()` batch and once per
|
||||||
|
single-operation write (`add`/`update`/`remove`/`relate`/…), so
|
||||||
|
`brain.generation()` is always a meaningful watermark. It is persisted in
|
||||||
|
`_system/generation.json` and never reissued for anything durable.
|
||||||
|
- `brain.now()` **pins** the current generation in O(1) and returns a `Db` —
|
||||||
|
an immutable view. Pins are refcounted; `db.release()` (with a
|
||||||
|
`FinalizationRegistry` backstop for leaked values) ends the pin.
|
||||||
|
- `brain.transact(ops, { meta, ifAtGeneration })` commits a declarative
|
||||||
|
batch atomically as **exactly one generation**, with whole-store
|
||||||
|
compare-and-swap (`ifAtGeneration` → `GenerationConflictError`) and
|
||||||
|
reified transaction metadata appended to `_system/tx-log.jsonl`.
|
||||||
|
- `brain.asOf(generation | Date | snapshotPath)` opens past state;
|
||||||
|
`db.with(ops)` layers a speculative in-memory overlay (never touching
|
||||||
|
disk, the counter, or index providers); `db.persist(path)` cuts an
|
||||||
|
instant snapshot; `brain.restore(path, { confirm: true })` replaces state
|
||||||
|
from one; `Brainy.load(path)` opens a snapshot read-only with the full
|
||||||
|
query surface.
|
||||||
|
|
||||||
|
### Persisted layout
|
||||||
|
|
||||||
|
All paths are storage-root-relative:
|
||||||
|
|
||||||
|
```
|
||||||
|
_system/generation.json { generation, updatedAt } atomic tmp+rename
|
||||||
|
_system/manifest.json { version, generation, atomic tmp+rename
|
||||||
|
committedAt, horizon } (the commit point)
|
||||||
|
_system/tx-log.jsonl one line per committed append-only
|
||||||
|
transact: { generation,
|
||||||
|
timestamp, meta? }
|
||||||
|
_generations/<N>/tx.json the generation-N delta: immutable
|
||||||
|
touched noun/verb ids + meta
|
||||||
|
_generations/<N>/prev/<id>.json before-image of <id> as of immutable
|
||||||
|
commit N (raw stored bytes;
|
||||||
|
null parts = file was absent)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Why per-generation deltas instead of a global `id → latest generation`
|
||||||
|
map in the manifest:** a global map makes every commit O(all ids) — the
|
||||||
|
whole map must be rewritten to swap it atomically. The delta layout makes a
|
||||||
|
commit O(ids touched) and keeps the manifest a fixed-size watermark, while
|
||||||
|
point-in-time resolution stays correct (see "Read resolution" below). The
|
||||||
|
trade is that resolution at a pinned generation scans the deltas of later
|
||||||
|
commits — bounded by the number of commits since the pin, which is exactly
|
||||||
|
the window compaction keeps short.
|
||||||
|
|
||||||
|
Before-images are deliberately the *only* per-id records. They serve both
|
||||||
|
roles the layer needs — the crash-recovery undo log and the point-in-time
|
||||||
|
read source. After-images would duplicate state that is already readable
|
||||||
|
(the canonical entity files hold the latest bytes; earlier states resolve
|
||||||
|
from later before-images) and would double record I/O per commit.
|
||||||
|
|
||||||
|
### Commit protocol (durability)
|
||||||
|
|
||||||
|
`transact()` commits under a store-wide mutex:
|
||||||
|
|
||||||
|
1. **CAS check.** A stale `ifAtGeneration` throws `GenerationConflictError`
|
||||||
|
before anything is staged.
|
||||||
|
2. **Reserve** generation `N` (counter increment).
|
||||||
|
3. **Stage the undo log:** write the before-image of every touched id plus
|
||||||
|
`tx.json` under `_generations/N/`, then **fsync** the files and their
|
||||||
|
directories. From this point, any crash is recoverable to the exact
|
||||||
|
pre-transaction bytes.
|
||||||
|
4. **Execute** the planned batch through the TransactionManager (which has
|
||||||
|
its own operation-level rollback for in-flight failures).
|
||||||
|
5. **Commit point:** persist the counter, then write `_system/manifest.json`
|
||||||
|
via atomic tmp+rename and fsync it. The rename *is* the commit: a
|
||||||
|
generation directory is committed if and only if `N ≤
|
||||||
|
manifest.generation`.
|
||||||
|
6. Append the tx-log line (advisory metadata — a crash between 5 and 6
|
||||||
|
keeps the transaction).
|
||||||
|
|
||||||
|
**Crash recovery (on open):** any `_generations/<N>` directory with
|
||||||
|
`N > manifest.generation` is an uncommitted transaction. Its before-images
|
||||||
|
are restored to the canonical paths (idempotently — recovery itself can
|
||||||
|
crash and rerun) and the directory is removed. Because recovery runs before
|
||||||
|
any index is built, and a recovery that rolled something back forces a full
|
||||||
|
index rebuild, derived indexes never observe rolled-back state. Reader-mode
|
||||||
|
instances skip recovery (readers never write; the next writer repairs).
|
||||||
|
|
||||||
|
A failed (non-crash) transaction takes the same staging directory down the
|
||||||
|
abort path: the TransactionManager rolls back applied operations, the
|
||||||
|
staging directory is removed, and the generation reservation is returned —
|
||||||
|
a failed batch leaves the generation counter unchanged.
|
||||||
|
|
||||||
|
### Read resolution at a pinned generation
|
||||||
|
|
||||||
|
The state of id X at pinned generation G is:
|
||||||
|
|
||||||
|
- the before-image stored by the **first committed generation after G that
|
||||||
|
touched X**, or
|
||||||
|
- the live canonical bytes, when nothing after G touched X.
|
||||||
|
|
||||||
|
While nothing has committed past G, *every* read on the `Db` delegates to
|
||||||
|
the live fast paths untouched — `now()` adds no read overhead until history
|
||||||
|
actually moves.
|
||||||
|
|
||||||
|
**Two read paths, one result set.** `get()`, metadata-level `find()`, and
|
||||||
|
filter-based `related()` resolve directly through the record layer at any
|
||||||
|
reachable pinned generation — no extra cost beyond scanning the deltas of
|
||||||
|
later commits. Index-accelerated dimensions (semantic/vector search, graph
|
||||||
|
traversal, cursors, aggregation) are served by **at-generation index
|
||||||
|
materialization**: the first such query on a historical `Db` copies the
|
||||||
|
exact at-G record set (live bytes for ids untouched since the pin,
|
||||||
|
before-images for the rest; a final reconciliation pass runs under the
|
||||||
|
commit mutex so transactions racing the copy cannot skew it) into an
|
||||||
|
ephemeral in-memory store and opens a read-only engine over it — the same
|
||||||
|
vector/metadata/graph index classes the live brain uses, sharing the host's
|
||||||
|
embedder and aggregate definitions. The handle is cached on the `Db` and
|
||||||
|
freed by `release()`.
|
||||||
|
|
||||||
|
**Cost, stated plainly:** materialization is O(n at G) time and memory,
|
||||||
|
once per `Db`. That is the open-core price of historical index queries. A
|
||||||
|
native `VersionedIndexProvider` (`isGenerationVisible()` + pins over
|
||||||
|
retained LSM segments) serves the same reads with no rebuild at all — the
|
||||||
|
materializer is the correctness baseline, the provider is the accelerator.
|
||||||
|
|
||||||
|
**The one remaining boundary.** Speculative `with()` overlays throw
|
||||||
|
`SpeculativeOverlayError` for index-accelerated queries and `persist()`:
|
||||||
|
overlay entities carry no embeddings (`with()` never invokes the embedder),
|
||||||
|
so a "full" index query over an overlay would silently exclude the
|
||||||
|
overlay's own entities. Commit with `transact()` to get the full surface.
|
||||||
|
|
||||||
|
**History granularity (Model-B).** EVERY write is its own immutable generation
|
||||||
|
— `transact()` batches AND single-operation `add`/`update`/`remove`/`relate`.
|
||||||
|
Single-ops stage a before-image and are reported by `db.since()`/`asOf()`/
|
||||||
|
`diff()`/`history()` exactly like transacts; a pin always freezes against later
|
||||||
|
writes. `transact()` groups several operations into ONE atomic generation.
|
||||||
|
|
||||||
|
Single-op history durability is **async group-commit**: the live write hits
|
||||||
|
canonical storage immediately (acknowledged), while its before-image is buffered
|
||||||
|
and persisted to disk in one batched fsync on a size/timer trigger (or forced by
|
||||||
|
`flush()`/`close()`/`transact()`/`compactHistory()`). The buffer participates in
|
||||||
|
point-in-time resolution exactly like on-disk generations, so the synchronous
|
||||||
|
`now()` freezes with no forced flush. A hard crash before the flush loses only
|
||||||
|
the buffered *history* of the last window — never live data — and a crash
|
||||||
|
*mid-flush* is recovered by **drop-without-restore** (the partial generation's
|
||||||
|
before-images are discarded, never replayed, because the live write was already
|
||||||
|
acknowledged; restoring them would silently revert it).
|
||||||
|
|
||||||
|
### Pinning, retention, compaction
|
||||||
|
|
||||||
|
- Each live `Db` holds one refcounted pin on its generation (plus a
|
||||||
|
`pin(generation)` on every registered `VersionedIndexProvider`, whose
|
||||||
|
explicit pin lifetime overrides any time-based snapshot retention the
|
||||||
|
provider has).
|
||||||
|
- The constructor **`retention`** knob governs auto-compaction (on `flush()`/
|
||||||
|
`close()`): unset → ADAPTIVE (disk/RAM-pressure byte budget, zero-config;
|
||||||
|
driven by a coordinator's `budgetBytes` or a local `os.freemem` probe) ·
|
||||||
|
`'all'` → unbounded · `{ maxGenerations?, maxAge?, maxBytes? }` → explicit
|
||||||
|
CAPS. `compactHistory({ maxGenerations?, maxAge?, maxBytes? })` reclaims
|
||||||
|
manually on the same caps — the oldest unpinned record-sets are reclaimed
|
||||||
|
while ANY supplied cap is exceeded.
|
||||||
|
- A record-set `N` is reclaimed only when `N` is at or below **every** live
|
||||||
|
pin — deleting `N` can only break readers pinned *below* `N`, because
|
||||||
|
resolution reads before-images from generations strictly greater than the
|
||||||
|
pin. Live pins are ALWAYS exempt, in every retention mode.
|
||||||
|
- The manifest records the **horizon** (highest reclaimed generation).
|
||||||
|
Generations below the horizon are unreachable; `asOf()` on them throws
|
||||||
|
`GenerationCompactedError`. The horizon itself stays reachable, resolved
|
||||||
|
from the record-sets above it. To keep a generation readable forever,
|
||||||
|
`persist()` it first — snapshots are self-contained.
|
||||||
|
|
||||||
|
### Snapshots and restore
|
||||||
|
|
||||||
|
`db.persist(path)` flushes indexes, then cuts the snapshot under the
|
||||||
|
store's commit mutex (no commit, compaction, or counter write can
|
||||||
|
interleave). On filesystem storage it is a **hard-link farm**: every data
|
||||||
|
file is immutable-by-rename, so linking is safe — later rewrites swap
|
||||||
|
inodes and the snapshot keeps the old bytes. The two exceptions are handled
|
||||||
|
explicitly: the append-in-place tx-log is byte-copied, and process-local
|
||||||
|
lock state is excluded. Cross-device targets (and filesystems that refuse
|
||||||
|
links) fall back to per-file byte copies. In-memory stores serialize to the
|
||||||
|
same directory layout, so persisting a memory brain produces a real,
|
||||||
|
durable, loadable store.
|
||||||
|
|
||||||
|
`persist()` requires the view to still be the store's latest generation
|
||||||
|
(a snapshot captures current bytes); a view that history has moved past
|
||||||
|
throws rather than persisting the wrong state.
|
||||||
|
|
||||||
|
`restore(path, { confirm: true })` replaces the store's contents from a
|
||||||
|
snapshot via byte copy (never links — the snapshot stays independent),
|
||||||
|
reloads all adapter-internal derived state, rebuilds all indexes, and
|
||||||
|
floors the generation counter at its pre-restore value so observed
|
||||||
|
generation numbers are never reissued. Live pins do not survive a restore;
|
||||||
|
a warning is logged when any exist.
|
||||||
|
|
||||||
|
### Versioned index providers
|
||||||
|
|
||||||
|
Native index providers may implement the optional 4-method
|
||||||
|
`VersionedIndexProvider` capability (`generation()`,
|
||||||
|
`isGenerationVisible()`, `pin()`, `release()` — BigInt generations at the
|
||||||
|
boundary). The locked consistency model: providers are **post-commit
|
||||||
|
appliers**. The storage-record commit is the source of truth; provider
|
||||||
|
index state is derived. On open, a provider behind the committed watermark
|
||||||
|
replays the gap from storage (or requests a rebuild) — there are no
|
||||||
|
provider rollback hooks, because uncommitted transactions are repaired at
|
||||||
|
the storage layer before any index opens. Speculative `with()` overlays
|
||||||
|
never reach providers.
|
||||||
|
|
||||||
|
## Guarantees (and their proofs)
|
||||||
|
|
||||||
|
Each stated guarantee has a test that proves it, not merely exercises it
|
||||||
|
(`tests/integration/db-mvcc.test.ts`, plus
|
||||||
|
`tests/unit/db/generationStore.test.ts` for the record layer in isolation):
|
||||||
|
|
||||||
|
| Guarantee | Proof |
|
||||||
|
|---|---|
|
||||||
|
| Snapshot isolation: a pinned `Db` reads exactly its pinned state, forever | proof 1 (200 mutations, including deletes, against a pinned view) |
|
||||||
|
| Atomicity: a failing batch applies nothing; generation unchanged | proofs 2a/2b/2c (plan-time failure, injected execution-phase storage failure, `ifRev` conflict) |
|
||||||
|
| Whole-store CAS | proof 3 (`ifAtGeneration` success + conflict with exact expected/actual) |
|
||||||
|
| Snapshot integrity under source mutation (hard-link safety) | proofs 4a/4b/4c |
|
||||||
|
| Compaction never breaks a pinned read; release enables reclaim | proof 5 |
|
||||||
|
| `with()` overlays touch nothing durable | proof 6 |
|
||||||
|
| Generation monotonicity across close/reopen | proof 7 |
|
||||||
|
| Crash before the manifest rename recovers to exact pre-transaction state through the real recovery path | proof 8 (fault injection that skips abort cleanup, exactly as a dead process would) |
|
||||||
|
| Balanced provider pin/release lockstep | proof 9 |
|
||||||
|
|
||||||
|
One deliberate softness: single-operation generation bumps persist the
|
||||||
|
counter coalesced (per write burst), not per write. Durable artifacts —
|
||||||
|
records, manifests, snapshots — always persist the counter synchronously at
|
||||||
|
their own commit points, so a crash inside the coalescing window can lose
|
||||||
|
only counter values that nothing durable ever referenced.
|
||||||
|
|
||||||
|
## Failure modes
|
||||||
|
|
||||||
|
| Failure | Outcome |
|
||||||
|
|---|---|
|
||||||
|
| Crash before staging completes | Partial staging directory > manifest watermark → removed on next open; canonical state untouched. |
|
||||||
|
| Crash after staging, before/during batch execution | Before-images restored on next open; indexes rebuilt; byte-identical pre-transaction state. |
|
||||||
|
| Crash after execution, before manifest rename | Same as above — the rename is the only commit point. |
|
||||||
|
| Crash after manifest rename, before tx-log append | Transaction kept (committed); tx-log misses one advisory line; `asOf(Date)` resolution for that commit falls back to neighboring entries. |
|
||||||
|
| Batch fails mid-execution (no crash) | TransactionManager operation rollback + staging-directory removal + reservation return; generation unchanged. |
|
||||||
|
| `asOf()` below the compaction horizon | `GenerationCompactedError` — explicit, never partial data. |
|
||||||
|
| Index-accelerated query on a `with()` overlay | `SpeculativeOverlayError` — explicit, never silently-incomplete results (overlay entities carry no embeddings). |
|
||||||
|
| `persist()` of a view history has moved past | `GenerationConflictError` — a snapshot captures current bytes; persist before further writes. |
|
||||||
|
| Torn trailing tx-log line (crashed append) | Tolerated; unparseable lines are skipped by readers. |
|
||||||
|
|
||||||
|
## Lineage
|
||||||
|
|
||||||
|
The design is an assembly of well-understood prior art, chosen for being
|
||||||
|
boring where it counts:
|
||||||
|
|
||||||
|
- **Datomic** — the database-as-a-value: an immutable `Db` you query, with
|
||||||
|
`with()` for speculation and reified transaction metadata instead of
|
||||||
|
commit messages.
|
||||||
|
- **LMDB** — reader pins: readers never block writers; a reader's view
|
||||||
|
stays valid because nothing overwrites the pages (here: records) it
|
||||||
|
references; reclamation waits for the last reader.
|
||||||
|
- **LSM trees / Cassandra** — immutable segments make snapshots hard links
|
||||||
|
and make compaction a retention policy instead of a locking problem.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
- One mechanism replaces the COW and versioning subsystems (their removal
|
||||||
|
is the companion change to this ADR).
|
||||||
|
- In-place branch switching (`checkout`) is gone by design; the replacement
|
||||||
|
is opening a persisted snapshot as a separate instance — a name→path
|
||||||
|
mapping where a product needs named branches.
|
||||||
|
- Every commit pays O(ids touched) extra writes (before-images + delta +
|
||||||
|
manifest). Single-operation writes pay only an in-memory counter bump
|
||||||
|
with coalesced persistence.
|
||||||
|
- The full query surface works at every reachable pinned generation.
|
||||||
|
Record-path reads (`get`, metadata `find`, filter `related`) are
|
||||||
|
effectively free; index-accelerated historical queries pay a one-time
|
||||||
|
O(n at G) materialization per `Db` on the open-core path (freed on
|
||||||
|
`release()`), and run rebuild-free on a native `VersionedIndexProvider`.
|
||||||
339
docs/BATCHING.md
339
docs/BATCHING.md
|
|
@ -5,31 +5,24 @@ public: true
|
||||||
category: guides
|
category: guides
|
||||||
template: guide
|
template: guide
|
||||||
order: 5
|
order: 5
|
||||||
description: Eliminate N+1 query patterns with batchGet() and storage-level batch APIs. Achieve 90%+ faster cloud storage access — from 12.7 seconds down to under 1 second.
|
description: Eliminate N+1 query patterns with batchGet() and storage-level batch APIs for fast multi-entity reads against filesystem and memory storage.
|
||||||
next:
|
next:
|
||||||
- api/reference
|
- api/reference
|
||||||
- guides/find-system
|
- guides/find-system
|
||||||
---
|
---
|
||||||
|
|
||||||
# Batch Operations API
|
# Batch Operations API
|
||||||
> **Enterprise Production-Ready** | Zero N+1 Query Patterns | 90%+ Performance Improvement
|
> **Production-Ready** | Zero N+1 Query Patterns
|
||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
Brainy introduces comprehensive batch operations at the storage layer, eliminating N+1 query patterns and dramatically improving performance for VFS operations, relationship queries, and entity retrieval on cloud storage.
|
Brainy provides batch operations at the storage layer to eliminate N+1 query patterns for VFS operations, relationship queries, and entity retrieval.
|
||||||
|
|
||||||
### Problem Solved
|
### Problem Solved
|
||||||
|
|
||||||
**Before optimization:**
|
The naive pattern of looping and calling `brain.get(id)` once per item issues sequential reads through the storage layer. Batched APIs collapse that into a single read pass.
|
||||||
- VFS `getTreeStructure()` on cloud storage: **12.7 seconds** for directory with 12 files
|
|
||||||
- N+1 query pattern: 1 directory query + N individual file queries (22 sequential calls × 580ms latency)
|
|
||||||
|
|
||||||
**After optimization:**
|
**IMPORTANT:** The batch optimizations apply **ONLY to `getTreeStructure()`** at the VFS layer and the explicit `batchGet()` / `getNounMetadataBatch()` calls — not to `readFile()` or individual `get()` operations.
|
||||||
- VFS `getTreeStructure()`: **<1 second** for 12 files (90%+ improvement)
|
|
||||||
- 2-3 batched calls instead of 22 sequential calls
|
|
||||||
- Native cloud storage batch APIs for maximum throughput
|
|
||||||
|
|
||||||
**IMPORTANT:** The batch optimizations apply **ONLY to `getTreeStructure()`**, not to `readFile()` or individual `get()` operations. See changes for comprehensive storage path optimizations.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -54,8 +47,7 @@ results.size // → 3 (number of found entities)
|
||||||
|
|
||||||
**Performance:**
|
**Performance:**
|
||||||
- Memory storage: Instant (parallel reads)
|
- Memory storage: Instant (parallel reads)
|
||||||
- Cloud storage (GCS/S3/Azure): <500ms for 100 entities
|
- Filesystem storage: Parallel reads, scales with available IOPS
|
||||||
- Throughput: 50-200+ entities/second depending on adapter
|
|
||||||
|
|
||||||
**Use Cases:**
|
**Use Cases:**
|
||||||
- Loading multiple entities for display
|
- Loading multiple entities for display
|
||||||
|
|
@ -85,13 +77,13 @@ for (const [id, metadata] of metadataMap) {
|
||||||
**Features:**
|
**Features:**
|
||||||
- ✅ Direct O(1) path construction from ID (no type lookup needed!)
|
- ✅ Direct O(1) path construction from ID (no type lookup needed!)
|
||||||
- ✅ Sharding preservation (all paths include `{shard}/{id}`)
|
- ✅ Sharding preservation (all paths include `{shard}/{id}`)
|
||||||
- ✅ COW-aware (respects branch paths)
|
- ✅ Write-cache coherent (read-after-write consistency)
|
||||||
- ✅ 40x faster than v5.x type-first architecture
|
- ✅ O(1) path construction — eliminates the per-entity type search of the old type-first layout
|
||||||
|
|
||||||
**Performance:**
|
**Performance:**
|
||||||
- ~1ms per 100 entities (consistent, no cache misses!)
|
- Constant-time path construction per ID — no type-cache misses
|
||||||
- Cloud storage: Parallel downloads (100-150 concurrent)
|
- Filesystem: parallel reads bounded by IOPS
|
||||||
- No type search delays - every ID maps directly to storage path
|
- No type search delays — every ID maps directly to storage path
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -129,130 +121,8 @@ for (const [sourceId, verbs] of results) {
|
||||||
- Bulk export of relationship data
|
- Bulk export of relationship data
|
||||||
|
|
||||||
**Performance:**
|
**Performance:**
|
||||||
- Memory storage: <10ms for 1000 relationships
|
- Memory storage: single in-memory pass over the metadata index
|
||||||
- Cloud storage: Batched reads with parallel metadata fetches
|
- Filesystem storage: parallel reads through the metadata index
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 4. `storage.readBatchWithInheritance(paths, targetBranch?)`
|
|
||||||
|
|
||||||
COW-aware batch path resolution with branch inheritance.
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const storage = brain.storage as BaseStorage
|
|
||||||
|
|
||||||
const paths = [
|
|
||||||
'entities/nouns/{shard}/id1/metadata.json',
|
|
||||||
'entities/nouns/{shard}/id2/metadata.json'
|
|
||||||
]
|
|
||||||
|
|
||||||
// Resolves to: branches/{branch}/entities/nouns/{shard}/{id}/metadata.json
|
|
||||||
const results: Map<string, any> = await storage.readBatchWithInheritance(paths, 'my-branch')
|
|
||||||
|
|
||||||
// Automatically inherits from parent branches for missing entities
|
|
||||||
```
|
|
||||||
|
|
||||||
**Features:**
|
|
||||||
- ✅ Branch path resolution (`branches/{branch}/...`)
|
|
||||||
- ✅ Write cache integration (read-after-write consistency)
|
|
||||||
- ✅ COW inheritance (fallback to parent commits for missing entities)
|
|
||||||
- ✅ Adapter-agnostic (works with all storage adapters)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Cloud Adapter Native Batch APIs
|
|
||||||
|
|
||||||
### GCS Storage
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const gcsStorage = new GCSStorage({ bucketName: 'my-bucket' })
|
|
||||||
|
|
||||||
// Native batch API with 100 concurrent downloads
|
|
||||||
const results = await gcsStorage.readBatch(paths)
|
|
||||||
|
|
||||||
// Configuration
|
|
||||||
gcsStorage.getBatchConfig() // → {
|
|
||||||
// maxBatchSize: 1000,
|
|
||||||
// maxConcurrent: 100,
|
|
||||||
// operationsPerSecond: 1000
|
|
||||||
// }
|
|
||||||
```
|
|
||||||
|
|
||||||
**Performance:**
|
|
||||||
- 100 concurrent downloads
|
|
||||||
- ~300-500ms for 100 objects
|
|
||||||
- HTTP/2 multiplexing for optimal throughput
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### S3 Compatible Storage
|
|
||||||
|
|
||||||
Works with Amazon S3, Cloudflare R2, and other S3-compatible services.
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const s3Storage = new S3CompatibleStorage({ bucketName: 'my-bucket' })
|
|
||||||
|
|
||||||
// Native batch API with 150 concurrent downloads
|
|
||||||
const results = await s3Storage.readBatch(paths)
|
|
||||||
|
|
||||||
// Configuration
|
|
||||||
s3Storage.getBatchConfig() // → {
|
|
||||||
// maxBatchSize: 1000,
|
|
||||||
// maxConcurrent: 150,
|
|
||||||
// operationsPerSecond: 5000
|
|
||||||
// }
|
|
||||||
```
|
|
||||||
|
|
||||||
**Performance:**
|
|
||||||
- 150 concurrent downloads
|
|
||||||
- ~200-500ms for 150 objects
|
|
||||||
- S3 handles 5000+ ops/second with burst capacity
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### R2 Storage (Cloudflare)
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const r2Storage = new R2Storage({ bucketName: 'my-bucket' })
|
|
||||||
|
|
||||||
// Fastest cloud storage with zero egress fees
|
|
||||||
const results = await r2Storage.readBatch(paths)
|
|
||||||
|
|
||||||
// Configuration
|
|
||||||
r2Storage.getBatchConfig() // → {
|
|
||||||
// maxBatchSize: 1000,
|
|
||||||
// maxConcurrent: 150,
|
|
||||||
// operationsPerSecond: 6000
|
|
||||||
// }
|
|
||||||
```
|
|
||||||
|
|
||||||
**Performance:**
|
|
||||||
- 150 concurrent downloads
|
|
||||||
- ~200-400ms for 150 objects (fastest!)
|
|
||||||
- Zero egress fees enable aggressive caching
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Azure Blob Storage
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const azureStorage = new AzureBlobStorage({ containerName: 'my-container' })
|
|
||||||
|
|
||||||
// Native batch API with 100 concurrent downloads
|
|
||||||
const results = await azureStorage.readBatch(paths)
|
|
||||||
|
|
||||||
// Configuration
|
|
||||||
azureStorage.getBatchConfig() // → {
|
|
||||||
// maxBatchSize: 1000,
|
|
||||||
// maxConcurrent: 100,
|
|
||||||
// operationsPerSecond: 3000
|
|
||||||
// }
|
|
||||||
```
|
|
||||||
|
|
||||||
**Performance:**
|
|
||||||
- 100 concurrent downloads
|
|
||||||
- ~400-600ms for 100 blobs
|
|
||||||
- Good throughput with Azure's global network
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -263,12 +133,10 @@ VFS operations automatically use batch APIs for maximum performance.
|
||||||
### Directory Traversal
|
### Directory Traversal
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// OLD: Sequential N+1 pattern (12.7 seconds for 12 files)
|
// Tree traversal uses batched reads under the hood
|
||||||
const tree = await brain.vfs.getTreeStructure('/my-dir')
|
const tree = await brain.vfs.getTreeStructure('/my-dir')
|
||||||
|
|
||||||
// NEW Parallel breadth-first with batching (<1 second)
|
|
||||||
// ✅ PathResolver.getChildren() uses brain.batchGet() internally
|
// ✅ PathResolver.getChildren() uses brain.batchGet() internally
|
||||||
// ✅ Parallel traversal of directories at same tree level
|
// ✅ Parallel traversal of directories at the same tree level
|
||||||
// ✅ 2-3 batched calls instead of 22 sequential calls
|
// ✅ 2-3 batched calls instead of 22 sequential calls
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -282,17 +150,11 @@ VFS.getTreeStructure()
|
||||||
→ brain.batchGet(childIds) [1 call instead of N]
|
→ brain.batchGet(childIds) [1 call instead of N]
|
||||||
↓ BATCHED
|
↓ BATCHED
|
||||||
→ storage.getNounMetadataBatch(ids) [1 call instead of N]
|
→ storage.getNounMetadataBatch(ids) [1 call instead of N]
|
||||||
↓ ADAPTER-SPECIFIC
|
↓ ADAPTER
|
||||||
→ GCS: readBatch() with 100 concurrent downloads
|
→ Filesystem: Promise.all() parallel reads
|
||||||
→ S3: readBatch() with 150 concurrent downloads
|
|
||||||
→ Memory: Promise.all() parallel reads
|
→ Memory: Promise.all() parallel reads
|
||||||
```
|
```
|
||||||
|
|
||||||
**Performance Gains:**
|
|
||||||
- **Before**: 22 sequential calls × 580ms = 12.7 seconds
|
|
||||||
- **After**: 2-3 batched calls = <1 second
|
|
||||||
- **Improvement**: **90%+ faster** on cloud storage
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Advanced Features Compatibility
|
## Advanced Features Compatibility
|
||||||
|
|
@ -309,7 +171,7 @@ entities/verbs/{SHARD}/{ID}/metadata.json
|
||||||
|
|
||||||
**Direct O(1) Path Construction:**
|
**Direct O(1) Path Construction:**
|
||||||
```typescript
|
```typescript
|
||||||
// Every ID maps directly to exactly ONE path - 40x faster!
|
// Every ID maps directly to exactly ONE path - O(1), no type search
|
||||||
const id = 'abc-123'
|
const id = 'abc-123'
|
||||||
const shard = getShardIdFromUuid(id) // → 'ab' (first 2 hex chars)
|
const shard = getShardIdFromUuid(id) // → 'ab' (first 2 hex chars)
|
||||||
const path = `entities/nouns/${shard}/${id}/metadata.json`
|
const path = `entities/nouns/${shard}/${id}/metadata.json`
|
||||||
|
|
@ -321,9 +183,9 @@ const path = `entities/nouns/${shard}/${id}/metadata.json`
|
||||||
```
|
```
|
||||||
|
|
||||||
**Benefits:**
|
**Benefits:**
|
||||||
- **40x faster** on GCS/S3 (eliminates 42-type sequential search)
|
- **O(1)** path lookups (eliminates the 42-type sequential search the old type-first layout required)
|
||||||
- **Simpler code** - removed 500+ lines of type cache complexity
|
- **Simpler code** - removed 500+ lines of type cache complexity
|
||||||
- **Scalable** - works at billion-scale without type tracking overhead
|
- **Scalable** - works at large scale without type tracking overhead
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -341,95 +203,39 @@ const path = `entities/nouns/${shard}/${id}/metadata.json`
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### ✅ COW (Copy-on-Write)
|
### ✅ Generational MVCC (8.0)
|
||||||
|
|
||||||
Batch operations respect branch isolation and time-travel:
|
Batch reads always serve the **live** generation through the fast paths
|
||||||
|
shown above. Point-in-time reads go through the Db API instead: a pinned
|
||||||
|
`Db` (`brain.now()`, `brain.asOf()`) resolves changed ids from immutable
|
||||||
|
generation records and unchanged ids from the same live paths batch reads
|
||||||
|
use — see the [consistency model](concepts/consistency-model.md).
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Main branch
|
const db = brain.now() // pinned view
|
||||||
const brain = await Brainy.create({ enableCOW: true })
|
const entity = await db.get(id) // correct at the pinned generation
|
||||||
await brain.add({ type: 'document', data: 'Main' })
|
const results = await brain.batchGet(ids) // live state, batched
|
||||||
|
await db.release()
|
||||||
// Create fork
|
|
||||||
const fork = await brain.fork('experiment')
|
|
||||||
|
|
||||||
// Batch operations are isolated
|
|
||||||
await brain.batchGet([id1, id2]) // → Reads from: branches/main/...
|
|
||||||
await fork.batchGet([id1, id2]) // → Reads from: branches/experiment/...
|
|
||||||
```
|
|
||||||
|
|
||||||
**Inheritance:**
|
|
||||||
- Entities missing from child branch automatically inherit from parent commits
|
|
||||||
- `readBatchWithInheritance()` walks commit history for missing items
|
|
||||||
- Preserves fork semantics while maintaining performance
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### ✅ fork() and checkout()
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const fork = await brain.fork('my-branch')
|
|
||||||
await fork.add({ type: 'document', data: 'Fork entity' })
|
|
||||||
|
|
||||||
// Batch operations use correct branch
|
|
||||||
const results = await fork.batchGet([id1, id2])
|
|
||||||
// → Reads from: branches/my-branch/...
|
|
||||||
|
|
||||||
// Checkout changes active branch
|
|
||||||
await fork.checkout('main')
|
|
||||||
const mainResults = await fork.batchGet([id1, id2])
|
|
||||||
// → Reads from: branches/main/...
|
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### ✅ asOf() Time-Travel
|
## Why Batching Is Faster
|
||||||
|
|
||||||
```typescript
|
Batching's advantage is structural, not a fixed multiplier (the actual speedup
|
||||||
// Create historical snapshot
|
depends on storage backend, IOPS, and batch size):
|
||||||
await brain.commit('v1.0')
|
|
||||||
const snapshot = await brain.asOf('v1.0')
|
|
||||||
|
|
||||||
// Batch operations on historical data
|
- **N+1 elimination** — N sequential reads collapse into a single parallel pass
|
||||||
const results = await snapshot.batchGet([id1, id2])
|
(`Promise.all` over the batch).
|
||||||
// → Reads from historical tree state
|
- **O(1) path construction** — every ID maps directly to one storage path, with
|
||||||
```
|
no per-type cache lookup.
|
||||||
|
- **One metadata round-trip** — relationship batches fetch all sources' metadata
|
||||||
|
in a single pass instead of one query per source.
|
||||||
|
|
||||||
Historical queries use `HistoricalStorageAdapter` which wraps batch operations to point at specific commits.
|
The integration test `tests/integration/storage-batch-operations.test.ts`
|
||||||
|
exercises batch vs. individual reads and asserts that batch retrieval is not
|
||||||
---
|
slower than the per-entity loop for large batches; it does not pin a specific
|
||||||
|
multiplier, since that is hardware- and IOPS-dependent.
|
||||||
## Performance Benchmarks
|
|
||||||
|
|
||||||
### VFS Operations (12 Files)
|
|
||||||
|
|
||||||
| Storage | Before optimization | After optimization | Improvement |
|
|
||||||
|---------|---------------|---------------|-------------|
|
|
||||||
| **GCS** | 12.7s | <1s | **92% faster** |
|
|
||||||
| **S3** | 13.2s | <1s | **92% faster** |
|
|
||||||
| **R2** | 11.8s | <0.8s | **93% faster** |
|
|
||||||
| **Azure** | 14.5s | <1s | **93% faster** |
|
|
||||||
| **Memory** | 150ms | 50ms | **67% faster** |
|
|
||||||
|
|
||||||
### Entity Batch Retrieval (100 Entities)
|
|
||||||
|
|
||||||
| Storage | Individual Gets | Batch Get | Improvement |
|
|
||||||
|---------|----------------|-----------|-------------|
|
|
||||||
| **GCS** | 5.8s | 0.4s | **93% faster** |
|
|
||||||
| **S3** | 5.2s | 0.3s | **94% faster** |
|
|
||||||
| **R2** | 4.9s | 0.25s | **95% faster** |
|
|
||||||
| **Azure** | 6.5s | 0.5s | **92% faster** |
|
|
||||||
| **Memory** | 180ms | 15ms | **92% faster** |
|
|
||||||
|
|
||||||
### Throughput (Entities/Second)
|
|
||||||
|
|
||||||
| Storage | Individual | Batch | Improvement |
|
|
||||||
|---------|-----------|-------|-------------|
|
|
||||||
| **GCS** | 17 ent/s | 250 ent/s | **14.7x** |
|
|
||||||
| **S3** | 19 ent/s | 333 ent/s | **17.5x** |
|
|
||||||
| **R2** | 20 ent/s | 400 ent/s | **20x** |
|
|
||||||
| **Azure** | 15 ent/s | 200 ent/s | **13.3x** |
|
|
||||||
| **Memory** | 556 ent/s | 6667 ent/s | **12x** |
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -494,7 +300,7 @@ const results = await brain.batchGet(ids)
|
||||||
const entities = Array.from(results.values())
|
const entities = Array.from(results.values())
|
||||||
```
|
```
|
||||||
|
|
||||||
**Performance Gain:** 10-20x faster on cloud storage.
|
**Performance Gain:** Replaces N sequential reads with a single batched pass — no fixed multiplier, it scales with storage IOPS.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -504,7 +310,7 @@ const entities = Array.from(results.values())
|
||||||
```typescript
|
```typescript
|
||||||
const allVerbs = []
|
const allVerbs = []
|
||||||
for (const sourceId of sourceIds) {
|
for (const sourceId of sourceIds) {
|
||||||
const verbs = await brain.getRelations({ from: sourceId })
|
const verbs = await brain.related({ from: sourceId })
|
||||||
allVerbs.push(...verbs)
|
allVerbs.push(...verbs)
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
@ -520,7 +326,7 @@ for (const verbs of results.values()) {
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
**Performance Gain:** 5-10x faster due to batched metadata fetches.
|
**Performance Gain:** One batched metadata fetch instead of one query per source entity.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -541,12 +347,9 @@ for (const id of ids) {
|
||||||
### 2. **Batch Size Recommendations**
|
### 2. **Batch Size Recommendations**
|
||||||
|
|
||||||
| Storage | Optimal Batch Size | Max Batch Size |
|
| Storage | Optimal Batch Size | Max Batch Size |
|
||||||
|---------|-------------------|----------------|
|
|---------|--------------------|----------------|
|
||||||
| **Memory** | Unlimited | Unlimited |
|
| **Memory** | Unlimited | Unlimited |
|
||||||
| **FileSystem** | 100-500 | 1000 |
|
| **Filesystem** | 100-500 | 1000 |
|
||||||
| **GCS** | 100-500 | 1000 |
|
|
||||||
| **S3/R2** | 100-1000 | 1000 |
|
|
||||||
| **Azure** | 100-500 | 1000 |
|
|
||||||
|
|
||||||
**Guideline:** For batches >1000, split into chunks of 500-1000.
|
**Guideline:** For batches >1000, split into chunks of 500-1000.
|
||||||
|
|
||||||
|
|
@ -614,68 +417,42 @@ High-Level API (src/brainy.ts)
|
||||||
↓
|
↓
|
||||||
Storage Layer (src/storage/baseStorage.ts)
|
Storage Layer (src/storage/baseStorage.ts)
|
||||||
↓
|
↓
|
||||||
COW Layer (readBatchWithInheritance)
|
|
||||||
↓
|
|
||||||
Adapter Layer (readBatchFromAdapter)
|
Adapter Layer (readBatchFromAdapter)
|
||||||
↓
|
↓
|
||||||
Cloud Adapter (GCS/S3/Azure native batch APIs)
|
Storage Adapter (FileSystemStorage / MemoryStorage)
|
||||||
```
|
```
|
||||||
|
|
||||||
### Automatic Fallback
|
### Parallel Reads
|
||||||
|
|
||||||
If an adapter doesn't implement `readBatch()`, the system automatically falls back to parallel individual reads:
|
Both shipped adapters fall back to `Promise.all` over individual reads:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// BaseStorage.readBatchFromAdapter()
|
// BaseStorage.readBatchFromAdapter()
|
||||||
if (typeof selfWithBatch.readBatch === 'function') {
|
return await Promise.all(resolvedPaths.map(path => this.read(path)))
|
||||||
// Use native batch API
|
|
||||||
return await selfWithBatch.readBatch(resolvedPaths)
|
|
||||||
} else {
|
|
||||||
// Automatic parallel fallback
|
|
||||||
return await Promise.all(resolvedPaths.map(path => this.read(path)))
|
|
||||||
}
|
|
||||||
```
|
```
|
||||||
|
|
||||||
**Adapters with Native Batch:**
|
**Shipped Adapters:**
|
||||||
- ✅ GCSStorage
|
|
||||||
- ✅ S3CompatibleStorage
|
|
||||||
- ✅ R2Storage
|
|
||||||
- ✅ AzureBlobStorage
|
|
||||||
|
|
||||||
**Adapters with Parallel Fallback:**
|
|
||||||
- MemoryStorage
|
- MemoryStorage
|
||||||
- FileSystemStorage
|
- FileSystemStorage
|
||||||
- OPFSStorage
|
|
||||||
- HistoricalStorageAdapter (delegates to underlying)
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Release Notes
|
## API Summary
|
||||||
|
|
||||||
**Version:** 5.12.0
|
|
||||||
**Release Date:** 2025-11-19
|
|
||||||
**Status:** Production-Ready
|
|
||||||
|
|
||||||
**Breaking Changes:** None (backward compatible)
|
|
||||||
|
|
||||||
**New APIs:**
|
|
||||||
- `brain.batchGet(ids, options?)` - High-level batch entity retrieval
|
- `brain.batchGet(ids, options?)` - High-level batch entity retrieval
|
||||||
- `storage.getNounMetadataBatch(ids)` - Storage-level metadata batch
|
- `storage.getNounMetadataBatch(ids)` - Storage-level metadata batch
|
||||||
- `storage.getVerbsBySourceBatch(sourceIds, verbType?)` - Batch relationship queries
|
- `storage.getVerbsBySourceBatch(sourceIds, verbType?)` - Batch relationship queries
|
||||||
- `storage.readBatchWithInheritance(paths, targetBranch?)` - COW-aware batch reads
|
|
||||||
|
|
||||||
**Performance Improvements:**
|
**Performance Improvements:**
|
||||||
- VFS operations: 90%+ faster on cloud storage
|
- VFS operations: single batched pass instead of N sequential reads
|
||||||
- Entity retrieval: 10-20x throughput improvement
|
- Entity retrieval: N+1 reads collapsed into one batched pass
|
||||||
- Zero N+1 query patterns
|
- Zero N+1 query patterns
|
||||||
|
|
||||||
**Compatibility:**
|
**Compatibility:**
|
||||||
- ✅ ID-first storage
|
- ✅ ID-first storage
|
||||||
- ✅ Sharding (256 shards)
|
- ✅ Sharding (256 shards)
|
||||||
- ✅ COW (branch isolation, inheritance)
|
- ✅ Generational MVCC — batch reads serve the live generation; pinned `Db` views serve the past
|
||||||
- ✅ fork() and checkout()
|
- ✅ All indexes respected (vector, metadata, graph adjacency)
|
||||||
- ✅ asOf() time-travel
|
|
||||||
- ✅ All 6 indexes respected (HNSW, TypeAwareHNSW, MetadataIndex, GraphAdjacency, Version, DeletedItems)
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,435 +0,0 @@
|
||||||
# Creating Augmentations for Brainy
|
|
||||||
|
|
||||||
> **Updated** - Includes metadata structure changes and type system improvements
|
|
||||||
|
|
||||||
## The BrainyAugmentation Interface
|
|
||||||
|
|
||||||
Every augmentation implements this simple yet powerful interface:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
interface BrainyAugmentation {
|
|
||||||
// Identification
|
|
||||||
name: string // Unique name for your augmentation
|
|
||||||
|
|
||||||
// Execution control
|
|
||||||
timing: 'before' | 'after' | 'around' | 'replace' // When to execute
|
|
||||||
operations: string[] // Which operations to intercept
|
|
||||||
priority: number // Execution order (higher = first)
|
|
||||||
|
|
||||||
// Lifecycle methods
|
|
||||||
initialize(context: AugmentationContext): Promise<void>
|
|
||||||
execute<T>(operation: string, params: any, next: () => Promise<T>): Promise<T>
|
|
||||||
shutdown?(): Promise<void> // Optional cleanup
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Breaking Changes for Augmentation Developers
|
|
||||||
|
|
||||||
### 1. Metadata Structure Separation
|
|
||||||
Brainy introduces strict metadata/vector separation for billion-scale performance:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// ✅ Metadata has required type field
|
|
||||||
interface NounMetadata {
|
|
||||||
noun: NounType // Required! Must be a valid noun type
|
|
||||||
[key: string]: any // Your custom metadata
|
|
||||||
}
|
|
||||||
|
|
||||||
interface VerbMetadata {
|
|
||||||
verb: VerbType // Required! Must be a valid verb type
|
|
||||||
sourceId: string
|
|
||||||
targetId: string
|
|
||||||
[key: string]: any
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Storage Adapter Return Types
|
|
||||||
Storage adapters now return different types at different boundaries:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Internal methods: Pure structures (no metadata)
|
|
||||||
abstract _getNoun(id: string): Promise<HNSWNoun | null>
|
|
||||||
|
|
||||||
// Public API: WithMetadata structures
|
|
||||||
abstract getNoun(id: string): Promise<HNSWNounWithMetadata | null>
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Verb Property Renamed
|
|
||||||
The verb relationship field changed from `type` to `verb`:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// ❌ v3.x
|
|
||||||
verb.type === 'relatedTo'
|
|
||||||
|
|
||||||
// ✅ Current
|
|
||||||
verb.verb === 'relatedTo'
|
|
||||||
```
|
|
||||||
|
|
||||||
## Creating a Storage Augmentation
|
|
||||||
|
|
||||||
Storage augmentations are special - they provide the storage backend for Brainy.
|
|
||||||
|
|
||||||
### Important: Storage Requirements
|
|
||||||
|
|
||||||
Your storage adapter MUST:
|
|
||||||
1. **Wrap metadata** with required `noun`/`verb` fields
|
|
||||||
2. **Return pure structures** from internal `_methods`
|
|
||||||
3. **Return WithMetadata types** from public methods
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { StorageAugmentation } from 'brainy/augmentations'
|
|
||||||
import { BaseStorageAdapter, HNSWNoun, HNSWNounWithMetadata, NounMetadata } from 'brainy'
|
|
||||||
|
|
||||||
export class MyCustomStorage extends BaseStorageAdapter {
|
|
||||||
// Internal method: Returns pure structure
|
|
||||||
async _getNoun(id: string): Promise<HNSWNoun | null> {
|
|
||||||
const data = await this.fetchFromDatabase(id)
|
|
||||||
return data ? {
|
|
||||||
id: data.id,
|
|
||||||
vector: data.vector,
|
|
||||||
nounType: data.type
|
|
||||||
} : null
|
|
||||||
}
|
|
||||||
|
|
||||||
// Public method: Returns WithMetadata structure
|
|
||||||
async getNoun(id: string): Promise<HNSWNounWithMetadata | null> {
|
|
||||||
const noun = await this._getNoun(id)
|
|
||||||
if (!noun) return null
|
|
||||||
|
|
||||||
// Fetch metadata separately
|
|
||||||
const metadata = await this.getNounMetadata(id)
|
|
||||||
|
|
||||||
return {
|
|
||||||
...noun,
|
|
||||||
metadata: metadata || { noun: noun.nounType || 'thing' }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// CRITICAL: Always save with proper metadata structure
|
|
||||||
async saveNoun(noun: HNSWNoun, metadata?: NounMetadata): Promise<void> {
|
|
||||||
// Validate metadata has required 'noun' field
|
|
||||||
if (!metadata?.noun) {
|
|
||||||
throw new Error('NounMetadata requires "noun" field')
|
|
||||||
}
|
|
||||||
|
|
||||||
await this.database.save({
|
|
||||||
id: noun.id,
|
|
||||||
vector: noun.vector,
|
|
||||||
nounType: noun.nounType,
|
|
||||||
metadata: metadata // Stored separately
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export class MyStorageAugmentation extends StorageAugmentation {
|
|
||||||
private config: MyStorageConfig
|
|
||||||
|
|
||||||
constructor(config: MyStorageConfig) {
|
|
||||||
super()
|
|
||||||
this.name = 'my-custom-storage'
|
|
||||||
this.config = config
|
|
||||||
}
|
|
||||||
|
|
||||||
// Called during storage resolution phase
|
|
||||||
async provideStorage(): Promise<StorageAdapter> {
|
|
||||||
const storage = new MyCustomStorage(this.config)
|
|
||||||
this.storageAdapter = storage
|
|
||||||
return storage
|
|
||||||
}
|
|
||||||
|
|
||||||
// Called during augmentation initialization
|
|
||||||
protected async onInitialize(): Promise<void> {
|
|
||||||
await this.storageAdapter!.init()
|
|
||||||
this.log(`Custom storage initialized`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Using Your Storage Augmentation
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Register before brain.init()
|
|
||||||
const brain = new Brainy()
|
|
||||||
brain.augmentations.register(new MyStorageAugmentation({
|
|
||||||
connectionString: 'redis://localhost:6379'
|
|
||||||
}))
|
|
||||||
await brain.init() // Will use your storage!
|
|
||||||
```
|
|
||||||
|
|
||||||
## Creating a Feature Augmentation
|
|
||||||
|
|
||||||
Here's a complete example of a caching augmentation:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { BaseAugmentation, BrainyAugmentation } from 'brainy/augmentations'
|
|
||||||
|
|
||||||
export class CachingAugmentation extends BaseAugmentation {
|
|
||||||
private cache = new Map<string, any>()
|
|
||||||
|
|
||||||
constructor() {
|
|
||||||
super()
|
|
||||||
this.name = 'smart-cache'
|
|
||||||
this.timing = 'around' // Wrap operations
|
|
||||||
this.operations = ['search'] // Only cache searches
|
|
||||||
this.priority = 50 // Mid-priority
|
|
||||||
}
|
|
||||||
|
|
||||||
async execute<T>(operation: string, params: any, next: () => Promise<T>): Promise<T> {
|
|
||||||
if (operation === 'search') {
|
|
||||||
// Check cache
|
|
||||||
const cacheKey = JSON.stringify(params)
|
|
||||||
if (this.cache.has(cacheKey)) {
|
|
||||||
this.log('Cache hit!')
|
|
||||||
return this.cache.get(cacheKey)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Execute and cache
|
|
||||||
const result = await next()
|
|
||||||
this.cache.set(cacheKey, result)
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
// Pass through other operations
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onInitialize(): Promise<void> {
|
|
||||||
this.log('Cache initialized')
|
|
||||||
}
|
|
||||||
|
|
||||||
async shutdown(): Promise<void> {
|
|
||||||
this.cache.clear()
|
|
||||||
await super.shutdown()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## The Four Timing Modes
|
|
||||||
|
|
||||||
### 1. `before` - Pre-processing
|
|
||||||
```typescript
|
|
||||||
timing = 'before'
|
|
||||||
async execute(op, params, next) {
|
|
||||||
// Validate/transform input
|
|
||||||
const validated = await validate(params)
|
|
||||||
return next(validated) // Pass modified params
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. `after` - Post-processing
|
|
||||||
```typescript
|
|
||||||
timing = 'after'
|
|
||||||
async execute(op, params, next) {
|
|
||||||
const result = await next()
|
|
||||||
// Log, analyze, or modify result
|
|
||||||
console.log(`Operation ${op} returned:`, result)
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. `around` - Wrapping (middleware)
|
|
||||||
```typescript
|
|
||||||
timing = 'around'
|
|
||||||
async execute(op, params, next) {
|
|
||||||
console.log('Starting', op)
|
|
||||||
try {
|
|
||||||
const result = await next()
|
|
||||||
console.log('Success', op)
|
|
||||||
return result
|
|
||||||
} catch (error) {
|
|
||||||
console.log('Failed', op, error)
|
|
||||||
throw error
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4. `replace` - Complete replacement
|
|
||||||
```typescript
|
|
||||||
timing = 'replace'
|
|
||||||
async execute(op, params, next) {
|
|
||||||
// Don't call next() - replace entirely!
|
|
||||||
return myCustomImplementation(params)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Operations You Can Intercept
|
|
||||||
|
|
||||||
Common operations in Brainy:
|
|
||||||
- `'storage'` - Storage resolution (special)
|
|
||||||
- `'add'` - Adding data
|
|
||||||
- `'search'`, `'similar'` - Searching
|
|
||||||
- `'update'`, `'delete'` - Modifications
|
|
||||||
- `'saveNoun'`, `'saveVerb'` - Storage operations
|
|
||||||
- `'all'` - Intercept everything
|
|
||||||
|
|
||||||
## Context Available to Augmentations
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
interface AugmentationContext {
|
|
||||||
brain: Brainy // The brain instance
|
|
||||||
storage: StorageAdapter // Storage backend
|
|
||||||
config: BrainyConfig // Configuration
|
|
||||||
log: (message: string, level?: 'info' | 'warn' | 'error') => void
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Real-World Examples
|
|
||||||
|
|
||||||
### 1. Redis Storage Augmentation
|
|
||||||
```typescript
|
|
||||||
export class RedisStorageAugmentation extends StorageAugmentation {
|
|
||||||
async provideStorage(): Promise<StorageAdapter> {
|
|
||||||
return new RedisAdapter({
|
|
||||||
host: 'localhost',
|
|
||||||
port: 6379,
|
|
||||||
// Implement full StorageAdapter interface
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Audit Trail Augmentation
|
|
||||||
```typescript
|
|
||||||
export class AuditAugmentation extends BaseAugmentation {
|
|
||||||
timing = 'after'
|
|
||||||
operations = ['add', 'update', 'delete']
|
|
||||||
|
|
||||||
async execute(op, params, next) {
|
|
||||||
const result = await next()
|
|
||||||
|
|
||||||
// Log to audit trail
|
|
||||||
await this.logAudit({
|
|
||||||
operation: op,
|
|
||||||
params,
|
|
||||||
result,
|
|
||||||
timestamp: new Date(),
|
|
||||||
user: this.context.config.currentUser
|
|
||||||
})
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Rate Limiting Augmentation
|
|
||||||
```typescript
|
|
||||||
export class RateLimitAugmentation extends BaseAugmentation {
|
|
||||||
timing = 'before'
|
|
||||||
operations = ['search']
|
|
||||||
private limiter = new RateLimiter({ rps: 100 })
|
|
||||||
|
|
||||||
async execute(op, params, next) {
|
|
||||||
await this.limiter.acquire() // Wait if rate limited
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Publishing to Brain Cloud Marketplace
|
|
||||||
|
|
||||||
Future capability for premium augmentations:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// package.json
|
|
||||||
{
|
|
||||||
"name": "@brain-cloud/redis-storage",
|
|
||||||
"brainy": {
|
|
||||||
"type": "augmentation",
|
|
||||||
"category": "storage",
|
|
||||||
"premium": true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Users can install via:
|
|
||||||
// brainy augment install redis-storage
|
|
||||||
```
|
|
||||||
|
|
||||||
## Best Practices
|
|
||||||
|
|
||||||
### General Practices
|
|
||||||
|
|
||||||
1. **Use BaseAugmentation** - Provides common functionality
|
|
||||||
2. **Set appropriate priority** - Storage (100), System (80-99), Features (10-50)
|
|
||||||
3. **Be selective with operations** - Don't use 'all' unless necessary
|
|
||||||
4. **Handle errors gracefully** - Don't break the chain
|
|
||||||
5. **Clean up in shutdown()** - Release resources
|
|
||||||
6. **Log appropriately** - Use context.log() for consistent output
|
|
||||||
7. **Document your augmentation** - Include examples
|
|
||||||
|
|
||||||
### Specific Best Practices
|
|
||||||
|
|
||||||
8. **Always include `noun` field** when creating/modifying NounMetadata:
|
|
||||||
```typescript
|
|
||||||
const metadata: NounMetadata = {
|
|
||||||
noun: 'thing', // REQUIRED!
|
|
||||||
yourField: 'value'
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
9. **Use `verb` property** not `type` when working with relationships:
|
|
||||||
```typescript
|
|
||||||
// ✅ Correct
|
|
||||||
if (verb.verb === 'relatedTo') { ... }
|
|
||||||
|
|
||||||
// ❌ Wrong (v3.x pattern)
|
|
||||||
if (verb.type === 'relatedTo') { ... }
|
|
||||||
```
|
|
||||||
|
|
||||||
10. **Access metadata correctly** from storage:
|
|
||||||
```typescript
|
|
||||||
// ✅ Correct - metadata is already structured
|
|
||||||
const nounType = noun.metadata.noun
|
|
||||||
|
|
||||||
// ⚠️ Fallback pattern for robustness
|
|
||||||
const nounType = noun.metadata?.noun || 'thing'
|
|
||||||
```
|
|
||||||
|
|
||||||
11. **Respect the two-file storage pattern** - Don't mix vector and metadata operations:
|
|
||||||
```typescript
|
|
||||||
// ✅ Good - Separate concerns
|
|
||||||
await storage.saveNoun(noun)
|
|
||||||
await storage.saveMetadata(noun.id, metadata)
|
|
||||||
|
|
||||||
// ❌ Bad - Mixing concerns
|
|
||||||
await storage.saveNounWithEverything(combinedData)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Testing Your Augmentation
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { Brainy } from 'brainy'
|
|
||||||
import { MyAugmentation } from './my-augmentation'
|
|
||||||
|
|
||||||
describe('MyAugmentation', () => {
|
|
||||||
let brain: Brainy
|
|
||||||
|
|
||||||
beforeEach(async () => {
|
|
||||||
brain = new Brainy()
|
|
||||||
brain.augmentations.register(new MyAugmentation())
|
|
||||||
await brain.init()
|
|
||||||
})
|
|
||||||
|
|
||||||
afterEach(async () => {
|
|
||||||
await brain.destroy()
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should enhance searches', async () => {
|
|
||||||
// Test your augmentation's effect
|
|
||||||
const results = await brain.search('test')
|
|
||||||
expect(results).toHaveProperty('enhanced', true)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
Augmentations are Brainy's extension system. They can:
|
|
||||||
- Replace storage backends
|
|
||||||
- Add caching layers
|
|
||||||
- Implement audit trails
|
|
||||||
- Add rate limiting
|
|
||||||
- Sync with external systems
|
|
||||||
- Transform data
|
|
||||||
- And much more!
|
|
||||||
|
|
||||||
The unified BrainyAugmentation interface makes it easy to create powerful extensions while maintaining consistency across the entire system.
|
|
||||||
|
|
@ -238,7 +238,7 @@ await brain.relate({
|
||||||
Fast-path filter on the verb side:
|
Fast-path filter on the verb side:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
const direct = await brain.getRelations({
|
const direct = await brain.related({
|
||||||
from: ceoId,
|
from: ceoId,
|
||||||
type: VerbType.ReportsTo,
|
type: VerbType.ReportsTo,
|
||||||
subtype: 'direct'
|
subtype: 'direct'
|
||||||
|
|
|
||||||
|
|
@ -193,21 +193,21 @@ console.log('✅ Created relationships')
|
||||||
console.log('\n🔍 Querying relationships...')
|
console.log('\n🔍 Querying relationships...')
|
||||||
|
|
||||||
// Who works for TechCorp?
|
// Who works for TechCorp?
|
||||||
const techcorpEmployees = await brain.getRelations({
|
const techcorpEmployees = await brain.related({
|
||||||
to: techcorpId,
|
to: techcorpId,
|
||||||
type: VerbType.WorksWith
|
type: VerbType.WorksWith
|
||||||
})
|
})
|
||||||
console.log(`TechCorp has ${techcorpEmployees.length} employees`)
|
console.log(`TechCorp has ${techcorpEmployees.length} employees`)
|
||||||
|
|
||||||
// Who works on the AI project?
|
// Who works on the AI project?
|
||||||
const projectContributors = await brain.getRelations({
|
const projectContributors = await brain.related({
|
||||||
to: projectId,
|
to: projectId,
|
||||||
type: [VerbType.WorksOn, VerbType.Manages]
|
type: [VerbType.WorksOn, VerbType.Manages]
|
||||||
})
|
})
|
||||||
console.log(`AI Project has ${projectContributors.length} contributors`)
|
console.log(`AI Project has ${projectContributors.length} contributors`)
|
||||||
|
|
||||||
// Who does John collaborate with?
|
// Who does John collaborate with?
|
||||||
const johnsCollaborators = await brain.getRelations({
|
const johnsCollaborators = await brain.related({
|
||||||
from: johnId,
|
from: johnId,
|
||||||
type: VerbType.CollaboratesWith
|
type: VerbType.CollaboratesWith
|
||||||
})
|
})
|
||||||
|
|
@ -294,7 +294,7 @@ await brain.update({
|
||||||
1. Create an organizational hierarchy (CEO → Managers → Engineers)
|
1. Create an organizational hierarchy (CEO → Managers → Engineers)
|
||||||
2. Build a project dependency graph
|
2. Build a project dependency graph
|
||||||
3. Model a social network with CollaboratesWith relationships
|
3. Model a social network with CollaboratesWith relationships
|
||||||
4. Query "Who reports to Alice?" using getRelations()
|
4. Query "Who reports to Alice?" using related()
|
||||||
5. Batch update all projects to add a "year: 2024" field
|
5. Batch update all projects to add a "year: 2024" field
|
||||||
|
|
||||||
### Next Steps
|
### Next Steps
|
||||||
|
|
@ -429,23 +429,7 @@ fusionResults.forEach(r => {
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
// 5. NEURAL API: Automatic clustering
|
// 5. SIMILARITY: Find similar documents
|
||||||
console.log('\n\n🤖 NEURAL API: Automatic Clustering')
|
|
||||||
const neural = brain.neural()
|
|
||||||
|
|
||||||
const clusters = await neural.clusters({
|
|
||||||
maxClusters: 3,
|
|
||||||
minClusterSize: 1
|
|
||||||
})
|
|
||||||
|
|
||||||
console.log(`Found ${clusters.length} semantic clusters:`)
|
|
||||||
clusters.forEach((cluster, i) => {
|
|
||||||
console.log(`\n Cluster ${i + 1}: ${cluster.label || cluster.id}`)
|
|
||||||
console.log(` Members: ${cluster.members.length}`)
|
|
||||||
console.log(` Centroid topics: ${cluster.metadata?.topics?.join(', ') || 'N/A'}`)
|
|
||||||
})
|
|
||||||
|
|
||||||
// 6. SIMILARITY: Find similar documents
|
|
||||||
console.log('\n\n🔍 SIMILARITY: Find Similar Documents')
|
console.log('\n\n🔍 SIMILARITY: Find Similar Documents')
|
||||||
const similarTo = await brain.similar({
|
const similarTo = await brain.similar({
|
||||||
to: paper1, // Entity ID of first AI paper
|
to: paper1, // Entity ID of first AI paper
|
||||||
|
|
@ -459,19 +443,6 @@ similarTo.forEach(r => {
|
||||||
console.log(` [${r.score.toFixed(3)}] ${r.entity.data?.substring(0, 50)}...`)
|
console.log(` [${r.score.toFixed(3)}] ${r.entity.data?.substring(0, 50)}...`)
|
||||||
})
|
})
|
||||||
|
|
||||||
// 7. OUTLIER DETECTION
|
|
||||||
console.log('\n\n🚨 OUTLIER DETECTION')
|
|
||||||
const outliers = await neural.outliers({
|
|
||||||
method: 'statistical',
|
|
||||||
threshold: 2.0 // 2 standard deviations
|
|
||||||
})
|
|
||||||
|
|
||||||
console.log(`Found ${outliers.length} outlier documents:`)
|
|
||||||
outliers.forEach(o => {
|
|
||||||
const entity = await brain.get(o.id)
|
|
||||||
console.log(` [Anomaly score: ${o.score.toFixed(3)}] ${entity?.data?.substring(0, 50)}...`)
|
|
||||||
})
|
|
||||||
|
|
||||||
await brain.close()
|
await brain.close()
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -676,7 +647,7 @@ if (readmeEntity.length > 0) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Query relationships
|
// Query relationships
|
||||||
const conceptDocs = await brain.getRelations({
|
const conceptDocs = await brain.related({
|
||||||
from: conceptId,
|
from: conceptId,
|
||||||
type: VerbType.DocumentedBy
|
type: VerbType.DocumentedBy
|
||||||
})
|
})
|
||||||
|
|
@ -798,7 +769,7 @@ await brain.relate({
|
||||||
})
|
})
|
||||||
|
|
||||||
// Query across boundaries
|
// Query across boundaries
|
||||||
const conceptDocs = await brain.getRelations({
|
const conceptDocs = await brain.related({
|
||||||
from: conceptId,
|
from: conceptId,
|
||||||
type: VerbType.DocumentedBy
|
type: VerbType.DocumentedBy
|
||||||
})
|
})
|
||||||
|
|
@ -852,7 +823,7 @@ Ready for production deployment? Level 5 covers **planet-scale architecture**.
|
||||||
## Level 5: Production Scale (90 minutes)
|
## Level 5: Production Scale (90 minutes)
|
||||||
|
|
||||||
### What You'll Learn
|
### What You'll Learn
|
||||||
- Cloud storage (GCS, S3, R2)
|
- Production filesystem storage and off-site backup
|
||||||
- Performance optimization
|
- Performance optimization
|
||||||
- Batch imports (CSV, Excel, PDF)
|
- Batch imports (CSV, Excel, PDF)
|
||||||
- Metadata query optimization
|
- Metadata query optimization
|
||||||
|
|
@ -863,18 +834,13 @@ Ready for production deployment? Level 5 covers **planet-scale architecture**.
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy, NounType } from '@soulcraft/brainy'
|
import { Brainy, NounType } from '@soulcraft/brainy'
|
||||||
|
|
||||||
// 1. PRODUCTION STORAGE - Google Cloud Storage (Native SDK)
|
// 1. PRODUCTION STORAGE - Filesystem with off-site snapshots
|
||||||
console.log('☁️ Initializing production storage...\n')
|
console.log('Initializing production storage...\n')
|
||||||
|
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: {
|
storage: {
|
||||||
type: 'gcs-native', // Native GCS SDK (recommended)
|
type: 'filesystem',
|
||||||
gcsNativeStorage: {
|
path: '/var/lib/brainy'
|
||||||
bucketName: 'my-brainy-production',
|
|
||||||
// ADC (Application Default Credentials) - zero config in Cloud Run/GCE!
|
|
||||||
// Or provide credentials:
|
|
||||||
// keyFilename: '/path/to/service-account.json'
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
|
|
||||||
// Performance tuning
|
// Performance tuning
|
||||||
|
|
@ -888,7 +854,8 @@ const brain = new Brainy({
|
||||||
})
|
})
|
||||||
|
|
||||||
await brain.init()
|
await brain.init()
|
||||||
console.log('✅ Brainy initialized with GCS Native storage\n')
|
console.log('Brainy initialized with filesystem storage')
|
||||||
|
console.log('Snapshot /var/lib/brainy off-site via cron with `gsutil rsync` / `aws s3 sync` / `rclone`.\n')
|
||||||
|
|
||||||
// 2. BATCH IMPORT - CSV File
|
// 2. BATCH IMPORT - CSV File
|
||||||
console.log('📊 Importing CSV data...\n')
|
console.log('📊 Importing CSV data...\n')
|
||||||
|
|
@ -967,26 +934,6 @@ console.log(` Failed: ${batchResult.failed.length}`)
|
||||||
console.log(` Duration: ${duration}ms`)
|
console.log(` Duration: ${duration}ms`)
|
||||||
console.log(` Throughput: ${(batchResult.successful.length / (duration / 1000)).toFixed(0)} entities/sec`)
|
console.log(` Throughput: ${(batchResult.successful.length / (duration / 1000)).toFixed(0)} entities/sec`)
|
||||||
|
|
||||||
// 5. ADVANCED CLUSTERING (Large Dataset)
|
|
||||||
console.log('\n\n🤖 Clustering 1000 entities...\n')
|
|
||||||
|
|
||||||
const neural = brain.neural()
|
|
||||||
|
|
||||||
// Use fast clustering for large datasets
|
|
||||||
const clusters = await neural.clusterFast({
|
|
||||||
maxClusters: 5
|
|
||||||
})
|
|
||||||
|
|
||||||
console.log(`Found ${clusters.length} clusters:`)
|
|
||||||
clusters.forEach((cluster, i) => {
|
|
||||||
console.log(`\n Cluster ${i + 1}: ${cluster.label || cluster.id}`)
|
|
||||||
console.log(` Size: ${cluster.members.length} members`)
|
|
||||||
console.log(` Density: ${(cluster.density || 0).toFixed(3)}`)
|
|
||||||
if (cluster.metadata?.keywords) {
|
|
||||||
console.log(` Keywords: ${cluster.metadata.keywords.slice(0, 5).join(', ')}`)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// 6. PRODUCTION STATISTICS
|
// 6. PRODUCTION STATISTICS
|
||||||
console.log('\n\n📊 Production Statistics:\n')
|
console.log('\n\n📊 Production Statistics:\n')
|
||||||
|
|
||||||
|
|
@ -1044,7 +991,7 @@ console.log('✅ Brain closed cleanly')
|
||||||
|
|
||||||
console.log('\n\n🎓 Production Deployment Complete!')
|
console.log('\n\n🎓 Production Deployment Complete!')
|
||||||
console.log('\n📚 Key Production Learnings:')
|
console.log('\n📚 Key Production Learnings:')
|
||||||
console.log(' 1. Use native cloud storage (GCS, S3, R2) for persistence')
|
console.log(' 1. Use filesystem storage and snapshot the data directory off-site from your scheduler')
|
||||||
console.log(' 2. Batch operations = 100x faster than individual ops')
|
console.log(' 2. Batch operations = 100x faster than individual ops')
|
||||||
console.log(' 3. Metadata query optimization for complex filters')
|
console.log(' 3. Metadata query optimization for complex filters')
|
||||||
console.log(' 4. Monitor query performance with explain: true')
|
console.log(' 4. Monitor query performance with explain: true')
|
||||||
|
|
@ -1057,41 +1004,22 @@ console.log(' 7. Stream large imports with progress callbacks')
|
||||||
|
|
||||||
#### 1. **Storage Options Comparison**
|
#### 1. **Storage Options Comparison**
|
||||||
|
|
||||||
| Storage | Use Case | Performance | Cost | Setup |
|
| Storage | Use Case | Performance | Setup |
|
||||||
|---------|----------|-------------|------|-------|
|
|---------|----------|-------------|-------|
|
||||||
| Memory | Dev/testing | Fastest | Free | Zero config |
|
| Memory | Dev/testing | Fastest | Zero config |
|
||||||
| Filesystem | Local prod | Fast | Free | Local path |
|
| Filesystem | Production | Fast | Local path + scheduled off-site backup |
|
||||||
| GCS Native | GCP prod | Fast | $$$ | Service account |
|
|
||||||
| S3 | AWS prod | Fast | $$$ | Access keys |
|
|
||||||
| R2 | Cloudflare | Fast | $ | Access keys |
|
|
||||||
|
|
||||||
#### 2. **GCS Native vs S3-Compatible**
|
#### 2. **Off-Site Backup**
|
||||||
|
|
||||||
```typescript
|
```bash
|
||||||
// ✅ RECOMMENDED: GCS Native SDK
|
# Cron / systemd timer / k8s CronJob — pick whatever you already operate
|
||||||
{
|
*/15 * * * * rclone sync /var/lib/brainy remote:brainy-backup
|
||||||
storage: {
|
*/15 * * * * aws s3 sync /var/lib/brainy s3://my-bucket/brainy-backup
|
||||||
type: 'gcs-native',
|
*/15 * * * * gsutil rsync -r /var/lib/brainy gs://my-bucket/brainy-backup
|
||||||
gcsNativeStorage: {
|
|
||||||
bucketName: 'my-bucket'
|
|
||||||
// ADC handles auth automatically in Cloud Run/GCE!
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ⚠️ LEGACY: S3-compatible mode
|
|
||||||
{
|
|
||||||
storage: {
|
|
||||||
type: 'gcs',
|
|
||||||
gcsStorage: {
|
|
||||||
bucketName: 'my-bucket',
|
|
||||||
accessKeyId: process.env.GCS_ACCESS_KEY,
|
|
||||||
secretAccessKey: process.env.GCS_SECRET_KEY
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Brainy itself never reaches out to an object store. Snapshot `path` from your scheduler.
|
||||||
|
|
||||||
#### 3. **Performance Optimization**
|
#### 3. **Performance Optimization**
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
|
|
@ -1164,8 +1092,8 @@ const brain = new Brainy({ verbose: true })
|
||||||
|
|
||||||
#### Before Deployment
|
#### Before Deployment
|
||||||
|
|
||||||
- [ ] Choose cloud storage (GCS/S3/R2)
|
- [ ] Provision a writable `path` on the host
|
||||||
- [ ] Set up authentication (service account/access keys)
|
- [ ] Set up an off-site snapshot job (`rclone` / `aws s3 sync` / `gsutil rsync`)
|
||||||
- [ ] Configure caching
|
- [ ] Configure caching
|
||||||
- [ ] Test batch operations
|
- [ ] Test batch operations
|
||||||
- [ ] Benchmark query performance
|
- [ ] Benchmark query performance
|
||||||
|
|
@ -1189,12 +1117,12 @@ const brain = new Brainy({ verbose: true })
|
||||||
|
|
||||||
### Practice Exercises
|
### Practice Exercises
|
||||||
|
|
||||||
1. Deploy Brainy with GCS Native storage
|
1. Deploy Brainy with filesystem storage + scheduled off-site backup
|
||||||
2. Import a 10,000 row CSV file
|
2. Import a 10,000 row CSV file
|
||||||
3. Measure query performance for different filters
|
3. Measure query performance for different filters
|
||||||
4. Optimize a slow query using getOptimalQueryPlan()
|
4. Optimize a slow query using getOptimalQueryPlan()
|
||||||
5. Set up monitoring dashboard
|
5. Set up monitoring dashboard
|
||||||
6. Create backup/restore scripts
|
6. Test restore from off-site snapshot
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -1212,7 +1140,7 @@ const brain = new Brainy({ verbose: true })
|
||||||
|
|
||||||
#### Advanced Topics
|
#### Advanced Topics
|
||||||
|
|
||||||
- **Distributed Systems**: Sharding, replication, coordination
|
- **Multi-instance Deployments**: Run multiple Brainy processes behind your own routing layer
|
||||||
- **Custom Augmentations**: Extend Brainy with plugins
|
- **Custom Augmentations**: Extend Brainy with plugins
|
||||||
- **Streaming Pipelines**: Real-time data ingestion
|
- **Streaming Pipelines**: Real-time data ingestion
|
||||||
- **Security**: Encryption, access control, audit logs
|
- **Security**: Encryption, access control, audit logs
|
||||||
|
|
@ -1223,7 +1151,6 @@ const brain = new Brainy({ verbose: true })
|
||||||
- 📚 [API Reference](../api/README.md) - Complete API documentation
|
- 📚 [API Reference](../api/README.md) - Complete API documentation
|
||||||
- 📁 [VFS Guide](../vfs/VFS_API_GUIDE.md) - Virtual Filesystem deep dive
|
- 📁 [VFS Guide](../vfs/VFS_API_GUIDE.md) - Virtual Filesystem deep dive
|
||||||
- 🤖 [Neural API](../guides/neural-api.md) - Advanced neural operations
|
- 🤖 [Neural API](../guides/neural-api.md) - Advanced neural operations
|
||||||
- 🌐 [Distributed Guide](../guides/distributed-system.md) - Planet-scale architecture
|
|
||||||
- 💬 [Discord Community](https://discord.gg/brainy) - Get help, share projects
|
- 💬 [Discord Community](https://discord.gg/brainy) - Get help, share projects
|
||||||
|
|
||||||
#### Share Your Success
|
#### Share Your Success
|
||||||
|
|
|
||||||
|
|
@ -1,396 +0,0 @@
|
||||||
# 🔌 Extending Brainy Storage with Augmentations
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
Brainy's zero-config system is **fully extensible**. Augmentations can register new storage providers, presets, and auto-detection logic that integrates seamlessly with the existing system.
|
|
||||||
|
|
||||||
## How Storage Extensions Work
|
|
||||||
|
|
||||||
### 1. Storage Provider Registration
|
|
||||||
|
|
||||||
When an augmentation is installed, it can register a new storage provider:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { StorageProvider, registerStorageAugmentation } from '@soulcraft/brainy/config'
|
|
||||||
|
|
||||||
const redisProvider: StorageProvider = {
|
|
||||||
type: 'redis',
|
|
||||||
name: 'Redis Storage',
|
|
||||||
description: 'High-performance in-memory data store',
|
|
||||||
priority: 10, // Higher priority = checked first in auto-detection
|
|
||||||
|
|
||||||
// Auto-detection logic
|
|
||||||
async detect(): Promise<boolean> {
|
|
||||||
// Check if Redis is available
|
|
||||||
if (process.env.REDIS_URL) {
|
|
||||||
try {
|
|
||||||
const redis = await import('ioredis')
|
|
||||||
const client = new redis.default(process.env.REDIS_URL)
|
|
||||||
await client.ping()
|
|
||||||
await client.quit()
|
|
||||||
return true
|
|
||||||
} catch {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
},
|
|
||||||
|
|
||||||
// Configuration builder
|
|
||||||
async getConfig(): Promise<any> {
|
|
||||||
return {
|
|
||||||
type: 'redis',
|
|
||||||
redisStorage: {
|
|
||||||
url: process.env.REDIS_URL,
|
|
||||||
prefix: 'brainy:',
|
|
||||||
ttl: 3600
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Register the provider
|
|
||||||
registerStorageAugmentation(redisProvider)
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Using Extended Storage
|
|
||||||
|
|
||||||
Once registered, the new storage type works with zero-config:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Auto-detection will now check Redis
|
|
||||||
const brain = new Brainy() // Will use Redis if available!
|
|
||||||
|
|
||||||
// Or explicitly specify
|
|
||||||
const brain = new Brainy({ storage: 'redis' })
|
|
||||||
|
|
||||||
// Or with custom config
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 'redis',
|
|
||||||
redisStorage: {
|
|
||||||
url: 'redis://localhost:6379',
|
|
||||||
prefix: 'myapp:'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## Real-World Examples
|
|
||||||
|
|
||||||
### Redis Augmentation
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// @soulcraft/brainy-redis package
|
|
||||||
export class RedisStorageAugmentation {
|
|
||||||
async init() {
|
|
||||||
// Register the storage provider
|
|
||||||
registerStorageAugmentation({
|
|
||||||
type: 'redis',
|
|
||||||
name: 'Redis Storage',
|
|
||||||
priority: 10,
|
|
||||||
|
|
||||||
async detect() {
|
|
||||||
return !!(process.env.REDIS_URL || process.env.REDIS_HOST)
|
|
||||||
},
|
|
||||||
|
|
||||||
async getConfig() {
|
|
||||||
return {
|
|
||||||
type: 'redis',
|
|
||||||
redisStorage: {
|
|
||||||
url: process.env.REDIS_URL ||
|
|
||||||
`redis://${process.env.REDIS_HOST}:${process.env.REDIS_PORT || 6379}`
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Register Redis-specific presets
|
|
||||||
registerPresetAugmentation('redis-cache', {
|
|
||||||
storage: 'redis',
|
|
||||||
model: ModelPrecision.Q8,
|
|
||||||
features: ['core', 'cache'],
|
|
||||||
distributed: true,
|
|
||||||
description: 'Redis-backed cache layer',
|
|
||||||
category: PresetCategory.SERVICE
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### MongoDB Augmentation
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// @soulcraft/brainy-mongodb package
|
|
||||||
export class MongoStorageAugmentation {
|
|
||||||
async init() {
|
|
||||||
registerStorageAugmentation({
|
|
||||||
type: 'mongodb',
|
|
||||||
name: 'MongoDB Storage',
|
|
||||||
priority: 8,
|
|
||||||
|
|
||||||
async detect() {
|
|
||||||
return !!(process.env.MONGODB_URI || process.env.MONGO_URL)
|
|
||||||
},
|
|
||||||
|
|
||||||
async getConfig() {
|
|
||||||
return {
|
|
||||||
type: 'mongodb',
|
|
||||||
mongoStorage: {
|
|
||||||
uri: process.env.MONGODB_URI,
|
|
||||||
database: 'brainy',
|
|
||||||
collection: 'vectors'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### PostgreSQL + pgvector Augmentation
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// @soulcraft/brainy-postgres package
|
|
||||||
export class PostgresStorageAugmentation {
|
|
||||||
async init() {
|
|
||||||
registerStorageAugmentation({
|
|
||||||
type: 'postgres',
|
|
||||||
name: 'PostgreSQL + pgvector',
|
|
||||||
priority: 9,
|
|
||||||
|
|
||||||
async detect() {
|
|
||||||
const url = process.env.DATABASE_URL
|
|
||||||
if (url?.includes('postgres')) {
|
|
||||||
// Check for pgvector extension
|
|
||||||
const client = new Client({ connectionString: url })
|
|
||||||
await client.connect()
|
|
||||||
const result = await client.query(
|
|
||||||
"SELECT * FROM pg_extension WHERE extname = 'vector'"
|
|
||||||
)
|
|
||||||
await client.end()
|
|
||||||
return result.rows.length > 0
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
},
|
|
||||||
|
|
||||||
async getConfig() {
|
|
||||||
return {
|
|
||||||
type: 'postgres',
|
|
||||||
postgresStorage: {
|
|
||||||
connectionString: process.env.DATABASE_URL,
|
|
||||||
table: 'brainy_vectors'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Auto-Detection Priority
|
|
||||||
|
|
||||||
Storage providers are checked in priority order:
|
|
||||||
|
|
||||||
1. **Custom providers** (highest priority first)
|
|
||||||
2. **Cloud storage** (S3, GCS, R2)
|
|
||||||
3. **Database storage** (Redis, MongoDB, PostgreSQL)
|
|
||||||
4. **Local storage** (filesystem, OPFS)
|
|
||||||
5. **Memory** (fallback)
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Example priority chain
|
|
||||||
Redis (priority: 10) → PostgreSQL (9) → MongoDB (8) → S3 (5) → Filesystem (1) → Memory (0)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Creating Custom Presets
|
|
||||||
|
|
||||||
Augmentations can also register new presets:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
registerPresetAugmentation('redis-cluster', {
|
|
||||||
storage: 'redis',
|
|
||||||
model: ModelPrecision.Q8,
|
|
||||||
features: ['core', 'cache', 'cluster'],
|
|
||||||
distributed: true,
|
|
||||||
role: DistributedRole.HYBRID,
|
|
||||||
cache: {
|
|
||||||
hotCacheMaxSize: 100000, // Large distributed cache
|
|
||||||
autoTune: true
|
|
||||||
},
|
|
||||||
description: 'Redis Cluster configuration',
|
|
||||||
category: PresetCategory.SERVICE
|
|
||||||
})
|
|
||||||
|
|
||||||
// Users can then use:
|
|
||||||
const brain = new Brainy('redis-cluster')
|
|
||||||
```
|
|
||||||
|
|
||||||
## Type Safety with Extensions
|
|
||||||
|
|
||||||
To maintain type safety with dynamic storage types:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Augmentation declares its types
|
|
||||||
declare module '@soulcraft/brainy' {
|
|
||||||
interface StorageTypes {
|
|
||||||
redis: {
|
|
||||||
url: string
|
|
||||||
prefix?: string
|
|
||||||
ttl?: number
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
interface PresetNames {
|
|
||||||
'redis-cache': 'redis-cache'
|
|
||||||
'redis-cluster': 'redis-cluster'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Best Practices for Storage Augmentations
|
|
||||||
|
|
||||||
1. **Always provide auto-detection** - Check environment variables and connectivity
|
|
||||||
2. **Set appropriate priority** - Higher for specialized storage, lower for general
|
|
||||||
3. **Handle failures gracefully** - Return false from detect() if not available
|
|
||||||
4. **Document requirements** - List required packages and environment variables
|
|
||||||
5. **Provide presets** - Include common configuration patterns
|
|
||||||
6. **Maintain compatibility** - Ensure model precision matches across instances
|
|
||||||
|
|
||||||
## Example: Complete Redis Augmentation
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import {
|
|
||||||
StorageProvider,
|
|
||||||
registerStorageAugmentation,
|
|
||||||
registerPresetAugmentation,
|
|
||||||
PresetCategory,
|
|
||||||
ModelPrecision,
|
|
||||||
DistributedRole
|
|
||||||
} from '@soulcraft/brainy/config'
|
|
||||||
import Redis from 'ioredis'
|
|
||||||
|
|
||||||
export class BrainyRedisAugmentation {
|
|
||||||
private client: Redis
|
|
||||||
|
|
||||||
async init() {
|
|
||||||
// Register storage provider
|
|
||||||
registerStorageAugmentation({
|
|
||||||
type: 'redis',
|
|
||||||
name: 'Redis Vector Storage',
|
|
||||||
description: 'Redis with RediSearch for vector similarity',
|
|
||||||
priority: 10,
|
|
||||||
|
|
||||||
requirements: {
|
|
||||||
env: ['REDIS_URL'],
|
|
||||||
packages: ['ioredis', 'redis']
|
|
||||||
},
|
|
||||||
|
|
||||||
async detect() {
|
|
||||||
if (!process.env.REDIS_URL) return false
|
|
||||||
|
|
||||||
try {
|
|
||||||
const client = new Redis(process.env.REDIS_URL)
|
|
||||||
|
|
||||||
// Check for RediSearch module
|
|
||||||
const modules = await client.call('MODULE', 'LIST')
|
|
||||||
const hasRediSearch = modules.some(m => m[1] === 'search')
|
|
||||||
|
|
||||||
await client.quit()
|
|
||||||
return hasRediSearch
|
|
||||||
} catch {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
},
|
|
||||||
|
|
||||||
async getConfig() {
|
|
||||||
return {
|
|
||||||
type: 'redis',
|
|
||||||
redisStorage: {
|
|
||||||
url: process.env.REDIS_URL,
|
|
||||||
prefix: process.env.REDIS_PREFIX || 'brainy:',
|
|
||||||
index: process.env.REDIS_INDEX || 'brainy-vectors',
|
|
||||||
ttl: process.env.REDIS_TTL ? parseInt(process.env.REDIS_TTL) : undefined
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Register presets
|
|
||||||
this.registerPresets()
|
|
||||||
}
|
|
||||||
|
|
||||||
private registerPresets() {
|
|
||||||
// Fast cache preset
|
|
||||||
registerPresetAugmentation('redis-fast-cache', {
|
|
||||||
storage: 'redis' as any,
|
|
||||||
model: ModelPrecision.Q8,
|
|
||||||
features: ['core', 'cache', 'search'],
|
|
||||||
distributed: false,
|
|
||||||
cache: {
|
|
||||||
hotCacheMaxSize: 10000,
|
|
||||||
autoTune: true
|
|
||||||
},
|
|
||||||
description: 'Redis-backed fast cache',
|
|
||||||
category: PresetCategory.SERVICE
|
|
||||||
})
|
|
||||||
|
|
||||||
// Distributed cache preset
|
|
||||||
registerPresetAugmentation('redis-distributed', {
|
|
||||||
storage: 'redis' as any,
|
|
||||||
model: ModelPrecision.AUTO,
|
|
||||||
features: ['core', 'cache', 'search', 'cluster'],
|
|
||||||
distributed: true,
|
|
||||||
role: DistributedRole.HYBRID,
|
|
||||||
cache: {
|
|
||||||
hotCacheMaxSize: 50000,
|
|
||||||
autoTune: true
|
|
||||||
},
|
|
||||||
description: 'Redis distributed cache cluster',
|
|
||||||
category: PresetCategory.SERVICE
|
|
||||||
})
|
|
||||||
|
|
||||||
// Session store preset
|
|
||||||
registerPresetAugmentation('redis-sessions', {
|
|
||||||
storage: 'redis' as any,
|
|
||||||
model: ModelPrecision.Q8,
|
|
||||||
features: ['core', 'cache'],
|
|
||||||
distributed: false,
|
|
||||||
cache: {
|
|
||||||
hotCacheMaxSize: 5000,
|
|
||||||
autoTune: false
|
|
||||||
},
|
|
||||||
description: 'Redis session storage',
|
|
||||||
category: PresetCategory.SERVICE
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Usage after installing the augmentation:
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
import '@soulcraft/brainy-redis' // Registers the augmentation
|
|
||||||
|
|
||||||
// Now Redis is automatically detected!
|
|
||||||
const brain = new Brainy() // Uses Redis if REDIS_URL is set
|
|
||||||
|
|
||||||
// Or use a Redis preset
|
|
||||||
const brain = new Brainy('redis-fast-cache')
|
|
||||||
|
|
||||||
// Or explicitly configure
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: 'redis',
|
|
||||||
model: ModelPrecision.FP32
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
The extensible configuration system allows:
|
|
||||||
|
|
||||||
1. **New storage types** via `registerStorageAugmentation()`
|
|
||||||
2. **Custom presets** via `registerPresetAugmentation()`
|
|
||||||
3. **Auto-detection logic** that integrates with zero-config
|
|
||||||
4. **Type-safe extensions** with TypeScript declarations
|
|
||||||
5. **Priority-based selection** for intelligent defaults
|
|
||||||
|
|
||||||
This ensures Brainy can grow with new storage technologies while maintaining its zero-configuration philosophy!
|
|
||||||
|
|
@ -22,7 +22,7 @@ Brainy's `find()` method is the most advanced query system in any vector databas
|
||||||
### 1. Vector Intelligence (HNSW Index)
|
### 1. Vector Intelligence (HNSW Index)
|
||||||
- **Purpose**: Semantic similarity search using embeddings
|
- **Purpose**: Semantic similarity search using embeddings
|
||||||
- **Algorithm**: Hierarchical Navigable Small World (HNSW)
|
- **Algorithm**: Hierarchical Navigable Small World (HNSW)
|
||||||
- **Performance**: O(log n) search, ~1.8ms typical
|
- **Performance**: O(log n) search
|
||||||
- **Data Structure**: Multi-layer graph with 16 connections per node
|
- **Data Structure**: Multi-layer graph with 16 connections per node
|
||||||
- **Use Cases**: "Find similar documents", "Content like this"
|
- **Use Cases**: "Find similar documents", "Content like this"
|
||||||
|
|
||||||
|
|
@ -36,14 +36,14 @@ Brainy's `find()` method is the most advanced query system in any vector databas
|
||||||
### 3. Metadata Intelligence (Incremental Indices)
|
### 3. Metadata Intelligence (Incremental Indices)
|
||||||
- **Purpose**: Fast filtering on structured data
|
- **Purpose**: Fast filtering on structured data
|
||||||
- **Algorithm**: HashMap for exact matches, Sorted arrays for ranges
|
- **Algorithm**: HashMap for exact matches, Sorted arrays for ranges
|
||||||
- **Performance**: O(1) exact, O(log n) ranges, <1ms typical
|
- **Performance**: O(1) exact, O(log n) ranges
|
||||||
- **Data Structure**: `Map<field:value, Set<id>>` + sorted value arrays
|
- **Data Structure**: `Map<field:value, Set<id>>` + sorted value arrays
|
||||||
- **Use Cases**: "Documents from 2023", "Status equals active"
|
- **Use Cases**: "Documents from 2023", "Status equals active"
|
||||||
|
|
||||||
### 4. Graph Intelligence (Adjacency Maps)
|
### 4. Graph Intelligence (Adjacency Maps)
|
||||||
- **Purpose**: Relationship traversal and connection analysis
|
- **Purpose**: Relationship traversal and connection analysis
|
||||||
- **Algorithm**: Pure O(1) neighbor lookups via Map operations
|
- **Algorithm**: Pure O(1) neighbor lookups via Map operations
|
||||||
- **Performance**: O(1) per hop, ~0.1ms typical
|
- **Performance**: O(1) per hop (measured <1 ms per neighbor lookup, validated up to 1M relationships — `tests/performance/graph-scale-performance.test.ts:238`)
|
||||||
- **Data Structure**: `Map<sourceId, Set<targetId>>`
|
- **Data Structure**: `Map<sourceId, Set<targetId>>`
|
||||||
- **Use Cases**: "Papers connected to MIT", "Authors who collaborated"
|
- **Use Cases**: "Papers connected to MIT", "Authors who collaborated"
|
||||||
|
|
||||||
|
|
@ -373,36 +373,40 @@ return results.slice(offset, offset + limit)
|
||||||
|
|
||||||
### Query Performance by Type
|
### Query Performance by Type
|
||||||
|
|
||||||
| Query Type | Index Used | Performance | Example |
|
| Query Type | Index Used | Complexity | Example |
|
||||||
|------------|------------|-------------|---------|
|
|------------|------------|------------|---------|
|
||||||
| **Semantic Search** | HNSW Vector | O(log n), ~1.8ms | `"AI research papers"` |
|
| **Semantic Search** | HNSW Vector | O(log n) | `"AI research papers"` |
|
||||||
| **Exact Metadata** | HashMap | O(1), ~0.8ms | `{status: "published"}` |
|
| **Exact Metadata** | HashMap | O(1) | `{status: "published"}` |
|
||||||
| **Range Metadata** | Sorted Array | O(log n), ~0.6ms | `{year: {greaterThan: 2020}}` |
|
| **Range Metadata** | Sorted Array | O(log n) | `{year: {greaterThan: 2020}}` |
|
||||||
| **Graph Traversal** | Adjacency Map | O(1), ~0.1ms | `{connected: {to: "mit"}}` |
|
| **Graph Traversal** | Adjacency Map | O(1) | `{connected: {to: "mit"}}` |
|
||||||
| **Type Detection** | Pre-embedded Types | O(t), ~0.3ms | `"documents"` → `NounType.Document` |
|
| **Type Detection** | Pre-embedded Types | O(t) | `"documents"` → `NounType.Document` |
|
||||||
| **Field Matching** | Field Embeddings | O(f), ~0.1ms | `"by"` → `"author"` |
|
| **Field Matching** | Field Embeddings | O(f) | `"by"` → `"author"` |
|
||||||
| **Combined Query** | All Indices | O(log n), ~1.8ms | NLP + filters + graph |
|
| **Combined Query** | All Indices | O(log n) | NLP + filters + graph |
|
||||||
|
|
||||||
Where:
|
Where:
|
||||||
- n = number of entities in database
|
- n = number of entities in database
|
||||||
- t = number of types (169 total: 42 noun + 127 verb)
|
- t = number of types (169 total: 42 noun + 127 verb)
|
||||||
- f = number of fields for detected entity type (typically 5-15)
|
- f = number of fields for detected entity type (typically 5-15)
|
||||||
|
|
||||||
|
Absolute latencies depend on hardware, embedding model, and storage backend; the columns above describe how each stage scales. The graph stage is the one path with a committed scale benchmark (see [Scalability](#scalability)).
|
||||||
|
|
||||||
### Scalability
|
### Scalability
|
||||||
|
|
||||||
| Database Size | Vector Search | Metadata Filter | Graph Query | Combined |
|
The cost of each query stage is governed by its algorithmic complexity, not a fixed millisecond figure — absolute latency depends on hardware, embedding model, and storage backend. Only the graph adjacency index carries a committed scale assertion:
|
||||||
|---------------|---------------|-----------------|-------------|----------|
|
|
||||||
| **1K entities** | 0.8ms | 0.3ms | 0.05ms | 1.1ms |
|
| Query stage | Complexity | Scaling behavior |
|
||||||
| **10K entities** | 1.2ms | 0.5ms | 0.08ms | 1.5ms |
|
|-------------|------------|------------------|
|
||||||
| **100K entities** | 1.8ms | 0.8ms | 0.1ms | 2.1ms |
|
| Metadata filter (exact) | O(1) | Constant — independent of dataset size |
|
||||||
| **1M entities** | 2.5ms | 1.2ms | 0.1ms | 2.8ms |
|
| Metadata filter (range) | O(log n) + O(k) | Sub-linear; k = matching results |
|
||||||
| **10M entities** | 3.8ms | 1.8ms | 0.1ms | 4.2ms |
|
| Vector search (HNSW) | O(log n) | Degrades gracefully via hierarchical layers |
|
||||||
|
| Graph hop | O(1) | Measured <1 ms per neighbor lookup, validated up to 1M relationships (`tests/performance/graph-scale-performance.test.ts:238`) |
|
||||||
|
| Combined query | O(log n) | Bounded by the vector stage; metadata and graph stages stay O(1)/O(log n) |
|
||||||
|
|
||||||
**Key Performance Notes:**
|
**Key Performance Notes:**
|
||||||
- Graph queries stay O(1) regardless of scale
|
- Graph queries stay O(1) regardless of scale
|
||||||
- Metadata ranges scale as O(log n), not O(n)
|
- Metadata ranges scale as O(log n), not O(n)
|
||||||
- Vector search degrades gracefully due to HNSW
|
- Vector search degrades gracefully due to HNSW
|
||||||
- Type-aware NLP adds minimal overhead (~0.4ms)
|
- Type-aware NLP adds minimal overhead (single embedding pass, no full scan)
|
||||||
|
|
||||||
## Example Query Flows
|
## Example Query Flows
|
||||||
|
|
||||||
|
|
@ -455,7 +459,7 @@ await brain.find({
|
||||||
},
|
},
|
||||||
limit: 10
|
limit: 10
|
||||||
})
|
})
|
||||||
// Total performance: ~1.2ms for 100K entities
|
// Executes as O(1) type filter + O(1)/O(log n) metadata + O(1) graph lookup — no full scan
|
||||||
```
|
```
|
||||||
|
|
||||||
## Filter Syntax Reference
|
## Filter Syntax Reference
|
||||||
|
|
@ -1001,7 +1005,7 @@ const results = await brain.find({
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
**Performance**: O(log n) vector search + O(log n) metadata filters + O(1) graph traversal = ~2-3ms total.
|
**Performance**: O(log n) vector search + O(log n) metadata filters + O(1) graph traversal — bounded by the vector stage.
|
||||||
|
|
||||||
### Excluding Soft-Deleted Entities
|
### Excluding Soft-Deleted Entities
|
||||||
|
|
||||||
|
|
@ -1228,7 +1232,7 @@ where: { age: { gt: 18 } }
|
||||||
**No connected entities found**:
|
**No connected entities found**:
|
||||||
```typescript
|
```typescript
|
||||||
// Verify relationship exists
|
// Verify relationship exists
|
||||||
const relations = await brain.getRelations({
|
const relations = await brain.related({
|
||||||
from: 'entity-a',
|
from: 'entity-a',
|
||||||
to: 'entity-b'
|
to: 'entity-b'
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -48,7 +48,7 @@ await storage.setLifecyclePolicy({
|
||||||
```typescript
|
```typescript
|
||||||
// v3: Delete one at a time (slow, expensive)
|
// v3: Delete one at a time (slow, expensive)
|
||||||
for (const id of idsToDelete) {
|
for (const id of idsToDelete) {
|
||||||
await brain.delete(id) // 1000 API calls for 1000 entities
|
await brain.remove(id) // 1000 API calls for 1000 entities
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.0.0: Batch delete (fast, cheap)
|
// v4.0.0: Batch delete (fast, cheap)
|
||||||
|
|
@ -426,16 +426,6 @@ npm install @soulcraft/brainy@^3.50.0
|
||||||
7. Verify data integrity thoroughly
|
7. Verify data integrity thoroughly
|
||||||
8. Enable lifecycle policies gradually
|
8. Enable lifecycle policies gradually
|
||||||
|
|
||||||
### Scenario 4: Multi-Node Distributed System
|
|
||||||
|
|
||||||
**Recommended approach:**
|
|
||||||
1. Perform blue-green deployment:
|
|
||||||
- Keep v3 nodes running (blue)
|
|
||||||
- Deploy v4 nodes (green)
|
|
||||||
- Migrate data once
|
|
||||||
- Switch traffic to v4 nodes
|
|
||||||
- Decommission v3 nodes
|
|
||||||
|
|
||||||
## Cost Savings After Migration
|
## Cost Savings After Migration
|
||||||
|
|
||||||
### Enable All v4.0.0 Features
|
### Enable All v4.0.0 Features
|
||||||
|
|
|
||||||
|
|
@ -2,21 +2,23 @@
|
||||||
|
|
||||||
## Performance Characteristics
|
## Performance Characteristics
|
||||||
|
|
||||||
Brainy achieves industry-leading performance through carefully optimized data structures and algorithms. All performance claims are verified through actual benchmarks on production code.
|
Brainy achieves high performance through carefully optimized data structures and algorithms. The tables below describe each component by its **algorithmic complexity** — the durable, defensible guarantee. The example latencies are figures from a single 100-item run on one machine (see [Benchmarks](#benchmarks)); they are illustrative, not a committed benchmark, and vary with hardware, embedding model, and storage backend. The one component with a committed scale assertion is the graph adjacency index (`tests/performance/graph-scale-performance.test.ts:238`).
|
||||||
|
|
||||||
### Core Performance Summary
|
### Core Performance Summary
|
||||||
|
|
||||||
| Component | Operation | Time Complexity | Measured Performance | Data Structure |
|
| Component | Operation | Time Complexity | Example latency (100-item run)\* | Data Structure |
|
||||||
|-----------|-----------|-----------------|---------------------|----------------|
|
|-----------|-----------|-----------------|---------------------|----------------|
|
||||||
| **Metadata Index** | Exact match | **O(1)** | 0.8ms | `Map<string, Set<string>>` |
|
| **Metadata Index** | Exact match | **O(1)** | 0.8ms | `Map<string, Set<string>>` |
|
||||||
| **Metadata Index** | Range query | **O(log n) + O(k)** | 0.6ms | Sorted array + binary search |
|
| **Metadata Index** | Range query | **O(log n) + O(k)** | 0.6ms | Sorted array + binary search |
|
||||||
| **Graph Index** | Get neighbors | **O(1)** | 0.09ms | `Map<string, Set<string>>` |
|
| **Graph Index** | Get neighbors | **O(1)** | 0.09ms | `Map<string, Set<string>>` |
|
||||||
| **Vector Search** | k-NN search | **O(log n)** | 1.8ms | HNSW hierarchical graph |
|
| **Vector Search** | k-NN search | **O(log n)** | 1.8ms | Hierarchical graph |
|
||||||
| **NLP Parser** | Query parsing | **O(m)** | 8.9ms | 220 pre-computed patterns |
|
| **NLP Parser** | Query parsing | **O(m)** | 8.9ms | 220 pre-computed patterns |
|
||||||
| **Type-Field Affinity** | Field matching | **O(f)** | 0.1ms | Type-specific field cache |
|
| **Type-Field Affinity** | Field matching | **O(f)** | 0.1ms | Type-specific field cache |
|
||||||
| **Type Detection** | Noun/Verb matching | **O(t)** | 0.3ms | Pre-embedded type vectors |
|
| **Type Detection** | Noun/Verb matching | **O(t)** | 0.3ms | Pre-embedded type vectors |
|
||||||
| **Triple Intelligence** | Combined query | **O(1) to O(log n)** | 1.8ms | Parallel execution |
|
| **Triple Intelligence** | Combined query | **O(1) to O(log n)** | 1.8ms | Parallel execution |
|
||||||
|
|
||||||
|
\* Illustrative single-run figures at 100 items on one machine — not a committed benchmark. Only the graph index carries an asserted scale bound (measured <1 ms per neighbor lookup up to 1M relationships, `tests/performance/graph-scale-performance.test.ts:238`).
|
||||||
|
|
||||||
Where:
|
Where:
|
||||||
- `n` = number of items in index
|
- `n` = number of items in index
|
||||||
- `k` = number of results returned
|
- `k` = number of results returned
|
||||||
|
|
@ -26,25 +28,32 @@ Where:
|
||||||
|
|
||||||
### brain.get() Metadata-Only Optimization
|
### brain.get() Metadata-Only Optimization
|
||||||
|
|
||||||
✨ **Massive Performance Improvement**: `brain.get()` is now **76-81% faster** by default!
|
`brain.get()` returns **metadata only by default**, skipping the 384-dimensional
|
||||||
|
embedding — the bulk of an entity's payload. Callers that need the vector opt in
|
||||||
|
with `{ includeVectors: true }`.
|
||||||
|
|
||||||
| Operation | Before | After | Speedup | Use Case |
|
| Operation | Default (metadata-only) | With `includeVectors: true` | Use Case |
|
||||||
|-----------|------------------|-----------------|---------|----------|
|
|-----------|-------------------------|-----------------------------|----------|
|
||||||
| **brain.get() (metadata-only)** | 43ms, 6KB | **10ms, 300 bytes** | **76-81%** | VFS, existence checks, metadata |
|
| **brain.get()** | Skips vector load | Loads full vector | VFS, existence checks, metadata |
|
||||||
| **brain.get({ includeVectors: true })** | 43ms, 6KB | 43ms, 6KB | 0% | Similarity calculations |
|
| **VFS readFile() / readdir()** | Inherits metadata-only path | n/a | File operations, directory listings |
|
||||||
| **VFS readFile()** | 53ms | **~13ms** | **75%** | File operations |
|
|
||||||
| **VFS readdir(100 files)** | 5.3s | **~1.3s** | **75%** | Directory listings |
|
|
||||||
|
|
||||||
**Key Innovation**: Lazy vector loading - only load 384-dimensional embeddings when explicitly needed.
|
**Key Innovation**: Lazy vector loading — only load the 384-dimensional embedding when explicitly needed.
|
||||||
|
|
||||||
|
The integration test `tests/integration/metadata-only-comprehensive.test.ts:306`
|
||||||
|
asserts metadata-only `get()` is faster than the full-entity `get()`
|
||||||
|
(`metadataTime < fullTime`). The *magnitude* of the speedup is
|
||||||
|
environment-dependent (the percentage assertion in
|
||||||
|
`tests/integration/vfs-performance-v5.11.1.test.ts` is intentionally skipped on
|
||||||
|
CI for that reason), so no fixed percentage is quoted here.
|
||||||
|
|
||||||
**Why this matters**:
|
**Why this matters**:
|
||||||
- **94% of brain.get() calls** don't need vectors (VFS, admin tools, import utilities, data APIs)
|
- Most `brain.get()` calls don't need vectors (VFS, admin tools, import utilities, data APIs)
|
||||||
- **Vector data is 95% of entity size** (6KB vectors vs 300 bytes metadata)
|
- The embedding dominates an entity's serialized size, so skipping it is the largest win
|
||||||
- **Zero code changes** for most applications - automatic speedup!
|
- **Zero code changes** for most applications — automatic by default
|
||||||
|
|
||||||
**When to use what**:
|
**When to use what**:
|
||||||
```typescript
|
```typescript
|
||||||
// DEFAULT: Metadata-only (76-81% faster) - use for:
|
// DEFAULT: Metadata-only (skips the vector load) - use for:
|
||||||
const entity = await brain.get(id)
|
const entity = await brain.get(id)
|
||||||
// - VFS operations (readFile, stat, readdir)
|
// - VFS operations (readFile, stat, readdir)
|
||||||
// - Existence checks: if (await brain.get(id)) ...
|
// - Existence checks: if (await brain.get(id)) ...
|
||||||
|
|
@ -55,7 +64,7 @@ const entity = await brain.get(id)
|
||||||
const entity = await brain.get(id, { includeVectors: true })
|
const entity = await brain.get(id, { includeVectors: true })
|
||||||
// - Computing similarity on THIS entity
|
// - Computing similarity on THIS entity
|
||||||
// - Manual vector operations
|
// - Manual vector operations
|
||||||
// - HNSW graph traversal
|
// - Vector index graph traversal
|
||||||
```
|
```
|
||||||
|
|
||||||
## Architecture Deep Dive
|
## Architecture Deep Dive
|
||||||
|
|
@ -143,12 +152,12 @@ class GraphAdjacencyIndex {
|
||||||
|
|
||||||
**Key Innovation:** Pure Map/Set operations - no database queries, no loops, just direct memory access.
|
**Key Innovation:** Pure Map/Set operations - no database queries, no loops, just direct memory access.
|
||||||
|
|
||||||
### 4. HNSW Vector Search - O(log n)
|
### 4. Vector Index - O(log n)
|
||||||
|
|
||||||
Hierarchical Navigable Small World graphs provide logarithmic approximate nearest neighbor search:
|
The default vector index (`JsHnswVectorIndex`) provides logarithmic approximate nearest neighbor search through a hierarchical graph:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
class HNSWIndex {
|
class JsHnswVectorIndex {
|
||||||
private nouns: Map<string, HNSWNoun> = new Map()
|
private nouns: Map<string, HNSWNoun> = new Map()
|
||||||
|
|
||||||
interface HNSWNoun {
|
interface HNSWNoun {
|
||||||
|
|
@ -238,7 +247,7 @@ const results = await Promise.all(searchPromises)
|
||||||
|-----------|--------------|---------|
|
|-----------|--------------|---------|
|
||||||
| Metadata Index | ~40 bytes/entry | `(key_size + 8) × unique_values + 8 × total_items` |
|
| Metadata Index | ~40 bytes/entry | `(key_size + 8) × unique_values + 8 × total_items` |
|
||||||
| Graph Index | ~24 bytes/edge | `16 × edges + 8 × nodes` |
|
| Graph Index | ~24 bytes/edge | `16 × edges + 8 × nodes` |
|
||||||
| HNSW | ~1.5KB/item | `vector_size × 4 + M × 8 × layers` |
|
| Vector Index | ~1.5KB/item | `vector_size × 4 + M × 8 × layers` |
|
||||||
| Pattern Library | 394KB fixed | Pre-computed, shared across instances |
|
| Pattern Library | 394KB fixed | Pre-computed, shared across instances |
|
||||||
| Type Embeddings | ~60KB fixed | 70 types × 384 dimensions × 4 bytes, cached |
|
| Type Embeddings | ~60KB fixed | 70 types × 384 dimensions × 4 bytes, cached |
|
||||||
| Field Embeddings | ~5KB dynamic | Actual fields × 384 dimensions × 4 bytes |
|
| Field Embeddings | ~5KB dynamic | Actual fields × 384 dimensions × 4 bytes |
|
||||||
|
|
@ -252,34 +261,40 @@ const results = await Promise.all(searchPromises)
|
||||||
|
|
||||||
## Benchmarks
|
## Benchmarks
|
||||||
|
|
||||||
### Real-world Performance Test (100 items)
|
### Illustrative Single Run (100 items, one machine)
|
||||||
|
|
||||||
|
Example output from a single 100-item run — illustrative only, not a committed
|
||||||
|
benchmark; absolute numbers vary by hardware. The values feed the
|
||||||
|
[Core Performance Summary](#core-performance-summary) example-latency column.
|
||||||
|
|
||||||
```
|
```
|
||||||
📊 Metadata exact match: 0.818ms (50 items matched)
|
Metadata exact match: 0.818ms (50 items matched)
|
||||||
📊 Metadata range query: 0.631ms (40 items in range)
|
Metadata range query: 0.631ms (40 items in range)
|
||||||
🔗 Graph neighbor lookup: 0.092ms (2 connections)
|
Graph neighbor lookup: 0.092ms (2 connections)
|
||||||
🎯 Vector k-NN search: 1.773ms (10 nearest neighbors)
|
Vector k-NN search: 1.773ms (10 nearest neighbors)
|
||||||
🧠 NLP query parsing: 8.906ms (full natural language)
|
NLP query parsing: 8.906ms (full natural language)
|
||||||
⚡ Triple Intelligence: 1.830ms (combined query)
|
Triple Intelligence: 1.830ms (combined query)
|
||||||
```
|
```
|
||||||
|
|
||||||
### Scaling Characteristics
|
### Scaling Characteristics
|
||||||
|
|
||||||
| Items | Metadata O(1) | Range O(log n) | Graph O(1) | Vector O(log n) |
|
Each stage scales by its algorithmic complexity, not a fixed millisecond figure
|
||||||
|-------|---------------|----------------|------------|-----------------|
|
— absolute latency depends on hardware, embedding model, and storage backend.
|
||||||
| 100 | 0.8ms | 0.6ms | 0.09ms | 1.8ms |
|
Only the graph adjacency index carries a committed scale assertion:
|
||||||
| 1,000 | 0.8ms | 0.9ms | 0.09ms | 2.5ms |
|
|
||||||
| 10,000 | 0.8ms | 1.2ms | 0.09ms | 3.2ms |
|
|
||||||
| 100,000 | 0.8ms | 1.5ms | 0.09ms | 4.1ms |
|
|
||||||
| 1,000,000 | 0.8ms | 1.8ms | 0.09ms | 5.0ms |
|
|
||||||
|
|
||||||
*Note: O(1) operations maintain constant time regardless of scale*
|
| Query stage | Complexity | Scaling behavior |
|
||||||
|
|-------------|------------|------------------|
|
||||||
|
| Metadata filter (exact) | O(1) | Constant — independent of dataset size |
|
||||||
|
| Metadata filter (range) | O(log n) + O(k) | Sub-linear; k = matching results |
|
||||||
|
| Vector search (HNSW) | O(log n) | Degrades gracefully via hierarchical layers |
|
||||||
|
| Graph hop | O(1) | Measured <1 ms per neighbor lookup, validated up to 1M relationships (`tests/performance/graph-scale-performance.test.ts:238`) |
|
||||||
|
| Combined query | O(log n) | Bounded by the vector stage; metadata and graph stages stay O(1)/O(log n) |
|
||||||
|
|
||||||
## Comparison with Other Systems
|
## Comparison with Other Systems
|
||||||
|
|
||||||
| System | Metadata Filter | Graph Traversal | Vector Search | Natural Language |
|
| System | Metadata Filter | Graph Traversal | Vector Search | Natural Language |
|
||||||
|--------|-----------------|-----------------|---------------|------------------|
|
|--------|-----------------|-----------------|---------------|------------------|
|
||||||
| **Brainy** | O(1) HashMap | O(1) Adjacency | O(log n) HNSW | 220 patterns |
|
| **Brainy** | O(1) HashMap | O(1) Adjacency | O(log n) vector index | 220 patterns |
|
||||||
| Neo4j | O(log n) B-tree | O(k) traversal | Not native | Not native |
|
| Neo4j | O(log n) B-tree | O(k) traversal | Not native | Not native |
|
||||||
| Elasticsearch | O(log n) inverted | Not native | O(n) brute force* | Basic tokenization |
|
| Elasticsearch | O(log n) inverted | Not native | O(n) brute force* | Basic tokenization |
|
||||||
| PostgreSQL | O(log n) B-tree | O(k) recursive | O(n) brute force* | Full-text only |
|
| PostgreSQL | O(log n) B-tree | O(k) recursive | O(n) brute force* | Full-text only |
|
||||||
|
|
@ -305,7 +320,7 @@ const results = await Promise.all(searchPromises)
|
||||||
- ✅ **No Network Calls**: Everything runs locally, including embeddings
|
- ✅ **No Network Calls**: Everything runs locally, including embeddings
|
||||||
- ✅ **Thread-Safe**: Immutable data structures where possible
|
- ✅ **Thread-Safe**: Immutable data structures where possible
|
||||||
- ✅ **Memory Bounded**: Configurable cache sizes and automatic cleanup
|
- ✅ **Memory Bounded**: Configurable cache sizes and automatic cleanup
|
||||||
- ✅ **Horizontally Scalable**: Stateless operations support clustering
|
- ✅ **Single-Node by Design**: One process owns one `path`; scale out at the service layer
|
||||||
- ✅ **Zero Stubs**: Every line of code is production-ready
|
- ✅ **Zero Stubs**: Every line of code is production-ready
|
||||||
|
|
||||||
## Lazy Loading Performance
|
## Lazy Loading Performance
|
||||||
|
|
@ -375,110 +390,47 @@ const brain = new Brainy({ disableAutoRebuild: true })
|
||||||
await brain.init() // Instant (0-10ms)
|
await brain.init() // Instant (0-10ms)
|
||||||
```
|
```
|
||||||
|
|
||||||
### Automatic Self-Tuning (Current & Planned)
|
### Automatic Self-Tuning
|
||||||
|
|
||||||
**✅ Currently Implemented:**
|
|
||||||
- **Metadata Index**: Auto-builds sorted indices for range queries on first use
|
- **Metadata Index**: Auto-builds sorted indices for range queries on first use
|
||||||
- **Graph Index**: Auto-flushes every 30 seconds
|
- **Graph Index**: Auto-flushes every 30 seconds
|
||||||
- **Default Tuning**: Research-based defaults (M=16, ef=200)
|
- **Default Tuning**: Research-based vector index defaults
|
||||||
- **Lazy Loading**: Indices built only when needed
|
- **Lazy Loading**: Indices built only when needed
|
||||||
- **Cache Management**: LRU caches with TTL
|
- **Cache Management**: LRU caches with TTL
|
||||||
|
|
||||||
**🚧 Planned Enhancements:**
|
|
||||||
- **Dynamic Storage Selection**: Auto-switch between memory/disk based on size
|
|
||||||
- **Adaptive Index Parameters**: Adjust M and ef based on query patterns
|
|
||||||
- **Smart Cache Sizing**: Scale caches based on available memory
|
|
||||||
- **Predictive Optimization**: Learn from usage patterns
|
|
||||||
|
|
||||||
### Intelligent Defaults
|
### Intelligent Defaults
|
||||||
|
|
||||||
All defaults are research-based and production-tested:
|
- **Vector recall** = `'balanced'` (M=16, ef=200): right for most datasets
|
||||||
- **HNSW M=16**: Optimal balance of recall/speed for most datasets
|
- **Cache TTL** = 5 min: balances freshness and performance
|
||||||
- **efConstruction=200**: High quality graph construction
|
- **Flush interval** = 30 s: non-blocking background persistence
|
||||||
- **Cache TTL=5min**: Balances freshness with performance
|
|
||||||
- **Flush Interval=30s**: Non-blocking background persistence
|
|
||||||
|
|
||||||
### Progressive Enhancement
|
### Vector Index Tuning Knobs
|
||||||
|
|
||||||
Brainy learns and improves over time:
|
Brainy 8.0 exposes two knobs on `config.vector`:
|
||||||
1. **Query Pattern Learning**: Frequently used patterns get cached
|
|
||||||
2. **Index Optimization**: Auto-rebuilds indices when fragmented
|
|
||||||
3. **Memory Management**: Coordinates caches across all components
|
|
||||||
4. **Predictive Loading**: Pre-warms caches for common queries
|
|
||||||
|
|
||||||
### Massive Scale Deployment
|
|
||||||
|
|
||||||
For enterprise and massive scale deployments, Brainy's architecture scales to billions of items with implemented S3 storage and distributed sharding.
|
|
||||||
|
|
||||||
**Currently Implemented:**
|
|
||||||
- Memory storage (production-ready)
|
|
||||||
- Disk storage (production-ready)
|
|
||||||
- S3-compatible storage (AWS S3, Cloudflare R2, Google Cloud Storage, MinIO, Backblaze B2)
|
|
||||||
- Distributed sharding with ConsistentHashRing
|
|
||||||
- Single-node deployment (scales to ~1M items)
|
|
||||||
- Multi-node deployment with sharding (scales to billions)
|
|
||||||
|
|
||||||
**Available Today:**
|
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// S3-compatible storage for unlimited scale - WORKS NOW
|
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: {
|
vector: {
|
||||||
type: 's3',
|
recall: 'fast', // 'fast' | 'balanced' | 'accurate'
|
||||||
bucketName: 'my-brainy-data',
|
persistMode: 'deferred' // 'immediate' | 'deferred'
|
||||||
region: 'us-east-1',
|
|
||||||
credentials: {
|
|
||||||
accessKeyId: 'YOUR_ACCESS_KEY',
|
|
||||||
secretAccessKey: 'YOUR_SECRET_KEY'
|
|
||||||
}
|
|
||||||
// Works with: AWS S3, MinIO, Cloudflare R2, Backblaze B2, Google Cloud Storage
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Cloudflare R2 storage - WORKS NOW
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 'r2',
|
|
||||||
bucketName: 'my-brainy-data',
|
|
||||||
accountId: 'YOUR_ACCOUNT_ID',
|
|
||||||
accessKeyId: 'YOUR_R2_ACCESS_KEY',
|
|
||||||
secretAccessKey: 'YOUR_R2_SECRET_KEY'
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Google Cloud Storage - WORKS NOW
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 'gcs',
|
|
||||||
bucketName: 'my-brainy-data',
|
|
||||||
region: 'us-central1',
|
|
||||||
credentials: {
|
|
||||||
accessKeyId: 'YOUR_ACCESS_KEY',
|
|
||||||
secretAccessKey: 'YOUR_SECRET_KEY'
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The default JS index is `JsHnswVectorIndex`. An optional native acceleration package (`@soulcraft/cortex`) can replace it with a higher-performing implementation; the public knobs stay the same.
|
||||||
|
|
||||||
### Scale Scenarios
|
### Scale Scenarios
|
||||||
|
|
||||||
| Scale | Items | Storage Strategy | Performance | Status |
|
| Scale | Items | Storage Strategy | Performance |
|
||||||
|-------|-------|-----------------|-------------|--------|
|
|-------|-------|------------------|-------------|
|
||||||
| **Small** | <10K | Memory (automatic) | Sub-millisecond | ✅ Implemented |
|
| **Small** | <10K | Memory | Sub-millisecond |
|
||||||
| **Medium** | 10K-1M | Disk with memory cache | 1-5ms | ✅ Implemented |
|
| **Medium** | 10K-1M | Filesystem | 1-5ms |
|
||||||
| **Large** | 1M-100M | S3 with memory cache | 2-10ms | ✅ Implemented |
|
| **Large** | 1M-10M | Filesystem + tuned cache | 2-10ms |
|
||||||
| **Massive** | 100M-10B | S3 + distributed sharding | 5-20ms | ✅ Implemented |
|
| **Massive** | 10M+ | Filesystem + native vector provider + service-layer sharding | 5-20ms |
|
||||||
| **Planetary** | 10B+ | Multi-region S3 + Edge cache | 10-50ms | 🚧 Roadmap |
|
|
||||||
|
|
||||||
### S3-Compatible Storage Benefits
|
For >10M entities, run multiple Brainy processes behind your own routing layer — Brainy 8.0 doesn't ship cluster coordination.
|
||||||
|
|
||||||
- **Unlimited Scale**: No practical limit on dataset size
|
### Architecture
|
||||||
- **Cost Effective**: $0.023/GB/month for standard storage
|
|
||||||
- **Durability**: 99.999999999% (11 9's) durability
|
|
||||||
- **Global**: Multi-region replication available
|
|
||||||
- **Compatible**: Works with any S3-compatible API (MinIO, R2, B2)
|
|
||||||
|
|
||||||
### Distributed Architecture (Implemented)
|
|
||||||
|
|
||||||
```
|
```
|
||||||
┌─────────────────────────────────────────┐
|
┌─────────────────────────────────────────┐
|
||||||
|
|
@ -490,103 +442,51 @@ const brain = new Brainy({
|
||||||
│ Brainy Core │
|
│ Brainy Core │
|
||||||
│ (Triple Intelligence Engine) │
|
│ (Triple Intelligence Engine) │
|
||||||
├─────────────────────────────────────────┤
|
├─────────────────────────────────────────┤
|
||||||
│ Memory │ Shard │ Metadata │
|
│ Memory │ Vector │ Metadata │
|
||||||
│ Cache │ Manager │ Index │
|
│ Cache │ Index │ Index │
|
||||||
└─────────────┬───────────────────────────┘
|
└─────────────┬───────────────────────────┘
|
||||||
│
|
│
|
||||||
┌─────────────▼───────────────────────────┐
|
┌─────────────▼───────────────────────────┐
|
||||||
│ Storage Layer │
|
│ Storage Layer │
|
||||||
├──────────┬──────────┬──────────────────┤
|
├──────────┬──────────┬──────────────────┤
|
||||||
│ HNSW │ Graph │ Objects │
|
│ Vectors │ Graph │ Files │
|
||||||
│ Vectors │ Edges │ (S3/R2/GCS) │
|
│ (sharded)│ Edges │ (filesystem) │
|
||||||
└──────────┴──────────┴──────────────────┘
|
└──────────┴──────────┴──────────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
**Distributed Sharding (Implemented):**
|
For off-site replication, snapshot `path` from your scheduler (`gsutil rsync`, `aws s3 sync`, `rclone`, or `tar`).
|
||||||
- ConsistentHashRing with 150 virtual nodes
|
|
||||||
- 64 shards by default
|
|
||||||
- Replication factor of 3
|
|
||||||
- Automatic rebalancing on node addition/removal
|
|
||||||
|
|
||||||
### Auto-Sharding for Horizontal Scale (Implemented)
|
|
||||||
|
|
||||||
Brainy includes a complete sharding implementation with ConsistentHashRing:
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
import { ShardManager } from '@soulcraft/brainy/distributed'
|
|
||||||
|
|
||||||
// Create shard manager with custom configuration
|
|
||||||
const shardManager = new ShardManager({
|
|
||||||
shardCount: 64, // Default: 64 shards
|
|
||||||
replicationFactor: 3, // Default: 3 replicas
|
|
||||||
virtualNodes: 150, // Default: 150 virtual nodes
|
|
||||||
autoRebalance: true // Default: true
|
|
||||||
})
|
|
||||||
|
|
||||||
// Add nodes to the cluster
|
|
||||||
shardManager.addNode('node-1')
|
|
||||||
shardManager.addNode('node-2')
|
|
||||||
shardManager.addNode('node-3')
|
|
||||||
|
|
||||||
// Sharding automatically:
|
|
||||||
// - Uses consistent hashing for even distribution
|
|
||||||
// - Maintains replicas for fault tolerance
|
|
||||||
// - Rebalances on node changes
|
|
||||||
// - Provides O(1) shard lookups
|
|
||||||
```
|
|
||||||
|
|
||||||
### Performance at Scale
|
### Performance at Scale
|
||||||
|
|
||||||
Even at massive scale, Brainy maintains excellent performance:
|
- **Metadata queries**: O(1) HashMap
|
||||||
|
- **Graph traversal**: O(1) adjacency lookup
|
||||||
- **Metadata queries**: Still O(1) with distributed hash tables
|
- **Vector search**: O(log n)
|
||||||
- **Graph traversal**: O(1) with edge locality optimization
|
- **Write throughput**: 50K+ writes/second per process (filesystem, batched)
|
||||||
- **Vector search**: O(log n) with hierarchical sharding
|
|
||||||
- **Write throughput**: 100K+ writes/second with S3 batching
|
|
||||||
- **Read throughput**: 1M+ reads/second with caching
|
- **Read throughput**: 1M+ reads/second with caching
|
||||||
|
|
||||||
### Zero-Config with Autoscaling (Implemented)
|
### Zero-Config with Autoscaling
|
||||||
|
|
||||||
Brainy includes extensive autoscaling capabilities:
|
|
||||||
|
|
||||||
**✅ Implemented Autoscaling:**
|
|
||||||
- **AutoConfiguration System**: Detects environment and adjusts settings
|
- **AutoConfiguration System**: Detects environment and adjusts settings
|
||||||
- **Learning from Performance**: `learnFromPerformance()` adapts based on metrics
|
- **Learning from Performance**: `learnFromPerformance()` adapts based on metrics
|
||||||
- **Auto-flush**: Graph index (30s), Metadata index (configurable)
|
- **Auto-flush**: Graph index (30s), Metadata index (configurable)
|
||||||
- **Auto-optimize**: Enabled by default in Graph and HNSW indices
|
- **Auto-optimize**: Enabled by default in graph and vector indices
|
||||||
- **Auto-rebalance**: Shards automatically rebalance on node changes
|
|
||||||
- **Zero-config presets**: Production, development, minimal modes
|
- **Zero-config presets**: Production, development, minimal modes
|
||||||
- **Adaptive memory**: Scales caches based on available memory
|
- **Adaptive memory**: Scales caches based on available memory
|
||||||
- **Environment detection**: Browser vs Node.js vs Serverless
|
|
||||||
|
|
||||||
**🚧 Roadmap Autoscaling:**
|
|
||||||
- Dynamic HNSW parameter adjustment (M, ef)
|
|
||||||
- Predictive query pattern caching
|
|
||||||
- Multi-region auto-replication
|
|
||||||
- Automatic cross-node data migration
|
|
||||||
|
|
||||||
## Implementation Status
|
## Implementation Status
|
||||||
|
|
||||||
### ✅ Fully Implemented and Production-Ready
|
### Fully Implemented and Production-Ready
|
||||||
- **O(1) metadata lookups** via HashMaps (exact match)
|
- **O(1) metadata lookups** via HashMaps (exact match)
|
||||||
- **O(log n) range queries** via sorted arrays with lazy building
|
- **O(log n) range queries** via sorted arrays with lazy building
|
||||||
- **O(1) graph traversal** via adjacency maps
|
- **O(1) graph traversal** via adjacency maps
|
||||||
- **O(log n) vector search** via HNSW
|
- **O(log n) vector search** via the default JS index, swappable for a native provider
|
||||||
- **220 NLP patterns** with pre-computed embeddings
|
- **220 NLP patterns** with pre-computed embeddings
|
||||||
- **S3-compatible storage** (AWS S3, R2, GCS, MinIO, B2)
|
- **Filesystem and memory storage** adapters
|
||||||
- **Distributed sharding** with ConsistentHashRing
|
|
||||||
- **Auto-configuration system** with environment detection
|
- **Auto-configuration system** with environment detection
|
||||||
- **Zero-config operation** with intelligent defaults
|
- **Zero-config operation** with intelligent defaults
|
||||||
- **Auto-flush and auto-optimize** in indices
|
- **Auto-flush and auto-optimize** in indices
|
||||||
- **Sub-2ms response times** for complex queries
|
- **Low-latency Triple Intelligence queries** (O(log n) vector + O(1) metadata/graph)
|
||||||
|
|
||||||
### 🚧 Roadmap Features
|
|
||||||
- Dynamic HNSW parameter tuning
|
|
||||||
- Predictive query pattern caching
|
|
||||||
- Multi-region S3 replication
|
|
||||||
- Automatic cross-node data migration
|
|
||||||
- Edge caching layer
|
|
||||||
|
|
||||||
## Conclusion
|
## Conclusion
|
||||||
|
|
||||||
Brainy delivers on its promise of **production-ready Triple Intelligence** with measured, verified performance characteristics. All listed features are fully implemented, tested, and benchmarked. No stubs, no mocks, no theoretical claims - just real, working code with measured performance.
|
Brainy delivers on its promise of **production-ready Triple Intelligence** with documented algorithmic-complexity guarantees and a committed graph-scale benchmark (`tests/performance/graph-scale-performance.test.ts`). All listed features are fully implemented and tested. No stubs, no mocks — just real, working code with characterized performance.
|
||||||
|
|
@ -5,15 +5,14 @@ public: true
|
||||||
category: guides
|
category: guides
|
||||||
template: guide
|
template: guide
|
||||||
order: 4
|
order: 4
|
||||||
description: Replace any Brainy subsystem — distance functions, embeddings, HNSW index, metadata index, aggregation — with a custom implementation or native Rust via Cortex.
|
description: Replace any Brainy subsystem — distance functions, embeddings, vector index, metadata index, aggregation — with a custom implementation or optional native acceleration.
|
||||||
next:
|
next:
|
||||||
- cortex/comparison
|
|
||||||
- guides/storage-adapters
|
- guides/storage-adapters
|
||||||
---
|
---
|
||||||
|
|
||||||
# Plugin Development Guide
|
# Plugin Development Guide
|
||||||
|
|
||||||
Brainy has a plugin system that allows third-party packages to replace internal subsystems with custom implementations. This is how `@soulcraft/cortex` provides native Rust acceleration, and it's the same system available to any developer.
|
Brainy has a plugin system that allows third-party packages to replace internal subsystems with custom implementations. This is how `@soulcraft/cortex` provides optional native acceleration, and it's the same system available to any developer.
|
||||||
|
|
||||||
## Architecture Overview
|
## Architecture Overview
|
||||||
|
|
||||||
|
|
@ -28,7 +27,7 @@ Plugins are **opt-in** — brainy never auto-imports packages. You must explicit
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
plugins: ['@soulcraft/cortex'] // explicitly load cortex
|
plugins: ['@soulcraft/cortex'] // explicitly load the native acceleration package
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -109,7 +108,7 @@ Each key has a specific expected signature. Brainy checks for these during `init
|
||||||
#### `distance`
|
#### `distance`
|
||||||
**Type:** `(a: number[], b: number[]) => number`
|
**Type:** `(a: number[], b: number[]) => number`
|
||||||
|
|
||||||
Replaces the default cosine distance function used in HNSW search and neural APIs. This is the highest-impact single provider — it's called for every vector comparison.
|
Replaces the default cosine distance function used in vector search and neural APIs. This is the highest-impact single provider — it's called for every vector comparison.
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
context.registerProvider('distance', (a: number[], b: number[]): number => {
|
context.registerProvider('distance', (a: number[], b: number[]): number => {
|
||||||
|
|
@ -151,10 +150,10 @@ context.registerProvider('embedBatch', async (texts: string[]) => {
|
||||||
|
|
||||||
### Index Providers
|
### Index Providers
|
||||||
|
|
||||||
#### `hnsw`
|
#### `vector`
|
||||||
**Type:** `(config: object, distanceFunction: Function, options: object) => HNSWIndex-compatible`
|
**Type:** `(config: object, distanceFunction: Function, options: object) => VectorIndexProvider-compatible`
|
||||||
|
|
||||||
Factory function that creates an HNSW index instance. The returned object must implement the `HNSWIndex` public API:
|
Factory function that creates a vector index instance. The returned object must implement the `VectorIndexProvider` public API:
|
||||||
|
|
||||||
- `addItem(item: { id: string, vector: number[] }): Promise<string>`
|
- `addItem(item: { id: string, vector: number[] }): Promise<string>`
|
||||||
- `search(queryVector: number[], k: number, filter?, options?): Promise<Array<[string, number]>>`
|
- `search(queryVector: number[], k: number, filter?, options?): Promise<Array<[string, number]>>`
|
||||||
|
|
@ -174,12 +173,12 @@ Factory function that creates an HNSW index instance. The returned object must i
|
||||||
- `setUseParallelization(boolean): void`
|
- `setUseParallelization(boolean): void`
|
||||||
|
|
||||||
For type-aware indexes (separate graph per noun type), also implement:
|
For type-aware indexes (separate graph per noun type), also implement:
|
||||||
- `getIndexForType(type: string): HNSWIndex` (duck-typed detection)
|
- `getIndexForType(type: string): VectorIndexProvider` (duck-typed detection)
|
||||||
- `search(queryVector, k, type?, filter?, options?): Promise<Array<[string, number]>>`
|
- `search(queryVector, k, type?, filter?, options?): Promise<Array<[string, number]>>`
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
context.registerProvider('hnsw', (config, distanceFn, options) => {
|
context.registerProvider('vector', (config, distanceFn, options) => {
|
||||||
return new MyNativeHNSWIndex(config, distanceFn, options)
|
return new MyNativeVectorIndex(config, distanceFn, options)
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -216,7 +215,7 @@ context.registerProvider('aggregation', (storage) => {
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
When provided by a native plugin like `@soulcraft/cortex`, this enables:
|
When provided by an optional native acceleration plugin (such as `@soulcraft/cortex`), this enables:
|
||||||
- Compiled source filters (vs per-entity JS object traversal)
|
- Compiled source filters (vs per-entity JS object traversal)
|
||||||
- Precise MIN/MAX via sorted data structures (vs lazy recompute)
|
- Precise MIN/MAX via sorted data structures (vs lazy recompute)
|
||||||
- Parallel aggregate rebuild across CPU cores
|
- Parallel aggregate rebuild across CPU cores
|
||||||
|
|
@ -227,7 +226,7 @@ When provided by a native plugin like `@soulcraft/cortex`, this enables:
|
||||||
#### `cache`
|
#### `cache`
|
||||||
**Type:** `UnifiedCache`
|
**Type:** `UnifiedCache`
|
||||||
|
|
||||||
Replaces the global `UnifiedCache` singleton used for VFS path resolution, semantic caching, and HNSW vector caching. Must implement the `UnifiedCache` interface (available from `@soulcraft/brainy/internals`).
|
Replaces the global `UnifiedCache` singleton used for VFS path resolution, semantic caching, and vector index caching. Must implement the `UnifiedCache` interface (available from `@soulcraft/brainy/internals`).
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import type { UnifiedCache } from '@soulcraft/brainy/internals'
|
import type { UnifiedCache } from '@soulcraft/brainy/internals'
|
||||||
|
|
@ -252,7 +251,7 @@ Native msgpack encode/decode for SSTable serialization.
|
||||||
|
|
||||||
### Analytics Providers (Native-Only)
|
### Analytics Providers (Native-Only)
|
||||||
|
|
||||||
These provider keys have **no JavaScript fallback** — they represent capabilities that require native code (SIMD, mmap, sub-microsecond latency). They are available when a native plugin like `@soulcraft/cortex` is installed.
|
These provider keys have **no JavaScript fallback** — they represent capabilities that require native code (SIMD, mmap, sub-microsecond latency). They are available when an optional native acceleration plugin (such as `@soulcraft/cortex`) is installed.
|
||||||
|
|
||||||
Use `brain.getProvider('analytics:hyperloglog')` to check availability. Returns `undefined` if no plugin provides it.
|
Use `brain.getProvider('analytics:hyperloglog')` to check availability. Returns `undefined` if no plugin provides it.
|
||||||
|
|
||||||
|
|
@ -338,11 +337,11 @@ console.log(diag)
|
||||||
// embeddings: { source: 'plugin' },
|
// embeddings: { source: 'plugin' },
|
||||||
// embedBatch: { source: 'plugin' },
|
// embedBatch: { source: 'plugin' },
|
||||||
// distance: { source: 'plugin' },
|
// distance: { source: 'plugin' },
|
||||||
// hnsw: { source: 'default' },
|
// vector: { source: 'default' },
|
||||||
// ...
|
// ...
|
||||||
// },
|
// },
|
||||||
// indexes: {
|
// indexes: {
|
||||||
// hnsw: { size: 0, type: 'TypeAwareHNSWIndex' },
|
// vector: { size: 0, type: 'JsHnswVectorIndex' },
|
||||||
// metadata: { type: 'MetadataIndexManager', initialized: true },
|
// metadata: { type: 'MetadataIndexManager', initialized: true },
|
||||||
// graph: { type: 'GraphAdjacencyIndex', initialized: true, wiredToStorage: true }
|
// graph: { type: 'GraphAdjacencyIndex', initialized: true, wiredToStorage: true }
|
||||||
// }
|
// }
|
||||||
|
|
@ -361,7 +360,7 @@ When a plugin is active, brainy automatically logs a provider summary after `ini
|
||||||
|
|
||||||
```
|
```
|
||||||
[brainy] Plugin activated: @soulcraft/cortex
|
[brainy] Plugin activated: @soulcraft/cortex
|
||||||
[brainy] Providers: 8/10 native (@soulcraft/cortex) | default: hnsw, cache
|
[brainy] Providers: 8/10 native (@soulcraft/cortex) | default: vector, cache
|
||||||
```
|
```
|
||||||
|
|
||||||
This tells you at a glance how many subsystems are accelerated and which ones are falling back to JavaScript. The log respects `config.silent`.
|
This tells you at a glance how many subsystems are accelerated and which ones are falling back to JavaScript. The log respects `config.silent`.
|
||||||
|
|
|
||||||
|
|
@ -215,9 +215,9 @@ console.log('Server running on http://localhost:3000')
|
||||||
```
|
```
|
||||||
|
|
||||||
**Benefits:**
|
**Benefits:**
|
||||||
- ✅ Native Bun performance (~2x faster than Node.js)
|
- ✅ Native Bun runtime performance
|
||||||
- ✅ No framework dependencies
|
- ✅ No framework dependencies
|
||||||
- ✅ Works with `bun --compile` for single-binary deployment
|
- ✅ Pure WASM — no native binaries, bundler-friendly
|
||||||
- ✅ Built-in TypeScript support
|
- ✅ Built-in TypeScript support
|
||||||
|
|
||||||
### Pattern 4: Express/Node.js Middleware (Legacy)
|
### Pattern 4: Express/Node.js Middleware (Legacy)
|
||||||
|
|
|
||||||
|
|
@ -10,8 +10,8 @@ All operators work with `find({ where: { ... } })` and filter on **metadata fiel
|
||||||
|
|
||||||
| Operator | Alias | Description | Example |
|
| Operator | Alias | Description | Example |
|
||||||
|----------|-------|-------------|---------|
|
|----------|-------|-------------|---------|
|
||||||
| `equals` | `eq`, `is` | Exact match | `{ status: { equals: 'active' } }` |
|
| `eq` | `equals` | Exact match | `{ status: { eq: 'active' } }` |
|
||||||
| `notEquals` | `ne`, `isNot` | Not equal | `{ status: { notEquals: 'deleted' } }` |
|
| `ne` | `notEquals` | Not equal | `{ status: { ne: 'deleted' } }` |
|
||||||
|
|
||||||
**Shorthand:** A bare value is treated as `equals`:
|
**Shorthand:** A bare value is treated as `equals`:
|
||||||
|
|
||||||
|
|
@ -27,10 +27,10 @@ brain.find({ where: { status: { equals: 'active' } } })
|
||||||
|
|
||||||
| Operator | Alias | Description | Example |
|
| Operator | Alias | Description | Example |
|
||||||
|----------|-------|-------------|---------|
|
|----------|-------|-------------|---------|
|
||||||
| `greaterThan` | `gt` | Greater than | `{ age: { greaterThan: 18 } }` |
|
| `gt` | `greaterThan` | Greater than | `{ age: { gt: 18 } }` |
|
||||||
| `greaterEqual` | `gte` | Greater or equal | `{ score: { greaterEqual: 90 } }` |
|
| `gte` | `greaterThanOrEqual` | Greater or equal | `{ score: { gte: 90 } }` |
|
||||||
| `lessThan` | `lt` | Less than | `{ price: { lessThan: 100 } }` |
|
| `lt` | `lessThan` | Less than | `{ price: { lt: 100 } }` |
|
||||||
| `lessEqual` | `lte` | Less or equal | `{ rating: { lessEqual: 3 } }` |
|
| `lte` | `lessThanOrEqual` | Less or equal | `{ rating: { lte: 3 } }` |
|
||||||
| `between` | — | Inclusive range `[min, max]` | `{ year: { between: [2020, 2025] } }` |
|
| `between` | — | Inclusive range `[min, max]` | `{ year: { between: [2020, 2025] } }` |
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
|
|
@ -160,9 +160,9 @@ Brainy's MetadataIndex supports a subset of operators natively for O(1) field lo
|
||||||
| `equals` / `eq` | Yes | Yes |
|
| `equals` / `eq` | Yes | Yes |
|
||||||
| `notEquals` / `ne` | — | Yes |
|
| `notEquals` / `ne` | — | Yes |
|
||||||
| `greaterThan` / `gt` | Yes | Yes |
|
| `greaterThan` / `gt` | Yes | Yes |
|
||||||
| `greaterEqual` / `gte` | Yes | Yes |
|
| `greaterThanOrEqual` / `gte` | Yes | Yes |
|
||||||
| `lessThan` / `lt` | Yes | Yes |
|
| `lessThan` / `lt` | Yes | Yes |
|
||||||
| `lessEqual` / `lte` | Yes | Yes |
|
| `lessThanOrEqual` / `lte` | Yes | Yes |
|
||||||
| `between` | Yes | Yes |
|
| `between` | Yes | Yes |
|
||||||
| `oneOf` / `in` | Yes | Yes |
|
| `oneOf` / `in` | Yes | Yes |
|
||||||
| `noneOf` | — | Yes |
|
| `noneOf` | — | Yes |
|
||||||
|
|
@ -216,18 +216,18 @@ See the **[Subtypes & Facets guide](./guides/subtypes-and-facets.md)** for the f
|
||||||
|
|
||||||
### Filter relationships by subtype (7.30+)
|
### Filter relationships by subtype (7.30+)
|
||||||
|
|
||||||
Verbs are first-class peers — `getRelations()` and graph traversal both honor subtype filters on the fast path:
|
Verbs are first-class peers — `related()` and graph traversal both honor subtype filters on the fast path:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Filter relationships by VerbType subtype
|
// Filter relationships by VerbType subtype
|
||||||
const direct = await brain.getRelations({
|
const direct = await brain.related({
|
||||||
from: ceoId,
|
from: ceoId,
|
||||||
type: VerbType.ReportsTo,
|
type: VerbType.ReportsTo,
|
||||||
subtype: 'direct'
|
subtype: 'direct'
|
||||||
})
|
})
|
||||||
|
|
||||||
// Set membership on verb subtype
|
// Set membership on verb subtype
|
||||||
const all = await brain.getRelations({
|
const all = await brain.related({
|
||||||
from: ceoId,
|
from: ceoId,
|
||||||
type: VerbType.ReportsTo,
|
type: VerbType.ReportsTo,
|
||||||
subtype: ['direct', 'dotted-line']
|
subtype: ['direct', 'dotted-line']
|
||||||
|
|
|
||||||
|
|
@ -35,6 +35,7 @@ const results = await brain.find({
|
||||||
| **[Data Model](./DATA_MODEL.md)** | Entity structure, data vs metadata, storage fields |
|
| **[Data Model](./DATA_MODEL.md)** | Entity structure, data vs metadata, storage fields |
|
||||||
| **[Query Operators](./QUERY_OPERATORS.md)** | All BFO operators with examples and indexed/in-memory matrix |
|
| **[Query Operators](./QUERY_OPERATORS.md)** | All BFO operators with examples and indexed/in-memory matrix |
|
||||||
| [Find System](./FIND_SYSTEM.md) | Natural language `find()` and hybrid search details |
|
| [Find System](./FIND_SYSTEM.md) | Natural language `find()` and hybrid search details |
|
||||||
|
| [Consistency Model](./concepts/consistency-model.md) | The Db API guarantees — snapshot isolation, atomic transactions, time travel |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -47,7 +48,7 @@ const results = await brain.find({
|
||||||
| [Noun-Verb Taxonomy](./architecture/noun-verb-taxonomy.md) | 42 nouns + 127 verbs type system |
|
| [Noun-Verb Taxonomy](./architecture/noun-verb-taxonomy.md) | 42 nouns + 127 verbs type system |
|
||||||
| [Stage 3 Canonical Taxonomy](./STAGE3-CANONICAL-TAXONOMY.md) | Complete type reference |
|
| [Stage 3 Canonical Taxonomy](./STAGE3-CANONICAL-TAXONOMY.md) | Complete type reference |
|
||||||
| [Storage Architecture](./architecture/storage-architecture.md) | Storage adapters and optimization |
|
| [Storage Architecture](./architecture/storage-architecture.md) | Storage adapters and optimization |
|
||||||
| [Index Architecture](./architecture/index-architecture.md) | HNSW, Graph, and Metadata indexing |
|
| [Index Architecture](./architecture/index-architecture.md) | Vector, Graph, and Metadata indexing |
|
||||||
| [Zero Configuration](./architecture/zero-config.md) | Auto-adapts to any environment |
|
| [Zero Configuration](./architecture/zero-config.md) | Auto-adapts to any environment |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
@ -70,6 +71,7 @@ See [vfs/](./vfs/) for the complete VFS documentation set.
|
||||||
| Document | Description |
|
| Document | Description |
|
||||||
|----------|-------------|
|
|----------|-------------|
|
||||||
| [Import Anything](./guides/import-anything.md) | CSV, Excel, PDF, URL imports |
|
| [Import Anything](./guides/import-anything.md) | CSV, Excel, PDF, URL imports |
|
||||||
|
| [Snapshots & Time Travel](./guides/snapshots-and-time-travel.md) | Backups, restore, what-if analysis, audit trails |
|
||||||
| [Natural Language](./guides/natural-language.md) | Query in plain English |
|
| [Natural Language](./guides/natural-language.md) | Query in plain English |
|
||||||
| [Neural API](./guides/neural-api.md) | AI-powered features |
|
| [Neural API](./guides/neural-api.md) | AI-powered features |
|
||||||
| [Enterprise for Everyone](./guides/enterprise-for-everyone.md) | No limits, no tiers |
|
| [Enterprise for Everyone](./guides/enterprise-for-everyone.md) | No limits, no tiers |
|
||||||
|
|
@ -81,24 +83,16 @@ See [vfs/](./vfs/) for the complete VFS documentation set.
|
||||||
|
|
||||||
| Document | Description |
|
| Document | Description |
|
||||||
|----------|-------------|
|
|----------|-------------|
|
||||||
| [Cloud Deployment](./deployment/CLOUD_DEPLOYMENT_GUIDE.md) | Deploy on AWS, GCP, Azure, Cloudflare |
|
| [Storage Architecture](./architecture/storage-architecture.md) | Filesystem and memory adapters, on-disk artifact layout, operator-layer backup |
|
||||||
| [Extending Storage](./EXTENDING_STORAGE.md) | Create custom storage adapters |
|
|
||||||
| [AWS S3 Cost Optimization](./operations/cost-optimization-aws-s3.md) | 96% cost savings |
|
|
||||||
| [GCS Cost Optimization](./operations/cost-optimization-gcs.md) | 94% savings with Autoclass |
|
|
||||||
| [Azure Cost Optimization](./operations/cost-optimization-azure.md) | 95% savings |
|
|
||||||
| [R2 Cost Optimization](./operations/cost-optimization-cloudflare-r2.md) | Zero egress fees |
|
|
||||||
| [Capacity Planning](./operations/capacity-planning.md) | Scale to millions of entities |
|
| [Capacity Planning](./operations/capacity-planning.md) | Scale to millions of entities |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Plugins & Augmentations
|
## Plugins
|
||||||
|
|
||||||
| Document | Description |
|
| Document | Description |
|
||||||
|----------|-------------|
|
|----------|-------------|
|
||||||
| [Plugins](./PLUGINS.md) | Plugin system overview |
|
| [Plugins](./PLUGINS.md) | Plugin system overview — providers, `plugins` config, `brain.use()` |
|
||||||
| [Creating Augmentations](./CREATING-AUGMENTATIONS.md) | Build custom plugins |
|
|
||||||
| [Augmentations Reference](./augmentations/COMPLETE-REFERENCE.md) | Full augmentation API |
|
|
||||||
| [Augmentations Developer Guide](./augmentations/DEVELOPER-GUIDE.md) | Plugin development guide |
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
|
||||||
567
docs/SCALING.md
567
docs/SCALING.md
|
|
@ -1,502 +1,239 @@
|
||||||
# 🚀 Brainy Scaling Guide - Enterprise for Everyone
|
# Brainy Scaling Guide
|
||||||
|
|
||||||
> **One Line Summary**: Start with one node, scale to hundreds. Zero configuration required.
|
> **One Line Summary**: Single-node by design — Brainy scales by getting the most out of one machine plus operator-layer backup.
|
||||||
|
|
||||||
## 📖 Table of Contents
|
## Table of Contents
|
||||||
- [Quick Start](#quick-start)
|
- [Quick Start](#quick-start)
|
||||||
- [How It Works](#how-it-works)
|
- [How Brainy Scales](#how-brainy-scales)
|
||||||
- [Storage Configurations](#storage-configurations)
|
- [Storage Configurations](#storage-configurations)
|
||||||
- [Scaling Patterns](#scaling-patterns)
|
- [Scaling Patterns](#scaling-patterns)
|
||||||
- [Real World Examples](#real-world-examples)
|
- [Real World Examples](#real-world-examples)
|
||||||
|
|
||||||
## Quick Start
|
## Quick Start
|
||||||
|
|
||||||
### Single Node (Default)
|
### In-Memory
|
||||||
```typescript
|
```typescript
|
||||||
import Brainy from '@soulcraft/brainy'
|
import Brainy from '@soulcraft/brainy'
|
||||||
const brain = new Brainy() // That's it!
|
const brain = new Brainy({ storage: { type: 'memory' } })
|
||||||
```
|
```
|
||||||
|
|
||||||
### Multi-Node (Auto-Discovery)
|
### On-Disk (Default for Node)
|
||||||
```typescript
|
```typescript
|
||||||
// Node 1
|
|
||||||
const brain = new Brainy() // Starts as primary
|
|
||||||
|
|
||||||
// Node 2 (different server)
|
|
||||||
const brain = new Brainy() // Auto-discovers Node 1, becomes replica!
|
|
||||||
```
|
|
||||||
|
|
||||||
**That's literally all you need!** Brainy handles everything else automatically.
|
|
||||||
|
|
||||||
## How It Works
|
|
||||||
|
|
||||||
### 🎯 The Magic: Zero Configuration
|
|
||||||
|
|
||||||
Brainy uses **intelligent defaults** and **auto-discovery** to eliminate configuration:
|
|
||||||
|
|
||||||
1. **First node starts** → Becomes primary automatically
|
|
||||||
2. **Second node starts** → Discovers first node via UDP broadcast
|
|
||||||
3. **Nodes negotiate** → Elect leader, distribute shards
|
|
||||||
4. **Data flows** → Automatic replication and routing
|
|
||||||
5. **Node fails** → Automatic failover in <1 second
|
|
||||||
|
|
||||||
### 🔄 Automatic Node Discovery
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Three ways Brainy finds other nodes (auto-selected):
|
|
||||||
|
|
||||||
// 1. LOCAL NETWORK (Default)
|
|
||||||
// Uses UDP broadcast on port 7946
|
|
||||||
// Perfect for: On-premise, same VPC
|
|
||||||
|
|
||||||
// 2. CLOUD NATIVE (Auto-detected)
|
|
||||||
// Kubernetes: Uses k8s DNS service discovery
|
|
||||||
// AWS: Uses EC2 tags or Route53
|
|
||||||
// Azure: Uses Azure DNS
|
|
||||||
|
|
||||||
// 3. EXPLICIT (When needed)
|
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
peers: ['node1.example.com', 'node2.example.com']
|
storage: { type: 'filesystem', path: './brainy-data' }
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
### 📊 Data Distribution
|
## How Brainy Scales
|
||||||
|
|
||||||
When you add data, Brainy automatically:
|
Brainy 8.0 is a **single-node library**. There is no cluster, no peer discovery, no S3 coordination. Scaling means:
|
||||||
|
|
||||||
```typescript
|
- **Up**: give the process more RAM, CPU, and IOPS
|
||||||
brain.add({ name: "John" }, 'person')
|
- **Out**: stand up multiple independent Brainy instances behind your own service layer
|
||||||
|
- **Cold storage**: snapshot the on-disk artifact off-site so you can rehydrate elsewhere
|
||||||
|
|
||||||
// Behind the scenes:
|
The three knobs that matter most:
|
||||||
// 1. Hash ID to determine shard (consistent hashing)
|
|
||||||
// 2. Find nodes responsible for this shard
|
1. **`config.vector.recall`** — `'fast'`, `'balanced'`, or `'accurate'` (default `'balanced'`)
|
||||||
// 3. Write to primary shard owner
|
2. **`config.vector.persistMode`** — `'immediate'` for durability, `'deferred'` for throughput
|
||||||
// 4. Replicate to N backup nodes (default: 2)
|
|
||||||
// 5. Confirm write when majority acknowledge
|
The native vector provider (via the optional `@soulcraft/cortex` package) extends this with a higher-performing index — and its own at-scale acceleration such as on-disk compressed indexing — when installed.
|
||||||
```
|
|
||||||
|
## Measured Performance
|
||||||
|
|
||||||
|
Numbers below are **measured** by `tests/benchmarks/find-composition-scale.js` (a single
|
||||||
|
Node 22 process, in-memory storage, 384-dim vectors, `balanced` recall). They are the
|
||||||
|
open-core (pure-TypeScript) path — what you get from `@soulcraft/brainy` with no native
|
||||||
|
provider installed. Run it yourself: `node --max-old-space-size=8192 tests/benchmarks/find-composition-scale.js 100000`.
|
||||||
|
|
||||||
|
`find()` query latency, p50 / p95 (200 queries each):
|
||||||
|
|
||||||
|
| Query | 5,000 entities | 100,000 entities |
|
||||||
|
|---|---|---|
|
||||||
|
| Vector similarity (`{ vector }`) | 0.8 / 1.3 ms | 1.4 / 4.7 ms |
|
||||||
|
| Graph 1-hop (`{ connected }`) | 0.5 / 0.7 ms | 0.7 / 0.8 ms |
|
||||||
|
| Metadata filter (`{ where }`, low-selectivity) | 0.7 / 1.2 ms | 23.5 / 30.1 ms |
|
||||||
|
| Vector + metadata | 7.7 / 8.3 ms | 78.8 / 93.8 ms |
|
||||||
|
|
||||||
|
What the shape tells you:
|
||||||
|
|
||||||
|
- **Vector and graph lookups scale ~logarithmically** — they barely move from 5k to 100k,
|
||||||
|
because HNSW search is ~O(ef·log n) and graph adjacency is O(degree).
|
||||||
|
- **Metadata-filtered paths scale with the size of the match set, not the database.** The
|
||||||
|
benchmark's `category` filter matches ~10% of rows (10,000 at 100k); the cost is
|
||||||
|
materializing that candidate set and running the vector search *inside* it (`find()` does
|
||||||
|
metadata-first hard filtering, then ranks within the candidates — see
|
||||||
|
[How find works](./FIND_SYSTEM.md)). A **high-selectivity** filter (few matches) is far
|
||||||
|
cheaper; a 10%-of-everything filter is the worst case. This candidate-restricted search is
|
||||||
|
precisely the path the native provider accelerates (Rust roaring-bitmap candidate
|
||||||
|
intersection).
|
||||||
|
- **Composition is correct, not lossy.** Combining vector + metadata + graph returns exactly
|
||||||
|
the entities satisfying all constraints — verified by
|
||||||
|
`tests/integration/find-triple-composition.test.ts`.
|
||||||
|
|
||||||
|
Memory: ~62 KB resident per entity at 100k (6.2 GB RSS for 100k × 384-dim including the HNSW
|
||||||
|
graph, metadata index, and 100k edges).
|
||||||
|
|
||||||
|
**Scale ceiling (open-core).** The pure-JS HNSW *build* cost (~100 inserts/s at 384-dim on
|
||||||
|
one core) makes the in-process open-core path most appropriate up to ~10⁵–10⁶ entities.
|
||||||
|
*Query* latency stays low well beyond that, but for the 10⁸–10¹⁰ regime install the native
|
||||||
|
provider (`@soulcraft/cortex`, on-disk DiskANN) — same API, no code change. _Projected from
|
||||||
|
the two measured points, vector p50 at 1M is ~2 ms; metadata-heavy composition grows with
|
||||||
|
match-set size and is the path to move onto the native provider first._
|
||||||
|
|
||||||
## Storage Configurations
|
## Storage Configurations
|
||||||
|
|
||||||
### 🗂️ Storage Adapter Patterns
|
### Filesystem (Recommended for Production)
|
||||||
|
|
||||||
Brainy intelligently adapts to your storage setup:
|
|
||||||
|
|
||||||
#### Pattern 1: Separate Storage Per Node (Recommended)
|
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Node 1 - Own filesystem
|
|
||||||
const brain1 = new Brainy({
|
|
||||||
storage: '/data/node1' // or auto: './brainy-data'
|
|
||||||
})
|
|
||||||
|
|
||||||
// Node 2 - Own filesystem
|
|
||||||
const brain2 = new Brainy({
|
|
||||||
storage: '/data/node2' // or auto: './brainy-data'
|
|
||||||
})
|
|
||||||
|
|
||||||
// ✅ BENEFITS:
|
|
||||||
// - No conflicts between nodes
|
|
||||||
// - Fast local reads
|
|
||||||
// - True horizontal scaling
|
|
||||||
// - Survives network partitions
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Pattern 2: Separate S3 Buckets Per Node
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Node 1 - Own S3 bucket
|
|
||||||
const brain1 = new Brainy({
|
|
||||||
storage: 's3://brainy-node-1' // Auto-uses AWS credentials
|
|
||||||
})
|
|
||||||
|
|
||||||
// Node 2 - Own S3 bucket
|
|
||||||
const brain2 = new Brainy({
|
|
||||||
storage: 's3://brainy-node-2'
|
|
||||||
})
|
|
||||||
|
|
||||||
// ✅ BENEFITS:
|
|
||||||
// - Infinite storage capacity
|
|
||||||
// - Geographic distribution
|
|
||||||
// - No local disk needed
|
|
||||||
// - Built-in durability
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Pattern 3: Shared S3 Bucket (Coordinated)
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// All nodes - Shared bucket with coordination
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: 's3://shared-brainy-data',
|
|
||||||
// Brainy automatically adds node-specific prefixes!
|
|
||||||
})
|
|
||||||
|
|
||||||
// What happens automatically:
|
|
||||||
// - Node 1 writes to: s3://shared-brainy-data/node-1/
|
|
||||||
// - Node 2 writes to: s3://shared-brainy-data/node-2/
|
|
||||||
// - Metadata in: s3://shared-brainy-data/_cluster/
|
|
||||||
// - Coordination via S3 conditional writes
|
|
||||||
|
|
||||||
// ✅ BENEFITS:
|
|
||||||
// - Single bucket to manage
|
|
||||||
// - Easy backup/restore
|
|
||||||
// - Cost effective
|
|
||||||
// - Automatic namespace isolation
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Pattern 4: Mixed Storage (Hybrid)
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Hot data on local SSD, cold data in S3
|
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: {
|
storage: {
|
||||||
hot: '/fast-ssd/brainy', // Recent/frequent data
|
type: 'filesystem',
|
||||||
cold: 's3://brainy-archive' // Older data
|
path: '/var/lib/brainy'
|
||||||
}
|
}
|
||||||
// Brainy automatically promotes/demotes data!
|
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
- Stores everything in a sharded JSON tree under `path`
|
||||||
|
- Atomic writes via rename
|
||||||
|
- Survives process restarts
|
||||||
|
- Snapshot it off-site with `gsutil rsync`, `aws s3 sync`, `rclone`, or `tar` from your scheduler
|
||||||
|
|
||||||
### 🌍 Cloud Provider Auto-Detection
|
### Memory
|
||||||
|
```typescript
|
||||||
|
const brain = new Brainy({ storage: { type: 'memory' } })
|
||||||
|
```
|
||||||
|
- Zero I/O, fastest possible
|
||||||
|
- No persistence — process exit discards everything
|
||||||
|
- Use for tests and ephemeral caches
|
||||||
|
|
||||||
|
### Auto
|
||||||
```typescript
|
```typescript
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: 'cloud://brainy-data' // Auto-detects provider!
|
storage: { type: 'auto', path: './data' }
|
||||||
})
|
})
|
||||||
|
|
||||||
// Automatically uses:
|
|
||||||
// - AWS: S3 + DynamoDB for metadata
|
|
||||||
// - Google Cloud: GCS + Firestore
|
|
||||||
// - Azure: Blob Storage + Cosmos DB
|
|
||||||
// - Cloudflare: R2 + D1
|
|
||||||
// - Vercel: Blob + KV
|
|
||||||
```
|
```
|
||||||
|
- Picks `filesystem` when running on Node with a writable `path`
|
||||||
### 📝 Storage Coordination Rules
|
- Falls back to `memory` otherwise
|
||||||
|
|
||||||
When multiple nodes share storage, Brainy automatically:
|
|
||||||
|
|
||||||
1. **Namespace Isolation**: Each node gets unique prefix
|
|
||||||
2. **Lock-Free Writes**: Uses atomic operations
|
|
||||||
3. **Consistent Metadata**: Coordinated via consensus
|
|
||||||
4. **Conflict Resolution**: Version vectors for conflicts
|
|
||||||
5. **Garbage Collection**: Automatic cleanup of old data
|
|
||||||
|
|
||||||
## Scaling Patterns
|
## Scaling Patterns
|
||||||
|
|
||||||
### 📈 Progressive Scaling Journey
|
### Stage 1: Prototype (Memory)
|
||||||
|
|
||||||
#### Stage 1: Prototype (1 node, memory)
|
|
||||||
```typescript
|
```typescript
|
||||||
const brain = new Brainy() // Memory storage, single node
|
const brain = new Brainy({ storage: { type: 'memory' } })
|
||||||
// Perfect for: Development, testing, <1000 items
|
// Development, tests, <100K items
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Stage 2: Production (1 node, disk)
|
### Stage 2: Production (Filesystem)
|
||||||
```typescript
|
```typescript
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: './data' // Persistent storage
|
storage: { type: 'filesystem', path: '/var/lib/brainy' }
|
||||||
})
|
})
|
||||||
// Perfect for: Small apps, <100K items
|
// Most production workloads up to ~10M entities on a single host
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Stage 3: High Availability (2-3 nodes)
|
### Stage 3: Higher Throughput (Tune the Vector Index)
|
||||||
```typescript
|
```typescript
|
||||||
// Just start same code on multiple servers!
|
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: './data' // Each node's own storage
|
storage: { type: 'filesystem', path: '/var/lib/brainy' },
|
||||||
|
vector: {
|
||||||
|
recall: 'fast', // Trade recall for latency
|
||||||
|
persistMode: 'deferred' // Batch persistence
|
||||||
|
}
|
||||||
})
|
})
|
||||||
// Automatic: Leader election, replication, failover
|
|
||||||
// Perfect for: Critical apps, <1M items
|
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Stage 4: Scale Out (N nodes)
|
### Stage 4: Multi-Instance (Operator-Layer)
|
||||||
```typescript
|
Run multiple Brainy processes behind your own routing/service layer. Each process owns its own `path`. Sync each artifact off-site independently. Brainy itself does not coordinate between processes.
|
||||||
// Same code, more servers!
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: 's3://brainy-{{nodeId}}' // Template auto-filled
|
|
||||||
})
|
|
||||||
// Automatic: Sharding, load balancing, geo-distribution
|
|
||||||
// Perfect for: Large apps, unlimited items
|
|
||||||
```
|
|
||||||
|
|
||||||
### 🎯 Common Scaling Scenarios
|
|
||||||
|
|
||||||
#### Scenario: Read-Heavy Application
|
|
||||||
```typescript
|
|
||||||
// Brainy auto-detects read-heavy pattern and:
|
|
||||||
// 1. Increases cache size
|
|
||||||
// 2. Creates more read replicas
|
|
||||||
// 3. Routes reads to nearest node
|
|
||||||
// 4. Caches popular items on all nodes
|
|
||||||
|
|
||||||
const brain = new Brainy() // No config needed!
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Scenario: Multi-Tenant SaaS
|
|
||||||
```typescript
|
|
||||||
// Brainy auto-detects tenant patterns and:
|
|
||||||
// 1. Shards by tenant ID
|
|
||||||
// 2. Isolates tenant data
|
|
||||||
// 3. Routes by tenant
|
|
||||||
// 4. Separate rate limits per tenant
|
|
||||||
|
|
||||||
const brain = new Brainy() // Detects from your queries!
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Scenario: Geographic Distribution
|
|
||||||
```typescript
|
|
||||||
// Deploy nodes in different regions
|
|
||||||
// Brainy automatically:
|
|
||||||
// 1. Detects node locations (via latency)
|
|
||||||
// 2. Replicates data geographically
|
|
||||||
// 3. Routes to nearest node
|
|
||||||
// 4. Handles region failures
|
|
||||||
|
|
||||||
// US-East
|
|
||||||
const brain = new Brainy({ region: 'us-east' }) // Optional hint
|
|
||||||
|
|
||||||
// EU-West (auto-discovers US-East)
|
|
||||||
const brain = new Brainy({ region: 'eu-west' })
|
|
||||||
```
|
|
||||||
|
|
||||||
## Real World Examples
|
## Real World Examples
|
||||||
|
|
||||||
### Example 1: Blog Platform
|
### Example 1: Single-Node App With Backup
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Day 1: Single server
|
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: './blog-data'
|
storage: { type: 'filesystem', path: '/var/lib/brainy' }
|
||||||
})
|
})
|
||||||
|
```
|
||||||
// Month 6: Add redundancy (on second server)
|
Schedule (cron / systemd timer):
|
||||||
const brain = new Brainy({
|
```bash
|
||||||
storage: './blog-data' // Different machine!
|
*/15 * * * * rclone sync /var/lib/brainy remote:brainy-backup
|
||||||
})
|
|
||||||
// Automatically syncs with first server
|
|
||||||
|
|
||||||
// Year 2: Global scale
|
|
||||||
// US Server
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: 's3://blog-us/data'
|
|
||||||
})
|
|
||||||
|
|
||||||
// EU Server
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: 's3://blog-eu/data'
|
|
||||||
})
|
|
||||||
|
|
||||||
// Asia Server
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: 's3://blog-asia/data'
|
|
||||||
})
|
|
||||||
// All automatically coordinate!
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Example 2: E-Commerce Site
|
### Example 2: Tests
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Development
|
const brain = new Brainy({ storage: { type: 'memory' } })
|
||||||
const brain = new Brainy() // Memory storage
|
// Fast, no cleanup needed between runs
|
||||||
|
|
||||||
// Staging (Kubernetes)
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: process.env.STORAGE_PATH // Uses PVC
|
|
||||||
})
|
|
||||||
// Auto-discovers other pods via K8s DNS
|
|
||||||
|
|
||||||
// Production (AWS)
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: 's3://shop-data',
|
|
||||||
cache: 'elasticache://shop-cache' // Optional
|
|
||||||
})
|
|
||||||
// Auto-scales with ECS/EKS
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Example 3: Analytics Platform
|
### Example 3: Multi-Tenant Service
|
||||||
|
Spin up one Brainy instance per tenant, each in its own directory:
|
||||||
```typescript
|
```typescript
|
||||||
// Ingestion nodes (write-optimized)
|
function brainForTenant(tenantId: string) {
|
||||||
const brain = new Brainy({
|
return new Brainy({
|
||||||
role: 'writer', // Hint for optimization
|
storage: {
|
||||||
storage: '/fast-nvme/ingest'
|
type: 'filesystem',
|
||||||
})
|
path: `/var/lib/brainy/${tenantId}`
|
||||||
|
}
|
||||||
// Query nodes (read-optimized)
|
})
|
||||||
const brain = new Brainy({
|
|
||||||
role: 'reader', // More cache, indexes
|
|
||||||
storage: 's3://analytics-archive'
|
|
||||||
})
|
|
||||||
|
|
||||||
// Automatically coordinates between writers and readers!
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🔧 Storage Adapter Specifics
|
|
||||||
|
|
||||||
### Local Filesystem
|
|
||||||
```typescript
|
|
||||||
{
|
|
||||||
storage: './data' // or absolute: '/var/lib/brainy'
|
|
||||||
// Each node MUST have separate directory
|
|
||||||
// Can be network mounted (NFS, EFS)
|
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
Your service layer handles routing and isolation; Brainy stays simple.
|
||||||
|
|
||||||
### AWS S3
|
### Example 4: Higher Recall at Scale
|
||||||
```typescript
|
```typescript
|
||||||
{
|
const brain = new Brainy({
|
||||||
storage: 's3://bucket-name/prefix'
|
storage: { type: 'filesystem', path: '/var/lib/brainy' },
|
||||||
// Uses AWS SDK credentials (env, IAM role, etc)
|
vector: {
|
||||||
// Supports S3-compatible (MinIO, Ceph)
|
recall: 'accurate'
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cloudflare R2
|
|
||||||
```typescript
|
|
||||||
{
|
|
||||||
storage: 'r2://bucket-name'
|
|
||||||
// Uses Wrangler or API tokens
|
|
||||||
// Zero egress fees!
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Google Cloud Storage
|
|
||||||
```typescript
|
|
||||||
{
|
|
||||||
storage: 'gs://bucket-name'
|
|
||||||
// Uses Application Default Credentials
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Azure Blob Storage
|
|
||||||
```typescript
|
|
||||||
{
|
|
||||||
storage: 'azure://container-name'
|
|
||||||
// Uses DefaultAzureCredential
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Mixed/Tiered
|
|
||||||
```typescript
|
|
||||||
{
|
|
||||||
storage: {
|
|
||||||
hot: './local-cache', // Fast SSD
|
|
||||||
warm: 's3://regular-data', // Standard storage
|
|
||||||
cold: 's3://glacier-archive' // Cheap archive
|
|
||||||
}
|
}
|
||||||
// Automatic tiering based on access patterns
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🎭 Advanced Patterns
|
|
||||||
|
|
||||||
### Pattern: Blue-Green Deployment
|
|
||||||
```typescript
|
|
||||||
// Blue cluster (current)
|
|
||||||
const brain = new Brainy({
|
|
||||||
cluster: 'blue',
|
|
||||||
storage: 's3://prod-blue'
|
|
||||||
})
|
|
||||||
|
|
||||||
// Green cluster (new version)
|
|
||||||
const brain = new Brainy({
|
|
||||||
cluster: 'green',
|
|
||||||
storage: 's3://prod-green',
|
|
||||||
syncFrom: 'blue' // Real-time sync during migration
|
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
### Pattern: Federation
|
## Tuning Knobs Summary
|
||||||
```typescript
|
|
||||||
// Region 1 Cluster
|
|
||||||
const brain1 = new Brainy({
|
|
||||||
federation: 'global',
|
|
||||||
region: 'us-east',
|
|
||||||
storage: 's3://us-east-data'
|
|
||||||
})
|
|
||||||
|
|
||||||
// Region 2 Cluster
|
| Setting | Values | When to change |
|
||||||
const brain2 = new Brainy({
|
|---------|--------|----------------|
|
||||||
federation: 'global',
|
| `vector.recall` | `'fast'` / `'balanced'` / `'accurate'` | Trade recall for latency |
|
||||||
region: 'eu-west',
|
| `vector.persistMode` | `'immediate'` / `'deferred'` | Throughput vs. durability |
|
||||||
storage: 's3://eu-west-data'
|
| `storage.cache.maxSize` | integer | Hot-path read cache size |
|
||||||
})
|
| `storage.cache.ttl` | ms | Cache freshness |
|
||||||
// Clusters coordinate for global queries!
|
|
||||||
```
|
|
||||||
|
|
||||||
### Pattern: Edge Computing
|
## Monitoring & Observability
|
||||||
```typescript
|
|
||||||
// Edge nodes (in CDN POPs)
|
|
||||||
const brain = new Brainy({
|
|
||||||
mode: 'edge',
|
|
||||||
storage: 'memory', // RAM only
|
|
||||||
upstream: 'https://main-cluster.example.com'
|
|
||||||
})
|
|
||||||
// Caches frequently accessed data at edge
|
|
||||||
```
|
|
||||||
|
|
||||||
## 📊 Monitoring & Observability
|
|
||||||
|
|
||||||
Brainy automatically exposes metrics:
|
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
const metrics = brain.getMetrics()
|
const stats = await brain.stats()
|
||||||
// {
|
// {
|
||||||
// nodes: { total: 5, healthy: 5 },
|
// nounCount: 50000,
|
||||||
// shards: { total: 20, local: 4 },
|
// verbCount: 80000,
|
||||||
// replication: { factor: 2, lag: 45 },
|
// vectorIndex: { ... },
|
||||||
// operations: { reads: 10000, writes: 1000 },
|
// storage: { used: '45GB' }
|
||||||
// storage: { used: '45GB', available: '955GB' }
|
|
||||||
// }
|
// }
|
||||||
```
|
```
|
||||||
|
|
||||||
## 🚨 Troubleshooting
|
## Troubleshooting
|
||||||
|
|
||||||
### Issue: Nodes don't discover each other
|
### Issue: Slow queries
|
||||||
```typescript
|
1. Switch to `vector.recall: 'fast'`
|
||||||
// Solution 1: Check network allows UDP 7946
|
2. Increase the read cache (`storage.cache.maxSize`)
|
||||||
// Solution 2: Use explicit peers
|
3. Consider the optional native vector provider via `@soulcraft/cortex`
|
||||||
const brain = new Brainy({
|
|
||||||
peers: ['10.0.0.1:7946', '10.0.0.2:7946']
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Issue: Storage conflicts
|
### Issue: Memory pressure
|
||||||
```typescript
|
1. Reduce `storage.cache.maxSize`
|
||||||
// Ensure each node has unique storage path
|
2. Move to `vector.persistMode: 'deferred'` to batch writes
|
||||||
// ❌ WRONG: All nodes use './data'
|
3. Consider the optional native vector provider via `@soulcraft/cortex` for at-scale index acceleration
|
||||||
// ✅ RIGHT: Node1: './data1', Node2: './data2'
|
|
||||||
// ✅ RIGHT: Use {{nodeId}} template
|
|
||||||
```
|
|
||||||
|
|
||||||
### Issue: Slow performance
|
### Issue: Slow startup after a crash
|
||||||
```typescript
|
1. Use `vector.persistMode: 'immediate'` so the index file stays in sync with storage
|
||||||
// Brainy auto-tunes, but you can hint:
|
2. Verify backup integrity periodically
|
||||||
const brain = new Brainy({
|
|
||||||
profile: 'read-heavy' // or 'write-heavy', 'balanced'
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🎯 Best Practices
|
## Best Practices
|
||||||
|
|
||||||
1. **Let Brainy Auto-Configure**: Don't over-configure
|
1. **One process = one `path`** — never share a directory between processes
|
||||||
2. **Separate Storage Per Node**: Avoids conflicts
|
2. **Snapshot from your scheduler** — Brainy doesn't ship cloud SDKs; use `rclone` / `aws s3 sync` / `gsutil`
|
||||||
3. **Use S3 for Large Scale**: Infinite capacity
|
3. **Profile before tuning** — `recall: 'balanced'` is right for most workloads
|
||||||
4. **Start Simple**: Single node → Scale when needed
|
4. **Install the native vector provider only when measured profiling shows it pays off**
|
||||||
5. **Monitor Metrics**: Watch for bottlenecks
|
|
||||||
6. **Trust Auto-Scaling**: It learns your patterns
|
|
||||||
|
|
||||||
## 🚀 Summary
|
## Summary
|
||||||
|
|
||||||
- **Zero Config**: Just `new Brainy()` at any scale
|
- Brainy 8.0 is a **library**, not a cluster
|
||||||
- **Auto-Discovery**: Nodes find each other
|
- Storage adapters: `filesystem`, `memory`, `auto`
|
||||||
- **Smart Storage**: Adapts to any backend
|
- Vector tuning: `recall`, `persistMode`
|
||||||
- **Progressive Scaling**: 1 → 100 nodes seamlessly
|
- Backup is an operator-layer concern — snapshot `path`
|
||||||
- **Self-Tuning**: Learns and optimizes
|
|
||||||
- **No DevOps**: It just works!
|
|
||||||
|
|
||||||
**This is Enterprise for Everyone - enterprise-grade scaling with toy-like simplicity!**
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
*Questions? Issues? Visit [github.com/soullabs/brainy](https://github.com/soullabs/brainy)*
|
|
||||||
|
|
|
||||||
1319
docs/api/README.md
1319
docs/api/README.md
File diff suppressed because it is too large
Load diff
|
|
@ -1,306 +0,0 @@
|
||||||
# 🧠 **Brainy Clustering Algorithms - Complete Analysis**
|
|
||||||
|
|
||||||
## 🎯 **Current State & Capabilities**
|
|
||||||
|
|
||||||
### **✅ Existing Infrastructure (Excellent Foundation)**
|
|
||||||
|
|
||||||
#### **1. HNSW Hierarchical Clustering**
|
|
||||||
```typescript
|
|
||||||
// ALREADY IMPLEMENTED & OPTIMIZED
|
|
||||||
brain.neural.clusters({ algorithm: 'hierarchical', level: 2 })
|
|
||||||
```
|
|
||||||
|
|
||||||
**How it works:**
|
|
||||||
- **Leverages HNSW natural hierarchy**: Uses existing index levels as natural cluster boundaries
|
|
||||||
- **O(n) performance**: Much faster than O(n²) traditional clustering
|
|
||||||
- **Multi-level granularity**: Higher levels = fewer, broader clusters; Lower levels = more, specific clusters
|
|
||||||
- **Representative sampling**: Uses HNSW level nodes as natural cluster centers
|
|
||||||
|
|
||||||
**Performance characteristics:**
|
|
||||||
- **Excellent for large datasets** (millions of items)
|
|
||||||
- **Preserves semantic relationships** from vector space
|
|
||||||
- **Automatic granularity control** via level parameter
|
|
||||||
|
|
||||||
#### **2. Distance-Based Algorithms**
|
|
||||||
```typescript
|
|
||||||
// COMPREHENSIVE DISTANCE FUNCTIONS AVAILABLE
|
|
||||||
euclideanDistance, cosineDistance, manhattanDistance, dotProductDistance
|
|
||||||
```
|
|
||||||
|
|
||||||
**Optimized implementations:**
|
|
||||||
- **Batch processing**: `calculateDistancesBatch()` with parallelization
|
|
||||||
- **Multiple metrics**: Choose optimal distance function per use case
|
|
||||||
- **Performance optimized**: Faster than GPU for small vectors due to no transfer overhead
|
|
||||||
|
|
||||||
#### **3. Rich Semantic Taxonomy**
|
|
||||||
```typescript
|
|
||||||
// 25+ NOUN TYPES & 35+ VERB TYPES
|
|
||||||
NounType: Person, Organization, Document, Concept, Event, Media, etc.
|
|
||||||
VerbType: RelatedTo, Contains, PartOf, Causes, CreatedBy, etc.
|
|
||||||
```
|
|
||||||
|
|
||||||
**Semantic clustering capabilities:**
|
|
||||||
- **Type-based clustering**: Group by semantic categories
|
|
||||||
- **Cross-type relationships**: Use verb types to find semantic bridges
|
|
||||||
- **Hierarchical taxonomies**: Natural clustering within and across types
|
|
||||||
|
|
||||||
#### **4. Graph Structure**
|
|
||||||
```typescript
|
|
||||||
// VERB RELATIONSHIPS CREATE RICH GRAPH
|
|
||||||
await brain.relate(sourceId, targetId, VerbType.Causes, { strength: 0.8 })
|
|
||||||
```
|
|
||||||
|
|
||||||
**Graph-based clustering potential:**
|
|
||||||
- **Connected components**: Find strongly connected groups
|
|
||||||
- **Community detection**: Use relationship strength for clustering
|
|
||||||
- **Multi-modal clustering**: Combine graph + vector + taxonomy
|
|
||||||
|
|
||||||
## 🚀 **Advanced Clustering Algorithms We Can Implement**
|
|
||||||
|
|
||||||
### **1. ✅ Already Implemented: HNSW Hierarchical**
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// PRODUCTION READY - Uses existing HNSW levels
|
|
||||||
const clusters = await brain.neural.clusters({
|
|
||||||
algorithm: 'hierarchical',
|
|
||||||
level: 2, // Control granularity
|
|
||||||
maxClusters: 15
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
**Performance:** **A+** - O(n) leveraging existing index structure
|
|
||||||
|
|
||||||
### **2. 🔥 Semantic Taxonomy Clustering**
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// IMPLEMENT: Fast type-based clustering with cross-type bridges
|
|
||||||
const clusters = await brain.neural.clusterByDomain('nounType', {
|
|
||||||
preserveTypeBoundaries: false, // Allow cross-type clusters
|
|
||||||
bridgeStrength: 0.8, // Minimum relationship strength for bridges
|
|
||||||
hybridWeighting: {
|
|
||||||
taxonomy: 0.4, // 40% weight to type similarity
|
|
||||||
vector: 0.4, // 40% weight to vector similarity
|
|
||||||
graph: 0.2 // 20% weight to relationship strength
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
**Algorithm approach:**
|
|
||||||
1. **Primary clustering by taxonomy**: Group by NounType/VerbType first
|
|
||||||
2. **Vector refinement**: Sub-cluster within types using vector similarity
|
|
||||||
3. **Cross-type bridging**: Find relationships that bridge type boundaries
|
|
||||||
4. **Weighted fusion**: Combine taxonomy + vector + graph signals
|
|
||||||
|
|
||||||
**Performance:** **A+** - O(n log n) - taxonomy grouping is O(n), refinement is HNSW-accelerated
|
|
||||||
|
|
||||||
### **3. 🔥 Graph Community Detection**
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// IMPLEMENT: Relationship-based clustering
|
|
||||||
const clusters = await brain.neural.clusterByConnections({
|
|
||||||
algorithm: 'modularity', // or 'louvain', 'leiden'
|
|
||||||
minCommunitySize: 3,
|
|
||||||
relationshipWeights: {
|
|
||||||
[VerbType.Creates]: 1.0,
|
|
||||||
[VerbType.PartOf]: 0.8,
|
|
||||||
[VerbType.RelatedTo]: 0.5
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
**Algorithm approach:**
|
|
||||||
1. **Build weighted graph**: Use verbs as edges, weights from relationship types + metadata
|
|
||||||
2. **Community detection**: Apply Louvain or Leiden algorithm for modularity optimization
|
|
||||||
3. **Semantic enhancement**: Use vector similarity to refine community boundaries
|
|
||||||
|
|
||||||
**Performance:** **A** - O(n log n) for sparse graphs, handles millions of relationships efficiently
|
|
||||||
|
|
||||||
### **4. 🔥 Multi-Modal Fusion Clustering**
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// IMPLEMENT: Best of all worlds
|
|
||||||
const clusters = await brain.neural.clusters({
|
|
||||||
algorithm: 'multimodal',
|
|
||||||
signals: {
|
|
||||||
vector: { weight: 0.5, metric: 'cosine' },
|
|
||||||
graph: { weight: 0.3, algorithm: 'modularity' },
|
|
||||||
taxonomy: { weight: 0.2, crossTypeThreshold: 0.8 }
|
|
||||||
},
|
|
||||||
fusion: 'weighted_ensemble' // or 'consensus', 'hierarchical'
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
**Algorithm approach:**
|
|
||||||
1. **Independent clustering**: Run HNSW, graph, and taxonomy clustering separately
|
|
||||||
2. **Consensus building**: Find agreement between different clustering results
|
|
||||||
3. **Conflict resolution**: Use weighted voting or hierarchical merging for disagreements
|
|
||||||
4. **Quality optimization**: Iteratively refine based on silhouette scores
|
|
||||||
|
|
||||||
**Performance:** **A** - O(n log n) - parallel execution of component algorithms
|
|
||||||
|
|
||||||
### **5. 💎 Temporal Pattern Clustering**
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// IMPLEMENT: Time-aware clustering using existing infrastructure
|
|
||||||
const clusters = await brain.neural.clusterByTime('createdAt', [
|
|
||||||
{ start: new Date('2024-01-01'), end: new Date('2024-06-30'), label: 'H1 2024' },
|
|
||||||
{ start: new Date('2024-07-01'), end: new Date('2024-12-31'), label: 'H2 2024' }
|
|
||||||
], {
|
|
||||||
evolution: 'track', // Track how clusters evolve over time
|
|
||||||
stability: 0.7, // Minimum stability threshold
|
|
||||||
trendAnalysis: true // Include trend detection
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
**Algorithm approach:**
|
|
||||||
1. **Time window clustering**: Apply HNSW clustering within each time window
|
|
||||||
2. **Cluster evolution tracking**: Match clusters across time windows using vector similarity
|
|
||||||
3. **Trend analysis**: Detect growing, shrinking, merging, splitting patterns
|
|
||||||
4. **Stability scoring**: Measure cluster consistency over time
|
|
||||||
|
|
||||||
**Performance:** **A+** - O(k*n log n) where k = number of time windows
|
|
||||||
|
|
||||||
### **6. 💎 DBSCAN with Adaptive Parameters**
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// IMPLEMENT: Density-based clustering with smart parameter selection
|
|
||||||
const clusters = await brain.neural.clusters({
|
|
||||||
algorithm: 'dbscan',
|
|
||||||
autoParams: true, // Automatically select eps and minPts
|
|
||||||
distanceMetric: 'cosine',
|
|
||||||
outlierHandling: 'soft' // Soft assignment instead of hard outliers
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
**Algorithm approach:**
|
|
||||||
1. **Adaptive parameter selection**: Use HNSW k-NN distances to estimate optimal eps
|
|
||||||
2. **Multi-scale analysis**: Run DBSCAN at multiple scales and merge results
|
|
||||||
3. **Soft outlier assignment**: Assign outliers to nearest clusters with confidence scores
|
|
||||||
|
|
||||||
**Performance:** **A** - O(n log n) using HNSW for neighbor queries
|
|
||||||
|
|
||||||
## 📊 **Performance Comparison Matrix**
|
|
||||||
|
|
||||||
| Algorithm | Time Complexity | Space | Large Scale | Semantic Quality | Graph Aware |
|
|
||||||
|-----------|----------------|-------|-------------|------------------|-------------|
|
|
||||||
| **HNSW Hierarchical** | O(n) | O(n) | ✅ Excellent | ✅ Very Good | ❌ No |
|
|
||||||
| **Taxonomy Fusion** | O(n log n) | O(n) | ✅ Excellent | 🔥 Exceptional | ⚡ Partial |
|
|
||||||
| **Graph Communities** | O(n log n) | O(e) | ✅ Very Good | ✅ Very Good | 🔥 Exceptional |
|
|
||||||
| **Multi-Modal** | O(n log n) | O(n) | ✅ Very Good | 🔥 Exceptional | 🔥 Exceptional |
|
|
||||||
| **Temporal Patterns** | O(k*n log n) | O(n) | ⚡ Good | ✅ Very Good | ⚡ Partial |
|
|
||||||
| **Adaptive DBSCAN** | O(n log n) | O(n) | ✅ Very Good | ✅ Very Good | ❌ No |
|
|
||||||
|
|
||||||
## 🎯 **Specific Improvements Using Existing Capabilities**
|
|
||||||
|
|
||||||
### **1. Enhanced HNSW Clustering (Easy Win)**
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// IMPROVE EXISTING: Add semantic post-processing
|
|
||||||
private async enhanceHNSWClusters(clusters: SemanticCluster[]): Promise<SemanticCluster[]> {
|
|
||||||
return Promise.all(clusters.map(async cluster => {
|
|
||||||
// Get actual metadata for cluster members
|
|
||||||
const members = await this.brain.getNouns(cluster.members.map(id => ({ id })))
|
|
||||||
|
|
||||||
// Analyze semantic characteristics
|
|
||||||
const semanticProfile = this.analyzeSemanticProfile(members)
|
|
||||||
|
|
||||||
// Generate meaningful cluster labels
|
|
||||||
const label = await this.generateClusterLabel(members, semanticProfile)
|
|
||||||
|
|
||||||
// Calculate cluster coherence using multiple signals
|
|
||||||
const coherence = this.calculateMultiModalCoherence(members)
|
|
||||||
|
|
||||||
return {
|
|
||||||
...cluster,
|
|
||||||
label,
|
|
||||||
semanticProfile,
|
|
||||||
coherence,
|
|
||||||
quality: coherence.overall
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### **2. Intelligent Algorithm Selection**
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// IMPLEMENT: Smart routing based on data characteristics
|
|
||||||
private selectOptimalAlgorithm(dataCharacteristics: {
|
|
||||||
size: number,
|
|
||||||
dimensionality: number,
|
|
||||||
graphDensity: number,
|
|
||||||
typeDistribution: Record<string, number>
|
|
||||||
}): string {
|
|
||||||
if (dataCharacteristics.size > 100000) {
|
|
||||||
return 'hierarchical' // HNSW scales best
|
|
||||||
}
|
|
||||||
|
|
||||||
if (dataCharacteristics.graphDensity > 0.1) {
|
|
||||||
return 'multimodal' // Rich graph structure
|
|
||||||
}
|
|
||||||
|
|
||||||
if (Object.keys(dataCharacteristics.typeDistribution).length > 10) {
|
|
||||||
return 'taxonomy' // Diverse semantic types
|
|
||||||
}
|
|
||||||
|
|
||||||
return 'hierarchical' // Safe default
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### **3. Streaming Cluster Updates**
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// IMPLEMENT: Incremental clustering using existing infrastructure
|
|
||||||
public async updateClusters(newItems: string[]): Promise<SemanticCluster[]> {
|
|
||||||
// Use HNSW nearest neighbor for fast cluster assignment
|
|
||||||
const assignments = await Promise.all(
|
|
||||||
newItems.map(async itemId => {
|
|
||||||
const neighbors = await this.brain.neural.neighbors(itemId, { limit: 5 })
|
|
||||||
return this.assignToNearestCluster(itemId, neighbors, this.existingClusters)
|
|
||||||
})
|
|
||||||
)
|
|
||||||
|
|
||||||
// Incrementally update cluster centroids and boundaries
|
|
||||||
return this.updateClusterBoundaries(assignments)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🏆 **Recommended Implementation Priority**
|
|
||||||
|
|
||||||
### **🔥 Phase 1: High Impact, Easy Implementation**
|
|
||||||
1. **Enhanced HNSW Clustering**: Add semantic post-processing to existing algorithm
|
|
||||||
2. **Taxonomy-Aware Clustering**: Leverage existing NounType/VerbType enums
|
|
||||||
3. **Intelligent Algorithm Selection**: Route based on data characteristics
|
|
||||||
|
|
||||||
### **⚡ Phase 2: Advanced Features**
|
|
||||||
4. **Graph Community Detection**: Use existing verb relationships
|
|
||||||
5. **Multi-Modal Fusion**: Combine all signals intelligently
|
|
||||||
6. **Streaming Updates**: Incremental cluster maintenance
|
|
||||||
|
|
||||||
### **💎 Phase 3: Cutting Edge**
|
|
||||||
7. **Temporal Pattern Analysis**: Track cluster evolution over time
|
|
||||||
8. **Adaptive DBSCAN**: Dynamic parameter selection
|
|
||||||
9. **Explainable Clustering**: Generate cluster explanations and reasoning
|
|
||||||
|
|
||||||
## 🎯 **Key Advantages of Our Approach**
|
|
||||||
|
|
||||||
### **✅ Leverages Existing Infrastructure**
|
|
||||||
- **HNSW index**: Already optimized for large-scale vector operations
|
|
||||||
- **Distance functions**: Battle-tested and performance-optimized
|
|
||||||
- **Semantic taxonomy**: Rich type system with 60+ semantic categories
|
|
||||||
- **Graph structure**: Relationship network from verb connections
|
|
||||||
|
|
||||||
### **✅ Multiple Clustering Paradigms**
|
|
||||||
- **Vector similarity**: Traditional embedding-based clustering
|
|
||||||
- **Graph structure**: Relationship-based community detection
|
|
||||||
- **Semantic taxonomy**: Type-aware intelligent grouping
|
|
||||||
- **Temporal patterns**: Time-aware cluster evolution
|
|
||||||
- **Multi-modal fusion**: Best of all worlds
|
|
||||||
|
|
||||||
### **✅ Scalability & Performance**
|
|
||||||
- **O(n) hierarchical clustering**: Leveraging HNSW levels
|
|
||||||
- **Parallel processing**: Batch distance calculations optimized
|
|
||||||
- **Streaming support**: Real-time cluster updates
|
|
||||||
- **Memory efficient**: Existing index structures reused
|
|
||||||
|
|
||||||
**Our clustering algorithms are not just competitive - they're architecturally superior by leveraging Brainy's unique multi-modal semantic infrastructure.**
|
|
||||||
|
|
@ -97,7 +97,7 @@ On `brain.update(entity)`, the engine reverses the old entity's contribution and
|
||||||
|
|
||||||
### Delete Handling
|
### Delete Handling
|
||||||
|
|
||||||
On `brain.delete(id)`, the engine reverses the entity's contribution:
|
On `brain.remove(id)`, the engine reverses the entity's contribution:
|
||||||
|
|
||||||
- `count` and `sum` are decremented
|
- `count` and `sum` are decremented
|
||||||
- `min`/`max` may become stale (marked in `staleMinMax` for lazy recompute)
|
- `min`/`max` may become stale (marked in `staleMinMax` for lazy recompute)
|
||||||
|
|
|
||||||
|
|
@ -1,359 +0,0 @@
|
||||||
# 🔍 Brainy 2.0 Augmentation System Architecture Audit (REVISED)
|
|
||||||
|
|
||||||
**Author**: Senior Architecture Review
|
|
||||||
**Date**: 2025-08-25
|
|
||||||
**Status**: 🟡 **WORKING BUT NEEDS MARKETPLACE FEATURES**
|
|
||||||
|
|
||||||
## Executive Summary
|
|
||||||
|
|
||||||
The augmentation system core execution is WORKING correctly through `AugmentationRegistry`. The system properly executes augmentations before/after operations. However, there's no discovery, installation, or marketplace integration for the brain-cloud registry vision.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🟢 What's Actually Working
|
|
||||||
|
|
||||||
### 1. Execution Mechanism ✅
|
|
||||||
The `AugmentationRegistry` class properly implements:
|
|
||||||
```typescript
|
|
||||||
async execute<T>(operation: string, params: any, mainOperation: () => Promise<T>): Promise<T>
|
|
||||||
```
|
|
||||||
- Chains augmentations correctly
|
|
||||||
- Respects timing (before/after/around)
|
|
||||||
- Handles operation filtering
|
|
||||||
- Works with all 27 augmentations
|
|
||||||
|
|
||||||
### 2. Registration System ✅
|
|
||||||
```typescript
|
|
||||||
brain.augmentations.register(augmentation)
|
|
||||||
```
|
|
||||||
- Two-phase initialization works (storage first)
|
|
||||||
- Context injection works
|
|
||||||
- Lifecycle management works
|
|
||||||
|
|
||||||
### 3. Clean Interface ✅
|
|
||||||
- 100% of augmentations use `BrainyAugmentation`
|
|
||||||
- `BaseAugmentation` provides solid foundation
|
|
||||||
- Proper TypeScript types
|
|
||||||
|
|
||||||
### 4. Auto-Configuration ✅
|
|
||||||
```typescript
|
|
||||||
new Brainy({
|
|
||||||
cache: true, // Auto-registers CacheAugmentation
|
|
||||||
index: true, // Auto-registers IndexAugmentation
|
|
||||||
storage: 's3' // Auto-registers S3StorageAugmentation
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🟡 Missing for Marketplace Vision
|
|
||||||
|
|
||||||
### 1. No Package Discovery
|
|
||||||
**Current**: Manual registration only
|
|
||||||
```typescript
|
|
||||||
// Current (manual)
|
|
||||||
import { NotionSynapse } from './my-custom-synapse'
|
|
||||||
brain.augmentations.register(new NotionSynapse())
|
|
||||||
|
|
||||||
// Needed
|
|
||||||
await brain.discover('notion') // Search npm/brain-cloud
|
|
||||||
await brain.install('@soulcraft/notion-synapse')
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. No Installation Mechanism
|
|
||||||
**Current**: Must be bundled at build time
|
|
||||||
**Needed**: Dynamic installation
|
|
||||||
```typescript
|
|
||||||
interface AugmentationMarketplace {
|
|
||||||
search(query: string): Promise<Package[]>
|
|
||||||
install(packageId: string): Promise<void>
|
|
||||||
uninstall(packageId: string): Promise<void>
|
|
||||||
listInstalled(): Promise<Package[]>
|
|
||||||
checkUpdates(): Promise<Update[]>
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. No Brain Cloud Registry Client
|
|
||||||
**Current**: No registry concept
|
|
||||||
**Needed**: Registry integration
|
|
||||||
```typescript
|
|
||||||
class BrainCloudRegistry {
|
|
||||||
private apiUrl = 'https://api.soulcraft.com/brain-cloud'
|
|
||||||
|
|
||||||
async search(query: string): Promise<AugmentationPackage[]> {
|
|
||||||
const response = await fetch(`${this.apiUrl}/augmentations/search?q=${query}`)
|
|
||||||
return response.json()
|
|
||||||
}
|
|
||||||
|
|
||||||
async getPackage(id: string): Promise<AugmentationPackage> {
|
|
||||||
const response = await fetch(`${this.apiUrl}/augmentations/${id}`)
|
|
||||||
return response.json()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4. No License Management
|
|
||||||
**Current**: All augmentations free/bundled
|
|
||||||
**Needed**: License verification
|
|
||||||
```typescript
|
|
||||||
interface LicenseManager {
|
|
||||||
verify(packageId: string, licenseKey: string): Promise<boolean>
|
|
||||||
activate(packageId: string, licenseKey: string): Promise<void>
|
|
||||||
deactivate(packageId: string): Promise<void>
|
|
||||||
getStatus(packageId: string): Promise<LicenseStatus>
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 5. No Version Management
|
|
||||||
**Current**: No versioning
|
|
||||||
**Needed**: Semver support
|
|
||||||
```typescript
|
|
||||||
interface VersionManager {
|
|
||||||
checkCompatibility(pkg: Package, brainyVersion: string): boolean
|
|
||||||
resolveConflicts(packages: Package[]): Package[]
|
|
||||||
upgrade(packageId: string, toVersion: string): Promise<void>
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📋 Implementation Plan for Marketplace
|
|
||||||
|
|
||||||
### Phase 1: Local Package Discovery (1 week)
|
|
||||||
```typescript
|
|
||||||
class LocalPackageDiscovery {
|
|
||||||
async discover(): Promise<Package[]> {
|
|
||||||
// 1. Search node_modules for brainy augmentations
|
|
||||||
const packages = await glob('node_modules/@*/package.json')
|
|
||||||
|
|
||||||
// 2. Filter for brainy augmentations
|
|
||||||
return packages.filter(pkg => pkg.brainy?.type === 'augmentation')
|
|
||||||
}
|
|
||||||
|
|
||||||
async load(packageId: string): Promise<BrainyAugmentation> {
|
|
||||||
// Dynamic import
|
|
||||||
const module = await import(packageId)
|
|
||||||
return new module.default()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Phase 2: NPM Integration (1 week)
|
|
||||||
```typescript
|
|
||||||
class NPMRegistry {
|
|
||||||
async search(query: string): Promise<Package[]> {
|
|
||||||
// Search npm for packages with brainy keyword
|
|
||||||
const response = await fetch(
|
|
||||||
`https://registry.npmjs.org/-/v1/search?text=${query}+keywords:brainy-augmentation`
|
|
||||||
)
|
|
||||||
return response.json()
|
|
||||||
}
|
|
||||||
|
|
||||||
async install(packageId: string): Promise<void> {
|
|
||||||
// Use npm programmatically
|
|
||||||
await exec(`npm install ${packageId}`)
|
|
||||||
|
|
||||||
// Auto-register after install
|
|
||||||
const aug = await this.load(packageId)
|
|
||||||
this.brain.augmentations.register(aug)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Phase 3: Brain Cloud Registry (2 weeks)
|
|
||||||
```typescript
|
|
||||||
class BrainCloudMarketplace {
|
|
||||||
private registry = new BrainCloudRegistry()
|
|
||||||
private licenses = new LicenseManager()
|
|
||||||
private installer = new AugmentationInstaller()
|
|
||||||
|
|
||||||
async browse(category?: string): Promise<MarketplaceListing[]> {
|
|
||||||
const packages = await this.registry.list(category)
|
|
||||||
|
|
||||||
return packages.map(pkg => ({
|
|
||||||
...pkg,
|
|
||||||
installed: this.isInstalled(pkg.id),
|
|
||||||
licensed: this.isLicensed(pkg.id),
|
|
||||||
updates: this.hasUpdates(pkg.id)
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
async purchase(packageId: string): Promise<void> {
|
|
||||||
// 1. Process payment
|
|
||||||
const license = await this.processPayment(packageId)
|
|
||||||
|
|
||||||
// 2. Activate license
|
|
||||||
await this.licenses.activate(packageId, license)
|
|
||||||
|
|
||||||
// 3. Install package
|
|
||||||
await this.install(packageId)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Phase 4: Developer Tools (1 week)
|
|
||||||
```typescript
|
|
||||||
// CLI for augmentation development
|
|
||||||
class AugmentationCLI {
|
|
||||||
async create(name: string): Promise<void> {
|
|
||||||
// Scaffold new augmentation project
|
|
||||||
await this.scaffold(name, 'augmentation-template')
|
|
||||||
}
|
|
||||||
|
|
||||||
async test(path: string): Promise<void> {
|
|
||||||
// Test augmentation locally
|
|
||||||
const aug = await this.load(path)
|
|
||||||
await this.runTests(aug)
|
|
||||||
}
|
|
||||||
|
|
||||||
async publish(path: string): Promise<void> {
|
|
||||||
// Publish to brain-cloud
|
|
||||||
const pkg = await this.package(path)
|
|
||||||
await this.registry.publish(pkg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🏗️ Recommended Architecture
|
|
||||||
|
|
||||||
### 1. Augmentation Package Structure
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"name": "@soulcraft/notion-synapse",
|
|
||||||
"version": "1.0.0",
|
|
||||||
"main": "dist/index.js",
|
|
||||||
"types": "dist/index.d.ts",
|
|
||||||
"brainy": {
|
|
||||||
"type": "augmentation",
|
|
||||||
"class": "NotionSynapse",
|
|
||||||
"timing": "after",
|
|
||||||
"operations": ["addNoun", "updateNoun"],
|
|
||||||
"priority": 20,
|
|
||||||
"license": "premium",
|
|
||||||
"price": 9.99,
|
|
||||||
"compatibility": ">=2.0.0",
|
|
||||||
"dependencies": []
|
|
||||||
},
|
|
||||||
"keywords": ["brainy-augmentation", "notion", "sync"]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Installation Flow
|
|
||||||
```typescript
|
|
||||||
// User flow
|
|
||||||
await brain.marketplace.search('notion')
|
|
||||||
// Returns: [@soulcraft/notion-synapse, @community/notion-sync, ...]
|
|
||||||
|
|
||||||
await brain.marketplace.install('@soulcraft/notion-synapse')
|
|
||||||
// 1. Check license (prompt for purchase if needed)
|
|
||||||
// 2. Check compatibility
|
|
||||||
// 3. Install dependencies
|
|
||||||
// 4. Download package
|
|
||||||
// 5. Load augmentation
|
|
||||||
// 6. Register with brain
|
|
||||||
// 7. Initialize
|
|
||||||
|
|
||||||
// Now it's working!
|
|
||||||
brain.augmentations.list()
|
|
||||||
// [..., { name: '@soulcraft/notion-synapse', enabled: true }]
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Discovery UI
|
|
||||||
```typescript
|
|
||||||
// Web UI component
|
|
||||||
<AugmentationMarketplace>
|
|
||||||
<SearchBar />
|
|
||||||
<Categories>
|
|
||||||
<Category name="Storage" count={12} />
|
|
||||||
<Category name="Sync" count={8} />
|
|
||||||
<Category name="AI" count={15} />
|
|
||||||
</Categories>
|
|
||||||
<Results>
|
|
||||||
<AugmentationCard
|
|
||||||
name="Notion Synapse"
|
|
||||||
author="Soulcraft"
|
|
||||||
rating={4.8}
|
|
||||||
installs={1200}
|
|
||||||
price={9.99}
|
|
||||||
onInstall={...}
|
|
||||||
/>
|
|
||||||
</Results>
|
|
||||||
</AugmentationMarketplace>
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🎯 Priority for 2.0 Release
|
|
||||||
|
|
||||||
### Must Have (Release Blockers)
|
|
||||||
- ✅ Working execution (DONE)
|
|
||||||
- ✅ Clean interface (DONE)
|
|
||||||
- ✅ Documentation (DONE)
|
|
||||||
- ⏳ Fix augmentationPipeline.ts removal
|
|
||||||
- ⏳ Test all 27 augmentations work
|
|
||||||
|
|
||||||
### Nice to Have (2.0.x)
|
|
||||||
- Local package discovery
|
|
||||||
- NPM integration
|
|
||||||
- Basic CLI tools
|
|
||||||
|
|
||||||
### Future (2.1+)
|
|
||||||
- Brain Cloud Registry
|
|
||||||
- License management
|
|
||||||
- Payment processing
|
|
||||||
- Marketplace UI
|
|
||||||
- Developer portal
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📊 Current State Assessment
|
|
||||||
|
|
||||||
| Component | Status | Notes |
|
|
||||||
|-----------|--------|-------|
|
|
||||||
| Core Execution | ✅ Working | AugmentationRegistry.execute() works |
|
|
||||||
| Registration | ✅ Working | Manual registration works |
|
|
||||||
| Auto-Config | ✅ Working | Cache, index, storage auto-register |
|
|
||||||
| Lifecycle | ✅ Working | Init, execute, shutdown work |
|
|
||||||
| Discovery | ❌ Missing | No package discovery |
|
|
||||||
| Installation | ❌ Missing | No dynamic installation |
|
|
||||||
| Marketplace | ❌ Missing | No registry client |
|
|
||||||
| Licensing | ❌ Missing | No license management |
|
|
||||||
| Versioning | ❌ Missing | No version checks |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 💡 Recommendations
|
|
||||||
|
|
||||||
### For 2.0 Release
|
|
||||||
1. **Ship with manual registration** - It works!
|
|
||||||
2. **Document how to create augmentations** - Critical for adoption
|
|
||||||
3. **Create 2-3 example augmentations** - Show the patterns
|
|
||||||
4. **Add basic CLI for testing** - Help developers
|
|
||||||
|
|
||||||
### For 2.1 (Q1 2025)
|
|
||||||
1. **Add NPM discovery** - Find installed augmentations
|
|
||||||
2. **Dynamic loading** - Import augmentations at runtime
|
|
||||||
3. **Basic marketplace API** - List available augmentations
|
|
||||||
4. **Version checking** - Ensure compatibility
|
|
||||||
|
|
||||||
### For 3.0 (Q2 2025)
|
|
||||||
1. **Full marketplace** - Browse, search, install
|
|
||||||
2. **Payment integration** - Premium augmentations
|
|
||||||
3. **Developer portal** - Publish augmentations
|
|
||||||
4. **Enterprise features** - Private registries
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ✅ Good News Summary
|
|
||||||
|
|
||||||
The augmentation system WORKS! The core architecture is solid:
|
|
||||||
- Execution mechanism is correct
|
|
||||||
- Registration works
|
|
||||||
- Lifecycle management works
|
|
||||||
- All 27 augmentations function properly
|
|
||||||
|
|
||||||
What's missing is the marketplace/discovery layer, which can be added incrementally without breaking the core system. The 2.0 release can ship with manual augmentation registration, and the marketplace features can be added in 2.1+.
|
|
||||||
|
|
||||||
**Recommendation: Ship 2.0 with current system, add marketplace in 2.1**
|
|
||||||
|
|
@ -73,10 +73,10 @@ import { MemoryStorageAugmentation } from 'brainy'
|
||||||
```
|
```
|
||||||
|
|
||||||
### 11. Server Search Augmentation ✅
|
### 11. Server Search Augmentation ✅
|
||||||
Distributed search capabilities.
|
Server-side search delegation over a conduit.
|
||||||
```typescript
|
```typescript
|
||||||
import { ServerSearchConduitAugmentation } from 'brainy'
|
import { ServerSearchConduitAugmentation } from 'brainy'
|
||||||
// Distributed query execution
|
// Forwards queries to a remote Brainy server
|
||||||
```
|
```
|
||||||
|
|
||||||
### 12. Neural Import Augmentation ✅
|
### 12. Neural Import Augmentation ✅
|
||||||
|
|
@ -100,7 +100,7 @@ await neuralImport.detectRelationships(entities)
|
||||||
await neuralImport.generateInsights(data)
|
await neuralImport.generateInsights(data)
|
||||||
```
|
```
|
||||||
|
|
||||||
### Distributed Operation Modes (Fully Implemented!)
|
### Operation Modes (Fully Implemented!)
|
||||||
```typescript
|
```typescript
|
||||||
// Read-only mode with optimized caching
|
// Read-only mode with optimized caching
|
||||||
const readerMode = new ReaderMode()
|
const readerMode = new ReaderMode()
|
||||||
|
|
@ -239,15 +239,10 @@ const cacheConfig = await getCacheAutoConfig()
|
||||||
|
|
||||||
## 🎨 How to Use Hidden Features
|
## 🎨 How to Use Hidden Features
|
||||||
|
|
||||||
### Enable Distributed Modes
|
### Enable Reader / Writer Modes
|
||||||
```typescript
|
```typescript
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
mode: 'reader', // or 'writer' or 'hybrid'
|
mode: 'reader' // or 'writer' or 'hybrid'
|
||||||
distributed: {
|
|
||||||
role: 'reader',
|
|
||||||
cacheStrategy: 'aggressive',
|
|
||||||
prefetch: true
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -285,7 +280,7 @@ const freshStats = await brain.getStatistics({
|
||||||
## 📝 What Needs Documentation
|
## 📝 What Needs Documentation
|
||||||
|
|
||||||
These features EXIST but need better docs:
|
These features EXIST but need better docs:
|
||||||
1. Distributed operation modes
|
1. Reader / writer operation modes
|
||||||
2. Neural import full API
|
2. Neural import full API
|
||||||
3. 3-level cache configuration
|
3. 3-level cache configuration
|
||||||
4. Performance monitoring API
|
4. Performance monitoring API
|
||||||
|
|
@ -297,7 +292,7 @@ These features EXIST but need better docs:
|
||||||
## 💡 The Truth
|
## 💡 The Truth
|
||||||
|
|
||||||
Brainy is MORE powerful than its own documentation suggests! Most "missing" features are actually implemented but hidden or not properly exposed. The codebase contains sophisticated systems for:
|
Brainy is MORE powerful than its own documentation suggests! Most "missing" features are actually implemented but hidden or not properly exposed. The codebase contains sophisticated systems for:
|
||||||
- Distributed operations
|
- Reader / writer operation modes
|
||||||
- AI-powered import
|
- AI-powered import
|
||||||
- Advanced caching
|
- Advanced caching
|
||||||
- Performance monitoring
|
- Performance monitoring
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,550 +0,0 @@
|
||||||
# 🏗️ Distributed Storage Architecture
|
|
||||||
|
|
||||||
> **Technical deep-dive**: How Brainy coordinates storage across multiple nodes and adapters
|
|
||||||
|
|
||||||
## Storage Adapter Layer
|
|
||||||
|
|
||||||
### Base Storage Interface
|
|
||||||
|
|
||||||
Every storage adapter implements this interface:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
interface StorageAdapter {
|
|
||||||
// Basic operations
|
|
||||||
get(key: string): Promise<any>
|
|
||||||
set(key: string, value: any): Promise<void>
|
|
||||||
delete(key: string): Promise<void>
|
|
||||||
|
|
||||||
// Batch operations
|
|
||||||
getBatch(keys: string[]): Promise<Map<string, any>>
|
|
||||||
setBatch(items: Map<string, any>): Promise<void>
|
|
||||||
|
|
||||||
// Atomic operations (for coordination)
|
|
||||||
compareAndSwap(key: string, oldVal: any, newVal: any): Promise<boolean>
|
|
||||||
increment(key: string, delta: number): Promise<number>
|
|
||||||
|
|
||||||
// Namespace support
|
|
||||||
withNamespace(namespace: string): StorageAdapter
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Storage Coordination Strategies
|
|
||||||
|
|
||||||
#### Strategy 1: Isolated Storage (Default)
|
|
||||||
|
|
||||||
Each node has completely separate storage:
|
|
||||||
|
|
||||||
```
|
|
||||||
Node-1 → Local FS: /data/node1/
|
|
||||||
└── shards/
|
|
||||||
├── shard-001/
|
|
||||||
├── shard-045/
|
|
||||||
└── shard-127/
|
|
||||||
|
|
||||||
Node-2 → Local FS: /data/node2/
|
|
||||||
└── shards/
|
|
||||||
├── shard-023/
|
|
||||||
├── shard-067/
|
|
||||||
└── shard-089/
|
|
||||||
```
|
|
||||||
|
|
||||||
**Coordination**: Via network messages only
|
|
||||||
- Shard ownership tracked in distributed consensus
|
|
||||||
- Data transfer via direct node-to-node communication
|
|
||||||
- No storage-level conflicts possible
|
|
||||||
|
|
||||||
#### Strategy 2: Shared Storage with Namespacing
|
|
||||||
|
|
||||||
Multiple nodes share storage but use namespaces:
|
|
||||||
|
|
||||||
```
|
|
||||||
S3 Bucket: brainy-cluster/
|
|
||||||
├── node-abc123/
|
|
||||||
│ ├── shards/
|
|
||||||
│ └── wal/
|
|
||||||
├── node-def456/
|
|
||||||
│ ├── shards/
|
|
||||||
│ └── wal/
|
|
||||||
└── _cluster/
|
|
||||||
├── topology.json
|
|
||||||
├── shard-map.json
|
|
||||||
└── elections/
|
|
||||||
```
|
|
||||||
|
|
||||||
**Coordination**: Via storage-level atomic operations
|
|
||||||
- Each node owns its namespace
|
|
||||||
- Cluster metadata in shared `_cluster/` namespace
|
|
||||||
- Atomic operations for leader election
|
|
||||||
- Conditional writes prevent conflicts
|
|
||||||
|
|
||||||
#### Strategy 3: Shared Storage with Fine-Grained Locking
|
|
||||||
|
|
||||||
Advanced mode for full shared storage:
|
|
||||||
|
|
||||||
```
|
|
||||||
S3 Bucket: brainy-shared/
|
|
||||||
├── shards/
|
|
||||||
│ ├── 001/
|
|
||||||
│ │ ├── data.bin
|
|
||||||
│ │ └── .lock (atomic)
|
|
||||||
│ ├── 002/
|
|
||||||
│ │ ├── data.bin
|
|
||||||
│ │ └── .lock
|
|
||||||
└── metadata/
|
|
||||||
├── index/
|
|
||||||
└── locks/
|
|
||||||
```
|
|
||||||
|
|
||||||
**Coordination**: Via distributed locking
|
|
||||||
- Shard-level locks using atomic operations
|
|
||||||
- Lock acquisition via compare-and-swap
|
|
||||||
- Automatic lock expiry (lease-based)
|
|
||||||
- Deadlock detection and recovery
|
|
||||||
|
|
||||||
## Storage Adapter Implementations
|
|
||||||
|
|
||||||
### 1. Filesystem Adapter
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class FilesystemAdapter implements StorageAdapter {
|
|
||||||
constructor(private basePath: string) {}
|
|
||||||
|
|
||||||
async get(key: string) {
|
|
||||||
const path = this.keyToPath(key)
|
|
||||||
return fs.readFile(path, 'json')
|
|
||||||
}
|
|
||||||
|
|
||||||
async compareAndSwap(key: string, oldVal: any, newVal: any) {
|
|
||||||
// Use file locking for atomicity
|
|
||||||
const lockfile = `${this.keyToPath(key)}.lock`
|
|
||||||
await flock(lockfile, 'ex') // Exclusive lock
|
|
||||||
try {
|
|
||||||
const current = await this.get(key)
|
|
||||||
if (deepEqual(current, oldVal)) {
|
|
||||||
await this.set(key, newVal)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
} finally {
|
|
||||||
await funlock(lockfile)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
withNamespace(ns: string) {
|
|
||||||
return new FilesystemAdapter(path.join(this.basePath, ns))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. S3 Adapter
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class S3Adapter implements StorageAdapter {
|
|
||||||
constructor(
|
|
||||||
private bucket: string,
|
|
||||||
private prefix: string = ''
|
|
||||||
) {}
|
|
||||||
|
|
||||||
async get(key: string) {
|
|
||||||
const result = await s3.getObject({
|
|
||||||
Bucket: this.bucket,
|
|
||||||
Key: `${this.prefix}${key}`
|
|
||||||
})
|
|
||||||
return JSON.parse(result.Body)
|
|
||||||
}
|
|
||||||
|
|
||||||
async compareAndSwap(key: string, oldVal: any, newVal: any) {
|
|
||||||
// Use S3's conditional writes
|
|
||||||
const fullKey = `${this.prefix}${key}`
|
|
||||||
|
|
||||||
// Get current version
|
|
||||||
const head = await s3.headObject({
|
|
||||||
Bucket: this.bucket,
|
|
||||||
Key: fullKey
|
|
||||||
})
|
|
||||||
|
|
||||||
// Conditional put with ETag
|
|
||||||
try {
|
|
||||||
await s3.putObject({
|
|
||||||
Bucket: this.bucket,
|
|
||||||
Key: fullKey,
|
|
||||||
Body: JSON.stringify(newVal),
|
|
||||||
IfMatch: head.ETag // Only succeeds if unchanged
|
|
||||||
})
|
|
||||||
return true
|
|
||||||
} catch (err) {
|
|
||||||
if (err.code === 'PreconditionFailed') {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
throw err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
withNamespace(ns: string) {
|
|
||||||
const newPrefix = `${this.prefix}${ns}/`
|
|
||||||
return new S3Adapter(this.bucket, newPrefix)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Cloudflare R2 Adapter
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class R2Adapter implements StorageAdapter {
|
|
||||||
// Similar to S3 but with R2-specific optimizations
|
|
||||||
|
|
||||||
async compareAndSwap(key: string, oldVal: any, newVal: any) {
|
|
||||||
// R2 supports conditional headers
|
|
||||||
const response = await fetch(`${this.endpoint}/${key}`, {
|
|
||||||
method: 'PUT',
|
|
||||||
body: JSON.stringify(newVal),
|
|
||||||
headers: {
|
|
||||||
'If-Match': await this.getETag(key)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
return response.ok
|
|
||||||
}
|
|
||||||
|
|
||||||
// R2-specific: Use Workers for edge computing
|
|
||||||
async getWithCache(key: string) {
|
|
||||||
// Check Cloudflare edge cache first
|
|
||||||
const cached = await caches.default.match(key)
|
|
||||||
if (cached) return cached.json()
|
|
||||||
|
|
||||||
// Fallback to R2
|
|
||||||
const value = await this.get(key)
|
|
||||||
|
|
||||||
// Cache at edge
|
|
||||||
await caches.default.put(key, new Response(JSON.stringify(value)))
|
|
||||||
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Distributed Coordination Patterns
|
|
||||||
|
|
||||||
### Pattern 1: Leader-Based Coordination
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class LeaderCoordinator {
|
|
||||||
async acquireShardOwnership(shardId: string) {
|
|
||||||
if (!this.isLeader()) {
|
|
||||||
// Only leader assigns shards
|
|
||||||
return this.requestFromLeader('acquireShard', shardId)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Leader logic
|
|
||||||
const shardMap = await this.storage.get('_cluster/shard-map')
|
|
||||||
if (!shardMap[shardId].owner) {
|
|
||||||
shardMap[shardId].owner = this.nodeId
|
|
||||||
|
|
||||||
// Atomic update
|
|
||||||
const success = await this.storage.compareAndSwap(
|
|
||||||
'_cluster/shard-map',
|
|
||||||
shardMap,
|
|
||||||
{ ...shardMap, [shardId]: { owner: this.nodeId } }
|
|
||||||
)
|
|
||||||
|
|
||||||
if (success) {
|
|
||||||
this.broadcast('shardAssigned', { shardId, owner: this.nodeId })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Pattern 2: Consensus-Based Coordination
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class ConsensusCoordinator {
|
|
||||||
async acquireShardOwnership(shardId: string) {
|
|
||||||
// Propose to all nodes
|
|
||||||
const proposal = {
|
|
||||||
type: 'ACQUIRE_SHARD',
|
|
||||||
shardId,
|
|
||||||
nodeId: this.nodeId,
|
|
||||||
term: this.currentTerm
|
|
||||||
}
|
|
||||||
|
|
||||||
// Raft consensus
|
|
||||||
const votes = await this.gatherVotes(proposal)
|
|
||||||
|
|
||||||
if (votes.length > this.nodes.length / 2) {
|
|
||||||
// Majority agreed
|
|
||||||
await this.commitProposal(proposal)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Pattern 3: Storage-Native Coordination
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class StorageNativeCoordinator {
|
|
||||||
async acquireShardOwnership(shardId: string) {
|
|
||||||
// Use storage adapter's native coordination
|
|
||||||
const lockKey = `_locks/shard-${shardId}`
|
|
||||||
const lease = {
|
|
||||||
owner: this.nodeId,
|
|
||||||
expires: Date.now() + 30000 // 30 second lease
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try to acquire lock atomically
|
|
||||||
const acquired = await this.storage.compareAndSwap(
|
|
||||||
lockKey,
|
|
||||||
null, // Must not exist
|
|
||||||
lease
|
|
||||||
)
|
|
||||||
|
|
||||||
if (acquired) {
|
|
||||||
// Start lease renewal
|
|
||||||
this.startLeaseRenewal(lockKey, lease)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
private startLeaseRenewal(key: string, lease: any) {
|
|
||||||
setInterval(async () => {
|
|
||||||
const renewed = await this.storage.compareAndSwap(
|
|
||||||
key,
|
|
||||||
lease,
|
|
||||||
{ ...lease, expires: Date.now() + 30000 }
|
|
||||||
)
|
|
||||||
|
|
||||||
if (!renewed) {
|
|
||||||
// Lost lease
|
|
||||||
this.handleLeaseLoss(key)
|
|
||||||
}
|
|
||||||
}, 10000) // Renew every 10s
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Multi-Storage Patterns
|
|
||||||
|
|
||||||
### Hybrid Storage (Hot/Cold)
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class HybridStorageAdapter implements StorageAdapter {
|
|
||||||
constructor(
|
|
||||||
private hot: StorageAdapter, // Fast SSD
|
|
||||||
private cold: StorageAdapter // Cheap S3
|
|
||||||
) {}
|
|
||||||
|
|
||||||
async get(key: string) {
|
|
||||||
// Try hot storage first
|
|
||||||
const hotValue = await this.hot.get(key).catch(() => null)
|
|
||||||
if (hotValue) {
|
|
||||||
this.updateAccessTime(key)
|
|
||||||
return hotValue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback to cold storage
|
|
||||||
const coldValue = await this.cold.get(key)
|
|
||||||
|
|
||||||
// Promote to hot storage if frequently accessed
|
|
||||||
if (this.shouldPromote(key)) {
|
|
||||||
await this.hot.set(key, coldValue)
|
|
||||||
}
|
|
||||||
|
|
||||||
return coldValue
|
|
||||||
}
|
|
||||||
|
|
||||||
async set(key: string, value: any) {
|
|
||||||
// Write to hot storage
|
|
||||||
await this.hot.set(key, value)
|
|
||||||
|
|
||||||
// Async write to cold storage
|
|
||||||
setImmediate(() => {
|
|
||||||
this.cold.set(key, value).catch(console.error)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Background process to demote cold data
|
|
||||||
async runTiering() {
|
|
||||||
const hotKeys = await this.hot.listKeys()
|
|
||||||
|
|
||||||
for (const key of hotKeys) {
|
|
||||||
const lastAccess = await this.getAccessTime(key)
|
|
||||||
|
|
||||||
if (Date.now() - lastAccess > 7 * 24 * 60 * 60 * 1000) {
|
|
||||||
// Not accessed in 7 days, demote to cold
|
|
||||||
await this.cold.set(key, await this.hot.get(key))
|
|
||||||
await this.hot.delete(key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Geo-Distributed Storage
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class GeoDistributedAdapter implements StorageAdapter {
|
|
||||||
constructor(
|
|
||||||
private regions: Map<string, StorageAdapter>
|
|
||||||
) {}
|
|
||||||
|
|
||||||
async get(key: string) {
|
|
||||||
// Determine closest region
|
|
||||||
const region = await this.getClosestRegion()
|
|
||||||
|
|
||||||
// Try local region first
|
|
||||||
const localValue = await this.regions.get(region)
|
|
||||||
.get(key)
|
|
||||||
.catch(() => null)
|
|
||||||
|
|
||||||
if (localValue) return localValue
|
|
||||||
|
|
||||||
// Fallback to other regions
|
|
||||||
for (const [name, adapter] of this.regions) {
|
|
||||||
if (name !== region) {
|
|
||||||
const value = await adapter.get(key).catch(() => null)
|
|
||||||
if (value) {
|
|
||||||
// Replicate to local region for next time
|
|
||||||
this.regions.get(region).set(key, value)
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new Error('Key not found in any region')
|
|
||||||
}
|
|
||||||
|
|
||||||
async set(key: string, value: any) {
|
|
||||||
// Write to local region immediately
|
|
||||||
const region = await this.getClosestRegion()
|
|
||||||
await this.regions.get(region).set(key, value)
|
|
||||||
|
|
||||||
// Async replication to other regions
|
|
||||||
for (const [name, adapter] of this.regions) {
|
|
||||||
if (name !== region) {
|
|
||||||
adapter.set(key, value).catch(console.error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Storage Optimization Strategies
|
|
||||||
|
|
||||||
### 1. Write Batching
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class BatchingAdapter implements StorageAdapter {
|
|
||||||
private writeBatch = new Map()
|
|
||||||
private batchTimer?: NodeJS.Timeout
|
|
||||||
|
|
||||||
async set(key: string, value: any) {
|
|
||||||
this.writeBatch.set(key, value)
|
|
||||||
|
|
||||||
if (!this.batchTimer) {
|
|
||||||
this.batchTimer = setTimeout(() => this.flush(), 100)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.writeBatch.size >= 1000) {
|
|
||||||
await this.flush()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async flush() {
|
|
||||||
if (this.writeBatch.size === 0) return
|
|
||||||
|
|
||||||
const batch = new Map(this.writeBatch)
|
|
||||||
this.writeBatch.clear()
|
|
||||||
|
|
||||||
await this.underlying.setBatch(batch)
|
|
||||||
|
|
||||||
if (this.batchTimer) {
|
|
||||||
clearTimeout(this.batchTimer)
|
|
||||||
this.batchTimer = undefined
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Read Caching
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class CachingAdapter implements StorageAdapter {
|
|
||||||
private cache = new LRU({ max: 10000 })
|
|
||||||
|
|
||||||
async get(key: string) {
|
|
||||||
// Check cache
|
|
||||||
if (this.cache.has(key)) {
|
|
||||||
return this.cache.get(key)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Read from storage
|
|
||||||
const value = await this.underlying.get(key)
|
|
||||||
|
|
||||||
// Cache for next time
|
|
||||||
this.cache.set(key, value)
|
|
||||||
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
|
|
||||||
async set(key: string, value: any) {
|
|
||||||
// Invalidate cache
|
|
||||||
this.cache.delete(key)
|
|
||||||
|
|
||||||
// Write through
|
|
||||||
await this.underlying.set(key, value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Compression
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class CompressingAdapter implements StorageAdapter {
|
|
||||||
async set(key: string, value: any) {
|
|
||||||
const json = JSON.stringify(value)
|
|
||||||
|
|
||||||
// Compress if beneficial
|
|
||||||
if (json.length > 1024) {
|
|
||||||
const compressed = await gzip(json)
|
|
||||||
await this.underlying.set(key, {
|
|
||||||
_compressed: true,
|
|
||||||
data: compressed.toString('base64')
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
await this.underlying.set(key, value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async get(key: string) {
|
|
||||||
const stored = await this.underlying.get(key)
|
|
||||||
|
|
||||||
if (stored._compressed) {
|
|
||||||
const compressed = Buffer.from(stored.data, 'base64')
|
|
||||||
const json = await gunzip(compressed)
|
|
||||||
return JSON.parse(json)
|
|
||||||
}
|
|
||||||
|
|
||||||
return stored
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
Brainy's storage layer is designed for:
|
|
||||||
|
|
||||||
1. **Flexibility**: Works with any storage backend
|
|
||||||
2. **Coordination**: Multiple strategies for different needs
|
|
||||||
3. **Performance**: Batching, caching, compression
|
|
||||||
4. **Scalability**: From single file to geo-distributed
|
|
||||||
5. **Simplicity**: Complexity hidden behind simple interface
|
|
||||||
|
|
||||||
The key insight: **Storage is just a plugin**. The intelligence is in the coordination layer above it!
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
*For user-facing documentation, see [SCALING.md](../SCALING.md)*
|
|
||||||
|
|
@ -10,14 +10,14 @@ Brainy has **3 main indexes** at the top level, each with multiple sub-indexes m
|
||||||
|
|
||||||
| Index | Purpose | Data Structure | Complexity | File Location | rebuild() Method |
|
| Index | Purpose | Data Structure | Complexity | File Location | rebuild() Method |
|
||||||
|-------|---------|----------------|------------|---------------|------------------|
|
|-------|---------|----------------|------------|---------------|------------------|
|
||||||
| **TypeAwareHNSWIndex** | Type-aware vector similarity search | 42 type-specific HNSW hierarchical graphs | O(log n) search | `src/hnsw/typeAwareHNSWIndex.ts` | ✅ Line 403 |
|
| **TypeAwareVectorIndex** | Type-aware vector similarity search | 42 type-specific hierarchical graphs | O(log n) search | `src/hnsw/typeAwareHNSWIndex.ts` | ✅ Line 403 |
|
||||||
| **MetadataIndexManager** | Fast metadata filtering | Chunked sparse indices with bloom filters + zone maps + roaring bitmaps | O(1) exact, O(log n) ranges | `src/utils/metadataIndex.ts` | ✅ Line 2318 |
|
| **MetadataIndexManager** | Fast metadata filtering | Chunked sparse indices with bloom filters + zone maps + roaring bitmaps | O(1) exact, O(log n) ranges | `src/utils/metadataIndex.ts` | ✅ Line 2318 |
|
||||||
| **GraphAdjacencyIndex** | Relationship traversal | 4 LSM-trees + bidirectional adjacency maps | O(1) per hop | `src/graph/graphAdjacencyIndex.ts` | ✅ Line 389 |
|
| **GraphAdjacencyIndex** | Relationship traversal | 2 verb-id LSM-trees + tombstone-filtered adjacency derivation | O(degree) per hop | `src/graph/graphAdjacencyIndex.ts` | ✅ Line 389 |
|
||||||
|
|
||||||
### Sub-Indexes (Level 2)
|
### Sub-Indexes (Level 2)
|
||||||
|
|
||||||
**TypeAwareHNSWIndex contains:**
|
**TypeAwareVectorIndex contains:**
|
||||||
- **42 type-specific HNSW indexes** - One per NounType (automatically rebuilt via parent)
|
- **42 type-specific vector indexes** - One per NounType (automatically rebuilt via parent)
|
||||||
|
|
||||||
**MetadataIndexManager contains:**
|
**MetadataIndexManager contains:**
|
||||||
- **ChunkManager** - Adaptive chunked sparse indexing
|
- **ChunkManager** - Adaptive chunked sparse indexing
|
||||||
|
|
@ -183,7 +183,7 @@ async getIdsForMultipleFields(pairs: [{field, value}, ...]): Promise<string[]> {
|
||||||
- Memory usage: **90% reduction** (17.17 MB → 2.01 MB for 100K entities)
|
- Memory usage: **90% reduction** (17.17 MB → 2.01 MB for 100K entities)
|
||||||
- Hardware acceleration: SIMD instructions make bitmap operations nearly free
|
- Hardware acceleration: SIMD instructions make bitmap operations nearly free
|
||||||
|
|
||||||
**Benchmark Results** (1,000 queries on various dataset sizes):
|
**Benchmark Results** — example output from a single run of `tests/performance/roaring-bitmap-benchmark.ts` (1,000 queries per size, one machine; absolute times vary by hardware, the relative speedup and memory savings are the durable signal):
|
||||||
| Dataset Size | Operation | Set Time | Roaring Time | Speedup | Memory Savings |
|
| Dataset Size | Operation | Set Time | Roaring Time | Speedup | Memory Savings |
|
||||||
|--------------|-----------|----------|--------------|---------|----------------|
|
|--------------|-----------|----------|--------------|---------|----------------|
|
||||||
| 10,000 entities | 3-field intersection | 3.74ms | 1.14ms | **3.3x faster** | 90% |
|
| 10,000 entities | 3-field intersection | 3.74ms | 1.14ms | **3.3x faster** | 90% |
|
||||||
|
|
@ -398,14 +398,16 @@ const DEFAULT_EXCLUDE_FIELDS = [
|
||||||
|
|
||||||
**Note**: Timestamp fields like `modified`, `accessed`, `created` are NO LONGER excluded as of they are indexed with automatic bucketing.
|
**Note**: Timestamp fields like `modified`, `accessed`, `created` are NO LONGER excluded as of they are indexed with automatic bucketing.
|
||||||
|
|
||||||
## 2. HNSWIndex - Vector Similarity Search
|
## 2. Vector Index - Vector Similarity Search
|
||||||
|
|
||||||
**Purpose**: O(log n) semantic similarity search using vector embeddings.
|
**Purpose**: O(log n) semantic similarity search using vector embeddings.
|
||||||
|
|
||||||
|
The default JS implementation is `JsHnswVectorIndex`; an optional native acceleration package (`@soulcraft/cortex`) can register a higher-performing `VectorIndexProvider` through the plugin system. The public API stays the same either way.
|
||||||
|
|
||||||
### Internal Architecture
|
### Internal Architecture
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
class HNSWIndex {
|
class JsHnswVectorIndex {
|
||||||
// Per-noun indexes for efficiency
|
// Per-noun indexes for efficiency
|
||||||
private nouns: Map<string, HNSWNoun> = new Map()
|
private nouns: Map<string, HNSWNoun> = new Map()
|
||||||
|
|
||||||
|
|
@ -437,7 +439,7 @@ class HNSWNode {
|
||||||
|
|
||||||
### Hierarchical Graph Structure
|
### Hierarchical Graph Structure
|
||||||
|
|
||||||
HNSW builds a multi-layered graph:
|
The default vector index builds a multi-layered graph:
|
||||||
|
|
||||||
```
|
```
|
||||||
Layer 2: [entry] ←→ [node1] (sparse, long-range connections)
|
Layer 2: [entry] ←→ [node1] (sparse, long-range connections)
|
||||||
|
|
@ -576,16 +578,6 @@ const reachable = await this.graphIndex.traverse({
|
||||||
// Complexity: O(V + E) breadth-first search, but each neighbor lookup is O(1)
|
// Complexity: O(V + E) breadth-first search, but each neighbor lookup is O(1)
|
||||||
```
|
```
|
||||||
|
|
||||||
## Notes on Other Indexes
|
|
||||||
|
|
||||||
### DeletedItemsIndex (Not Currently Used)
|
|
||||||
|
|
||||||
**Status**: Utility class exists in `src/utils/deletedItemsIndex.ts` but is **not instantiated** in the Brainy class.
|
|
||||||
|
|
||||||
**Purpose** (if used): O(1) tracking of soft-deleted items without removing data.
|
|
||||||
|
|
||||||
**Current Behavior**: Brainy uses hard deletes via `storage.deleteNoun()` and `storage.deleteVerb()`. Soft-delete functionality is not currently integrated.
|
|
||||||
|
|
||||||
## Shared Memory Management: UnifiedCache
|
## Shared Memory Management: UnifiedCache
|
||||||
|
|
||||||
All three main indexes share a single **UnifiedCache** instance for coordinated memory management.
|
All three main indexes share a single **UnifiedCache** instance for coordinated memory management.
|
||||||
|
|
@ -603,7 +595,7 @@ class UnifiedCache {
|
||||||
// Each index gets the same cache instance
|
// Each index gets the same cache instance
|
||||||
const unifiedCache = new UnifiedCache({ maxSize: 1000 })
|
const unifiedCache = new UnifiedCache({ maxSize: 1000 })
|
||||||
this.metadataIndex = new MetadataIndexManager(storage, { unifiedCache })
|
this.metadataIndex = new MetadataIndexManager(storage, { unifiedCache })
|
||||||
this.hnswIndex = new HNSWIndex(storage, { unifiedCache })
|
this.vectorIndex = new JsHnswVectorIndex(storage, { unifiedCache })
|
||||||
this.graphIndex = new GraphAdjacencyIndex(storage, { unifiedCache })
|
this.graphIndex = new GraphAdjacencyIndex(storage, { unifiedCache })
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -623,7 +615,7 @@ Each index uses different key prefixes:
|
||||||
// Metadata index
|
// Metadata index
|
||||||
cache.set(`meta:${field}:${value}`, indexEntry)
|
cache.set(`meta:${field}:${value}`, indexEntry)
|
||||||
|
|
||||||
// HNSW index
|
// Vector index
|
||||||
cache.set(`vector:${id}`, vectorData)
|
cache.set(`vector:${id}`, vectorData)
|
||||||
|
|
||||||
// Graph index
|
// Graph index
|
||||||
|
|
@ -645,7 +637,7 @@ async add(params: AddParams): Promise<string> {
|
||||||
// Add to metadata index (field filtering)
|
// Add to metadata index (field filtering)
|
||||||
await this.metadataIndex.addToIndex(id, params.metadata)
|
await this.metadataIndex.addToIndex(id, params.metadata)
|
||||||
|
|
||||||
// Add to HNSW index (vector search)
|
// Add to vector index (vector search)
|
||||||
await this.index.addEntity(id, vector, params.noun)
|
await this.index.addEntity(id, vector, params.noun)
|
||||||
|
|
||||||
// Relationships added via separate relate() calls
|
// Relationships added via separate relate() calls
|
||||||
|
|
@ -682,9 +674,6 @@ async find(query: FindQuery): Promise<Result[]> {
|
||||||
results = results.filter(r => connectedIds.includes(r.id))
|
results = results.filter(r => connectedIds.includes(r.id))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Step 4: Filter deleted items
|
|
||||||
results = results.filter(r => !this.deletedItemsIndex.isDeleted(r.id))
|
|
||||||
|
|
||||||
return results
|
return results
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
@ -700,7 +689,7 @@ async update(params: UpdateParams): Promise<void> {
|
||||||
await this.metadataIndex.removeFromIndex(params.id, existing.metadata)
|
await this.metadataIndex.removeFromIndex(params.id, existing.metadata)
|
||||||
await this.metadataIndex.addToIndex(params.id, params.metadata)
|
await this.metadataIndex.addToIndex(params.id, params.metadata)
|
||||||
|
|
||||||
// Update HNSW index (re-embed if content changed)
|
// Update vector index (re-embed if content changed)
|
||||||
if (params.content) {
|
if (params.content) {
|
||||||
const newVector = await this.embedder(params.content)
|
const newVector = await this.embedder(params.content)
|
||||||
await this.index.updateEntity(params.id, newVector)
|
await this.index.updateEntity(params.id, newVector)
|
||||||
|
|
@ -726,10 +715,7 @@ async stats(): Promise<Statistics> {
|
||||||
relationships: this.graphIndex.getTotalRelationshipCount(),
|
relationships: this.graphIndex.getTotalRelationshipCount(),
|
||||||
relationshipTypes: this.graphIndex.getRelationshipCountsByType(),
|
relationshipTypes: this.graphIndex.getRelationshipCountsByType(),
|
||||||
|
|
||||||
// From deleted items index
|
// From vector index
|
||||||
deletedItems: this.deletedItemsIndex.getDeletedCount(),
|
|
||||||
|
|
||||||
// From HNSW index
|
|
||||||
vectorIndexSize: this.index.getSize()
|
vectorIndexSize: this.index.getSize()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -746,16 +732,16 @@ async stats(): Promise<Statistics> {
|
||||||
async init(): Promise<void> {
|
async init(): Promise<void> {
|
||||||
// When disableAutoRebuild: false (default)
|
// When disableAutoRebuild: false (default)
|
||||||
const metadataStats = await this.metadataIndex.getStats()
|
const metadataStats = await this.metadataIndex.getStats()
|
||||||
const hnswIndexSize = this.index.size()
|
const vectorIndexSize = this.index.size()
|
||||||
const graphIndexSize = await this.graphIndex.size()
|
const graphIndexSize = await this.graphIndex.size()
|
||||||
|
|
||||||
if (metadataStats.totalEntries === 0 ||
|
if (metadataStats.totalEntries === 0 ||
|
||||||
hnswIndexSize === 0 ||
|
vectorIndexSize === 0 ||
|
||||||
graphIndexSize === 0) {
|
graphIndexSize === 0) {
|
||||||
// Rebuild all indexes in parallel
|
// Rebuild all indexes in parallel
|
||||||
await Promise.all([
|
await Promise.all([
|
||||||
metadataStats.totalEntries === 0 ? this.metadataIndex.rebuild() : Promise.resolve(),
|
metadataStats.totalEntries === 0 ? this.metadataIndex.rebuild() : Promise.resolve(),
|
||||||
hnswIndexSize === 0 ? this.index.rebuild() : Promise.resolve(),
|
vectorIndexSize === 0 ? this.index.rebuild() : Promise.resolve(),
|
||||||
graphIndexSize === 0 ? this.graphIndex.rebuild() : Promise.resolve()
|
graphIndexSize === 0 ? this.graphIndex.rebuild() : Promise.resolve()
|
||||||
])
|
])
|
||||||
}
|
}
|
||||||
|
|
@ -795,7 +781,7 @@ The **TripleIntelligenceSystem** (`src/triple/TripleIntelligenceSystem.ts`) comb
|
||||||
class TripleIntelligenceSystem {
|
class TripleIntelligenceSystem {
|
||||||
constructor(
|
constructor(
|
||||||
private metadataIndex: MetadataIndexManager,
|
private metadataIndex: MetadataIndexManager,
|
||||||
private hnswIndex: HNSWIndex,
|
private vectorIndex: VectorIndexProvider,
|
||||||
private graphIndex: GraphAdjacencyIndex,
|
private graphIndex: GraphAdjacencyIndex,
|
||||||
private embedder: EmbedderFunction,
|
private embedder: EmbedderFunction,
|
||||||
private storage: BaseStorage
|
private storage: BaseStorage
|
||||||
|
|
@ -808,7 +794,7 @@ class TripleIntelligenceSystem {
|
||||||
// Execute across all three indexes
|
// Execute across all three indexes
|
||||||
const [metadataResults, vectorResults, graphResults] = await Promise.all([
|
const [metadataResults, vectorResults, graphResults] = await Promise.all([
|
||||||
this.metadataIndex.getIdsForFilter(parsed.filters),
|
this.metadataIndex.getIdsForFilter(parsed.filters),
|
||||||
this.hnswIndex.search(parsed.vector, parsed.limit),
|
this.vectorIndex.search(parsed.vector, parsed.limit),
|
||||||
this.graphIndex.traverse(parsed.graphConstraints)
|
this.graphIndex.traverse(parsed.graphConstraints)
|
||||||
])
|
])
|
||||||
|
|
||||||
|
|
@ -822,7 +808,7 @@ class TripleIntelligenceSystem {
|
||||||
|
|
||||||
### Operation Complexity by Index
|
### Operation Complexity by Index
|
||||||
|
|
||||||
| Operation | MetadataIndexManager | TypeAwareHNSWIndex | GraphAdjacencyIndex |
|
| Operation | MetadataIndexManager | TypeAwareVectorIndex | GraphAdjacencyIndex |
|
||||||
|-----------|---------------------|-------------------|---------------------|
|
|-----------|---------------------|-------------------|---------------------|
|
||||||
| **Add** | O(1) per field | O(log n) | O(1) |
|
| **Add** | O(1) per field | O(log n) | O(1) |
|
||||||
| **Remove** | O(1) per field | O(log n) | O(1) |
|
| **Remove** | O(1) per field | O(log n) | O(1) |
|
||||||
|
|
@ -837,15 +823,15 @@ Where:
|
||||||
- n = total number of entities
|
- n = total number of entities
|
||||||
- k = number of matching results
|
- k = number of matching results
|
||||||
|
|
||||||
**Note**: All 3 main indexes have rebuild() methods that load persisted data (O(n)) rather than recomputing (which would be O(n log n) for HNSW).
|
**Note**: All 3 main indexes have rebuild() methods that load persisted data (O(n)) rather than recomputing (which would be O(n log n) for the vector index).
|
||||||
|
|
||||||
### Memory Footprint
|
### Memory Footprint
|
||||||
|
|
||||||
| Index | Per-Entity Memory | Notes |
|
| Index | Per-Entity Memory | Notes |
|
||||||
|-------|-------------------|-------|
|
|-------|-------------------|-------|
|
||||||
| **MetadataIndexManager** | ~100 bytes | Depends on field count and cardinality (RoaringBitmap32 compression) |
|
| **MetadataIndexManager** | ~100 bytes | Depends on field count and cardinality (RoaringBitmap32 compression) |
|
||||||
| **TypeAwareHNSWIndex** | ~1.5 KB | Vector (384 dims × 4 bytes) + graph connections across 42 type-specific indexes |
|
| **TypeAwareVectorIndex** | ~1.5 KB | Vector (384 dims × 4 bytes) + graph connections across 42 type-specific indexes |
|
||||||
| **GraphAdjacencyIndex** | ~50 bytes per relationship | Bidirectional references + metadata in 4 LSM-trees |
|
| **GraphAdjacencyIndex** | ~50 bytes per relationship | Bidirectional verb-id references in 2 LSM-trees |
|
||||||
|
|
||||||
**Total overhead**: ~1.6 KB per entity + ~50 bytes per relationship
|
**Total overhead**: ~1.6 KB per entity + ~50 bytes per relationship
|
||||||
|
|
||||||
|
|
@ -856,19 +842,20 @@ Where:
|
||||||
|
|
||||||
### Scalability
|
### Scalability
|
||||||
|
|
||||||
All indexes scale gracefully:
|
All indexes scale gracefully. The cost of each stage is governed by its algorithmic complexity, not a fixed millisecond figure — absolute latency depends on hardware, embedding model, and storage backend. Only the graph adjacency index carries a committed scale assertion:
|
||||||
|
|
||||||
| Database Size | Metadata Filter | Vector Search | Graph Hop | Combined Query |
|
| Query stage | Complexity | Scaling behavior |
|
||||||
|---------------|----------------|---------------|-----------|----------------|
|
|-------------|------------|------------------|
|
||||||
| **1K entities** | 0.3ms | 0.8ms | 0.05ms | 1.1ms |
|
| Metadata filter (exact) | O(1) | Constant — independent of dataset size |
|
||||||
| **10K entities** | 0.5ms | 1.2ms | 0.08ms | 1.5ms |
|
| Metadata filter (range) | O(log n) + O(k) | Sub-linear; k = matching results |
|
||||||
| **100K entities** | 0.8ms | 1.8ms | 0.1ms | 2.1ms |
|
| Vector search (HNSW) | O(log n) | Degrades gracefully via hierarchical layers |
|
||||||
| **1M entities** | 1.2ms | 2.5ms | 0.1ms | 2.8ms |
|
| Graph hop | O(1) | Measured <1 ms per neighbor lookup, validated up to 1M relationships (`tests/performance/graph-scale-performance.test.ts:238`) |
|
||||||
|
| Combined query | O(log n) | Bounded by the vector stage; metadata and graph stages stay O(1)/O(log n) |
|
||||||
|
|
||||||
**Key observations**:
|
**Key observations**:
|
||||||
- Graph queries stay O(1) regardless of scale
|
- Graph queries stay O(1) regardless of scale
|
||||||
- Metadata filtering scales sub-linearly
|
- Metadata filtering scales sub-linearly
|
||||||
- Vector search degrades gracefully due to HNSW
|
- Vector search degrades gracefully due to the hierarchical index
|
||||||
- Combined queries remain fast even at scale
|
- Combined queries remain fast even at scale
|
||||||
|
|
||||||
## Best Practices
|
## Best Practices
|
||||||
|
|
@ -881,7 +868,7 @@ All indexes scale gracefully:
|
||||||
- Field discovery (what filters are available)
|
- Field discovery (what filters are available)
|
||||||
- Type-based querying (find all characters, all items)
|
- Type-based querying (find all characters, all items)
|
||||||
|
|
||||||
**HNSWIndex**:
|
**Vector Index**:
|
||||||
- Semantic similarity search ("find similar documents")
|
- Semantic similarity search ("find similar documents")
|
||||||
- Content-based retrieval ("find posts about AI")
|
- Content-based retrieval ("find posts about AI")
|
||||||
- Fuzzy matching (when exact matches aren't required)
|
- Fuzzy matching (when exact matches aren't required)
|
||||||
|
|
@ -906,9 +893,9 @@ All indexes scale gracefully:
|
||||||
### Memory Management
|
### Memory Management
|
||||||
|
|
||||||
1. **Configure UnifiedCache appropriately** - Balance between speed and memory
|
1. **Configure UnifiedCache appropriately** - Balance between speed and memory
|
||||||
2. **Use lazy loading** - HNSW loads vectors on-demand
|
2. **Use lazy loading** - Vector index loads vectors on-demand
|
||||||
3. **Monitor cache hit rates** - Adjust cache size if hit rate is low
|
3. **Monitor cache hit rates** - Adjust cache size if hit rate is low
|
||||||
4. **Consider storage adapter** - Memory storage = fastest, S3 = most scalable
|
4. **Consider storage adapter** - Memory = fastest, filesystem = persistent
|
||||||
|
|
||||||
## Related Documentation
|
## Related Documentation
|
||||||
|
|
||||||
|
|
@ -922,15 +909,15 @@ All indexes scale gracefully:
|
||||||
|
|
||||||
### Level 1: Main Indexes (3)
|
### Level 1: Main Indexes (3)
|
||||||
All have rebuild() methods and are covered by lazy loading:
|
All have rebuild() methods and are covered by lazy loading:
|
||||||
1. **TypeAwareHNSWIndex** - `src/hnsw/typeAwareHNSWIndex.ts:403`
|
1. **TypeAwareVectorIndex** - `src/hnsw/typeAwareHNSWIndex.ts:403`
|
||||||
2. **MetadataIndexManager** - `src/utils/metadataIndex.ts:2318`
|
2. **MetadataIndexManager** - `src/utils/metadataIndex.ts:2318`
|
||||||
3. **GraphAdjacencyIndex** - `src/graph/graphAdjacencyIndex.ts:389`
|
3. **GraphAdjacencyIndex** - `src/graph/graphAdjacencyIndex.ts:389`
|
||||||
|
|
||||||
### Level 2: Sub-Indexes (~50+)
|
### Level 2: Sub-Indexes (~50+)
|
||||||
Automatically managed by parent rebuild():
|
Automatically managed by parent rebuild():
|
||||||
- **42 type-specific HNSW indexes** (one per NounType)
|
- **42 type-specific vector indexes** (one per NounType)
|
||||||
- **6 metadata components** (ChunkManager, EntityIdMapper, FieldTypeInference, Field Sparse Indexes, Sorted Indexes)
|
- **6 metadata components** (ChunkManager, EntityIdMapper, FieldTypeInference, Field Sparse Indexes, Sorted Indexes)
|
||||||
- **4 LSM-trees** (lsmTreeSource, lsmTreeTarget, lsmTreeVerbsBySource, lsmTreeVerbsByTarget)
|
- **2 LSM-trees** (lsmTreeVerbsBySource, lsmTreeVerbsByTarget — the verb set is the single adjacency source of truth; neighbor reads derive from live verbs so removals are honored)
|
||||||
- **In-memory graph structures** (sourceIndex, targetIndex, verbIndex)
|
- **In-memory graph structures** (sourceIndex, targetIndex, verbIndex)
|
||||||
|
|
||||||
### Lazy Loading
|
### Lazy Loading
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
# Initialization and Rebuild Processes
|
# Initialization and Rebuild Processes
|
||||||
|
|
||||||
This document explains how Brainy's four indexes (MetadataIndex, HNSWIndex, GraphAdjacencyIndex, DeletedItemsIndex) initialize and rebuild from persisted storage.
|
This document explains how Brainy's four indexes (MetadataIndex, vector index, GraphAdjacencyIndex, DeletedItemsIndex) initialize and rebuild from persisted storage.
|
||||||
|
|
||||||
## Core Principle: All Indexes Are Disk-Based
|
## Core Principle: All Indexes Are Disk-Based
|
||||||
|
|
||||||
|
|
@ -11,7 +11,7 @@ This document explains how Brainy's four indexes (MetadataIndex, HNSWIndex, Grap
|
||||||
| Index | Persisted Data | Storage Method | Since Version |
|
| Index | Persisted Data | Storage Method | Since Version |
|
||||||
|-------|---------------|----------------|---------------|
|
|-------|---------------|----------------|---------------|
|
||||||
| **MetadataIndex** | Field registry + chunked sparse indices with bloom filters + zone maps | `storage.saveMetadata()` | v3.42.0 (chunks), v4.2.1 (registry) |
|
| **MetadataIndex** | Field registry + chunked sparse indices with bloom filters + zone maps | `storage.saveMetadata()` | v3.42.0 (chunks), v4.2.1 (registry) |
|
||||||
| **HNSWIndex** | Vector embeddings + HNSW graph connections | `storage.saveHNSWData()` + `storage.saveHNSWSystem()` | v3.35.0 |
|
| **Vector Index** | Vector embeddings + graph connections | `storage.saveHNSWData()` + `storage.saveHNSWSystem()` | v3.35.0 |
|
||||||
| **GraphAdjacencyIndex** | Relationships via LSM-tree SSTables | LSM-tree auto-persistence | v3.44.0 |
|
| **GraphAdjacencyIndex** | Relationships via LSM-tree SSTables | LSM-tree auto-persistence | v3.44.0 |
|
||||||
| **DeletedItemsIndex** | Set of deleted IDs | `storage.saveDeletedItems()` | v3.0.0 |
|
| **DeletedItemsIndex** | Set of deleted IDs | `storage.saveDeletedItems()` | v3.0.0 |
|
||||||
|
|
||||||
|
|
@ -32,7 +32,7 @@ The MetadataIndex now persists two components:
|
||||||
- Roaring bitmaps for compressed entity ID storage
|
- Roaring bitmaps for compressed entity ID storage
|
||||||
- Loaded on-demand based on query patterns
|
- Loaded on-demand based on query patterns
|
||||||
|
|
||||||
All storage operations use the **StorageAdapter** interface, which works with FileSystem, OPFS, S3, GCS, R2, and Memory backends.
|
All storage operations use the **StorageAdapter** interface, which works with FileSystem and Memory backends.
|
||||||
|
|
||||||
## Initialization Process
|
## Initialization Process
|
||||||
|
|
||||||
|
|
@ -84,12 +84,12 @@ async init(): Promise<void> {
|
||||||
|
|
||||||
// STEP 2: Check index sizes (lazy initialization triggers here)
|
// STEP 2: Check index sizes (lazy initialization triggers here)
|
||||||
const metadataStats = await this.metadataIndex.getStats()
|
const metadataStats = await this.metadataIndex.getStats()
|
||||||
const hnswIndexSize = this.index.size()
|
const vectorIndexSize = this.index.size()
|
||||||
const graphIndexSize = await this.graphIndex.size()
|
const graphIndexSize = await this.graphIndex.size()
|
||||||
|
|
||||||
// STEP 3: Rebuild empty indexes from storage in parallel
|
// STEP 3: Rebuild empty indexes from storage in parallel
|
||||||
if (metadataStats.totalEntries === 0 ||
|
if (metadataStats.totalEntries === 0 ||
|
||||||
hnswIndexSize === 0 ||
|
vectorIndexSize === 0 ||
|
||||||
graphIndexSize === 0) {
|
graphIndexSize === 0) {
|
||||||
|
|
||||||
const rebuildStartTime = Date.now()
|
const rebuildStartTime = Date.now()
|
||||||
|
|
@ -97,7 +97,7 @@ async init(): Promise<void> {
|
||||||
metadataStats.totalEntries === 0
|
metadataStats.totalEntries === 0
|
||||||
? this.metadataIndex.rebuild()
|
? this.metadataIndex.rebuild()
|
||||||
: Promise.resolve(),
|
: Promise.resolve(),
|
||||||
hnswIndexSize === 0
|
vectorIndexSize === 0
|
||||||
? this.index.rebuild()
|
? this.index.rebuild()
|
||||||
: Promise.resolve(),
|
: Promise.resolve(),
|
||||||
graphIndexSize === 0
|
graphIndexSize === 0
|
||||||
|
|
@ -207,13 +207,13 @@ private async ensureIndexesLoaded(): Promise<void> {
|
||||||
### What "Rebuild" Actually Means
|
### What "Rebuild" Actually Means
|
||||||
|
|
||||||
**IMPORTANT**: "Rebuild" does NOT mean recomputing data. It means:
|
**IMPORTANT**: "Rebuild" does NOT mean recomputing data. It means:
|
||||||
1. **Load persisted data** from storage (HNSW connections, metadata chunks, LSM-tree SSTables)
|
1. **Load persisted data** from storage (vector index connections, metadata chunks, LSM-tree SSTables)
|
||||||
2. **Populate in-memory structures** (Maps, Sets, graphs)
|
2. **Populate in-memory structures** (Maps, Sets, graphs)
|
||||||
3. **Apply adaptive caching** (preload vectors if small dataset, lazy load if large)
|
3. **Apply adaptive caching** (preload vectors if small dataset, lazy load if large)
|
||||||
|
|
||||||
**Complexity**: O(N) - linear scan through storage, NOT O(N log N) recomputation!
|
**Complexity**: O(N) - linear scan through storage, NOT O(N log N) recomputation!
|
||||||
|
|
||||||
### 1. HNSWIndex Rebuild (Correct Pattern)
|
### 1. Vector Index Rebuild (Correct Pattern)
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// src/hnsw/hnswIndex.ts (lines 809-947)
|
// src/hnsw/hnswIndex.ts (lines 809-947)
|
||||||
|
|
@ -236,7 +236,7 @@ public async rebuild(options: {
|
||||||
const availableCache = this.unifiedCache.getRemainingCapacity()
|
const availableCache = this.unifiedCache.getRemainingCapacity()
|
||||||
const shouldPreload = vectorMemory < availableCache * 0.3
|
const shouldPreload = vectorMemory < availableCache * 0.3
|
||||||
|
|
||||||
// STEP 4: Load entities with persisted HNSW connections
|
// STEP 4: Load entities with persisted vector index connections
|
||||||
let hasMore = true
|
let hasMore = true
|
||||||
let cursor: string | undefined = undefined
|
let cursor: string | undefined = undefined
|
||||||
|
|
||||||
|
|
@ -246,7 +246,7 @@ public async rebuild(options: {
|
||||||
})
|
})
|
||||||
|
|
||||||
for (const nounData of result.items) {
|
for (const nounData of result.items) {
|
||||||
// Load HNSW graph data from storage (NOT recomputed!)
|
// Load vector graph data from storage (NOT recomputed!)
|
||||||
const hnswData = await this.storage.getHNSWData(nounData.id)
|
const hnswData = await this.storage.getHNSWData(nounData.id)
|
||||||
|
|
||||||
// Create noun with restored connections
|
// Create noun with restored connections
|
||||||
|
|
@ -274,14 +274,14 @@ public async rebuild(options: {
|
||||||
```
|
```
|
||||||
|
|
||||||
**Key Points**:
|
**Key Points**:
|
||||||
- ✅ Loads HNSW connections from storage via `getHNSWData()`
|
- ✅ Loads vector index connections from storage via `getHNSWData()`
|
||||||
- ✅ Uses adaptive caching (preload vectors if < 30% of available cache)
|
- ✅ Uses adaptive caching (preload vectors if < 30% of available cache)
|
||||||
- ✅ O(N) complexity - just loads existing data
|
- ✅ O(N) complexity - just loads existing data
|
||||||
- ❌ Does NOT call `addItem()` which would recompute connections (O(N log N))
|
- ❌ Does NOT call `addItem()` which would recompute connections (O(N log N))
|
||||||
|
|
||||||
### 2. TypeAwareHNSWIndex Rebuild (Fixed in v3.45.0)
|
### 2. TypeAwareVectorIndex Rebuild (Fixed in v3.45.0)
|
||||||
|
|
||||||
**Critical Architectural Fix**: TypeAwareHNSWIndex previously had TWO major bugs:
|
**Critical Architectural Fix**: The type-aware vector index previously had TWO major bugs:
|
||||||
|
|
||||||
1. **Bug #1**: Called `addItem()` during rebuild → O(N log N) recomputation instead of O(N) loading
|
1. **Bug #1**: Called `addItem()` during rebuild → O(N log N) recomputation instead of O(N) loading
|
||||||
2. **Bug #2**: Loaded ALL nouns 31 times in parallel (once per type) → O(31*N) complexity causing timeouts
|
2. **Bug #2**: Loaded ALL nouns 31 times in parallel (once per type) → O(31*N) complexity causing timeouts
|
||||||
|
|
@ -300,7 +300,7 @@ public async rebuild(options?: {
|
||||||
index.clear()
|
index.clear()
|
||||||
}
|
}
|
||||||
|
|
||||||
// STEP 2: Determine preloading strategy (same as HNSWIndex)
|
// STEP 2: Determine preloading strategy (same as vector index)
|
||||||
const totalNouns = await this.storage.getNounCount()
|
const totalNouns = await this.storage.getNounCount()
|
||||||
const vectorMemory = totalNouns * 384 * 4
|
const vectorMemory = totalNouns * 384 * 4
|
||||||
const availableCache = this.unifiedCache.getRemainingCapacity()
|
const availableCache = this.unifiedCache.getRemainingCapacity()
|
||||||
|
|
@ -319,7 +319,7 @@ public async rebuild(options?: {
|
||||||
})
|
})
|
||||||
|
|
||||||
for (const nounData of result.items) {
|
for (const nounData of result.items) {
|
||||||
// CORRECT: Load persisted HNSW data (not recomputed!)
|
// CORRECT: Load persisted vector index data (not recomputed!)
|
||||||
const hnswData = await this.storage.getHNSWData(nounData.id)
|
const hnswData = await this.storage.getHNSWData(nounData.id)
|
||||||
|
|
||||||
const noun = {
|
const noun = {
|
||||||
|
|
@ -533,7 +533,7 @@ const unifiedCache = getGlobalCache() // Singleton, 100MB default
|
||||||
// MetadataIndex
|
// MetadataIndex
|
||||||
this.unifiedCache = unifiedCache
|
this.unifiedCache = unifiedCache
|
||||||
|
|
||||||
// HNSWIndex
|
// Vector index
|
||||||
this.unifiedCache = unifiedCache
|
this.unifiedCache = unifiedCache
|
||||||
|
|
||||||
// GraphAdjacencyIndex
|
// GraphAdjacencyIndex
|
||||||
|
|
@ -549,7 +549,7 @@ this.unifiedCache = unifiedCache
|
||||||
|
|
||||||
### Rebuild Times (Typical Hardware)
|
### Rebuild Times (Typical Hardware)
|
||||||
|
|
||||||
| Dataset Size | Metadata | HNSW | Graph | Total (Parallel) |
|
| Dataset Size | Metadata | Vector | Graph | Total (Parallel) |
|
||||||
|--------------|----------|------|-------|------------------|
|
|--------------|----------|------|-------|------------------|
|
||||||
| 1K entities | 50ms | 100ms | 30ms | **150ms** |
|
| 1K entities | 50ms | 100ms | 30ms | **150ms** |
|
||||||
| 10K entities | 200ms | 500ms | 150ms | **600ms** |
|
| 10K entities | 200ms | 500ms | 150ms | **600ms** |
|
||||||
|
|
@ -563,7 +563,7 @@ this.unifiedCache = unifiedCache
|
||||||
| Index | In-Memory Overhead | Disk Storage |
|
| Index | In-Memory Overhead | Disk Storage |
|
||||||
|-------|-------------------|--------------|
|
|-------|-------------------|--------------|
|
||||||
| **MetadataIndex** | ~100 bytes/entity | ~500 bytes/entity (chunks) |
|
| **MetadataIndex** | ~100 bytes/entity | ~500 bytes/entity (chunks) |
|
||||||
| **HNSWIndex** | ~200 bytes/entity (no vectors) | ~1.5 KB/entity (vectors + connections) |
|
| **Vector Index** | ~200 bytes/entity (no vectors) | ~1.5 KB/entity (vectors + connections) |
|
||||||
| **GraphAdjacencyIndex** | ~128 bytes/relationship | ~200 bytes/relationship (LSM-tree) |
|
| **GraphAdjacencyIndex** | ~128 bytes/relationship | ~200 bytes/relationship (LSM-tree) |
|
||||||
| **DeletedItemsIndex** | ~40 bytes/deleted ID | ~50 bytes/deleted ID |
|
| **DeletedItemsIndex** | ~40 bytes/deleted ID | ~50 bytes/deleted ID |
|
||||||
|
|
||||||
|
|
@ -573,9 +573,9 @@ this.unifiedCache = unifiedCache
|
||||||
|
|
||||||
### O(N) vs O(N log N) Comparison
|
### O(N) vs O(N log N) Comparison
|
||||||
|
|
||||||
**Before fix** (TypeAwareHNSWIndex bug):
|
**Before fix** (TypeAwareVectorIndex bug):
|
||||||
```typescript
|
```typescript
|
||||||
// BAD: Recomputes HNSW connections during rebuild
|
// BAD: Recomputes vector index connections during rebuild
|
||||||
for (const noun of nouns) {
|
for (const noun of nouns) {
|
||||||
await index.addItem(noun) // O(log N) per item → O(N log N) total
|
await index.addItem(noun) // O(log N) per item → O(N log N) total
|
||||||
}
|
}
|
||||||
|
|
@ -652,7 +652,7 @@ console.timeEnd('rebuild')
|
||||||
|
|
||||||
// For 10K entities:
|
// For 10K entities:
|
||||||
// - Expected: 500-800ms (loading from storage)
|
// - Expected: 500-800ms (loading from storage)
|
||||||
// - Bug: 5-10 minutes (recomputing HNSW connections)
|
// - Bug: 5-10 minutes (recomputing vector index connections)
|
||||||
```
|
```
|
||||||
|
|
||||||
**Solution**: Ensure index is loading from storage, not calling `addItem()` during rebuild.
|
**Solution**: Ensure index is loading from storage, not calling `addItem()` during rebuild.
|
||||||
|
|
@ -706,8 +706,8 @@ console.log('Nouns in storage:', nouns.items.length)
|
||||||
## Version History
|
## Version History
|
||||||
|
|
||||||
- **v5.7.7** (November 2025): Added production-scale lazy loading with `ensureIndexesLoaded()` helper. Fixed critical bug where `disableAutoRebuild: true` left indexes empty forever. Added concurrency control (mutex) to prevent duplicate rebuilds from concurrent queries. Added `getIndexStatus()` diagnostic method. Zero-config operation - works automatically.
|
- **v5.7.7** (November 2025): Added production-scale lazy loading with `ensureIndexesLoaded()` helper. Fixed critical bug where `disableAutoRebuild: true` left indexes empty forever. Added concurrency control (mutex) to prevent duplicate rebuilds from concurrent queries. Added `getIndexStatus()` diagnostic method. Zero-config operation - works automatically.
|
||||||
- **v3.45.0** (October 2025): Fixed TypeAwareHNSWIndex.rebuild() to load from storage instead of recomputing. Removed all snapshot code (unnecessary with correct rebuild pattern). 200-600x speedup.
|
- **v3.45.0** (October 2025): Fixed type-aware vector index `rebuild()` to load from storage instead of recomputing. Removed all snapshot code (unnecessary with correct rebuild pattern). 200-600x speedup.
|
||||||
- **v3.44.0** (October 2025): GraphAdjacencyIndex migrated to LSM-tree storage for billion-scale relationships
|
- **v3.44.0** (October 2025): GraphAdjacencyIndex migrated to LSM-tree storage for billion-scale relationships
|
||||||
- **v3.42.0** (October 2025): MetadataIndex migrated to chunked sparse indexing
|
- **v3.42.0** (October 2025): MetadataIndex migrated to chunked sparse indexing
|
||||||
- **v3.35.0** (August 2025): HNSW connections first persisted to storage
|
- **v3.35.0** (August 2025): Vector index connections first persisted to storage
|
||||||
- **v3.0.0** (September 2025): Initial 3-tier index architecture
|
- **v3.0.0** (September 2025): Initial 3-tier index architecture
|
||||||
|
|
|
||||||
|
|
@ -32,7 +32,7 @@ on `BaseStorage`.
|
||||||
|
|
||||||
`BaseStorage` already mixes several concerns:
|
`BaseStorage` already mixes several concerns:
|
||||||
- entity / verb CRUD primitives
|
- entity / verb CRUD primitives
|
||||||
- COW (copy-on-write) lifecycle
|
- generational record hooks (8.0 MVCC)
|
||||||
- type-statistics tracking
|
- type-statistics tracking
|
||||||
- count persistence
|
- count persistence
|
||||||
- multi-process safety (new)
|
- multi-process safety (new)
|
||||||
|
|
@ -86,9 +86,8 @@ Benefits:
|
||||||
`FileSystemStorage` is the only one in-tree, but Cortex's
|
`FileSystemStorage` is the only one in-tree, but Cortex's
|
||||||
`MmapFileSystemStorage` inherits from it — interface relocation would
|
`MmapFileSystemStorage` inherits from it — interface relocation would
|
||||||
ripple through the plugin ecosystem.
|
ripple through the plugin ecosystem.
|
||||||
- The current state works. Real failure modes
|
- The current state works. The real failure modes seen in the field
|
||||||
(BR-CX-INTERFACE-GAP) were build/install artifacts, not type-system
|
were build/install artifacts, not type-system failures.
|
||||||
failures.
|
|
||||||
- 7.22.0 just shipped a clean fix. Stacking another refactor before
|
- 7.22.0 just shipped a clean fix. Stacking another refactor before
|
||||||
consumers absorb it adds churn without urgency.
|
consumers absorb it adds churn without urgency.
|
||||||
- The `hasStorageMethod()` guard accomplishes the same runtime safety the
|
- The `hasStorageMethod()` guard accomplishes the same runtime safety the
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -6,14 +6,14 @@ Brainy is a multi-dimensional AI database that combines vector similarity, graph
|
||||||
|
|
||||||
### Brainy (Main Entry Point)
|
### Brainy (Main Entry Point)
|
||||||
The central orchestrator that manages all subsystems:
|
The central orchestrator that manages all subsystems:
|
||||||
- **4-Index Architecture**: MetadataIndex, HNSWIndex, GraphAdjacencyIndex, DeletedItemsIndex (see [Index Architecture](./index-architecture.md))
|
- **4-Index Architecture**: MetadataIndex, vector index, GraphAdjacencyIndex, DeletedItemsIndex (see [Index Architecture](./index-architecture.md))
|
||||||
- **Storage System**: Universal storage adapters (FileSystem, S3, OPFS, Memory)
|
- **Storage System**: FileSystem and Memory adapters
|
||||||
- **Augmentation System**: Extensible plugin architecture
|
- **Augmentation System**: Extensible plugin architecture
|
||||||
- **Triple Intelligence**: Unified query engine
|
- **Triple Intelligence**: Unified query engine
|
||||||
|
|
||||||
### Triple Intelligence Engine
|
### Triple Intelligence Engine
|
||||||
Brainy's revolutionary feature that unifies three types of search:
|
Brainy's revolutionary feature that unifies three types of search:
|
||||||
- **Vector Search**: Semantic similarity using HNSW indexing
|
- **Vector Search**: Semantic similarity via the pluggable vector index
|
||||||
- **Graph Traversal**: Relationship-based queries
|
- **Graph Traversal**: Relationship-based queries
|
||||||
- **Field Filtering**: Precise metadata filtering with O(1) performance
|
- **Field Filtering**: Precise metadata filtering with O(1) performance
|
||||||
|
|
||||||
|
|
@ -42,12 +42,13 @@ brainy-data/
|
||||||
└── locks/ # Concurrent access control
|
└── locks/ # Concurrent access control
|
||||||
```
|
```
|
||||||
|
|
||||||
### HNSW Index
|
### Vector Index
|
||||||
Hierarchical Navigable Small World index for efficient vector search:
|
Pluggable vector index (`VectorIndexProvider`) for efficient nearest-neighbor search. The default JS implementation, `JsHnswVectorIndex`, uses a hierarchical graph:
|
||||||
- **Performance**: O(log n) search complexity
|
- **Performance**: O(log n) search complexity
|
||||||
- **Memory Efficient**: Product quantization support
|
- **Configurable recall**: `fast` / `balanced` / `accurate` presets trade recall for latency
|
||||||
- **Scalable**: Handles millions of vectors
|
- **Scalable**: Handles millions of vectors per process
|
||||||
- **Persistent**: Serializable to storage
|
- **Persistent**: Serializable to storage
|
||||||
|
- **Swappable**: Replace with a native implementation (such as `@soulcraft/cortex`) via the plugin system without changing application code
|
||||||
|
|
||||||
### Metadata Index Manager
|
### Metadata Index Manager
|
||||||
High-performance field indexing system:
|
High-performance field indexing system:
|
||||||
|
|
@ -59,7 +60,7 @@ High-performance field indexing system:
|
||||||
## Performance Characteristics
|
## Performance Characteristics
|
||||||
|
|
||||||
### Operation Complexity
|
### Operation Complexity
|
||||||
- **Vector Search**: O(log n) via HNSW
|
- **Vector Search**: O(log n) via the vector index
|
||||||
- **Field Filtering**: O(1) via inverted indexes
|
- **Field Filtering**: O(1) via inverted indexes
|
||||||
- **Graph Traversal**: O(V + E) for breadth-first search
|
- **Graph Traversal**: O(V + E) for breadth-first search
|
||||||
- **Add Operation**: O(log n) for index insertion
|
- **Add Operation**: O(log n) for index insertion
|
||||||
|
|
@ -83,7 +84,6 @@ Brainy's extensible plugin architecture allows for powerful enhancements:
|
||||||
### Core Augmentations
|
### Core Augmentations
|
||||||
- **Entity Registry**: High-speed deduplication for streaming data
|
- **Entity Registry**: High-speed deduplication for streaming data
|
||||||
- **Batch Processing**: Optimized bulk operations
|
- **Batch Processing**: Optimized bulk operations
|
||||||
- **Connection Pool**: Efficient resource management
|
|
||||||
- **Request Deduplicator**: Prevents duplicate processing
|
- **Request Deduplicator**: Prevents duplicate processing
|
||||||
|
|
||||||
### Creating Custom Augmentations
|
### Creating Custom Augmentations
|
||||||
|
|
@ -111,7 +111,7 @@ Multi-layered caching for optimal performance:
|
||||||
## Integration Points
|
## Integration Points
|
||||||
|
|
||||||
### Key Objects for Extensions
|
### Key Objects for Extensions
|
||||||
- `brain.index`: Access HNSW vector index
|
- `brain.index`: Access the vector index
|
||||||
- `brain.metadataIndex`: Access field indexing
|
- `brain.metadataIndex`: Access field indexing
|
||||||
- `brain.graphIndex`: Access graph adjacency index
|
- `brain.graphIndex`: Access graph adjacency index
|
||||||
- `brain.storage`: Access storage layer
|
- `brain.storage`: Access storage layer
|
||||||
|
|
|
||||||
|
|
@ -1,46 +1,46 @@
|
||||||
# Storage Architecture
|
# Storage Architecture
|
||||||
|
|
||||||
> **Updated**: Metadata/vector separation, UUID-based sharding, lifecycle management
|
> **Updated**: Metadata/vector separation, UUID-based sharding, on-disk artifact for operator-layer backup
|
||||||
|
|
||||||
## Storage Structure
|
## Storage Structure
|
||||||
|
|
||||||
### Architecture: Metadata/Vector Separation
|
### Architecture: Metadata/Vector Separation
|
||||||
|
|
||||||
entities and relationships are split into **2 separate files** for optimal performance at billion-entity scale:
|
Entities and relationships are split into **2 separate files** for optimal performance at billion-entity scale:
|
||||||
|
|
||||||
```
|
```
|
||||||
brainy-data/
|
brainy-data/
|
||||||
├── _system/ # System metadata (not sharded)
|
├── _system/ # System metadata (not sharded)
|
||||||
│ ├── statistics.json # Performance metrics
|
│ ├── statistics.json # Performance metrics
|
||||||
│ ├── __metadata_field_index__*.json # Field indexes
|
│ ├── __metadata_field_index__*.json # Field indexes
|
||||||
│ └── __metadata_sorted_index__*.json # Sorted indexes
|
│ └── __metadata_sorted_index__*.json # Sorted indexes
|
||||||
│
|
│
|
||||||
├── entities/
|
├── entities/
|
||||||
│ ├── nouns/
|
│ ├── nouns/
|
||||||
│ │ ├── vectors/ # HNSW graph data (sharded by UUID)
|
│ │ ├── vectors/ # Vector graph data (sharded by UUID)
|
||||||
│ │ │ ├── 00/ # Shard 00 (first 2 hex digits)
|
│ │ │ ├── 00/ # Shard 00 (first 2 hex digits)
|
||||||
│ │ │ │ ├── 00123456-....json # Vector + HNSW connections
|
│ │ │ │ ├── 00123456-....json # Vector + graph connections
|
||||||
│ │ │ │ └── 00abcdef-....json
|
│ │ │ │ └── 00abcdef-....json
|
||||||
│ │ │ ├── 01/ ... ff/ # 256 shards total
|
│ │ │ ├── 01/ ... ff/ # 256 shards total
|
||||||
│ │ │
|
│ │ │
|
||||||
│ │ └── metadata/ # Business data (sharded by UUID)
|
│ │ └── metadata/ # Business data (sharded by UUID)
|
||||||
│ │ ├── 00/
|
│ │ ├── 00/
|
||||||
│ │ │ ├── 00123456-....json # Entity metadata only
|
│ │ │ ├── 00123456-....json # Entity metadata only
|
||||||
│ │ │ └── 00abcdef-....json
|
│ │ │ └── 00abcdef-....json
|
||||||
│ │ ├── 01/ ... ff/
|
│ │ ├── 01/ ... ff/
|
||||||
│ │
|
│ │
|
||||||
│ └── verbs/
|
│ └── verbs/
|
||||||
│ ├── vectors/ # Relationship vectors (sharded)
|
│ ├── vectors/ # Relationship vectors (sharded)
|
||||||
│ │ ├── 00/ ... ff/
|
│ │ ├── 00/ ... ff/
|
||||||
│ │
|
│ │
|
||||||
│ └── metadata/ # Relationship data (sharded)
|
│ └── metadata/ # Relationship data (sharded)
|
||||||
│ ├── 00/ ... ff/
|
│ ├── 00/ ... ff/
|
||||||
```
|
```
|
||||||
|
|
||||||
### Why Split Metadata and Vectors?
|
### Why Split Metadata and Vectors?
|
||||||
|
|
||||||
**Performance at scale:**
|
**Performance at scale:**
|
||||||
- **HNSW operations**: Only load vectors (4KB) during search, not metadata (2-10KB)
|
- **Vector search operations**: Only load vectors (4KB) during search, not metadata (2-10KB)
|
||||||
- **Filtering**: Only load metadata during filtering, not vectors
|
- **Filtering**: Only load metadata during filtering, not vectors
|
||||||
- **Pagination**: Load metadata IDs first, fetch vectors/metadata on-demand
|
- **Pagination**: Load metadata IDs first, fetch vectors/metadata on-demand
|
||||||
- **Result**: 60-70% reduction in I/O for typical queries at million-entity scale
|
- **Result**: 60-70% reduction in I/O for typical queries at million-entity scale
|
||||||
|
|
@ -52,115 +52,69 @@ brainy-data/
|
||||||
const uuid = "3fa85f64-5717-4562-b3fc-2c963f66afa6"
|
const uuid = "3fa85f64-5717-4562-b3fc-2c963f66afa6"
|
||||||
const shard = uuid.substring(0, 2) // "3f"
|
const shard = uuid.substring(0, 2) // "3f"
|
||||||
|
|
||||||
// Vector path: entities/nouns/vectors/3f/3fa85f64-....json
|
// Vector path: entities/nouns/vectors/3f/3fa85f64-....json
|
||||||
// Metadata path: entities/nouns/metadata/3f/3fa85f64-....json
|
// Metadata path: entities/nouns/metadata/3f/3fa85f64-....json
|
||||||
```
|
```
|
||||||
|
|
||||||
**Benefits:**
|
**Benefits:**
|
||||||
- **Uniform distribution**: ~3,900 entities per shard (at 1M scale)
|
- **Uniform distribution**: ~3,900 entities per shard (at 1M scale)
|
||||||
- **Cloud storage optimization**: 200x faster than unsharded (30s → 150ms)
|
- **Filesystem optimization**: avoids huge flat directories that bog down `readdir`
|
||||||
- **Parallel operations**: Load 256 shards in parallel
|
- **Parallel operations**: walk 256 shards in parallel
|
||||||
- **Predictable**: Deterministic shard assignment
|
- **Predictable**: Deterministic shard assignment
|
||||||
|
|
||||||
## Storage Adapters
|
## Storage Adapters
|
||||||
|
|
||||||
Brainy provides multiple storage adapters with identical APIs and production features:
|
Brainy 8.0 ships two adapters, both implementing the same `StorageAdapter` interface:
|
||||||
|
|
||||||
### FileSystem Storage (Node.js)
|
### FileSystem Storage (Node.js, default)
|
||||||
```typescript
|
```typescript
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: {
|
storage: {
|
||||||
type: 'filesystem',
|
type: 'filesystem',
|
||||||
path: './data',
|
path: './data'
|
||||||
compression: true // Gzip compression (60-80% space savings)
|
}
|
||||||
}
|
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
- **Use case**: Server applications, CLI tools
|
- **Use case**: Server applications, CLI tools, single-node deployments
|
||||||
- **Performance**: Direct file I/O with optional compression
|
- **Performance**: Direct file I/O
|
||||||
- **Persistence**: Permanent on disk
|
- **Persistence**: Permanent on disk
|
||||||
- **Features**:
|
- **Features**:
|
||||||
- **Gzip Compression**: 60-80% storage savings with minimal CPU overhead
|
- **Batch Delete**: Efficient bulk deletion with retries
|
||||||
- **Batch Delete**: Efficient bulk deletion with retries
|
- **UUID Sharding**: Automatic 256-shard distribution
|
||||||
- **UUID Sharding**: Automatic 256-shard distribution
|
|
||||||
|
|
||||||
### S3 Compatible Storage (AWS, MinIO, R2)
|
### Memory Storage
|
||||||
```typescript
|
```typescript
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: {
|
storage: {
|
||||||
type: 's3',
|
type: 'memory'
|
||||||
bucket: 'my-brainy-data',
|
}
|
||||||
region: 'us-east-1',
|
|
||||||
credentials: {
|
|
||||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
|
||||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
- **Use case**: Distributed applications, cloud deployments
|
- **Use case**: Tests, ephemeral workloads, single-process caches
|
||||||
- **Performance**: Network dependent, with intelligent caching
|
- **Performance**: No I/O — all data lives in process memory
|
||||||
- **Persistence**: Cloud storage durability (99.999999999%)
|
- **Persistence**: None — data is lost when the process exits
|
||||||
- **Features**:
|
|
||||||
- **Lifecycle Policies**: Automatic tier transitions (Standard → IA → Glacier → Deep Archive)
|
|
||||||
- **Intelligent-Tiering**: Automatic optimization based on access patterns (up to 95% savings)
|
|
||||||
- **Batch Delete**: Efficient bulk deletion (1000 objects per request)
|
|
||||||
- **Cost Impact**: $138k/year → $5.9k/year at 500TB (96% savings!)
|
|
||||||
|
|
||||||
### Google Cloud Storage (GCS)
|
### Auto
|
||||||
```typescript
|
```typescript
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: {
|
storage: {
|
||||||
type: 'gcs',
|
type: 'auto',
|
||||||
bucketName: 'my-brainy-data',
|
path: './data'
|
||||||
keyFilename: './service-account.json' // Or use ADC
|
}
|
||||||
}
|
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
- **Use case**: Google Cloud deployments
|
`'auto'` picks `'filesystem'` when running on Node.js with a writable `path`, and falls back to `'memory'` otherwise.
|
||||||
- **Performance**: Global CDN with edge caching
|
|
||||||
- **Persistence**: 99.999999999% durability
|
|
||||||
- **Features**:
|
|
||||||
- **Lifecycle Policies**: Automatic tier transitions (Standard → Nearline → Coldline → Archive)
|
|
||||||
- **Autoclass**: Intelligent automatic tier optimization
|
|
||||||
- **Batch Delete**: Efficient bulk operations
|
|
||||||
- **Cost Impact**: $138k/year → $8.3k/year at 500TB (94% savings!)
|
|
||||||
|
|
||||||
### Azure Blob Storage
|
## Backup and Off-Site Replication
|
||||||
```typescript
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 'azure',
|
|
||||||
connectionString: process.env.AZURE_STORAGE_CONNECTION_STRING,
|
|
||||||
containerName: 'brainy-data'
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
- **Use case**: Azure cloud deployments
|
|
||||||
- **Performance**: Global replication with CDN
|
|
||||||
- **Persistence**: LRS, ZRS, GRS, RA-GRS options
|
|
||||||
- **Features**:
|
|
||||||
- **Blob Tier Management**: Hot/Cool/Archive tiers (99% cost savings)
|
|
||||||
- **Lifecycle Policies**: Automatic tier transitions and deletions
|
|
||||||
- **Batch Delete**: BlobBatchClient for efficient bulk operations
|
|
||||||
- **Batch Tier Changes**: Move thousands of blobs efficiently
|
|
||||||
- **Archive Rehydration**: Smart rehydration with priority options
|
|
||||||
|
|
||||||
### Origin Private File System (Browser)
|
Brainy 8.0 does not embed cloud SDKs. The on-disk artifact at `path` is a plain directory tree of JSON files, so backup is an operator-layer concern. Typical patterns:
|
||||||
```typescript
|
|
||||||
const brain = new Brainy({
|
- `gsutil rsync -r ./data gs://my-bucket/brainy-data`
|
||||||
storage: {
|
- `aws s3 sync ./data s3://my-bucket/brainy-data`
|
||||||
type: 'opfs'
|
- `rclone sync ./data remote:brainy-data`
|
||||||
}
|
- Periodic `tar` snapshots to any object store
|
||||||
})
|
|
||||||
```
|
Run these from your scheduler (cron, systemd timer, k8s CronJob) — Brainy itself only reads and writes the local directory.
|
||||||
- **Use case**: Browser applications, PWAs
|
|
||||||
- **Performance**: Near-native file system speed
|
|
||||||
- **Persistence**: Permanent in browser (with quota limits)
|
|
||||||
- **Features**:
|
|
||||||
- **Quota Monitoring**: Real-time quota tracking and warnings
|
|
||||||
- **Batch Delete**: Efficient bulk deletion
|
|
||||||
- **Storage Status**: Detailed usage/available reporting
|
|
||||||
|
|
||||||
## Metadata Indexing System
|
## Metadata Indexing System
|
||||||
|
|
||||||
|
|
@ -170,12 +124,12 @@ Tracks all unique values for each field:
|
||||||
```json
|
```json
|
||||||
// __metadata_field_index__field_category.json
|
// __metadata_field_index__field_category.json
|
||||||
{
|
{
|
||||||
"values": {
|
"values": {
|
||||||
"technology": 45,
|
"technology": 45,
|
||||||
"science": 32,
|
"science": 32,
|
||||||
"business": 28
|
"business": 28
|
||||||
},
|
},
|
||||||
"lastUpdated": 1699564234567
|
"lastUpdated": 1699564234567
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -185,11 +139,11 @@ Maps field+value combinations to entity IDs:
|
||||||
```json
|
```json
|
||||||
// __metadata_index__category_technology_chunk0.json
|
// __metadata_index__category_technology_chunk0.json
|
||||||
{
|
{
|
||||||
"field": "category",
|
"field": "category",
|
||||||
"value": "technology",
|
"value": "technology",
|
||||||
"ids": ["uuid1", "uuid2", "uuid3", ...],
|
"ids": ["uuid1", "uuid2", "uuid3", ...],
|
||||||
"chunk": 0,
|
"chunk": 0,
|
||||||
"total": 45
|
"total": 45
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -207,14 +161,14 @@ High-performance deduplication system for streaming data:
|
||||||
```json
|
```json
|
||||||
// __entity_registry__.json
|
// __entity_registry__.json
|
||||||
{
|
{
|
||||||
"mappings": {
|
"mappings": {
|
||||||
"did:plc:alice123": "550e8400-e29b-41d4-a716-446655440000",
|
"did:plc:alice123": "550e8400-e29b-41d4-a716-446655440000",
|
||||||
"handle:alice.bsky.social": "550e8400-e29b-41d4-a716-446655440000"
|
"handle:alice.bsky.social": "550e8400-e29b-41d4-a716-446655440000"
|
||||||
},
|
},
|
||||||
"stats": {
|
"stats": {
|
||||||
"totalMappings": 10000,
|
"totalMappings": 10000,
|
||||||
"lastSync": 1699564234567
|
"lastSync": 1699564234567
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -224,216 +178,46 @@ High-performance deduplication system for streaming data:
|
||||||
- **Cache**: LRU with configurable TTL
|
- **Cache**: LRU with configurable TTL
|
||||||
- **Sync**: Periodic or on-demand
|
- **Sync**: Periodic or on-demand
|
||||||
|
|
||||||
|
## Durability
|
||||||
|
|
||||||
Ensures durability and enables recovery:
|
Brainy persists writes to disk through the filesystem adapter. Each save is a rename-based atomic write of a JSON file under the appropriate shard. Operators that need point-in-time recovery should snapshot `path` (see [Backup and Off-Site Replication](#backup-and-off-site-replication)).
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"timestamp": 1699564234567,
|
|
||||||
"operation": "add",
|
|
||||||
"data": {
|
|
||||||
"id": "550e8400-e29b-41d4-a716-446655440000",
|
|
||||||
"content": "...",
|
|
||||||
"metadata": {}
|
|
||||||
},
|
|
||||||
"checksum": "sha256:..."
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Recovery Process
|
|
||||||
2. Replay operations from last checkpoint
|
|
||||||
3. Verify checksums for integrity
|
|
||||||
|
|
||||||
## Storage Optimization
|
## Storage Optimization
|
||||||
|
|
||||||
### 1. Lifecycle Policies (Cloud Storage)
|
### 1. Batch Operations
|
||||||
|
|
||||||
**Automatic cost optimization through tier transitions:**
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// S3: Set lifecycle policy for automatic archival
|
|
||||||
await storage.setLifecyclePolicy({
|
|
||||||
rules: [{
|
|
||||||
id: 'archive-old-data',
|
|
||||||
prefix: 'entities/',
|
|
||||||
status: 'Enabled',
|
|
||||||
transitions: [
|
|
||||||
{ days: 30, storageClass: 'STANDARD_IA' }, // Move to IA after 30 days
|
|
||||||
{ days: 90, storageClass: 'GLACIER' }, // Archive after 90 days
|
|
||||||
{ days: 365, storageClass: 'DEEP_ARCHIVE' } // Deep archive after 1 year
|
|
||||||
]
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
|
|
||||||
// GCS: Set lifecycle policy
|
|
||||||
await storage.setLifecyclePolicy({
|
|
||||||
rules: [{
|
|
||||||
condition: { age: 30 },
|
|
||||||
action: { type: 'SetStorageClass', storageClass: 'NEARLINE' }
|
|
||||||
}, {
|
|
||||||
condition: { age: 90 },
|
|
||||||
action: { type: 'SetStorageClass', storageClass: 'COLDLINE' }
|
|
||||||
}, {
|
|
||||||
condition: { age: 365 },
|
|
||||||
action: { type: 'SetStorageClass', storageClass: 'ARCHIVE' }
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
|
|
||||||
// Azure: Set lifecycle policy
|
|
||||||
await storage.setLifecyclePolicy({
|
|
||||||
rules: [{
|
|
||||||
name: 'archiveOldData',
|
|
||||||
enabled: true,
|
|
||||||
type: 'Lifecycle',
|
|
||||||
definition: {
|
|
||||||
filters: { blobTypes: ['blockBlob'] },
|
|
||||||
actions: {
|
|
||||||
baseBlob: {
|
|
||||||
tierToCool: { daysAfterModificationGreaterThan: 30 },
|
|
||||||
tierToArchive: { daysAfterModificationGreaterThan: 90 }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
**Cost Impact (500TB dataset):**
|
|
||||||
| Storage | Before | After | Savings |
|
|
||||||
|---------|--------|-------|---------|
|
|
||||||
| **AWS S3** | $138,000/yr | $5,940/yr | **96%** |
|
|
||||||
| **GCS** | $138,000/yr | $8,300/yr | **94%** |
|
|
||||||
| **Azure** | $107,520/yr | $5,016/yr | **95%** |
|
|
||||||
|
|
||||||
### 2. Intelligent-Tiering (S3)
|
|
||||||
|
|
||||||
**Automatic optimization without retrieval fees:**
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Enable S3 Intelligent-Tiering
|
|
||||||
await storage.enableIntelligentTiering('entities/', 'auto-optimize')
|
|
||||||
|
|
||||||
// Benefits:
|
|
||||||
// - Automatic tier transitions based on access patterns
|
|
||||||
// - No retrieval fees (unlike Glacier)
|
|
||||||
// - Up to 95% cost savings
|
|
||||||
// - No performance impact on frequently accessed data
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Autoclass (GCS)
|
|
||||||
|
|
||||||
**Google Cloud's intelligent automatic optimization:**
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Enable GCS Autoclass
|
|
||||||
await storage.enableAutoclass({
|
|
||||||
terminalStorageClass: 'ARCHIVE' // Optional: Set lowest tier
|
|
||||||
})
|
|
||||||
|
|
||||||
// Benefits:
|
|
||||||
// - Automatic optimization based on access patterns
|
|
||||||
// - No data retrieval delays
|
|
||||||
// - Transparent tier transitions
|
|
||||||
// - Up to 94% cost savings
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4. Compression (FileSystem)
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Enable gzip compression for local storage
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 'filesystem',
|
|
||||||
path: './data',
|
|
||||||
compression: true // 60-80% space savings
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Performance impact:
|
|
||||||
// - Write: +10-20ms per file (gzip compression)
|
|
||||||
// - Read: +5-10ms per file (gzip decompression)
|
|
||||||
// - Space savings: 60-80% for typical JSON data
|
|
||||||
// - CPU overhead: Minimal (~5% CPU)
|
|
||||||
```
|
|
||||||
|
|
||||||
### 5. Batch Operations
|
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Efficient batch delete
|
// Efficient batch delete
|
||||||
await storage.batchDelete([
|
await storage.batchDelete([
|
||||||
'entities/nouns/vectors/00/00123456-....json',
|
'entities/nouns/vectors/00/00123456-....json',
|
||||||
'entities/nouns/metadata/00/00123456-....json',
|
'entities/nouns/metadata/00/00123456-....json'
|
||||||
// ... up to 1000 objects
|
// ...
|
||||||
])
|
])
|
||||||
|
|
||||||
// Benefits:
|
|
||||||
// - S3: 1000 objects per request (vs 1 per request)
|
|
||||||
// - GCS: 100 objects per request
|
|
||||||
// - Azure: 256 objects per batch
|
|
||||||
// - Automatic retry logic with exponential backoff
|
|
||||||
// - Throttling protection
|
|
||||||
|
|
||||||
// Batch writes for performance
|
// Batch writes for performance
|
||||||
await brain.addBatch([
|
await brain.addBatch([
|
||||||
{ content: "item1", metadata: {} },
|
{ content: "item1", metadata: {} },
|
||||||
{ content: "item2", metadata: {} },
|
{ content: "item2", metadata: {} },
|
||||||
{ content: "item3", metadata: {} }
|
{ content: "item3", metadata: {} }
|
||||||
])
|
])
|
||||||
// Single transaction, optimized I/O
|
// Single transaction, optimized I/O
|
||||||
```
|
```
|
||||||
|
|
||||||
### 6. Quota Monitoring (OPFS)
|
### 2. Caching Strategy
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Get quota status for browser storage
|
// Configure caching
|
||||||
const status = await storage.getStorageStatus()
|
|
||||||
|
|
||||||
console.log(status)
|
|
||||||
// {
|
|
||||||
// type: 'opfs',
|
|
||||||
// available: true,
|
|
||||||
// details: {
|
|
||||||
// usage: 45829120, // 43.7 MB used
|
|
||||||
// quota: 536870912, // 512 MB available
|
|
||||||
// usagePercent: 8.5,
|
|
||||||
// quotaExceeded: false
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// Proactive quota management:
|
|
||||||
// - Monitor usage before writes
|
|
||||||
// - Warn users when approaching quota
|
|
||||||
// - Automatically clean up old data
|
|
||||||
```
|
|
||||||
|
|
||||||
### 7. Tier Management (Azure)
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Change blob tier for cost optimization
|
|
||||||
await storage.changeBlobTier(blobPath, 'Cool') // Hot → Cool (50% savings)
|
|
||||||
await storage.changeBlobTier(blobPath, 'Archive') // Cool → Archive (99% savings)
|
|
||||||
|
|
||||||
// Batch tier changes (efficient)
|
|
||||||
await storage.batchChangeTier([blob1, blob2, blob3], 'Cool')
|
|
||||||
|
|
||||||
// Rehydrate from Archive when needed
|
|
||||||
await storage.rehydrateBlob(blobPath, 'Standard') // Standard or High priority
|
|
||||||
```
|
|
||||||
|
|
||||||
### 8. Caching Strategy
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Configure caching per storage type
|
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: {
|
storage: {
|
||||||
type: 'filesystem',
|
type: 'filesystem',
|
||||||
cache: {
|
path: './data',
|
||||||
enabled: true,
|
cache: {
|
||||||
maxSize: 1000, // Maximum cached items
|
enabled: true,
|
||||||
ttl: 300000, // 5 minutes
|
maxSize: 1000, // Maximum cached items
|
||||||
strategy: 'lru' // Least recently used
|
ttl: 300000, // 5 minutes
|
||||||
}
|
strategy: 'lru' // Least recently used
|
||||||
}
|
}
|
||||||
|
}
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -443,8 +227,8 @@ const brain = new Brainy({
|
||||||
```typescript
|
```typescript
|
||||||
// Automatic locking for write operations
|
// Automatic locking for write operations
|
||||||
await brain.storage.withLock('resource-id', async () => {
|
await brain.storage.withLock('resource-id', async () => {
|
||||||
// Exclusive access to resource
|
// Exclusive access to resource
|
||||||
await brain.storage.saveNoun(id, data)
|
await brain.storage.saveNoun(id, data)
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -455,54 +239,39 @@ await brain.storage.withLock('resource-id', async () => {
|
||||||
|
|
||||||
## Migration and Backup
|
## Migration and Backup
|
||||||
|
|
||||||
### Export Data
|
Backup and restore go through the Db API — see
|
||||||
|
[Snapshots & Time Travel](../guides/snapshots-and-time-travel.md) for the full
|
||||||
|
recipe book.
|
||||||
|
|
||||||
|
### Snapshot (backup)
|
||||||
```typescript
|
```typescript
|
||||||
// Export the whole brain to a portable PortableGraph document
|
// Instant, self-contained snapshot (hard links on filesystem storage)
|
||||||
const graph = await brain.data().then(d => d.export(undefined, { includeVectors: true }))
|
const db = brain.now()
|
||||||
|
await db.persist('/backups/2026-06-11')
|
||||||
|
await db.release()
|
||||||
```
|
```
|
||||||
|
|
||||||
### Import Data
|
### Restore
|
||||||
```typescript
|
```typescript
|
||||||
// Restore a PortableGraph document (dedup-by-id merge by default)
|
// Replace the store's entire state from a snapshot (destructive — confirm required)
|
||||||
await brain.data().then(d => d.import(graph, { onConflict: 'merge' }))
|
await brain.restore('/backups/2026-06-11', { confirm: true })
|
||||||
```
|
```
|
||||||
|
|
||||||
See the [Export & Import guide](../guides/export-and-import.md) for partial exports
|
### Move to a new directory
|
||||||
(by id, collection, connected neighbourhood, VFS subtree, or predicate).
|
|
||||||
|
|
||||||
### Storage Migration
|
|
||||||
```typescript
|
```typescript
|
||||||
// Migrate between storage types
|
// A snapshot directory is a complete store: restore it into a fresh brain
|
||||||
const oldBrain = new Brainy({ storage: { type: 'filesystem' } })
|
const brain = new Brainy({ storage: { type: 'filesystem', path: './new' } })
|
||||||
const newBrain = new Brainy({ storage: { type: 's3' } })
|
await brain.init()
|
||||||
|
await brain.restore('/backups/2026-06-11', { confirm: true })
|
||||||
await oldBrain.init()
|
|
||||||
await newBrain.init()
|
|
||||||
|
|
||||||
// Transfer all data via a portable graph
|
|
||||||
const data = await oldBrain.data().then(d => d.export(undefined, { includeVectors: true }))
|
|
||||||
await newBrain.data().then(d => d.import(data))
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Performance Tuning
|
## Performance Tuning
|
||||||
|
|
||||||
### Storage-Specific Optimizations
|
### FileSystem Optimizations
|
||||||
|
- **Directory sharding**: 256 shards spread files across subdirectories
|
||||||
#### FileSystem
|
|
||||||
- **Directory sharding**: Split files across subdirectories
|
|
||||||
- **Async I/O**: Non-blocking file operations
|
- **Async I/O**: Non-blocking file operations
|
||||||
- **Buffer pooling**: Reuse buffers for efficiency
|
- **Buffer pooling**: Reuse buffers for efficiency
|
||||||
|
|
||||||
#### S3
|
|
||||||
- **Multipart uploads**: For large objects
|
|
||||||
- **Request batching**: Combine small operations
|
|
||||||
- **CDN integration**: Edge caching for reads
|
|
||||||
|
|
||||||
#### OPFS
|
|
||||||
- **Quota management**: Monitor and request increases
|
|
||||||
- **Worker offloading**: Heavy operations in workers
|
|
||||||
- **Transaction batching**: Group operations
|
|
||||||
|
|
||||||
### Monitoring
|
### Monitoring
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
|
|
@ -510,58 +279,34 @@ await newBrain.data().then(d => d.import(data))
|
||||||
const stats = await brain.storage.getStatistics()
|
const stats = await brain.storage.getStatistics()
|
||||||
console.log(stats)
|
console.log(stats)
|
||||||
// {
|
// {
|
||||||
// totalSize: 1048576,
|
// totalSize: 1048576,
|
||||||
// entityCount: 1000,
|
// entityCount: 1000,
|
||||||
// indexSize: 204800,
|
// indexSize: 204800,
|
||||||
// walSize: 10240,
|
// walSize: 10240,
|
||||||
// cacheHitRate: 0.85
|
// cacheHitRate: 0.85
|
||||||
// }
|
// }
|
||||||
```
|
```
|
||||||
|
|
||||||
## Best Practices
|
## Best Practices
|
||||||
|
|
||||||
### Choose the Right Adapter
|
### Choose the Right Adapter
|
||||||
1. **Development**: FileSystem with compression (local persistence, small storage footprint)
|
1. **Development & tests**: `memory` for speed, `filesystem` when you need persistence
|
||||||
2. **Production Server**: FileSystem with compression or cloud storage with lifecycle policies
|
2. **Single-process production**: `filesystem` with off-site backup via `gsutil` / `aws s3 sync` / `rclone`
|
||||||
3. **Browser Apps**: OPFS with quota monitoring
|
3. **Horizontal scaling**: Brainy runs in one process — there is no built-in cluster. Run independent instances behind a service layer and replicate the on-disk artifact with your operator tooling; or run many reader processes against one shared store with a single writer
|
||||||
4. **Distributed**: S3/GCS/Azure with Intelligent-Tiering/Autoclass
|
|
||||||
|
|
||||||
### Optimize for Your Use Case
|
### Optimize for Your Use Case
|
||||||
1. **Read-heavy**: Enable aggressive caching + cloud CDN
|
1. **Read-heavy**: Enable caching and let the OS page cache do its job
|
||||||
2. **Write-heavy**: Batch operations + async writes
|
2. **Write-heavy**: Batch operations and tune the cache `maxSize`
|
||||||
3. **Real-time**: FileSystem with periodic snapshots
|
3. **Real-time**: FileSystem with periodic snapshots
|
||||||
4. **Archival**: Cloud storage with lifecycle policies (96% cost savings!)
|
4. **Archival**: Snapshot `path` to cold object storage on a schedule
|
||||||
5. **Large-scale**: Metadata/vector separation + UUID sharding + lifecycle policies
|
5. **Large-scale**: Rely on metadata/vector separation + UUID sharding
|
||||||
|
|
||||||
### Cost Optimization
|
|
||||||
1. **Enable lifecycle policies** for cloud storage (automated cost reduction)
|
|
||||||
2. **Use Intelligent-Tiering (S3)** or Autoclass (GCS) for automatic optimization
|
|
||||||
3. **Enable compression** for FileSystem storage (60-80% space savings)
|
|
||||||
4. **Monitor quota** for OPFS (prevent quota exceeded errors)
|
|
||||||
5. **Use batch operations** for bulk deletions (efficient API usage)
|
|
||||||
6. **Consider tier management** for Azure (Hot/Cool/Archive tiers)
|
|
||||||
|
|
||||||
**Example Cost Savings (500TB dataset):**
|
|
||||||
- Without lifecycle policies: **$138,000/year**
|
|
||||||
- With lifecycle policies: **$5,940/year**
|
|
||||||
- **Savings: $132,060/year (96%)**
|
|
||||||
|
|
||||||
### Monitor and Maintain
|
### Monitor and Maintain
|
||||||
1. Regular statistics collection
|
1. Regular statistics collection
|
||||||
2. Monitor lifecycle policy effectiveness
|
2. Watch disk usage and shard balance
|
||||||
3. Index optimization
|
3. Index optimization
|
||||||
4. Cache tuning based on hit rates
|
4. Cache tuning based on hit rates
|
||||||
5. Track storage costs and tier distribution
|
5. Verify backup runs (test restore quarterly)
|
||||||
6. Review quota usage (OPFS) and storage growth patterns
|
|
||||||
|
|
||||||
### Production Deployment Checklist
|
|
||||||
- ✅ Enable lifecycle policies on cloud storage
|
|
||||||
- ✅ Configure batch delete for cleanup operations
|
|
||||||
- ✅ Enable compression for FileSystem storage
|
|
||||||
- ✅ Set up quota monitoring for OPFS
|
|
||||||
- ✅ Configure appropriate tier transitions
|
|
||||||
- ✅ Enable Intelligent-Tiering (S3) or Autoclass (GCS)
|
|
||||||
- ✅ Monitor storage costs and optimize regularly
|
|
||||||
|
|
||||||
## API Reference
|
## API Reference
|
||||||
|
|
||||||
|
|
@ -569,6 +314,5 @@ See the [Storage API](../api/storage.md) for complete method documentation.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
**Version**: 4.0.0
|
**Last Updated**: 2026
|
||||||
**Last Updated**: 2025-10-17
|
**Key Features**: Metadata/vector separation, UUID sharding, filesystem-and-memory adapters, operator-layer backup
|
||||||
**Key Features**: Metadata/vector separation, UUID sharding, lifecycle management, tier optimization
|
|
||||||
|
|
|
||||||
|
|
@ -23,29 +23,37 @@ Traditional databases force you to choose between vector search, graph traversal
|
||||||
|
|
||||||
### Unified Query Structure
|
### Unified Query Structure
|
||||||
|
|
||||||
```typescript
|
`find()` accepts a single `FindParams` object (or a natural-language string). One
|
||||||
interface TripleQuery {
|
object combines all three intelligences:
|
||||||
// Vector/Semantic search
|
|
||||||
like?: string | Vector | any
|
|
||||||
similar?: string | Vector | any
|
|
||||||
|
|
||||||
// Graph/Relationship search
|
```typescript
|
||||||
|
interface FindParams {
|
||||||
|
// Vector intelligence — semantic similarity
|
||||||
|
query?: string // Natural-language / semantic query (embedded, matched via HNSW + text index)
|
||||||
|
vector?: number[] // Pre-computed embedding for direct vector search
|
||||||
|
|
||||||
|
// Metadata intelligence — structured field filters
|
||||||
|
type?: NounType | NounType[] // Filter by entity type
|
||||||
|
subtype?: string | string[] // Filter by per-product subtype
|
||||||
|
where?: Record<string, any> // Field predicates with bare operators (gte, lt, in, contains, exists…)
|
||||||
|
|
||||||
|
// Graph intelligence — relationship traversal
|
||||||
connected?: {
|
connected?: {
|
||||||
to?: string | string[]
|
to?: string // Reachable to this entity
|
||||||
from?: string | string[]
|
from?: string // Reachable from this entity
|
||||||
type?: string | string[]
|
via?: VerbType | VerbType[] // Relationship type(s) to traverse (alias: type)
|
||||||
depth?: number
|
depth?: number // Max traversal depth (default: 1)
|
||||||
direction?: 'in' | 'out' | 'both'
|
direction?: 'in' | 'out' | 'both'
|
||||||
}
|
}
|
||||||
|
|
||||||
// Field/Attribute search
|
// Proximity — nearest neighbours of a known entity
|
||||||
where?: Record<string, any>
|
near?: { id: string; threshold?: number }
|
||||||
|
|
||||||
// Advanced options
|
// Control
|
||||||
limit?: number
|
limit?: number // Max results (default: 10)
|
||||||
boost?: 'recent' | 'popular' | 'verified' | string
|
offset?: number // Skip N results
|
||||||
explain?: boolean
|
orderBy?: string // Field to sort by (e.g. 'createdAt')
|
||||||
threshold?: number
|
order?: 'asc' | 'desc' // Sort direction
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -68,16 +76,16 @@ const articles = await brain.find("verified articles by John Smith about machine
|
||||||
|
|
||||||
#### Simple Vector Search
|
#### Simple Vector Search
|
||||||
```typescript
|
```typescript
|
||||||
const results = await brain.search("machine learning concepts")
|
const results = await brain.find("machine learning concepts")
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Combined Intelligence Query
|
#### Combined Intelligence Query
|
||||||
```typescript
|
```typescript
|
||||||
const results = await brain.find({
|
const results = await brain.find({
|
||||||
like: "neural networks",
|
query: "neural networks",
|
||||||
where: {
|
where: {
|
||||||
category: "research",
|
category: "research",
|
||||||
year: { $gte: 2023 }
|
year: { gte: 2023 }
|
||||||
},
|
},
|
||||||
connected: {
|
connected: {
|
||||||
to: "deep-learning-team",
|
to: "deep-learning-team",
|
||||||
|
|
@ -109,8 +117,8 @@ All three search types execute simultaneously:
|
||||||
```typescript
|
```typescript
|
||||||
// Parallel execution for balanced query
|
// Parallel execution for balanced query
|
||||||
const results = await brain.find({
|
const results = await brain.find({
|
||||||
like: "AI research", // ~1000 potential matches
|
query: "AI research", // ~1000 potential matches
|
||||||
where: { type: "paper" }, // ~500 potential matches
|
where: { kind: "paper" }, // ~500 potential matches
|
||||||
connected: { to: "stanford" } // ~200 potential matches
|
connected: { to: "stanford" } // ~200 potential matches
|
||||||
})
|
})
|
||||||
// All three execute in parallel, results fused
|
// All three execute in parallel, results fused
|
||||||
|
|
@ -126,7 +134,7 @@ Operations chain for maximum efficiency:
|
||||||
// Progressive execution for selective query
|
// Progressive execution for selective query
|
||||||
const results = await brain.find({
|
const results = await brain.find({
|
||||||
where: { userId: "user123" }, // Very selective (1-10 matches)
|
where: { userId: "user123" }, // Very selective (1-10 matches)
|
||||||
like: "recent posts", // Applied to filtered set
|
query: "recent posts", // Applied to filtered set
|
||||||
limit: 5
|
limit: 5
|
||||||
})
|
})
|
||||||
// Metadata filter first, then vector search on results
|
// Metadata filter first, then vector search on results
|
||||||
|
|
@ -161,15 +169,15 @@ Brainy includes 220+ embedded patterns for natural language understanding:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Natural language automatically parsed
|
// Natural language automatically parsed
|
||||||
const results = await brain.search(
|
const results = await brain.find(
|
||||||
"show me recent AI papers from Stanford published this year"
|
"show me recent AI papers from Stanford published this year"
|
||||||
)
|
)
|
||||||
// Automatically converts to:
|
// Automatically converts to:
|
||||||
// {
|
// {
|
||||||
// like: "AI papers",
|
// query: "AI papers",
|
||||||
// where: {
|
// where: {
|
||||||
// institution: "Stanford",
|
// institution: "Stanford",
|
||||||
// published: { $gte: "2024-01-01" }
|
// published: { gte: "2024-01-01" }
|
||||||
// }
|
// }
|
||||||
// }
|
// }
|
||||||
```
|
```
|
||||||
|
|
@ -188,11 +196,11 @@ The NLP processor identifies query intent:
|
||||||
|
|
||||||
Successful execution plans are cached:
|
Successful execution plans are cached:
|
||||||
```typescript
|
```typescript
|
||||||
// First query: 50ms (plan generation + execution)
|
// First call parses the natural-language query and builds an execution plan
|
||||||
await brain.search("machine learning papers")
|
await brain.find("machine learning papers")
|
||||||
|
|
||||||
// Subsequent similar queries: 10ms (cached plan)
|
// A structurally similar query reuses that plan, skipping plan generation
|
||||||
await brain.search("deep learning papers")
|
await brain.find("deep learning papers")
|
||||||
```
|
```
|
||||||
|
|
||||||
### Self-Optimization
|
### Self-Optimization
|
||||||
|
|
@ -213,50 +221,45 @@ Triple Intelligence leverages all available indexes:
|
||||||
|
|
||||||
### Explain Mode
|
### Explain Mode
|
||||||
|
|
||||||
Understand how your query was executed:
|
Diagnose how a query's `where` fields map to the index. Run `brain.explain()`
|
||||||
|
first whenever `find()` returns surprising or empty results:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
const results = await brain.find({
|
const plan = await brain.explain({
|
||||||
like: "quantum computing",
|
query: "quantum computing",
|
||||||
where: { category: "research" },
|
where: { category: "research" }
|
||||||
explain: true
|
|
||||||
})
|
})
|
||||||
|
|
||||||
console.log(results[0].explanation)
|
console.log(plan.fieldPlan)
|
||||||
// {
|
// [
|
||||||
// plan: "field-first-progressive",
|
// { field: 'category', path: 'column-store', notes: '...' }
|
||||||
// timing: {
|
// ]
|
||||||
// fieldFilter: 2,
|
|
||||||
// vectorSearch: 8,
|
console.log(plan.warnings)
|
||||||
// fusion: 1
|
// e.g. ['Field "category" has no index entries. find() will return [] silently...']
|
||||||
// },
|
|
||||||
// selectivity: {
|
|
||||||
// field: 0.1,
|
|
||||||
// vector: 0.3
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Boosting
|
### Result Ordering
|
||||||
|
|
||||||
Apply custom ranking boosts:
|
Sort results by any stored field with `orderBy` / `order`:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
const results = await brain.find({
|
const results = await brain.find({
|
||||||
like: "news articles",
|
query: "news articles",
|
||||||
boost: 'recent', // Boost recent items
|
where: { verified: true },
|
||||||
where: { verified: true }
|
orderBy: 'createdAt', // Newest first
|
||||||
|
order: 'desc'
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
### Threshold Control
|
### Similarity Threshold
|
||||||
|
|
||||||
Set minimum similarity thresholds:
|
Find the nearest neighbours of a known entity and keep only close matches with
|
||||||
|
`near`:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
const results = await brain.find({
|
const results = await brain.find({
|
||||||
like: "exact match needed",
|
near: { id: anchorId, threshold: 0.9 }, // Only results >= 0.9 similarity
|
||||||
threshold: 0.9, // Only very similar results
|
|
||||||
limit: 10
|
limit: 10
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
@ -283,7 +286,7 @@ const results = await brain.find({
|
||||||
```typescript
|
```typescript
|
||||||
// Find similar content with constraints
|
// Find similar content with constraints
|
||||||
const results = await brain.find({
|
const results = await brain.find({
|
||||||
like: query,
|
query: searchText,
|
||||||
where: {
|
where: {
|
||||||
status: 'published',
|
status: 'published',
|
||||||
language: 'en'
|
language: 'en'
|
||||||
|
|
@ -298,7 +301,7 @@ const results = await brain.find({
|
||||||
connected: {
|
connected: {
|
||||||
to: itemId,
|
to: itemId,
|
||||||
depth: 2,
|
depth: 2,
|
||||||
type: 'similar'
|
via: VerbType.RelatedTo
|
||||||
},
|
},
|
||||||
limit: 20
|
limit: 20
|
||||||
})
|
})
|
||||||
|
|
@ -309,10 +312,11 @@ const results = await brain.find({
|
||||||
// Recent items matching criteria
|
// Recent items matching criteria
|
||||||
const results = await brain.find({
|
const results = await brain.find({
|
||||||
where: {
|
where: {
|
||||||
timestamp: { $gte: Date.now() - 86400000 }
|
timestamp: { gte: Date.now() - 86400000 }
|
||||||
},
|
},
|
||||||
like: "trending topics",
|
query: "trending topics",
|
||||||
boost: 'recent'
|
orderBy: 'timestamp',
|
||||||
|
order: 'desc'
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,11 +1,11 @@
|
||||||
---
|
---
|
||||||
title: Zero Configuration
|
title: Zero Configuration
|
||||||
slug: concepts/zero-config
|
slug: concepts/zero-config
|
||||||
public: true
|
public: false
|
||||||
category: concepts
|
category: concepts
|
||||||
template: concept
|
template: concept
|
||||||
order: 3
|
order: 3
|
||||||
description: Brainy auto-detects storage, initializes embeddings, and builds indexes — no configuration required. Works in Node.js, Bun, OPFS, and cloud environments.
|
description: Brainy auto-detects storage, initializes embeddings, and builds indexes — no configuration required. Works in Node.js and Bun (server-only since 8.0).
|
||||||
next:
|
next:
|
||||||
- getting-started/installation
|
- getting-started/installation
|
||||||
- guides/storage-adapters
|
- guides/storage-adapters
|
||||||
|
|
@ -13,770 +13,145 @@ next:
|
||||||
|
|
||||||
# Zero Configuration & Auto-Adaptation
|
# Zero Configuration & Auto-Adaptation
|
||||||
|
|
||||||
> **Current Status**: Basic zero-config is fully functional. Advanced auto-adaptation features are in development.
|
> **"Zero config by default, fully tunable when you need it."** Construct a
|
||||||
|
> `Brainy()` with no options and it picks sensible, environment-aware defaults.
|
||||||
|
> Every default below is overridable through the constructor — see the
|
||||||
|
> [API Reference](../api/README.md#configuration).
|
||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
Brainy is designed with **"Zero Config by Default, Infinite Tunability"** philosophy. It automatically detects your environment, adapts to available resources, learns from usage patterns, and optimizes itself for your specific workload—all without any configuration.
|
Brainy 8.0 is server-only (Node.js 22+ / Bun). With no configuration it:
|
||||||
|
|
||||||
## Zero Configuration Magic
|
- selects a storage adapter from the runtime,
|
||||||
|
- initializes the embedding model (all-MiniLM-L6-v2, 384 dimensions),
|
||||||
|
- builds and maintains the metadata, graph, and vector indexes,
|
||||||
|
- sizes its caches and write buffers to the detected memory budget,
|
||||||
|
- chooses a persistence mode that matches the storage backend, and
|
||||||
|
- quiets its own logging when it detects a production environment.
|
||||||
|
|
||||||
### Instant Start
|
There is no public config-generation function — adaptation happens inside the
|
||||||
|
constructor and `init()`.
|
||||||
|
|
||||||
|
## Instant Start
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy } from 'brainy'
|
import { Brainy } from '@soulcraft/brainy'
|
||||||
|
|
||||||
// That's it. No config needed.
|
// That's it. No config needed.
|
||||||
const brain = new Brainy()
|
const brain = new Brainy()
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
||||||
// Brainy automatically:
|
await brain.add({ data: 'First entity', type: 'concept' })
|
||||||
// ✓ Detects environment (Node.js, Browser, Edge, Deno)
|
const results = await brain.find('first')
|
||||||
// ✓ Chooses optimal storage (FileSystem, OPFS, Memory)
|
|
||||||
// ✓ Downloads required models (if needed)
|
|
||||||
// ✓ Configures vector dimensions (384 optimal)
|
|
||||||
// ✓ Sets up indexing strategies
|
|
||||||
// ✓ Enables appropriate augmentations
|
|
||||||
// ✓ Configures caching layers
|
|
||||||
// ✓ Optimizes for your hardware
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Environment Detection ✅ Available
|
## What Auto-Adaptation Covers
|
||||||
|
|
||||||
Brainy automatically detects and adapts to your runtime:
|
### 1. Storage auto-detection
|
||||||
|
|
||||||
|
With no `storage` option, Brainy uses `type: 'auto'`:
|
||||||
|
|
||||||
|
- **Filesystem** when running on a runtime with a writable Node filesystem and a
|
||||||
|
resolvable root directory. This is the default for typical Node/Bun servers and
|
||||||
|
persists across restarts.
|
||||||
|
- **In-memory** otherwise (no filesystem access, or an explicit memory request).
|
||||||
|
Fast, zero I/O, discarded on process exit — ideal for tests and ephemeral
|
||||||
|
caches.
|
||||||
|
|
||||||
|
8.0 ships exactly two storage adapters — `memory` and `filesystem` — plus the
|
||||||
|
`auto` selector that resolves to one of them. See
|
||||||
|
[Storage Adapters](../concepts/storage-adapters.md) for the full contract.
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Brainy's environment detection
|
// Explicit override when you want a specific root
|
||||||
const environment = {
|
const brain = new Brainy({
|
||||||
// Runtime detection
|
storage: { type: 'filesystem', path: './brainy-data' }
|
||||||
isNode: typeof process !== 'undefined',
|
|
||||||
isBrowser: typeof window !== 'undefined',
|
|
||||||
isDeno: typeof Deno !== 'undefined',
|
|
||||||
isEdge: typeof EdgeRuntime !== 'undefined',
|
|
||||||
isWebWorker: typeof WorkerGlobalScope !== 'undefined',
|
|
||||||
|
|
||||||
// Capability detection
|
|
||||||
hasFileSystem: /* auto-detected */,
|
|
||||||
hasIndexedDB: /* auto-detected */,
|
|
||||||
hasOPFS: /* auto-detected */,
|
|
||||||
hasWebGPU: /* auto-detected */,
|
|
||||||
hasWASM: /* auto-detected */,
|
|
||||||
|
|
||||||
// Resource detection
|
|
||||||
cpuCores: /* auto-detected */,
|
|
||||||
memory: /* auto-detected */,
|
|
||||||
storage: /* auto-detected */
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Auto-Adaptive Storage ✅ Available
|
|
||||||
|
|
||||||
> **Current**: Brainy automatically selects the best storage adapter for your environment.
|
|
||||||
|
|
||||||
### Storage Selection Logic
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Brainy's intelligent storage selection
|
|
||||||
async function autoSelectStorage() {
|
|
||||||
// Server environments
|
|
||||||
if (environment.isNode) {
|
|
||||||
if (await hasWritePermission('./data')) {
|
|
||||||
return 'filesystem' // Best for servers
|
|
||||||
} else if (process.env.S3_BUCKET) {
|
|
||||||
return 's3' // Cloud deployment
|
|
||||||
} else {
|
|
||||||
return 'memory' // Fallback for restricted environments
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Browser environments
|
|
||||||
if (environment.isBrowser) {
|
|
||||||
if (await navigator.storage.estimate() > 1GB) {
|
|
||||||
return 'opfs' // Best for modern browsers
|
|
||||||
} else if (indexedDB) {
|
|
||||||
return 'indexeddb' // Fallback for older browsers
|
|
||||||
} else {
|
|
||||||
return 'memory' // In-memory for restricted contexts
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Edge environments
|
|
||||||
if (environment.isEdge) {
|
|
||||||
return 'kv' // Use edge KV stores (Cloudflare, Vercel)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Storage Migration
|
|
||||||
|
|
||||||
Brainy seamlessly migrates between storage types:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Start with memory storage (development)
|
|
||||||
const brain = new Brainy() // Auto-selects memory
|
|
||||||
|
|
||||||
// Later, migrate to production storage
|
|
||||||
await brain.migrate({
|
|
||||||
to: 'filesystem',
|
|
||||||
path: './production-data'
|
|
||||||
})
|
})
|
||||||
// All data seamlessly transferred
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Learning & Optimization 🚧 Coming Soon
|
### 2. HNSW quality from the `recall` preset
|
||||||
|
|
||||||
> **Note**: These features are planned for Q2 2025. Currently, Brainy uses static optimizations.
|
Vector-index quality comes from a single preset rather than hand-tuned graph
|
||||||
|
parameters. `config.vector.recall` accepts `'fast'`, `'balanced'`, or
|
||||||
### Query Pattern Learning 🚧 Planned
|
`'accurate'` and defaults to `'balanced'`. The preset maps internally to the
|
||||||
|
HNSW construction and search parameters (`M` / `efConstruction` / `efSearch`),
|
||||||
Brainy learns from your query patterns and optimizes accordingly:
|
so you trade recall against latency with one knob instead of three.
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Brainy observes query patterns
|
const brain = new Brainy({
|
||||||
class QueryPatternLearner {
|
vector: { recall: 'fast' } // favor latency over recall
|
||||||
analyze(queries: Query[]) {
|
})
|
||||||
return {
|
|
||||||
// Frequency analysis
|
|
||||||
mostCommonFields: this.getTopFields(queries),
|
|
||||||
avgResultSize: this.getAvgSize(queries),
|
|
||||||
temporalPatterns: this.getTimePatterns(queries),
|
|
||||||
|
|
||||||
// Relationship analysis
|
|
||||||
commonTraversals: this.getGraphPatterns(queries),
|
|
||||||
typicalDepth: this.getAvgDepth(queries),
|
|
||||||
|
|
||||||
// Performance analysis
|
|
||||||
slowQueries: this.getSlowQueries(queries),
|
|
||||||
cacheability: this.getCacheability(queries)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Automatic optimizations based on learning:
|
|
||||||
// - Creates indexes for frequently queried fields
|
|
||||||
// - Pre-computes common graph traversals
|
|
||||||
// - Adjusts cache sizes based on working set
|
|
||||||
// - Optimizes vector search parameters
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Auto-Indexing 🚧 Planned
|
The default JS index is `JsHnswVectorIndex`. An optional native acceleration
|
||||||
|
provider (the `@soulcraft/cortex` package) can replace it with a
|
||||||
|
higher-performing implementation; the public knobs stay the same. Quantization
|
||||||
|
and other index-internal acceleration are the native provider's concern, not a
|
||||||
|
Brainy configuration option.
|
||||||
|
|
||||||
Brainy automatically creates indexes based on usage:
|
### 3. Persistence mode follows the backend
|
||||||
|
|
||||||
|
`config.vector.persistMode` accepts `'immediate'` or `'deferred'`. Left unset,
|
||||||
|
Brainy chooses for you:
|
||||||
|
|
||||||
|
- **Immediate** on filesystem storage, so the index file stays in lock-step with
|
||||||
|
the data and survives a crash.
|
||||||
|
- **Deferred** on in-memory storage, where there is nothing durable to sync to,
|
||||||
|
so writes are batched for throughput.
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// No manual index configuration needed
|
const brain = new Brainy({
|
||||||
await brain.find({ where: { category: "tech" } }) // First query
|
vector: { persistMode: 'deferred' } // batch persistence for write-heavy loads
|
||||||
// Brainy notices 'category' field usage
|
})
|
||||||
|
|
||||||
await brain.find({ where: { category: "science" } }) // Second query
|
|
||||||
// Pattern detected - auto-creates category index
|
|
||||||
|
|
||||||
await brain.find({ where: { category: "tech" } }) // Third query
|
|
||||||
// Now using index - 100x faster!
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Adaptive Caching 🚧 Planned
|
### 4. Memory-aware cache and buffer sizing
|
||||||
|
|
||||||
Cache strategies adapt to your access patterns:
|
Brainy reads the container's memory budget — `CLOUD_RUN_MEMORY`, `MEMORY_LIMIT`,
|
||||||
|
or the cgroup memory limit when running in a container — and sizes its read
|
||||||
|
caches and write buffers to fit. On a small instance it stays conservative; on a
|
||||||
|
large one it uses more of the available headroom. Query-result limits are capped
|
||||||
|
against the same budget (roughly 25 KB per result) to keep a single oversized
|
||||||
|
query from exhausting memory.
|
||||||
|
|
||||||
|
You can pin the cache explicitly:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
class AdaptiveCache {
|
const brain = new Brainy({
|
||||||
async adapt(metrics: AccessMetrics) {
|
cache: { maxSize: 10000, ttl: 3_600_000 }
|
||||||
if (metrics.hitRate < 0.3) {
|
})
|
||||||
// Low hit rate - switch strategy
|
|
||||||
this.strategy = 'lfu' // Least Frequently Used
|
|
||||||
} else if (metrics.workingSet > this.size) {
|
|
||||||
// Working set too large - increase size
|
|
||||||
this.size = Math.min(metrics.workingSet * 1.5, maxMemory)
|
|
||||||
} else if (metrics.temporalLocality > 0.8) {
|
|
||||||
// High temporal locality - use time-based eviction
|
|
||||||
this.strategy = 'ttl'
|
|
||||||
this.ttl = metrics.avgAccessInterval * 2
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Performance Auto-Scaling 🚧 Coming Soon
|
### 5. Logging quiets in production
|
||||||
|
|
||||||
### Dynamic Batch Sizing
|
Brainy detects production-style environments (for example `NODE_ENV` set to a
|
||||||
|
non-development value) and reduces its own log verbosity automatically. This is
|
||||||
Brainy adjusts batch sizes based on system load:
|
logging-only behavior — it does not change indexing, storage, or query results.
|
||||||
|
|
||||||
```typescript
|
|
||||||
class DynamicBatcher {
|
|
||||||
calculateOptimalBatch() {
|
|
||||||
const cpuUsage = process.cpuUsage()
|
|
||||||
const memoryUsage = process.memoryUsage()
|
|
||||||
|
|
||||||
if (cpuUsage < 30 && memoryUsage < 50) {
|
|
||||||
return 1000 // System idle - large batches
|
|
||||||
} else if (cpuUsage < 60 && memoryUsage < 70) {
|
|
||||||
return 100 // Moderate load - medium batches
|
|
||||||
} else {
|
|
||||||
return 10 // High load - small batches
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Automatically applied during bulk operations
|
|
||||||
for (const item of millionItems) {
|
|
||||||
await brain.add(item) // Internally batched optimally
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Memory Management
|
|
||||||
|
|
||||||
Automatic memory pressure handling:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class MemoryManager {
|
|
||||||
async handlePressure() {
|
|
||||||
const usage = process.memoryUsage()
|
|
||||||
const available = os.freemem()
|
|
||||||
|
|
||||||
if (available < 100 * 1024 * 1024) { // Less than 100MB free
|
|
||||||
// Emergency mode
|
|
||||||
await this.flushCaches()
|
|
||||||
await this.compactIndexes()
|
|
||||||
await this.offloadToDisk()
|
|
||||||
} else if (usage.heapUsed / usage.heapTotal > 0.9) {
|
|
||||||
// Preventive mode
|
|
||||||
await this.reduceCacheSizes()
|
|
||||||
await this.pauseBackgroundTasks()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Connection Pooling
|
|
||||||
|
|
||||||
Automatic connection management for storage backends:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class ConnectionPool {
|
|
||||||
async getOptimalPoolSize() {
|
|
||||||
// Adapts based on workload
|
|
||||||
const metrics = await this.getMetrics()
|
|
||||||
|
|
||||||
if (metrics.waitTime > 100) {
|
|
||||||
// Queries waiting - increase pool
|
|
||||||
this.size = Math.min(this.size * 1.5, this.maxSize)
|
|
||||||
} else if (metrics.idleConnections > this.size * 0.5) {
|
|
||||||
// Too many idle - decrease pool
|
|
||||||
this.size = Math.max(this.size * 0.7, this.minSize)
|
|
||||||
}
|
|
||||||
|
|
||||||
return this.size
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Model Auto-Selection
|
|
||||||
|
|
||||||
### Embedding Model Selection
|
|
||||||
|
|
||||||
Brainy chooses the best embedding model for your use case:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
async function autoSelectModel(data: Sample[]) {
|
|
||||||
const analysis = {
|
|
||||||
languages: detectLanguages(data),
|
|
||||||
domainSpecific: detectDomain(data),
|
|
||||||
averageLength: getAvgLength(data),
|
|
||||||
requiresMultilingual: languages.length > 1
|
|
||||||
}
|
|
||||||
|
|
||||||
if (analysis.requiresMultilingual) {
|
|
||||||
return 'multilingual-e5-base' // Handles 100+ languages
|
|
||||||
} else if (analysis.domainSpecific === 'code') {
|
|
||||||
return 'codebert-base' // Optimized for code
|
|
||||||
} else if (analysis.averageLength > 512) {
|
|
||||||
return 'all-mpnet-base-v2' // Better for long text
|
|
||||||
} else {
|
|
||||||
return 'all-MiniLM-L6-v2' // Fast and efficient default
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Model Downloading
|
|
||||||
|
|
||||||
Models are automatically downloaded when needed:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// First use - model auto-downloads
|
|
||||||
const brain = new Brainy()
|
|
||||||
await brain.init() // Downloads model if not cached
|
|
||||||
|
|
||||||
// Intelligent model caching
|
|
||||||
const modelCache = {
|
|
||||||
location: process.env.MODEL_CACHE || '~/.brainy/models',
|
|
||||||
maxSize: 5 * 1024 * 1024 * 1024, // 5GB max
|
|
||||||
strategy: 'lru', // Least recently used eviction
|
|
||||||
|
|
||||||
// CDN selection based on location
|
|
||||||
cdn: await selectFastestCDN([
|
|
||||||
'https://cdn.brainy.io',
|
|
||||||
'https://brainy.b-cdn.net',
|
|
||||||
'https://models.huggingface.co'
|
|
||||||
])
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Workload Detection
|
|
||||||
|
|
||||||
### Pattern Recognition
|
|
||||||
|
|
||||||
Brainy identifies your workload type and optimizes:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
enum WorkloadType {
|
|
||||||
OLTP = 'oltp', // Many small transactions
|
|
||||||
OLAP = 'olap', // Analytical queries
|
|
||||||
STREAMING = 'streaming', // Real-time ingestion
|
|
||||||
BATCH = 'batch', // Bulk processing
|
|
||||||
HYBRID = 'hybrid' // Mixed workload
|
|
||||||
}
|
|
||||||
|
|
||||||
class WorkloadDetector {
|
|
||||||
detect(metrics: OperationMetrics): WorkloadType {
|
|
||||||
if (metrics.writesPerSecond > 1000 && metrics.avgWriteSize < 1024) {
|
|
||||||
return WorkloadType.STREAMING
|
|
||||||
} else if (metrics.avgQueryComplexity > 0.8 && metrics.avgResultSize > 10000) {
|
|
||||||
return WorkloadType.OLAP
|
|
||||||
} else if (metrics.batchOperations > metrics.singleOperations) {
|
|
||||||
return WorkloadType.BATCH
|
|
||||||
} else if (metrics.writeReadRatio > 0.3 && metrics.writeReadRatio < 0.7) {
|
|
||||||
return WorkloadType.HYBRID
|
|
||||||
} else {
|
|
||||||
return WorkloadType.OLTP
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Optimization Strategies
|
|
||||||
|
|
||||||
Different optimizations for different workloads:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class WorkloadOptimizer {
|
|
||||||
optimize(workload: WorkloadType) {
|
|
||||||
switch (workload) {
|
|
||||||
case WorkloadType.STREAMING:
|
|
||||||
return {
|
|
||||||
entityRegistry: true, // Deduplication
|
|
||||||
batchSize: 1000,
|
|
||||||
walEnabled: true,
|
|
||||||
cacheSize: 'small',
|
|
||||||
indexStrategy: 'lazy'
|
|
||||||
}
|
|
||||||
|
|
||||||
case WorkloadType.OLAP:
|
|
||||||
return {
|
|
||||||
entityRegistry: false,
|
|
||||||
batchSize: 10000,
|
|
||||||
walEnabled: false,
|
|
||||||
cacheSize: 'large',
|
|
||||||
indexStrategy: 'eager',
|
|
||||||
parallelQueries: true
|
|
||||||
}
|
|
||||||
|
|
||||||
case WorkloadType.BATCH:
|
|
||||||
return {
|
|
||||||
entityRegistry: false,
|
|
||||||
batchSize: 50000,
|
|
||||||
walEnabled: false,
|
|
||||||
cacheSize: 'minimal',
|
|
||||||
indexStrategy: 'deferred'
|
|
||||||
}
|
|
||||||
|
|
||||||
default:
|
|
||||||
return this.defaultConfig
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Hardware Adaptation 🚧 Coming Soon
|
|
||||||
|
|
||||||
> **Note**: GPU acceleration and hardware optimization planned for Q3 2025.
|
|
||||||
|
|
||||||
### CPU Optimization
|
|
||||||
|
|
||||||
Adapts to available CPU resources:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class CPUAdapter {
|
|
||||||
async optimize() {
|
|
||||||
const cores = os.cpus().length
|
|
||||||
const type = os.cpus()[0].model
|
|
||||||
|
|
||||||
// Parallel processing based on cores
|
|
||||||
this.parallelism = Math.max(1, cores - 1) // Leave one core free
|
|
||||||
|
|
||||||
// SIMD detection for vector operations
|
|
||||||
if (type.includes('Intel') || type.includes('AMD')) {
|
|
||||||
this.enableSIMD = await checkSIMDSupport()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Thread pool sizing
|
|
||||||
this.threadPoolSize = cores * 2 // Optimal for I/O bound
|
|
||||||
|
|
||||||
// Vector search optimization
|
|
||||||
if (cores >= 8) {
|
|
||||||
this.hnswConstruction = 200 // Higher quality index
|
|
||||||
this.hnswSearch = 100 // More accurate search
|
|
||||||
} else {
|
|
||||||
this.hnswConstruction = 100 // Balanced
|
|
||||||
this.hnswSearch = 50 // Faster search
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Memory Adaptation
|
|
||||||
|
|
||||||
Intelligent memory allocation:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class MemoryAdapter {
|
|
||||||
async configure() {
|
|
||||||
const totalMemory = os.totalmem()
|
|
||||||
const availableMemory = os.freemem()
|
|
||||||
|
|
||||||
// Allocate based on available memory
|
|
||||||
const allocation = {
|
|
||||||
cache: Math.min(availableMemory * 0.25, 2 * GB),
|
|
||||||
vectors: Math.min(availableMemory * 0.30, 4 * GB),
|
|
||||||
indexes: Math.min(availableMemory * 0.20, 2 * GB),
|
|
||||||
working: Math.min(availableMemory * 0.25, 2 * GB)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Adjust for low memory systems
|
|
||||||
if (totalMemory < 4 * GB) {
|
|
||||||
allocation.cache *= 0.5
|
|
||||||
allocation.vectors *= 0.7
|
|
||||||
this.enableSwapping = true
|
|
||||||
}
|
|
||||||
|
|
||||||
return allocation
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### GPU Acceleration
|
|
||||||
|
|
||||||
Automatic GPU detection and utilization:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class GPUAdapter {
|
|
||||||
async detect() {
|
|
||||||
// WebGPU in browsers
|
|
||||||
if (navigator?.gpu) {
|
|
||||||
const adapter = await navigator.gpu.requestAdapter()
|
|
||||||
return {
|
|
||||||
available: true,
|
|
||||||
type: 'webgpu',
|
|
||||||
memory: adapter.limits.maxBufferSize,
|
|
||||||
compute: adapter.limits.maxComputeWorkgroupsPerDimension
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// CUDA in Node.js
|
|
||||||
if (process.platform === 'linux' || process.platform === 'win32') {
|
|
||||||
const hasCuda = await checkCudaSupport()
|
|
||||||
if (hasCuda) {
|
|
||||||
return {
|
|
||||||
available: true,
|
|
||||||
type: 'cuda',
|
|
||||||
memory: await getCudaMemory(),
|
|
||||||
compute: await getCudaCores()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return { available: false }
|
|
||||||
}
|
|
||||||
|
|
||||||
async optimize(gpu: GPUInfo) {
|
|
||||||
if (gpu.available) {
|
|
||||||
// Offload vector operations to GPU
|
|
||||||
this.vectorOps = 'gpu'
|
|
||||||
this.embeddingGeneration = 'gpu'
|
|
||||||
this.matrixMultiplication = 'gpu'
|
|
||||||
|
|
||||||
// Larger batch sizes for GPU
|
|
||||||
this.batchSize = gpu.memory > 8 * GB ? 10000 : 1000
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Network Adaptation
|
|
||||||
|
|
||||||
### Bandwidth Detection
|
|
||||||
|
|
||||||
Optimizes for available network bandwidth:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class NetworkAdapter {
|
|
||||||
async measureBandwidth() {
|
|
||||||
const testSize = 1 * MB
|
|
||||||
const start = Date.now()
|
|
||||||
await this.transfer(testSize)
|
|
||||||
const duration = Date.now() - start
|
|
||||||
|
|
||||||
const bandwidth = (testSize / duration) * 1000 // bytes/sec
|
|
||||||
|
|
||||||
if (bandwidth < 1 * MB) {
|
|
||||||
// Low bandwidth - optimize
|
|
||||||
this.compression = 'aggressive'
|
|
||||||
this.batchTransfers = true
|
|
||||||
this.cacheRemote = true
|
|
||||||
} else if (bandwidth > 100 * MB) {
|
|
||||||
// High bandwidth
|
|
||||||
this.compression = 'minimal'
|
|
||||||
this.parallelTransfers = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Latency Optimization
|
|
||||||
|
|
||||||
Adapts to network latency:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class LatencyOptimizer {
|
|
||||||
async optimize() {
|
|
||||||
const latency = await this.measureLatency()
|
|
||||||
|
|
||||||
if (latency > 100) { // High latency
|
|
||||||
// Batch operations
|
|
||||||
this.minBatchSize = 100
|
|
||||||
|
|
||||||
// Aggressive prefetching
|
|
||||||
this.prefetchDepth = 3
|
|
||||||
|
|
||||||
// Local caching
|
|
||||||
this.cacheStrategy = 'aggressive'
|
|
||||||
|
|
||||||
// Connection pooling
|
|
||||||
this.connectionPool = Math.min(latency / 10, 50)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Cloud Provider Detection 🚧 Coming Soon
|
|
||||||
|
|
||||||
> **Note**: Cloud provider auto-detection planned for Q3 2025.
|
|
||||||
|
|
||||||
### Automatic Cloud Optimization
|
|
||||||
|
|
||||||
Detects and optimizes for cloud providers:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class CloudDetector {
|
|
||||||
async detect() {
|
|
||||||
// AWS Detection
|
|
||||||
if (process.env.AWS_REGION || await canReachMetadata('169.254.169.254')) {
|
|
||||||
return {
|
|
||||||
provider: 'aws',
|
|
||||||
instance: await getEC2InstanceType(),
|
|
||||||
region: process.env.AWS_REGION,
|
|
||||||
services: {
|
|
||||||
storage: 's3',
|
|
||||||
cache: 'elasticache',
|
|
||||||
compute: 'lambda'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Google Cloud Detection
|
|
||||||
if (process.env.GOOGLE_CLOUD_PROJECT || await canReachMetadata('metadata.google.internal')) {
|
|
||||||
return {
|
|
||||||
provider: 'gcp',
|
|
||||||
instance: await getGCEInstanceType(),
|
|
||||||
region: process.env.GOOGLE_CLOUD_REGION,
|
|
||||||
services: {
|
|
||||||
storage: 'gcs',
|
|
||||||
cache: 'memorystore',
|
|
||||||
compute: 'cloud-run'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Vercel Edge Detection
|
|
||||||
if (process.env.VERCEL) {
|
|
||||||
return {
|
|
||||||
provider: 'vercel',
|
|
||||||
region: process.env.VERCEL_REGION,
|
|
||||||
services: {
|
|
||||||
storage: 'vercel-kv',
|
|
||||||
cache: 'edge-config',
|
|
||||||
compute: 'edge-runtime'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Development vs Production
|
|
||||||
|
|
||||||
### Automatic Environment Detection
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class EnvironmentDetector {
|
|
||||||
detect() {
|
|
||||||
const indicators = {
|
|
||||||
// Development indicators
|
|
||||||
isDevelopment:
|
|
||||||
process.env.NODE_ENV === 'development' ||
|
|
||||||
process.env.DEBUG ||
|
|
||||||
process.argv.includes('--dev') ||
|
|
||||||
isLocalhost() ||
|
|
||||||
hasDevTools(),
|
|
||||||
|
|
||||||
// Test indicators
|
|
||||||
isTest:
|
|
||||||
process.env.NODE_ENV === 'test' ||
|
|
||||||
process.env.CI ||
|
|
||||||
isTestRunner(),
|
|
||||||
|
|
||||||
// Production indicators
|
|
||||||
isProduction:
|
|
||||||
process.env.NODE_ENV === 'production' ||
|
|
||||||
process.env.VERCEL ||
|
|
||||||
process.env.NETLIFY ||
|
|
||||||
!isLocalhost()
|
|
||||||
}
|
|
||||||
|
|
||||||
return indicators
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Different defaults for different environments
|
|
||||||
const config = environment.isProduction ? {
|
|
||||||
storage: 'filesystem',
|
|
||||||
wal: true,
|
|
||||||
monitoring: true,
|
|
||||||
compression: true,
|
|
||||||
caching: 'aggressive'
|
|
||||||
} : {
|
|
||||||
storage: 'memory',
|
|
||||||
wal: false,
|
|
||||||
monitoring: false,
|
|
||||||
compression: false,
|
|
||||||
caching: 'minimal'
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Error Recovery
|
|
||||||
|
|
||||||
### Automatic Fallbacks
|
|
||||||
|
|
||||||
Brainy automatically recovers from errors:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class AutoRecovery {
|
|
||||||
async handleStorageFailure() {
|
|
||||||
try {
|
|
||||||
await this.primaryStorage.write(data)
|
|
||||||
} catch (error) {
|
|
||||||
console.warn('Primary storage failed, trying fallback')
|
|
||||||
|
|
||||||
// Try secondary storage
|
|
||||||
if (this.secondaryStorage) {
|
|
||||||
await this.secondaryStorage.write(data)
|
|
||||||
} else {
|
|
||||||
// Fall back to memory
|
|
||||||
await this.memoryStorage.write(data)
|
|
||||||
|
|
||||||
// Schedule retry
|
|
||||||
this.scheduleRetry(data)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async handleModelFailure() {
|
|
||||||
try {
|
|
||||||
return await this.primaryModel.embed(text)
|
|
||||||
} catch (error) {
|
|
||||||
// Fall back to simpler model
|
|
||||||
return await this.fallbackModel.embed(text)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Configuration Override
|
## Configuration Override
|
||||||
|
|
||||||
While zero-config is default, you can override when needed:
|
Zero-config is the default, not a ceiling. Every adaptive decision above has an
|
||||||
|
explicit constructor option:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Explicit configuration when needed
|
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
// Override auto-detection
|
storage: { type: 'filesystem', path: '/var/lib/brainy' },
|
||||||
storage: {
|
vector: {
|
||||||
type: 'filesystem',
|
recall: 'accurate',
|
||||||
path: '/custom/path'
|
persistMode: 'immediate'
|
||||||
},
|
},
|
||||||
|
cache: { maxSize: 50000, ttl: 600_000 }
|
||||||
// Override auto-optimization
|
|
||||||
optimization: {
|
|
||||||
autoIndex: false,
|
|
||||||
autoCache: false,
|
|
||||||
autoBatch: false
|
|
||||||
},
|
|
||||||
|
|
||||||
// Override auto-scaling
|
|
||||||
scaling: {
|
|
||||||
maxMemory: 2 * GB,
|
|
||||||
maxConnections: 100,
|
|
||||||
maxBatchSize: 1000
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## Monitoring Auto-Adaptation
|
|
||||||
|
|
||||||
Brainy provides visibility into its auto-adaptation:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
brain.on('adaptation', (event) => {
|
|
||||||
console.log(`Brainy adapted: ${event.type}`)
|
|
||||||
console.log(`Reason: ${event.reason}`)
|
|
||||||
console.log(`Before: ${JSON.stringify(event.before)}`)
|
|
||||||
console.log(`After: ${JSON.stringify(event.after)}`)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
// Example events:
|
await brain.init()
|
||||||
// - Index created for frequently queried field
|
|
||||||
// - Cache strategy changed due to low hit rate
|
|
||||||
// - Batch size increased due to high throughput
|
|
||||||
// - Storage migrated due to space constraints
|
|
||||||
// - Model switched due to multilingual content
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Conclusion
|
See the [API Reference](../api/README.md#configuration) for the complete option
|
||||||
|
list.
|
||||||
Brainy's zero-configuration and auto-adaptation capabilities mean you can focus on your application logic while Brainy handles:
|
|
||||||
|
|
||||||
- Environment detection and optimization
|
|
||||||
- Storage selection and migration
|
|
||||||
- Performance tuning and scaling
|
|
||||||
- Resource management
|
|
||||||
- Error recovery
|
|
||||||
- Workload optimization
|
|
||||||
|
|
||||||
Just create a Brainy instance and start using it. Brainy will learn, adapt, and optimize itself for your specific use case—no configuration required.
|
|
||||||
|
|
||||||
## See Also
|
## See Also
|
||||||
|
|
||||||
- [Architecture Overview](./overview.md)
|
- [Architecture Overview](./overview.md)
|
||||||
- [Storage Architecture](./storage.md)
|
- [Storage Adapters](../concepts/storage-adapters.md)
|
||||||
- [Performance Guide](../guides/performance.md)
|
- [Scaling Guide](../SCALING.md)
|
||||||
- [Augmentations System](./augmentations.md)
|
- [API Reference](../api/README.md)
|
||||||
|
|
|
||||||
|
|
@ -1,451 +0,0 @@
|
||||||
# 🔌 Brainy Augmentations Complete Reference
|
|
||||||
|
|
||||||
> **All augmentations that power Brainy's extensibility - with locations, usage, and examples**
|
|
||||||
>
|
|
||||||
> **⚠️ Update**: Updated for metadata structure changes and billion-scale optimizations
|
|
||||||
|
|
||||||
## Quick Start
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
const brain = new Brainy({
|
|
||||||
// Augmentations auto-configure based on environment
|
|
||||||
storage: 'auto', // Storage augmentation
|
|
||||||
cache: true, // Cache augmentation
|
|
||||||
index: true // Index augmentation
|
|
||||||
})
|
|
||||||
|
|
||||||
await brain.init() // Augmentations initialize automatically
|
|
||||||
```
|
|
||||||
|
|
||||||
## Augmentation Architecture
|
|
||||||
|
|
||||||
### Key Improvements for Billion-Scale Performance
|
|
||||||
|
|
||||||
1. **Metadata/Vector Separation**: Augmentations now work with separated metadata and vectors
|
|
||||||
- Metadata stored separately from vector data
|
|
||||||
- 99.2% memory reduction for type tracking
|
|
||||||
- Two-file storage pattern for optimal I/O
|
|
||||||
|
|
||||||
2. **Type System Enforcement**: All metadata requires type fields
|
|
||||||
- `NounMetadata` requires `noun: NounType`
|
|
||||||
- `VerbMetadata` requires `verb: VerbType`
|
|
||||||
- Type inference system available as public API
|
|
||||||
|
|
||||||
3. **Storage Adapter Pattern**: Internal vs public method distinction
|
|
||||||
- `_methods`: Return pure structures (HNSWNoun, HNSWVerb)
|
|
||||||
- Public methods: Return WithMetadata types
|
|
||||||
- MetadataEnforcer Proxy ensures proper access
|
|
||||||
|
|
||||||
### What This Means for Augmentation Users
|
|
||||||
|
|
||||||
**✅ If you use built-in augmentations**: No changes needed! They're all updated.
|
|
||||||
|
|
||||||
**⚠️ If you created custom storage augmentations**: Update your storage adapter to:
|
|
||||||
- Wrap metadata with required `noun`/`verb` fields
|
|
||||||
- Follow the internal/public method pattern
|
|
||||||
- Use two-file storage approach
|
|
||||||
|
|
||||||
**⚠️ If you access relationship data**: Change `verb.type` to `verb.verb`
|
|
||||||
|
|
||||||
## Core Concepts
|
|
||||||
|
|
||||||
### What are Augmentations?
|
|
||||||
Augmentations are modular extensions that add functionality to Brainy without cluttering the core API. They follow a unified interface and can be:
|
|
||||||
- **Auto-enabled**: Based on configuration (cache, index, storage)
|
|
||||||
- **Manually registered**: For custom functionality
|
|
||||||
- **Chained**: Multiple augmentations work together seamlessly
|
|
||||||
- **Billion-scale ready**: Optimized for datasets with billions of nouns and verbs
|
|
||||||
|
|
||||||
### Augmentation Lifecycle
|
|
||||||
1. **Registration**: Augmentations register before init()
|
|
||||||
2. **Initialization**: Two-phase init (storage first, then others)
|
|
||||||
3. **Execution**: Hook into operations (before/after/both)
|
|
||||||
4. **Shutdown**: Clean teardown on brain.shutdown()
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Storage Augmentations (8 total)
|
|
||||||
|
|
||||||
### MemoryStorageAugmentation
|
|
||||||
**Location**: `src/augmentations/storageAugmentations.ts`
|
|
||||||
**Auto-enabled**: When `storage: 'memory'` or in test environments
|
|
||||||
**Purpose**: In-memory storage for testing and temporary data
|
|
||||||
```typescript
|
|
||||||
const brain = new Brainy({ storage: 'memory' })
|
|
||||||
```
|
|
||||||
|
|
||||||
### FileSystemStorageAugmentation
|
|
||||||
**Location**: `src/augmentations/storageAugmentations.ts`
|
|
||||||
**Auto-enabled**: When `storage: 'filesystem'` or Node.js detected
|
|
||||||
**Purpose**: Persistent file-based storage for Node.js applications
|
|
||||||
```typescript
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: { type: 'filesystem', path: './data' }
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### OPFSStorageAugmentation
|
|
||||||
**Location**: `src/augmentations/storageAugmentations.ts`
|
|
||||||
**Auto-enabled**: When `storage: 'opfs'` or browser with OPFS support
|
|
||||||
**Purpose**: Browser-based persistent storage using Origin Private File System
|
|
||||||
```typescript
|
|
||||||
const brain = new Brainy({ storage: 'opfs' })
|
|
||||||
```
|
|
||||||
|
|
||||||
### S3StorageAugmentation
|
|
||||||
**Location**: `src/augmentations/storageAugmentations.ts`
|
|
||||||
**Manual**: Requires AWS credentials
|
|
||||||
**Purpose**: AWS S3-compatible cloud storage
|
|
||||||
```typescript
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 's3',
|
|
||||||
bucket: 'my-bucket',
|
|
||||||
region: 'us-east-1',
|
|
||||||
credentials: { accessKeyId, secretAccessKey }
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### R2StorageAugmentation
|
|
||||||
**Location**: `src/augmentations/storageAugmentations.ts`
|
|
||||||
**Manual**: Requires Cloudflare credentials
|
|
||||||
**Purpose**: Cloudflare R2 storage (S3-compatible)
|
|
||||||
```typescript
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 'r2',
|
|
||||||
accountId: 'xxx',
|
|
||||||
bucket: 'my-bucket',
|
|
||||||
credentials: { accessKeyId, secretAccessKey }
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### GCSStorageAugmentation
|
|
||||||
**Location**: `src/augmentations/storageAugmentations.ts`
|
|
||||||
**Manual**: Requires Google Cloud credentials
|
|
||||||
**Purpose**: Google Cloud Storage
|
|
||||||
```typescript
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 'gcs',
|
|
||||||
bucket: 'my-bucket',
|
|
||||||
projectId: 'my-project'
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### StorageAugmentation (base)
|
|
||||||
**Location**: `src/augmentations/storageAugmentation.ts`
|
|
||||||
**Purpose**: Base class for custom storage implementations
|
|
||||||
|
|
||||||
### DynamicStorageAugmentation
|
|
||||||
**Location**: `src/augmentations/storageAugmentation.ts`
|
|
||||||
**Purpose**: Runtime storage adapter switching
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Performance Augmentations (7 total)
|
|
||||||
|
|
||||||
### CacheAugmentation
|
|
||||||
**Location**: `src/augmentations/cacheAugmentation.ts`
|
|
||||||
**Auto-enabled**: When `cache: true` (default)
|
|
||||||
**Purpose**: LRU cache for search results and frequent queries
|
|
||||||
```typescript
|
|
||||||
brain.clearCache() // Exposed via API
|
|
||||||
brain.getCacheStats() // Cache hit/miss statistics
|
|
||||||
```
|
|
||||||
|
|
||||||
### IndexAugmentation
|
|
||||||
**Location**: `src/augmentations/indexAugmentation.ts`
|
|
||||||
**Auto-enabled**: When `index: true` (default)
|
|
||||||
**Purpose**: Metadata indexing for O(1) field lookups
|
|
||||||
```typescript
|
|
||||||
brain.rebuildMetadataIndex() // Exposed via API
|
|
||||||
// Enables fast where queries:
|
|
||||||
brain.find({ where: { category: 'tech' } })
|
|
||||||
```
|
|
||||||
|
|
||||||
### MetricsAugmentation
|
|
||||||
**Location**: `src/augmentations/metricsAugmentation.ts`
|
|
||||||
**Auto-enabled**: Always active
|
|
||||||
**Purpose**: Performance metrics and statistics collection
|
|
||||||
```typescript
|
|
||||||
brain.getStats() // Comprehensive metrics
|
|
||||||
```
|
|
||||||
|
|
||||||
### MonitoringAugmentation
|
|
||||||
**Location**: `src/augmentations/monitoringAugmentation.ts`
|
|
||||||
**Manual**: Register for detailed monitoring
|
|
||||||
**Purpose**: Real-time performance monitoring and alerts
|
|
||||||
|
|
||||||
### BatchProcessingAugmentation
|
|
||||||
**Location**: `src/augmentations/batchProcessingAugmentation.ts`
|
|
||||||
**Auto-enabled**: For batch operations
|
|
||||||
**Purpose**: Optimizes bulk add/update/delete operations
|
|
||||||
```typescript
|
|
||||||
brain.addNouns([...]) // Automatically batched
|
|
||||||
```
|
|
||||||
|
|
||||||
### RequestDeduplicatorAugmentation
|
|
||||||
**Location**: `src/augmentations/requestDeduplicatorAugmentation.ts`
|
|
||||||
**Auto-enabled**: Always active
|
|
||||||
**Purpose**: Prevents duplicate concurrent operations
|
|
||||||
|
|
||||||
### ConnectionPoolAugmentation
|
|
||||||
**Location**: `src/augmentations/connectionPoolAugmentation.ts`
|
|
||||||
**Auto-enabled**: For network storage
|
|
||||||
**Purpose**: Connection pooling for cloud storage adapters
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Data Integrity Augmentations (3 total)
|
|
||||||
|
|
||||||
**Auto-enabled**: When `wal: true`
|
|
||||||
**Purpose**: Write-ahead logging for crash recovery
|
|
||||||
```typescript
|
|
||||||
const brain = new Brainy({ wal: true })
|
|
||||||
// Automatic recovery on restart after crash
|
|
||||||
```
|
|
||||||
|
|
||||||
### EntityRegistryAugmentation
|
|
||||||
**Location**: `src/augmentations/entityRegistryAugmentation.ts`
|
|
||||||
**Auto-enabled**: For streaming operations
|
|
||||||
**Purpose**: High-speed deduplication for real-time data
|
|
||||||
```typescript
|
|
||||||
// Prevents duplicate entities in streaming scenarios
|
|
||||||
brain.add(data) // Automatically deduplicated
|
|
||||||
```
|
|
||||||
|
|
||||||
### AutoRegisterEntitiesAugmentation
|
|
||||||
**Location**: `src/augmentations/entityRegistryAugmentation.ts`
|
|
||||||
**Manual**: For automatic entity discovery
|
|
||||||
**Purpose**: Auto-discovers and registers entities from data
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Intelligence Augmentations (2 total)
|
|
||||||
|
|
||||||
### NeuralImportAugmentation
|
|
||||||
**Location**: `src/augmentations/neuralImport.ts`
|
|
||||||
**Manual**: Via `brain.neuralImport()`
|
|
||||||
**Purpose**: AI-powered smart data import
|
|
||||||
```typescript
|
|
||||||
const result = await brain.neuralImport(data, {
|
|
||||||
confidenceThreshold: 0.7,
|
|
||||||
autoApply: true
|
|
||||||
})
|
|
||||||
// Automatically detects entities and relationships
|
|
||||||
```
|
|
||||||
|
|
||||||
### IntelligentVerbScoringAugmentation
|
|
||||||
**Location**: `src/augmentations/intelligentVerbScoringAugmentation.ts`
|
|
||||||
**Auto-enabled**: When verbs are used
|
|
||||||
**Purpose**: ML-based relationship strength scoring
|
|
||||||
```typescript
|
|
||||||
brain.verbScoring.train(feedback)
|
|
||||||
brain.verbScoring.getScore(verbId)
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Communication Augmentations (4 total)
|
|
||||||
|
|
||||||
### APIServerAugmentation
|
|
||||||
**Location**: `src/augmentations/apiServerAugmentation.ts`
|
|
||||||
**Manual**: For server deployments
|
|
||||||
**Purpose**: REST/WebSocket/MCP API server
|
|
||||||
```typescript
|
|
||||||
const augmentation = new APIServerAugmentation()
|
|
||||||
await brain.registerAugmentation(augmentation)
|
|
||||||
// Exposes full Brainy API over network
|
|
||||||
```
|
|
||||||
|
|
||||||
### WebSocketConduitAugmentation
|
|
||||||
**Location**: `src/augmentations/conduitAugmentations.ts`
|
|
||||||
**Manual**: For Brainy-to-Brainy sync
|
|
||||||
**Purpose**: Real-time sync between Brainy instances
|
|
||||||
```typescript
|
|
||||||
const conduit = new WebSocketConduitAugmentation()
|
|
||||||
await conduit.establishConnection('ws://other-brain')
|
|
||||||
```
|
|
||||||
|
|
||||||
### ServerSearchConduitAugmentation
|
|
||||||
**Location**: `src/augmentations/serverSearchAugmentations.ts`
|
|
||||||
**Manual**: For client-server search
|
|
||||||
**Purpose**: Search remote Brainy instance, cache locally
|
|
||||||
|
|
||||||
### ServerSearchActivationAugmentation
|
|
||||||
**Location**: `src/augmentations/serverSearchAugmentations.ts`
|
|
||||||
**Manual**: Works with ServerSearchConduit
|
|
||||||
**Purpose**: Triggers and manages server search operations
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## External Integration (2 total)
|
|
||||||
|
|
||||||
### SynapseAugmentation (base)
|
|
||||||
**Location**: `src/augmentations/synapseAugmentation.ts`
|
|
||||||
**Purpose**: Base class for external platform integrations
|
|
||||||
```typescript
|
|
||||||
// Example: NotionSynapse, SlackSynapse, etc.
|
|
||||||
class NotionSynapse extends SynapseAugmentation {
|
|
||||||
async fetchData() { /* Notion API calls */ }
|
|
||||||
async pushData() { /* Sync to Notion */ }
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### ExampleFileSystemSynapse
|
|
||||||
**Location**: `src/augmentations/synapseAugmentation.ts`
|
|
||||||
**Purpose**: Example implementation for file system sync
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Augmentation Configuration
|
|
||||||
|
|
||||||
### Auto-Configuration
|
|
||||||
```typescript
|
|
||||||
const brain = new Brainy({
|
|
||||||
// These auto-register augmentations:
|
|
||||||
storage: 'auto', // Storage augmentation
|
|
||||||
cache: true, // Cache augmentation
|
|
||||||
index: true, // Index augmentation
|
|
||||||
metrics: true // Metrics augmentation
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Manual Registration
|
|
||||||
```typescript
|
|
||||||
const brain = new Brainy()
|
|
||||||
|
|
||||||
// Register before init()
|
|
||||||
const customAug = new MyCustomAugmentation()
|
|
||||||
await brain.registerAugmentation(customAug)
|
|
||||||
|
|
||||||
await brain.init()
|
|
||||||
```
|
|
||||||
|
|
||||||
### Creating Custom Augmentations
|
|
||||||
```typescript
|
|
||||||
import { BaseAugmentation } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
class MyAugmentation extends BaseAugmentation {
|
|
||||||
readonly name = 'my-augmentation'
|
|
||||||
readonly timing = 'after' // before | after | both
|
|
||||||
readonly operations = ['addNoun', 'search'] // Which ops to hook
|
|
||||||
readonly priority = 10 // Execution order (lower = earlier)
|
|
||||||
|
|
||||||
protected async onInit(): Promise<void> {
|
|
||||||
// Initialize your augmentation
|
|
||||||
}
|
|
||||||
|
|
||||||
async execute<T>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
context?: AugmentationContext
|
|
||||||
): Promise<T | void> {
|
|
||||||
// Your augmentation logic
|
|
||||||
if (operation === 'addNoun') {
|
|
||||||
console.log('Noun added:', params)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onShutdown(): Promise<void> {
|
|
||||||
// Cleanup
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Augmentation Timing & Priority
|
|
||||||
|
|
||||||
### Timing Options
|
|
||||||
- **`before`**: Runs before the operation (can modify params)
|
|
||||||
- **`after`**: Runs after the operation (can see results)
|
|
||||||
- **`both`**: Runs before AND after
|
|
||||||
|
|
||||||
### Priority (lower = earlier)
|
|
||||||
1. Storage augmentations (priority: 0)
|
|
||||||
2. Cache/Index augmentations (priority: 5-10)
|
|
||||||
3. Monitoring/Metrics (priority: 15-20)
|
|
||||||
4. Conduits/Synapses (priority: 20-30)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Key Integration Points
|
|
||||||
|
|
||||||
### Where Augmentations Hook In
|
|
||||||
|
|
||||||
**Brainy Constructor**:
|
|
||||||
- Storage augmentations register based on config
|
|
||||||
- Cache/Index augmentations auto-register if enabled
|
|
||||||
|
|
||||||
**brain.init()**:
|
|
||||||
- Two-phase initialization (storage first, then others)
|
|
||||||
- Augmentations can access brain instance via context
|
|
||||||
|
|
||||||
**Operations** (addNoun, search, etc.):
|
|
||||||
- Augmentations execute based on timing and operations filter
|
|
||||||
- Can modify params (before) or see results (after)
|
|
||||||
|
|
||||||
**brain.shutdown()**:
|
|
||||||
- All augmentations cleaned up in reverse order
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Performance Impact
|
|
||||||
|
|
||||||
Most augmentations have minimal overhead:
|
|
||||||
- **Cache**: ~1ms per search (saves 10-100ms on hits)
|
|
||||||
- **Index**: ~1ms per operation (saves 100ms+ on queries)
|
|
||||||
- **Metrics**: <1ms per operation
|
|
||||||
- **Storage**: Varies by adapter (memory: 0ms, S3: 50-200ms)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Best Practices
|
|
||||||
|
|
||||||
1. **Let auto-configuration work**: Most apps need zero manual config
|
|
||||||
2. **Storage first**: Always configure storage before other augmentations
|
|
||||||
3. **Use built-in augmentations**: They're optimized and battle-tested
|
|
||||||
4. **Custom augmentations**: Extend BaseAugmentation for consistency
|
|
||||||
5. **Respect timing**: Use 'before' to modify, 'after' to observe
|
|
||||||
6. **Mind priority**: Lower numbers execute first
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Troubleshooting
|
|
||||||
|
|
||||||
### Augmentation not working?
|
|
||||||
```typescript
|
|
||||||
// Check if registered
|
|
||||||
brain.listAugmentations()
|
|
||||||
|
|
||||||
// Check if enabled
|
|
||||||
brain.isAugmentationEnabled('cache')
|
|
||||||
|
|
||||||
// Enable/disable at runtime
|
|
||||||
brain.enableAugmentation('cache')
|
|
||||||
brain.disableAugmentation('cache')
|
|
||||||
```
|
|
||||||
|
|
||||||
### Performance issues?
|
|
||||||
```typescript
|
|
||||||
// Check augmentation overhead
|
|
||||||
const stats = brain.getStats()
|
|
||||||
console.log(stats.augmentations)
|
|
||||||
|
|
||||||
// Disable non-critical augmentations
|
|
||||||
brain.disableAugmentation('monitoring')
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
*Augmentations make Brainy infinitely extensible while keeping the core API clean and simple!*
|
|
||||||
|
|
@ -1,620 +0,0 @@
|
||||||
# Augmentation Configuration System
|
|
||||||
|
|
||||||
**Version**: 2.0.0
|
|
||||||
**Status**: Production Ready
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
The Brainy Augmentation Configuration System provides a VSCode-style extension architecture with multiple configuration sources, schema validation, and tool discovery. This system maintains Brainy's zero-config philosophy while enabling sophisticated enterprise configuration management.
|
|
||||||
|
|
||||||
## Table of Contents
|
|
||||||
- [Quick Start](#quick-start)
|
|
||||||
- [Configuration Sources](#configuration-sources)
|
|
||||||
- [Creating Configurable Augmentations](#creating-configurable-augmentations)
|
|
||||||
- [Configuration Discovery](#configuration-discovery)
|
|
||||||
- [Runtime Configuration](#runtime-configuration)
|
|
||||||
- [Environment Variables](#environment-variables)
|
|
||||||
- [Configuration Files](#configuration-files)
|
|
||||||
- [CLI Commands](#cli-commands)
|
|
||||||
- [Tool Integration](#tool-integration)
|
|
||||||
- [Migration Guide](#migration-guide)
|
|
||||||
|
|
||||||
## Quick Start
|
|
||||||
|
|
||||||
### Using an Augmentation with Configuration
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
// Zero-config (uses defaults)
|
|
||||||
const brain = new Brainy()
|
|
||||||
|
|
||||||
// With custom configuration
|
|
||||||
immediateWrites: true,
|
|
||||||
checkpointInterval: 300000 // 5 minutes
|
|
||||||
}))
|
|
||||||
```
|
|
||||||
|
|
||||||
### Configuring via Environment Variables
|
|
||||||
|
|
||||||
```bash
|
|
||||||
export BRAINY_AUG_CACHE_TTL=600000
|
|
||||||
```
|
|
||||||
|
|
||||||
### Configuring via Files
|
|
||||||
|
|
||||||
Create a `.brainyrc` file in your project root:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"augmentations": {
|
|
||||||
"wal": {
|
|
||||||
"enabled": true,
|
|
||||||
"immediateWrites": true,
|
|
||||||
"maxSize": 20971520
|
|
||||||
},
|
|
||||||
"cache": {
|
|
||||||
"ttl": 600000,
|
|
||||||
"maxSize": 2000
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Configuration Sources
|
|
||||||
|
|
||||||
Configuration is resolved in the following priority order (highest to lowest):
|
|
||||||
|
|
||||||
1. **Runtime Updates** - Dynamic configuration changes via API
|
|
||||||
2. **Constructor Parameters** - Code-time configuration
|
|
||||||
3. **Environment Variables** - `BRAINY_AUG_<NAME>_<KEY>`
|
|
||||||
4. **Configuration Files** - `.brainyrc`, `brainy.config.json`
|
|
||||||
5. **Schema Defaults** - Default values from manifest
|
|
||||||
|
|
||||||
### Resolution Example
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Schema default
|
|
||||||
{ maxSize: 10485760 }
|
|
||||||
|
|
||||||
// File configuration (.brainyrc)
|
|
||||||
{ maxSize: 20971520 }
|
|
||||||
|
|
||||||
// Environment variable
|
|
||||||
|
|
||||||
// Constructor parameter
|
|
||||||
|
|
||||||
// Final resolved value: 41943040 (constructor wins)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Creating Configurable Augmentations
|
|
||||||
|
|
||||||
### Step 1: Extend ConfigurableAugmentation
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { ConfigurableAugmentation, AugmentationManifest } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
export class MyAugmentation extends ConfigurableAugmentation {
|
|
||||||
name = 'my-augmentation'
|
|
||||||
timing = 'around' as const
|
|
||||||
metadata = 'none' as const
|
|
||||||
operations = ['search', 'add']
|
|
||||||
priority = 50
|
|
||||||
|
|
||||||
constructor(config?: MyConfig) {
|
|
||||||
super(config) // Handles configuration resolution
|
|
||||||
}
|
|
||||||
|
|
||||||
// Required: Provide manifest for discovery
|
|
||||||
getManifest(): AugmentationManifest {
|
|
||||||
return {
|
|
||||||
id: 'my-augmentation',
|
|
||||||
name: 'My Augmentation',
|
|
||||||
version: '1.0.0',
|
|
||||||
description: 'Does something amazing',
|
|
||||||
category: 'performance',
|
|
||||||
configSchema: {
|
|
||||||
type: 'object',
|
|
||||||
properties: {
|
|
||||||
enabled: {
|
|
||||||
type: 'boolean',
|
|
||||||
default: true,
|
|
||||||
description: 'Enable this augmentation'
|
|
||||||
},
|
|
||||||
threshold: {
|
|
||||||
type: 'number',
|
|
||||||
default: 100,
|
|
||||||
minimum: 1,
|
|
||||||
maximum: 1000,
|
|
||||||
description: 'Processing threshold'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Optional: Handle runtime configuration changes
|
|
||||||
protected async onConfigChange(newConfig: MyConfig, oldConfig: MyConfig): Promise<void> {
|
|
||||||
if (newConfig.threshold !== oldConfig.threshold) {
|
|
||||||
// React to threshold change
|
|
||||||
this.updateThreshold(newConfig.threshold)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async execute<T>(operation: string, params: any, next: () => Promise<T>): Promise<T> {
|
|
||||||
if (!this.config.enabled) {
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Your augmentation logic here
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 2: Define Configuration Interface
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
interface MyConfig {
|
|
||||||
enabled?: boolean
|
|
||||||
threshold?: number
|
|
||||||
mode?: 'fast' | 'balanced' | 'thorough'
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 3: Add JSON Schema in Manifest
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
configSchema: {
|
|
||||||
type: 'object',
|
|
||||||
properties: {
|
|
||||||
enabled: {
|
|
||||||
type: 'boolean',
|
|
||||||
default: true,
|
|
||||||
description: 'Enable augmentation'
|
|
||||||
},
|
|
||||||
threshold: {
|
|
||||||
type: 'number',
|
|
||||||
default: 100,
|
|
||||||
minimum: 1,
|
|
||||||
maximum: 1000,
|
|
||||||
description: 'Processing threshold'
|
|
||||||
},
|
|
||||||
mode: {
|
|
||||||
type: 'string',
|
|
||||||
default: 'balanced',
|
|
||||||
enum: ['fast', 'balanced', 'thorough'],
|
|
||||||
description: 'Processing mode'
|
|
||||||
}
|
|
||||||
},
|
|
||||||
required: [],
|
|
||||||
additionalProperties: false
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Configuration Discovery
|
|
||||||
|
|
||||||
The Discovery API allows tools to discover and configure augmentations dynamically:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { AugmentationDiscovery } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
const discovery = new AugmentationDiscovery(brain.augmentations)
|
|
||||||
|
|
||||||
// Discover all augmentations with manifests
|
|
||||||
const listings = await discovery.discover({
|
|
||||||
includeConfig: true,
|
|
||||||
includeSchema: true
|
|
||||||
})
|
|
||||||
|
|
||||||
// Get configuration schema
|
|
||||||
const schema = await discovery.getConfigSchema('wal')
|
|
||||||
|
|
||||||
// Validate configuration
|
|
||||||
const validation = await discovery.validateConfig('wal', {
|
|
||||||
enabled: true,
|
|
||||||
maxSize: 'invalid' // Will fail validation
|
|
||||||
})
|
|
||||||
|
|
||||||
// Update configuration at runtime
|
|
||||||
await discovery.updateConfig('wal', {
|
|
||||||
checkpointInterval: 120000
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## Runtime Configuration
|
|
||||||
|
|
||||||
### Update Configuration Dynamically
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Get augmentation
|
|
||||||
const wal = brain.augmentations.get('wal')
|
|
||||||
|
|
||||||
// Update configuration
|
|
||||||
await wal.updateConfig({
|
|
||||||
checkpointInterval: 300000
|
|
||||||
})
|
|
||||||
|
|
||||||
// Get current configuration
|
|
||||||
const config = wal.getConfig()
|
|
||||||
```
|
|
||||||
|
|
||||||
### React to Configuration Changes
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class MyAugmentation extends ConfigurableAugmentation {
|
|
||||||
protected async onConfigChange(newConfig: any, oldConfig: any): Promise<void> {
|
|
||||||
// Stop old processes
|
|
||||||
if (oldConfig.enabled && !newConfig.enabled) {
|
|
||||||
await this.stop()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Start new processes
|
|
||||||
if (!oldConfig.enabled && newConfig.enabled) {
|
|
||||||
await this.start()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update settings
|
|
||||||
if (newConfig.interval !== oldConfig.interval) {
|
|
||||||
this.rescheduleTimer(newConfig.interval)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Environment Variables
|
|
||||||
|
|
||||||
### Naming Convention
|
|
||||||
|
|
||||||
```bash
|
|
||||||
BRAINY_AUG_<AUGMENTATION_ID>_<CONFIG_KEY>=value
|
|
||||||
```
|
|
||||||
|
|
||||||
### Examples
|
|
||||||
|
|
||||||
```bash
|
|
||||||
|
|
||||||
# Cache augmentation
|
|
||||||
BRAINY_AUG_CACHE_ENABLED=true
|
|
||||||
BRAINY_AUG_CACHE_MAX_SIZE=2000
|
|
||||||
BRAINY_AUG_CACHE_TTL=600000
|
|
||||||
|
|
||||||
# Complex values (JSON)
|
|
||||||
BRAINY_AUG_MYAUG_FILTERS='["*.js","*.ts"]'
|
|
||||||
BRAINY_AUG_MYAUG_OPTIONS='{"deep":true,"follow":false}'
|
|
||||||
```
|
|
||||||
|
|
||||||
### Docker Example
|
|
||||||
|
|
||||||
```dockerfile
|
|
||||||
ENV BRAINY_AUG_CACHE_TTL=600000
|
|
||||||
```
|
|
||||||
|
|
||||||
## Configuration Files
|
|
||||||
|
|
||||||
### File Locations (Priority Order)
|
|
||||||
|
|
||||||
1. `.brainyrc` (current directory)
|
|
||||||
2. `.brainyrc.json` (current directory)
|
|
||||||
3. `brainy.config.json` (current directory)
|
|
||||||
4. `~/.brainy/config.json` (user home)
|
|
||||||
5. `~/.brainyrc` (user home)
|
|
||||||
|
|
||||||
### File Format
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"augmentations": {
|
|
||||||
"wal": {
|
|
||||||
"enabled": true,
|
|
||||||
"immediateWrites": true,
|
|
||||||
"maxSize": 20971520,
|
|
||||||
"checkpointInterval": 300000
|
|
||||||
},
|
|
||||||
"cache": {
|
|
||||||
"enabled": true,
|
|
||||||
"maxSize": 2000,
|
|
||||||
"ttl": 600000
|
|
||||||
},
|
|
||||||
"metrics": {
|
|
||||||
"enabled": false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Per-Environment Configuration
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"augmentations": {
|
|
||||||
"wal": {
|
|
||||||
"development": {
|
|
||||||
"enabled": true,
|
|
||||||
"immediateWrites": true,
|
|
||||||
"maxSize": 5242880
|
|
||||||
},
|
|
||||||
"production": {
|
|
||||||
"enabled": true,
|
|
||||||
"immediateWrites": false,
|
|
||||||
"maxSize": 104857600,
|
|
||||||
"checkpointInterval": 60000
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## CLI Commands
|
|
||||||
|
|
||||||
### List Augmentations with Configuration
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Show all augmentations with config status
|
|
||||||
brainy augment list --detailed
|
|
||||||
|
|
||||||
# Show configuration for specific augmentation
|
|
||||||
brainy augment config wal
|
|
||||||
|
|
||||||
# Set configuration value
|
|
||||||
brainy augment config wal --set immediateWrites=true
|
|
||||||
|
|
||||||
# Show environment variable names
|
|
||||||
brainy augment config wal --env
|
|
||||||
|
|
||||||
# Export configuration schema
|
|
||||||
brainy augment schema wal > wal-schema.json
|
|
||||||
|
|
||||||
# Validate configuration file
|
|
||||||
brainy augment validate --file config.json
|
|
||||||
```
|
|
||||||
|
|
||||||
### Interactive Configuration
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Interactive configuration wizard
|
|
||||||
brainy augment configure wal
|
|
||||||
|
|
||||||
? Operation mode?
|
|
||||||
❯ Performance (immediate writes)
|
|
||||||
Durability (synchronous writes)
|
|
||||||
Custom
|
|
||||||
? Maximum log size? (10MB) 20MB
|
|
||||||
? Checkpoint interval? (1 minute) 5 minutes
|
|
||||||
|
|
||||||
Configuration saved to .brainyrc
|
|
||||||
```
|
|
||||||
|
|
||||||
## Tool Integration
|
|
||||||
|
|
||||||
### Brain-Cloud Explorer UI
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Auto-generate configuration form from schema
|
|
||||||
const ConfigurationUI = ({ augmentationId }) => {
|
|
||||||
const [manifest, setManifest] = useState(null)
|
|
||||||
const [config, setConfig] = useState({})
|
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
// Fetch manifest with schema
|
|
||||||
fetch(`/api/augmentations/${augmentationId}/manifest`)
|
|
||||||
.then(res => res.json())
|
|
||||||
.then(setManifest)
|
|
||||||
|
|
||||||
// Get current configuration
|
|
||||||
discovery.getConfig(augmentationId)
|
|
||||||
.then(setConfig)
|
|
||||||
}, [augmentationId])
|
|
||||||
|
|
||||||
const handleSave = async (newConfig) => {
|
|
||||||
// Validate configuration
|
|
||||||
const validation = await fetch(`/api/augmentations/${augmentationId}/validate`, {
|
|
||||||
method: 'POST',
|
|
||||||
body: JSON.stringify(newConfig)
|
|
||||||
}).then(res => res.json())
|
|
||||||
|
|
||||||
if (validation.valid) {
|
|
||||||
// Apply configuration
|
|
||||||
await discovery.updateConfig(augmentationId, newConfig)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Render form based on schema
|
|
||||||
return <SchemaForm
|
|
||||||
schema={manifest?.configSchema}
|
|
||||||
values={config}
|
|
||||||
onSubmit={handleSave}
|
|
||||||
/>
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### VS Code Extension
|
|
||||||
|
|
||||||
```json
|
|
||||||
// package.json contribution points
|
|
||||||
{
|
|
||||||
"contributes": {
|
|
||||||
"configuration": {
|
|
||||||
"title": "Brainy Augmentations",
|
|
||||||
"properties": {
|
|
||||||
"brainy.augmentations.wal.enabled": {
|
|
||||||
"type": "boolean",
|
|
||||||
"default": true,
|
|
||||||
},
|
|
||||||
"brainy.augmentations.wal.maxSize": {
|
|
||||||
"type": "number",
|
|
||||||
"default": 10485760,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Migration Guide
|
|
||||||
|
|
||||||
### Migrating from BaseAugmentation
|
|
||||||
|
|
||||||
**Before:**
|
|
||||||
```typescript
|
|
||||||
export class MyAugmentation extends BaseAugmentation {
|
|
||||||
constructor(config: MyConfig = {}) {
|
|
||||||
super()
|
|
||||||
this.config = {
|
|
||||||
enabled: config.enabled ?? true,
|
|
||||||
threshold: config.threshold ?? 100
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// No manifest
|
|
||||||
// No config discovery
|
|
||||||
// No runtime updates
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
**After:**
|
|
||||||
```typescript
|
|
||||||
export class MyAugmentation extends ConfigurableAugmentation {
|
|
||||||
constructor(config?: MyConfig) {
|
|
||||||
super(config) // Config resolution handled automatically
|
|
||||||
}
|
|
||||||
|
|
||||||
getManifest(): AugmentationManifest {
|
|
||||||
return {
|
|
||||||
id: 'my-augmentation',
|
|
||||||
name: 'My Augmentation',
|
|
||||||
version: '1.0.0',
|
|
||||||
description: 'Does something amazing',
|
|
||||||
category: 'performance',
|
|
||||||
configSchema: {
|
|
||||||
type: 'object',
|
|
||||||
properties: {
|
|
||||||
enabled: { type: 'boolean', default: true },
|
|
||||||
threshold: { type: 'number', default: 100 }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Optional: Handle config changes
|
|
||||||
protected async onConfigChange(newConfig: MyConfig, oldConfig: MyConfig): Promise<void> {
|
|
||||||
// React to changes
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Backwards Compatibility
|
|
||||||
|
|
||||||
The system maintains full backwards compatibility:
|
|
||||||
|
|
||||||
1. **BaseAugmentation still works** - Existing augmentations continue to function
|
|
||||||
2. **Constructor config still works** - Existing configuration patterns preserved
|
|
||||||
3. **Zero-config still works** - Defaults are applied automatically
|
|
||||||
4. **Progressive enhancement** - Add features as needed
|
|
||||||
|
|
||||||
## Best Practices
|
|
||||||
|
|
||||||
### 1. Always Provide Defaults
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
configSchema: {
|
|
||||||
properties: {
|
|
||||||
enabled: {
|
|
||||||
type: 'boolean',
|
|
||||||
default: true, // Always provide defaults
|
|
||||||
description: 'Enable this feature'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Use Descriptive Configuration Keys
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Good
|
|
||||||
checkpointInterval: 60000
|
|
||||||
|
|
||||||
// Bad
|
|
||||||
ci: 60000
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Validate Configuration
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
protected async onConfigChange(newConfig: any, oldConfig: any): Promise<void> {
|
|
||||||
// Validate before applying
|
|
||||||
if (newConfig.maxSize < 1048576) {
|
|
||||||
throw new Error('maxSize must be at least 1MB')
|
|
||||||
}
|
|
||||||
|
|
||||||
// Apply changes
|
|
||||||
this.maxSize = newConfig.maxSize
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4. Document Environment Variables
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
/**
|
|
||||||
* Environment Variables:
|
|
||||||
* - BRAINY_AUG_MYAUG_ENABLED: Enable augmentation (boolean)
|
|
||||||
* - BRAINY_AUG_MYAUG_THRESHOLD: Processing threshold (number)
|
|
||||||
* - BRAINY_AUG_MYAUG_MODE: Processing mode (fast|balanced|thorough)
|
|
||||||
*/
|
|
||||||
```
|
|
||||||
|
|
||||||
### 5. Provide Configuration Examples
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
configExamples: [
|
|
||||||
{
|
|
||||||
name: 'Production',
|
|
||||||
description: 'Optimized for production use',
|
|
||||||
config: {
|
|
||||||
enabled: true,
|
|
||||||
mode: 'thorough',
|
|
||||||
threshold: 500
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: 'Development',
|
|
||||||
description: 'Lightweight for development',
|
|
||||||
config: {
|
|
||||||
enabled: true,
|
|
||||||
mode: 'fast',
|
|
||||||
threshold: 10
|
|
||||||
}
|
|
||||||
}
|
|
||||||
]
|
|
||||||
```
|
|
||||||
|
|
||||||
## Troubleshooting
|
|
||||||
|
|
||||||
### Configuration Not Loading
|
|
||||||
|
|
||||||
1. Check file locations and names
|
|
||||||
2. Verify JSON syntax in config files
|
|
||||||
3. Check environment variable names (case-sensitive)
|
|
||||||
4. Use `brainy augment config <name> --debug` to see resolution
|
|
||||||
|
|
||||||
### Validation Errors
|
|
||||||
|
|
||||||
1. Check schema requirements
|
|
||||||
2. Verify data types match schema
|
|
||||||
3. Check minimum/maximum constraints
|
|
||||||
4. Use discovery API to validate before applying
|
|
||||||
|
|
||||||
### Runtime Updates Not Working
|
|
||||||
|
|
||||||
1. Ensure augmentation extends ConfigurableAugmentation
|
|
||||||
2. Implement onConfigChange if needed
|
|
||||||
3. Check for validation errors
|
|
||||||
4. Verify augmentation is initialized
|
|
||||||
|
|
||||||
## API Reference
|
|
||||||
|
|
||||||
See the [Discovery API Documentation](./discovery-api.md) for complete API details.
|
|
||||||
|
|
||||||
## Examples
|
|
||||||
|
|
||||||
See the [examples directory](../../examples/augmentation-config/) for complete working examples.
|
|
||||||
|
|
@ -1,527 +0,0 @@
|
||||||
# 🛠️ Brainy Augmentation Developer Guide
|
|
||||||
|
|
||||||
> **How to create, test, and use augmentations in Brainy**
|
|
||||||
>
|
|
||||||
> **⚠️ Update**: This guide has been updated with breaking changes for metadata structure and type system improvements.
|
|
||||||
|
|
||||||
## Migration Guide
|
|
||||||
|
|
||||||
### What Changed?
|
|
||||||
|
|
||||||
1. **Metadata Structure**: All metadata now requires type fields (`noun` or `verb`)
|
|
||||||
2. **Property Rename**: `verb.type` → `verb.verb` for relationships
|
|
||||||
3. **Two-File Storage**: Vectors and metadata stored separately for performance
|
|
||||||
4. **Return Types**: Storage methods distinguish between internal (pure) and public (WithMetadata) returns
|
|
||||||
|
|
||||||
### Migration Checklist
|
|
||||||
|
|
||||||
- [ ] Update metadata creation to include required `noun` field
|
|
||||||
- [ ] Change `verb.type` to `verb.verb` in all relationship code
|
|
||||||
- [ ] Update storage adapter methods to follow internal/public pattern
|
|
||||||
- [ ] Ensure metadata access uses correct structure
|
|
||||||
|
|
||||||
### Quick Migration Example
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// ❌ v3.x
|
|
||||||
const verb = {
|
|
||||||
type: 'relatedTo',
|
|
||||||
sourceId: 'a',
|
|
||||||
targetId: 'b'
|
|
||||||
}
|
|
||||||
if (verb.type === 'relatedTo') { ... }
|
|
||||||
|
|
||||||
// ✅ Current
|
|
||||||
const verb = {
|
|
||||||
verb: 'relatedTo',
|
|
||||||
sourceId: 'a',
|
|
||||||
targetId: 'b'
|
|
||||||
}
|
|
||||||
const metadata: VerbMetadata = {
|
|
||||||
verb: 'relatedTo',
|
|
||||||
sourceId: 'a',
|
|
||||||
targetId: 'b'
|
|
||||||
}
|
|
||||||
if (verb.verb === 'relatedTo') { ... }
|
|
||||||
```
|
|
||||||
|
|
||||||
## Quick Start: Your First Augmentation
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { BaseAugmentation, BrainyAugmentation, AugmentationContext } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
export class MyFirstAugmentation extends BaseAugmentation {
|
|
||||||
readonly name = 'my-first-augmentation'
|
|
||||||
readonly timing = 'after' as const // When to run: before | after | both
|
|
||||||
readonly operations = ['add'] as const // Which operations to hook
|
|
||||||
readonly priority = 10 // Execution order (lower = first)
|
|
||||||
|
|
||||||
protected async onInit(): Promise<void> {
|
|
||||||
// Initialize your augmentation
|
|
||||||
console.log('MyFirstAugmentation initialized!')
|
|
||||||
}
|
|
||||||
|
|
||||||
async execute<T = any>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
context?: AugmentationContext
|
|
||||||
): Promise<T | void> {
|
|
||||||
// Your augmentation logic
|
|
||||||
if (operation === 'add') {
|
|
||||||
console.log('Noun added:', params.noun)
|
|
||||||
|
|
||||||
// Access metadata correctly
|
|
||||||
if (params.noun?.metadata) {
|
|
||||||
console.log('Noun type:', params.noun.metadata.noun) // Required field
|
|
||||||
}
|
|
||||||
|
|
||||||
// You can access the brain instance
|
|
||||||
const stats = await context?.brain.getStats()
|
|
||||||
console.log('Total nouns:', stats.totalNouns)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected async onShutdown(): Promise<void> {
|
|
||||||
// Cleanup
|
|
||||||
console.log('MyFirstAugmentation shutting down')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Using Your Augmentation
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
import { MyFirstAugmentation } from './my-first-augmentation'
|
|
||||||
|
|
||||||
const brain = new Brainy()
|
|
||||||
|
|
||||||
// Register before init()
|
|
||||||
brain.augmentations.register(new MyFirstAugmentation())
|
|
||||||
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Now your augmentation runs automatically!
|
|
||||||
await brain.add('Hello World')
|
|
||||||
// Console: "Noun added: { id: '...', vector: [...], metadata: {} }"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Augmentation Lifecycle
|
|
||||||
|
|
||||||
### 1. Registration Phase
|
|
||||||
```typescript
|
|
||||||
const aug = new MyAugmentation()
|
|
||||||
brain.augmentations.register(aug) // Before brain.init()!
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Initialization Phase
|
|
||||||
```typescript
|
|
||||||
await brain.init() // Calls aug.initialize() internally
|
|
||||||
// Your onInit() method runs here
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Execution Phase
|
|
||||||
```typescript
|
|
||||||
await brain.add('data') // Your execute() method runs
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4. Shutdown Phase
|
|
||||||
```typescript
|
|
||||||
await brain.shutdown() // Your onShutdown() method runs
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Timing Options
|
|
||||||
|
|
||||||
### `before` - Modify Input
|
|
||||||
```typescript
|
|
||||||
class ValidationAugmentation extends BaseAugmentation {
|
|
||||||
readonly timing = 'before' as const
|
|
||||||
|
|
||||||
async execute<T>(operation: string, params: any): Promise<any> {
|
|
||||||
if (operation === 'add') {
|
|
||||||
// Validate and/or modify params
|
|
||||||
if (!params.content) {
|
|
||||||
throw new Error('Content required')
|
|
||||||
}
|
|
||||||
// Return modified params
|
|
||||||
return { ...params, validated: true }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### `after` - React to Results
|
|
||||||
```typescript
|
|
||||||
class LoggingAugmentation extends BaseAugmentation {
|
|
||||||
readonly timing = 'after' as const
|
|
||||||
|
|
||||||
async execute<T>(operation: string, params: any): Promise<void> {
|
|
||||||
if (operation === 'search') {
|
|
||||||
console.log(`Search for "${params.query}" returned ${params.result.length} results`)
|
|
||||||
}
|
|
||||||
// Don't return anything - just observe
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### `both` - Before AND After
|
|
||||||
```typescript
|
|
||||||
class TimingAugmentation extends BaseAugmentation {
|
|
||||||
readonly timing = 'both' as const
|
|
||||||
private startTime?: number
|
|
||||||
|
|
||||||
async execute<T>(operation: string, params: any, context?: AugmentationContext): Promise<void> {
|
|
||||||
if (!this.startTime) {
|
|
||||||
// Before execution
|
|
||||||
this.startTime = Date.now()
|
|
||||||
} else {
|
|
||||||
// After execution
|
|
||||||
const duration = Date.now() - this.startTime
|
|
||||||
console.log(`${operation} took ${duration}ms`)
|
|
||||||
this.startTime = undefined
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Operation Hooks
|
|
||||||
|
|
||||||
### Core Operations You Can Hook
|
|
||||||
```typescript
|
|
||||||
readonly operations = [
|
|
||||||
'add', // Adding data
|
|
||||||
'update', // Updating data
|
|
||||||
'delete', // Deleting data
|
|
||||||
'get', // Retrieving data
|
|
||||||
'search', // Searching
|
|
||||||
'find', // Triple Intelligence queries
|
|
||||||
'relate', // Adding relationships
|
|
||||||
'unrelate', // Removing relationships
|
|
||||||
'clear', // Clearing data
|
|
||||||
'all' // Hook ALL operations
|
|
||||||
] as const
|
|
||||||
```
|
|
||||||
|
|
||||||
### Example: Multi-Operation Hook
|
|
||||||
```typescript
|
|
||||||
class AuditAugmentation extends BaseAugmentation {
|
|
||||||
readonly operations = ['add', 'update', 'delete'] as const
|
|
||||||
|
|
||||||
async execute<T>(operation: string, params: any): Promise<void> {
|
|
||||||
// Log all data modifications
|
|
||||||
await this.logToAuditTrail(operation, params)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Accessing Brain Context
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class ContextAwareAugmentation extends BaseAugmentation {
|
|
||||||
async execute<T>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
context?: AugmentationContext
|
|
||||||
): Promise<void> {
|
|
||||||
// Access the brain instance
|
|
||||||
const brain = context?.brain
|
|
||||||
if (!brain) return
|
|
||||||
|
|
||||||
// Use any brain method
|
|
||||||
const stats = await brain.getStats()
|
|
||||||
const size = await brain.size()
|
|
||||||
const results = await brain.search('query')
|
|
||||||
|
|
||||||
// Access other augmentations
|
|
||||||
const cache = brain.augmentations.get('cache')
|
|
||||||
if (cache) {
|
|
||||||
await cache.clear()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Real-World Examples
|
|
||||||
|
|
||||||
### 1. Backup Augmentation
|
|
||||||
```typescript
|
|
||||||
class BackupAugmentation extends BaseAugmentation {
|
|
||||||
readonly name = 'backup'
|
|
||||||
readonly timing = 'after' as const
|
|
||||||
readonly operations = ['add', 'update', 'delete'] as const
|
|
||||||
readonly priority = 5
|
|
||||||
|
|
||||||
private changes = 0
|
|
||||||
private readonly backupThreshold = 100
|
|
||||||
|
|
||||||
async execute<T>(operation: string, params: any, context?: AugmentationContext): Promise<void> {
|
|
||||||
this.changes++
|
|
||||||
|
|
||||||
if (this.changes >= this.backupThreshold) {
|
|
||||||
await this.performBackup(context?.brain)
|
|
||||||
this.changes = 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async performBackup(brain?: any): Promise<void> {
|
|
||||||
if (!brain) return
|
|
||||||
// Create instant COW snapshot
|
|
||||||
const snapshotName = `backup-${Date.now()}`
|
|
||||||
await brain.fork(snapshotName)
|
|
||||||
console.log(`Automatic snapshot created: ${snapshotName}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Rate Limiting Augmentation
|
|
||||||
```typescript
|
|
||||||
class RateLimitAugmentation extends BaseAugmentation {
|
|
||||||
readonly name = 'rate-limit'
|
|
||||||
readonly timing = 'before' as const
|
|
||||||
readonly operations = ['search', 'find'] as const
|
|
||||||
readonly priority = 100 // High priority - run first
|
|
||||||
|
|
||||||
private requests = new Map<string, number[]>()
|
|
||||||
private readonly limit = 100 // 100 requests
|
|
||||||
private readonly window = 60000 // per minute
|
|
||||||
|
|
||||||
async execute<T>(operation: string, params: any): Promise<void> {
|
|
||||||
const now = Date.now()
|
|
||||||
const key = params.userId || 'anonymous'
|
|
||||||
|
|
||||||
// Get request timestamps
|
|
||||||
const timestamps = this.requests.get(key) || []
|
|
||||||
|
|
||||||
// Remove old timestamps
|
|
||||||
const recent = timestamps.filter(t => now - t < this.window)
|
|
||||||
|
|
||||||
// Check limit
|
|
||||||
if (recent.length >= this.limit) {
|
|
||||||
throw new Error('Rate limit exceeded')
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add current request
|
|
||||||
recent.push(now)
|
|
||||||
this.requests.set(key, recent)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Encryption Augmentation
|
|
||||||
```typescript
|
|
||||||
class EncryptionAugmentation extends BaseAugmentation {
|
|
||||||
readonly name = 'encryption'
|
|
||||||
readonly timing = 'both' as const
|
|
||||||
readonly operations = ['add', 'get'] as const
|
|
||||||
readonly priority = 90 // Run early
|
|
||||||
|
|
||||||
async execute<T>(operation: string, params: any): Promise<any> {
|
|
||||||
if (operation === 'add') {
|
|
||||||
// Encrypt before storing
|
|
||||||
if (params.metadata?.sensitive) {
|
|
||||||
params.content = await this.encrypt(params.content)
|
|
||||||
params.encrypted = true
|
|
||||||
}
|
|
||||||
return params
|
|
||||||
}
|
|
||||||
|
|
||||||
if (operation === 'get' && params.result?.encrypted) {
|
|
||||||
// Decrypt after retrieval
|
|
||||||
params.result.content = await this.decrypt(params.result.content)
|
|
||||||
delete params.result.encrypted
|
|
||||||
return params.result
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Testing Your Augmentation
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { describe, it, expect } from 'vitest'
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
import { MyAugmentation } from './my-augmentation'
|
|
||||||
|
|
||||||
describe('MyAugmentation', () => {
|
|
||||||
it('should hook into addNoun', async () => {
|
|
||||||
const brain = new Brainy({ storage: 'memory' })
|
|
||||||
const aug = new MyAugmentation()
|
|
||||||
|
|
||||||
// Spy on the execute method
|
|
||||||
const executeSpy = vi.spyOn(aug, 'execute')
|
|
||||||
|
|
||||||
brain.augmentations.register(aug)
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Trigger the augmentation
|
|
||||||
await brain.add('test data')
|
|
||||||
|
|
||||||
// Verify it was called
|
|
||||||
expect(executeSpy).toHaveBeenCalledWith(
|
|
||||||
'add',
|
|
||||||
expect.objectContaining({ content: 'test data' }),
|
|
||||||
expect.any(Object)
|
|
||||||
)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Best Practices
|
|
||||||
|
|
||||||
### 1. Use Proper Timing
|
|
||||||
- `before`: Validation, modification, rate limiting
|
|
||||||
- `after`: Logging, metrics, side effects
|
|
||||||
- `both`: Timing, tracing, wrapping
|
|
||||||
|
|
||||||
### 2. Set Appropriate Priority
|
|
||||||
```typescript
|
|
||||||
// Priority guidelines
|
|
||||||
100: Critical (auth, rate limiting)
|
|
||||||
50: Important (validation, transformation)
|
|
||||||
10: Normal (logging, metrics)
|
|
||||||
1: Optional (debugging, tracing)
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Handle Errors Gracefully
|
|
||||||
```typescript
|
|
||||||
async execute<T>(operation: string, params: any): Promise<void> {
|
|
||||||
try {
|
|
||||||
await this.riskyOperation()
|
|
||||||
} catch (error) {
|
|
||||||
// Log but don't break the main operation
|
|
||||||
console.error(`Augmentation error in ${this.name}:`, error)
|
|
||||||
// Optionally report to monitoring
|
|
||||||
this.reportError(error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4. Be Performance Conscious
|
|
||||||
```typescript
|
|
||||||
class CachedAugmentation extends BaseAugmentation {
|
|
||||||
private cache = new Map<string, any>()
|
|
||||||
|
|
||||||
async execute<T>(operation: string, params: any): Promise<any> {
|
|
||||||
const key = this.getCacheKey(params)
|
|
||||||
|
|
||||||
// Check cache first
|
|
||||||
if (this.cache.has(key)) {
|
|
||||||
return this.cache.get(key)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Expensive operation
|
|
||||||
const result = await this.expensiveOperation(params)
|
|
||||||
this.cache.set(key, result)
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 5. Clean Up Resources
|
|
||||||
```typescript
|
|
||||||
protected async onShutdown(): Promise<void> {
|
|
||||||
// Close connections
|
|
||||||
await this.connection?.close()
|
|
||||||
|
|
||||||
// Clear intervals
|
|
||||||
clearInterval(this.interval)
|
|
||||||
|
|
||||||
// Flush buffers
|
|
||||||
await this.flush()
|
|
||||||
|
|
||||||
// Clear caches
|
|
||||||
this.cache.clear()
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Publishing Your Augmentation (Future)
|
|
||||||
|
|
||||||
### Package Structure
|
|
||||||
```
|
|
||||||
my-augmentation/
|
|
||||||
├── src/
|
|
||||||
│ └── index.ts # Your augmentation
|
|
||||||
├── dist/ # Built output
|
|
||||||
├── tests/
|
|
||||||
│ └── augmentation.test.ts
|
|
||||||
├── package.json
|
|
||||||
├── tsconfig.json
|
|
||||||
└── README.md
|
|
||||||
```
|
|
||||||
|
|
||||||
### package.json
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"name": "@mycompany/brainy-custom-augmentation",
|
|
||||||
"version": "1.0.0",
|
|
||||||
"main": "dist/index.js",
|
|
||||||
"types": "dist/index.d.ts",
|
|
||||||
"keywords": ["brainy-augmentation"],
|
|
||||||
"peerDependencies": {
|
|
||||||
"@soulcraft/brainy": ">=2.0.0"
|
|
||||||
},
|
|
||||||
"brainy": {
|
|
||||||
"type": "augmentation",
|
|
||||||
"class": "CustomAugmentation",
|
|
||||||
"timing": "after",
|
|
||||||
"operations": ["add"],
|
|
||||||
"priority": 10
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Future: Brain Cloud Registry
|
|
||||||
```bash
|
|
||||||
# Coming in 2.1+
|
|
||||||
npm run build
|
|
||||||
npm test
|
|
||||||
brainy publish # Publishes to brain-cloud registry
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## FAQ
|
|
||||||
|
|
||||||
### Q: Can I modify the operation result?
|
|
||||||
**A**: Yes, if `timing: 'before'`, return modified params. If `timing: 'after'`, you can see but not modify results.
|
|
||||||
|
|
||||||
### Q: Can augmentations communicate?
|
|
||||||
**A**: Yes, through the context: `context.brain.augmentations.get('other-augmentation')`
|
|
||||||
|
|
||||||
### Q: What if my augmentation fails?
|
|
||||||
**A**: Handle errors internally. Don't break the main operation unless critical.
|
|
||||||
|
|
||||||
### Q: Can I use async operations?
|
|
||||||
**A**: Yes, everything is async-friendly.
|
|
||||||
|
|
||||||
### Q: How do I access storage directly?
|
|
||||||
**A**: Through context: `context.brain.storage` (but prefer using brain methods)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Get Help
|
|
||||||
|
|
||||||
- **GitHub**: [github.com/soulcraft/brainy](https://github.com/soulcraft/brainy)
|
|
||||||
- **Discord**: [discord.gg/brainy](https://discord.gg/brainy)
|
|
||||||
- **Examples**: See `/examples/augmentations/` in the repo
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
*Start building your augmentation today! The marketplace is coming in 2.1 🚀*
|
|
||||||
|
|
@ -1,667 +0,0 @@
|
||||||
# Augmentation Examples - Import, Store, Export
|
|
||||||
|
|
||||||
This guide shows two complete workflows:
|
|
||||||
1. **Simple Handler** - Just import a new file type
|
|
||||||
2. **Full Augmentation** - Import + Store + Export (premium-ready)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Workflow 1: Simple Handler (Import Only)
|
|
||||||
|
|
||||||
**Use case:** You want to import a new file type (e.g., CAD files) into Brainy's knowledge graph.
|
|
||||||
|
|
||||||
### Step 1: Create the Handler
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// src/handlers/CADHandler.ts
|
|
||||||
import { BaseFormatHandler } from '@soulcraft/brainy/augmentations/intelligentImport'
|
|
||||||
import type { FormatHandlerOptions, ProcessedData } from '@soulcraft/brainy/augmentations/intelligentImport'
|
|
||||||
import { parseCAD } from 'cad-parser' // Your parsing library
|
|
||||||
|
|
||||||
export class CADHandler extends BaseFormatHandler {
|
|
||||||
readonly format = 'cad'
|
|
||||||
|
|
||||||
canHandle(data: Buffer | string | { filename?: string; ext?: string }): boolean {
|
|
||||||
if (typeof data === 'object' && 'filename' in data) {
|
|
||||||
const mimeType = this.getMimeType(data)
|
|
||||||
return this.mimeTypeMatches(mimeType, ['image/vnd.dwg', 'image/vnd.dxf'])
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
async process(data: Buffer | string, options: FormatHandlerOptions): Promise<ProcessedData> {
|
|
||||||
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data)
|
|
||||||
|
|
||||||
// Parse CAD file
|
|
||||||
const cad = await parseCAD(buffer)
|
|
||||||
|
|
||||||
// Extract entities for knowledge graph
|
|
||||||
const entities = []
|
|
||||||
|
|
||||||
// Document entity
|
|
||||||
entities.push({
|
|
||||||
type: 'CADDocument',
|
|
||||||
filename: options.filename,
|
|
||||||
units: cad.units,
|
|
||||||
bounds: cad.bounds,
|
|
||||||
version: cad.version
|
|
||||||
})
|
|
||||||
|
|
||||||
// Layer entities
|
|
||||||
for (const layer of cad.layers) {
|
|
||||||
entities.push({
|
|
||||||
type: 'CADLayer',
|
|
||||||
name: layer.name,
|
|
||||||
color: layer.color,
|
|
||||||
visible: layer.visible
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Object entities
|
|
||||||
for (const obj of cad.objects) {
|
|
||||||
entities.push({
|
|
||||||
type: 'CADObject',
|
|
||||||
objectType: obj.type,
|
|
||||||
layer: obj.layer,
|
|
||||||
geometry: obj.geometry
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
format: 'cad',
|
|
||||||
data: entities,
|
|
||||||
metadata: {
|
|
||||||
rowCount: entities.length,
|
|
||||||
fields: ['type', 'name', 'geometry'],
|
|
||||||
processingTime: Date.now() - startTime,
|
|
||||||
layerCount: cad.layers.length,
|
|
||||||
objectCount: cad.objects.length
|
|
||||||
},
|
|
||||||
filename: options.filename
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 2: Register the Handler
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// src/index.ts
|
|
||||||
import { globalHandlerRegistry } from '@soulcraft/brainy/augmentations/intelligentImport'
|
|
||||||
import { CADHandler } from './handlers/CADHandler.js'
|
|
||||||
|
|
||||||
// Register handler globally
|
|
||||||
globalHandlerRegistry.registerHandler({
|
|
||||||
name: 'cad',
|
|
||||||
mimeTypes: ['image/vnd.dwg', 'image/vnd.dxf'],
|
|
||||||
extensions: ['.dwg', '.dxf', '.dwf'],
|
|
||||||
loader: async () => new CADHandler()
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 3: Use It
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
const brain = new Brainy()
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Import CAD file - automatically routed to CADHandler
|
|
||||||
const result = await brain.import({
|
|
||||||
type: 'file',
|
|
||||||
data: cadFileBuffer,
|
|
||||||
filename: 'floor-plan.dwg'
|
|
||||||
})
|
|
||||||
|
|
||||||
console.log(`Imported ${result.entities.length} CAD entities`)
|
|
||||||
|
|
||||||
// Query the imported data
|
|
||||||
const layers = await brain.find({ type: 'CADLayer' })
|
|
||||||
const objects = await brain.find({ type: 'CADObject', layer: 'WALLS' })
|
|
||||||
```
|
|
||||||
|
|
||||||
**That's it!** Simple handlers just import data. Brainy handles storage automatically.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Workflow 2: Full Augmentation (Import + Store + Export)
|
|
||||||
|
|
||||||
**Use case:** You want a complete solution that imports project files, stores them with special logic, and exports results (e.g., React project analyzer).
|
|
||||||
|
|
||||||
### Step 1: Create the Augmentation
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// @yourcompany/brainy-react-analyzer
|
|
||||||
|
|
||||||
import { BaseAugmentation, type AugmentationContext } from '@soulcraft/brainy'
|
|
||||||
import { BaseFormatHandler } from '@soulcraft/brainy/augmentations/intelligentImport'
|
|
||||||
import type { ProcessedData } from '@soulcraft/brainy/augmentations/intelligentImport'
|
|
||||||
import { parse } from '@babel/parser'
|
|
||||||
import traverse from '@babel/traverse'
|
|
||||||
import * as t from '@babel/types'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* React Project Analyzer Augmentation
|
|
||||||
*
|
|
||||||
* Features:
|
|
||||||
* - Import: Parse React components, extract props, hooks, imports
|
|
||||||
* - Store: Create relationships between components
|
|
||||||
* - Export: Generate component diagram, dependency graph
|
|
||||||
*/
|
|
||||||
export class ReactAnalyzerAugmentation extends BaseAugmentation {
|
|
||||||
readonly name = 'react-analyzer'
|
|
||||||
readonly timing = 'before' as const
|
|
||||||
readonly operations = ['import', 'export'] as any[]
|
|
||||||
readonly priority = 75
|
|
||||||
|
|
||||||
private handler: ReactComponentHandler
|
|
||||||
|
|
||||||
constructor(config = {}) {
|
|
||||||
super(config)
|
|
||||||
this.handler = new ReactComponentHandler()
|
|
||||||
}
|
|
||||||
|
|
||||||
async execute<T = any>(
|
|
||||||
operation: string,
|
|
||||||
params: any,
|
|
||||||
next: () => Promise<T>
|
|
||||||
): Promise<T> {
|
|
||||||
// IMPORT: Parse React files
|
|
||||||
if (operation === 'import' && this.isReactFile(params)) {
|
|
||||||
return this.handleImport(params, next)
|
|
||||||
}
|
|
||||||
|
|
||||||
// EXPORT: Generate diagrams/reports
|
|
||||||
if (operation === 'export' && params.format === 'react-diagram') {
|
|
||||||
return this.handleExport(params, next)
|
|
||||||
}
|
|
||||||
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
private isReactFile(params: any): boolean {
|
|
||||||
const filename = params.filename || ''
|
|
||||||
return (
|
|
||||||
(filename.endsWith('.tsx') || filename.endsWith('.jsx')) &&
|
|
||||||
params.data?.includes('React')
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
private async handleImport<T>(params: any, next: () => Promise<T>): Promise<T> {
|
|
||||||
// Parse React component
|
|
||||||
const processed = await this.handler.process(params.data, params.options)
|
|
||||||
|
|
||||||
// Enrich with relationships
|
|
||||||
params.data = processed.data
|
|
||||||
params.metadata = {
|
|
||||||
...params.metadata,
|
|
||||||
reactAnalysis: processed.metadata
|
|
||||||
}
|
|
||||||
|
|
||||||
// Continue to next augmentation/storage
|
|
||||||
const result = await next()
|
|
||||||
|
|
||||||
// Post-process: Create component relationships
|
|
||||||
await this.createComponentRelationships(processed, result)
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
private async createComponentRelationships(
|
|
||||||
processed: ProcessedData,
|
|
||||||
result: any
|
|
||||||
): Promise<void> {
|
|
||||||
const brain = this.getBrain()
|
|
||||||
if (!brain) return
|
|
||||||
|
|
||||||
// Find the component entity that was created
|
|
||||||
const component = processed.data.find(d => d.type === 'ReactComponent')
|
|
||||||
if (!component) return
|
|
||||||
|
|
||||||
// Create relationships for imports
|
|
||||||
for (const imp of component.imports || []) {
|
|
||||||
// Find or create imported component
|
|
||||||
const imported = await brain.findOne({
|
|
||||||
type: 'ReactComponent',
|
|
||||||
name: imp.name
|
|
||||||
})
|
|
||||||
|
|
||||||
if (imported) {
|
|
||||||
// Create "Imports" relationship
|
|
||||||
await brain.createRelation({
|
|
||||||
source: result.entities[0].id,
|
|
||||||
verb: 'Imports',
|
|
||||||
target: imported.id,
|
|
||||||
metadata: {
|
|
||||||
importPath: imp.path,
|
|
||||||
importType: imp.type
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create relationships for prop types
|
|
||||||
for (const prop of component.props || []) {
|
|
||||||
if (prop.typeRef) {
|
|
||||||
const typeEntity = await brain.findOne({
|
|
||||||
type: 'TypeDefinition',
|
|
||||||
name: prop.typeRef
|
|
||||||
})
|
|
||||||
|
|
||||||
if (typeEntity) {
|
|
||||||
await brain.createRelation({
|
|
||||||
source: result.entities[0].id,
|
|
||||||
verb: 'UsesPropType',
|
|
||||||
target: typeEntity.id
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async handleExport<T>(params: any, next: () => Promise<T>): Promise<T> {
|
|
||||||
const brain = this.getBrain()
|
|
||||||
if (!brain) return next()
|
|
||||||
|
|
||||||
// Query all React components
|
|
||||||
const components = await brain.find({ type: 'ReactComponent' })
|
|
||||||
|
|
||||||
// Build dependency graph
|
|
||||||
const graph = await this.buildDependencyGraph(components)
|
|
||||||
|
|
||||||
// Generate diagram
|
|
||||||
const diagram = this.generateMermaidDiagram(graph)
|
|
||||||
|
|
||||||
return {
|
|
||||||
format: 'react-diagram',
|
|
||||||
diagram,
|
|
||||||
components: components.length,
|
|
||||||
dependencies: graph.edges.length
|
|
||||||
} as T
|
|
||||||
}
|
|
||||||
|
|
||||||
private async buildDependencyGraph(components: any[]): Promise<any> {
|
|
||||||
const brain = this.getBrain()
|
|
||||||
const nodes = components.map(c => ({
|
|
||||||
id: c.id,
|
|
||||||
name: c.name,
|
|
||||||
props: c.props
|
|
||||||
}))
|
|
||||||
|
|
||||||
const edges = []
|
|
||||||
for (const component of components) {
|
|
||||||
const imports = await brain.getRelated(component.id, 'Imports')
|
|
||||||
for (const imp of imports) {
|
|
||||||
edges.push({
|
|
||||||
from: component.id,
|
|
||||||
to: imp.id,
|
|
||||||
type: 'imports'
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return { nodes, edges }
|
|
||||||
}
|
|
||||||
|
|
||||||
private generateMermaidDiagram(graph: any): string {
|
|
||||||
let mermaid = 'graph TD\n'
|
|
||||||
|
|
||||||
for (const node of graph.nodes) {
|
|
||||||
mermaid += ` ${node.id}[${node.name}]\n`
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const edge of graph.edges) {
|
|
||||||
mermaid += ` ${edge.from} --> ${edge.to}\n`
|
|
||||||
}
|
|
||||||
|
|
||||||
return mermaid
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* React Component Handler
|
|
||||||
*/
|
|
||||||
class ReactComponentHandler extends BaseFormatHandler {
|
|
||||||
readonly format = 'react'
|
|
||||||
|
|
||||||
canHandle(data: any): boolean {
|
|
||||||
if (typeof data === 'object' && 'filename' in data) {
|
|
||||||
return data.filename?.match(/\.(tsx|jsx)$/) !== null
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
async process(data: Buffer | string, options: any): Promise<ProcessedData> {
|
|
||||||
const code = Buffer.isBuffer(data) ? data.toString('utf-8') : data
|
|
||||||
|
|
||||||
// Parse with Babel
|
|
||||||
const ast = parse(code, {
|
|
||||||
sourceType: 'module',
|
|
||||||
plugins: ['jsx', 'typescript']
|
|
||||||
})
|
|
||||||
|
|
||||||
// Extract component info
|
|
||||||
const components: any[] = []
|
|
||||||
const imports: any[] = []
|
|
||||||
const exports: any[] = []
|
|
||||||
|
|
||||||
traverse(ast, {
|
|
||||||
// Detect function components
|
|
||||||
FunctionDeclaration(path) {
|
|
||||||
if (this.isReactComponent(path.node)) {
|
|
||||||
components.push({
|
|
||||||
type: 'ReactComponent',
|
|
||||||
name: path.node.id?.name,
|
|
||||||
componentType: 'function',
|
|
||||||
props: this.extractProps(path),
|
|
||||||
hooks: this.extractHooks(path),
|
|
||||||
state: this.extractState(path)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
},
|
|
||||||
|
|
||||||
// Detect class components
|
|
||||||
ClassDeclaration(path) {
|
|
||||||
if (this.isReactClassComponent(path.node)) {
|
|
||||||
components.push({
|
|
||||||
type: 'ReactComponent',
|
|
||||||
name: path.node.id.name,
|
|
||||||
componentType: 'class',
|
|
||||||
props: this.extractClassProps(path),
|
|
||||||
state: this.extractClassState(path),
|
|
||||||
lifecycle: this.extractLifecycleMethods(path)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
},
|
|
||||||
|
|
||||||
// Extract imports
|
|
||||||
ImportDeclaration(path) {
|
|
||||||
imports.push({
|
|
||||||
type: 'Import',
|
|
||||||
from: path.node.source.value,
|
|
||||||
imports: path.node.specifiers.map(s => ({
|
|
||||||
name: s.local.name,
|
|
||||||
imported: t.isImportSpecifier(s) ? s.imported.name : null
|
|
||||||
}))
|
|
||||||
})
|
|
||||||
},
|
|
||||||
|
|
||||||
// Extract exports
|
|
||||||
ExportNamedDeclaration(path) {
|
|
||||||
exports.push({
|
|
||||||
type: 'Export',
|
|
||||||
name: path.node.declaration?.id?.name
|
|
||||||
})
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Enrich components with import info
|
|
||||||
for (const component of components) {
|
|
||||||
component.imports = imports
|
|
||||||
component.exports = exports.find(e => e.name === component.name)
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
format: 'react',
|
|
||||||
data: components,
|
|
||||||
metadata: {
|
|
||||||
rowCount: components.length,
|
|
||||||
fields: ['type', 'name', 'props', 'hooks'],
|
|
||||||
processingTime: Date.now() - startTime,
|
|
||||||
componentCount: components.length,
|
|
||||||
importCount: imports.length,
|
|
||||||
exportCount: exports.length
|
|
||||||
},
|
|
||||||
filename: options.filename
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private isReactComponent(node: any): boolean {
|
|
||||||
// Check if function returns JSX
|
|
||||||
return node.body?.body?.some(stmt =>
|
|
||||||
t.isReturnStatement(stmt) && this.isJSX(stmt.argument)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
private isJSX(node: any): boolean {
|
|
||||||
return t.isJSXElement(node) || t.isJSXFragment(node)
|
|
||||||
}
|
|
||||||
|
|
||||||
private extractProps(path: any): any[] {
|
|
||||||
const params = path.node.params
|
|
||||||
if (params.length === 0) return []
|
|
||||||
|
|
||||||
const propsParam = params[0]
|
|
||||||
if (t.isObjectPattern(propsParam)) {
|
|
||||||
return propsParam.properties.map(p => ({
|
|
||||||
name: p.key.name,
|
|
||||||
type: p.typeAnnotation?.typeAnnotation?.type
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
return []
|
|
||||||
}
|
|
||||||
|
|
||||||
private extractHooks(path: any): string[] {
|
|
||||||
const hooks: string[] = []
|
|
||||||
|
|
||||||
path.traverse({
|
|
||||||
CallExpression(hookPath) {
|
|
||||||
const callee = hookPath.node.callee
|
|
||||||
if (t.isIdentifier(callee) && callee.name.startsWith('use')) {
|
|
||||||
hooks.push(callee.name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
return hooks
|
|
||||||
}
|
|
||||||
|
|
||||||
private extractState(path: any): any[] {
|
|
||||||
const stateVars: any[] = []
|
|
||||||
|
|
||||||
path.traverse({
|
|
||||||
CallExpression(hookPath) {
|
|
||||||
if (
|
|
||||||
t.isIdentifier(hookPath.node.callee) &&
|
|
||||||
hookPath.node.callee.name === 'useState'
|
|
||||||
) {
|
|
||||||
const parent = hookPath.parent
|
|
||||||
if (t.isVariableDeclarator(parent) && t.isArrayPattern(parent.id)) {
|
|
||||||
const [stateVar] = parent.id.elements
|
|
||||||
if (t.isIdentifier(stateVar)) {
|
|
||||||
stateVars.push({
|
|
||||||
name: stateVar.name,
|
|
||||||
initialValue: hookPath.node.arguments[0]
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
return stateVars
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 2: Package as NPM Module
|
|
||||||
|
|
||||||
```json
|
|
||||||
// package.json
|
|
||||||
{
|
|
||||||
"name": "@yourcompany/brainy-react-analyzer",
|
|
||||||
"version": "1.0.0",
|
|
||||||
"description": "React project analyzer for Brainy",
|
|
||||||
"main": "dist/index.js",
|
|
||||||
"types": "dist/index.d.ts",
|
|
||||||
"exports": {
|
|
||||||
".": {
|
|
||||||
"import": "./dist/index.js",
|
|
||||||
"require": "./dist/index.cjs"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"keywords": ["brainy", "react", "analyzer", "augmentation"],
|
|
||||||
"peerDependencies": {
|
|
||||||
"@soulcraft/brainy": "^5.2.0"
|
|
||||||
},
|
|
||||||
"dependencies": {
|
|
||||||
"@babel/parser": "^7.23.0",
|
|
||||||
"@babel/traverse": "^7.23.0",
|
|
||||||
"@babel/types": "^7.23.0"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 3: Use the Augmentation
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Install
|
|
||||||
// npm install @yourcompany/brainy-react-analyzer
|
|
||||||
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
import { ReactAnalyzerAugmentation } from '@yourcompany/brainy-react-analyzer'
|
|
||||||
|
|
||||||
const brain = new Brainy()
|
|
||||||
|
|
||||||
// Add augmentation
|
|
||||||
brain.addAugmentation(new ReactAnalyzerAugmentation())
|
|
||||||
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Import React project
|
|
||||||
await brain.import({
|
|
||||||
type: 'directory',
|
|
||||||
path: '/path/to/react-project/src',
|
|
||||||
recursive: true
|
|
||||||
})
|
|
||||||
|
|
||||||
// Query components
|
|
||||||
const components = await brain.find({ type: 'ReactComponent' })
|
|
||||||
console.log(`Found ${components.length} React components`)
|
|
||||||
|
|
||||||
// Find component dependencies
|
|
||||||
const appComponent = await brain.findOne({ type: 'ReactComponent', name: 'App' })
|
|
||||||
const imports = await brain.getRelated(appComponent.id, 'Imports')
|
|
||||||
console.log(`App component imports:`, imports.map(c => c.name))
|
|
||||||
|
|
||||||
// Export the analyzed component graph as a portable PortableGraph document
|
|
||||||
const graph = await brain.data().then(d => d.export({ type: 'ReactComponent' }))
|
|
||||||
console.log(`Exported ${graph.stats.entityCount} components, ${graph.stats.relationCount} edges`)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 4: Premium Licensing (Optional)
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Add license checking
|
|
||||||
export class ReactAnalyzerAugmentation extends BaseAugmentation {
|
|
||||||
private licenseKey?: string
|
|
||||||
|
|
||||||
constructor(config: { licenseKey?: string } = {}) {
|
|
||||||
super(config)
|
|
||||||
this.licenseKey = config.licenseKey
|
|
||||||
}
|
|
||||||
|
|
||||||
async onInitialize(): Promise<void> {
|
|
||||||
if (!this.licenseKey) {
|
|
||||||
throw new Error('React Analyzer requires a license key. Get one at https://yourcompany.com/brainy-react')
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify license
|
|
||||||
const valid = await this.verifyLicense(this.licenseKey)
|
|
||||||
if (!valid) {
|
|
||||||
throw new Error('Invalid license key')
|
|
||||||
}
|
|
||||||
|
|
||||||
this.log('React Analyzer initialized successfully')
|
|
||||||
}
|
|
||||||
|
|
||||||
private async verifyLicense(key: string): Promise<boolean> {
|
|
||||||
// Check with your license server
|
|
||||||
const response = await fetch('https://api.yourcompany.com/verify', {
|
|
||||||
method: 'POST',
|
|
||||||
headers: { 'Content-Type': 'application/json' },
|
|
||||||
body: JSON.stringify({ key, product: 'brainy-react-analyzer' })
|
|
||||||
})
|
|
||||||
|
|
||||||
const data = await response.json()
|
|
||||||
return data.valid
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Usage with license
|
|
||||||
brain.addAugmentation(new ReactAnalyzerAugmentation({
|
|
||||||
licenseKey: 'YOUR-LICENSE-KEY'
|
|
||||||
}))
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Comparison: Handler vs Augmentation
|
|
||||||
|
|
||||||
| Feature | Simple Handler | Full Augmentation |
|
|
||||||
|---------|---------------|-------------------|
|
|
||||||
| **Import** | ✅ Yes (automatic) | ✅ Yes (with custom logic) |
|
|
||||||
| **Storage** | ✅ Automatic (Brainy core) | ✅ Custom logic + relationships |
|
|
||||||
| **Export** | ❌ No | ✅ Yes (custom formats) |
|
|
||||||
| **Relationships** | ❌ No | ✅ Yes (create custom relationships) |
|
|
||||||
| **Premium licensing** | ❌ Difficult | ✅ Easy (augmentation-level) |
|
|
||||||
| **Custom operations** | ❌ Import only | ✅ Any operation |
|
|
||||||
| **Complexity** | Low (50-100 lines) | Medium (200-500 lines) |
|
|
||||||
|
|
||||||
### When to use Handler:
|
|
||||||
- Just need to import a new file type
|
|
||||||
- Don't need custom export
|
|
||||||
- Don't need special relationships
|
|
||||||
- Simple use case
|
|
||||||
|
|
||||||
### When to use Augmentation:
|
|
||||||
- Need import + export workflow
|
|
||||||
- Need custom relationship logic
|
|
||||||
- Want premium licensing capability
|
|
||||||
- Complex business logic
|
|
||||||
- Multiple operations (import + export + query)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## More Examples
|
|
||||||
|
|
||||||
### Example: Python Project Analyzer
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class PythonAnalyzerAugmentation extends BaseAugmentation {
|
|
||||||
// Import Python files, extract classes/functions
|
|
||||||
// Create relationships between modules
|
|
||||||
// Export: Dependency diagram, call graph
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Example: Database Schema Sync
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class DatabaseSyncAugmentation extends BaseAugmentation {
|
|
||||||
// Import: Parse SQL schema
|
|
||||||
// Store: Tables, columns, relationships
|
|
||||||
// Export: Generate migration scripts
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Example: API Documentation Generator
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class APIDocAugmentation extends BaseAugmentation {
|
|
||||||
// Import: Parse TypeScript types
|
|
||||||
// Store: Endpoints, parameters, responses
|
|
||||||
// Export: OpenAPI spec, Markdown docs
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## See Also
|
|
||||||
|
|
||||||
- [FORMAT_HANDLERS.md](./FORMAT_HANDLERS.md) - Creating format handlers
|
|
||||||
- [AUGMENTATIONS.md](./AUGMENTATIONS.md) - Augmentation system details
|
|
||||||
- [ImageHandler source](../../src/augmentations/intelligentImport/handlers/imageHandler.ts) - Handler example
|
|
||||||
- [IntelligentImportAugmentation source](../../src/augmentations/intelligentImport/IntelligentImportAugmentation.ts) - Augmentation example
|
|
||||||
|
|
@ -1,687 +0,0 @@
|
||||||
# Creating Custom Format Handlers
|
|
||||||
|
|
||||||
**Version:** 5.2.0+
|
|
||||||
|
|
||||||
Format handlers enable you to import ANY file type into Brainy as structured knowledge graph data. This guide shows how to create custom format handlers for your specific file formats.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Quick Start
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { BaseFormatHandler, globalHandlerRegistry } from '@soulcraft/brainy/augmentations/intelligentImport'
|
|
||||||
import type { FormatHandlerOptions, ProcessedData } from '@soulcraft/brainy/augmentations/intelligentImport'
|
|
||||||
|
|
||||||
class MyFormatHandler extends BaseFormatHandler {
|
|
||||||
readonly format = 'myformat'
|
|
||||||
|
|
||||||
canHandle(data: Buffer | string | { filename?: string; ext?: string }): boolean {
|
|
||||||
// Option 1: Check by MIME type
|
|
||||||
if (typeof data === 'object' && 'filename' in data) {
|
|
||||||
const mimeType = this.getMimeType(data)
|
|
||||||
return this.mimeTypeMatches(mimeType, ['application/x-myformat'])
|
|
||||||
}
|
|
||||||
|
|
||||||
// Option 2: Check by magic bytes
|
|
||||||
if (Buffer.isBuffer(data)) {
|
|
||||||
return data[0] === 0x4D && data[1] === 0x59 // "MY" magic bytes
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
async process(
|
|
||||||
data: Buffer | string,
|
|
||||||
options: FormatHandlerOptions
|
|
||||||
): Promise<ProcessedData> {
|
|
||||||
// Convert to Buffer
|
|
||||||
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data)
|
|
||||||
|
|
||||||
// Parse your format
|
|
||||||
const parsed = this.parseMyFormat(buffer)
|
|
||||||
|
|
||||||
// Return structured data
|
|
||||||
return {
|
|
||||||
format: 'myformat',
|
|
||||||
data: [
|
|
||||||
{
|
|
||||||
type: 'MyEntity',
|
|
||||||
name: parsed.name,
|
|
||||||
metadata: parsed.metadata
|
|
||||||
}
|
|
||||||
],
|
|
||||||
metadata: {
|
|
||||||
rowCount: 1,
|
|
||||||
fields: ['type', 'name', 'metadata'],
|
|
||||||
processingTime: Date.now() - startTime
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private parseMyFormat(buffer: Buffer): any {
|
|
||||||
// Your parsing logic here
|
|
||||||
return { name: 'example', metadata: {} }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Register globally
|
|
||||||
globalHandlerRegistry.registerHandler({
|
|
||||||
name: 'myformat',
|
|
||||||
mimeTypes: ['application/x-myformat'],
|
|
||||||
extensions: ['.myf', '.myfmt'],
|
|
||||||
loader: async () => new MyFormatHandler()
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
Now Brainy automatically handles your format:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
await brain.import({
|
|
||||||
type: 'file',
|
|
||||||
data: myFormatBuffer,
|
|
||||||
filename: 'data.myf'
|
|
||||||
})
|
|
||||||
// Automatically routes to MyFormatHandler!
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## BaseFormatHandler
|
|
||||||
|
|
||||||
All format handlers should extend `BaseFormatHandler`, which provides:
|
|
||||||
|
|
||||||
### MIME Type Detection
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
protected getMimeType(data: Buffer | string | { filename?: string }): string
|
|
||||||
```
|
|
||||||
|
|
||||||
Detects MIME type from filename or buffer. Uses Brainy's comprehensive MIME detection (2000+ types).
|
|
||||||
|
|
||||||
**Example:**
|
|
||||||
```typescript
|
|
||||||
const mimeType = this.getMimeType({ filename: 'data.dwg' })
|
|
||||||
// Returns: 'image/vnd.dwg'
|
|
||||||
```
|
|
||||||
|
|
||||||
### MIME Type Matching
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
protected mimeTypeMatches(mimeType: string, patterns: string[]): boolean
|
|
||||||
```
|
|
||||||
|
|
||||||
Checks if MIME type matches patterns. Supports wildcards (`image/*`).
|
|
||||||
|
|
||||||
**Example:**
|
|
||||||
```typescript
|
|
||||||
if (this.mimeTypeMatches(mimeType, ['image/*', 'video/*'])) {
|
|
||||||
// Handle all images and videos
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Extension Detection
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
protected detectExtension(data: string | Buffer | { filename?: string; ext?: string }): string | null
|
|
||||||
```
|
|
||||||
|
|
||||||
Extracts file extension for fallback detection.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## canHandle() Method
|
|
||||||
|
|
||||||
The `canHandle()` method determines if your handler can process the given data.
|
|
||||||
|
|
||||||
### Strategy 1: MIME Type Detection (Recommended)
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
canHandle(data: Buffer | string | { filename?: string; ext?: string }): boolean {
|
|
||||||
if (typeof data === 'object' && 'filename' in data) {
|
|
||||||
const mimeType = this.getMimeType(data)
|
|
||||||
return this.mimeTypeMatches(mimeType, [
|
|
||||||
'application/x-myformat',
|
|
||||||
'application/myformat'
|
|
||||||
])
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
✅ **Pros:** Automatic, comprehensive, works with 2000+ types
|
|
||||||
❌ **Cons:** Requires filename
|
|
||||||
|
|
||||||
### Strategy 2: Magic Byte Detection
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
canHandle(data: Buffer | string | { filename?: string; ext?: string }): boolean {
|
|
||||||
if (Buffer.isBuffer(data)) {
|
|
||||||
// Check magic bytes
|
|
||||||
return (
|
|
||||||
data[0] === 0x50 && // 'P'
|
|
||||||
data[1] === 0x4B && // 'K'
|
|
||||||
data[2] === 0x03 &&
|
|
||||||
data[3] === 0x04 // ZIP signature
|
|
||||||
)
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
✅ **Pros:** Works without filename, robust
|
|
||||||
❌ **Cons:** Requires knowledge of format structure
|
|
||||||
|
|
||||||
### Strategy 3: Combined Approach
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
canHandle(data: Buffer | string | { filename?: string; ext?: string }): boolean {
|
|
||||||
// Try MIME type first
|
|
||||||
if (typeof data === 'object' && 'filename' in data) {
|
|
||||||
const mimeType = this.getMimeType(data)
|
|
||||||
if (this.mimeTypeMatches(mimeType, ['application/x-myformat'])) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback to magic bytes
|
|
||||||
if (Buffer.isBuffer(data)) {
|
|
||||||
return this.checkMagicBytes(data)
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
✅ **Pros:** Robust, works in all scenarios
|
|
||||||
❌ **Cons:** More complex
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## process() Method
|
|
||||||
|
|
||||||
The `process()` method extracts structured data from the file.
|
|
||||||
|
|
||||||
### Return Format
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
interface ProcessedData {
|
|
||||||
/** Format identifier */
|
|
||||||
format: string
|
|
||||||
|
|
||||||
/** Array of extracted entities */
|
|
||||||
data: Array<Record<string, any>>
|
|
||||||
|
|
||||||
/** Metadata about processing */
|
|
||||||
metadata: {
|
|
||||||
rowCount: number
|
|
||||||
fields: string[]
|
|
||||||
processingTime: number
|
|
||||||
[key: string]: any
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Original filename (optional) */
|
|
||||||
filename?: string
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Example: CAD File Handler
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
async process(data: Buffer | string, options: FormatHandlerOptions): Promise<ProcessedData> {
|
|
||||||
const startTime = Date.now()
|
|
||||||
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data)
|
|
||||||
|
|
||||||
// Parse CAD file
|
|
||||||
const cad = await this.parseCAD(buffer)
|
|
||||||
|
|
||||||
// Extract entities
|
|
||||||
const entities = []
|
|
||||||
|
|
||||||
// Main CAD document
|
|
||||||
entities.push({
|
|
||||||
type: 'CADDocument',
|
|
||||||
filename: options.filename,
|
|
||||||
units: cad.units,
|
|
||||||
bounds: cad.bounds
|
|
||||||
})
|
|
||||||
|
|
||||||
// Layers
|
|
||||||
for (const layer of cad.layers) {
|
|
||||||
entities.push({
|
|
||||||
type: 'CADLayer',
|
|
||||||
name: layer.name,
|
|
||||||
color: layer.color,
|
|
||||||
visible: layer.visible,
|
|
||||||
objectCount: layer.objects.length
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Objects
|
|
||||||
for (const obj of cad.objects) {
|
|
||||||
entities.push({
|
|
||||||
type: 'CADObject',
|
|
||||||
objectType: obj.type,
|
|
||||||
layer: obj.layer,
|
|
||||||
geometry: obj.geometry,
|
|
||||||
properties: obj.properties
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
format: 'cad',
|
|
||||||
data: entities,
|
|
||||||
metadata: {
|
|
||||||
rowCount: entities.length,
|
|
||||||
fields: ['type', 'name', 'geometry', 'properties'],
|
|
||||||
processingTime: Date.now() - startTime,
|
|
||||||
layerCount: cad.layers.length,
|
|
||||||
objectCount: cad.objects.length,
|
|
||||||
units: cad.units
|
|
||||||
},
|
|
||||||
filename: options.filename
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Best Practices
|
|
||||||
|
|
||||||
1. **Always track processing time:**
|
|
||||||
```typescript
|
|
||||||
const startTime = Date.now()
|
|
||||||
// ... processing ...
|
|
||||||
metadata.processingTime = Date.now() - startTime
|
|
||||||
```
|
|
||||||
|
|
||||||
2. **Include rich metadata:**
|
|
||||||
```typescript
|
|
||||||
metadata: {
|
|
||||||
rowCount: entities.length,
|
|
||||||
fields: ['type', 'name', ...],
|
|
||||||
processingTime: 123,
|
|
||||||
// Format-specific metadata
|
|
||||||
layerCount: 5,
|
|
||||||
objectCount: 150,
|
|
||||||
version: '2.0'
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
3. **Handle errors gracefully:**
|
|
||||||
```typescript
|
|
||||||
try {
|
|
||||||
const parsed = this.parse(buffer)
|
|
||||||
return { format: 'myformat', data: parsed, ... }
|
|
||||||
} catch (error) {
|
|
||||||
throw new Error(
|
|
||||||
`Failed to parse myformat: ${error instanceof Error ? error.message : String(error)}`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
4. **Support progress reporting (optional):**
|
|
||||||
```typescript
|
|
||||||
if (options.progressHooks?.onCurrentItem) {
|
|
||||||
options.progressHooks.onCurrentItem(`Processing layer ${i}/${total}`)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## FormatHandlerRegistry
|
|
||||||
|
|
||||||
### Global Registry
|
|
||||||
|
|
||||||
Use the global registry for application-wide handlers:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { globalHandlerRegistry } from '@soulcraft/brainy/augmentations/intelligentImport'
|
|
||||||
|
|
||||||
globalHandlerRegistry.registerHandler({
|
|
||||||
name: 'myformat',
|
|
||||||
mimeTypes: ['application/x-myformat'],
|
|
||||||
extensions: ['.myf'],
|
|
||||||
loader: async () => new MyFormatHandler()
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Local Registry
|
|
||||||
|
|
||||||
Create a local registry for scoped handlers:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { FormatHandlerRegistry } from '@soulcraft/brainy/augmentations/intelligentImport'
|
|
||||||
|
|
||||||
const registry = new FormatHandlerRegistry()
|
|
||||||
registry.registerHandler({ ... })
|
|
||||||
```
|
|
||||||
|
|
||||||
### Lazy Loading
|
|
||||||
|
|
||||||
Handlers are lazy-loaded for performance:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
globalHandlerRegistry.registerHandler({
|
|
||||||
name: 'heavy',
|
|
||||||
mimeTypes: ['application/x-heavy'],
|
|
||||||
extensions: ['.heavy'],
|
|
||||||
loader: async () => {
|
|
||||||
// Only loaded when first needed
|
|
||||||
const { HeavyHandler } = await import('./HeavyHandler.js')
|
|
||||||
return new HeavyHandler()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Getting Handlers
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// By filename (automatic MIME detection)
|
|
||||||
const handler = await registry.getHandler('data.myf')
|
|
||||||
|
|
||||||
// By MIME type
|
|
||||||
const handler = await registry.getHandlerByMimeType('application/x-myformat')
|
|
||||||
|
|
||||||
// By extension
|
|
||||||
const handler = await registry.getHandlerByExtension('.myf')
|
|
||||||
|
|
||||||
// By name
|
|
||||||
const handler = await registry.getHandlerByName('myformat')
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Real-World Examples
|
|
||||||
|
|
||||||
### Example 1: Video Metadata Extractor
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { BaseFormatHandler } from '@soulcraft/brainy/augmentations/intelligentImport'
|
|
||||||
import ffmpeg from 'fluent-ffmpeg'
|
|
||||||
|
|
||||||
class VideoHandler extends BaseFormatHandler {
|
|
||||||
readonly format = 'video'
|
|
||||||
|
|
||||||
canHandle(data) {
|
|
||||||
const mimeType = this.getMimeType(data)
|
|
||||||
return this.mimeTypeMatches(mimeType, ['video/*'])
|
|
||||||
}
|
|
||||||
|
|
||||||
async process(data, options) {
|
|
||||||
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data)
|
|
||||||
|
|
||||||
// Extract video metadata with ffmpeg
|
|
||||||
const metadata = await this.extractVideoMetadata(buffer)
|
|
||||||
|
|
||||||
return {
|
|
||||||
format: 'video',
|
|
||||||
data: [{
|
|
||||||
type: 'Video',
|
|
||||||
duration: metadata.duration,
|
|
||||||
codec: metadata.codec,
|
|
||||||
resolution: metadata.resolution,
|
|
||||||
frameRate: metadata.frameRate,
|
|
||||||
bitrate: metadata.bitrate,
|
|
||||||
audioTracks: metadata.audioTracks,
|
|
||||||
subtitles: metadata.subtitles
|
|
||||||
}],
|
|
||||||
metadata: {
|
|
||||||
rowCount: 1,
|
|
||||||
fields: ['type', 'duration', 'codec', 'resolution'],
|
|
||||||
processingTime: metadata.processingTime
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async extractVideoMetadata(buffer: Buffer) {
|
|
||||||
// Use ffmpeg to extract metadata
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
ffmpeg(buffer)
|
|
||||||
.ffprobe((err, data) => {
|
|
||||||
if (err) reject(err)
|
|
||||||
else resolve(this.parseFFmpegOutput(data))
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Example 2: Git Repository Parser
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { BaseFormatHandler } from '@soulcraft/brainy/augmentations/intelligentImport'
|
|
||||||
import { simpleGit } from 'simple-git'
|
|
||||||
|
|
||||||
class GitRepoHandler extends BaseFormatHandler {
|
|
||||||
readonly format = 'git-repo'
|
|
||||||
|
|
||||||
canHandle(data) {
|
|
||||||
// Check for .git directory
|
|
||||||
if (typeof data === 'object' && 'filename' in data) {
|
|
||||||
return data.filename?.includes('.git') || false
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
async process(data, options) {
|
|
||||||
const repoPath = options.filename || ''
|
|
||||||
const git = simpleGit(repoPath)
|
|
||||||
|
|
||||||
// Extract commits
|
|
||||||
const log = await git.log()
|
|
||||||
const commits = log.all.map(commit => ({
|
|
||||||
type: 'GitCommit',
|
|
||||||
hash: commit.hash,
|
|
||||||
message: commit.message,
|
|
||||||
author: commit.author_name,
|
|
||||||
date: commit.date
|
|
||||||
}))
|
|
||||||
|
|
||||||
// Extract branches
|
|
||||||
const branchSummary = await git.branchLocal()
|
|
||||||
const branches = Object.keys(branchSummary.branches).map(name => ({
|
|
||||||
type: 'GitBranch',
|
|
||||||
name,
|
|
||||||
current: branchSummary.current === name
|
|
||||||
}))
|
|
||||||
|
|
||||||
return {
|
|
||||||
format: 'git-repo',
|
|
||||||
data: [...commits, ...branches],
|
|
||||||
metadata: {
|
|
||||||
rowCount: commits.length + branches.length,
|
|
||||||
fields: ['type', 'hash', 'message', 'author'],
|
|
||||||
processingTime: Date.now() - startTime,
|
|
||||||
commitCount: commits.length,
|
|
||||||
branchCount: branches.length
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Example 3: Database Schema Importer
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { BaseFormatHandler } from '@soulcraft/brainy/augmentations/intelligentImport'
|
|
||||||
import { Client } from 'pg'
|
|
||||||
|
|
||||||
class PostgreSQLSchemaHandler extends BaseFormatHandler {
|
|
||||||
readonly format = 'postgres-schema'
|
|
||||||
|
|
||||||
canHandle(data) {
|
|
||||||
if (typeof data === 'object' && 'filename' in data) {
|
|
||||||
return data.filename?.endsWith('.sql') || false
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
async process(data, options) {
|
|
||||||
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data)
|
|
||||||
const sql = buffer.toString('utf-8')
|
|
||||||
|
|
||||||
// Parse SQL or connect to database
|
|
||||||
const schema = await this.parseSchema(sql)
|
|
||||||
|
|
||||||
const entities = []
|
|
||||||
|
|
||||||
// Tables
|
|
||||||
for (const table of schema.tables) {
|
|
||||||
entities.push({
|
|
||||||
type: 'Table',
|
|
||||||
name: table.name,
|
|
||||||
schema: table.schema,
|
|
||||||
columnCount: table.columns.length
|
|
||||||
})
|
|
||||||
|
|
||||||
// Columns
|
|
||||||
for (const column of table.columns) {
|
|
||||||
entities.push({
|
|
||||||
type: 'Column',
|
|
||||||
name: column.name,
|
|
||||||
table: table.name,
|
|
||||||
dataType: column.dataType,
|
|
||||||
nullable: column.nullable,
|
|
||||||
primaryKey: column.primaryKey
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Foreign keys
|
|
||||||
for (const fk of schema.foreignKeys) {
|
|
||||||
entities.push({
|
|
||||||
type: 'ForeignKey',
|
|
||||||
from: `${fk.fromTable}.${fk.fromColumn}`,
|
|
||||||
to: `${fk.toTable}.${fk.toColumn}`
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
format: 'postgres-schema',
|
|
||||||
data: entities,
|
|
||||||
metadata: {
|
|
||||||
rowCount: entities.length,
|
|
||||||
fields: ['type', 'name', 'table', 'dataType'],
|
|
||||||
processingTime: Date.now() - startTime,
|
|
||||||
tableCount: schema.tables.length,
|
|
||||||
columnCount: schema.tables.reduce((sum, t) => sum + t.columns.length, 0),
|
|
||||||
foreignKeyCount: schema.foreignKeys.length
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Creating Premium Augmentations
|
|
||||||
|
|
||||||
Package your handler as a premium augmentation:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// @yourcompany/brainy-cad-importer
|
|
||||||
|
|
||||||
import { BaseAugmentation } from '@soulcraft/brainy'
|
|
||||||
import { CADHandler } from './CADHandler.js'
|
|
||||||
|
|
||||||
export class CADImportAugmentation extends BaseAugmentation {
|
|
||||||
readonly name = 'cad-import'
|
|
||||||
readonly timing = 'before'
|
|
||||||
readonly operations = ['import', 'importFile']
|
|
||||||
|
|
||||||
private handler: CADHandler
|
|
||||||
|
|
||||||
constructor(config = {}) {
|
|
||||||
super(config)
|
|
||||||
this.handler = new CADHandler()
|
|
||||||
}
|
|
||||||
|
|
||||||
async execute(operation, params, next) {
|
|
||||||
// Check if this is a CAD file
|
|
||||||
if (this.isCADFile(params)) {
|
|
||||||
const processed = await this.handler.process(params.data, params.options)
|
|
||||||
params.data = processed.data
|
|
||||||
params.metadata = { ...params.metadata, ...processed.metadata }
|
|
||||||
}
|
|
||||||
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
|
|
||||||
private isCADFile(params: any): boolean {
|
|
||||||
return this.handler.canHandle(params.data || params)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Usage:
|
|
||||||
// npm install @yourcompany/brainy-cad-importer
|
|
||||||
// brain.addAugmentation(new CADImportAugmentation())
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Testing
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { describe, it, expect } from 'vitest'
|
|
||||||
import { MyFormatHandler } from './MyFormatHandler.js'
|
|
||||||
|
|
||||||
describe('MyFormatHandler', () => {
|
|
||||||
let handler: MyFormatHandler
|
|
||||||
|
|
||||||
beforeEach(() => {
|
|
||||||
handler = new MyFormatHandler()
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('canHandle', () => {
|
|
||||||
it('should handle .myf files', () => {
|
|
||||||
expect(handler.canHandle({ filename: 'data.myf' })).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle by MIME type', () => {
|
|
||||||
expect(handler.canHandle({ filename: 'data.myformat' })).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should reject non-myformat files', () => {
|
|
||||||
expect(handler.canHandle({ filename: 'data.txt' })).toBe(false)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('process', () => {
|
|
||||||
it('should extract structured data', async () => {
|
|
||||||
const testData = Buffer.from('MY format data')
|
|
||||||
|
|
||||||
const result = await handler.process(testData)
|
|
||||||
|
|
||||||
expect(result.format).toBe('myformat')
|
|
||||||
expect(result.data).toHaveLength(1)
|
|
||||||
expect(result.metadata.processingTime).toBeGreaterThan(0)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle errors gracefully', async () => {
|
|
||||||
const invalidData = Buffer.from('invalid')
|
|
||||||
|
|
||||||
await expect(handler.process(invalidData)).rejects.toThrow()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Best Practices
|
|
||||||
|
|
||||||
1. **Always extend BaseFormatHandler** - provides MIME detection and utilities
|
|
||||||
2. **Use MIME types for routing** - automatic, comprehensive, maintainable
|
|
||||||
3. **Lazy load heavy dependencies** - better performance
|
|
||||||
4. **Extract rich metadata** - make data queryable in knowledge graph
|
|
||||||
5. **Handle errors gracefully** - fail fast with clear error messages
|
|
||||||
6. **Test thoroughly** - test canHandle() and process() with real data
|
|
||||||
7. **Document your format** - explain what data is extracted and how
|
|
||||||
8. **Follow ProcessedData format** - ensures compatibility with Brainy
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## See Also
|
|
||||||
|
|
||||||
- [ImageHandler source](../../src/augmentations/intelligentImport/handlers/imageHandler.ts) - Reference implementation
|
|
||||||
- [BaseFormatHandler source](../../src/augmentations/intelligentImport/handlers/base.ts) - Base class
|
|
||||||
- [FormatHandlerRegistry source](../../src/augmentations/intelligentImport/FormatHandlerRegistry.ts) - Registry implementation
|
|
||||||
- [Augmentations Guide](./AUGMENTATIONS.md) - Creating augmentations
|
|
||||||
|
|
@ -1,204 +0,0 @@
|
||||||
# Brainy Augmentations
|
|
||||||
|
|
||||||
Augmentations are the core extensibility mechanism in Brainy. They allow you to modify, enhance, and extend Brainy's behavior without changing the core code.
|
|
||||||
|
|
||||||
## Core Principle: One Interface, Infinite Possibilities
|
|
||||||
|
|
||||||
Every augmentation implements the same simple `BrainyAugmentation` interface:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
interface BrainyAugmentation {
|
|
||||||
name: string
|
|
||||||
timing: 'before' | 'after' | 'around' | 'replace'
|
|
||||||
operations: string[]
|
|
||||||
priority: number
|
|
||||||
initialize(context): Promise<void>
|
|
||||||
execute(operation, params, next): Promise<any>
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
This single interface can handle EVERYTHING - from adding AI capabilities to exposing APIs to replacing storage backends.
|
|
||||||
|
|
||||||
## Available Augmentations
|
|
||||||
|
|
||||||
### 🧠 Data Processing
|
|
||||||
Augmentations that enhance how data is processed and stored.
|
|
||||||
|
|
||||||
| Augmentation | Description | Timing | Status |
|
|
||||||
|-------------|-------------|--------|--------|
|
|
||||||
| **NeuralImportAugmentation** | AI-powered entity and relationship extraction | `before` | ✅ Production |
|
|
||||||
| **EntityRegistryAugmentation** | High-performance entity deduplication | `before` | ✅ Production |
|
|
||||||
| **BatchProcessingAugmentation** | Optimizes bulk operations | `around` | ✅ Production |
|
|
||||||
| **IntelligentVerbScoringAugmentation** | Learns relationship importance over time | `after` | ✅ Production |
|
|
||||||
|
|
||||||
### 🔌 External Connections (Synapses)
|
|
||||||
Connect Brainy to external services and data sources.
|
|
||||||
|
|
||||||
| Augmentation | Description | Timing | Status |
|
|
||||||
|-------------|-------------|--------|--------|
|
|
||||||
| **NotionSynapse** | Sync with Notion databases | `after` | 📝 Example |
|
|
||||||
| **SalesforceSynapse** | Connect to Salesforce CRM | `after` | 📝 Example |
|
|
||||||
| **SlackSynapse** | Import Slack conversations | `after` | 📝 Example |
|
|
||||||
| **GoogleDriveSynapse** | Sync Google Drive documents | `after` | 📝 Example |
|
|
||||||
|
|
||||||
### 🌐 API Exposure
|
|
||||||
Expose Brainy through various protocols.
|
|
||||||
|
|
||||||
| Augmentation | Description | Timing | Status |
|
|
||||||
|-------------|-------------|--------|--------|
|
|
||||||
| **APIServerAugmentation** | REST, WebSocket, and MCP server | `after` | ✅ Production |
|
|
||||||
| **GraphQLAugmentation** | GraphQL API endpoint | `after` | 🚧 Planned |
|
|
||||||
| **gRPCAugmentation** | gRPC service | `after` | 🚧 Planned |
|
|
||||||
|
|
||||||
### 💾 Storage Backends
|
|
||||||
Replace or enhance the storage layer.
|
|
||||||
|
|
||||||
| Augmentation | Description | Timing | Status |
|
|
||||||
|-------------|-------------|--------|--------|
|
|
||||||
| **S3StorageAugmentation** | Use S3 as storage backend | `replace` | 📝 Example |
|
|
||||||
| **RedisAugmentation** | Redis caching layer | `around` | 📝 Example |
|
|
||||||
| **PostgresAugmentation** | PostgreSQL persistence | `replace` | 📝 Example |
|
|
||||||
|
|
||||||
### 🔄 Real-time & Sync
|
|
||||||
Handle real-time updates and synchronization.
|
|
||||||
|
|
||||||
| Augmentation | Description | Timing | Status |
|
|
||||||
|-------------|-------------|--------|--------|
|
|
||||||
| **WebSocketConduitAugmentation** | WebSocket client connections | `after` | ⚠️ Legacy |
|
|
||||||
| **ServerSearchAugmentation** | Connect to remote Brainy servers | `after` | ⚠️ Legacy |
|
|
||||||
| **TeamCoordinationAugmentation** | Multi-agent synchronization | `after` | 📝 Example |
|
|
||||||
|
|
||||||
### 🛡️ Infrastructure
|
|
||||||
Core infrastructure and reliability features.
|
|
||||||
|
|
||||||
| Augmentation | Description | Timing | Status |
|
|
||||||
|-------------|-------------|--------|--------|
|
|
||||||
| **ConnectionPoolAugmentation** | Optimize cloud storage connections | `before` | ✅ Production |
|
|
||||||
| **RequestDeduplicatorAugmentation** | Prevent duplicate concurrent requests | `before` | ✅ Production |
|
|
||||||
| **TransactionAugmentation** | ACID transaction support | `around` | 🚧 Planned |
|
|
||||||
| **CacheAugmentation** | Multi-level caching | `around` | ✅ Production |
|
|
||||||
|
|
||||||
### 📊 Monitoring & Analytics
|
|
||||||
Track and analyze Brainy's behavior.
|
|
||||||
|
|
||||||
| Augmentation | Description | Timing | Status |
|
|
||||||
|-------------|-------------|--------|--------|
|
|
||||||
| **MetricsAugmentation** | Prometheus metrics | `after` | 📝 Example |
|
|
||||||
| **LoggingAugmentation** | Structured logging | `after` | 📝 Example |
|
|
||||||
| **TracingAugmentation** | Distributed tracing | `around` | 🚧 Planned |
|
|
||||||
|
|
||||||
### 🤖 AI & Chat
|
|
||||||
AI-powered interfaces and chat capabilities.
|
|
||||||
|
|
||||||
| Augmentation | Description | Timing | Status |
|
|
||||||
|-------------|-------------|--------|--------|
|
|
||||||
| **ChatInterfaceAugmentation** | Natural language interface | `before` | 📝 Example |
|
|
||||||
| **MCPAgentMemoryAugmentation** | AI agent memory via MCP | `after` | 📝 Example |
|
|
||||||
| **LLMQueryAugmentation** | LLM-enhanced queries | `before` | 📝 Example |
|
|
||||||
|
|
||||||
### 📈 Visualization
|
|
||||||
Visual representations of data.
|
|
||||||
|
|
||||||
| Augmentation | Description | Timing | Status |
|
|
||||||
|-------------|-------------|--------|--------|
|
|
||||||
| **GraphVisualizationAugmentation** | Real-time graph visualization | `after` | 📝 Example |
|
|
||||||
| **DashboardAugmentation** | Web-based dashboard | `after` | 🚧 Planned |
|
|
||||||
|
|
||||||
## Status Legend
|
|
||||||
|
|
||||||
- ✅ **Production**: Fully implemented and tested
|
|
||||||
- 📝 **Example**: Example implementation available
|
|
||||||
- 🚧 **Planned**: On the roadmap
|
|
||||||
- ⚠️ **Legacy**: Being replaced by newer augmentations
|
|
||||||
|
|
||||||
## Using Augmentations
|
|
||||||
|
|
||||||
### Zero-Config Approach
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const brain = new Brainy()
|
|
||||||
|
|
||||||
// Just register augmentations - they work automatically!
|
|
||||||
brain.augmentations.register(new EntityRegistryAugmentation())
|
|
||||||
brain.augmentations.register(new APIServerAugmentation())
|
|
||||||
|
|
||||||
await brain.init()
|
|
||||||
```
|
|
||||||
|
|
||||||
### With Configuration
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const brain = new Brainy()
|
|
||||||
|
|
||||||
brain.augmentations.register(
|
|
||||||
new APIServerAugmentation({
|
|
||||||
port: 8080,
|
|
||||||
auth: { required: true }
|
|
||||||
})
|
|
||||||
)
|
|
||||||
|
|
||||||
await brain.init()
|
|
||||||
```
|
|
||||||
|
|
||||||
## Creating Custom Augmentations
|
|
||||||
|
|
||||||
See [Creating Custom Augmentations](./creating-augmentations.md) for a complete guide.
|
|
||||||
|
|
||||||
Quick example:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class MyAugmentation extends BaseAugmentation {
|
|
||||||
readonly name = 'my-augmentation'
|
|
||||||
readonly timing = 'after'
|
|
||||||
readonly operations = ['add', 'search']
|
|
||||||
readonly priority = 50
|
|
||||||
|
|
||||||
async execute<T>(operation: string, params: any, next: () => Promise<T>): Promise<T> {
|
|
||||||
console.log(`Before ${operation}`)
|
|
||||||
const result = await next()
|
|
||||||
console.log(`After ${operation}`)
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Augmentation Timing
|
|
||||||
|
|
||||||
### `before`
|
|
||||||
Executes before the main operation. Used for:
|
|
||||||
- Input validation
|
|
||||||
- Data transformation
|
|
||||||
- Authentication checks
|
|
||||||
|
|
||||||
### `after`
|
|
||||||
Executes after the main operation. Used for:
|
|
||||||
- Broadcasting updates
|
|
||||||
- Syncing to external services
|
|
||||||
- Logging and metrics
|
|
||||||
|
|
||||||
### `around`
|
|
||||||
Wraps the main operation. Used for:
|
|
||||||
- Transactions
|
|
||||||
- Caching
|
|
||||||
- Error handling
|
|
||||||
|
|
||||||
### `replace`
|
|
||||||
Completely replaces the main operation. Used for:
|
|
||||||
- Alternative storage backends
|
|
||||||
- Mock implementations
|
|
||||||
- Proxy operations
|
|
||||||
|
|
||||||
## Priority System
|
|
||||||
|
|
||||||
Higher numbers execute first:
|
|
||||||
- **100**: Critical system operations
|
|
||||||
- **50**: Performance optimizations
|
|
||||||
- **10**: Enhancement features
|
|
||||||
- **1**: Optional features
|
|
||||||
|
|
||||||
## Related Documentation
|
|
||||||
|
|
||||||
- [API Server Augmentation](./api-server.md) - Complete API server documentation
|
|
||||||
- [Creating Augmentations](./creating-augmentations.md) - How to build your own
|
|
||||||
- [Augmentation Examples](../AUGMENTATION-EXAMPLES.md) - Real-world examples
|
|
||||||
- [Architecture Overview](../COMPLETE-ARCHITECTURE-VISION.md) - System architecture
|
|
||||||
|
|
@ -1,429 +0,0 @@
|
||||||
# API Server Augmentation
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
The `APIServerAugmentation` is a powerful augmentation that exposes your Brainy instance through REST, WebSocket, and MCP (Model Context Protocol) APIs. It transforms Brainy into a full-featured API server with zero configuration required.
|
|
||||||
|
|
||||||
## Features
|
|
||||||
|
|
||||||
### 🌐 REST API
|
|
||||||
Complete CRUD operations and advanced queries through HTTP endpoints.
|
|
||||||
|
|
||||||
### 🔌 WebSocket Server
|
|
||||||
Real-time bidirectional communication with automatic operation broadcasting.
|
|
||||||
|
|
||||||
### 🧠 MCP Integration
|
|
||||||
Built-in Model Context Protocol support for AI agent communication.
|
|
||||||
|
|
||||||
### 📊 Operation Broadcasting
|
|
||||||
Automatically broadcasts all Brainy operations to subscribed WebSocket clients.
|
|
||||||
|
|
||||||
### 🔒 Optional Security
|
|
||||||
Built-in authentication and rate limiting when needed.
|
|
||||||
|
|
||||||
## Installation
|
|
||||||
|
|
||||||
The APIServerAugmentation is included in Brainy core. No additional installation required.
|
|
||||||
|
|
||||||
**Bun** (recommended): No additional dependencies needed - use `Bun.serve()` directly.
|
|
||||||
|
|
||||||
**Node.js**: Install optional Express dependencies:
|
|
||||||
```bash
|
|
||||||
npm install express cors ws
|
|
||||||
```
|
|
||||||
|
|
||||||
## Zero-Config Usage
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { Brainy } from 'brainy'
|
|
||||||
import { APIServerAugmentation } from 'brainy/augmentations'
|
|
||||||
|
|
||||||
const brain = new Brainy()
|
|
||||||
|
|
||||||
// Register the API server augmentation
|
|
||||||
brain.augmentations.register(new APIServerAugmentation())
|
|
||||||
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Server is now running at http://localhost:3000
|
|
||||||
console.log('API Server ready!')
|
|
||||||
console.log('REST: http://localhost:3000/api/*')
|
|
||||||
console.log('WebSocket: ws://localhost:3000/ws')
|
|
||||||
console.log('MCP: http://localhost:3000/api/mcp')
|
|
||||||
```
|
|
||||||
|
|
||||||
## Configuration Options
|
|
||||||
|
|
||||||
While zero-config works great, you can customize the server:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const apiServer = new APIServerAugmentation({
|
|
||||||
enabled: true, // Enable/disable the server
|
|
||||||
port: 3000, // HTTP port
|
|
||||||
host: '0.0.0.0', // Bind address
|
|
||||||
|
|
||||||
cors: {
|
|
||||||
origin: '*', // CORS allowed origins
|
|
||||||
credentials: true // Allow credentials
|
|
||||||
},
|
|
||||||
|
|
||||||
auth: {
|
|
||||||
required: false, // Require authentication
|
|
||||||
apiKeys: [], // Valid API keys
|
|
||||||
bearerTokens: [] // Valid bearer tokens
|
|
||||||
},
|
|
||||||
|
|
||||||
rateLimit: {
|
|
||||||
windowMs: 60000, // Rate limit window (ms)
|
|
||||||
max: 100 // Max requests per window
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## REST API Endpoints
|
|
||||||
|
|
||||||
### Health Check
|
|
||||||
```http
|
|
||||||
GET /health
|
|
||||||
```
|
|
||||||
Returns server status and basic metrics.
|
|
||||||
|
|
||||||
### Search
|
|
||||||
```http
|
|
||||||
POST /api/search
|
|
||||||
Content-Type: application/json
|
|
||||||
|
|
||||||
{
|
|
||||||
"query": "search text",
|
|
||||||
"limit": 10,
|
|
||||||
"options": {}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Add Data
|
|
||||||
```http
|
|
||||||
POST /api/add
|
|
||||||
Content-Type: application/json
|
|
||||||
|
|
||||||
{
|
|
||||||
"content": "data to add",
|
|
||||||
"metadata": {
|
|
||||||
"key": "value"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Get by ID
|
|
||||||
```http
|
|
||||||
GET /api/get/:id
|
|
||||||
```
|
|
||||||
|
|
||||||
### Delete
|
|
||||||
```http
|
|
||||||
DELETE /api/delete/:id
|
|
||||||
```
|
|
||||||
|
|
||||||
### Create Relationship
|
|
||||||
```http
|
|
||||||
POST /api/relate
|
|
||||||
Content-Type: application/json
|
|
||||||
|
|
||||||
{
|
|
||||||
"source": "id1",
|
|
||||||
"target": "id2",
|
|
||||||
"verb": "relates_to",
|
|
||||||
"metadata": {}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Complex Queries
|
|
||||||
```http
|
|
||||||
POST /api/find
|
|
||||||
Content-Type: application/json
|
|
||||||
|
|
||||||
{
|
|
||||||
"where": { "type": "document" },
|
|
||||||
"like": "machine learning",
|
|
||||||
"limit": 10
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Clustering
|
|
||||||
```http
|
|
||||||
POST /api/cluster
|
|
||||||
Content-Type: application/json
|
|
||||||
|
|
||||||
{
|
|
||||||
"algorithm": "kmeans",
|
|
||||||
"options": {
|
|
||||||
"k": 5
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Statistics
|
|
||||||
```http
|
|
||||||
GET /api/stats
|
|
||||||
```
|
|
||||||
|
|
||||||
### Operation History
|
|
||||||
```http
|
|
||||||
GET /api/history
|
|
||||||
```
|
|
||||||
|
|
||||||
## WebSocket API
|
|
||||||
|
|
||||||
### Connection
|
|
||||||
```javascript
|
|
||||||
const ws = new WebSocket('ws://localhost:3000/ws')
|
|
||||||
|
|
||||||
ws.onopen = () => {
|
|
||||||
console.log('Connected to Brainy WebSocket')
|
|
||||||
}
|
|
||||||
|
|
||||||
ws.onmessage = (event) => {
|
|
||||||
const msg = JSON.parse(event.data)
|
|
||||||
console.log('Received:', msg)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Subscribe to Operations
|
|
||||||
```javascript
|
|
||||||
ws.send(JSON.stringify({
|
|
||||||
type: 'subscribe',
|
|
||||||
operations: ['all'] // or specific: ['add', 'search', 'delete']
|
|
||||||
}))
|
|
||||||
```
|
|
||||||
|
|
||||||
### Search via WebSocket
|
|
||||||
```javascript
|
|
||||||
ws.send(JSON.stringify({
|
|
||||||
type: 'search',
|
|
||||||
query: 'your search',
|
|
||||||
limit: 10,
|
|
||||||
requestId: 'unique-id'
|
|
||||||
}))
|
|
||||||
```
|
|
||||||
|
|
||||||
### Add Data via WebSocket
|
|
||||||
```javascript
|
|
||||||
ws.send(JSON.stringify({
|
|
||||||
type: 'add',
|
|
||||||
content: 'data to add',
|
|
||||||
metadata: {},
|
|
||||||
requestId: 'unique-id'
|
|
||||||
}))
|
|
||||||
```
|
|
||||||
|
|
||||||
### Operation Broadcasts
|
|
||||||
When subscribed, you'll receive real-time updates:
|
|
||||||
```javascript
|
|
||||||
{
|
|
||||||
"type": "operation",
|
|
||||||
"operation": "add",
|
|
||||||
"params": { /* sanitized parameters */ },
|
|
||||||
"timestamp": 1234567890,
|
|
||||||
"duration": 15
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## MCP (Model Context Protocol)
|
|
||||||
|
|
||||||
The MCP endpoint allows AI agents to interact with Brainy:
|
|
||||||
|
|
||||||
```http
|
|
||||||
POST /api/mcp
|
|
||||||
Content-Type: application/json
|
|
||||||
|
|
||||||
{
|
|
||||||
"method": "search",
|
|
||||||
"params": {
|
|
||||||
"query": "find documents about AI"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Authentication
|
|
||||||
|
|
||||||
When authentication is enabled:
|
|
||||||
|
|
||||||
### API Key
|
|
||||||
```http
|
|
||||||
GET /api/stats
|
|
||||||
X-API-Key: your-api-key
|
|
||||||
```
|
|
||||||
|
|
||||||
### Bearer Token
|
|
||||||
```http
|
|
||||||
GET /api/stats
|
|
||||||
Authorization: Bearer your-token
|
|
||||||
```
|
|
||||||
|
|
||||||
## Environment Support
|
|
||||||
|
|
||||||
### Bun ✅ (Recommended)
|
|
||||||
Native performance with Bun.serve() - no Express required. Works with `bun --compile` for single-binary deployment.
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Simple Bun server with Brainy
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
const brain = new Brainy()
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
Bun.serve({
|
|
||||||
port: 3000,
|
|
||||||
async fetch(req) {
|
|
||||||
const url = new URL(req.url)
|
|
||||||
if (url.pathname === '/api/search') {
|
|
||||||
const { query } = await req.json()
|
|
||||||
const results = await brain.search(query)
|
|
||||||
return Response.json(results)
|
|
||||||
}
|
|
||||||
return new Response('Not Found', { status: 404 })
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Node.js ✅
|
|
||||||
Full support with Express, WebSocket, and all features.
|
|
||||||
|
|
||||||
### Deno 🚧
|
|
||||||
Planned support using Deno.serve() or oak framework.
|
|
||||||
|
|
||||||
### Browser/Service Worker 🚧
|
|
||||||
Planned support for intercepting fetch() calls locally.
|
|
||||||
|
|
||||||
## How It Works
|
|
||||||
|
|
||||||
The APIServerAugmentation hooks into Brainy's augmentation pipeline:
|
|
||||||
|
|
||||||
1. **Timing**: Executes `after` operations complete
|
|
||||||
2. **Operations**: Monitors `all` operations
|
|
||||||
3. **Broadcasting**: Sends operation details to subscribed clients
|
|
||||||
4. **History**: Maintains operation history (last 1000 operations)
|
|
||||||
|
|
||||||
## Example: Multi-Client Sync
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Server
|
|
||||||
const brain = new Brainy()
|
|
||||||
brain.augmentations.register(new APIServerAugmentation())
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Client 1 - WebSocket subscriber
|
|
||||||
const ws1 = new WebSocket('ws://localhost:3000/ws')
|
|
||||||
ws1.onopen = () => {
|
|
||||||
ws1.send(JSON.stringify({
|
|
||||||
type: 'subscribe',
|
|
||||||
operations: ['add', 'delete']
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
ws1.onmessage = (e) => {
|
|
||||||
console.log('Client 1 received update:', JSON.parse(e.data))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Client 2 - REST API user
|
|
||||||
fetch('http://localhost:3000/api/add', {
|
|
||||||
method: 'POST',
|
|
||||||
headers: { 'Content-Type': 'application/json' },
|
|
||||||
body: JSON.stringify({
|
|
||||||
content: 'New data',
|
|
||||||
metadata: { source: 'client2' }
|
|
||||||
})
|
|
||||||
})
|
|
||||||
// Client 1 automatically receives notification!
|
|
||||||
```
|
|
||||||
|
|
||||||
## Performance Considerations
|
|
||||||
|
|
||||||
- **Operation History**: Limited to last 1000 operations
|
|
||||||
- **WebSocket Heartbeat**: Every 30 seconds
|
|
||||||
- **Client Timeout**: 60 seconds of inactivity
|
|
||||||
- **Parameter Sanitization**: Sensitive fields removed, large content truncated
|
|
||||||
- **Rate Limiting**: In-memory tracking (use Redis in production)
|
|
||||||
|
|
||||||
## Security Notes
|
|
||||||
|
|
||||||
1. **Default Configuration**: No auth, open CORS - suitable for development
|
|
||||||
2. **Production**: Enable auth, configure CORS, use HTTPS
|
|
||||||
3. **Sensitive Data**: Parameters are sanitized before broadcasting
|
|
||||||
4. **Rate Limiting**: Basic in-memory implementation included
|
|
||||||
|
|
||||||
## Comparison with Previous Implementations
|
|
||||||
|
|
||||||
The APIServerAugmentation unifies and replaces:
|
|
||||||
- `BrainyMCPBroadcast` - Node-specific WebSocket/HTTP server
|
|
||||||
- `WebSocketConduitAugmentation` - WebSocket client functionality
|
|
||||||
- `ServerSearchAugmentations` - Remote Brainy connections
|
|
||||||
|
|
||||||
Benefits of the unified approach:
|
|
||||||
- Single augmentation for all API needs
|
|
||||||
- Consistent interface across protocols
|
|
||||||
- Automatic operation broadcasting
|
|
||||||
- Environment-aware implementation
|
|
||||||
- Zero-configuration philosophy
|
|
||||||
|
|
||||||
## Advanced Usage
|
|
||||||
|
|
||||||
### Custom Operation Filtering
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
class FilteredAPIServer extends APIServerAugmentation {
|
|
||||||
shouldExecute(operation: string, params: any): boolean {
|
|
||||||
// Don't broadcast sensitive operations
|
|
||||||
if (operation === 'delete' && params.sensitive) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Integration with Other Augmentations
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const brain = new Brainy()
|
|
||||||
|
|
||||||
// Stack augmentations for complete system
|
|
||||||
brain.augmentations.register(new EntityRegistryAugmentation()) // Dedup
|
|
||||||
brain.augmentations.register(new APIServerAugmentation()) // API
|
|
||||||
|
|
||||||
await brain.init()
|
|
||||||
// All augmentations work together seamlessly!
|
|
||||||
```
|
|
||||||
|
|
||||||
## Troubleshooting
|
|
||||||
|
|
||||||
### Server won't start
|
|
||||||
- Check if port is already in use
|
|
||||||
- Verify Node.js dependencies are installed: `npm install express cors ws`
|
|
||||||
- Check console for error messages
|
|
||||||
|
|
||||||
### WebSocket connections drop
|
|
||||||
- Ensure heartbeat responses are handled
|
|
||||||
- Check for proxy/firewall issues
|
|
||||||
- Verify CORS configuration
|
|
||||||
|
|
||||||
### Authentication not working
|
|
||||||
- Ensure `auth.required` is set to `true`
|
|
||||||
- Verify API keys or bearer tokens are correctly configured
|
|
||||||
- Check request headers are properly formatted
|
|
||||||
|
|
||||||
## Future Enhancements
|
|
||||||
|
|
||||||
- [ ] Deno server implementation
|
|
||||||
- [ ] Service Worker implementation
|
|
||||||
- [ ] GraphQL endpoint
|
|
||||||
- [ ] gRPC support
|
|
||||||
- [ ] Built-in SSL/TLS
|
|
||||||
- [ ] Redis-based rate limiting
|
|
||||||
- [ ] Prometheus metrics endpoint
|
|
||||||
- [ ] OpenAPI/Swagger documentation
|
|
||||||
|
|
||||||
## Related Documentation
|
|
||||||
|
|
||||||
- [Augmentation System Overview](../AUGMENTATION-SYSTEM.md)
|
|
||||||
- [BrainyAugmentation Interface](./brainy-augmentation.md)
|
|
||||||
- [MCP Integration](../mcp/README.md)
|
|
||||||
- [Zero-Config Philosophy](../ZERO-CONFIG.md)
|
|
||||||
386
docs/concepts/consistency-model.md
Normal file
386
docs/concepts/consistency-model.md
Normal file
|
|
@ -0,0 +1,386 @@
|
||||||
|
---
|
||||||
|
title: Consistency Model
|
||||||
|
slug: concepts/consistency-model
|
||||||
|
public: true
|
||||||
|
category: concepts
|
||||||
|
template: concept
|
||||||
|
order: 4
|
||||||
|
description: The exact guarantees behind Brainy's Db API — snapshot isolation, atomic transactions, two levels of compare-and-swap, time travel, retention, snapshots, crash recovery, and the reserved-field contract.
|
||||||
|
next:
|
||||||
|
- guides/snapshots-and-time-travel
|
||||||
|
- guides/optimistic-concurrency
|
||||||
|
---
|
||||||
|
|
||||||
|
# Consistency Model
|
||||||
|
|
||||||
|
Brainy 8.0's consistency story rests on one mechanism: **generational MVCC**
|
||||||
|
— multi-version concurrency control over immutable, generation-stamped
|
||||||
|
records. It is exposed through a single value type, the **`Db`**: an
|
||||||
|
immutable, point-in-time view of the whole store that you query like the
|
||||||
|
live brain.
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const db = brain.now() // pin the current state — O(1), no I/O
|
||||||
|
|
||||||
|
await brain.transact([
|
||||||
|
{ op: 'update', id: invoiceId, metadata: { status: 'paid' } }
|
||||||
|
])
|
||||||
|
|
||||||
|
await db.get(invoiceId) // still 'pending' — pinned, forever
|
||||||
|
await brain.get(invoiceId) // 'paid' — live
|
||||||
|
await db.release() // unpin when done
|
||||||
|
```
|
||||||
|
|
||||||
|
This page states the guarantees precisely — what is promised, what it costs,
|
||||||
|
and where the honest limits are. The design record is
|
||||||
|
[ADR-001](../ADR-001-generational-mvcc.md); every guarantee below is proven
|
||||||
|
by a dedicated test in `tests/integration/db-mvcc.test.ts`.
|
||||||
|
|
||||||
|
## The generation clock
|
||||||
|
|
||||||
|
A **monotonic generation counter** is the store's logical clock:
|
||||||
|
|
||||||
|
- It advances **once per committed `transact()` batch** and once per
|
||||||
|
single-operation write (`add`/`update`/`remove`/`relate`/…).
|
||||||
|
- `brain.generation()` reads it; it is persisted in the data directory and
|
||||||
|
**never reissued** — not across restarts, and not across `restore()`
|
||||||
|
(the counter is floored at its pre-restore value).
|
||||||
|
|
||||||
|
Every `Db` is pinned at one generation. `db.generation` and `db.timestamp`
|
||||||
|
identify the view; `newerDb.since(olderDb)` returns exactly the entity and
|
||||||
|
relationship ids that committed transactions touched between two views.
|
||||||
|
|
||||||
|
## Snapshot isolation for reads
|
||||||
|
|
||||||
|
**Guarantee:** a `Db` reads exactly the state at its pinned generation, no
|
||||||
|
matter what commits afterwards — including deletes. There are no torn reads,
|
||||||
|
no partially applied batches, and no drift over time.
|
||||||
|
|
||||||
|
- `brain.now()` pins the current generation in O(1).
|
||||||
|
- `brain.transact()` returns a `Db` pinned at the freshly committed
|
||||||
|
generation.
|
||||||
|
- `brain.asOf(generation | Date | snapshotPath)` pins past state.
|
||||||
|
|
||||||
|
While nothing has committed past the pin, reads delegate to the live fast
|
||||||
|
paths — pinning is free until history actually moves. Once later
|
||||||
|
transactions commit, the view keeps serving the **full query surface** at
|
||||||
|
its generation (see "Reading the past" below).
|
||||||
|
|
||||||
|
Writers are never blocked by readers and readers never block writers: a
|
||||||
|
pinned view stays valid because nothing overwrites the immutable records it
|
||||||
|
resolves from (the LMDB reader-pin model).
|
||||||
|
|
||||||
|
## Transaction atomicity
|
||||||
|
|
||||||
|
`brain.transact(ops)` executes a declarative batch — `add`, `update`,
|
||||||
|
`remove`, `relate`, `unrelate` — **atomically as exactly one generation**:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const db = await brain.transact([
|
||||||
|
{ op: 'add', id: orderId, type: NounType.Document, subtype: 'order', data: 'Order #1042' },
|
||||||
|
{ op: 'add', id: itemId, type: NounType.Thing, subtype: 'line-item', data: 'Widget x3' },
|
||||||
|
{ op: 'relate', from: orderId, to: itemId, type: VerbType.Contains, subtype: 'order-line' }
|
||||||
|
], { meta: { author: 'order-service', requestId: 'req-9f2' } })
|
||||||
|
|
||||||
|
db.receipt.ids // resolved id per operation, in input order
|
||||||
|
```
|
||||||
|
|
||||||
|
Either every operation applies, or none do and the store is byte-identical
|
||||||
|
to its pre-transaction state. Operation semantics mirror the corresponding
|
||||||
|
single-operation methods — validation, subtype enforcement, relationship
|
||||||
|
deduplication, delete cascades — and later operations may reference ids
|
||||||
|
created earlier in the same batch.
|
||||||
|
|
||||||
|
**The commit point is one atomic rename.** The durability protocol:
|
||||||
|
|
||||||
|
1. Before-images of every touched id are staged into an immutable
|
||||||
|
generation directory and **fsynced**.
|
||||||
|
2. The batch executes through the transaction manager (which has its own
|
||||||
|
operation-level rollback for non-crash failures).
|
||||||
|
3. The store manifest is replaced via atomic temp-file rename and fsynced.
|
||||||
|
**The rename is the commit** — a generation is committed if and only if
|
||||||
|
the manifest says so.
|
||||||
|
|
||||||
|
**Crash recovery:** on the next open, any staged generation above the
|
||||||
|
manifest watermark is an uncommitted transaction; its before-images are
|
||||||
|
restored (idempotently — recovery can itself crash and rerun) and derived
|
||||||
|
indexes never observe the rolled-back state. A crash anywhere before the
|
||||||
|
rename rolls back to the exact pre-transaction bytes; a crash after it keeps
|
||||||
|
the transaction.
|
||||||
|
|
||||||
|
Transaction metadata (`meta`) is reified Datomic-style: recorded in an
|
||||||
|
append-only transaction log readable via `brain.transactionLog()` — audit
|
||||||
|
fields live in the database, not in commit messages.
|
||||||
|
|
||||||
|
## Two levels of compare-and-swap
|
||||||
|
|
||||||
|
Concurrent `transact()` calls commit serially (snapshot-isolated batches).
|
||||||
|
For lost-update protection across a read–modify–write cycle, Brainy offers
|
||||||
|
CAS at two granularities:
|
||||||
|
|
||||||
|
| Granularity | Mechanism | Conflict error | Use when |
|
||||||
|
|---|---|---|---|
|
||||||
|
| **Per entity** | `_rev` + `{ op: 'update', ifRev }` (also on `brain.update()`) | `RevisionConflictError` | "This entity must not have changed since I read it." |
|
||||||
|
| **Whole store** | `transact(ops, { ifAtGeneration })` | `GenerationConflictError` | "*Nothing* may have committed since I read." |
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const view = brain.now()
|
||||||
|
const order = await view.get(orderId)
|
||||||
|
|
||||||
|
try {
|
||||||
|
await brain.transact(
|
||||||
|
[{ op: 'update', id: orderId, metadata: { total: recompute(order) }, ifRev: order._rev }],
|
||||||
|
{ ifAtGeneration: view.generation }
|
||||||
|
)
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof GenerationConflictError) {
|
||||||
|
// Something committed since the pin — re-read and retry.
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
await view.release()
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
An `ifRev` conflict on any operation rejects the **whole batch**; an
|
||||||
|
`ifAtGeneration` conflict is detected before anything is staged. Both leave
|
||||||
|
the store untouched and the generation counter unchanged. See
|
||||||
|
[Optimistic concurrency with `_rev`](../guides/optimistic-concurrency.md)
|
||||||
|
for the per-entity pattern in depth.
|
||||||
|
|
||||||
|
## Reading the past
|
||||||
|
|
||||||
|
`brain.asOf()` accepts a generation number, a `Date` (resolved through the
|
||||||
|
transaction log to the newest generation committed at or before it), or a
|
||||||
|
snapshot directory path. Historical views serve the **full query surface**
|
||||||
|
— `get()`, `find()` in every mode, semantic search, graph traversal,
|
||||||
|
cursors, aggregation — through two complementary paths:
|
||||||
|
|
||||||
|
- **Record path** (free): `get()`, metadata-level `find()`, and
|
||||||
|
filter-based `related()` resolve directly through the immutable record
|
||||||
|
layer. Ids untouched since the pin still ride the live fast paths.
|
||||||
|
- **Index path** (paid once): index-accelerated queries — semantic/vector
|
||||||
|
search, graph traversal, cursors, aggregation — are served by an
|
||||||
|
**at-generation index materialization** built lazily on first use:
|
||||||
|
Brainy reconstructs in-memory indexes over the exact record set at that
|
||||||
|
generation. This costs O(n at the pinned generation) time and memory,
|
||||||
|
**once per `Db`**, cached until `release()`. That is the open-core price
|
||||||
|
of historical index queries, stated plainly.
|
||||||
|
|
||||||
|
A native index provider implementing the optional
|
||||||
|
`VersionedIndexProvider` plugin capability serves the same historical reads
|
||||||
|
from its retained index segments **without any rebuild** — the materializer
|
||||||
|
is the correctness baseline, the provider is the accelerator. Semantics are
|
||||||
|
identical on both paths.
|
||||||
|
|
||||||
|
### History granularity — the honest limit
|
||||||
|
|
||||||
|
Generation *records* are written per `transact()` batch only.
|
||||||
|
Single-operation writes (`add`/`update`/`remove`/`relate`/… outside
|
||||||
|
`transact()`) advance the generation counter — so watermarks and CAS stay
|
||||||
|
sound — but do **not** stage before-images: they remain visible through
|
||||||
|
earlier pins and are not reported by `db.since()`. Code that needs pinned
|
||||||
|
isolation across its own writes uses `transact()`. This is the documented
|
||||||
|
8.0 contract, not an accident.
|
||||||
|
|
||||||
|
## Speculative writes: `db.with()`
|
||||||
|
|
||||||
|
`db.with(ops)` returns a new `Db` whose reads see the operations applied
|
||||||
|
**in memory, on top of the view** — Datomic's `with`. Nothing touches disk,
|
||||||
|
the generation counter, or index providers:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const current = brain.now()
|
||||||
|
const whatIf = await current.with([
|
||||||
|
{ op: 'update', id: employeeId, metadata: { team: 'platform' } }
|
||||||
|
])
|
||||||
|
|
||||||
|
await whatIf.find({ where: { team: 'platform' } }) // sees the change
|
||||||
|
await brain.get(employeeId) // unchanged — nothing committed
|
||||||
|
```
|
||||||
|
|
||||||
|
**The one boundary:** overlay entities carry no embeddings (`with()` never
|
||||||
|
invokes the embedder), so index-accelerated queries and `persist()` on a
|
||||||
|
speculative view throw `SpeculativeOverlayError` rather than returning
|
||||||
|
silently incomplete results. `get()`, metadata-filter `find()`, and
|
||||||
|
filter-based `related()` work fully on overlays. To get the full surface,
|
||||||
|
commit the same operations with `brain.transact()`.
|
||||||
|
|
||||||
|
## Retention and compaction
|
||||||
|
|
||||||
|
Historical records cost disk space, so retention is explicit:
|
||||||
|
|
||||||
|
- Every live `Db` holds a refcounted **pin**; a record-set is never
|
||||||
|
reclaimed while any pin could need it — pinned reads stay correct across
|
||||||
|
compaction, always.
|
||||||
|
- The **`retention`** knob governs auto-compaction (on every `flush()`/
|
||||||
|
`close()`): unset → ADAPTIVE (disk/RAM-pressure, zero-config) · `'all'` →
|
||||||
|
unbounded · `{ maxGenerations?, maxAge?, maxBytes? }` → explicit caps.
|
||||||
|
`brain.compactHistory({ maxGenerations?, maxAge?, maxBytes? })` reclaims
|
||||||
|
manually on the same caps, and records the **horizon** — `asOf()` below it
|
||||||
|
throws `GenerationCompactedError`, explicitly, never partial data.
|
||||||
|
- To keep a state readable forever, `persist()` it first: snapshots are
|
||||||
|
self-contained and unaffected by compaction of the source store.
|
||||||
|
|
||||||
|
Release `Db` values you do not keep (including the ones `transact()`
|
||||||
|
returns). A `FinalizationRegistry` backstop releases leaked pins at garbage
|
||||||
|
collection, but explicit `release()` is what makes compaction
|
||||||
|
deterministic.
|
||||||
|
|
||||||
|
## Durability: snapshots and restore
|
||||||
|
|
||||||
|
`db.persist(path)` cuts a **self-contained snapshot** under the store's
|
||||||
|
commit mutex, so no commit or compaction can interleave. On filesystem
|
||||||
|
storage it is built from **hard links**: because every data file is
|
||||||
|
immutable-by-rename, linking is safe — the snapshot is created without
|
||||||
|
copying entity data, shares disk space with the source, and later writes to
|
||||||
|
the source can never alter it (rewrites swap inodes; the snapshot keeps the
|
||||||
|
old bytes). Cross-device targets fall back to byte copies; in-memory stores
|
||||||
|
serialize to the same directory layout, producing a real, durable store.
|
||||||
|
|
||||||
|
Two rules keep snapshots honest:
|
||||||
|
|
||||||
|
- `persist()` requires the view to still be the store's **latest**
|
||||||
|
generation (a snapshot captures current bytes); a view that history has
|
||||||
|
moved past throws `GenerationConflictError` instead of persisting the
|
||||||
|
wrong state.
|
||||||
|
- `brain.restore(path, { confirm: true })` replaces the store's entire
|
||||||
|
state from a snapshot via byte copy (never links — the snapshot stays
|
||||||
|
independent), rebuilds all indexes, and floors the generation counter so
|
||||||
|
observed generation numbers are never reissued. Live pins do not survive
|
||||||
|
a restore — release them first (a warning is logged when any exist).
|
||||||
|
|
||||||
|
`Brainy.load(path)` (or `brain.asOf(path)`) opens a snapshot as a
|
||||||
|
self-contained **read-only** store with the full query surface, including
|
||||||
|
vector search.
|
||||||
|
|
||||||
|
## Reserved fields
|
||||||
|
|
||||||
|
Some field names belong to Brainy, not to your metadata. They live at **top
|
||||||
|
level** on every entity and relationship, have dedicated write paths, and may
|
||||||
|
never appear inside a `metadata` bag:
|
||||||
|
|
||||||
|
| Entities (nouns) | Relationships (verbs) | Canonical write path |
|
||||||
|
|---|---|---|
|
||||||
|
| `noun` | `verb` | the `type` param of `add()` / `relate()` |
|
||||||
|
| `subtype` | `subtype` | the `subtype` param |
|
||||||
|
| `visibility` | `visibility` | the `visibility` param (`'public'` \| `'internal'`) |
|
||||||
|
| `confidence` | `confidence` | the `confidence` param |
|
||||||
|
| `weight` | `weight` | the `weight` param |
|
||||||
|
| `service` | `service` | the `service` param (fixed at create time) |
|
||||||
|
| `data` | `data` | the `data` param |
|
||||||
|
| `createdBy` | `createdBy` | the `createdBy` param of `add()` (system-managed on verbs) |
|
||||||
|
| `createdAt`, `updatedAt`, `_rev` | `createdAt`, `updatedAt`, `_rev` | system-managed (`ifRev` for CAS) |
|
||||||
|
|
||||||
|
The canonical machine-readable lists are exported as
|
||||||
|
`RESERVED_ENTITY_FIELDS` and `RESERVED_RELATION_FIELDS` (defined in
|
||||||
|
`src/types/reservedFields.ts`, the single source of truth). Three layers
|
||||||
|
enforce the contract:
|
||||||
|
|
||||||
|
1. **Compile time** — every `metadata` param (`add`, `update`, `relate`,
|
||||||
|
`updateRelation`, and the matching `transact()` operations) rejects a
|
||||||
|
literal reserved key as a TypeScript error.
|
||||||
|
2. **Write time** — untyped (JavaScript) callers that pass one anyway are
|
||||||
|
normalized: user-settable fields (`confidence`, `weight`, `subtype`, and
|
||||||
|
`service`/`createdBy` at create time) are remapped to their dedicated
|
||||||
|
param — **top-level wins** when both are supplied — and system-managed
|
||||||
|
fields are dropped with a one-shot warning naming the correct write path.
|
||||||
|
`update({ metadata: { confidence: 0.9 } })` therefore behaves exactly
|
||||||
|
like `update({ confidence: 0.9 })`.
|
||||||
|
3. **Read time** — every read path (`get`, `find`, `search`,
|
||||||
|
`related`, batch reads, and historical `asOf()` materialization)
|
||||||
|
surfaces reserved fields **only at top level**: `entity.metadata` and
|
||||||
|
`relation.metadata` contain only your custom fields, always.
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const id = await brain.add({
|
||||||
|
type: 'document', subtype: 'invoice',
|
||||||
|
data: 'Invoice #42', confidence: 0.95, // reserved → top-level params
|
||||||
|
metadata: { customer: 'acme', total: 129.5 } // custom fields only
|
||||||
|
})
|
||||||
|
|
||||||
|
const entity = await brain.get(id)
|
||||||
|
entity.confidence // 0.95 — top level
|
||||||
|
entity.metadata // { customer: 'acme', total: 129.5 }
|
||||||
|
```
|
||||||
|
|
||||||
|
### Visibility — `public` / `internal` / `system`
|
||||||
|
|
||||||
|
`visibility` is a reserved tier that controls whether an entity or relationship
|
||||||
|
surfaces on Brainy's **default** user-facing reads. The absence of the field is
|
||||||
|
exactly equivalent to `'public'`.
|
||||||
|
|
||||||
|
| Tier | Counted in `getNounCount()` / `stats()`? | Returned by default `find()` / `related()`? | Opt-in |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `'public'` (default, or field absent) | yes | yes | — |
|
||||||
|
| `'internal'` | no | no | `find({ includeInternal: true })` / `related({ includeInternal: true })` |
|
||||||
|
| `'system'` | no | no | `find({ includeSystem: true })` / `related({ includeSystem: true })` |
|
||||||
|
|
||||||
|
- **`'public'`** — normal data. Counted and returned everywhere. Stored lean:
|
||||||
|
the field is omitted on disk for public records, so existing data needs no
|
||||||
|
migration.
|
||||||
|
- **`'internal'`** — your app's own bookkeeping (audit trails, derived caches,
|
||||||
|
scratch entities) that should not pollute default queries, counts, or
|
||||||
|
`stats()`, yet must stay retrievable on demand. Set it via the `visibility`
|
||||||
|
param; read it back with the `includeInternal` opt-in.
|
||||||
|
- **`'system'`** — Brainy's own plumbing (for example the Virtual File System
|
||||||
|
root entity). Hidden everywhere by default — even when `includeInternal` is
|
||||||
|
set — and surfaced only with the explicit `includeSystem` opt-in. The
|
||||||
|
`'system'` tier is **not** part of the public `add()` / `relate()` param type
|
||||||
|
(`'public' | 'internal'`); only internal Brainy code assigns it.
|
||||||
|
|
||||||
|
The opt-ins are applied as a **hard candidate filter** — hidden entities are
|
||||||
|
removed before `limit` / `offset` are applied, so a default `find({ limit: 10 })`
|
||||||
|
always returns ten *visible* results when that many exist, never a short page.
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// App-internal scratch entity: present, retrievable, but out of the way.
|
||||||
|
await brain.add({ type: 'task', data: 'reindex job', visibility: 'internal' })
|
||||||
|
|
||||||
|
await brain.getNounCount() // unchanged — internal not counted
|
||||||
|
await brain.find({ type: 'task' }) // [] — hidden by default
|
||||||
|
await brain.find({ type: 'task', includeInternal: true }) // includes it
|
||||||
|
|
||||||
|
// A brand-new brain reports zero user entities even though the VFS root exists:
|
||||||
|
const fresh = new Brainy()
|
||||||
|
await fresh.init()
|
||||||
|
await fresh.getNounCount() // 0 — the root is visibility:'system'
|
||||||
|
```
|
||||||
|
|
||||||
|
> **Note (8.0):** the structural `Contains` edges the VFS creates between
|
||||||
|
> directories and files are left at the default (`public`) visibility for now —
|
||||||
|
> only the VFS *root entity* is `'system'`. Marking those edges system requires
|
||||||
|
> companion changes to VFS traversal and is out of scope for this change.
|
||||||
|
|
||||||
|
## What is not guaranteed
|
||||||
|
|
||||||
|
Stated plainly, so nothing surprises you in production:
|
||||||
|
|
||||||
|
- **Single-writer.** Brainy is a single-writer, many-reader database
|
||||||
|
([multi-process model](./multi-process.md)). Transactions are atomic
|
||||||
|
within one writer process — there is no distributed or cross-process
|
||||||
|
transaction coordination.
|
||||||
|
- **History granularity.** Every write is its own immutable generation —
|
||||||
|
`transact()` batches AND single-operation `add`/`update`/`remove`/`relate`.
|
||||||
|
A pin always freezes against later writes, and every write is addressable
|
||||||
|
via `asOf()`. (`transact()` groups several operations into ONE atomic
|
||||||
|
generation; durability of single-op history is batched via async
|
||||||
|
group-commit — a hard crash can lose only the last un-flushed window's
|
||||||
|
*history*, never live data.)
|
||||||
|
- **Compacted history is gone.** `asOf()` below the compaction horizon
|
||||||
|
fails explicitly; persist what you must keep.
|
||||||
|
- **Counter persistence is coalesced for single-operation writes.** Durable
|
||||||
|
artifacts (records, manifests, snapshots) always persist the counter at
|
||||||
|
their own commit points, so a crash inside the coalescing window can lose
|
||||||
|
only counter values nothing durable ever referenced.
|
||||||
|
- **Speculative overlays are metadata-only readers.** Index-accelerated
|
||||||
|
queries on `with()` views throw rather than guess.
|
||||||
|
|
||||||
|
## Where to go next
|
||||||
|
|
||||||
|
- [Snapshots & Time Travel](../guides/snapshots-and-time-travel.md) — the
|
||||||
|
recipes: backup, restore, time-travel debugging, what-if analysis, audit
|
||||||
|
trails.
|
||||||
|
- [Optimistic concurrency with `_rev`](../guides/optimistic-concurrency.md)
|
||||||
|
— the per-entity CAS pattern.
|
||||||
|
- [ADR-001: Generational MVCC](../ADR-001-generational-mvcc.md) — the full
|
||||||
|
design record, including the persisted layout and the proof table.
|
||||||
|
|
@ -105,7 +105,7 @@ coexists with whatever the writer is doing:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
const reader = await Brainy.openReadOnly({
|
const reader = await Brainy.openReadOnly({
|
||||||
storage: { type: 'filesystem', rootDirectory: '/data/brain' }
|
storage: { type: 'filesystem', path: '/data/brain' }
|
||||||
})
|
})
|
||||||
|
|
||||||
const stats = await reader.stats()
|
const stats = await reader.stats()
|
||||||
|
|
@ -119,7 +119,7 @@ you need fresher state, ask the writer to flush before opening:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
const reader = await Brainy.openReadOnly({
|
const reader = await Brainy.openReadOnly({
|
||||||
storage: { type: 'filesystem', rootDirectory: '/data/brain' }
|
storage: { type: 'filesystem', path: '/data/brain' }
|
||||||
})
|
})
|
||||||
|
|
||||||
const acked = await reader.requestFlush({ timeoutMs: 5000 })
|
const acked = await reader.requestFlush({ timeoutMs: 5000 })
|
||||||
|
|
@ -135,11 +135,12 @@ The CLI `brainy inspect` subcommands all do this for you by default
|
||||||
|
|
||||||
## What's not enforced (yet)
|
## What's not enforced (yet)
|
||||||
|
|
||||||
- **Cloud storage backends** (S3, GCS, R2, Azure) do not currently enforce
|
- **Non-filesystem backends** are out of scope in 8.0, which ships only the
|
||||||
multi-process locking. Two processes pointing at the same bucket can both
|
filesystem and memory adapters. A custom `BaseStorage` subclass that is not
|
||||||
succeed at `init()` in writer mode and clobber each other's writes. A
|
filesystem-backed does not enforce multi-process locking by default: two
|
||||||
best-effort warning is logged in writer mode against a non-filesystem
|
processes can both succeed at `init()` in writer mode and clobber each
|
||||||
backend. Lock semantics for cloud backends will land in a future release.
|
other's writes. A best-effort warning is logged in writer mode against a
|
||||||
|
non-filesystem backend.
|
||||||
- **Long-running readers** do not automatically pick up new Cortex segments
|
- **Long-running readers** do not automatically pick up new Cortex segments
|
||||||
the writer publishes. One-shot inspector calls re-open the store and see
|
the writer publishes. One-shot inspector calls re-open the store and see
|
||||||
fresh segments; a reader that stays open for hours sees its column store
|
fresh segments; a reader that stays open for hours sees its column store
|
||||||
|
|
|
||||||
|
|
@ -27,8 +27,8 @@ override, and the install-time failure modes that the defensive
|
||||||
```
|
```
|
||||||
BaseStorageAdapter (counts, batch ops, multi-tenancy hooks)
|
BaseStorageAdapter (counts, batch ops, multi-tenancy hooks)
|
||||||
↑
|
↑
|
||||||
BaseStorage (COW, type-statistics, lifecycle helpers,
|
BaseStorage (type-statistics, lifecycle helpers, generation
|
||||||
default no-op multi-process methods)
|
hooks, default no-op multi-process methods)
|
||||||
↑
|
↑
|
||||||
FileSystemStorage (real filesystem I/O, writer-lock implementation,
|
FileSystemStorage (real filesystem I/O, writer-lock implementation,
|
||||||
flush-request watcher, atomic writes)
|
flush-request watcher, atomic writes)
|
||||||
|
|
@ -75,7 +75,7 @@ That's the full ceremony for inheriting multi-process safety.
|
||||||
|
|
||||||
## When NOT to extend FileSystemStorage
|
## When NOT to extend FileSystemStorage
|
||||||
|
|
||||||
If your storage is **not filesystem-backed** (S3, GCS, R2, Azure, a custom
|
If your storage is **not filesystem-backed** (a custom
|
||||||
network backend), extend `BaseStorage` directly:
|
network backend), extend `BaseStorage` directly:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
|
|
|
||||||
|
|
@ -1,833 +0,0 @@
|
||||||
# Brainy Cloud Deployment Guide
|
|
||||||
|
|
||||||
This guide provides production-ready deployment configurations for Brainy using S3CompatibleStorage (preferred) or FileSystemStorage across major cloud platforms. All examples are verified against the actual Brainy codebase.
|
|
||||||
|
|
||||||
## 🆕 Production Features
|
|
||||||
|
|
||||||
**Cost Optimization at Scale:**
|
|
||||||
- **Lifecycle Policies**: Automatic tier transitions for 96% cost savings
|
|
||||||
- **Intelligent-Tiering**: S3 Intelligent-Tiering and GCS Autoclass support
|
|
||||||
- **Batch Operations**: Efficient bulk delete (1000 objects per request)
|
|
||||||
- **Compression**: Gzip compression for 60-80% storage savings
|
|
||||||
|
|
||||||
**Example Impact (500TB dataset):**
|
|
||||||
- Before: $138,000/year
|
|
||||||
- With lifecycle policies: $5,940/year
|
|
||||||
- **Savings: $132,060/year (96%)**
|
|
||||||
|
|
||||||
See the [Cost Optimization](#cost-optimization-v40) section below for implementation details.
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
The API Server augmentation provides a universal handler that works with standard Request/Response objects, making Brainy deployable on any JavaScript runtime.
|
|
||||||
|
|
||||||
## Storage Adapter
|
|
||||||
|
|
||||||
**S3CompatibleStorage** is the preferred storage adapter for cloud deployments. It works with:
|
|
||||||
- Amazon S3
|
|
||||||
- Cloudflare R2
|
|
||||||
- Google Cloud Storage
|
|
||||||
- Azure Blob Storage
|
|
||||||
- Any S3-compatible service
|
|
||||||
|
|
||||||
## Deployment Examples
|
|
||||||
|
|
||||||
### AWS Lambda
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// handler.js
|
|
||||||
const { Brainy } = require('@soulcraft/brainy')
|
|
||||||
const { S3CompatibleStorage } = require('@soulcraft/brainy/storage')
|
|
||||||
|
|
||||||
let brain
|
|
||||||
let handler
|
|
||||||
|
|
||||||
exports.handler = async (event) => {
|
|
||||||
if (!brain) {
|
|
||||||
const storage = new S3CompatibleStorage({
|
|
||||||
endpoint: 's3.amazonaws.com',
|
|
||||||
region: process.env.AWS_REGION,
|
|
||||||
bucket: process.env.BRAINY_BUCKET,
|
|
||||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
|
||||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
|
|
||||||
prefix: 'brainy-data/'
|
|
||||||
})
|
|
||||||
|
|
||||||
brain = new Brainy({
|
|
||||||
storage,
|
|
||||||
augmentations: [{
|
|
||||||
name: 'api-server',
|
|
||||||
config: {
|
|
||||||
enabled: true,
|
|
||||||
auth: {
|
|
||||||
required: true,
|
|
||||||
apiKeys: [process.env.API_KEY]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Get the universal handler from the augmentation
|
|
||||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
|
||||||
handler = apiAugmentation.createUniversalHandler()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Convert Lambda event to Request
|
|
||||||
const url = `https://${event.requestContext.domainName}${event.rawPath}`
|
|
||||||
const request = new Request(url, {
|
|
||||||
method: event.requestContext.http.method,
|
|
||||||
headers: event.headers,
|
|
||||||
body: event.body
|
|
||||||
})
|
|
||||||
|
|
||||||
// Use the universal handler
|
|
||||||
const response = await handler(request)
|
|
||||||
|
|
||||||
return {
|
|
||||||
statusCode: response.status,
|
|
||||||
headers: Object.fromEntries(response.headers),
|
|
||||||
body: await response.text()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Google Cloud Functions
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// index.js
|
|
||||||
const { Brainy } = require('@soulcraft/brainy')
|
|
||||||
const { S3CompatibleStorage } = require('@soulcraft/brainy/storage')
|
|
||||||
|
|
||||||
let brain
|
|
||||||
let handler
|
|
||||||
|
|
||||||
exports.brainyAPI = async (req, res) => {
|
|
||||||
if (!brain) {
|
|
||||||
const storage = new S3CompatibleStorage({
|
|
||||||
endpoint: 'storage.googleapis.com',
|
|
||||||
bucket: process.env.GCS_BUCKET,
|
|
||||||
accessKeyId: process.env.GCS_ACCESS_KEY,
|
|
||||||
secretAccessKey: process.env.GCS_SECRET_KEY,
|
|
||||||
prefix: 'brainy/',
|
|
||||||
forcePathStyle: false,
|
|
||||||
region: 'US'
|
|
||||||
})
|
|
||||||
|
|
||||||
brain = new Brainy({ storage })
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
|
||||||
handler = apiAugmentation.createUniversalHandler()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Convert Express req/res to Request/Response
|
|
||||||
const request = new Request(`https://${req.hostname}${req.originalUrl}`, {
|
|
||||||
method: req.method,
|
|
||||||
headers: req.headers,
|
|
||||||
body: JSON.stringify(req.body)
|
|
||||||
})
|
|
||||||
|
|
||||||
const response = await handler(request)
|
|
||||||
|
|
||||||
res.status(response.status)
|
|
||||||
res.set(Object.fromEntries(response.headers))
|
|
||||||
res.send(await response.text())
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Google Cloud Run with Cloud Storage
|
|
||||||
|
|
||||||
Google Cloud Run is ideal for containerized deployments with automatic scaling. This example uses Google Cloud Storage via the S3-compatible API.
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// server.js
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
import { S3CompatibleStorage } from '@soulcraft/brainy/storage'
|
|
||||||
import express from 'express'
|
|
||||||
|
|
||||||
const app = express()
|
|
||||||
app.use(express.json())
|
|
||||||
|
|
||||||
const PORT = process.env.PORT || 8080
|
|
||||||
|
|
||||||
let brain
|
|
||||||
let handler
|
|
||||||
|
|
||||||
async function initBrainy() {
|
|
||||||
// Google Cloud Storage is S3-compatible
|
|
||||||
const storage = new S3CompatibleStorage({
|
|
||||||
endpoint: 'storage.googleapis.com',
|
|
||||||
bucket: process.env.GCS_BUCKET || 'brainy-data',
|
|
||||||
accessKeyId: process.env.GCS_ACCESS_KEY,
|
|
||||||
secretAccessKey: process.env.GCS_SECRET_KEY,
|
|
||||||
prefix: 'brainy/',
|
|
||||||
forcePathStyle: false,
|
|
||||||
region: process.env.GCS_REGION || 'US'
|
|
||||||
})
|
|
||||||
|
|
||||||
brain = new Brainy({
|
|
||||||
storage,
|
|
||||||
augmentations: [{
|
|
||||||
name: 'api-server',
|
|
||||||
config: {
|
|
||||||
enabled: true,
|
|
||||||
port: PORT,
|
|
||||||
cors: {
|
|
||||||
origin: process.env.CORS_ORIGIN || '*'
|
|
||||||
},
|
|
||||||
auth: {
|
|
||||||
required: process.env.AUTH_REQUIRED === 'true',
|
|
||||||
apiKeys: process.env.API_KEY ? [process.env.API_KEY] : []
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
|
||||||
handler = apiAugmentation.createUniversalHandler()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize on startup
|
|
||||||
await initBrainy()
|
|
||||||
|
|
||||||
// Health check endpoint
|
|
||||||
app.get('/health', (req, res) => {
|
|
||||||
res.json({ status: 'healthy', service: 'brainy-api' })
|
|
||||||
})
|
|
||||||
|
|
||||||
// Universal handler for all API routes
|
|
||||||
app.use('*', async (req, res) => {
|
|
||||||
const request = new Request(`https://${req.hostname}${req.originalUrl}`, {
|
|
||||||
method: req.method,
|
|
||||||
headers: req.headers,
|
|
||||||
body: req.method !== 'GET' ? JSON.stringify(req.body) : undefined
|
|
||||||
})
|
|
||||||
|
|
||||||
const response = await handler(request)
|
|
||||||
|
|
||||||
res.status(response.status)
|
|
||||||
res.set(Object.fromEntries(response.headers))
|
|
||||||
res.send(await response.text())
|
|
||||||
})
|
|
||||||
|
|
||||||
app.listen(PORT, () => {
|
|
||||||
console.log(`Brainy API running on port ${PORT}`)
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
```dockerfile
|
|
||||||
# Dockerfile
|
|
||||||
FROM node:20-alpine AS builder
|
|
||||||
WORKDIR /app
|
|
||||||
COPY package*.json ./
|
|
||||||
RUN npm ci --only=production
|
|
||||||
|
|
||||||
FROM node:20-alpine
|
|
||||||
WORKDIR /app
|
|
||||||
COPY --from=builder /app/node_modules ./node_modules
|
|
||||||
COPY . .
|
|
||||||
EXPOSE 8080
|
|
||||||
CMD ["node", "server.js"]
|
|
||||||
```
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# cloudbuild.yaml
|
|
||||||
steps:
|
|
||||||
# Build the container image
|
|
||||||
- name: 'gcr.io/cloud-builders/docker'
|
|
||||||
args: ['build', '-t', 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA', '.']
|
|
||||||
|
|
||||||
# Push to Container Registry
|
|
||||||
- name: 'gcr.io/cloud-builders/docker'
|
|
||||||
args: ['push', 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA']
|
|
||||||
|
|
||||||
# Deploy to Cloud Run
|
|
||||||
- name: 'gcr.io/google.com/cloudsdktool/cloud-sdk'
|
|
||||||
entrypoint: gcloud
|
|
||||||
args:
|
|
||||||
- 'run'
|
|
||||||
- 'deploy'
|
|
||||||
- 'brainy-api'
|
|
||||||
- '--image'
|
|
||||||
- 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA'
|
|
||||||
- '--region'
|
|
||||||
- 'us-central1'
|
|
||||||
- '--platform'
|
|
||||||
- 'managed'
|
|
||||||
- '--allow-unauthenticated'
|
|
||||||
- '--set-env-vars'
|
|
||||||
- 'GCS_BUCKET=brainy-storage,GCS_REGION=US'
|
|
||||||
- '--set-secrets'
|
|
||||||
- 'GCS_ACCESS_KEY=gcs-access-key:latest,GCS_SECRET_KEY=gcs-secret-key:latest'
|
|
||||||
- '--memory'
|
|
||||||
- '2Gi'
|
|
||||||
- '--cpu'
|
|
||||||
- '2'
|
|
||||||
- '--max-instances'
|
|
||||||
- '100'
|
|
||||||
- '--min-instances'
|
|
||||||
- '0'
|
|
||||||
|
|
||||||
images:
|
|
||||||
- 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA'
|
|
||||||
```
|
|
||||||
|
|
||||||
**Deploy with gcloud CLI:**
|
|
||||||
```bash
|
|
||||||
# Build and submit to Cloud Build
|
|
||||||
gcloud builds submit --config cloudbuild.yaml
|
|
||||||
|
|
||||||
# Or deploy directly
|
|
||||||
gcloud run deploy brainy-api \
|
|
||||||
--source . \
|
|
||||||
--platform managed \
|
|
||||||
--region us-central1 \
|
|
||||||
--allow-unauthenticated \
|
|
||||||
--set-env-vars GCS_BUCKET=brainy-storage \
|
|
||||||
--set-secrets "GCS_ACCESS_KEY=gcs-access-key:latest,GCS_SECRET_KEY=gcs-secret-key:latest"
|
|
||||||
```
|
|
||||||
|
|
||||||
**Create GCS Bucket with S3-compatible access:**
|
|
||||||
```bash
|
|
||||||
# Create bucket
|
|
||||||
gsutil mb -p PROJECT_ID -c STANDARD -l US gs://brainy-storage/
|
|
||||||
|
|
||||||
# Enable interoperability
|
|
||||||
gsutil iam ch serviceAccount:SERVICE_ACCOUNT@PROJECT_ID.iam.gserviceaccount.com:objectAdmin gs://brainy-storage
|
|
||||||
|
|
||||||
# Generate HMAC keys for S3-compatible access
|
|
||||||
gsutil hmac create SERVICE_ACCOUNT@PROJECT_ID.iam.gserviceaccount.com
|
|
||||||
|
|
||||||
# Store the access key and secret in Secret Manager
|
|
||||||
echo -n "YOUR_ACCESS_KEY" | gcloud secrets create gcs-access-key --data-file=-
|
|
||||||
echo -n "YOUR_SECRET_KEY" | gcloud secrets create gcs-secret-key --data-file=-
|
|
||||||
```
|
|
||||||
|
|
||||||
### Microsoft Azure Functions
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// index.js
|
|
||||||
module.exports = async function (context, req) {
|
|
||||||
const { Brainy } = require('@soulcraft/brainy')
|
|
||||||
const { S3CompatibleStorage } = require('@soulcraft/brainy/storage')
|
|
||||||
|
|
||||||
const storage = new S3CompatibleStorage({
|
|
||||||
endpoint: `${process.env.AZURE_STORAGE_ACCOUNT}.blob.core.windows.net`,
|
|
||||||
bucket: 'brainy-data',
|
|
||||||
accessKeyId: process.env.AZURE_STORAGE_ACCOUNT,
|
|
||||||
secretAccessKey: process.env.AZURE_STORAGE_KEY,
|
|
||||||
prefix: 'entities/',
|
|
||||||
forcePathStyle: false
|
|
||||||
})
|
|
||||||
|
|
||||||
const brain = new Brainy({ storage })
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
|
||||||
const handler = apiAugmentation.createUniversalHandler()
|
|
||||||
|
|
||||||
const request = new Request(`https://${context.req.headers.host}${context.req.url}`, {
|
|
||||||
method: context.req.method,
|
|
||||||
headers: context.req.headers,
|
|
||||||
body: JSON.stringify(context.req.body)
|
|
||||||
})
|
|
||||||
|
|
||||||
const response = await handler(request)
|
|
||||||
|
|
||||||
context.res = {
|
|
||||||
status: response.status,
|
|
||||||
headers: Object.fromEntries(response.headers),
|
|
||||||
body: await response.text()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cloudflare Workers
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// worker.js
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
import { R2Storage } from '@soulcraft/brainy/storage' // Alias for S3CompatibleStorage
|
|
||||||
|
|
||||||
let handler
|
|
||||||
|
|
||||||
export default {
|
|
||||||
async fetch(request, env, ctx) {
|
|
||||||
if (!handler) {
|
|
||||||
const storage = new R2Storage({
|
|
||||||
endpoint: `${env.ACCOUNT_ID}.r2.cloudflarestorage.com`,
|
|
||||||
bucket: 'brainy-data',
|
|
||||||
accessKeyId: env.R2_ACCESS_KEY_ID,
|
|
||||||
secretAccessKey: env.R2_SECRET_ACCESS_KEY,
|
|
||||||
region: 'auto',
|
|
||||||
forcePathStyle: true
|
|
||||||
})
|
|
||||||
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage,
|
|
||||||
augmentations: [{
|
|
||||||
name: 'api-server',
|
|
||||||
config: {
|
|
||||||
enabled: true,
|
|
||||||
cors: { origin: '*' }
|
|
||||||
}
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
|
||||||
handler = apiAugmentation.createUniversalHandler()
|
|
||||||
}
|
|
||||||
|
|
||||||
// The handler works directly with Request/Response!
|
|
||||||
return handler(request)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
```toml
|
|
||||||
# wrangler.toml
|
|
||||||
name = "brainy-api"
|
|
||||||
main = "worker.js"
|
|
||||||
compatibility_date = "2024-01-01"
|
|
||||||
|
|
||||||
[[r2_buckets]]
|
|
||||||
binding = "R2"
|
|
||||||
bucket_name = "brainy-data"
|
|
||||||
|
|
||||||
[vars]
|
|
||||||
ACCOUNT_ID = "your-account-id"
|
|
||||||
|
|
||||||
[env.production.vars]
|
|
||||||
R2_ACCESS_KEY_ID = "your-access-key"
|
|
||||||
R2_SECRET_ACCESS_KEY = "your-secret-key"
|
|
||||||
```
|
|
||||||
|
|
||||||
### Vercel Edge Functions
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// api/brainy.js
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
import { S3CompatibleStorage } from '@soulcraft/brainy/storage'
|
|
||||||
|
|
||||||
let handler
|
|
||||||
|
|
||||||
export const config = {
|
|
||||||
runtime: 'edge',
|
|
||||||
}
|
|
||||||
|
|
||||||
export default async (request) => {
|
|
||||||
if (!handler) {
|
|
||||||
const storage = new S3CompatibleStorage({
|
|
||||||
endpoint: 's3.amazonaws.com',
|
|
||||||
region: 'us-east-1',
|
|
||||||
bucket: process.env.S3_BUCKET,
|
|
||||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
|
||||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
|
||||||
})
|
|
||||||
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage,
|
|
||||||
augmentations: [{
|
|
||||||
name: 'api-server',
|
|
||||||
config: { enabled: true }
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
|
||||||
handler = apiAugmentation.createUniversalHandler()
|
|
||||||
}
|
|
||||||
|
|
||||||
return handler(request)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
```json
|
|
||||||
// vercel.json
|
|
||||||
{
|
|
||||||
"functions": {
|
|
||||||
"api/brainy.js": {
|
|
||||||
"maxDuration": 30,
|
|
||||||
"memory": 1024
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"rewrites": [
|
|
||||||
{
|
|
||||||
"source": "/api/:path*",
|
|
||||||
"destination": "/api/brainy"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Railway
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// server.js
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
import { S3CompatibleStorage } from '@soulcraft/brainy/storage'
|
|
||||||
import express from 'express'
|
|
||||||
|
|
||||||
const app = express()
|
|
||||||
const PORT = process.env.PORT || 3000
|
|
||||||
|
|
||||||
let brain
|
|
||||||
let handler
|
|
||||||
|
|
||||||
async function init() {
|
|
||||||
const storage = new S3CompatibleStorage({
|
|
||||||
endpoint: process.env.S3_ENDPOINT || 's3.amazonaws.com',
|
|
||||||
region: process.env.S3_REGION || 'us-east-1',
|
|
||||||
bucket: process.env.S3_BUCKET,
|
|
||||||
accessKeyId: process.env.S3_ACCESS_KEY,
|
|
||||||
secretAccessKey: process.env.S3_SECRET_KEY,
|
|
||||||
prefix: 'brainy/'
|
|
||||||
})
|
|
||||||
|
|
||||||
brain = new Brainy({
|
|
||||||
storage,
|
|
||||||
augmentations: [{
|
|
||||||
name: 'api-server',
|
|
||||||
config: {
|
|
||||||
enabled: true,
|
|
||||||
port: PORT
|
|
||||||
}
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
|
||||||
handler = apiAugmentation.createUniversalHandler()
|
|
||||||
}
|
|
||||||
|
|
||||||
await init()
|
|
||||||
|
|
||||||
// Universal handler for all routes
|
|
||||||
app.use('*', async (req, res) => {
|
|
||||||
const request = new Request(`http://localhost${req.originalUrl}`, {
|
|
||||||
method: req.method,
|
|
||||||
headers: req.headers,
|
|
||||||
body: JSON.stringify(req.body)
|
|
||||||
})
|
|
||||||
|
|
||||||
const response = await handler(request)
|
|
||||||
|
|
||||||
res.status(response.status)
|
|
||||||
res.set(Object.fromEntries(response.headers))
|
|
||||||
res.send(await response.text())
|
|
||||||
})
|
|
||||||
|
|
||||||
app.listen(PORT, () => {
|
|
||||||
console.log(`Brainy API running on port ${PORT}`)
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
```toml
|
|
||||||
# railway.toml
|
|
||||||
[build]
|
|
||||||
builder = "nixpacks"
|
|
||||||
buildCommand = "npm ci"
|
|
||||||
|
|
||||||
[deploy]
|
|
||||||
startCommand = "node server.js"
|
|
||||||
restartPolicyType = "always"
|
|
||||||
restartPolicyMaxRetries = 3
|
|
||||||
```
|
|
||||||
|
|
||||||
### Render
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// server.js (same as Railway example above)
|
|
||||||
// Use S3CompatibleStorage with your preferred object storage provider
|
|
||||||
```
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# render.yaml
|
|
||||||
services:
|
|
||||||
- type: web
|
|
||||||
name: brainy-api
|
|
||||||
runtime: node
|
|
||||||
buildCommand: npm install
|
|
||||||
startCommand: node server.js
|
|
||||||
envVars:
|
|
||||||
- key: S3_BUCKET
|
|
||||||
value: brainy-data
|
|
||||||
- key: S3_ENDPOINT
|
|
||||||
value: s3.amazonaws.com
|
|
||||||
- key: S3_REGION
|
|
||||||
value: us-east-1
|
|
||||||
- key: S3_ACCESS_KEY
|
|
||||||
sync: false
|
|
||||||
- key: S3_SECRET_KEY
|
|
||||||
sync: false
|
|
||||||
- key: API_KEY
|
|
||||||
generateValue: true
|
|
||||||
healthCheckPath: /health
|
|
||||||
autoDeploy: true
|
|
||||||
```
|
|
||||||
|
|
||||||
### Deno Deploy
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// main.ts
|
|
||||||
import { Brainy } from "npm:@soulcraft/brainy"
|
|
||||||
import { S3CompatibleStorage } from "npm:@soulcraft/brainy/storage"
|
|
||||||
|
|
||||||
const storage = new S3CompatibleStorage({
|
|
||||||
endpoint: Deno.env.get("S3_ENDPOINT") || "s3.amazonaws.com",
|
|
||||||
bucket: Deno.env.get("S3_BUCKET")!,
|
|
||||||
accessKeyId: Deno.env.get("S3_ACCESS_KEY")!,
|
|
||||||
secretAccessKey: Deno.env.get("S3_SECRET_KEY")!,
|
|
||||||
region: "auto"
|
|
||||||
})
|
|
||||||
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage,
|
|
||||||
augmentations: [{
|
|
||||||
name: 'api-server',
|
|
||||||
config: { enabled: true }
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
|
||||||
const handler = apiAugmentation.createUniversalHandler()
|
|
||||||
|
|
||||||
Deno.serve(handler)
|
|
||||||
```
|
|
||||||
|
|
||||||
## API Endpoints
|
|
||||||
|
|
||||||
The API Server augmentation provides these REST endpoints:
|
|
||||||
|
|
||||||
- `POST /api/brainy/add` - Add entity
|
|
||||||
- `GET /api/brainy/get?id=xxx` - Get entity by ID
|
|
||||||
- `PUT /api/brainy/update` - Update entity
|
|
||||||
- `DELETE /api/brainy/delete?id=xxx` - Delete entity
|
|
||||||
- `POST /api/brainy/find` - Search/find entities
|
|
||||||
- `POST /api/brainy/relate` - Create relationship
|
|
||||||
- `GET /api/brainy/insights` - Get statistics and insights
|
|
||||||
- `GET /health` - Health check
|
|
||||||
|
|
||||||
## WebSocket Support
|
|
||||||
|
|
||||||
The API Server augmentation includes WebSocket support for real-time updates through the `setupUniversalWebSocket()` method.
|
|
||||||
|
|
||||||
## MCP Support
|
|
||||||
|
|
||||||
Model Context Protocol (MCP) endpoints are available at `/mcp/*` for AI tool integration.
|
|
||||||
|
|
||||||
## Environment Variables
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Storage Configuration (S3Compatible)
|
|
||||||
S3_ENDPOINT=s3.amazonaws.com
|
|
||||||
S3_REGION=us-east-1
|
|
||||||
S3_BUCKET=brainy-data
|
|
||||||
S3_ACCESS_KEY=xxx
|
|
||||||
S3_SECRET_KEY=xxx
|
|
||||||
|
|
||||||
# API Configuration
|
|
||||||
API_KEY=your-secret-key
|
|
||||||
PORT=3000
|
|
||||||
|
|
||||||
# CORS
|
|
||||||
CORS_ORIGIN=*
|
|
||||||
|
|
||||||
# Rate Limiting
|
|
||||||
RATE_LIMIT_WINDOW=60000
|
|
||||||
RATE_LIMIT_MAX=100
|
|
||||||
```
|
|
||||||
|
|
||||||
## Client Usage
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// REST API Client
|
|
||||||
const response = await fetch('https://your-api.com/api/brainy/add', {
|
|
||||||
method: 'POST',
|
|
||||||
headers: {
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
'Authorization': 'Bearer YOUR_API_KEY'
|
|
||||||
},
|
|
||||||
body: JSON.stringify({
|
|
||||||
data: 'Your content here',
|
|
||||||
metadata: { type: 'document' }
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
const { id } = await response.json()
|
|
||||||
|
|
||||||
// Search
|
|
||||||
const searchResponse = await fetch('https://your-api.com/api/brainy/find', {
|
|
||||||
method: 'POST',
|
|
||||||
headers: {
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
'Authorization': 'Bearer YOUR_API_KEY'
|
|
||||||
},
|
|
||||||
body: JSON.stringify({
|
|
||||||
query: 'neural networks'
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
const results = await searchResponse.json()
|
|
||||||
|
|
||||||
// WebSocket Client
|
|
||||||
const ws = new WebSocket('wss://your-api.com/ws')
|
|
||||||
ws.onopen = () => {
|
|
||||||
ws.send(JSON.stringify({
|
|
||||||
type: 'subscribe',
|
|
||||||
pattern: 'technology'
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
ws.onmessage = (event) => {
|
|
||||||
const update = JSON.parse(event.data)
|
|
||||||
console.log('Real-time update:', update)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Storage Adapter Configuration
|
|
||||||
|
|
||||||
S3CompatibleStorage constructor parameters (verified from source):
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
{
|
|
||||||
endpoint: string, // Required (e.g., 's3.amazonaws.com')
|
|
||||||
bucket: string, // Required
|
|
||||||
accessKeyId: string, // Required
|
|
||||||
secretAccessKey: string, // Required
|
|
||||||
region?: string, // Optional (default: 'us-east-1')
|
|
||||||
prefix?: string, // Optional (e.g., 'brainy/')
|
|
||||||
forcePathStyle?: boolean, // Optional (needed for some S3-compatible services)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Important Notes
|
|
||||||
|
|
||||||
1. All examples use the **real** Brainy 3.0 APIs
|
|
||||||
2. The `createUniversalHandler()` method is provided by the API Server augmentation
|
|
||||||
3. S3CompatibleStorage works with any S3-compatible service
|
|
||||||
4. Always call `brain.init()` before using Brainy
|
|
||||||
5. The handler can be cached across requests for better performance
|
|
||||||
6. R2Storage is an alias for S3CompatibleStorage (for Cloudflare R2)
|
|
||||||
|
|
||||||
## Security Best Practices
|
|
||||||
|
|
||||||
1. **Always use environment variables for sensitive data** (API keys, secrets)
|
|
||||||
2. **Enable authentication** in the API Server augmentation config
|
|
||||||
3. **Use HTTPS/TLS** for all production deployments
|
|
||||||
4. **Implement rate limiting** to prevent abuse
|
|
||||||
5. **Configure CORS** appropriately for your use case
|
|
||||||
|
|
||||||
## Cost Optimization
|
|
||||||
|
|
||||||
### Enable Lifecycle Policies
|
|
||||||
|
|
||||||
**AWS S3: Automatic tier transitions**
|
|
||||||
```javascript
|
|
||||||
// After initializing brain with S3CompatibleStorage
|
|
||||||
const storage = brain.storage
|
|
||||||
|
|
||||||
// Set lifecycle policy for automatic archival
|
|
||||||
await storage.setLifecyclePolicy({
|
|
||||||
rules: [{
|
|
||||||
id: 'archive-old-data',
|
|
||||||
prefix: 'entities/',
|
|
||||||
status: 'Enabled',
|
|
||||||
transitions: [
|
|
||||||
{ days: 30, storageClass: 'STANDARD_IA' }, // $0.0125/GB after 30 days
|
|
||||||
{ days: 90, storageClass: 'GLACIER' }, // $0.004/GB after 90 days
|
|
||||||
{ days: 365, storageClass: 'DEEP_ARCHIVE' } // $0.00099/GB after 1 year
|
|
||||||
]
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
|
|
||||||
// Or enable Intelligent-Tiering for hands-off optimization
|
|
||||||
await storage.enableIntelligentTiering('entities/', 'auto-optimize')
|
|
||||||
```
|
|
||||||
|
|
||||||
**Cost Impact (500TB dataset):**
|
|
||||||
| Storage Class | Cost/GB/Month | 500TB/Year | Savings |
|
|
||||||
|---------------|---------------|------------|---------|
|
|
||||||
| Standard | $0.023 | $138,000 | Baseline |
|
|
||||||
| Intelligent-Tiering | Variable | $6,900 | **95%** |
|
|
||||||
| With lifecycle policy | Variable | $5,940 | **96%** |
|
|
||||||
|
|
||||||
### Enable Batch Operations
|
|
||||||
|
|
||||||
**Efficient bulk deletions:**
|
|
||||||
```javascript
|
|
||||||
// Batch delete (1000 objects per request)
|
|
||||||
const idsToDelete = [/* array of entity IDs */]
|
|
||||||
const paths = idsToDelete.flatMap(id => [
|
|
||||||
`entities/nouns/vectors/${id.substring(0, 2)}/${id}.json`,
|
|
||||||
`entities/nouns/metadata/${id.substring(0, 2)}/${id}.json`
|
|
||||||
])
|
|
||||||
|
|
||||||
await storage.batchDelete(paths) // Much faster than individual deletes
|
|
||||||
```
|
|
||||||
|
|
||||||
### Enable Compression (FileSystem)
|
|
||||||
|
|
||||||
**For local/server deployments:**
|
|
||||||
```javascript
|
|
||||||
const storage = new FileSystemStorage({
|
|
||||||
path: './data',
|
|
||||||
compression: true // 60-80% space savings
|
|
||||||
})
|
|
||||||
|
|
||||||
const brain = new Brainy({ storage })
|
|
||||||
```
|
|
||||||
|
|
||||||
## Performance Tips
|
|
||||||
|
|
||||||
1. **Cache the brain instance** - Initialize once and reuse across requests
|
|
||||||
2. **Use S3CompatibleStorage** for cloud deployments (better scalability)
|
|
||||||
3. **Enable the cache augmentation** for frequently accessed data
|
|
||||||
4. **Configure appropriate memory limits** for your runtime
|
|
||||||
5. Enable lifecycle policies to reduce storage costs by 96%
|
|
||||||
6. Use batch operations for cleanup tasks
|
|
||||||
|
|
||||||
## Troubleshooting
|
|
||||||
|
|
||||||
### Common Issues
|
|
||||||
|
|
||||||
1. **"Brainy not initialized"** - Make sure to call `await brain.init()` before use
|
|
||||||
2. **"augmentationRegistry.getAugmentation is not a function"** - The augmentation wasn't loaded properly
|
|
||||||
3. **S3 Access Denied** - Check your IAM permissions and credentials
|
|
||||||
4. **CORS errors** - Configure the CORS settings in the API Server augmentation
|
|
||||||
|
|
||||||
### Debug Mode
|
|
||||||
|
|
||||||
Enable debug logging by setting:
|
|
||||||
```javascript
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage,
|
|
||||||
debug: true,
|
|
||||||
augmentations: [{
|
|
||||||
name: 'api-server',
|
|
||||||
config: {
|
|
||||||
enabled: true,
|
|
||||||
verbose: true
|
|
||||||
}
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## Support
|
|
||||||
|
|
||||||
- Documentation: https://github.com/soulcraft/brainy/docs
|
|
||||||
- Issues: https://github.com/soulcraft/brainy/issues
|
|
||||||
- NPM Package: https://www.npmjs.com/package/@soulcraft/brainy
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
This guide contains only verified, working code from the actual Brainy 3.0 codebase. No mock implementations, no stub methods, only production-ready code.
|
|
||||||
|
|
@ -1,505 +0,0 @@
|
||||||
# AWS Deployment Guide for Brainy
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
Deploy Brainy on AWS with automatic scaling, high availability, and zero-config dynamic adaptation. Brainy automatically adapts to your AWS environment without manual configuration.
|
|
||||||
|
|
||||||
## Quick Start (Zero-Config)
|
|
||||||
|
|
||||||
### Option 1: AWS Lambda (Serverless)
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Install Brainy
|
|
||||||
npm install @soulcraft/brainy
|
|
||||||
|
|
||||||
# Create handler.js
|
|
||||||
cat > handler.js << 'EOF'
|
|
||||||
const { Brainy } = require('@soulcraft/brainy')
|
|
||||||
|
|
||||||
let brain
|
|
||||||
|
|
||||||
exports.handler = async (event) => {
|
|
||||||
// Brainy auto-detects Lambda environment and configures accordingly
|
|
||||||
if (!brain) {
|
|
||||||
brain = new Brainy() // Zero config - auto-adapts to Lambda
|
|
||||||
await brain.init()
|
|
||||||
}
|
|
||||||
|
|
||||||
const { method, ...params } = JSON.parse(event.body)
|
|
||||||
|
|
||||||
switch(method) {
|
|
||||||
case 'add':
|
|
||||||
const id = await brain.add(params)
|
|
||||||
return { statusCode: 200, body: JSON.stringify({ id }) }
|
|
||||||
case 'find':
|
|
||||||
const results = await brain.find(params)
|
|
||||||
return { statusCode: 200, body: JSON.stringify({ results }) }
|
|
||||||
default:
|
|
||||||
return { statusCode: 400, body: 'Unknown method' }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
EOF
|
|
||||||
|
|
||||||
# Deploy with AWS SAM
|
|
||||||
sam init --runtime nodejs20.x --name brainy-app
|
|
||||||
sam deploy --guided
|
|
||||||
```
|
|
||||||
|
|
||||||
### Option 2: ECS Fargate (Container)
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Build and push Docker image
|
|
||||||
aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin $ECR_URI
|
|
||||||
docker build -t brainy .
|
|
||||||
docker tag brainy:latest $ECR_URI/brainy:latest
|
|
||||||
docker push $ECR_URI/brainy:latest
|
|
||||||
|
|
||||||
# Deploy with minimal ECS task definition
|
|
||||||
cat > task-definition.json << 'EOF'
|
|
||||||
{
|
|
||||||
"family": "brainy",
|
|
||||||
"networkMode": "awsvpc",
|
|
||||||
"requiresCompatibilities": ["FARGATE"],
|
|
||||||
"cpu": "256",
|
|
||||||
"memory": "512",
|
|
||||||
"containerDefinitions": [{
|
|
||||||
"name": "brainy",
|
|
||||||
"image": "$ECR_URI/brainy:latest",
|
|
||||||
"environment": [
|
|
||||||
{"name": "NODE_ENV", "value": "production"}
|
|
||||||
],
|
|
||||||
"logConfiguration": {
|
|
||||||
"logDriver": "awslogs",
|
|
||||||
"options": {
|
|
||||||
"awslogs-group": "/ecs/brainy",
|
|
||||||
"awslogs-region": "us-east-1",
|
|
||||||
"awslogs-stream-prefix": "ecs"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}]
|
|
||||||
}
|
|
||||||
EOF
|
|
||||||
|
|
||||||
# Brainy auto-detects ECS environment and uses S3 for storage
|
|
||||||
aws ecs register-task-definition --cli-input-json file://task-definition.json
|
|
||||||
aws ecs create-service --cluster default --service-name brainy --task-definition brainy --desired-count 2
|
|
||||||
```
|
|
||||||
|
|
||||||
### Option 3: EC2 Auto-Scaling
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# User data script for EC2 instances
|
|
||||||
#!/bin/bash
|
|
||||||
curl -fsSL https://rpm.nodesource.com/setup_20.x | sudo bash -
|
|
||||||
sudo yum install -y nodejs git
|
|
||||||
|
|
||||||
# Clone and setup (or use your deployment method)
|
|
||||||
git clone https://github.com/yourorg/brainy-app.git /app
|
|
||||||
cd /app
|
|
||||||
npm install --production
|
|
||||||
|
|
||||||
# Create systemd service
|
|
||||||
cat > /etc/systemd/system/brainy.service << 'EOF'
|
|
||||||
[Unit]
|
|
||||||
Description=Brainy
|
|
||||||
After=network.target
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=simple
|
|
||||||
User=ec2-user
|
|
||||||
WorkingDirectory=/app
|
|
||||||
ExecStart=/usr/bin/node index.js
|
|
||||||
Restart=on-failure
|
|
||||||
Environment=NODE_ENV=production
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
EOF
|
|
||||||
|
|
||||||
systemctl start brainy
|
|
||||||
systemctl enable brainy
|
|
||||||
```
|
|
||||||
|
|
||||||
## Zero-Config Storage (Automatic)
|
|
||||||
|
|
||||||
Brainy automatically detects and uses the best available storage:
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// No configuration needed - Brainy auto-detects:
|
|
||||||
const brain = new Brainy()
|
|
||||||
|
|
||||||
// Auto-detection priority:
|
|
||||||
// 1. S3 (if IAM role has permissions)
|
|
||||||
// 2. EFS (if mounted at /mnt/efs)
|
|
||||||
// 3. EBS volume (if available)
|
|
||||||
// 4. Instance storage (fallback)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Manual S3 Configuration (Optional)
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 's3',
|
|
||||||
options: {
|
|
||||||
bucket: process.env.S3_BUCKET || 'auto', // 'auto' creates bucket
|
|
||||||
region: process.env.AWS_REGION || 'auto', // 'auto' detects region
|
|
||||||
// IAM role provides credentials automatically
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## Scaling Strategies
|
|
||||||
|
|
||||||
### 1. Horizontal Scaling (Recommended)
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# Auto-scaling policy
|
|
||||||
Resources:
|
|
||||||
AutoScalingTarget:
|
|
||||||
Type: AWS::ApplicationAutoScaling::ScalableTarget
|
|
||||||
Properties:
|
|
||||||
ServiceNamespace: ecs
|
|
||||||
ResourceId: service/default/brainy
|
|
||||||
ScalableDimension: ecs:service:DesiredCount
|
|
||||||
MinCapacity: 2
|
|
||||||
MaxCapacity: 100
|
|
||||||
|
|
||||||
AutoScalingPolicy:
|
|
||||||
Type: AWS::ApplicationAutoScaling::ScalingPolicy
|
|
||||||
Properties:
|
|
||||||
PolicyType: TargetTrackingScaling
|
|
||||||
TargetTrackingScalingPolicyConfiguration:
|
|
||||||
TargetValue: 70.0
|
|
||||||
PredefinedMetricSpecification:
|
|
||||||
PredefinedMetricType: ECSServiceAverageCPUUtilization
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Vertical Scaling
|
|
||||||
|
|
||||||
Brainy automatically adapts to available memory:
|
|
||||||
- **256MB**: Minimal mode, optimized caching
|
|
||||||
- **512MB**: Standard mode, balanced performance
|
|
||||||
- **1GB+**: Full mode, maximum performance
|
|
||||||
|
|
||||||
## High Availability Setup
|
|
||||||
|
|
||||||
### Multi-AZ Deployment
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Brainy automatically handles multi-AZ with S3
|
|
||||||
const brain = new Brainy({
|
|
||||||
distributed: {
|
|
||||||
enabled: true, // Auto-enables with S3 storage
|
|
||||||
coordinationMethod: 'auto' // Uses S3 for coordination
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Load Balancing
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Application Load Balancer with health checks
|
|
||||||
aws elbv2 create-load-balancer \
|
|
||||||
--name brainy-alb \
|
|
||||||
--subnets subnet-xxx subnet-yyy \
|
|
||||||
--security-groups sg-xxx
|
|
||||||
|
|
||||||
aws elbv2 create-target-group \
|
|
||||||
--name brainy-targets \
|
|
||||||
--protocol HTTP \
|
|
||||||
--port 3000 \
|
|
||||||
--vpc-id vpc-xxx \
|
|
||||||
--health-check-path /health \
|
|
||||||
--health-check-interval-seconds 30
|
|
||||||
```
|
|
||||||
|
|
||||||
## Monitoring & Observability
|
|
||||||
|
|
||||||
### CloudWatch Integration
|
|
||||||
|
|
||||||
Brainy automatically sends metrics when running on AWS:
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Automatic CloudWatch metrics (no config needed)
|
|
||||||
// - Request count
|
|
||||||
// - Response time
|
|
||||||
// - Error rate
|
|
||||||
// - Storage usage
|
|
||||||
// - Memory usage
|
|
||||||
```
|
|
||||||
|
|
||||||
### Custom Metrics
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
const brain = new Brainy({
|
|
||||||
monitoring: {
|
|
||||||
enabled: true,
|
|
||||||
customMetrics: {
|
|
||||||
namespace: 'Brainy/Production',
|
|
||||||
dimensions: [
|
|
||||||
{ Name: 'Environment', Value: 'production' },
|
|
||||||
{ Name: 'Service', Value: 'api' }
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## Security Best Practices
|
|
||||||
|
|
||||||
### 1. IAM Role (Recommended)
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"Version": "2012-10-17",
|
|
||||||
"Statement": [
|
|
||||||
{
|
|
||||||
"Effect": "Allow",
|
|
||||||
"Action": [
|
|
||||||
"s3:GetObject",
|
|
||||||
"s3:PutObject",
|
|
||||||
"s3:DeleteObject",
|
|
||||||
"s3:ListBucket"
|
|
||||||
],
|
|
||||||
"Resource": [
|
|
||||||
"arn:aws:s3:::brainy-*/*",
|
|
||||||
"arn:aws:s3:::brainy-*"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. VPC Configuration
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Private subnets with NAT Gateway
|
|
||||||
aws ec2 create-vpc --cidr-block 10.0.0.0/16
|
|
||||||
aws ec2 create-subnet --vpc-id vpc-xxx --cidr-block 10.0.1.0/24 --availability-zone us-east-1a
|
|
||||||
aws ec2 create-subnet --vpc-id vpc-xxx --cidr-block 10.0.2.0/24 --availability-zone us-east-1b
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Encryption
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Automatic encryption with S3
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 's3',
|
|
||||||
options: {
|
|
||||||
encryption: 'auto' // Uses S3 SSE-S3 by default
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## Cost Optimization
|
|
||||||
|
|
||||||
### 1. Spot Instances (70% savings)
|
|
||||||
|
|
||||||
```bash
|
|
||||||
aws ec2 request-spot-fleet --spot-fleet-request-config '{
|
|
||||||
"IamFleetRole": "arn:aws:iam::xxx:role/fleet-role",
|
|
||||||
"TargetCapacity": 2,
|
|
||||||
"SpotPrice": "0.05",
|
|
||||||
"LaunchSpecifications": [{
|
|
||||||
"ImageId": "ami-xxx",
|
|
||||||
"InstanceType": "t3.medium",
|
|
||||||
"UserData": "BASE64_ENCODED_STARTUP_SCRIPT"
|
|
||||||
}]
|
|
||||||
}'
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. S3 Intelligent-Tiering
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Brainy automatically uses S3 Intelligent-Tiering
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 's3',
|
|
||||||
options: {
|
|
||||||
storageClass: 'INTELLIGENT_TIERING' // Automatic cost optimization
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Lambda Reserved Concurrency
|
|
||||||
|
|
||||||
```bash
|
|
||||||
aws lambda put-function-concurrency \
|
|
||||||
--function-name brainy-handler \
|
|
||||||
--reserved-concurrent-executions 10
|
|
||||||
```
|
|
||||||
|
|
||||||
## Deployment Automation
|
|
||||||
|
|
||||||
### GitHub Actions CI/CD
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
name: Deploy to AWS
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches: [main]
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
deploy:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v2
|
|
||||||
|
|
||||||
- name: Configure AWS credentials
|
|
||||||
uses: aws-actions/configure-aws-credentials@v1
|
|
||||||
with:
|
|
||||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
||||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
||||||
aws-region: us-east-1
|
|
||||||
|
|
||||||
- name: Deploy to ECS
|
|
||||||
run: |
|
|
||||||
docker build -t brainy .
|
|
||||||
docker tag brainy:latest $ECR_URI/brainy:latest
|
|
||||||
docker push $ECR_URI/brainy:latest
|
|
||||||
aws ecs update-service --cluster default --service brainy --force-new-deployment
|
|
||||||
```
|
|
||||||
|
|
||||||
## Troubleshooting
|
|
||||||
|
|
||||||
### Common Issues
|
|
||||||
|
|
||||||
1. **Storage Auto-Detection Fails**
|
|
||||||
```javascript
|
|
||||||
// Explicitly specify storage
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: { type: 's3', options: { bucket: 'my-bucket' } }
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
2. **Memory Issues**
|
|
||||||
```javascript
|
|
||||||
// Optimize for low memory
|
|
||||||
const brain = new Brainy({
|
|
||||||
cache: { maxSize: 100 }, // Reduce cache size
|
|
||||||
index: { M: 8 } // Reduce HNSW connections
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
3. **Cold Starts (Lambda)**
|
|
||||||
|
|
||||||
**Progressive Initialization (Zero-Config)**
|
|
||||||
|
|
||||||
Brainy automatically detects Lambda environments (AWS_LAMBDA_FUNCTION_NAME)
|
|
||||||
and uses progressive initialization for <200ms cold starts:
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Zero-config - Brainy auto-detects Lambda
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 's3',
|
|
||||||
s3Storage: {
|
|
||||||
bucketName: 'my-bucket',
|
|
||||||
region: 'us-east-1',
|
|
||||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
|
||||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
await brain.init() // Returns in <200ms
|
|
||||||
|
|
||||||
// First write validates bucket (lazy validation)
|
|
||||||
await brain.add('noun', { name: 'test' }) // Validates here
|
|
||||||
```
|
|
||||||
|
|
||||||
**Manual Override (if needed)**
|
|
||||||
```javascript
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 's3',
|
|
||||||
s3Storage: {
|
|
||||||
bucketName: 'my-bucket',
|
|
||||||
// Force specific mode
|
|
||||||
initMode: 'progressive' // 'auto' | 'progressive' | 'strict'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
| Mode | Cold Start | Best For |
|
|
||||||
|------|------------|----------|
|
|
||||||
| `auto` (default) | <200ms in Lambda | Zero-config, auto-detects |
|
|
||||||
| `progressive` | <200ms always | Force fast init everywhere |
|
|
||||||
| `strict` | 100-500ms+ | Local dev, tests, debugging |
|
|
||||||
|
|
||||||
**Warm-up (Alternative)**
|
|
||||||
```javascript
|
|
||||||
exports.warmup = async () => {
|
|
||||||
if (!brain) {
|
|
||||||
brain = new Brainy({ warmup: true })
|
|
||||||
await brain.init()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
**Readiness Detection**
|
|
||||||
|
|
||||||
Use the `brain.ready` Promise to ensure Brainy is initialized before handling requests:
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
let brain
|
|
||||||
|
|
||||||
exports.handler = async (event) => {
|
|
||||||
if (!brain) {
|
|
||||||
brain = new Brainy({ storage: { type: 's3', ... } })
|
|
||||||
brain.init() // Fire and forget
|
|
||||||
}
|
|
||||||
|
|
||||||
// Wait for initialization to complete
|
|
||||||
await brain.ready
|
|
||||||
|
|
||||||
// Now safe to use brain methods
|
|
||||||
const results = await brain.find({ query: event.queryStringParameters.q })
|
|
||||||
return { statusCode: 200, body: JSON.stringify(results) }
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
For health checks, use `isFullyInitialized()` to verify all background tasks are complete:
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
exports.healthCheck = async () => {
|
|
||||||
try {
|
|
||||||
await brain.ready
|
|
||||||
return {
|
|
||||||
statusCode: 200,
|
|
||||||
body: JSON.stringify({
|
|
||||||
status: 'ready',
|
|
||||||
fullyInitialized: brain.isFullyInitialized()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
return {
|
|
||||||
statusCode: 503,
|
|
||||||
body: JSON.stringify({ status: 'initializing' })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Production Checklist
|
|
||||||
|
|
||||||
- [ ] IAM roles configured with minimal permissions
|
|
||||||
- [ ] VPC with private subnets
|
|
||||||
- [ ] Auto-scaling configured
|
|
||||||
- [ ] CloudWatch alarms set up
|
|
||||||
- [ ] Backup strategy (S3 versioning enabled)
|
|
||||||
- [ ] SSL/TLS certificates configured
|
|
||||||
- [ ] Rate limiting enabled
|
|
||||||
- [ ] Health checks configured
|
|
||||||
- [ ] Monitoring dashboard created
|
|
||||||
- [ ] Cost alerts configured
|
|
||||||
|
|
||||||
## Support
|
|
||||||
|
|
||||||
- Documentation: https://brainy.soulcraft.ai/docs
|
|
||||||
- Issues: https://github.com/soulcraft/brainy/issues
|
|
||||||
- Community: https://discord.gg/brainy
|
|
||||||
|
|
@ -1,627 +0,0 @@
|
||||||
# Google Cloud Platform Deployment Guide for Brainy
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
Deploy Brainy on GCP with automatic scaling, global distribution, and zero-config dynamic adaptation. Brainy automatically detects and optimizes for GCP services.
|
|
||||||
|
|
||||||
## Quick Start (Zero-Config)
|
|
||||||
|
|
||||||
### Option 1: Cloud Run (Serverless Containers)
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Build and deploy with one command
|
|
||||||
gcloud run deploy brainy \
|
|
||||||
--source . \
|
|
||||||
--platform managed \
|
|
||||||
--region us-central1 \
|
|
||||||
--allow-unauthenticated
|
|
||||||
|
|
||||||
# Brainy auto-detects Cloud Run and configures:
|
|
||||||
# - Memory-optimized caching
|
|
||||||
# - GCS for storage (if available)
|
|
||||||
# - Cloud SQL for metadata (if available)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Option 2: Cloud Functions (Serverless)
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// index.js
|
|
||||||
const { Brainy } = require('@soulcraft/brainy')
|
|
||||||
|
|
||||||
let brain
|
|
||||||
|
|
||||||
exports.brainyHandler = async (req, res) => {
|
|
||||||
// Zero-config - auto-adapts to Cloud Functions
|
|
||||||
if (!brain) {
|
|
||||||
brain = new Brainy() // Detects GCP environment automatically
|
|
||||||
await brain.init()
|
|
||||||
}
|
|
||||||
|
|
||||||
const { method, ...params } = req.body
|
|
||||||
|
|
||||||
try {
|
|
||||||
let result
|
|
||||||
switch(method) {
|
|
||||||
case 'add':
|
|
||||||
result = await brain.add(params)
|
|
||||||
break
|
|
||||||
case 'find':
|
|
||||||
result = await brain.find(params)
|
|
||||||
break
|
|
||||||
case 'relate':
|
|
||||||
result = await brain.relate(params)
|
|
||||||
break
|
|
||||||
default:
|
|
||||||
return res.status(400).json({ error: 'Unknown method' })
|
|
||||||
}
|
|
||||||
res.json({ result })
|
|
||||||
} catch (error) {
|
|
||||||
res.status(500).json({ error: error.message })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Deploy:
|
|
||||||
```bash
|
|
||||||
gcloud functions deploy brainy \
|
|
||||||
--runtime nodejs20 \
|
|
||||||
--trigger-http \
|
|
||||||
--entry-point brainyHandler \
|
|
||||||
--memory 512MB \
|
|
||||||
--timeout 60s
|
|
||||||
```
|
|
||||||
|
|
||||||
### Option 3: Google Kubernetes Engine (GKE)
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Create autopilot cluster (fully managed, zero-config)
|
|
||||||
gcloud container clusters create-auto brainy-cluster \
|
|
||||||
--region us-central1
|
|
||||||
|
|
||||||
# Deploy using Cloud Build
|
|
||||||
gcloud builds submit --tag gcr.io/$PROJECT_ID/brainy
|
|
||||||
|
|
||||||
# Apply Kubernetes manifest
|
|
||||||
kubectl apply -f - <<EOF
|
|
||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: brainy
|
|
||||||
spec:
|
|
||||||
replicas: 3
|
|
||||||
selector:
|
|
||||||
matchLabels:
|
|
||||||
app: brainy
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
app: brainy
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: brainy
|
|
||||||
image: gcr.io/$PROJECT_ID/brainy
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
memory: "512Mi"
|
|
||||||
cpu: "250m"
|
|
||||||
env:
|
|
||||||
- name: NODE_ENV
|
|
||||||
value: production
|
|
||||||
---
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: brainy-service
|
|
||||||
spec:
|
|
||||||
type: LoadBalancer
|
|
||||||
selector:
|
|
||||||
app: brainy
|
|
||||||
ports:
|
|
||||||
- port: 80
|
|
||||||
targetPort: 3000
|
|
||||||
EOF
|
|
||||||
```
|
|
||||||
|
|
||||||
## Zero-Config Storage (Automatic)
|
|
||||||
|
|
||||||
Brainy automatically detects and uses the best GCP storage:
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
const brain = new Brainy()
|
|
||||||
// Auto-detection priority:
|
|
||||||
// 1. Firestore (if available)
|
|
||||||
// 2. Cloud Storage (GCS)
|
|
||||||
// 3. Cloud SQL
|
|
||||||
// 4. Persistent Disk
|
|
||||||
// 5. Memory (fallback)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cloud Storage Configuration (Optional)
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 's3', // GCS is S3-compatible
|
|
||||||
options: {
|
|
||||||
endpoint: 'https://storage.googleapis.com',
|
|
||||||
bucket: process.env.GCS_BUCKET || 'auto', // Auto-creates bucket
|
|
||||||
// Uses Application Default Credentials automatically
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Firestore Integration (Optional)
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 'firestore',
|
|
||||||
options: {
|
|
||||||
projectId: process.env.GCP_PROJECT || 'auto',
|
|
||||||
collection: 'brainy-data'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## Scaling Strategies
|
|
||||||
|
|
||||||
### 1. Cloud Run Auto-scaling
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# service.yaml
|
|
||||||
apiVersion: serving.knative.dev/v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: brainy
|
|
||||||
annotations:
|
|
||||||
run.googleapis.com/execution-environment: gen2
|
|
||||||
spec:
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
annotations:
|
|
||||||
autoscaling.knative.dev/minScale: "1"
|
|
||||||
autoscaling.knative.dev/maxScale: "1000"
|
|
||||||
autoscaling.knative.dev/target: "80"
|
|
||||||
spec:
|
|
||||||
containerConcurrency: 100
|
|
||||||
containers:
|
|
||||||
- image: gcr.io/PROJECT_ID/brainy
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
cpu: "2"
|
|
||||||
memory: "2Gi"
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. GKE Horizontal Pod Autoscaling
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
apiVersion: autoscaling/v2
|
|
||||||
kind: HorizontalPodAutoscaler
|
|
||||||
metadata:
|
|
||||||
name: brainy-hpa
|
|
||||||
spec:
|
|
||||||
scaleTargetRef:
|
|
||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
name: brainy
|
|
||||||
minReplicas: 3
|
|
||||||
maxReplicas: 100
|
|
||||||
metrics:
|
|
||||||
- type: Resource
|
|
||||||
resource:
|
|
||||||
name: cpu
|
|
||||||
target:
|
|
||||||
type: Utilization
|
|
||||||
averageUtilization: 70
|
|
||||||
- type: Resource
|
|
||||||
resource:
|
|
||||||
name: memory
|
|
||||||
target:
|
|
||||||
type: Utilization
|
|
||||||
averageUtilization: 80
|
|
||||||
```
|
|
||||||
|
|
||||||
## Global Distribution
|
|
||||||
|
|
||||||
### Multi-Region Setup
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Brainy automatically handles multi-region with GCS
|
|
||||||
const brain = new Brainy({
|
|
||||||
distributed: {
|
|
||||||
enabled: true,
|
|
||||||
regions: ['us-central1', 'europe-west1', 'asia-northeast1'],
|
|
||||||
replication: 'auto' // Automatic cross-region replication
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Traffic Director Configuration
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Global load balancing with Traffic Director
|
|
||||||
gcloud compute backend-services create brainy-global \
|
|
||||||
--global \
|
|
||||||
--load-balancing-scheme=INTERNAL_SELF_MANAGED \
|
|
||||||
--protocol=HTTP2
|
|
||||||
|
|
||||||
gcloud compute backend-services add-backend brainy-global \
|
|
||||||
--global \
|
|
||||||
--network-endpoint-group=brainy-neg \
|
|
||||||
--network-endpoint-group-region=us-central1
|
|
||||||
```
|
|
||||||
|
|
||||||
## Monitoring & Observability
|
|
||||||
|
|
||||||
### Cloud Monitoring (Automatic)
|
|
||||||
|
|
||||||
Brainy automatically sends metrics to Cloud Monitoring:
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// No configuration needed - automatic when running on GCP
|
|
||||||
const brain = new Brainy()
|
|
||||||
|
|
||||||
// Automatic metrics:
|
|
||||||
// - Request latency
|
|
||||||
// - Error rate
|
|
||||||
// - Storage operations
|
|
||||||
// - Cache hit rate
|
|
||||||
// - Memory usage
|
|
||||||
```
|
|
||||||
|
|
||||||
### Custom Metrics
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
const { Monitoring } = require('@google-cloud/monitoring')
|
|
||||||
const monitoring = new Monitoring.MetricServiceClient()
|
|
||||||
|
|
||||||
const brain = new Brainy({
|
|
||||||
onMetric: async (metric) => {
|
|
||||||
// Send custom metrics to Cloud Monitoring
|
|
||||||
await monitoring.createTimeSeries({
|
|
||||||
name: monitoring.projectPath(projectId),
|
|
||||||
timeSeries: [{
|
|
||||||
metric: {
|
|
||||||
type: `custom.googleapis.com/brainy/${metric.name}`,
|
|
||||||
labels: metric.labels
|
|
||||||
},
|
|
||||||
points: [{
|
|
||||||
interval: { endTime: { seconds: Date.now() / 1000 } },
|
|
||||||
value: { doubleValue: metric.value }
|
|
||||||
}]
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cloud Trace Integration
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
const brain = new Brainy({
|
|
||||||
tracing: {
|
|
||||||
enabled: true,
|
|
||||||
sampleRate: 0.1 // Sample 10% of requests
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## Security Best Practices
|
|
||||||
|
|
||||||
### 1. Workload Identity (GKE)
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# Enable Workload Identity
|
|
||||||
apiVersion: v1
|
|
||||||
kind: ServiceAccount
|
|
||||||
metadata:
|
|
||||||
name: brainy-sa
|
|
||||||
annotations:
|
|
||||||
iam.gke.io/gcp-service-account: brainy@PROJECT_ID.iam.gserviceaccount.com
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Binary Authorization
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# Ensure only signed container images
|
|
||||||
apiVersion: binaryauthorization.grafeas.io/v1beta1
|
|
||||||
kind: Policy
|
|
||||||
metadata:
|
|
||||||
name: brainy-policy
|
|
||||||
spec:
|
|
||||||
defaultAdmissionRule:
|
|
||||||
requireAttestationsBy:
|
|
||||||
- projects/PROJECT_ID/attestors/prod-attestor
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. VPC Service Controls
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Create VPC Service Perimeter
|
|
||||||
gcloud access-context-manager perimeters create brainy_perimeter \
|
|
||||||
--resources=projects/PROJECT_NUMBER \
|
|
||||||
--restricted-services=storage.googleapis.com \
|
|
||||||
--title="Brainy Security Perimeter"
|
|
||||||
```
|
|
||||||
|
|
||||||
## Cost Optimization
|
|
||||||
|
|
||||||
### 1. Preemptible VMs (80% savings)
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# GKE node pool with preemptible VMs
|
|
||||||
apiVersion: container.cnrm.cloud.google.com/v1beta1
|
|
||||||
kind: ContainerNodePool
|
|
||||||
metadata:
|
|
||||||
name: brainy-preemptible-pool
|
|
||||||
spec:
|
|
||||||
clusterRef:
|
|
||||||
name: brainy-cluster
|
|
||||||
config:
|
|
||||||
preemptible: true
|
|
||||||
machineType: n2-standard-2
|
|
||||||
autoscaling:
|
|
||||||
minNodeCount: 1
|
|
||||||
maxNodeCount: 10
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Cloud CDN for Static Assets
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Enable Cloud CDN for frequently accessed data
|
|
||||||
gcloud compute backend-buckets create brainy-assets \
|
|
||||||
--gcs-bucket-name=brainy-static
|
|
||||||
|
|
||||||
gcloud compute backend-buckets update brainy-assets \
|
|
||||||
--enable-cdn \
|
|
||||||
--cache-mode=CACHE_ALL_STATIC
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Committed Use Discounts
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Purchase committed use for predictable workloads
|
|
||||||
gcloud compute commitments create brainy-commitment \
|
|
||||||
--plan=TWELVE_MONTH \
|
|
||||||
--resources=vcpu=100,memory=400
|
|
||||||
```
|
|
||||||
|
|
||||||
## Deployment Automation
|
|
||||||
|
|
||||||
### Cloud Build CI/CD
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# cloudbuild.yaml
|
|
||||||
steps:
|
|
||||||
# Build container
|
|
||||||
- name: 'gcr.io/cloud-builders/docker'
|
|
||||||
args: ['build', '-t', 'gcr.io/$PROJECT_ID/brainy:$SHORT_SHA', '.']
|
|
||||||
|
|
||||||
# Push to registry
|
|
||||||
- name: 'gcr.io/cloud-builders/docker'
|
|
||||||
args: ['push', 'gcr.io/$PROJECT_ID/brainy:$SHORT_SHA']
|
|
||||||
|
|
||||||
# Deploy to Cloud Run
|
|
||||||
- name: 'gcr.io/cloud-builders/gcloud'
|
|
||||||
args:
|
|
||||||
- 'run'
|
|
||||||
- 'deploy'
|
|
||||||
- 'brainy'
|
|
||||||
- '--image=gcr.io/$PROJECT_ID/brainy:$SHORT_SHA'
|
|
||||||
- '--region=us-central1'
|
|
||||||
- '--platform=managed'
|
|
||||||
|
|
||||||
# Trigger on push to main
|
|
||||||
trigger:
|
|
||||||
branch:
|
|
||||||
name: main
|
|
||||||
```
|
|
||||||
|
|
||||||
### Terraform Infrastructure
|
|
||||||
|
|
||||||
```hcl
|
|
||||||
# main.tf
|
|
||||||
resource "google_cloud_run_service" "brainy" {
|
|
||||||
name = "brainy"
|
|
||||||
location = "us-central1"
|
|
||||||
|
|
||||||
template {
|
|
||||||
spec {
|
|
||||||
containers {
|
|
||||||
image = "gcr.io/${var.project_id}/brainy"
|
|
||||||
|
|
||||||
resources {
|
|
||||||
limits = {
|
|
||||||
cpu = "2"
|
|
||||||
memory = "2Gi"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
env {
|
|
||||||
name = "NODE_ENV"
|
|
||||||
value = "production"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
traffic {
|
|
||||||
percent = 100
|
|
||||||
latest_revision = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
resource "google_cloud_run_service_iam_member" "public" {
|
|
||||||
service = google_cloud_run_service.brainy.name
|
|
||||||
location = google_cloud_run_service.brainy.location
|
|
||||||
role = "roles/run.invoker"
|
|
||||||
member = "allUsers"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Performance Optimization
|
|
||||||
|
|
||||||
### 1. Memory Store (Redis Compatible)
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Brainy can use Memorystore for caching
|
|
||||||
const brain = new Brainy({
|
|
||||||
cache: {
|
|
||||||
type: 'redis',
|
|
||||||
options: {
|
|
||||||
host: process.env.REDIS_HOST || 'auto-detect',
|
|
||||||
port: 6379
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Cloud Spanner for Global Consistency
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
const brain = new Brainy({
|
|
||||||
metadata: {
|
|
||||||
type: 'spanner',
|
|
||||||
options: {
|
|
||||||
instance: 'brainy-instance',
|
|
||||||
database: 'brainy-db'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## Troubleshooting
|
|
||||||
|
|
||||||
### Common Issues
|
|
||||||
|
|
||||||
1. **Quota Exceeded**
|
|
||||||
```bash
|
|
||||||
# Check quotas
|
|
||||||
gcloud compute project-info describe --project=$PROJECT_ID
|
|
||||||
|
|
||||||
# Request increase
|
|
||||||
gcloud compute project-info add-metadata \
|
|
||||||
--metadata google-compute-default-region=us-central1
|
|
||||||
```
|
|
||||||
|
|
||||||
2. **Cold Starts**
|
|
||||||
|
|
||||||
**Progressive Initialization (Zero-Config)**
|
|
||||||
|
|
||||||
Brainy automatically detects Cloud Run and Cloud Functions environments
|
|
||||||
and uses progressive initialization for <200ms cold starts:
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Zero-config - Brainy auto-detects Cloud Run (K_SERVICE env var)
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 'gcs',
|
|
||||||
gcsNativeStorage: { bucketName: 'my-bucket' }
|
|
||||||
}
|
|
||||||
})
|
|
||||||
await brain.init() // Returns in <200ms
|
|
||||||
|
|
||||||
// First write validates bucket (lazy validation)
|
|
||||||
await brain.add('noun', { name: 'test' }) // Validates here
|
|
||||||
```
|
|
||||||
|
|
||||||
**Manual Override (if needed)**
|
|
||||||
```javascript
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 'gcs',
|
|
||||||
gcsNativeStorage: {
|
|
||||||
bucketName: 'my-bucket',
|
|
||||||
// Force specific mode
|
|
||||||
initMode: 'progressive' // 'auto' | 'progressive' | 'strict'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
| Mode | Cold Start | Best For |
|
|
||||||
|------|------------|----------|
|
|
||||||
| `auto` (default) | <200ms in cloud | Zero-config, auto-detects |
|
|
||||||
| `progressive` | <200ms always | Force fast init everywhere |
|
|
||||||
| `strict` | 100-500ms+ | Local dev, tests, debugging |
|
|
||||||
|
|
||||||
**Keep Warm (Alternative)**
|
|
||||||
```javascript
|
|
||||||
// Keep minimum instances warm
|
|
||||||
const brain = new Brainy({
|
|
||||||
warmup: {
|
|
||||||
enabled: true,
|
|
||||||
interval: 60000 // Ping every minute
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
**Readiness Detection**
|
|
||||||
|
|
||||||
Use the `brain.ready` Promise to ensure Brainy is initialized before handling requests:
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
let brain
|
|
||||||
|
|
||||||
async function handleRequest(req, res) {
|
|
||||||
if (!brain) {
|
|
||||||
brain = new Brainy({ storage: { type: 'gcs', ... } })
|
|
||||||
brain.init() // Fire and forget
|
|
||||||
}
|
|
||||||
|
|
||||||
// Wait for initialization to complete
|
|
||||||
await brain.ready
|
|
||||||
|
|
||||||
// Now safe to use brain methods
|
|
||||||
const results = await brain.find({ query: req.query.q })
|
|
||||||
res.json(results)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
For Cloud Run health checks, use `isFullyInitialized()` to verify all background tasks are complete:
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Health check endpoint for Cloud Run
|
|
||||||
app.get('/health', async (req, res) => {
|
|
||||||
try {
|
|
||||||
await brain.ready
|
|
||||||
res.json({
|
|
||||||
status: 'ready',
|
|
||||||
fullyInitialized: brain.isFullyInitialized()
|
|
||||||
})
|
|
||||||
} catch (error) {
|
|
||||||
res.status(503).json({ status: 'initializing' })
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
3. **Memory Pressure**
|
|
||||||
```javascript
|
|
||||||
// Optimize for GCP memory constraints
|
|
||||||
const brain = new Brainy({
|
|
||||||
memory: {
|
|
||||||
mode: 'aggressive', // Aggressive garbage collection
|
|
||||||
maxHeap: 0.8 // Use 80% of available memory
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## Production Checklist
|
|
||||||
|
|
||||||
- [ ] Enable Workload Identity for secure access
|
|
||||||
- [ ] Configure Cloud Armor for DDoS protection
|
|
||||||
- [ ] Set up Cloud KMS for encryption keys
|
|
||||||
- [ ] Enable VPC Service Controls
|
|
||||||
- [ ] Configure Cloud IAP for authentication
|
|
||||||
- [ ] Set up Cloud Monitoring dashboards
|
|
||||||
- [ ] Configure Error Reporting
|
|
||||||
- [ ] Enable Cloud Trace
|
|
||||||
- [ ] Set up budget alerts
|
|
||||||
- [ ] Configure backup and disaster recovery
|
|
||||||
|
|
||||||
## Support
|
|
||||||
|
|
||||||
- Documentation: https://brainy.soulcraft.ai/docs
|
|
||||||
- Issues: https://github.com/soulcraft/brainy/issues
|
|
||||||
- GCP Marketplace: https://console.cloud.google.com/marketplace/product/brainy
|
|
||||||
|
|
@ -1,727 +0,0 @@
|
||||||
# Kubernetes Deployment Guide for Brainy
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
Deploy Brainy on Kubernetes with automatic scaling, high availability, and zero-config dynamic adaptation. Works with any Kubernetes distribution (vanilla, EKS, GKE, AKS, OpenShift, etc.).
|
|
||||||
|
|
||||||
## Quick Start (Zero-Config)
|
|
||||||
|
|
||||||
### Basic Deployment
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# brainy-deployment.yaml
|
|
||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: brainy
|
|
||||||
labels:
|
|
||||||
app: brainy
|
|
||||||
spec:
|
|
||||||
replicas: 3
|
|
||||||
selector:
|
|
||||||
matchLabels:
|
|
||||||
app: brainy
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
app: brainy
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: brainy
|
|
||||||
image: soulcraft/brainy:latest
|
|
||||||
ports:
|
|
||||||
- containerPort: 3000
|
|
||||||
env:
|
|
||||||
- name: NODE_ENV
|
|
||||||
value: production
|
|
||||||
# Brainy auto-detects Kubernetes and configures accordingly
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
memory: "256Mi"
|
|
||||||
cpu: "100m"
|
|
||||||
limits:
|
|
||||||
memory: "1Gi"
|
|
||||||
cpu: "500m"
|
|
||||||
---
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: brainy-service
|
|
||||||
spec:
|
|
||||||
selector:
|
|
||||||
app: brainy
|
|
||||||
ports:
|
|
||||||
- protocol: TCP
|
|
||||||
port: 80
|
|
||||||
targetPort: 3000
|
|
||||||
type: LoadBalancer
|
|
||||||
```
|
|
||||||
|
|
||||||
Deploy:
|
|
||||||
```bash
|
|
||||||
kubectl apply -f brainy-deployment.yaml
|
|
||||||
```
|
|
||||||
|
|
||||||
## Production-Grade Setup
|
|
||||||
|
|
||||||
### 1. StatefulSet with Persistent Storage
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
apiVersion: v1
|
|
||||||
kind: StorageClass
|
|
||||||
metadata:
|
|
||||||
name: brainy-storage
|
|
||||||
provisioner: kubernetes.io/aws-ebs # Or your cloud provider
|
|
||||||
parameters:
|
|
||||||
type: gp3
|
|
||||||
iopsPerGB: "10"
|
|
||||||
---
|
|
||||||
apiVersion: apps/v1
|
|
||||||
kind: StatefulSet
|
|
||||||
metadata:
|
|
||||||
name: brainy
|
|
||||||
spec:
|
|
||||||
serviceName: brainy-headless
|
|
||||||
replicas: 3
|
|
||||||
selector:
|
|
||||||
matchLabels:
|
|
||||||
app: brainy
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
app: brainy
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: brainy
|
|
||||||
image: soulcraft/brainy:latest
|
|
||||||
ports:
|
|
||||||
- containerPort: 3000
|
|
||||||
env:
|
|
||||||
- name: NODE_ENV
|
|
||||||
value: production
|
|
||||||
- name: BRAINY_STORAGE_TYPE
|
|
||||||
value: filesystem
|
|
||||||
- name: BRAINY_STORAGE_PATH
|
|
||||||
value: /data
|
|
||||||
volumeMounts:
|
|
||||||
- name: data
|
|
||||||
mountPath: /data
|
|
||||||
livenessProbe:
|
|
||||||
httpGet:
|
|
||||||
path: /health
|
|
||||||
port: 3000
|
|
||||||
initialDelaySeconds: 30
|
|
||||||
periodSeconds: 10
|
|
||||||
readinessProbe:
|
|
||||||
httpGet:
|
|
||||||
path: /ready
|
|
||||||
port: 3000
|
|
||||||
initialDelaySeconds: 5
|
|
||||||
periodSeconds: 5
|
|
||||||
volumeClaimTemplates:
|
|
||||||
- metadata:
|
|
||||||
name: data
|
|
||||||
spec:
|
|
||||||
accessModes: ["ReadWriteOnce"]
|
|
||||||
storageClassName: brainy-storage
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
storage: 10Gi
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Horizontal Pod Autoscaler
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
apiVersion: autoscaling/v2
|
|
||||||
kind: HorizontalPodAutoscaler
|
|
||||||
metadata:
|
|
||||||
name: brainy-hpa
|
|
||||||
spec:
|
|
||||||
scaleTargetRef:
|
|
||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
name: brainy
|
|
||||||
minReplicas: 2
|
|
||||||
maxReplicas: 100
|
|
||||||
metrics:
|
|
||||||
- type: Resource
|
|
||||||
resource:
|
|
||||||
name: cpu
|
|
||||||
target:
|
|
||||||
type: Utilization
|
|
||||||
averageUtilization: 70
|
|
||||||
- type: Resource
|
|
||||||
resource:
|
|
||||||
name: memory
|
|
||||||
target:
|
|
||||||
type: Utilization
|
|
||||||
averageUtilization: 80
|
|
||||||
behavior:
|
|
||||||
scaleUp:
|
|
||||||
stabilizationWindowSeconds: 60
|
|
||||||
policies:
|
|
||||||
- type: Percent
|
|
||||||
value: 100
|
|
||||||
periodSeconds: 60
|
|
||||||
scaleDown:
|
|
||||||
stabilizationWindowSeconds: 300
|
|
||||||
policies:
|
|
||||||
- type: Percent
|
|
||||||
value: 10
|
|
||||||
periodSeconds: 60
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Ingress Configuration
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
apiVersion: networking.k8s.io/v1
|
|
||||||
kind: Ingress
|
|
||||||
metadata:
|
|
||||||
name: brainy-ingress
|
|
||||||
annotations:
|
|
||||||
kubernetes.io/ingress.class: nginx
|
|
||||||
cert-manager.io/cluster-issuer: letsencrypt-prod
|
|
||||||
nginx.ingress.kubernetes.io/rate-limit: "100"
|
|
||||||
nginx.ingress.kubernetes.io/proxy-body-size: "50m"
|
|
||||||
spec:
|
|
||||||
tls:
|
|
||||||
- hosts:
|
|
||||||
- api.brainy.example.com
|
|
||||||
secretName: brainy-tls
|
|
||||||
rules:
|
|
||||||
- host: api.brainy.example.com
|
|
||||||
http:
|
|
||||||
paths:
|
|
||||||
- path: /
|
|
||||||
pathType: Prefix
|
|
||||||
backend:
|
|
||||||
service:
|
|
||||||
name: brainy-service
|
|
||||||
port:
|
|
||||||
number: 80
|
|
||||||
```
|
|
||||||
|
|
||||||
## Zero-Config Storage Options
|
|
||||||
|
|
||||||
### Option 1: S3-Compatible Storage (Recommended)
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Secret
|
|
||||||
metadata:
|
|
||||||
name: brainy-s3-credentials
|
|
||||||
type: Opaque
|
|
||||||
data:
|
|
||||||
access-key: <base64-encoded-key>
|
|
||||||
secret-key: <base64-encoded-secret>
|
|
||||||
---
|
|
||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: brainy
|
|
||||||
spec:
|
|
||||||
template:
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: brainy
|
|
||||||
env:
|
|
||||||
- name: BRAINY_STORAGE_TYPE
|
|
||||||
value: s3
|
|
||||||
- name: AWS_ACCESS_KEY_ID
|
|
||||||
valueFrom:
|
|
||||||
secretKeyRef:
|
|
||||||
name: brainy-s3-credentials
|
|
||||||
key: access-key
|
|
||||||
- name: AWS_SECRET_ACCESS_KEY
|
|
||||||
valueFrom:
|
|
||||||
secretKeyRef:
|
|
||||||
name: brainy-s3-credentials
|
|
||||||
key: secret-key
|
|
||||||
- name: S3_BUCKET
|
|
||||||
value: brainy-data
|
|
||||||
- name: AWS_REGION
|
|
||||||
value: us-east-1
|
|
||||||
```
|
|
||||||
|
|
||||||
### Option 2: MinIO (Self-Hosted S3)
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: minio
|
|
||||||
spec:
|
|
||||||
replicas: 1
|
|
||||||
selector:
|
|
||||||
matchLabels:
|
|
||||||
app: minio
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
app: minio
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: minio
|
|
||||||
image: minio/minio:latest
|
|
||||||
args:
|
|
||||||
- server
|
|
||||||
- /data
|
|
||||||
env:
|
|
||||||
- name: MINIO_ROOT_USER
|
|
||||||
value: brainy
|
|
||||||
- name: MINIO_ROOT_PASSWORD
|
|
||||||
value: brainy123456
|
|
||||||
volumeMounts:
|
|
||||||
- name: data
|
|
||||||
mountPath: /data
|
|
||||||
volumes:
|
|
||||||
- name: data
|
|
||||||
persistentVolumeClaim:
|
|
||||||
claimName: minio-pvc
|
|
||||||
---
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: minio-service
|
|
||||||
spec:
|
|
||||||
selector:
|
|
||||||
app: minio
|
|
||||||
ports:
|
|
||||||
- port: 9000
|
|
||||||
targetPort: 9000
|
|
||||||
```
|
|
||||||
|
|
||||||
### Option 3: Shared NFS Storage
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
apiVersion: v1
|
|
||||||
kind: PersistentVolume
|
|
||||||
metadata:
|
|
||||||
name: brainy-nfs-pv
|
|
||||||
spec:
|
|
||||||
capacity:
|
|
||||||
storage: 100Gi
|
|
||||||
accessModes:
|
|
||||||
- ReadWriteMany
|
|
||||||
nfs:
|
|
||||||
server: nfs-server.example.com
|
|
||||||
path: /export/brainy
|
|
||||||
---
|
|
||||||
apiVersion: v1
|
|
||||||
kind: PersistentVolumeClaim
|
|
||||||
metadata:
|
|
||||||
name: brainy-nfs-pvc
|
|
||||||
spec:
|
|
||||||
accessModes:
|
|
||||||
- ReadWriteMany
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
storage: 100Gi
|
|
||||||
```
|
|
||||||
|
|
||||||
## High Availability Configuration
|
|
||||||
|
|
||||||
### 1. Pod Anti-Affinity
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: brainy
|
|
||||||
spec:
|
|
||||||
template:
|
|
||||||
spec:
|
|
||||||
affinity:
|
|
||||||
podAntiAffinity:
|
|
||||||
requiredDuringSchedulingIgnoredDuringExecution:
|
|
||||||
- labelSelector:
|
|
||||||
matchExpressions:
|
|
||||||
- key: app
|
|
||||||
operator: In
|
|
||||||
values:
|
|
||||||
- brainy
|
|
||||||
topologyKey: kubernetes.io/hostname
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Pod Disruption Budget
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
apiVersion: policy/v1
|
|
||||||
kind: PodDisruptionBudget
|
|
||||||
metadata:
|
|
||||||
name: brainy-pdb
|
|
||||||
spec:
|
|
||||||
minAvailable: 2
|
|
||||||
selector:
|
|
||||||
matchLabels:
|
|
||||||
app: brainy
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Multi-Zone Deployment
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: brainy
|
|
||||||
spec:
|
|
||||||
template:
|
|
||||||
spec:
|
|
||||||
topologySpreadConstraints:
|
|
||||||
- maxSkew: 1
|
|
||||||
topologyKey: topology.kubernetes.io/zone
|
|
||||||
whenUnsatisfiable: DoNotSchedule
|
|
||||||
labelSelector:
|
|
||||||
matchLabels:
|
|
||||||
app: brainy
|
|
||||||
```
|
|
||||||
|
|
||||||
## Monitoring & Observability
|
|
||||||
|
|
||||||
### 1. Prometheus Metrics
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: brainy-metrics
|
|
||||||
annotations:
|
|
||||||
prometheus.io/scrape: "true"
|
|
||||||
prometheus.io/port: "9090"
|
|
||||||
prometheus.io/path: "/metrics"
|
|
||||||
spec:
|
|
||||||
selector:
|
|
||||||
app: brainy
|
|
||||||
ports:
|
|
||||||
- name: metrics
|
|
||||||
port: 9090
|
|
||||||
targetPort: 9090
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Grafana Dashboard
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
apiVersion: v1
|
|
||||||
kind: ConfigMap
|
|
||||||
metadata:
|
|
||||||
name: brainy-dashboard
|
|
||||||
data:
|
|
||||||
dashboard.json: |
|
|
||||||
{
|
|
||||||
"dashboard": {
|
|
||||||
"title": "Brainy Metrics",
|
|
||||||
"panels": [
|
|
||||||
{
|
|
||||||
"title": "Request Rate",
|
|
||||||
"targets": [
|
|
||||||
{
|
|
||||||
"expr": "rate(brainy_requests_total[5m])"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"title": "Response Time",
|
|
||||||
"targets": [
|
|
||||||
{
|
|
||||||
"expr": "histogram_quantile(0.95, brainy_response_time)"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Logging with Fluentd
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
apiVersion: v1
|
|
||||||
kind: ConfigMap
|
|
||||||
metadata:
|
|
||||||
name: fluentd-config
|
|
||||||
data:
|
|
||||||
fluent.conf: |
|
|
||||||
<source>
|
|
||||||
@type tail
|
|
||||||
path /var/log/containers/brainy*.log
|
|
||||||
pos_file /var/log/fluentd-brainy.log.pos
|
|
||||||
tag brainy.*
|
|
||||||
<parse>
|
|
||||||
@type json
|
|
||||||
</parse>
|
|
||||||
</source>
|
|
||||||
|
|
||||||
<match brainy.**>
|
|
||||||
@type elasticsearch
|
|
||||||
host elasticsearch.logging.svc.cluster.local
|
|
||||||
port 9200
|
|
||||||
logstash_format true
|
|
||||||
logstash_prefix brainy
|
|
||||||
</match>
|
|
||||||
```
|
|
||||||
|
|
||||||
## Security Best Practices
|
|
||||||
|
|
||||||
### 1. Network Policies
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
apiVersion: networking.k8s.io/v1
|
|
||||||
kind: NetworkPolicy
|
|
||||||
metadata:
|
|
||||||
name: brainy-netpol
|
|
||||||
spec:
|
|
||||||
podSelector:
|
|
||||||
matchLabels:
|
|
||||||
app: brainy
|
|
||||||
policyTypes:
|
|
||||||
- Ingress
|
|
||||||
- Egress
|
|
||||||
ingress:
|
|
||||||
- from:
|
|
||||||
- namespaceSelector:
|
|
||||||
matchLabels:
|
|
||||||
name: ingress-nginx
|
|
||||||
ports:
|
|
||||||
- protocol: TCP
|
|
||||||
port: 3000
|
|
||||||
egress:
|
|
||||||
- to:
|
|
||||||
- namespaceSelector: {}
|
|
||||||
ports:
|
|
||||||
- protocol: TCP
|
|
||||||
port: 443 # HTTPS
|
|
||||||
- protocol: TCP
|
|
||||||
port: 9000 # MinIO/S3
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Pod Security Policy
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
apiVersion: policy/v1beta1
|
|
||||||
kind: PodSecurityPolicy
|
|
||||||
metadata:
|
|
||||||
name: brainy-psp
|
|
||||||
spec:
|
|
||||||
privileged: false
|
|
||||||
allowPrivilegeEscalation: false
|
|
||||||
requiredDropCapabilities:
|
|
||||||
- ALL
|
|
||||||
volumes:
|
|
||||||
- 'configMap'
|
|
||||||
- 'emptyDir'
|
|
||||||
- 'projected'
|
|
||||||
- 'secret'
|
|
||||||
- 'persistentVolumeClaim'
|
|
||||||
runAsUser:
|
|
||||||
rule: 'MustRunAsNonRoot'
|
|
||||||
seLinux:
|
|
||||||
rule: 'RunAsAny'
|
|
||||||
fsGroup:
|
|
||||||
rule: 'RunAsAny'
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. RBAC Configuration
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
apiVersion: v1
|
|
||||||
kind: ServiceAccount
|
|
||||||
metadata:
|
|
||||||
name: brainy-sa
|
|
||||||
---
|
|
||||||
apiVersion: rbac.authorization.k8s.io/v1
|
|
||||||
kind: Role
|
|
||||||
metadata:
|
|
||||||
name: brainy-role
|
|
||||||
rules:
|
|
||||||
- apiGroups: [""]
|
|
||||||
resources: ["configmaps"]
|
|
||||||
verbs: ["get", "list", "watch"]
|
|
||||||
- apiGroups: [""]
|
|
||||||
resources: ["secrets"]
|
|
||||||
verbs: ["get"]
|
|
||||||
---
|
|
||||||
apiVersion: rbac.authorization.k8s.io/v1
|
|
||||||
kind: RoleBinding
|
|
||||||
metadata:
|
|
||||||
name: brainy-rolebinding
|
|
||||||
subjects:
|
|
||||||
- kind: ServiceAccount
|
|
||||||
name: brainy-sa
|
|
||||||
roleRef:
|
|
||||||
kind: Role
|
|
||||||
name: brainy-role
|
|
||||||
apiGroup: rbac.authorization.k8s.io
|
|
||||||
```
|
|
||||||
|
|
||||||
## GitOps with ArgoCD
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
apiVersion: argoproj.io/v1alpha1
|
|
||||||
kind: Application
|
|
||||||
metadata:
|
|
||||||
name: brainy
|
|
||||||
namespace: argocd
|
|
||||||
spec:
|
|
||||||
project: default
|
|
||||||
source:
|
|
||||||
repoURL: https://github.com/yourorg/brainy-k8s
|
|
||||||
targetRevision: HEAD
|
|
||||||
path: manifests
|
|
||||||
destination:
|
|
||||||
server: https://kubernetes.default.svc
|
|
||||||
namespace: brainy
|
|
||||||
syncPolicy:
|
|
||||||
automated:
|
|
||||||
prune: true
|
|
||||||
selfHeal: true
|
|
||||||
syncOptions:
|
|
||||||
- CreateNamespace=true
|
|
||||||
```
|
|
||||||
|
|
||||||
## Helm Chart Installation
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Add Brainy Helm repository
|
|
||||||
helm repo add brainy https://charts.brainy.io
|
|
||||||
helm repo update
|
|
||||||
|
|
||||||
# Install with custom values
|
|
||||||
cat > values.yaml << EOF
|
|
||||||
replicaCount: 3
|
|
||||||
image:
|
|
||||||
repository: soulcraft/brainy
|
|
||||||
tag: latest
|
|
||||||
pullPolicy: IfNotPresent
|
|
||||||
|
|
||||||
service:
|
|
||||||
type: LoadBalancer
|
|
||||||
port: 80
|
|
||||||
|
|
||||||
ingress:
|
|
||||||
enabled: true
|
|
||||||
className: nginx
|
|
||||||
hosts:
|
|
||||||
- host: api.brainy.example.com
|
|
||||||
paths:
|
|
||||||
- path: /
|
|
||||||
pathType: Prefix
|
|
||||||
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
cpu: 1000m
|
|
||||||
memory: 1Gi
|
|
||||||
requests:
|
|
||||||
cpu: 100m
|
|
||||||
memory: 256Mi
|
|
||||||
|
|
||||||
autoscaling:
|
|
||||||
enabled: true
|
|
||||||
minReplicas: 2
|
|
||||||
maxReplicas: 100
|
|
||||||
targetCPUUtilizationPercentage: 70
|
|
||||||
|
|
||||||
storage:
|
|
||||||
type: s3
|
|
||||||
s3:
|
|
||||||
bucket: brainy-data
|
|
||||||
region: us-east-1
|
|
||||||
EOF
|
|
||||||
|
|
||||||
helm install brainy brainy/brainy -f values.yaml
|
|
||||||
```
|
|
||||||
|
|
||||||
## Cost Optimization
|
|
||||||
|
|
||||||
### 1. Spot/Preemptible Nodes
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
apiVersion: v1
|
|
||||||
kind: NodePool
|
|
||||||
metadata:
|
|
||||||
name: brainy-spot-pool
|
|
||||||
spec:
|
|
||||||
nodeSelector:
|
|
||||||
node.kubernetes.io/lifecycle: spot
|
|
||||||
taints:
|
|
||||||
- key: spot
|
|
||||||
value: "true"
|
|
||||||
effect: NoSchedule
|
|
||||||
tolerations:
|
|
||||||
- key: spot
|
|
||||||
operator: Equal
|
|
||||||
value: "true"
|
|
||||||
effect: NoSchedule
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Vertical Pod Autoscaler
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
apiVersion: autoscaling.k8s.io/v1
|
|
||||||
kind: VerticalPodAutoscaler
|
|
||||||
metadata:
|
|
||||||
name: brainy-vpa
|
|
||||||
spec:
|
|
||||||
targetRef:
|
|
||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
name: brainy
|
|
||||||
updatePolicy:
|
|
||||||
updateMode: "Auto"
|
|
||||||
resourcePolicy:
|
|
||||||
containerPolicies:
|
|
||||||
- containerName: brainy
|
|
||||||
minAllowed:
|
|
||||||
cpu: 100m
|
|
||||||
memory: 128Mi
|
|
||||||
maxAllowed:
|
|
||||||
cpu: 2
|
|
||||||
memory: 2Gi
|
|
||||||
```
|
|
||||||
|
|
||||||
## Troubleshooting
|
|
||||||
|
|
||||||
### Common Issues
|
|
||||||
|
|
||||||
1. **Pod CrashLoopBackOff**
|
|
||||||
```bash
|
|
||||||
kubectl logs -f pod/brainy-xxx
|
|
||||||
kubectl describe pod brainy-xxx
|
|
||||||
```
|
|
||||||
|
|
||||||
2. **Storage Issues**
|
|
||||||
```bash
|
|
||||||
kubectl get pv,pvc
|
|
||||||
kubectl describe pvc brainy-data
|
|
||||||
```
|
|
||||||
|
|
||||||
3. **Network Connectivity**
|
|
||||||
```bash
|
|
||||||
kubectl exec -it pod/brainy-xxx -- curl http://brainy-service/health
|
|
||||||
kubectl get endpoints brainy-service
|
|
||||||
```
|
|
||||||
|
|
||||||
4. **Memory Pressure**
|
|
||||||
```bash
|
|
||||||
kubectl top pods -l app=brainy
|
|
||||||
kubectl describe node
|
|
||||||
```
|
|
||||||
|
|
||||||
## Production Checklist
|
|
||||||
|
|
||||||
- [ ] High availability with multiple replicas
|
|
||||||
- [ ] Pod disruption budgets configured
|
|
||||||
- [ ] Resource limits and requests set
|
|
||||||
- [ ] Horizontal and vertical autoscaling enabled
|
|
||||||
- [ ] Persistent storage configured
|
|
||||||
- [ ] Network policies in place
|
|
||||||
- [ ] RBAC properly configured
|
|
||||||
- [ ] Monitoring and alerting setup
|
|
||||||
- [ ] Backup and disaster recovery plan
|
|
||||||
- [ ] Security scanning enabled
|
|
||||||
- [ ] GitOps deployment pipeline
|
|
||||||
|
|
||||||
## Support
|
|
||||||
|
|
||||||
- Documentation: https://brainy.soulcraft.ai/docs
|
|
||||||
- Helm Charts: https://github.com/soulcraft/brainy-charts
|
|
||||||
- Issues: https://github.com/soulcraft/brainy/issues
|
|
||||||
- Slack: https://brainy-community.slack.com
|
|
||||||
10
docs/eli5.md
10
docs/eli5.md
|
|
@ -59,7 +59,7 @@ Brainy can narrow any result set down by exact labels or ranges in the same brea
|
||||||
|
|
||||||
- **Live dashboards.** You can define running totals that Brainy keeps updated automatically — things like "total sales this month by region" or "average response time per service." Every time new data comes in, the numbers stay current with no manual recalculation.
|
- **Live dashboards.** You can define running totals that Brainy keeps updated automatically — things like "total sales this month by region" or "average response time per service." Every time new data comes in, the numbers stay current with no manual recalculation.
|
||||||
|
|
||||||
- **Safe experiments.** You can branch your entire knowledge base — like branching code in version control — make changes on the branch, and either keep them or throw them away without ever touching the original.
|
- **Time travel.** Every committed change becomes part of the database's history. You can pin the current state as a frozen view, see the whole knowledge base exactly as it was last week, try out changes in a scratch copy that never touches the real data, and take instant backups.
|
||||||
|
|
||||||
- **Universal vocabulary.** Brainy ships with a shared language of 42 kinds of things (Person, Document, Task, Concept, Event…) and 127 kinds of connections (Contains, DependsOn, Creates, RelatedTo…). This means data from different sources speaks the same language without you having to translate.
|
- **Universal vocabulary.** Brainy ships with a shared language of 42 kinds of things (Person, Document, Task, Concept, Event…) and 127 kinds of connections (Contains, DependsOn, Creates, RelatedTo…). This means data from different sources speaks the same language without you having to translate.
|
||||||
|
|
||||||
|
|
@ -101,11 +101,11 @@ Most applications that need to store and search knowledge end up stitching toget
|
||||||
- Plus glue code, sync jobs, ETL pipelines, and 3am incidents
|
- Plus glue code, sync jobs, ETL pipelines, and 3am incidents
|
||||||
|
|
||||||
**After Brainy** — one thing:
|
**After Brainy** — one thing:
|
||||||
Search, graph, filter, files, branches, and imports — unified in a single query.
|
Search, graph, filter, files, time travel, and imports — unified in a single library.
|
||||||
|
|
||||||
### What Each Tool Is Missing
|
### What Each Tool Is Missing
|
||||||
|
|
||||||
| Tool | Search | Graph | Filter | VFS | Branch | Import |
|
| Tool | Search | Graph | Filter | VFS | Time travel | Import |
|
||||||
|---|:---:|:---:|:---:|:---:|:---:|:---:|
|
|---|:---:|:---:|:---:|:---:|:---:|:---:|
|
||||||
| **Brainy** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
| **Brainy** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||||
| *— Vector databases —* | | | | | | |
|
| *— Vector databases —* | | | | | | |
|
||||||
|
|
@ -133,7 +133,7 @@ Brainy is the only row with every box checked. And it runs all of them in a sing
|
||||||
|
|
||||||
### One library, any scale
|
### One library, any scale
|
||||||
|
|
||||||
Brainy scales from a single laptop to billions of entities without changing a line of code. Small datasets live in memory. Larger ones spill to disk. At cloud scale, Brainy uses S3-compatible storage and automatically shards across nodes — the same API the whole way. Up to ten billion entities is fully implemented today.
|
Brainy scales from a quick experiment to serious production datasets without changing a line of code. Small datasets live entirely in memory. Larger ones spill to disk, where Brainy shards and compresses everything automatically. Need a backup or a copy? Snapshots are instant — the same API the whole way.
|
||||||
|
|
||||||
Add Cortex and you also unlock memory-mapped storage — aggregate state lives directly in the operating system's memory with zero serialization overhead, as fast as the hardware allows.
|
Add Cortex and you also unlock memory-mapped storage — aggregate state lives directly in the operating system's memory with zero serialization overhead, as fast as the hardware allows.
|
||||||
|
|
||||||
|
|
@ -147,7 +147,7 @@ Add Cortex and you also unlock memory-mapped storage — aggregate state lives d
|
||||||
- **Searchable knowledge bases** — Build institutional memory that links documents automatically and surfaces answers across the full web of related information.
|
- **Searchable knowledge bases** — Build institutional memory that links documents automatically and surfaces answers across the full web of related information.
|
||||||
- **Semantic document search** — Index PDFs, code, or media and find them by meaning, not just keywords.
|
- **Semantic document search** — Index PDFs, code, or media and find them by meaning, not just keywords.
|
||||||
- **Relationship-aware recommendations** — Power product catalogs or content platforms where every recommendation understands what connects to what.
|
- **Relationship-aware recommendations** — Power product catalogs or content platforms where every recommendation understands what connects to what.
|
||||||
- **Safe experiments** — Let teams branch the knowledge base, experiment independently, and merge when ready — just like branching code.
|
- **Safe experiments** — Test risky changes against a scratch copy of the knowledge base, audit exactly what changed and when, and roll back to any snapshot instantly.
|
||||||
- **Unified business platforms** — Combine booking, CRM, inventory, and analytics in one queryable knowledge graph with no sync pipeline.
|
- **Unified business platforms** — Combine booking, CRM, inventory, and analytics in one queryable knowledge graph with no sync pipeline.
|
||||||
|
|
||||||
### What Brainy is good at
|
### What Brainy is good at
|
||||||
|
|
|
||||||
|
|
@ -1,413 +0,0 @@
|
||||||
# 🚀 Brainy 2.0 - Complete Feature List
|
|
||||||
|
|
||||||
> **The Truth**: Brainy is MORE powerful than previously documented! This is the complete list of ALL implemented features.
|
|
||||||
|
|
||||||
## 🧠 Core Intelligence Engine
|
|
||||||
|
|
||||||
### Triple Intelligence System ✅
|
|
||||||
Unified query system that automatically combines:
|
|
||||||
- **Vector Search**: HNSW-indexed semantic similarity (O(log n) performance)
|
|
||||||
- **Graph Traversal**: Relationship-based discovery
|
|
||||||
- **Field Filtering**: Metadata and attribute queries
|
|
||||||
- **Auto-optimization**: Queries are automatically optimized based on data patterns
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// All three intelligences work together automatically
|
|
||||||
const results = await brain.find({
|
|
||||||
like: 'AI research', // Vector search
|
|
||||||
where: { year: 2024 }, // Metadata filtering
|
|
||||||
connected: { to: authorId } // Graph traversal
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Neural Query Understanding ✅
|
|
||||||
- **220+ embedded patterns** for query intent detection
|
|
||||||
- Natural language query processing
|
|
||||||
- Automatic query type detection
|
|
||||||
- Query rewriting and optimization
|
|
||||||
|
|
||||||
## 🔧 12+ Production Augmentations
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Full crash recovery, checkpointing, replay
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Entity Registry ✅
|
|
||||||
```typescript
|
|
||||||
import { EntityRegistryAugmentation } from 'brainy'
|
|
||||||
// Bloom filter-based deduplication for streaming data
|
|
||||||
// Handles millions of entities with minimal memory
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Auto-Register Entities ✅
|
|
||||||
```typescript
|
|
||||||
import { AutoRegisterEntitiesAugmentation } from 'brainy'
|
|
||||||
// Automatically extracts and registers entities from text
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4. Intelligent Verb Scoring ✅
|
|
||||||
```typescript
|
|
||||||
import { IntelligentVerbScoringAugmentation } from 'brainy'
|
|
||||||
// Multi-factor relationship strength:
|
|
||||||
// - Semantic similarity
|
|
||||||
// - Temporal decay
|
|
||||||
// - Frequency amplification
|
|
||||||
// - Context awareness
|
|
||||||
```
|
|
||||||
|
|
||||||
### 5. Batch Processing ✅
|
|
||||||
```typescript
|
|
||||||
import { BatchProcessingAugmentation } from 'brainy'
|
|
||||||
// Adaptive batching with backpressure
|
|
||||||
// Dynamically adjusts batch size based on load
|
|
||||||
```
|
|
||||||
|
|
||||||
### 6. Connection Pool ✅
|
|
||||||
```typescript
|
|
||||||
import { ConnectionPoolAugmentation } from 'brainy'
|
|
||||||
// Auto-scaling connection management
|
|
||||||
// Optimized for distributed operations
|
|
||||||
```
|
|
||||||
|
|
||||||
### 7. Request Deduplicator ✅
|
|
||||||
```typescript
|
|
||||||
import { RequestDeduplicatorAugmentation } from 'brainy'
|
|
||||||
// In-flight request deduplication
|
|
||||||
// 3x performance boost for concurrent operations
|
|
||||||
```
|
|
||||||
|
|
||||||
### 8. WebSocket Conduit ✅
|
|
||||||
```typescript
|
|
||||||
import { WebSocketConduitAugmentation } from 'brainy'
|
|
||||||
// Real-time bidirectional streaming
|
|
||||||
// Auto-reconnection and heartbeat
|
|
||||||
```
|
|
||||||
|
|
||||||
### 9. WebRTC Conduit ✅
|
|
||||||
```typescript
|
|
||||||
import { WebRTCConduitAugmentation } from 'brainy'
|
|
||||||
// Peer-to-peer data channels
|
|
||||||
// Direct browser-to-browser communication
|
|
||||||
```
|
|
||||||
|
|
||||||
### 10. Memory Storage Optimization ✅
|
|
||||||
```typescript
|
|
||||||
import { MemoryStorageAugmentation } from 'brainy'
|
|
||||||
// Memory-specific optimizations
|
|
||||||
// Circular buffers, compression
|
|
||||||
```
|
|
||||||
|
|
||||||
### 11. Server Search Conduit ✅
|
|
||||||
```typescript
|
|
||||||
import { ServerSearchConduitAugmentation } from 'brainy'
|
|
||||||
// Distributed query execution
|
|
||||||
// Load balancing across nodes
|
|
||||||
```
|
|
||||||
|
|
||||||
### 12. Neural Import ✅
|
|
||||||
```typescript
|
|
||||||
import { NeuralImportAugmentation } from 'brainy'
|
|
||||||
// AI-powered data understanding
|
|
||||||
// Automatic entity detection and classification
|
|
||||||
// Relationship discovery
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🤖 Neural Import Capabilities (FULLY IMPLEMENTED!)
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const neuralImport = new NeuralImport(brain)
|
|
||||||
|
|
||||||
// ALL of these work TODAY:
|
|
||||||
await neuralImport.neuralImport('data.csv')
|
|
||||||
await neuralImport.detectEntitiesWithNeuralAnalysis(data)
|
|
||||||
await neuralImport.detectNounType(entity)
|
|
||||||
await neuralImport.detectRelationships(entities)
|
|
||||||
await neuralImport.generateInsights(data)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Features:
|
|
||||||
- **Auto-detects file format** (CSV, JSON, XML, etc.)
|
|
||||||
- **Identifies entity types** using AI
|
|
||||||
- **Discovers relationships** between entities
|
|
||||||
- **Generates insights** about the data
|
|
||||||
- **Creates optimal graph structure** automatically
|
|
||||||
|
|
||||||
## 🎯 Zero-Config Model Loading Cascade
|
|
||||||
|
|
||||||
Brainy automatically loads models with ZERO configuration required:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const brain = new Brainy() // That's it!
|
|
||||||
await brain.init()
|
|
||||||
// Models load automatically from best available source
|
|
||||||
```
|
|
||||||
|
|
||||||
### Loading Priority:
|
|
||||||
1. **Local Cache** (./models) - Instant, no network
|
|
||||||
2. **CDN** (models.soulcraft.com) - Fast, global [Coming Soon]
|
|
||||||
3. **GitHub Releases** - Reliable backup
|
|
||||||
4. **HuggingFace** - Ultimate fallback
|
|
||||||
|
|
||||||
### Key Features:
|
|
||||||
- **Automatic fallback** if sources fail
|
|
||||||
- **Model verification** with checksums
|
|
||||||
- **Offline support** with bundled models
|
|
||||||
- **No environment variables needed**
|
|
||||||
- **Works in all environments** (Node, Browser, Workers)
|
|
||||||
|
|
||||||
## 🏢 Distributed Operation Modes
|
|
||||||
|
|
||||||
### Reader Mode ✅
|
|
||||||
```typescript
|
|
||||||
const brain = new Brainy({ mode: 'reader' })
|
|
||||||
// Optimized for read-heavy workloads
|
|
||||||
// 80% cache ratio, aggressive prefetch
|
|
||||||
// 1 hour TTL, minimal writes
|
|
||||||
```
|
|
||||||
|
|
||||||
### Writer Mode ✅
|
|
||||||
```typescript
|
|
||||||
const brain = new Brainy({ mode: 'writer' })
|
|
||||||
// Optimized for write-heavy workloads
|
|
||||||
// Large write buffers, batch writes
|
|
||||||
// Minimal caching, fast ingestion
|
|
||||||
```
|
|
||||||
|
|
||||||
### Hybrid Mode ✅
|
|
||||||
```typescript
|
|
||||||
const brain = new Brainy({ mode: 'hybrid' })
|
|
||||||
// Balanced for mixed workloads
|
|
||||||
// Adaptive caching and batching
|
|
||||||
```
|
|
||||||
|
|
||||||
## 💾 Advanced Caching System
|
|
||||||
|
|
||||||
### 3-Level Cache Architecture ✅
|
|
||||||
```typescript
|
|
||||||
const cacheConfig = {
|
|
||||||
hotCache: {
|
|
||||||
size: 1000, // L1 - RAM
|
|
||||||
ttl: 60000 // 1 minute
|
|
||||||
},
|
|
||||||
warmCache: {
|
|
||||||
size: 10000, // L2 - Fast storage
|
|
||||||
ttl: 300000 // 5 minutes
|
|
||||||
},
|
|
||||||
coldCache: {
|
|
||||||
size: 100000, // L3 - Persistent
|
|
||||||
ttl: null // No expiry
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cache Features:
|
|
||||||
- **Automatic promotion/demotion** between levels
|
|
||||||
- **LRU eviction** within each level
|
|
||||||
- **Compression** for cold cache
|
|
||||||
- **Statistics tracking** for optimization
|
|
||||||
|
|
||||||
## 📊 Comprehensive Statistics
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const stats = await brain.getStats()
|
|
||||||
// Returns detailed metrics:
|
|
||||||
{
|
|
||||||
nouns: {
|
|
||||||
count, created, updated, deleted,
|
|
||||||
size, avgSize
|
|
||||||
},
|
|
||||||
verbs: {
|
|
||||||
count, created, types,
|
|
||||||
weights: { min, max, avg }
|
|
||||||
},
|
|
||||||
vectors: {
|
|
||||||
dimensions: 384,
|
|
||||||
indexSize, partitions,
|
|
||||||
avgSearchTime
|
|
||||||
},
|
|
||||||
cache: {
|
|
||||||
hits, misses, evictions,
|
|
||||||
hitRate, sizes
|
|
||||||
},
|
|
||||||
performance: {
|
|
||||||
operations, avgTimes,
|
|
||||||
p95Latency, p99Latency
|
|
||||||
},
|
|
||||||
storage: {
|
|
||||||
used, available,
|
|
||||||
compression, files
|
|
||||||
},
|
|
||||||
throttling: {
|
|
||||||
delays, rateLimited,
|
|
||||||
backoffMs, retries
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🚀 GPU Acceleration Support
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Automatic GPU detection
|
|
||||||
const device = await detectBestDevice()
|
|
||||||
// Returns: 'cpu' | 'webgpu' | 'cuda'
|
|
||||||
|
|
||||||
// WebGPU in browser (when available)
|
|
||||||
if (device === 'webgpu') {
|
|
||||||
// Transformer models use WebGPU automatically
|
|
||||||
}
|
|
||||||
|
|
||||||
// CUDA in Node.js (future GPU support)
|
|
||||||
if (device === 'cuda') {
|
|
||||||
// Future: GPU acceleration for embeddings
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🔄 Adaptive Systems
|
|
||||||
|
|
||||||
### Adaptive Backpressure ✅
|
|
||||||
```typescript
|
|
||||||
// Automatically adjusts flow based on system load
|
|
||||||
// Prevents OOM and maintains throughput
|
|
||||||
```
|
|
||||||
|
|
||||||
### Adaptive Socket Manager ✅
|
|
||||||
```typescript
|
|
||||||
// Dynamic connection pooling
|
|
||||||
// Scales connections based on traffic patterns
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cache Auto-Configuration ✅
|
|
||||||
```typescript
|
|
||||||
// Sizes cache based on available memory
|
|
||||||
// Adjusts strategies based on usage patterns
|
|
||||||
```
|
|
||||||
|
|
||||||
### S3 Throttling Protection ✅
|
|
||||||
```typescript
|
|
||||||
// Built-in exponential backoff
|
|
||||||
// Rate limit detection and adaptation
|
|
||||||
// Automatic retry with jitter
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🛠️ Storage Adapters
|
|
||||||
|
|
||||||
All included, auto-selected based on environment:
|
|
||||||
|
|
||||||
### FileSystem Storage ✅
|
|
||||||
- Default for Node.js
|
|
||||||
- Efficient file-based storage
|
|
||||||
- Automatic directory management
|
|
||||||
|
|
||||||
### Memory Storage ✅
|
|
||||||
- Ultra-fast in-memory operations
|
|
||||||
- Perfect for testing and temporary data
|
|
||||||
- Circular buffer support
|
|
||||||
|
|
||||||
### OPFS Storage ✅
|
|
||||||
- Browser persistent storage
|
|
||||||
- Survives page refreshes
|
|
||||||
- Quota management
|
|
||||||
|
|
||||||
### S3 Storage ✅
|
|
||||||
- AWS S3 compatible
|
|
||||||
- Automatic multipart uploads
|
|
||||||
- Throttling protection
|
|
||||||
- Batch operations
|
|
||||||
|
|
||||||
## 🎨 Natural Language Processing
|
|
||||||
|
|
||||||
### Built-in Patterns (220+)
|
|
||||||
- Question types (what, why, how, when, where)
|
|
||||||
- Temporal queries (yesterday, last week, 2024)
|
|
||||||
- Comparative queries (better than, similar to)
|
|
||||||
- Aggregations (count, sum, average)
|
|
||||||
- Filters (only, except, without)
|
|
||||||
- Relationships (related to, connected with)
|
|
||||||
|
|
||||||
### Coverage: 94-98% of typical queries!
|
|
||||||
|
|
||||||
## 🔐 Security Features
|
|
||||||
|
|
||||||
### Built-in Security ✅
|
|
||||||
- Automatic input sanitization
|
|
||||||
- SQL injection prevention
|
|
||||||
- XSS protection for web contexts
|
|
||||||
- Rate limiting support
|
|
||||||
|
|
||||||
### Encryption Ready ✅
|
|
||||||
```typescript
|
|
||||||
import { crypto } from 'brainy/utils'
|
|
||||||
// AES-256-GCM encryption utilities
|
|
||||||
// Key derivation functions
|
|
||||||
// Secure random generation
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🎯 Key Design Principles
|
|
||||||
|
|
||||||
### 1. Zero Configuration
|
|
||||||
```typescript
|
|
||||||
const brain = new Brainy()
|
|
||||||
await brain.init()
|
|
||||||
// Everything else is automatic!
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Fixed Dimensions (384)
|
|
||||||
- **ALWAYS** uses all-MiniLM-L6-v2 model
|
|
||||||
- **ALWAYS** 384 dimensions
|
|
||||||
- **NOT** configurable (by design)
|
|
||||||
- Ensures everything works together
|
|
||||||
|
|
||||||
### 3. Progressive Enhancement
|
|
||||||
- Starts simple, scales automatically
|
|
||||||
- Adapts to workload patterns
|
|
||||||
- Optimizes based on usage
|
|
||||||
|
|
||||||
### 4. Universal Compatibility
|
|
||||||
- Works in Node.js 18+
|
|
||||||
- Works in modern browsers
|
|
||||||
- Works in Web Workers
|
|
||||||
- Works in Edge environments
|
|
||||||
|
|
||||||
## 📦 What Ships in Core (MIT Licensed)
|
|
||||||
|
|
||||||
**EVERYTHING** is included in the core package:
|
|
||||||
- ✅ All engines (vector, graph, field, neural)
|
|
||||||
- ✅ All augmentations (12+)
|
|
||||||
- ✅ All storage adapters
|
|
||||||
- ✅ All distributed modes
|
|
||||||
- ✅ Complete statistics
|
|
||||||
- ✅ GPU support
|
|
||||||
- ✅ No feature limitations
|
|
||||||
- ✅ No premium tiers
|
|
||||||
- ✅ 100% MIT licensed
|
|
||||||
|
|
||||||
## 🚀 Quick Start
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { Brainy } from 'brainy'
|
|
||||||
|
|
||||||
// Zero config required!
|
|
||||||
const brain = new Brainy()
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Add data (auto-detects type)
|
|
||||||
await brain.add('Content here')
|
|
||||||
|
|
||||||
// Search with natural language
|
|
||||||
const results = await brain.find('related content from last week')
|
|
||||||
|
|
||||||
// Everything else is automatic!
|
|
||||||
```
|
|
||||||
|
|
||||||
## 📈 Performance Characteristics
|
|
||||||
|
|
||||||
- **Vector Search**: O(log n) with HNSW indexing
|
|
||||||
- **Graph Traversal**: O(k) for k-hop queries
|
|
||||||
- **Field Filtering**: O(1) with metadata index
|
|
||||||
- **Memory Usage**: ~100MB base + data
|
|
||||||
- **Embedding Speed**: ~100ms for batch of 10
|
|
||||||
- **Query Speed**: <10ms for most queries
|
|
||||||
|
|
||||||
## 🎉 Summary
|
|
||||||
|
|
||||||
Brainy 2.0 is a **complete**, **production-ready** AI database that requires **ZERO configuration**. Every feature listed here is **implemented and working** today. No configuration, no setup, no complexity - just powerful AI capabilities that work out of the box!
|
|
||||||
|
|
@ -1,693 +0,0 @@
|
||||||
# Instant Fork™
|
|
||||||
|
|
||||||
**Clone your entire Brainy database in 1-2 seconds. Test anything without fear.**
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## The Problem
|
|
||||||
|
|
||||||
You need to test a risky migration. Or run an A/B experiment. Or let your team fork production data for development.
|
|
||||||
|
|
||||||
**Traditional approach:**
|
|
||||||
```javascript
|
|
||||||
// Export entire database (20 minutes)
|
|
||||||
await database.export('backup.json')
|
|
||||||
|
|
||||||
// Modify data (cross your fingers)
|
|
||||||
await database.updateAll(riskyTransformation)
|
|
||||||
|
|
||||||
// If it fails... restore from backup (another 20 minutes)
|
|
||||||
// Total downtime: 40+ minutes
|
|
||||||
```
|
|
||||||
|
|
||||||
**The pain:**
|
|
||||||
- ❌ Slow (hours for large datasets)
|
|
||||||
- ❌ Risky (one mistake = data loss)
|
|
||||||
- ❌ Expensive (full copy = 2x storage)
|
|
||||||
- ❌ Complex (manual backup/restore workflows)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## The Solution
|
|
||||||
|
|
||||||
**Brainy's Instant Fork**:
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Clone entire database in 1-2 seconds
|
|
||||||
const experiment = await brain.fork('test-migration')
|
|
||||||
|
|
||||||
// Test your changes safely
|
|
||||||
await experiment.updateAll(riskyTransformation)
|
|
||||||
|
|
||||||
// Works? Great! Use the experimental branch.
|
|
||||||
// Failed? Just discard.
|
|
||||||
if (success) {
|
|
||||||
// Make experiment the new main branch
|
|
||||||
await brain.checkout('test-migration')
|
|
||||||
} else {
|
|
||||||
await experiment.destroy() // No harm done
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
**Benefits:**
|
|
||||||
- ✅ **Fast**: 1-2 seconds even with millions of entities
|
|
||||||
- ✅ **Safe**: Original data untouched
|
|
||||||
- ✅ **Cheap**: 70-90% storage savings (content-addressable blobs)
|
|
||||||
- ✅ **Simple**: One line of code
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## How It Works
|
|
||||||
|
|
||||||
Brainy uses **Snowflake-style Copy-on-Write (COW)** for instant forking:
|
|
||||||
|
|
||||||
### Architecture
|
|
||||||
|
|
||||||
1. **HNSW Index COW** (The Performance Bottleneck):
|
|
||||||
- **Shallow Copy**: O(1) Map reference copying (~10ms for 1M+ nodes)
|
|
||||||
- **Lazy Deep Copy**: Nodes copied only when modified (`ensureCOW()`)
|
|
||||||
- **Write Isolation**: Fork modifications don't affect parent
|
|
||||||
- **Implementation**: `src/hnsw/hnswIndex.ts` lines 2088-2150
|
|
||||||
|
|
||||||
2. **Metadata & Graph Indexes** (Fast Rebuild):
|
|
||||||
- **Rebuild from Storage**: < 500ms total for both indexes
|
|
||||||
- **Shared Storage**: Both indexes read from COW-enabled storage layer
|
|
||||||
- **Acceptable Overhead**: Fast enough not to need in-memory COW
|
|
||||||
|
|
||||||
3. **Storage Layer** (Shared):
|
|
||||||
- **RefManager**: Manages branch references
|
|
||||||
- **BlobStorage**: Content-addressable with deduplication
|
|
||||||
- **All Adapters**: Memory, FileSystem, S3, R2, GCS, Azure, OPFS
|
|
||||||
|
|
||||||
**Performance**:
|
|
||||||
- **Fork time**: < 100ms @ 10K entities (MEASURED in tests)
|
|
||||||
- **Storage overhead**: 10-20% (shared blobs, only changed data duplicated)
|
|
||||||
- **Memory overhead**: 10-20% (shared HNSW nodes, only modified nodes duplicated)
|
|
||||||
|
|
||||||
**Technical Details**:
|
|
||||||
```typescript
|
|
||||||
// Shallow copy HNSW (instant)
|
|
||||||
clone.index.enableCOW(this.index) // O(1) Map reference copy
|
|
||||||
|
|
||||||
// Fast rebuild small indexes from shared storage
|
|
||||||
clone.metadataIndex = new MetadataIndexManager(clone.storage) // <100ms
|
|
||||||
clone.graphIndex = new GraphAdjacencyIndex(clone.storage) // <500ms
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Basic Usage
|
|
||||||
|
|
||||||
### 1. Create a Fork
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
const brain = new Brainy({ storage: { adapter: 'memory' } })
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Add some data
|
|
||||||
await brain.add({ noun: 'user', data: { name: 'Alice' } })
|
|
||||||
await brain.add({ noun: 'user', data: { name: 'Bob' } })
|
|
||||||
|
|
||||||
// Fork instantly
|
|
||||||
const fork = await brain.fork('experiment')
|
|
||||||
|
|
||||||
console.log('Fork created!', fork)
|
|
||||||
```
|
|
||||||
|
|
||||||
**What happens:**
|
|
||||||
- Original brain: unchanged
|
|
||||||
- Fork: exact copy at this moment
|
|
||||||
- Changes in fork: don't affect original
|
|
||||||
- Changes in original: don't affect fork
|
|
||||||
|
|
||||||
### 2. Work with the Fork
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Fork is a full Brainy instance
|
|
||||||
await fork.add({ noun: 'user', data: { name: 'Charlie' } })
|
|
||||||
|
|
||||||
// All APIs work
|
|
||||||
const users = await fork.find({ noun: 'user' })
|
|
||||||
console.log(users.length) // 3 (Alice, Bob, Charlie)
|
|
||||||
|
|
||||||
// Original brain unchanged
|
|
||||||
const originalUsers = await brain.find({ noun: 'user' })
|
|
||||||
console.log(originalUsers.length) // 2 (Alice, Bob)
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Discard When Done
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Clean up fork when finished
|
|
||||||
await fork.destroy()
|
|
||||||
|
|
||||||
// Note: fork() creates independent branches for experimentation
|
|
||||||
// Use checkout() to switch between branches or keep them separate forever
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Use Cases
|
|
||||||
|
|
||||||
### 1. Safe Migrations
|
|
||||||
|
|
||||||
**Problem**: Migrating data is risky. One mistake = data corruption.
|
|
||||||
|
|
||||||
**Solution**: Test migration in fork first.
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
const brain = new Brainy({ storage: { adapter: 'filesystem', path: './data' } })
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Fork production data
|
|
||||||
const migration = await brain.fork('migration-test')
|
|
||||||
|
|
||||||
// Run migration on fork
|
|
||||||
const users = await migration.find({ noun: 'user' })
|
|
||||||
for (const user of users) {
|
|
||||||
await migration.update(user.id, {
|
|
||||||
email: user.data.email.toLowerCase(), // Normalize emails
|
|
||||||
verified: user.data.verified ?? false // Add missing field
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate migration
|
|
||||||
const allValid = (await migration.find({ noun: 'user' }))
|
|
||||||
.every(u => u.data.email === u.data.email.toLowerCase())
|
|
||||||
|
|
||||||
if (allValid) {
|
|
||||||
console.log('✅ Migration safe! Apply changes to production brain')
|
|
||||||
// Apply validated migration to main brain
|
|
||||||
const users = await brain.find({ noun: 'user' })
|
|
||||||
for (const user of users) {
|
|
||||||
await brain.update(user.id, {
|
|
||||||
email: user.data.email.toLowerCase(),
|
|
||||||
verified: user.data.verified ?? false
|
|
||||||
})
|
|
||||||
}
|
|
||||||
await migration.destroy()
|
|
||||||
} else {
|
|
||||||
console.log('❌ Migration failed! Discarding fork.')
|
|
||||||
await migration.destroy()
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. A/B Testing
|
|
||||||
|
|
||||||
**Problem**: Need to test two different algorithms on the same data.
|
|
||||||
|
|
||||||
**Solution**: Create two forks, run experiments in parallel.
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
const brain = new Brainy({ storage: { adapter: 's3', bucket: 'production' } })
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Create two variants
|
|
||||||
const variantA = await brain.fork('variant-a') // Control
|
|
||||||
const variantB = await brain.fork('variant-b') // Test
|
|
||||||
|
|
||||||
// Run different algorithms
|
|
||||||
await variantA.processWithAlgorithm('current')
|
|
||||||
await variantB.processWithAlgorithm('improved')
|
|
||||||
|
|
||||||
// Compare results
|
|
||||||
const metricsA = await variantA.getMetrics()
|
|
||||||
const metricsB = await variantB.getMetrics()
|
|
||||||
|
|
||||||
console.log('Variant A accuracy:', metricsA.accuracy)
|
|
||||||
console.log('Variant B accuracy:', metricsB.accuracy)
|
|
||||||
|
|
||||||
// Choose winner and update main brain
|
|
||||||
if (metricsB.accuracy > metricsA.accuracy) {
|
|
||||||
console.log('B wins! Apply algorithm B to production')
|
|
||||||
await brain.processWithAlgorithm('improved')
|
|
||||||
await variantA.destroy()
|
|
||||||
await variantB.destroy()
|
|
||||||
} else {
|
|
||||||
console.log('A wins! Keeping current algorithm.')
|
|
||||||
await variantA.destroy()
|
|
||||||
await variantB.destroy()
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Distributed Development
|
|
||||||
|
|
||||||
**Problem**: Multiple developers need to work with production data.
|
|
||||||
|
|
||||||
**Solution**: Each developer gets their own fork.
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Main production brain
|
|
||||||
const production = new Brainy({ storage: { adapter: 's3', bucket: 'prod' } })
|
|
||||||
await production.init()
|
|
||||||
|
|
||||||
// Alice's feature branch
|
|
||||||
const aliceBranch = await production.fork('alice-feature-x')
|
|
||||||
|
|
||||||
// Bob's feature branch
|
|
||||||
const bobBranch = await production.fork('bob-feature-y')
|
|
||||||
|
|
||||||
// Both work independently (zero conflicts!)
|
|
||||||
await aliceBranch.add({ noun: 'feature', data: { name: 'X' } })
|
|
||||||
await bobBranch.add({ noun: 'feature', data: { name: 'Y' } })
|
|
||||||
|
|
||||||
// When ready, manually copy validated changes to production
|
|
||||||
await production.add({ noun: 'feature', data: { name: 'X' } })
|
|
||||||
await production.add({ noun: 'feature', data: { name: 'Y' } })
|
|
||||||
|
|
||||||
await aliceBranch.destroy()
|
|
||||||
await bobBranch.destroy()
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4. Instant Backup/Restore
|
|
||||||
|
|
||||||
**Problem**: Need fast backups before risky operations.
|
|
||||||
|
|
||||||
**Solution**: Fork as backup.
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
const brain = new Brainy({ storage: { adapter: 'memory' } })
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Work with production data
|
|
||||||
await brain.add({ noun: 'important', data: { value: 'critical' } })
|
|
||||||
|
|
||||||
// Instant backup (1-2 seconds)
|
|
||||||
const backup = await brain.fork('backup-before-delete')
|
|
||||||
|
|
||||||
// Do risky operation
|
|
||||||
const entities = await brain.find({ noun: 'important' })
|
|
||||||
await brain.delete(entities[0].id)
|
|
||||||
|
|
||||||
// Oops! Need to restore
|
|
||||||
// Just discard current, use backup
|
|
||||||
await brain.destroy()
|
|
||||||
|
|
||||||
// Restore from backup (or switch to backup branch)
|
|
||||||
const restored = await backup.fork('main')
|
|
||||||
|
|
||||||
console.log('✅ Data restored!')
|
|
||||||
```
|
|
||||||
|
|
||||||
### 5. Snapshot Testing
|
|
||||||
|
|
||||||
**Problem**: Need to test code against specific data states.
|
|
||||||
|
|
||||||
**Solution**: Create fork snapshots before making changes.
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
const brain = new Brainy({ storage: { adapter: 'memory' } })
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Create initial state
|
|
||||||
await brain.add({ noun: 'doc', data: { version: 1 } })
|
|
||||||
|
|
||||||
// Take snapshot before changes
|
|
||||||
const snapshot = await brain.fork('before-update')
|
|
||||||
|
|
||||||
// Make changes to main brain
|
|
||||||
const docs = await brain.find({ noun: 'doc' })
|
|
||||||
await brain.update(docs[0].id, { version: 2 })
|
|
||||||
|
|
||||||
// Test against original state
|
|
||||||
const originalData = await snapshot.find({ noun: 'doc' })
|
|
||||||
console.log(originalData[0].data.version) // 1 (original state!)
|
|
||||||
|
|
||||||
// Clean up
|
|
||||||
await snapshot.destroy()
|
|
||||||
|
|
||||||
// Note: Time-travel queries (asOf) Planned
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Advanced Features
|
|
||||||
|
|
||||||
### Fork Options
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Custom branch name
|
|
||||||
const fork1 = await brain.fork('my-experiment')
|
|
||||||
|
|
||||||
// Auto-generated name (uses timestamp)
|
|
||||||
const fork2 = await brain.fork() // 'fork-1635789012345'
|
|
||||||
|
|
||||||
// Fork with metadata (for tracking)
|
|
||||||
const fork3 = await brain.fork('test', {
|
|
||||||
author: 'Alice',
|
|
||||||
message: 'Testing new feature'
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Branch Management
|
|
||||||
|
|
||||||
Full branch management now available!
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// List all branches
|
|
||||||
const branches = await brain.listBranches()
|
|
||||||
console.log(branches) // ['main', 'experiment', 'test']
|
|
||||||
|
|
||||||
// Get current branch
|
|
||||||
const current = await brain.getCurrentBranch()
|
|
||||||
console.log(current) // 'main'
|
|
||||||
|
|
||||||
// Switch between branches
|
|
||||||
await brain.checkout('experiment')
|
|
||||||
|
|
||||||
// Delete a branch
|
|
||||||
await brain.deleteBranch('old-experiment')
|
|
||||||
```
|
|
||||||
|
|
||||||
### Commit Tracking
|
|
||||||
|
|
||||||
Git-style commit tracking!
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Create a commit (snapshot of current state)
|
|
||||||
await brain.add({ type: 'user', data: { name: 'Alice' } })
|
|
||||||
|
|
||||||
const commitHash = await brain.commit({
|
|
||||||
message: 'Add Alice user',
|
|
||||||
author: 'dev@example.com'
|
|
||||||
})
|
|
||||||
|
|
||||||
console.log(commitHash) // 'a3f2c1b9...'
|
|
||||||
```
|
|
||||||
|
|
||||||
### Commit History
|
|
||||||
|
|
||||||
View commit history!
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Get commit history for current branch
|
|
||||||
const history = await brain.getHistory({ limit: 10 })
|
|
||||||
|
|
||||||
history.forEach(commit => {
|
|
||||||
console.log(`${commit.hash}: ${commit.message}`)
|
|
||||||
console.log(` By: ${commit.author} at ${new Date(commit.timestamp)}`)
|
|
||||||
})
|
|
||||||
|
|
||||||
## Performance Characteristics
|
|
||||||
|
|
||||||
### Fork Speed (Measured)
|
|
||||||
|
|
||||||
| Entities | Traditional Copy | Brainy Fork | Speedup |
|
|
||||||
|----------|------------------|-------------|---------|
|
|
||||||
| 1,000 | 2-5 seconds | 0.5 seconds | 4-10x |
|
|
||||||
| 10,000 | 20-40 seconds | 0.8 seconds | 25-50x |
|
|
||||||
| 100,000 | 3-5 minutes | 1.2 seconds | 150-250x |
|
|
||||||
| 1,000,000 | 30-60 minutes | 1.8 seconds | 1000-2000x |
|
|
||||||
|
|
||||||
### Storage Overhead
|
|
||||||
|
|
||||||
```
|
|
||||||
Scenario: 1M entities, 10 forks
|
|
||||||
|
|
||||||
Traditional: 10 full copies = 80GB × 10 = 800GB
|
|
||||||
Brainy: 1 base + 10% changes = 80GB + 8GB = 88GB
|
|
||||||
|
|
||||||
Savings: 89% less storage
|
|
||||||
```
|
|
||||||
|
|
||||||
### Memory Overhead
|
|
||||||
|
|
||||||
```
|
|
||||||
Scenario: 1M entities in memory
|
|
||||||
|
|
||||||
Traditional fork: 2x memory (10GB → 20GB)
|
|
||||||
Brainy fork: 1.2x memory (10GB → 12GB)
|
|
||||||
|
|
||||||
Savings: 40% less memory
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Zero Configuration
|
|
||||||
|
|
||||||
**Fork is enabled by default. No setup required.**
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// This is all you need:
|
|
||||||
const brain = new Brainy({ storage: { adapter: 'memory' } })
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Fork is ready to use:
|
|
||||||
const fork = await brain.fork()
|
|
||||||
|
|
||||||
// That's it!
|
|
||||||
```
|
|
||||||
|
|
||||||
**Automatic optimizations:**
|
|
||||||
- ✅ Compression: zstd for metadata, none for vectors (automatic)
|
|
||||||
- ✅ Deduplication: content-addressable (automatic)
|
|
||||||
- ✅ Caching: LRU with memory limits (automatic)
|
|
||||||
- ✅ Garbage collection: cleanup unused blobs (automatic)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Integration with Brainy Features
|
|
||||||
|
|
||||||
### Works with All Storage Adapters
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Memory
|
|
||||||
await new Brainy({ storage: { adapter: 'memory' } }).fork()
|
|
||||||
|
|
||||||
// FileSystem
|
|
||||||
await new Brainy({ storage: { adapter: 'filesystem', path: './data' } }).fork()
|
|
||||||
|
|
||||||
// S3
|
|
||||||
await new Brainy({ storage: { adapter: 's3', bucket: 'my-data' } }).fork()
|
|
||||||
|
|
||||||
// All adapters supported: Memory, OPFS, FileSystem, S3, R2, GCS, Azure, TypeAware
|
|
||||||
```
|
|
||||||
|
|
||||||
### Works with find(), VFS, Triple Intelligence
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: { adapter: 'memory' },
|
|
||||||
vfs: { enabled: true },
|
|
||||||
intelligence: { enabled: true }
|
|
||||||
})
|
|
||||||
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Create VFS files
|
|
||||||
await brain.vfs.writeFile('/project/README.md', '# My Project')
|
|
||||||
|
|
||||||
// Add entities
|
|
||||||
await brain.add({ noun: 'user', data: { name: 'Alice' } })
|
|
||||||
|
|
||||||
// Fork everything
|
|
||||||
const fork = await brain.fork('test')
|
|
||||||
|
|
||||||
// All features work on fork:
|
|
||||||
await fork.vfs.readFile('/project/README.md') // ✅ VFS
|
|
||||||
await fork.find({ noun: 'user' }) // ✅ find()
|
|
||||||
await fork.query('users named Alice') // ✅ Triple Intelligence
|
|
||||||
```
|
|
||||||
|
|
||||||
### Works at Billion Scale
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Tested at 1M entities, extrapolates to 1B
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: { adapter: 'gcs', bucket: 'billion-scale' },
|
|
||||||
hnsw: { typeAware: true } // 87% memory reduction
|
|
||||||
})
|
|
||||||
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Fork 1B entities: still < 2 seconds
|
|
||||||
const fork = await brain.fork()
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## FAQ
|
|
||||||
|
|
||||||
### Q: Does fork() copy all data?
|
|
||||||
|
|
||||||
**A: No.** Fork uses copy-on-write (COW). Unchanged data is shared between parent and fork via content-addressable blobs. Only modified data creates new blobs.
|
|
||||||
|
|
||||||
### Q: Is fork() safe for production?
|
|
||||||
|
|
||||||
**A: Yes.** Fork is battle-tested at scale. Uses proven Git-like COW technology. Zero risk to original data.
|
|
||||||
|
|
||||||
### Q: Does it work with all storage adapters?
|
|
||||||
|
|
||||||
**A: Yes.** Fork works with Memory, OPFS, FileSystem, S3, R2, GCS, Azure, and TypeAware adapters.
|
|
||||||
|
|
||||||
### Q: What happens to the fork if I modify the original?
|
|
||||||
|
|
||||||
**A: Nothing.** Fork is isolated. Changes in parent don't affect fork. Changes in fork don't affect parent.
|
|
||||||
|
|
||||||
### Q: Can I merge forks back to main?
|
|
||||||
|
|
||||||
**A: Use the "experimental branching" paradigm.** Instead of merging, either (1) make your experimental branch the new main with `checkout()`, or (2) manually copy specific entities you want. See CHANGELOG v6.0.0 for migration patterns.
|
|
||||||
|
|
||||||
### Q: How long are forks kept?
|
|
||||||
|
|
||||||
**A: Forever (or until you delete them).** Forks persist like branches. Delete with `fork.destroy()` or set retention policy (Enterprise).
|
|
||||||
|
|
||||||
### Q: What's the performance impact?
|
|
||||||
|
|
||||||
**A: Minimal.** Fork time: 1-2 seconds @ 1M entities. Storage: 10-20% overhead. Memory: 20-40% overhead.
|
|
||||||
|
|
||||||
### Q: Can I fork a fork?
|
|
||||||
|
|
||||||
**A: Yes.** Fork anything, anytime. Create branch trees as deep as needed.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Comparison to Other Databases
|
|
||||||
|
|
||||||
### vs PostgreSQL
|
|
||||||
|
|
||||||
**PostgreSQL:**
|
|
||||||
```sql
|
|
||||||
-- Create copy (full table scan, minutes)
|
|
||||||
CREATE TABLE users_backup AS SELECT * FROM users;
|
|
||||||
|
|
||||||
-- Modify (risky!)
|
|
||||||
UPDATE users SET email = LOWER(email);
|
|
||||||
|
|
||||||
-- Restore (if failed)
|
|
||||||
DROP TABLE users;
|
|
||||||
ALTER TABLE users_backup RENAME TO users;
|
|
||||||
```
|
|
||||||
|
|
||||||
**Brainy:**
|
|
||||||
```javascript
|
|
||||||
const fork = await brain.fork('test')
|
|
||||||
await fork.updateAll({ email: (u) => u.email.toLowerCase() })
|
|
||||||
if (success) await brain.checkout('test') // Make test branch active
|
|
||||||
else await fork.destroy()
|
|
||||||
```
|
|
||||||
|
|
||||||
**Winner: Brainy** (1000x faster, safer)
|
|
||||||
|
|
||||||
### vs MongoDB
|
|
||||||
|
|
||||||
**MongoDB:**
|
|
||||||
```javascript
|
|
||||||
// No native fork/clone
|
|
||||||
// Must manually export/import
|
|
||||||
|
|
||||||
// Export (slow)
|
|
||||||
mongoexport --db mydb --collection users --out users.json
|
|
||||||
|
|
||||||
// Import to new collection (slow)
|
|
||||||
mongoimport --db mydb --collection users_backup --file users.json
|
|
||||||
```
|
|
||||||
|
|
||||||
**Brainy:**
|
|
||||||
```javascript
|
|
||||||
const fork = await brain.fork() // Done!
|
|
||||||
```
|
|
||||||
|
|
||||||
**Winner: Brainy** (100x faster, built-in)
|
|
||||||
|
|
||||||
### vs Pinecone/Weaviate
|
|
||||||
|
|
||||||
**Pinecone/Weaviate:**
|
|
||||||
```
|
|
||||||
❌ No fork/clone feature at all
|
|
||||||
❌ Manual backup/restore only
|
|
||||||
❌ Downtime required for testing
|
|
||||||
```
|
|
||||||
|
|
||||||
**Brainy:**
|
|
||||||
```javascript
|
|
||||||
✅ Fork in 1-2 seconds
|
|
||||||
✅ Zero downtime
|
|
||||||
✅ Zero risk
|
|
||||||
```
|
|
||||||
|
|
||||||
**Winner: Brainy** (only vector DB with instant fork)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## What's Implemented vs. What's Next
|
|
||||||
|
|
||||||
### ✅ Available in - ✅ `fork()` - Instant clone in <100ms
|
|
||||||
- ✅ `listBranches()` - List all forks
|
|
||||||
- ✅ `getCurrentBranch()` - Get active branch
|
|
||||||
- ✅ `checkout()` - Switch between branches
|
|
||||||
- ✅ `deleteBranch()` - Delete branches
|
|
||||||
- ✅ `commit()` - Create state snapshots
|
|
||||||
- ✅ `getHistory()` - View commit history
|
|
||||||
|
|
||||||
### 🔮 Planned for:
|
|
||||||
|
|
||||||
**Temporal Features:**
|
|
||||||
- `asOf(timestamp)` - Query data at specific time (✅ available)
|
|
||||||
- `rollback(commitHash)` - Restore to previous state
|
|
||||||
- Full audit trail for all changes
|
|
||||||
|
|
||||||
These features require additional temporal infrastructure and are being carefully designed
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## CLI Support
|
|
||||||
|
|
||||||
All fork/merge/commit features are available via CLI:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Fork (instant clone)
|
|
||||||
brainy fork feature-x --message "Testing new feature" --author "dev@example.com"
|
|
||||||
|
|
||||||
# List branches
|
|
||||||
brainy branch list
|
|
||||||
|
|
||||||
# Switch branches
|
|
||||||
brainy checkout feature-x
|
|
||||||
|
|
||||||
# Create commit
|
|
||||||
brainy commit --message "Add new feature" --author "dev@example.com"
|
|
||||||
|
|
||||||
# View history
|
|
||||||
brainy history --limit 10
|
|
||||||
|
|
||||||
# Merge branches
|
|
||||||
brainy merge feature-x main --strategy last-write-wins
|
|
||||||
|
|
||||||
# Delete branch
|
|
||||||
brainy branch delete old-feature --force
|
|
||||||
```
|
|
||||||
|
|
||||||
## Try It Now
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npm install @soulcraft/brainy
|
|
||||||
```
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
const brain = new Brainy()
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
await brain.add({ noun: 'test', data: { value: 1 } })
|
|
||||||
|
|
||||||
const fork = await brain.fork('experiment')
|
|
||||||
|
|
||||||
console.log('Fork created in < 2 seconds! 🚀')
|
|
||||||
```
|
|
||||||
|
|
||||||
**Zero config. Zero complexity. Pure power.**
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Learn More
|
|
||||||
|
|
||||||
- [COW Architecture](../architecture/copy-on-write.md)
|
|
||||||
- [Performance Benchmarks](../benchmarks/fork-performance.md)
|
|
||||||
- [Enterprise Features](../enterprise/temporal-cloning.md)
|
|
||||||
- [API Reference](../api/fork.md)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Brainy** | [GitHub](https://github.com/soulcraftlabs/brainy) | [npm](https://npmjs.com/package/@soulcraft/brainy)
|
|
||||||
|
|
@ -1,296 +0,0 @@
|
||||||
# 🚀 Brainy - Production-Ready Features
|
|
||||||
|
|
||||||
> **Status**: All features listed here are IMPLEMENTED and TESTED
|
|
||||||
|
|
||||||
## 📊 Performance Metrics
|
|
||||||
- **Search Latency**: <10ms for 10,000+ items
|
|
||||||
- **Write Throughput**: 10,000+ ops/sec
|
|
||||||
- **Memory Efficiency**: <500MB for 10K items
|
|
||||||
- **Concurrent Operations**: 100+ simultaneous operations
|
|
||||||
|
|
||||||
## 🧠 Core Intelligence Features
|
|
||||||
|
|
||||||
### Triple Intelligence System ✅
|
|
||||||
Unified query system combining three types of intelligence:
|
|
||||||
```typescript
|
|
||||||
const results = await brain.find({
|
|
||||||
like: 'AI research', // Vector similarity search
|
|
||||||
where: { year: 2024 }, // Metadata filtering
|
|
||||||
connected: { to: authorId } // Graph relationships
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Intelligent Type Mapping ✅
|
|
||||||
Prevents semantic degradation by intelligently inferring types:
|
|
||||||
```typescript
|
|
||||||
// Automatically infers 'person' from email field
|
|
||||||
brain.add({ name: "John", email: "john@example.com" }, 'entity')
|
|
||||||
// → Stored as type: 'person', not generic 'entity'
|
|
||||||
```
|
|
||||||
|
|
||||||
### Neural Query Understanding ✅
|
|
||||||
- 220+ embedded patterns for intent detection
|
|
||||||
- Natural language query processing
|
|
||||||
- Automatic query optimization
|
|
||||||
- Pattern-based query rewriting
|
|
||||||
|
|
||||||
## 🏢 Enterprise Features
|
|
||||||
|
|
||||||
### Distributed Coordination ✅
|
|
||||||
Raft consensus for multi-node deployments:
|
|
||||||
```typescript
|
|
||||||
import { DistributedCoordinator } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
const coordinator = createCoordinator({
|
|
||||||
nodeId: 'node-1',
|
|
||||||
peers: ['node-2', 'node-3'],
|
|
||||||
electionTimeout: 500
|
|
||||||
})
|
|
||||||
// Automatic leader election and failover
|
|
||||||
```
|
|
||||||
|
|
||||||
### Horizontal Sharding ✅
|
|
||||||
Consistent hashing for data distribution:
|
|
||||||
```typescript
|
|
||||||
import { ShardManager } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
const shards = createShardManager({
|
|
||||||
nodes: ['node-1', 'node-2', 'node-3'],
|
|
||||||
replicationFactor: 2,
|
|
||||||
virtualNodes: 150
|
|
||||||
})
|
|
||||||
// Automatic shard rebalancing on node changes
|
|
||||||
```
|
|
||||||
|
|
||||||
### Read/Write Separation ✅
|
|
||||||
Primary-replica architecture for scale:
|
|
||||||
```typescript
|
|
||||||
import { ReadWriteSeparation } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
const replication = createReadWriteSeparation({
|
|
||||||
role: 'auto', // Automatic primary/replica detection
|
|
||||||
consistencyLevel: 'strong', // or 'eventual'
|
|
||||||
readPreference: 'nearest'
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cross-Instance Cache Sync ✅
|
|
||||||
Version vector-based cache synchronization:
|
|
||||||
```typescript
|
|
||||||
import { CacheSync } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
const cache = createCacheSync({
|
|
||||||
nodeId: 'node-1',
|
|
||||||
syncInterval: 100,
|
|
||||||
conflictResolution: 'version-vector'
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🔐 Security & Compliance
|
|
||||||
|
|
||||||
### Rate Limiting ✅
|
|
||||||
Per-operation configurable limits:
|
|
||||||
```typescript
|
|
||||||
const rateLimiter = createRateLimitAugmentation({
|
|
||||||
limits: {
|
|
||||||
searches: 1000, // per minute
|
|
||||||
writes: 100,
|
|
||||||
reads: 5000,
|
|
||||||
deletes: 50
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Audit Logging ✅
|
|
||||||
Comprehensive operation tracking:
|
|
||||||
```typescript
|
|
||||||
const auditLogger = createAuditLogAugmentation({
|
|
||||||
logLevel: 'detailed',
|
|
||||||
retention: 90, // days
|
|
||||||
includeMetadata: true
|
|
||||||
})
|
|
||||||
|
|
||||||
// Query audit logs
|
|
||||||
const logs = auditLogger.queryLogs({
|
|
||||||
operation: 'add',
|
|
||||||
startTime: Date.now() - 3600000
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## 📦 Storage & Persistence
|
|
||||||
|
|
||||||
Full crash recovery and replay:
|
|
||||||
```typescript
|
|
||||||
enabled: true,
|
|
||||||
checkpointInterval: 1000,
|
|
||||||
maxLogSize: 100 * 1024 * 1024 // 100MB
|
|
||||||
}))
|
|
||||||
```
|
|
||||||
|
|
||||||
### Multi-Tenancy ✅
|
|
||||||
Service-based data isolation:
|
|
||||||
```typescript
|
|
||||||
// Isolated data per service
|
|
||||||
await brain.add(data, 'document', { service: 'tenant-1' })
|
|
||||||
await brain.find('query', { service: 'tenant-1' })
|
|
||||||
```
|
|
||||||
|
|
||||||
### Write-Only Mode ✅
|
|
||||||
For dedicated write nodes:
|
|
||||||
```typescript
|
|
||||||
const brain = new Brainy({
|
|
||||||
mode: 'write-only',
|
|
||||||
storage: 's3://bucket/path'
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🚀 Performance Features
|
|
||||||
|
|
||||||
### Batch Operations ✅
|
|
||||||
Optimized bulk processing:
|
|
||||||
```typescript
|
|
||||||
// Parallel processing with automatic batching
|
|
||||||
await brain.addMany(items) // <10ms per item
|
|
||||||
await brain.updateMany(updates)
|
|
||||||
await brain.deleteMany(filters)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Request Deduplication ✅
|
|
||||||
Automatic duplicate request handling:
|
|
||||||
```typescript
|
|
||||||
brain.use(new RequestDeduplicatorAugmentation())
|
|
||||||
// Identical concurrent requests return same result
|
|
||||||
```
|
|
||||||
|
|
||||||
### Smart Caching ✅
|
|
||||||
Intelligent search result caching:
|
|
||||||
```typescript
|
|
||||||
brain.use(new CacheAugmentation({
|
|
||||||
maxSize: 10000,
|
|
||||||
ttl: 300000, // 5 minutes
|
|
||||||
invalidateOnWrite: true
|
|
||||||
}))
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🔄 Data Processing
|
|
||||||
|
|
||||||
### Entity Registry ✅
|
|
||||||
Bloom filter-based deduplication:
|
|
||||||
```typescript
|
|
||||||
brain.use(new EntityRegistryAugmentation())
|
|
||||||
// Handles millions of entities with minimal memory
|
|
||||||
```
|
|
||||||
|
|
||||||
### Neural Import ✅
|
|
||||||
Intelligent data import with type inference:
|
|
||||||
```typescript
|
|
||||||
await brain.import({
|
|
||||||
source: 'data.json',
|
|
||||||
autoDetectTypes: true,
|
|
||||||
batchSize: 1000
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Streaming Pipeline ✅
|
|
||||||
Real-time data processing:
|
|
||||||
```typescript
|
|
||||||
brain.stream()
|
|
||||||
.pipe(transform)
|
|
||||||
.pipe(enrich)
|
|
||||||
.pipe(brain.writer())
|
|
||||||
```
|
|
||||||
|
|
||||||
## 📊 Analytics & Monitoring
|
|
||||||
|
|
||||||
### Metrics Collection ✅
|
|
||||||
Built-in performance metrics:
|
|
||||||
```typescript
|
|
||||||
const metrics = brain.getMetrics()
|
|
||||||
// {
|
|
||||||
// operations: { add: 1000, find: 5000 },
|
|
||||||
// performance: { p95: 8, p99: 12 },
|
|
||||||
// cache: { hits: 4500, misses: 500 }
|
|
||||||
// }
|
|
||||||
```
|
|
||||||
|
|
||||||
### Health Monitoring ✅
|
|
||||||
Automatic health checks:
|
|
||||||
```typescript
|
|
||||||
const health = brain.getHealth()
|
|
||||||
// {
|
|
||||||
// status: 'healthy',
|
|
||||||
// storage: 'connected',
|
|
||||||
// memory: { used: 245, limit: 512 }
|
|
||||||
// }
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🛠️ Developer Experience
|
|
||||||
|
|
||||||
### Zero Configuration ✅
|
|
||||||
Works out of the box:
|
|
||||||
```typescript
|
|
||||||
import Brainy from '@soulcraft/brainy'
|
|
||||||
const brain = new Brainy() // Auto-configures everything
|
|
||||||
```
|
|
||||||
|
|
||||||
### TypeScript First ✅
|
|
||||||
Full type safety and inference:
|
|
||||||
```typescript
|
|
||||||
// Types are automatically inferred
|
|
||||||
const results = await brain.find<MyType>('query')
|
|
||||||
```
|
|
||||||
|
|
||||||
### Augmentation System ✅
|
|
||||||
Extensible plugin architecture:
|
|
||||||
```typescript
|
|
||||||
class CustomAugmentation extends BaseAugmentation {
|
|
||||||
execute(operation, params, next) {
|
|
||||||
// Your custom logic
|
|
||||||
return next()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🔧 Operational Features
|
|
||||||
|
|
||||||
### Graceful Shutdown ✅
|
|
||||||
Clean shutdown with data persistence:
|
|
||||||
```typescript
|
|
||||||
process.on('SIGTERM', async () => {
|
|
||||||
await brain.shutdown() // Saves all pending data
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Hot Reload ✅
|
|
||||||
Configuration updates without restart:
|
|
||||||
```typescript
|
|
||||||
brain.updateConfig({
|
|
||||||
cache: { enabled: false }
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Backup & Restore ✅
|
|
||||||
Full data backup capabilities:
|
|
||||||
```typescript
|
|
||||||
await brain.backup('backup.bin')
|
|
||||||
await brain.restore('backup.bin')
|
|
||||||
```
|
|
||||||
|
|
||||||
## 📈 Proven at Scale
|
|
||||||
|
|
||||||
- **10,000+ items**: Sub-10ms search
|
|
||||||
- **1M+ operations**: Stable memory usage
|
|
||||||
- **100+ concurrent users**: No performance degradation
|
|
||||||
- **Multi-node clusters**: Automatic failover
|
|
||||||
|
|
||||||
## 🚫 NOT Implemented (Planned)
|
|
||||||
|
|
||||||
These features are documented but NOT yet implemented:
|
|
||||||
- GraphQL API (use REST API instead)
|
|
||||||
- Kubernetes operators (use Docker)
|
|
||||||
- Some distributed features require manual configuration
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
*Last Updated: Latest Version*
|
|
||||||
*All features listed above are production-ready and tested*
|
|
||||||
|
|
@ -1,406 +0,0 @@
|
||||||
# Distributed Brainy System Guide
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
Brainy introduces a groundbreaking **zero-configuration distributed system** that transforms how vector databases scale. Unlike traditional distributed databases that require complex setup (Consul, etcd, Zookeeper), Brainy uses your existing storage (S3, GCS, R2) as the coordination layer.
|
|
||||||
|
|
||||||
## Key Innovation: Storage-Based Coordination
|
|
||||||
|
|
||||||
Instead of requiring separate infrastructure for coordination, Brainy leverages your storage backend:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Traditional distributed database setup:
|
|
||||||
// ❌ Setup Consul/etcd
|
|
||||||
// ❌ Configure node discovery
|
|
||||||
// ❌ Setup health checks
|
|
||||||
// ❌ Configure sharding
|
|
||||||
// ❌ Setup replication
|
|
||||||
|
|
||||||
// Brainy distributed setup:
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 's3',
|
|
||||||
options: { bucket: 'my-data' }
|
|
||||||
},
|
|
||||||
distributed: true // ✅ That's it!
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## Real-World Scenarios
|
|
||||||
|
|
||||||
### 1. Processing Multiple Streaming Data Sources (Bluesky, Twitter, etc.)
|
|
||||||
|
|
||||||
**Problem**: You need to ingest millions of posts from multiple social media firehoses simultaneously while maintaining search performance.
|
|
||||||
|
|
||||||
**Traditional Approach**:
|
|
||||||
- Separate ingestion and search clusters
|
|
||||||
- Complex queue systems (Kafka, RabbitMQ)
|
|
||||||
- Manual sharding configuration
|
|
||||||
- Complicated backpressure handling
|
|
||||||
|
|
||||||
**Brainy Solution**:
|
|
||||||
```typescript
|
|
||||||
// Node 1: Bluesky Ingestion
|
|
||||||
const ingestionNode1 = new Brainy({
|
|
||||||
storage: { type: 's3', options: { bucket: 'social-data' }},
|
|
||||||
distributed: true,
|
|
||||||
writeOnly: true // Optimized for writes
|
|
||||||
})
|
|
||||||
|
|
||||||
// Continuously ingest Bluesky firehose
|
|
||||||
blueskyStream.on('post', async (post) => {
|
|
||||||
await ingestionNode1.add({
|
|
||||||
content: post.text,
|
|
||||||
author: post.author,
|
|
||||||
timestamp: post.createdAt,
|
|
||||||
platform: 'bluesky'
|
|
||||||
}, 'social-post')
|
|
||||||
})
|
|
||||||
|
|
||||||
// Node 2: Twitter Ingestion (separate machine)
|
|
||||||
const ingestionNode2 = new Brainy({
|
|
||||||
storage: { type: 's3', options: { bucket: 'social-data' }},
|
|
||||||
distributed: true,
|
|
||||||
writeOnly: true
|
|
||||||
})
|
|
||||||
|
|
||||||
// Node 3-5: Search nodes (auto-balanced)
|
|
||||||
const searchNode = new Brainy({
|
|
||||||
storage: { type: 's3', options: { bucket: 'social-data' }},
|
|
||||||
distributed: true,
|
|
||||||
readOnly: true // Optimized for queries
|
|
||||||
})
|
|
||||||
|
|
||||||
// Search across ALL data from ALL sources
|
|
||||||
const results = await searchNode.find('AI trends', 100)
|
|
||||||
// Automatically queries all shards across all nodes!
|
|
||||||
```
|
|
||||||
|
|
||||||
**Benefits**:
|
|
||||||
- **Auto-sharding**: Data automatically distributed by content hash
|
|
||||||
- **No bottlenecks**: Each ingestion node writes directly to storage
|
|
||||||
- **Live search**: Search nodes see new data immediately
|
|
||||||
- **Auto-scaling**: Add nodes anytime, data rebalances automatically
|
|
||||||
|
|
||||||
### 2. Multi-Tenant SaaS Application
|
|
||||||
|
|
||||||
**Problem**: Each customer needs isolated, fast search across their documents, with ability to scale per customer.
|
|
||||||
|
|
||||||
**Brainy Solution**:
|
|
||||||
```typescript
|
|
||||||
// Spin up dedicated nodes per large customer
|
|
||||||
const enterpriseNode = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 's3',
|
|
||||||
options: {
|
|
||||||
bucket: 'customer-data',
|
|
||||||
prefix: 'customer-123/' // Isolated data
|
|
||||||
}
|
|
||||||
},
|
|
||||||
distributed: true
|
|
||||||
})
|
|
||||||
|
|
||||||
// Shared nodes for smaller customers
|
|
||||||
const sharedNode = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 's3',
|
|
||||||
options: { bucket: 'shared-customers' }
|
|
||||||
},
|
|
||||||
distributed: true
|
|
||||||
})
|
|
||||||
|
|
||||||
// Domain-based sharding ensures customer data stays together
|
|
||||||
await sharedNode.add(document, 'document', {
|
|
||||||
customerId: 'cust-456', // Used for shard assignment
|
|
||||||
domain: 'customer-456' // Keeps related data together
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Global Knowledge Graph with Local Inference
|
|
||||||
|
|
||||||
**Problem**: Building a knowledge graph that needs both global connectivity and local LLM inference.
|
|
||||||
|
|
||||||
**Brainy Solution**:
|
|
||||||
```typescript
|
|
||||||
// Edge nodes near users (with GPU)
|
|
||||||
const edgeNode = new Brainy({
|
|
||||||
storage: { type: 's3', options: { bucket: 'knowledge-graph' }},
|
|
||||||
distributed: true,
|
|
||||||
models: {
|
|
||||||
embed: { model: 'BAAI/bge-base-en-v1.5' },
|
|
||||||
chat: { model: 'meta-llama/Llama-3.2-3B-Instruct' }
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Central nodes for graph operations
|
|
||||||
const graphNode = new Brainy({
|
|
||||||
storage: { type: 's3', options: { bucket: 'knowledge-graph' }},
|
|
||||||
distributed: true,
|
|
||||||
augmentations: ['graph', 'triple-intelligence']
|
|
||||||
})
|
|
||||||
|
|
||||||
// Local inference with global knowledge
|
|
||||||
const context = await edgeNode.find(userQuery, 10)
|
|
||||||
const response = await edgeNode.chat(userQuery, { context })
|
|
||||||
|
|
||||||
// Graph traversal across all nodes
|
|
||||||
const connections = await graphNode.traverse({
|
|
||||||
start: 'concept:AI',
|
|
||||||
depth: 3,
|
|
||||||
relationship: 'related-to'
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## Competitive Advantages
|
|
||||||
|
|
||||||
### vs. Pinecone/Weaviate/Qdrant
|
|
||||||
|
|
||||||
| Feature | Traditional Vector DBs | Brainy Distributed |
|
|
||||||
|---------|----------------------|-------------------|
|
|
||||||
| Setup Complexity | High (k8s, operators) | Zero (just storage) |
|
|
||||||
| Minimum Nodes | 3-5 for HA | 1 (scale as needed) |
|
|
||||||
| Coordination | External (etcd, Consul) | Built-in (via storage) |
|
|
||||||
| Data Locality | Random sharding | Domain-aware sharding |
|
|
||||||
| Query Planning | Basic | Triple Intelligence |
|
|
||||||
| Cost at Scale | High (always-on clusters) | Low (scale to zero) |
|
|
||||||
|
|
||||||
### vs. Neo4j/ArangoDB (Graph Databases)
|
|
||||||
|
|
||||||
| Feature | Graph Databases | Brainy Distributed |
|
|
||||||
|---------|----------------|-------------------|
|
|
||||||
| Vector Search | Bolt-on/Limited | Native HNSW |
|
|
||||||
| Embedding Generation | External | Built-in (30+ models) |
|
|
||||||
| Distributed Transactions | Complex/Slow | Eventually consistent |
|
|
||||||
| Natural Language | No | Native (Triple Intelligence) |
|
|
||||||
| Setup | Very Complex | Zero config |
|
|
||||||
|
|
||||||
### vs. Elasticsearch/OpenSearch
|
|
||||||
|
|
||||||
| Feature | Elasticsearch | Brainy Distributed |
|
|
||||||
|---------|--------------|-------------------|
|
|
||||||
| Vector Support | Added later (slow) | Native (fast HNSW) |
|
|
||||||
| Cluster Management | Complex (master nodes) | Automatic (via storage) |
|
|
||||||
| Shard Rebalancing | Manual/Risky | Automatic/Safe |
|
|
||||||
| Memory Usage | Very High | Efficient |
|
|
||||||
| Query Language | Complex DSL | Natural language |
|
|
||||||
|
|
||||||
## Innovative Features
|
|
||||||
|
|
||||||
### 1. Domain-Aware Sharding
|
|
||||||
|
|
||||||
Unlike hash-based sharding, Brainy understands data relationships:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Documents about the same topic stay on the same shard
|
|
||||||
await brain.add(doc1, 'document', { domain: 'physics' })
|
|
||||||
await brain.add(doc2, 'document', { domain: 'physics' })
|
|
||||||
// Both documents on same shard = faster related queries
|
|
||||||
|
|
||||||
// Customer data stays together
|
|
||||||
await brain.add(order, 'order', { customerId: 'cust-123' })
|
|
||||||
await brain.add(invoice, 'invoice', { customerId: 'cust-123' })
|
|
||||||
// Same customer = same shard = better locality
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Streaming Shard Migration
|
|
||||||
|
|
||||||
Zero-downtime data movement between nodes:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Automatically triggered when nodes join/leave
|
|
||||||
// Uses HTTP streaming for efficiency
|
|
||||||
// Validates data integrity
|
|
||||||
// Atomic ownership transfer
|
|
||||||
// No query downtime!
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Storage-Based Consensus
|
|
||||||
|
|
||||||
No Raft/Paxos complexity:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Leader election via storage atomic operations
|
|
||||||
// Health monitoring via storage heartbeats
|
|
||||||
// Configuration consensus via storage CAS
|
|
||||||
// No split-brain issues!
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4. Intelligent Query Planning
|
|
||||||
|
|
||||||
The distributed query planner understands:
|
|
||||||
- Which shards contain relevant data
|
|
||||||
- Node health and latency
|
|
||||||
- Data locality and caching
|
|
||||||
- Triple Intelligence scoring
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Automatically optimizes query execution
|
|
||||||
const results = await brain.find('quantum physics')
|
|
||||||
// Planner knows:
|
|
||||||
// - Physics domain → shard-3
|
|
||||||
// - Node-2 has shard-3 cached
|
|
||||||
// - Route query to node-2
|
|
||||||
// - Merge results with Triple Intelligence
|
|
||||||
```
|
|
||||||
|
|
||||||
## Performance Characteristics
|
|
||||||
|
|
||||||
### Scalability
|
|
||||||
- **Horizontal**: Add nodes anytime
|
|
||||||
- **Vertical**: Nodes can differ in size
|
|
||||||
- **Geographic**: Nodes can be globally distributed
|
|
||||||
- **Elastic**: Scale to zero when idle
|
|
||||||
|
|
||||||
### Throughput
|
|
||||||
- **Writes**: Linear scaling with nodes
|
|
||||||
- **Reads**: Sub-linear (due to caching)
|
|
||||||
- **Mixed**: Read/write optimized nodes
|
|
||||||
|
|
||||||
### Latency
|
|
||||||
- **Local queries**: ~10ms p50
|
|
||||||
- **Distributed queries**: ~50ms p50
|
|
||||||
- **Shard migration**: Streaming (no bulk pause)
|
|
||||||
|
|
||||||
## Use Cases Where Brainy Excels
|
|
||||||
|
|
||||||
### ✅ Perfect For:
|
|
||||||
|
|
||||||
1. **Multi-source data ingestion** (social media, logs, events)
|
|
||||||
2. **Global search applications** (distributed teams)
|
|
||||||
3. **Multi-tenant SaaS** (customer isolation)
|
|
||||||
4. **Knowledge graphs** (with vector search)
|
|
||||||
5. **Edge AI applications** (local inference, global knowledge)
|
|
||||||
6. **Document intelligence** (contracts, research papers)
|
|
||||||
7. **Real-time analytics** (streaming + search)
|
|
||||||
|
|
||||||
### ⚠️ Consider Alternatives For:
|
|
||||||
|
|
||||||
1. **Strong consistency requirements** (use PostgreSQL)
|
|
||||||
2. **Sub-millisecond latency** (use Redis)
|
|
||||||
3. **Complex transactions** (use traditional RDBMS)
|
|
||||||
4. **Purely structured data** (use columnar stores)
|
|
||||||
|
|
||||||
## Migration from Other Systems
|
|
||||||
|
|
||||||
### From Pinecone/Weaviate:
|
|
||||||
```typescript
|
|
||||||
// Your existing vector search still works
|
|
||||||
const results = await brain.search(embedding, 10)
|
|
||||||
|
|
||||||
// But now you can scale horizontally!
|
|
||||||
// And add graph relationships!
|
|
||||||
// And use natural language!
|
|
||||||
```
|
|
||||||
|
|
||||||
### From Elasticsearch:
|
|
||||||
```typescript
|
|
||||||
// Import your documents
|
|
||||||
await brain.import('./elasticsearch-export.json')
|
|
||||||
|
|
||||||
// Queries are simpler
|
|
||||||
const results = await brain.find('user query')
|
|
||||||
// No complex DSL needed!
|
|
||||||
```
|
|
||||||
|
|
||||||
### From Neo4j:
|
|
||||||
```typescript
|
|
||||||
// Import your graph
|
|
||||||
await brain.importGraph('./neo4j-export.cypher')
|
|
||||||
|
|
||||||
// Now with vector search!
|
|
||||||
const similar = await brain.find('concepts like quantum computing')
|
|
||||||
```
|
|
||||||
|
|
||||||
## Deployment Patterns
|
|
||||||
|
|
||||||
### 1. Start Simple, Scale Later
|
|
||||||
```typescript
|
|
||||||
// Day 1: Single node
|
|
||||||
const brain = new Brainy({ storage: 's3' })
|
|
||||||
|
|
||||||
// Month 2: Growing data, add distribution
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: 's3',
|
|
||||||
distributed: true // Just add this!
|
|
||||||
})
|
|
||||||
|
|
||||||
// Month 6: Multiple nodes auto-balance
|
|
||||||
// No migration needed!
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Geographic Distribution
|
|
||||||
```typescript
|
|
||||||
// US Node
|
|
||||||
const usNode = new Brainy({
|
|
||||||
storage: { region: 'us-east-1' },
|
|
||||||
distributed: true
|
|
||||||
})
|
|
||||||
|
|
||||||
// EU Node
|
|
||||||
const euNode = new Brainy({
|
|
||||||
storage: { region: 'eu-west-1' },
|
|
||||||
distributed: true
|
|
||||||
})
|
|
||||||
|
|
||||||
// They automatically coordinate!
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Specialized Nodes
|
|
||||||
```typescript
|
|
||||||
// GPU nodes for embedding
|
|
||||||
const embedNode = new Brainy({
|
|
||||||
distributed: true,
|
|
||||||
writeOnly: true,
|
|
||||||
models: { embed: 'large-model' }
|
|
||||||
})
|
|
||||||
|
|
||||||
// CPU nodes for search
|
|
||||||
const searchNode = new Brainy({
|
|
||||||
distributed: true,
|
|
||||||
readOnly: true
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## Monitoring & Operations
|
|
||||||
|
|
||||||
### Health Checks
|
|
||||||
```typescript
|
|
||||||
const health = await brain.getClusterHealth()
|
|
||||||
// {
|
|
||||||
// nodes: 5,
|
|
||||||
// healthy: 5,
|
|
||||||
// shards: 16,
|
|
||||||
// status: 'green'
|
|
||||||
// }
|
|
||||||
```
|
|
||||||
|
|
||||||
### Shard Distribution
|
|
||||||
```typescript
|
|
||||||
const shards = await brain.getShardDistribution()
|
|
||||||
// Shows which nodes own which shards
|
|
||||||
```
|
|
||||||
|
|
||||||
### Migration Status
|
|
||||||
```typescript
|
|
||||||
const migrations = await brain.getActiveMigrations()
|
|
||||||
// Shows ongoing shard movements
|
|
||||||
```
|
|
||||||
|
|
||||||
## Conclusion
|
|
||||||
|
|
||||||
Brainy's distributed system is **production-ready** and offers:
|
|
||||||
|
|
||||||
1. **True zero-configuration** - Just add `distributed: true`
|
|
||||||
2. **Storage-based coordination** - No external dependencies
|
|
||||||
3. **Intelligent sharding** - Domain-aware data placement
|
|
||||||
4. **Automatic operations** - Rebalancing, failover, scaling
|
|
||||||
5. **Unified interface** - Vector + Graph + Document + LLM
|
|
||||||
|
|
||||||
This is not just another distributed database. It's a fundamental rethinking of how distributed systems should work in the cloud era.
|
|
||||||
|
|
||||||
## Next Steps
|
|
||||||
|
|
||||||
1. [Try the distributed quick start](./distributed-quickstart.md)
|
|
||||||
2. [Read the architecture deep dive](../architecture/distributed-storage.md)
|
|
||||||
3. [View benchmarks](../benchmarks/distributed-performance.md)
|
|
||||||
4. [Deploy to production](./production-deployment.md)
|
|
||||||
|
|
@ -165,32 +165,28 @@ await brain.syncWith({
|
||||||
- **Webhook support**: React to changes
|
- **Webhook support**: React to changes
|
||||||
- **API generation**: Auto-generate REST/GraphQL APIs
|
- **API generation**: Auto-generate REST/GraphQL APIs
|
||||||
|
|
||||||
### 🌍 Enterprise Scale 🚧 Coming Soon
|
### 🌍 Scale
|
||||||
|
|
||||||
**Everyone gets planetary scale:**
|
**Everyone gets the same scale model:**
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Same architecture Netflix uses, free for you
|
// Pure JS by default; install the optional native provider for billions of vectors
|
||||||
const brain = new Brainy({
|
const brain = new Brainy()
|
||||||
clustering: {
|
|
||||||
enabled: true, // Distributed mode
|
|
||||||
sharding: 'automatic', // Auto-sharding
|
|
||||||
replication: 3, // Triple replication
|
|
||||||
consensus: 'raft', // Strong consistency
|
|
||||||
geoDistribution: true // Multi-region support
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Handles everything from 1 to 1 billion entities
|
// 1 → ~1M vectors: pure-JS HNSW, zero extra setup
|
||||||
|
// 1M → 10B+ vectors: install @soulcraft/cortex for the native DiskANN provider
|
||||||
```
|
```
|
||||||
|
|
||||||
**Scaling features:**
|
**Scaling model:**
|
||||||
- **Horizontal scaling**: Add nodes as needed
|
- **Single process, no cluster**: Brainy runs in one process — no coordinator,
|
||||||
- **Auto-sharding**: Distributes data automatically
|
no peer discovery, no consensus to operate
|
||||||
- **Multi-region**: Global distribution
|
- **Optional native provider**: install `@soulcraft/cortex` to back the index with
|
||||||
- **Load balancing**: Automatic request distribution
|
on-disk DiskANN that scales to 10B+ vectors on one machine
|
||||||
- **Zero-downtime upgrades**: Rolling updates
|
- **Per-tenant pools**: isolate tenants by giving each its own Brainy instance and
|
||||||
- **Infinite scale**: No upper limits
|
storage directory
|
||||||
|
- **Horizontal read scaling**: run many reader processes against one shared on-disk
|
||||||
|
store (single writer, many readers); replicate the artifact with your operator
|
||||||
|
tooling
|
||||||
|
|
||||||
### 🛡️ Enterprise Compliance 🚧 Coming Soon
|
### 🛡️ Enterprise Compliance 🚧 Coming Soon
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ public: true
|
||||||
category: guides
|
category: guides
|
||||||
template: guide
|
template: guide
|
||||||
order: 9
|
order: 9
|
||||||
description: Export part or all of a brain to a portable, versioned PortableGraph document and import it back — by id, collection, connected neighbourhood, VFS subtree, predicate, or the whole brain. Covers vectors, edge policy, conflict handling, and id remapping.
|
description: Export part or all of a brain to a portable, versioned PortableGraph document and import it back — by id, collection, connected neighbourhood, VFS subtree, predicate, or the whole brain. export() lives on the immutable Db, so asOf()/with() give time-travel and what-if exports.
|
||||||
next:
|
next:
|
||||||
- guides/subtypes-and-facets
|
- guides/subtypes-and-facets
|
||||||
- api/README
|
- api/README
|
||||||
|
|
@ -13,38 +13,46 @@ next:
|
||||||
|
|
||||||
# Export & Import (portable graph)
|
# Export & Import (portable graph)
|
||||||
|
|
||||||
`brain.data()` exposes a **portable graph export/import** API. One method serializes a
|
Brainy serializes part or all of a brain — an item, a collection, a connected
|
||||||
graph — an item, a collection, a connected neighbourhood, a VFS subtree, a predicate
|
neighbourhood, a VFS subtree, a predicate match, or the whole brain — into a single
|
||||||
match, or the whole brain — into a single versioned JSON document (`PortableGraph`); the
|
versioned JSON document (`PortableGraph`), and restores it.
|
||||||
inverse restores it.
|
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
const data = await brain.data()
|
const graph = await brain.export() // whole brain → PortableGraph
|
||||||
|
await brain.import(graph) // restore (merge by id, re-embed if no vectors)
|
||||||
const graph = await data.export() // whole brain → PortableGraph
|
|
||||||
await data.import(graph) // restore (merge by id, re-embed if no vectors)
|
|
||||||
```
|
```
|
||||||
|
|
||||||
It is **portable** (human-readable JSON), **versioned** (`formatVersion`, so a 7.x export
|
It is **portable** (human-readable JSON), **versioned** (`formatVersion`, so a document
|
||||||
imports cleanly into 8.0), and **current-state** (the entities and edges as they are now —
|
written by 7.x imports cleanly into 8.0), and **current-state** (the entities and edges as
|
||||||
no generation history). Use it for portable artifacts, partial graphs, cross-environment
|
they are now — no generation history). Use it for portable artifacts, partial exports,
|
||||||
moves, and version upgrades.
|
cross-environment moves, and version upgrades.
|
||||||
|
|
||||||
|
`export()` is a method on the **immutable `Db` value**, so it composes with every way of
|
||||||
|
obtaining one:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
brain.export(sel) // = brain.now().export(sel)
|
||||||
|
;(await brain.asOf(gen)).export(sel) // time-travel export (a past generation)
|
||||||
|
brain.now().with(ops).export(sel) // what-if export (a speculative state)
|
||||||
|
```
|
||||||
|
|
||||||
## When to use which
|
## When to use which
|
||||||
|
|
||||||
| You want… | Use |
|
| You want… | Use |
|
||||||
|-----------|-----|
|
|-----------|-----|
|
||||||
| A portable, partial-or-whole, cross-version graph document | **`brain.data().export()` / `import()`** (this guide) |
|
| A portable, partial-or-whole, cross-version graph document | **`brain.export()` / `brain.import()`** (this guide) |
|
||||||
|
| A whole-brain snapshot **with generation history** | `brain.now().persist(path)` / `Brainy.load(path)` (native) |
|
||||||
| To ingest a CSV / PDF / Excel / JSON **file** as new entities | `brain.import(file)` — see [Import Anything](./import-anything.md) |
|
| To ingest a CSV / PDF / Excel / JSON **file** as new entities | `brain.import(file)` — see [Import Anything](./import-anything.md) |
|
||||||
|
|
||||||
The two are different operations that happen to share a verb: `import(file)` parses a
|
`import()` is **polymorphic**: hand it a `PortableGraph` and it does the graph round-trip;
|
||||||
foreign document into new entities; `data().import(graph)` restores a graph this API
|
hand it a file/buffer and it does foreign-file ingestion (dispatched on the document's
|
||||||
exported.
|
`format: 'brainy-portable-graph'` tag).
|
||||||
|
|
||||||
## Exporting
|
## Exporting
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
export(selector?, options?): Promise<PortableGraph>
|
brain.export(selector?, options?): Promise<PortableGraph>
|
||||||
|
// (also on any Db: brain.now().export(...), (await brain.asOf(g)).export(...))
|
||||||
```
|
```
|
||||||
|
|
||||||
### Selectors — *what* to export
|
### Selectors — *what* to export
|
||||||
|
|
@ -57,23 +65,19 @@ Omit the selector to export the whole brain. Otherwise pick a node set:
|
||||||
| A collection + its children | `{ collection: collectionId }` (alias `memberOf`) |
|
| A collection + its children | `{ collection: collectionId }` (alias `memberOf`) |
|
||||||
| A connected neighbourhood | `{ connected: { from: id, depth: 2, verbs?, direction? } }` |
|
| A connected neighbourhood | `{ connected: { from: id, depth: 2, verbs?, direction? } }` |
|
||||||
| A VFS directory / file (+ subtree) | `{ vfsPath: '/docs', recursive?: true }` |
|
| A VFS directory / file (+ subtree) | `{ vfsPath: '/docs', recursive?: true }` |
|
||||||
| Everything matching a predicate | `{ type, subtype, where, service }` |
|
| Everything matching a predicate | `{ type, subtype, where, service, visibility }` |
|
||||||
| The whole brain | *(omit)* |
|
| The whole brain | *(omit)* |
|
||||||
|
|
||||||
The selector reuses `find()`'s grammar — *"export what `find()` would match, minus ranking
|
The selector reuses `find()`'s grammar — *"export what `find()` would match, minus ranking
|
||||||
and limit."* Structural and predicate selectors **compose** — a structural selector picks
|
and limit."* Structural and predicate selectors **compose**:
|
||||||
the nodes, and predicate keys then filter them:
|
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Members of a collection whose status is "open"
|
// Members of a collection whose status is "open"
|
||||||
await data.export({ collection: collectionId, where: { status: 'open' } })
|
await brain.export({ collection: collectionId, where: { status: 'open' } })
|
||||||
```
|
|
||||||
|
|
||||||
If you already have results from `find()`, export exactly those with the `ids` selector:
|
// Already have find() results? Export exactly those with the ids selector
|
||||||
|
|
||||||
```typescript
|
|
||||||
const hits = await brain.find({ type: NounType.Document })
|
const hits = await brain.find({ type: NounType.Document })
|
||||||
await data.export({ ids: hits.map(r => r.id) })
|
await brain.export({ ids: hits.map(r => r.id) })
|
||||||
```
|
```
|
||||||
|
|
||||||
### Options — *how* to serialize
|
### Options — *how* to serialize
|
||||||
|
|
@ -82,56 +86,46 @@ await data.export({ ids: hits.map(r => r.id) })
|
||||||
|--------|---------|--------|
|
|--------|---------|--------|
|
||||||
| `includeVectors` | `false` | Carry embedding vectors verbatim. Off ⇒ `import()` re-embeds from `data`. |
|
| `includeVectors` | `false` | Carry embedding vectors verbatim. Off ⇒ `import()` re-embeds from `data`. |
|
||||||
| `includeContent` | `false` | Include VFS file bytes in `blobs` so files round-trip byte-identically. |
|
| `includeContent` | `false` | Include VFS file bytes in `blobs` so files round-trip byte-identically. |
|
||||||
| `includeSystem` | `false` | Include `system` entities such as the VFS root. |
|
| `includeSystem` | `false` | Include `visibility:'system'` entities such as the VFS root. |
|
||||||
| `edges` | `'induced'` | `'induced'` (both endpoints in the set), `'incident'` (also dangling edges, recorded in `danglingIds`), or `'none'` (nodes only). |
|
| `edges` | `'induced'` | `'induced'` (both endpoints in the set), `'incident'` (also dangling edges, recorded in `danglingIds`), or `'none'` (nodes only). |
|
||||||
|
|
||||||
```typescript
|
|
||||||
// A self-contained subgraph with vectors, ready to restore elsewhere
|
|
||||||
const subset = await data.export({ ids }, { includeVectors: true })
|
|
||||||
|
|
||||||
// A VFS subtree including file bytes
|
|
||||||
const tree = await data.export({ vfsPath: '/docs' }, { includeContent: true })
|
|
||||||
```
|
|
||||||
|
|
||||||
## Importing
|
## Importing
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import(graph, options?): Promise<ImportResult>
|
brain.import(graph, options?): Promise<ImportResult>
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The whole graph is applied as **one atomic transaction** — it advances the brain exactly
|
||||||
|
one generation, or none on failure.
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
const result = await brain.data().then(d => d.import(graph, { onConflict: 'merge' }))
|
const result = await brain.import(graph, { onConflict: 'merge' })
|
||||||
// → { imported, merged, skipped, reembedded, blobsWritten, errors }
|
// → { imported, merged, skipped, reembedded, blobsWritten, errors }
|
||||||
```
|
```
|
||||||
|
|
||||||
| Option | Default | Effect |
|
| Option | Default | Effect |
|
||||||
|--------|---------|--------|
|
|--------|---------|--------|
|
||||||
| `onConflict` | `'merge'` | `'merge'` (update existing id in place — assemble many graphs), `'replace'` (delete + recreate), or `'skip'`. |
|
| `onConflict` | `'merge'` | `'merge'` (update existing id in place — assemble many exported graphs), `'replace'` (delete + recreate), or `'skip'`. |
|
||||||
| `reembed` | `'auto'` | `'auto'` (use the carried vector, else re-embed from `data`) or `'never'` (require a carried vector; record an error if absent). |
|
| `reembed` | `'auto'` | `'auto'` (use the carried vector, else re-embed from `data`) or `'never'` (require a carried vector; record an error if absent). |
|
||||||
| `remapIds` | — | Rewrite every id on the way in, e.g. to clone a template subgraph under fresh ids. |
|
| `remapIds` | — | Rewrite every id on the way in, e.g. to clone a template subgraph under fresh ids. |
|
||||||
|
| `meta` | — | Transaction metadata recorded in the tx-log alongside the new generation. |
|
||||||
|
|
||||||
The default `onConflict: 'merge'` is what lets you assemble one working graph from many
|
The default `onConflict: 'merge'` lets you assemble one working graph from many exported
|
||||||
exported documents that share entity ids — re-importing an id merges rather than duplicates.
|
documents that share entity ids — re-importing an id merges rather than duplicates.
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Clone a subgraph under fresh ids (a copy, not a move)
|
|
||||||
const remap = new Map(graph.entities.map(e => [e.id, crypto.randomUUID()]))
|
|
||||||
await data.import(graph, { remapIds: id => remap.get(id) ?? id })
|
|
||||||
```
|
|
||||||
|
|
||||||
## The `PortableGraph` format
|
## The `PortableGraph` format
|
||||||
|
|
||||||
```jsonc
|
```jsonc
|
||||||
{
|
{
|
||||||
"format": "brainy-portable-graph",
|
"format": "brainy-portable-graph", // identifies the document type
|
||||||
"formatVersion": 1, // import gates on this (cross-version migration)
|
"formatVersion": 1, // import gates on this (cross-version migration)
|
||||||
"brainyVersion": "7.32.0",
|
"brainyVersion": "8.0.0",
|
||||||
"createdAt": "2026-06-16T…Z",
|
"createdAt": "2026-06-16T…Z",
|
||||||
"embedding": { "model": "all-MiniLM-L6-v2", "dimensions": 384 },
|
"embedding": { "model": "all-MiniLM-L6-v2", "dimensions": 384 },
|
||||||
"selector": { … }, // echoes what was exported (provenance)
|
"selector": { … }, // echoes what was exported (provenance)
|
||||||
"entities": [
|
"entities": [
|
||||||
{
|
{
|
||||||
"id": "…", "type": "Document", "subtype": "invoice",
|
"id": "…", "type": "Document", "subtype": "invoice", "visibility": "public",
|
||||||
"data": "…", // the embedding source
|
"data": "…", // the embedding source
|
||||||
"confidence": 1, "weight": 1, "service": "…",
|
"confidence": 1, "weight": 1, "service": "…",
|
||||||
"vector": [ … ], // only with includeVectors
|
"vector": [ … ], // only with includeVectors
|
||||||
|
|
@ -148,18 +142,32 @@ await data.import(graph, { remapIds: id => remap.get(id) ?? id })
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
Standard fields (`subtype`, `data`, `confidence`, `weight`, `service`) sit at the top level
|
Standard fields (`subtype`, `visibility`, `data`, `confidence`, `weight`, `service`) sit at
|
||||||
of each entity; `metadata` holds **only** custom user fields — mirroring the in-memory
|
the top level of each entity; `metadata` holds **only** custom user fields — mirroring the
|
||||||
`Entity` shape, so `import()` maps each field to its dedicated parameter.
|
in-memory `Entity` shape, so `import()` maps each field to its dedicated parameter. The
|
||||||
|
TypeScript types (`PortableGraph`, `PortableGraphEntity`, `PortableGraphRelation`, `ExportSelector`,
|
||||||
|
`ExportOptions`, `ImportOptions`, `ImportResult`) are exported from the package root.
|
||||||
|
|
||||||
|
## Generations & time-travel
|
||||||
|
|
||||||
|
The portable document is **current-state** — it never embeds generation history (that keeps
|
||||||
|
it cross-version-portable). History lives where it's queryable:
|
||||||
|
|
||||||
|
- **During a session:** `brain.asOf(g)` / `brain.now().with(ops)` on the live brain. Because
|
||||||
|
`export()` is on the `Db`, `(await brain.asOf(g)).export()` serializes a *past* generation
|
||||||
|
and `brain.now().with(ops).export()` serializes a *speculative* one.
|
||||||
|
- **A whole-brain snapshot with history:** `brain.now().persist(path)` / `Brainy.load(path)`
|
||||||
|
(native, generation-preserving) — a separate facility from this portable format.
|
||||||
|
|
||||||
|
Note: only `transact()` (and the write shortcuts that commit through it) advances a
|
||||||
|
generation, so time-travel export differs across transaction boundaries.
|
||||||
|
|
||||||
## Cross-version (7.x → 8.0)
|
## Cross-version (7.x → 8.0)
|
||||||
|
|
||||||
Because the document is shared and versioned, a graph written by 7.x imports into 8.0:
|
Because the document is shared and versioned, a PortableGraph written by 7.x imports into 8.0:
|
||||||
`formatVersion` is read forward, `subtype` is carried so 8.0 re-types correctly, and the
|
`formatVersion` is read forward, `subtype` is carried so 8.0 re-types correctly, and the
|
||||||
same 384-dimension model on both lines means `includeVectors:false` re-embeds identically
|
same 384-dimension model on both lines means `includeVectors:false` re-embeds identically
|
||||||
(or `true` carries vectors verbatim). The format is **current-state** — if you need a
|
(or `true` carries vectors verbatim).
|
||||||
whole-brain snapshot *with* generation history, that is a separate native facility
|
|
||||||
(`db.persist()` / `Brainy.load()` on 8.0).
|
|
||||||
|
|
||||||
## VFS
|
## VFS
|
||||||
|
|
||||||
|
|
@ -167,7 +175,7 @@ VFS directories are `Collection` entities and files are entities linked by `Cont
|
||||||
the whole filesystem (or any subtree) exports through the `vfsPath` selector:
|
the whole filesystem (or any subtree) exports through the `vfsPath` selector:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
await data.export({ vfsPath: '/' }, { includeContent: true }) // all VFS + bytes
|
await brain.export({ vfsPath: '/' }, { includeContent: true }) // all VFS + bytes
|
||||||
await data.export({ vfsPath: '/docs' }, { includeContent: true }) // one directory
|
await brain.export({ vfsPath: '/docs' }, { includeContent: true }) // one directory
|
||||||
await data.export({ vfsPath: '/a/b.txt' }, { includeContent: true }) // one file
|
await brain.export({ vfsPath: '/a/b.txt' }, { includeContent: true }) // one file
|
||||||
```
|
```
|
||||||
|
|
|
||||||
|
|
@ -74,7 +74,7 @@ When the auto-config is wrong for your workload (e.g. you know your entities are
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: { type: 'filesystem', options: { path: './data' } },
|
storage: { type: 'filesystem', path: './data' },
|
||||||
maxQueryLimit: 50_000 // raises the cap; still hard-clamped at 100 000
|
maxQueryLimit: 50_000 // raises the cap; still hard-clamped at 100 000
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
|
||||||
|
|
@ -1,15 +1,17 @@
|
||||||
# Framework Integration Guide
|
# Framework Integration Guide
|
||||||
|
|
||||||
Brainy 3.0 is **framework-first** - designed from the ground up to work seamlessly with modern JavaScript frameworks. This guide shows you how to integrate Brainy into any framework.
|
Brainy is **framework-friendly** - designed to drop into the server side of any modern JavaScript framework. This guide shows you how to integrate Brainy into framework-based apps.
|
||||||
|
|
||||||
## 🎯 Why Framework-First?
|
> **Runtime**: Brainy 8.0 runs on Node.js 22+ and Bun (server-side only). It is not a browser library. In framework apps, run Brainy in API routes, server components, server actions, loaders, or a dedicated backend service - never in client-side bundles.
|
||||||
|
|
||||||
Traditional AI databases require complex browser polyfills and bundler configurations. Brainy 3.0 trusts your framework to handle this:
|
## 🎯 Why Server-Side?
|
||||||
|
|
||||||
|
Brainy embeds an HNSW vector index, a graph engine, and a filesystem-backed persistence layer. These belong on the server:
|
||||||
|
|
||||||
- **Zero configuration**: Just `import { Brainy } from '@soulcraft/brainy'`
|
- **Zero configuration**: Just `import { Brainy } from '@soulcraft/brainy'`
|
||||||
- **Framework responsibility**: Let Next.js, Vite, Webpack handle Node.js polyfills
|
- **Auto storage detection**: `new Brainy()` auto-selects filesystem persistence on Node
|
||||||
- **Cleaner code**: No browser-specific entry points or conditional imports
|
- **Cleaner code**: No browser polyfills, no conditional client/server imports
|
||||||
- **Better DX**: Same API everywhere - browser, server, edge
|
- **Better DX**: One instance shared across your server routes
|
||||||
|
|
||||||
## 🚀 Quick Start
|
## 🚀 Quick Start
|
||||||
|
|
||||||
|
|
@ -24,7 +26,8 @@ npm install @soulcraft/brainy
|
||||||
```javascript
|
```javascript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraft/brainy'
|
||||||
|
|
||||||
// Works in any framework!
|
// Run on the server (API route, server component, backend service)
|
||||||
|
// new Brainy() auto-detects filesystem persistence on Node
|
||||||
const brain = new Brainy()
|
const brain = new Brainy()
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
||||||
|
|
@ -41,52 +44,48 @@ const results = await brain.find("framework integration")
|
||||||
|
|
||||||
## ⚛️ React Integration
|
## ⚛️ React Integration
|
||||||
|
|
||||||
|
Brainy runs on the server, so a React client component talks to it through an API endpoint (see the Next.js API route below). The hook fetches results; it never instantiates Brainy in the browser.
|
||||||
|
|
||||||
### Basic Hook Pattern
|
### Basic Hook Pattern
|
||||||
|
|
||||||
```jsx
|
```jsx
|
||||||
import { useState, useEffect, useCallback } from 'react'
|
import { useState, useCallback } from 'react'
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
function useBrainy() {
|
function useBrainySearch(endpoint = '/api/search') {
|
||||||
const [brain, setBrain] = useState(null)
|
const [results, setResults] = useState([])
|
||||||
const [isReady, setIsReady] = useState(false)
|
const [loading, setLoading] = useState(false)
|
||||||
|
|
||||||
useEffect(() => {
|
const search = useCallback(async (query) => {
|
||||||
const initBrain = async () => {
|
if (!query) return
|
||||||
const newBrain = new Brainy({
|
setLoading(true)
|
||||||
storage: { type: 'opfs' } // Browser storage
|
try {
|
||||||
|
const res = await fetch(endpoint, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ query })
|
||||||
})
|
})
|
||||||
await newBrain.init()
|
const { results } = await res.json()
|
||||||
setBrain(newBrain)
|
setResults(results)
|
||||||
setIsReady(true)
|
} finally {
|
||||||
|
setLoading(false)
|
||||||
}
|
}
|
||||||
|
}, [endpoint])
|
||||||
|
|
||||||
initBrain()
|
return { results, loading, search }
|
||||||
}, [])
|
|
||||||
|
|
||||||
return { brain, isReady }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Usage in component
|
// Usage in component
|
||||||
function SearchComponent() {
|
function SearchComponent() {
|
||||||
const { brain, isReady } = useBrainy()
|
const { results, loading, search } = useBrainySearch()
|
||||||
const [results, setResults] = useState([])
|
|
||||||
|
|
||||||
const handleSearch = useCallback(async (query) => {
|
|
||||||
if (!isReady) return
|
|
||||||
const searchResults = await brain.find(query)
|
|
||||||
setResults(searchResults)
|
|
||||||
}, [brain, isReady])
|
|
||||||
|
|
||||||
if (!isReady) return <div>Loading AI...</div>
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div>
|
<div>
|
||||||
<input
|
<input
|
||||||
type="text"
|
type="text"
|
||||||
placeholder="Search..."
|
placeholder="Search..."
|
||||||
onChange={(e) => handleSearch(e.target.value)}
|
onChange={(e) => search(e.target.value)}
|
||||||
/>
|
/>
|
||||||
|
{loading && <div>Searching...</div>}
|
||||||
<div>
|
<div>
|
||||||
{results.map(result => (
|
{results.map(result => (
|
||||||
<div key={result.id}>
|
<div key={result.id}>
|
||||||
|
|
@ -100,48 +99,34 @@ function SearchComponent() {
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
### React Context Pattern
|
### Shared Server Instance
|
||||||
|
|
||||||
```jsx
|
On the server, create one Brainy instance and reuse it across requests. This module is imported only by server code (API routes, server components), never by client components:
|
||||||
import React, { createContext, useContext, useEffect, useState } from 'react'
|
|
||||||
|
```javascript
|
||||||
|
// lib/brain.server.js
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraft/brainy'
|
||||||
|
|
||||||
const BrainyContext = createContext()
|
let brainPromise
|
||||||
|
|
||||||
export function BrainyProvider({ children }) {
|
export function getBrain() {
|
||||||
const [brain, setBrain] = useState(null)
|
if (!brainPromise) {
|
||||||
const [isReady, setIsReady] = useState(false)
|
brainPromise = (async () => {
|
||||||
|
// new Brainy() auto-detects filesystem persistence on Node
|
||||||
useEffect(() => {
|
const brain = new Brainy()
|
||||||
const initBrain = async () => {
|
await brain.init()
|
||||||
const newBrain = new Brainy()
|
return brain
|
||||||
await newBrain.init()
|
})()
|
||||||
setBrain(newBrain)
|
|
||||||
setIsReady(true)
|
|
||||||
}
|
|
||||||
|
|
||||||
initBrain()
|
|
||||||
}, [])
|
|
||||||
|
|
||||||
return (
|
|
||||||
<BrainyContext.Provider value={{ brain, isReady }}>
|
|
||||||
{children}
|
|
||||||
</BrainyContext.Provider>
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
export function useBrainContext() {
|
|
||||||
const context = useContext(BrainyContext)
|
|
||||||
if (!context) {
|
|
||||||
throw new Error('useBrainContext must be used within BrainyProvider')
|
|
||||||
}
|
}
|
||||||
return context
|
return brainPromise
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
## 🟢 Vue.js Integration
|
## 🟢 Vue.js Integration
|
||||||
|
|
||||||
### Composition API
|
Vue components call a server endpoint (see the Nuxt server route in the [Vue.js Integration Guide](vue-integration.md)); Brainy itself runs on the server.
|
||||||
|
|
||||||
|
### Composition API (client component)
|
||||||
|
|
||||||
```vue
|
```vue
|
||||||
<template>
|
<template>
|
||||||
|
|
@ -155,100 +140,70 @@ export function useBrainContext() {
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { ref, onMounted } from 'vue'
|
import { ref } from 'vue'
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
const brain = ref(null)
|
|
||||||
const isReady = ref(false)
|
|
||||||
const query = ref('')
|
const query = ref('')
|
||||||
const results = ref([])
|
const results = ref([])
|
||||||
|
|
||||||
onMounted(async () => {
|
|
||||||
brain.value = new Brainy({
|
|
||||||
storage: { type: 'opfs' }
|
|
||||||
})
|
|
||||||
await brain.value.init()
|
|
||||||
isReady.value = true
|
|
||||||
})
|
|
||||||
|
|
||||||
const search = async () => {
|
const search = async () => {
|
||||||
if (!isReady.value || !query.value) return
|
if (!query.value) return
|
||||||
results.value = await brain.value.find(query.value)
|
const res = await fetch('/api/search', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ query: query.value })
|
||||||
|
})
|
||||||
|
results.value = (await res.json()).results
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
```
|
```
|
||||||
|
|
||||||
### Vue 3 Plugin
|
### Shared Server Instance
|
||||||
|
|
||||||
|
On the server, create one Brainy instance and reuse it across requests:
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// plugins/brainy.js
|
// server/brain.js (server-only module)
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraft/brainy'
|
||||||
|
|
||||||
export default {
|
let brainPromise
|
||||||
install(app, options) {
|
|
||||||
const brain = new Brainy(options)
|
|
||||||
|
|
||||||
app.config.globalProperties.$brain = brain
|
export function getBrain() {
|
||||||
app.provide('brain', brain)
|
if (!brainPromise) {
|
||||||
|
brainPromise = (async () => {
|
||||||
// Initialize on app mount
|
// new Brainy() auto-detects filesystem persistence on Node
|
||||||
brain.init()
|
const brain = new Brainy()
|
||||||
|
await brain.init()
|
||||||
|
return brain
|
||||||
|
})()
|
||||||
}
|
}
|
||||||
|
return brainPromise
|
||||||
}
|
}
|
||||||
|
|
||||||
// main.js
|
|
||||||
import { createApp } from 'vue'
|
|
||||||
import BrainyPlugin from './plugins/brainy'
|
|
||||||
|
|
||||||
const app = createApp(App)
|
|
||||||
app.use(BrainyPlugin, {
|
|
||||||
storage: { type: 'opfs' }
|
|
||||||
})
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## 🅰️ Angular Integration
|
## 🅰️ Angular Integration
|
||||||
|
|
||||||
### Service Pattern
|
The Angular service calls your backend over HTTP; Brainy lives in that backend, not in the browser.
|
||||||
|
|
||||||
|
### Service Pattern (calls the backend)
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// brainy.service.ts
|
// brainy.service.ts
|
||||||
import { Injectable } from '@angular/core'
|
import { Injectable } from '@angular/core'
|
||||||
import { BehaviorSubject, Observable } from 'rxjs'
|
import { HttpClient } from '@angular/common/http'
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Observable } from 'rxjs'
|
||||||
|
|
||||||
@Injectable({
|
@Injectable({
|
||||||
providedIn: 'root'
|
providedIn: 'root'
|
||||||
})
|
})
|
||||||
export class BrainyService {
|
export class BrainyService {
|
||||||
private brain: Brainy
|
constructor(private http: HttpClient) {}
|
||||||
private readySubject = new BehaviorSubject<boolean>(false)
|
|
||||||
|
|
||||||
ready$: Observable<boolean> = this.readySubject.asObservable()
|
search(query: string): Observable<{ results: any[] }> {
|
||||||
|
return this.http.post<{ results: any[] }>('/api/search', { query })
|
||||||
constructor() {
|
|
||||||
this.initBrain()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private async initBrain() {
|
add(data: any, type: string, metadata?: any): Observable<{ id: string }> {
|
||||||
this.brain = new Brainy({
|
return this.http.post<{ id: string }>('/api/add', { data, type, metadata })
|
||||||
storage: { type: 'opfs' }
|
|
||||||
})
|
|
||||||
await this.brain.init()
|
|
||||||
this.readySubject.next(true)
|
|
||||||
}
|
|
||||||
|
|
||||||
async search(query: string): Promise<any[]> {
|
|
||||||
if (!this.readySubject.value) {
|
|
||||||
throw new Error('Brain not ready')
|
|
||||||
}
|
|
||||||
return await this.brain.find(query)
|
|
||||||
}
|
|
||||||
|
|
||||||
async add(data: any, type: string, metadata?: any): Promise<string> {
|
|
||||||
if (!this.readySubject.value) {
|
|
||||||
throw new Error('Brain not ready')
|
|
||||||
}
|
|
||||||
return await this.brain.add({ data, type, metadata })
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
@ -280,64 +235,52 @@ export class SearchComponent {
|
||||||
|
|
||||||
constructor(private brainyService: BrainyService) {}
|
constructor(private brainyService: BrainyService) {}
|
||||||
|
|
||||||
async search() {
|
search() {
|
||||||
if (!this.query) return
|
if (!this.query) return
|
||||||
this.results = await this.brainyService.search(this.query)
|
this.brainyService.search(this.query).subscribe(({ results }) => {
|
||||||
|
this.results = results
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The matching backend endpoint uses Brainy directly (Node/Bun):
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// server: api/search
|
||||||
|
import { Brainy } from '@soulcraft/brainy'
|
||||||
|
|
||||||
|
const brain = new Brainy() // auto-detects filesystem persistence on Node
|
||||||
|
await brain.init()
|
||||||
|
|
||||||
|
export async function handleSearch(query: string) {
|
||||||
|
return await brain.find(query)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
## 🚀 Next.js Integration
|
## 🚀 Next.js Integration
|
||||||
|
|
||||||
### App Router (Next.js 13+)
|
In Next.js, Brainy lives in server code only: API routes, server components, or server actions. Create one shared instance in a server-only module.
|
||||||
|
|
||||||
```jsx
|
### Shared Server Instance
|
||||||
// app/providers.jsx
|
|
||||||
'use client'
|
```javascript
|
||||||
import { createContext, useContext, useEffect, useState } from 'react'
|
// lib/brain.server.js (imported only by server code)
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraft/brainy'
|
||||||
|
|
||||||
const BrainyContext = createContext()
|
let brainPromise
|
||||||
|
|
||||||
export function BrainyProvider({ children }) {
|
export function getBrain() {
|
||||||
const [brain, setBrain] = useState(null)
|
if (!brainPromise) {
|
||||||
const [isReady, setIsReady] = useState(false)
|
brainPromise = (async () => {
|
||||||
|
const brain = new Brainy({
|
||||||
useEffect(() => {
|
storage: { type: 'filesystem', path: './data' }
|
||||||
const initBrain = async () => {
|
})
|
||||||
const newBrain = new Brainy()
|
await brain.init()
|
||||||
await newBrain.init()
|
return brain
|
||||||
setBrain(newBrain)
|
})()
|
||||||
setIsReady(true)
|
}
|
||||||
}
|
return brainPromise
|
||||||
|
|
||||||
initBrain()
|
|
||||||
}, [])
|
|
||||||
|
|
||||||
return (
|
|
||||||
<BrainyContext.Provider value={{ brain, isReady }}>
|
|
||||||
{children}
|
|
||||||
</BrainyContext.Provider>
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
export const useBrainy = () => useContext(BrainyContext)
|
|
||||||
```
|
|
||||||
|
|
||||||
```jsx
|
|
||||||
// app/layout.jsx
|
|
||||||
import { BrainyProvider } from './providers'
|
|
||||||
|
|
||||||
export default function RootLayout({ children }) {
|
|
||||||
return (
|
|
||||||
<html>
|
|
||||||
<body>
|
|
||||||
<BrainyProvider>
|
|
||||||
{children}
|
|
||||||
</BrainyProvider>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -345,45 +288,78 @@ export default function RootLayout({ children }) {
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// app/api/search/route.js
|
// app/api/search/route.js
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { getBrain } from '@/lib/brain.server'
|
||||||
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: { type: 'filesystem', path: './data' }
|
|
||||||
})
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
export async function POST(request) {
|
export async function POST(request) {
|
||||||
const { query } = await request.json()
|
const { query } = await request.json()
|
||||||
|
const brain = await getBrain()
|
||||||
const results = await brain.find(query)
|
const results = await brain.find(query)
|
||||||
|
|
||||||
return Response.json({ results })
|
return Response.json({ results })
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
## 🔷 Svelte Integration
|
### Server Action
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
// app/actions.js
|
||||||
|
'use server'
|
||||||
|
import { getBrain } from '@/lib/brain.server'
|
||||||
|
|
||||||
|
export async function search(query) {
|
||||||
|
const brain = await getBrain()
|
||||||
|
return await brain.find(query)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🔷 SvelteKit Integration
|
||||||
|
|
||||||
|
Brainy runs in a server-only module (`*.server.js`); the component fetches results from an endpoint.
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
// src/lib/server/brain.js (server-only — note the .server suffix)
|
||||||
|
import { Brainy } from '@soulcraft/brainy'
|
||||||
|
|
||||||
|
let brainPromise
|
||||||
|
|
||||||
|
export function getBrain() {
|
||||||
|
if (!brainPromise) {
|
||||||
|
brainPromise = (async () => {
|
||||||
|
const brain = new Brainy() // auto-detects filesystem persistence
|
||||||
|
await brain.init()
|
||||||
|
return brain
|
||||||
|
})()
|
||||||
|
}
|
||||||
|
return brainPromise
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
// src/routes/api/search/+server.js
|
||||||
|
import { json } from '@sveltejs/kit'
|
||||||
|
import { getBrain } from '$lib/server/brain'
|
||||||
|
|
||||||
|
export async function POST({ request }) {
|
||||||
|
const { query } = await request.json()
|
||||||
|
const brain = await getBrain()
|
||||||
|
return json({ results: await brain.find(query) })
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
```svelte
|
```svelte
|
||||||
<!-- SearchComponent.svelte -->
|
<!-- SearchComponent.svelte -->
|
||||||
<script>
|
<script>
|
||||||
import { onMount } from 'svelte'
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
let brain = null
|
|
||||||
let isReady = false
|
|
||||||
let query = ''
|
let query = ''
|
||||||
let results = []
|
let results = []
|
||||||
|
|
||||||
onMount(async () => {
|
|
||||||
brain = new Brainy({
|
|
||||||
storage: { type: 'opfs' }
|
|
||||||
})
|
|
||||||
await brain.init()
|
|
||||||
isReady = true
|
|
||||||
})
|
|
||||||
|
|
||||||
async function search() {
|
async function search() {
|
||||||
if (!isReady || !query) return
|
if (!query) return
|
||||||
results = await brain.find(query)
|
const res = await fetch('/api/search', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ query })
|
||||||
|
})
|
||||||
|
results = (await res.json()).results
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
|
|
@ -401,27 +377,23 @@ export async function POST(request) {
|
||||||
|
|
||||||
## 🌟 Solid.js Integration
|
## 🌟 Solid.js Integration
|
||||||
|
|
||||||
|
The component calls a server endpoint (use SolidStart server routes, or any backend, to host Brainy):
|
||||||
|
|
||||||
```jsx
|
```jsx
|
||||||
import { createSignal, onMount } from 'solid-js'
|
import { createSignal } from 'solid-js'
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
function SearchComponent() {
|
function SearchComponent() {
|
||||||
const [brain, setBrain] = createSignal(null)
|
|
||||||
const [isReady, setIsReady] = createSignal(false)
|
|
||||||
const [query, setQuery] = createSignal('')
|
const [query, setQuery] = createSignal('')
|
||||||
const [results, setResults] = createSignal([])
|
const [results, setResults] = createSignal([])
|
||||||
|
|
||||||
onMount(async () => {
|
|
||||||
const newBrain = new Brainy()
|
|
||||||
await newBrain.init()
|
|
||||||
setBrain(newBrain)
|
|
||||||
setIsReady(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
const search = async () => {
|
const search = async () => {
|
||||||
if (!isReady() || !query()) return
|
if (!query()) return
|
||||||
const searchResults = await brain().find(query())
|
const res = await fetch('/api/search', {
|
||||||
setResults(searchResults)
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ query: query() })
|
||||||
|
})
|
||||||
|
setResults((await res.json()).results)
|
||||||
}
|
}
|
||||||
|
|
||||||
return (
|
return (
|
||||||
|
|
@ -450,57 +422,25 @@ function SearchComponent() {
|
||||||
|
|
||||||
## 📦 Bundler Configuration
|
## 📦 Bundler Configuration
|
||||||
|
|
||||||
### Vite (Recommended)
|
Brainy is a server-side dependency, so keep it out of client bundles. Import it only from server-only modules (`*.server.js`, API routes, server components, server actions). If your bundler ever tries to pull Brainy into a client bundle, that's a sign it's being imported from a client component — move the import to a server module.
|
||||||
|
|
||||||
|
For server builds, mark Brainy as external so the bundler doesn't inline it:
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// vite.config.js
|
// vite.config.js (SSR build)
|
||||||
import { defineConfig } from 'vite'
|
import { defineConfig } from 'vite'
|
||||||
|
|
||||||
export default defineConfig({
|
export default defineConfig({
|
||||||
define: {
|
ssr: {
|
||||||
global: 'globalThis'
|
external: ['@soulcraft/brainy']
|
||||||
},
|
|
||||||
optimizeDeps: {
|
|
||||||
include: ['@soulcraft/brainy']
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
### Webpack
|
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// webpack.config.js
|
// rollup.config.js (server bundle)
|
||||||
module.exports = {
|
|
||||||
resolve: {
|
|
||||||
fallback: {
|
|
||||||
"fs": false,
|
|
||||||
"path": require.resolve("path-browserify"),
|
|
||||||
"crypto": require.resolve("crypto-browserify")
|
|
||||||
}
|
|
||||||
},
|
|
||||||
plugins: [
|
|
||||||
new webpack.ProvidePlugin({
|
|
||||||
global: 'global'
|
|
||||||
})
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Rollup
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// rollup.config.js
|
|
||||||
import { nodeResolve } from '@rollup/plugin-node-resolve'
|
|
||||||
import commonjs from '@rollup/plugin-commonjs'
|
|
||||||
|
|
||||||
export default {
|
export default {
|
||||||
plugins: [
|
external: ['@soulcraft/brainy', 'node:fs', 'node:path', 'node:crypto']
|
||||||
nodeResolve({
|
|
||||||
browser: true,
|
|
||||||
preferBuiltins: false
|
|
||||||
}),
|
|
||||||
commonjs()
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -508,30 +448,24 @@ export default {
|
||||||
|
|
||||||
### Server-Side Rendering
|
### Server-Side Rendering
|
||||||
|
|
||||||
|
Instantiate Brainy on the server and feed its results into the rendered page. Never construct it in client code.
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// Check if running in browser
|
// Server-side data loading (framework loader / getServerSideProps / load fn)
|
||||||
if (typeof window !== 'undefined') {
|
import { getBrain } from './brain.server'
|
||||||
// Browser-only code
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: { type: 'opfs' }
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Or use dynamic imports
|
export async function load({ url }) {
|
||||||
const initBrainForBrowser = async () => {
|
const brain = await getBrain()
|
||||||
if (typeof window === 'undefined') return null
|
const query = url.searchParams.get('q') ?? ''
|
||||||
|
const results = query ? await brain.find(query) : []
|
||||||
const { Brainy } = await import('@soulcraft/brainy')
|
return { results }
|
||||||
const brain = new Brainy()
|
|
||||||
await brain.init()
|
|
||||||
return brain
|
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
### Static Site Generation
|
### Static Site Generation
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// For build-time usage
|
// For build-time usage (runs in Node during the build)
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraft/brainy'
|
||||||
|
|
||||||
export async function generateStaticProps() {
|
export async function generateStaticProps() {
|
||||||
|
|
@ -540,11 +474,11 @@ export async function generateStaticProps() {
|
||||||
})
|
})
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
||||||
// Build a search index — export the whole brain as a portable PortableGraph document
|
// Build search index (paginate with { limit, offset } for larger stores)
|
||||||
const graph = await brain.data().then(d => d.export())
|
const allContent = await brain.find({ limit: 1000 })
|
||||||
|
|
||||||
return {
|
return {
|
||||||
props: { searchIndex: graph.entities }
|
props: { searchIndex: allContent }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
@ -552,67 +486,46 @@ export async function generateStaticProps() {
|
||||||
## 🔧 Framework-Specific Tips
|
## 🔧 Framework-Specific Tips
|
||||||
|
|
||||||
### React
|
### React
|
||||||
- Use `useCallback` for search functions to prevent re-renders
|
- Keep components client-side and call a Brainy-backed API route
|
||||||
- Consider `useMemo` for expensive brain operations
|
- Use `useCallback` for fetch handlers to prevent re-renders
|
||||||
- Implement cleanup in `useEffect` for proper memory management
|
- Debounce keystroke-driven searches before hitting the endpoint
|
||||||
|
|
||||||
### Vue
|
### Vue
|
||||||
- Use `shallowRef` for the brain instance (it's not reactive data)
|
- Components call an endpoint; the shared instance lives in a server module
|
||||||
- Consider Pinia for global brain state management
|
- Consider Pinia for caching results client-side
|
||||||
- Use `watchEffect` for reactive search queries
|
- Debounce reactive search queries
|
||||||
|
|
||||||
### Angular
|
### Angular
|
||||||
- Implement proper dependency injection with services
|
- Use `HttpClient` and RxJS to call the backend
|
||||||
- Use RxJS observables for reactive search
|
- Hold the shared Brainy instance in your Node backend, not the app
|
||||||
- Consider lazy loading brain in feature modules
|
- Consider lazy loading search features in feature modules
|
||||||
|
|
||||||
### Next.js
|
### Next.js
|
||||||
- Use dynamic imports for client-side only features
|
- Put Brainy in server-only modules (`*.server.js`), API routes, or server actions
|
||||||
- Consider API routes for server-side brain operations
|
- Reuse one shared instance across requests
|
||||||
- Implement proper error boundaries
|
- Implement proper error boundaries for failed fetches
|
||||||
|
|
||||||
## 🚨 Common Issues & Solutions
|
## 🚨 Common Issues & Solutions
|
||||||
|
|
||||||
### Issue: "crypto is not defined"
|
### Issue: "fs module not found" / "crypto is not defined" in the browser
|
||||||
**Solution**: Your framework should handle this automatically. If not:
|
**Cause**: Brainy was imported into a client bundle. Brainy 8.0 is a server-side library (Node 22+/Bun) and uses Node built-ins like `fs` and `crypto`.
|
||||||
```javascript
|
**Solution**: Import Brainy only from server code — server-only modules (`*.server.js`), API routes, server components, or server actions. From client components, call those endpoints instead.
|
||||||
// Add to your bundle config
|
|
||||||
define: {
|
|
||||||
global: 'globalThis'
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Issue: "fs module not found"
|
### Issue: Large client bundle size
|
||||||
**Solution**: This is expected in browsers. Use browser-compatible storage:
|
**Cause**: A client module is pulling in Brainy.
|
||||||
```javascript
|
**Solution**: Move the `import { Brainy } from '@soulcraft/brainy'` into a server-only module so it never reaches the browser bundle.
|
||||||
const brain = new Brainy({
|
|
||||||
storage: { type: 'opfs' } // Or 'memory' for development
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Issue: Large bundle size
|
|
||||||
**Solution**: Use dynamic imports for optional features:
|
|
||||||
```javascript
|
|
||||||
const brain = await import('@soulcraft/brainy').then(m => new m.Brainy())
|
|
||||||
```
|
|
||||||
|
|
||||||
### Issue: SSR hydration mismatch
|
### Issue: SSR hydration mismatch
|
||||||
**Solution**: Initialize brain only on client:
|
**Solution**: Run the search on the server (loader / server action / API route) and pass the results down as props, so server and client render the same markup.
|
||||||
```javascript
|
|
||||||
useEffect(() => {
|
|
||||||
// Browser-only initialization
|
|
||||||
initBrain()
|
|
||||||
}, [])
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🎯 Best Practices
|
## 🎯 Best Practices
|
||||||
|
|
||||||
1. **Initialize Once**: Create brain instance at app level, not component level
|
1. **Initialize Once**: Create one shared Brainy instance per server process, not per request
|
||||||
2. **Use Context**: Share brain instance across components with context/providers
|
2. **Server-Only**: Import Brainy only from server modules — never from client components
|
||||||
3. **Handle Loading**: Always show loading states during brain initialization
|
3. **Endpoint Boundary**: Expose search/add through API routes or server actions
|
||||||
4. **Error Boundaries**: Implement proper error handling for brain operations
|
4. **Handle Loading**: Show loading states in the client while the fetch is in flight
|
||||||
5. **Memory Management**: Clean up brain instances on unmount
|
5. **Error Handling**: Catch and surface failed endpoint calls gracefully
|
||||||
6. **Storage Strategy**: Choose appropriate storage for your deployment target
|
6. **Storage**: Use `filesystem` for persistence (the default on Node) or `memory` for ephemeral/tests
|
||||||
|
|
||||||
## 📚 Next Steps
|
## 📚 Next Steps
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -49,28 +49,23 @@ await brain.import(csv, { format: 'csv' })
|
||||||
|
|
||||||
### 📊 Import Excel - Multi-Sheet Support
|
### 📊 Import Excel - Multi-Sheet Support
|
||||||
```javascript
|
```javascript
|
||||||
// Import entire Excel workbook
|
// Import entire Excel workbook — every sheet is processed automatically
|
||||||
await brain.import('sales-report.xlsx')
|
await brain.import('sales-report.xlsx')
|
||||||
// ✨ Processes all sheets, preserves structure, infers types!
|
// ✨ Processes all sheets, preserves structure, infers types!
|
||||||
|
|
||||||
// Or specific sheets only
|
// Mirror the workbook into the VFS, grouped by sheet
|
||||||
await brain.import('data.xlsx', {
|
await brain.import('data.xlsx', {
|
||||||
excelSheets: ['Customers', 'Orders']
|
vfsPath: '/imports/data',
|
||||||
|
groupBy: 'sheet'
|
||||||
})
|
})
|
||||||
// ✨ Multi-sheet data becomes interconnected entities!
|
// ✨ Multi-sheet data becomes interconnected entities!
|
||||||
```
|
```
|
||||||
|
|
||||||
### 📑 Import PDF - Text & Tables
|
### 📑 Import PDF - Text & Tables
|
||||||
```javascript
|
```javascript
|
||||||
// Import PDF documents
|
// Import PDF documents — text and tables are extracted automatically
|
||||||
await brain.import('research-paper.pdf')
|
await brain.import('research-paper.pdf')
|
||||||
// ✨ Extracts text, detects tables, preserves metadata!
|
// ✨ Extracts text, detects tables, preserves metadata!
|
||||||
|
|
||||||
// With table extraction
|
|
||||||
await brain.import('report.pdf', {
|
|
||||||
pdfExtractTables: true
|
|
||||||
})
|
|
||||||
// ✨ Converts PDF tables to structured data automatically!
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### 📝 Import YAML - File or String
|
### 📝 Import YAML - File or String
|
||||||
|
|
@ -265,7 +260,7 @@ Once imported, use Triple Intelligence to query:
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// Vector search
|
// Vector search
|
||||||
const similar = await brain.search('engineers')
|
const similar = await brain.find('engineers')
|
||||||
|
|
||||||
// Natural language
|
// Natural language
|
||||||
const results = await brain.find('people in engineering who joined this year')
|
const results = await brain.find('people in engineering who joined this year')
|
||||||
|
|
|
||||||
|
|
@ -1173,21 +1173,22 @@ await this.graphIndex.addEdge(
|
||||||
```
|
```
|
||||||
|
|
||||||
**Benefits**:
|
**Benefits**:
|
||||||
- **O(1) relationship lookups**: `getRelations(entityId)` is instant
|
- **O(1) relationship lookups**: `related(entityId)` is instant
|
||||||
- **Bidirectional traversal**: Find incoming and outgoing edges
|
- **Bidirectional traversal**: Find incoming and outgoing edges
|
||||||
- **Type filtering**: Get only `CreatedBy` relationships
|
- **Type filtering**: Get only `CreatedBy` relationships
|
||||||
- **Global statistics**: Count relationships by type
|
- **Global statistics**: Count relationships by type
|
||||||
|
|
||||||
**Query Examples**:
|
**Query Examples**:
|
||||||
```typescript
|
```typescript
|
||||||
// What did Mona Lisa create?
|
// What did Mona Lisa create? (outgoing edges)
|
||||||
const outgoing = await brain.getRelations('ent_mona_lisa_...', { direction: 'outgoing' })
|
const outgoing = await brain.related({ from: 'ent_mona_lisa_...' })
|
||||||
|
|
||||||
// What created Mona Lisa?
|
// What created Mona Lisa? (incoming edges)
|
||||||
const incoming = await brain.getRelations('ent_mona_lisa_...', { direction: 'incoming' })
|
const incoming = await brain.related({ to: 'ent_mona_lisa_...' })
|
||||||
|
|
||||||
// Get only CreatedBy relationships
|
// Get only CreatedBy relationships
|
||||||
const createdBy = await brain.getRelations('ent_mona_lisa_...', {
|
const createdBy = await brain.related({
|
||||||
|
from: 'ent_mona_lisa_...',
|
||||||
type: VerbType.CreatedBy
|
type: VerbType.CreatedBy
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
@ -1308,7 +1309,7 @@ await this.history.recordImport(
|
||||||
```
|
```
|
||||||
|
|
||||||
**Use Cases**:
|
**Use Cases**:
|
||||||
- List all imports: `brain.data().listImports()`
|
- List all imports: `coordinator.getHistory().getHistory()` (the `ImportHistory` entries shown above)
|
||||||
- Reimport with same settings
|
- Reimport with same settings
|
||||||
- Audit trail for compliance
|
- Audit trail for compliance
|
||||||
- Rollback imports
|
- Rollback imports
|
||||||
|
|
@ -1687,11 +1688,11 @@ verbCounts['CreatedBy']
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 8. Storage by Cloud Provider
|
### 8. Storage Layout
|
||||||
|
|
||||||
Brainy supports multiple storage adapters:
|
Brainy 8.0 ships two adapters: filesystem and memory.
|
||||||
|
|
||||||
#### File System (Local)
|
#### Filesystem (Default)
|
||||||
```
|
```
|
||||||
.brainy/
|
.brainy/
|
||||||
├── nouns/
|
├── nouns/
|
||||||
|
|
@ -1701,43 +1702,18 @@ Brainy supports multiple storage adapters:
|
||||||
└── index.json
|
└── index.json
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Google Cloud Storage (GCS)
|
|
||||||
```
|
|
||||||
gs://my-bucket/brainy/
|
|
||||||
├── nouns/
|
|
||||||
│ └── ent_mona_lisa_1730000000000.json
|
|
||||||
├── nouns-metadata/
|
|
||||||
│ └── ent_mona_lisa_1730000000000.json
|
|
||||||
└── ...
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Amazon S3
|
|
||||||
```
|
|
||||||
s3://my-bucket/brainy/
|
|
||||||
├── nouns/
|
|
||||||
│ └── ent_mona_lisa_1730000000000.json
|
|
||||||
└── ...
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Cloudflare R2
|
|
||||||
```
|
|
||||||
r2://my-bucket/brainy/
|
|
||||||
├── nouns/
|
|
||||||
│ └── ent_mona_lisa_1730000000000.json
|
|
||||||
└── ...
|
|
||||||
```
|
|
||||||
|
|
||||||
**Configuration**:
|
**Configuration**:
|
||||||
```typescript
|
```typescript
|
||||||
const brain = await Brainy.create({
|
const brain = await Brainy.create({
|
||||||
storage: {
|
storage: {
|
||||||
type: 'gcs',
|
type: 'filesystem',
|
||||||
bucket: 'my-bucket',
|
path: './.brainy'
|
||||||
prefix: 'brainy/'
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
|
For off-site backup, snapshot `path` from your scheduler with `gsutil rsync`, `aws s3 sync`, `rclone`, or `tar`. Brainy itself doesn't reach out to object storage.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Performance & Scale
|
## Performance & Scale
|
||||||
|
|
@ -1767,7 +1743,7 @@ const brain = await Brainy.create({
|
||||||
- HNSW Index: O(log n) search (1B entities = ~30 hops)
|
- HNSW Index: O(log n) search (1B entities = ~30 hops)
|
||||||
- Metadata Index: O(1) filtering
|
- Metadata Index: O(1) filtering
|
||||||
- Graph Adjacency: O(1) relationship lookups
|
- Graph Adjacency: O(1) relationship lookups
|
||||||
- Storage: Unlimited (cloud buckets)
|
- Storage: Bounded by the filesystem volume backing `path`
|
||||||
|
|
||||||
### Optimization Tips
|
### Optimization Tips
|
||||||
|
|
||||||
|
|
@ -1911,7 +1887,7 @@ groupBy: 'type'
|
||||||
- ✅ O(1) metadata filtering
|
- ✅ O(1) metadata filtering
|
||||||
- ✅ O(1) relationship traversal
|
- ✅ O(1) relationship traversal
|
||||||
- ✅ Human-readable VFS structure
|
- ✅ Human-readable VFS structure
|
||||||
- ✅ Cloud storage support (GCS/S3/R2)
|
- ✅ Filesystem-backed persistence (snapshot/sync the directory for off-site backup)
|
||||||
- ✅ Billion-scale performance
|
- ✅ Billion-scale performance
|
||||||
- ✅ Zero mocks, production-ready!
|
- ✅ Zero mocks, production-ready!
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -351,7 +351,7 @@ const result = await brain.import(file)
|
||||||
const products = await brain.find({ type: 'Product' })
|
const products = await brain.find({ type: 'Product' })
|
||||||
|
|
||||||
// Get entity relationships
|
// Get entity relationships
|
||||||
const relations = await brain.getRelations(products[0].id)
|
const relations = await brain.related(products[0].id)
|
||||||
|
|
||||||
// Search VFS
|
// Search VFS
|
||||||
const vfsFiles = await brain.vfs().find(result.vfs.rootPath + '/**/*.json')
|
const vfsFiles = await brain.vfs().find(result.vfs.rootPath + '/**/*.json')
|
||||||
|
|
|
||||||
|
|
@ -17,10 +17,7 @@ store. This guide covers the safe ways to query a running Brainy directory.
|
||||||
|
|
||||||
## The cardinal rule
|
## The cardinal rule
|
||||||
|
|
||||||
**Never open a second writer on the same directory.** It will throw on
|
**Never open a second writer on the same directory.** Filesystem storage will throw, and any other write path will corrupt the live writer's state. Use `Brainy.openReadOnly()` or the `brainy inspect` CLI instead.
|
||||||
filesystem storage; on cloud storage it'll silently overwrite the live
|
|
||||||
writer's state. Use `Brainy.openReadOnly()` or the `brainy inspect` CLI
|
|
||||||
instead.
|
|
||||||
|
|
||||||
## The CLI is the fastest path
|
## The CLI is the fastest path
|
||||||
|
|
||||||
|
|
@ -97,7 +94,7 @@ $ brainy inspect health /data/brain
|
||||||
{
|
{
|
||||||
"overall": "warn",
|
"overall": "warn",
|
||||||
"checks": [
|
"checks": [
|
||||||
{ "name": "index-parity", "status": "pass", "message": "HNSW (1851) and metadata (1851) agree." },
|
{ "name": "index-parity", "status": "pass", "message": "Vector (1851) and metadata (1851) agree." },
|
||||||
{ "name": "field-registry", "status": "pass", "message": "23 fields registered for 1851 entities." },
|
{ "name": "field-registry", "status": "pass", "message": "23 fields registered for 1851 entities." },
|
||||||
{ "name": "seeded-records", "status": "warn", "message": "15 entities tagged _seeded:true." },
|
{ "name": "seeded-records", "status": "warn", "message": "15 entities tagged _seeded:true." },
|
||||||
{ "name": "writer-heartbeat", "status": "pass", "message": "Writer healthy (PID 1774431...)." }
|
{ "name": "writer-heartbeat", "status": "pass", "message": "Writer healthy (PID 1774431...)." }
|
||||||
|
|
@ -114,7 +111,7 @@ check fails — useful for piping into monitoring or CI.
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraft/brainy'
|
||||||
|
|
||||||
const reader = await Brainy.openReadOnly({
|
const reader = await Brainy.openReadOnly({
|
||||||
storage: { type: 'filesystem', rootDirectory: '/data/brain' }
|
storage: { type: 'filesystem', path: '/data/brain' }
|
||||||
})
|
})
|
||||||
|
|
||||||
// Force the writer to flush before reading
|
// Force the writer to flush before reading
|
||||||
|
|
@ -137,8 +134,8 @@ console.log(health.overall)
|
||||||
await reader.close()
|
await reader.close()
|
||||||
```
|
```
|
||||||
|
|
||||||
Every mutation method (`add`, `update`, `delete`, `relate`, `commit`,
|
Every mutation method (`add`, `update`, `remove`, `relate`, `transact`,
|
||||||
`fork`, `branch`, ...) throws on a read-only instance with a clear message.
|
`restore`, ...) throws on a read-only instance with a clear message.
|
||||||
|
|
||||||
## Backups
|
## Backups
|
||||||
|
|
||||||
|
|
@ -151,9 +148,10 @@ brainy inspect backup /data/brain /backups/brain-2026-05-15.tar
|
||||||
```
|
```
|
||||||
|
|
||||||
For periodic backups (hourly, daily), schedule this via cron or your
|
For periodic backups (hourly, daily), schedule this via cron or your
|
||||||
container scheduler. For point-in-time recovery, use Brainy's COW
|
container scheduler. For point-in-time recovery, use the Db API's
|
||||||
`commit()` API — the snapshots there are content-addressed and never
|
`db.persist(path)` — a self-contained hard-link snapshot that later writes
|
||||||
overwritten.
|
can never alter, restorable with `brain.restore(path, { confirm: true })`.
|
||||||
|
See [Snapshots & Time Travel](./snapshots-and-time-travel.md).
|
||||||
|
|
||||||
## Comparing two stores
|
## Comparing two stores
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -5,10 +5,10 @@ public: true
|
||||||
category: getting-started
|
category: getting-started
|
||||||
template: guide
|
template: guide
|
||||||
order: 1
|
order: 1
|
||||||
description: Install Brainy with npm, bun, yarn, or pnpm. Works in Node.js 22+, Bun 1.0+, and browser (OPFS). TypeScript included.
|
description: Install Brainy with npm, bun, yarn, or pnpm. Works in Node.js 22+ and Bun 1.0+ (server-only since 8.0). TypeScript included.
|
||||||
next:
|
next:
|
||||||
- getting-started/quick-start
|
- getting-started/quick-start
|
||||||
- concepts/zero-config
|
- guides/storage-adapters
|
||||||
---
|
---
|
||||||
|
|
||||||
# Installation
|
# Installation
|
||||||
|
|
@ -45,7 +45,7 @@ console.log('Brainy ready.')
|
||||||
|
|
||||||
## Native Acceleration (Optional)
|
## Native Acceleration (Optional)
|
||||||
|
|
||||||
For production workloads, add Cortex for Rust-accelerated SIMD distance calculations, vector quantization, and native embeddings:
|
For production workloads, add Cortex for Rust-accelerated SIMD distance calculations and native embeddings:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
npm install @soulcraft/cortex
|
npm install @soulcraft/cortex
|
||||||
|
|
@ -53,28 +53,18 @@ npm install @soulcraft/cortex
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraft/brainy'
|
||||||
import { registerCortex } from '@soulcraft/cortex'
|
|
||||||
|
|
||||||
registerCortex() // activates native acceleration globally
|
const brain = new Brainy({ plugins: ['@soulcraft/cortex'] })
|
||||||
|
await brain.init() // native providers registered during init
|
||||||
const brain = new Brainy()
|
|
||||||
await brain.init()
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Cortex delivers a **5.2x geometric mean speedup** — see [Brainy vs Cortex](/docs/cortex/comparison) for measured benchmarks.
|
Cortex registers native (Rust/SIMD) vector, metadata, and graph engines behind the same Brainy `find()` API — no code changes, an optional dependency for production-scale workloads.
|
||||||
|
|
||||||
## Browser (OPFS)
|
## Server-only since 8.0
|
||||||
|
|
||||||
Brainy works in the browser using the Origin Private File System:
|
Brainy 8.0 runs on Node.js 22+ and Bun 1.0+. Browser support (OPFS storage,
|
||||||
|
Web Workers, in-browser WASM embeddings) was removed in 8.0 — the 7.x line
|
||||||
```typescript
|
remains available on npm if you need it.
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
const brain = new Brainy({ storage: 'opfs' })
|
|
||||||
await brain.init()
|
|
||||||
```
|
|
||||||
|
|
||||||
No server required. Data persists across page refreshes in the browser's private storage.
|
|
||||||
|
|
||||||
## TypeScript
|
## TypeScript
|
||||||
|
|
||||||
|
|
@ -96,5 +86,4 @@ const id = await brain.add({
|
||||||
## Next Steps
|
## Next Steps
|
||||||
|
|
||||||
- [Quick Start](/docs/getting-started/quick-start) — build your first knowledge graph in 60 seconds
|
- [Quick Start](/docs/getting-started/quick-start) — build your first knowledge graph in 60 seconds
|
||||||
- [Zero Configuration](/docs/concepts/zero-config) — understand what Brainy auto-detects
|
|
||||||
- [Storage Adapters](/docs/guides/storage-adapters) — choose the right storage for your deployment
|
- [Storage Adapters](/docs/guides/storage-adapters) — choose the right storage for your deployment
|
||||||
|
|
|
||||||
|
|
@ -252,7 +252,7 @@ const result = await brain.import('./glossary.xlsx', {
|
||||||
// - "part_of"
|
// - "part_of"
|
||||||
// - "related_to"
|
// - "related_to"
|
||||||
|
|
||||||
const relations = await brain.getRelations({ limit: 100 })
|
const relations = await brain.related({ limit: 100 })
|
||||||
const types = new Set(relations.map(r => r.label))
|
const types = new Set(relations.map(r => r.label))
|
||||||
console.log(types)
|
console.log(types)
|
||||||
// Set { 'capital_of', 'guards', 'located_in', 'related_to' }
|
// Set { 'capital_of', 'guards', 'located_in', 'related_to' }
|
||||||
|
|
|
||||||
|
|
@ -35,15 +35,18 @@ This single WASM file contains everything needed for sentence embeddings.
|
||||||
|
|
||||||
### Bun (Recommended)
|
### Bun (Recommended)
|
||||||
|
|
||||||
```typescript
|
```bash
|
||||||
// Works with Bun runtime
|
# Bun as a runtime — supported and recommended
|
||||||
|
bun add @soulcraft/brainy
|
||||||
bun run server.ts
|
bun run server.ts
|
||||||
|
|
||||||
// Works with bun --compile (single binary deployment!)
|
|
||||||
bun build --compile --target=bun server.ts
|
|
||||||
./server // Self-contained binary with embedded model
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Brainy is pure WebAssembly with no native binaries, so the module graph stays
|
||||||
|
bundler-friendly. Single-binary `bun build --compile` is **not a supported
|
||||||
|
target** at present: Bun 1.3.10 has a `--compile` codegen regression
|
||||||
|
(`__promiseAll is not defined`) triggered by top-level `await` in the bundled
|
||||||
|
graph. Run Brainy under the Bun runtime (above) instead.
|
||||||
|
|
||||||
### Node.js
|
### Node.js
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
|
|
@ -164,7 +167,7 @@ await brain.init()
|
||||||
|
|
||||||
**What's new:**
|
**What's new:**
|
||||||
- Faster initialization
|
- Faster initialization
|
||||||
- Works with `bun --compile`
|
- Bundler-friendly (pure WASM, no native binaries)
|
||||||
- No network requirements
|
- No network requirements
|
||||||
|
|
||||||
### From Custom Embedding Functions
|
### From Custom Embedding Functions
|
||||||
|
|
@ -216,9 +219,8 @@ export { brain }
|
||||||
|
|
||||||
### Deployment
|
### Deployment
|
||||||
```bash
|
```bash
|
||||||
# Option 1: Bun compile (single binary)
|
# Option 1: Bun runtime
|
||||||
bun build --compile server.ts
|
bun run server.ts
|
||||||
./server # Contains everything
|
|
||||||
|
|
||||||
# Option 2: Docker
|
# Option 2: Docker
|
||||||
docker build -t my-app .
|
docker build -t my-app .
|
||||||
|
|
|
||||||
|
|
@ -1,371 +0,0 @@
|
||||||
# Neural API Guide
|
|
||||||
|
|
||||||
> Semantic intelligence features for clustering, similarity, and analysis
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
The Neural API provides advanced AI-powered features for understanding relationships and patterns in your data. Access it through `brain.neural()` (method call) after initializing Brainy.
|
|
||||||
|
|
||||||
## Quick Start
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
const brain = new Brainy()
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Access Neural API (note: neural() is a method call)
|
|
||||||
const neural = brain.neural()
|
|
||||||
|
|
||||||
// Find similar items
|
|
||||||
const similarity = await neural.similar('text1', 'text2')
|
|
||||||
|
|
||||||
// Auto-cluster your data
|
|
||||||
const clusters = await neural.clusters()
|
|
||||||
```
|
|
||||||
|
|
||||||
## Core Features
|
|
||||||
|
|
||||||
### 1. Semantic Clustering
|
|
||||||
|
|
||||||
Automatically group related items based on their meaning:
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Simple clustering - let Brainy decide
|
|
||||||
const clusters = await neural.clusters()
|
|
||||||
|
|
||||||
// Each cluster contains:
|
|
||||||
// - id: Unique identifier
|
|
||||||
// - members: Array of item IDs in this cluster
|
|
||||||
// - centroid: The "center" of the cluster
|
|
||||||
// - label: Optional descriptive label
|
|
||||||
// - confidence: How confident the clustering is
|
|
||||||
|
|
||||||
// Example: Organize customer feedback
|
|
||||||
const feedback = [
|
|
||||||
await brain.add("The app crashes when I upload photos"),
|
|
||||||
await brain.add("Photo upload feature is broken"),
|
|
||||||
await brain.add("Great customer service!"),
|
|
||||||
await brain.add("Support team was very helpful"),
|
|
||||||
await brain.add("Pricing is too high"),
|
|
||||||
await brain.add("Too expensive for what it offers")
|
|
||||||
]
|
|
||||||
|
|
||||||
const themes = await neural.clusters()
|
|
||||||
// Results in 3 clusters: bugs, support, pricing
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Advanced Clustering Options
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Control clustering behavior
|
|
||||||
const clusters = await neural.clusters({
|
|
||||||
algorithm: 'kmeans', // Algorithm to use
|
|
||||||
maxClusters: 5, // Maximum clusters to create
|
|
||||||
threshold: 0.7 // Minimum similarity within clusters
|
|
||||||
})
|
|
||||||
|
|
||||||
// Cluster specific items only
|
|
||||||
const techItems = ['id1', 'id2', 'id3', 'id4']
|
|
||||||
const techClusters = await neural.clusters(techItems)
|
|
||||||
|
|
||||||
// Find clusters near a specific item
|
|
||||||
const relatedClusters = await neural.clusters('central-item-id')
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Similarity Calculation
|
|
||||||
|
|
||||||
Compare any two items to see how similar they are:
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Compare by ID
|
|
||||||
const score = await neural.similar('item1-id', 'item2-id')
|
|
||||||
// Returns 0-1 (0 = completely different, 1 = identical)
|
|
||||||
|
|
||||||
// Compare text directly
|
|
||||||
const score = await neural.similar(
|
|
||||||
"Machine learning is fascinating",
|
|
||||||
"AI and deep learning are interesting"
|
|
||||||
)
|
|
||||||
// Returns ~0.75 (pretty similar)
|
|
||||||
|
|
||||||
// Compare vectors
|
|
||||||
const v1 = await brain.embed("concept 1")
|
|
||||||
const v2 = await brain.embed("concept 2")
|
|
||||||
const score = await neural.similar(v1, v2)
|
|
||||||
|
|
||||||
// Get detailed similarity analysis
|
|
||||||
const detailed = await neural.similar('id1', 'id2', {
|
|
||||||
detailed: true
|
|
||||||
})
|
|
||||||
// Returns: {
|
|
||||||
// score: 0.85,
|
|
||||||
// confidence: 0.92,
|
|
||||||
// explanation: "High semantic overlap in technology domain"
|
|
||||||
// }
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Finding Neighbors
|
|
||||||
|
|
||||||
Discover items similar to a given item:
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Find 5 most similar items
|
|
||||||
const neighbors = await neural.neighbors('item-id', 5)
|
|
||||||
|
|
||||||
// Each neighbor has:
|
|
||||||
// - id: The neighbor's ID
|
|
||||||
// - similarity: How similar (0-1)
|
|
||||||
// - data: The actual content
|
|
||||||
|
|
||||||
// Example: Recommend similar articles
|
|
||||||
const articleId = await brain.add("Guide to React Hooks")
|
|
||||||
const similar = await neural.neighbors(articleId, 3)
|
|
||||||
|
|
||||||
for (const article of similar) {
|
|
||||||
console.log(`${article.similarity * 100}% similar: ${article.data}`)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4. Semantic Hierarchy
|
|
||||||
|
|
||||||
Build a hierarchy showing relationships between items:
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
const hierarchy = await neural.hierarchy('item-id')
|
|
||||||
|
|
||||||
// Returns structure like:
|
|
||||||
// {
|
|
||||||
// self: { id: 'item-id', type: 'article' },
|
|
||||||
// parent: { id: 'parent-id', similarity: 0.8 },
|
|
||||||
// siblings: [
|
|
||||||
// { id: 'sibling1', similarity: 0.75 },
|
|
||||||
// { id: 'sibling2', similarity: 0.72 }
|
|
||||||
// ],
|
|
||||||
// children: [
|
|
||||||
// { id: 'child1', similarity: 0.85 }
|
|
||||||
// ]
|
|
||||||
// }
|
|
||||||
|
|
||||||
// Use for navigation or breadcrumbs
|
|
||||||
const hier = await neural.hierarchy(currentDoc)
|
|
||||||
console.log(`You are here: ${hier.self.id}`)
|
|
||||||
if (hier.parent) {
|
|
||||||
console.log(`Parent topic: ${hier.parent.id}`)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 5. Outlier Detection
|
|
||||||
|
|
||||||
Find unusual or anomalous items in your data:
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Find items that don't fit patterns
|
|
||||||
const outliers = await neural.outliers(0.3)
|
|
||||||
// Returns array of IDs that are > 0.3 distance from others
|
|
||||||
|
|
||||||
// Example: Detect spam or unusual content
|
|
||||||
const messages = [
|
|
||||||
await brain.add("Meeting at 3pm"),
|
|
||||||
await brain.add("Lunch plans for tomorrow"),
|
|
||||||
await brain.add("BUY NOW!!! AMAZING DEALS!!!"),
|
|
||||||
await brain.add("Project deadline next week")
|
|
||||||
]
|
|
||||||
|
|
||||||
const suspicious = await neural.outliers(0.4)
|
|
||||||
// Returns the spam message ID
|
|
||||||
```
|
|
||||||
|
|
||||||
### 6. Visualization Support
|
|
||||||
|
|
||||||
Generate data for visualization libraries:
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Create force-directed graph data
|
|
||||||
const vizData = await neural.visualize({
|
|
||||||
maxNodes: 100, // Limit nodes for performance
|
|
||||||
dimensions: 2, // 2D or 3D
|
|
||||||
algorithm: 'force' // Layout algorithm
|
|
||||||
})
|
|
||||||
|
|
||||||
// Returns:
|
|
||||||
// {
|
|
||||||
// nodes: [
|
|
||||||
// { id: 'n1', x: 10, y: 20, cluster: 'c1' },
|
|
||||||
// { id: 'n2', x: 30, y: 40, cluster: 'c1' }
|
|
||||||
// ],
|
|
||||||
// edges: [
|
|
||||||
// { source: 'n1', target: 'n2', weight: 0.8 }
|
|
||||||
// ],
|
|
||||||
// clusters: [
|
|
||||||
// { id: 'c1', color: '#ff6b6b', size: 15 }
|
|
||||||
// ]
|
|
||||||
// }
|
|
||||||
|
|
||||||
// Use with D3.js, Cytoscape, or other viz libraries
|
|
||||||
const data = await neural.visualize({ dimensions: 3 })
|
|
||||||
// Now feed to Three.js for 3D visualization
|
|
||||||
```
|
|
||||||
|
|
||||||
## Practical Examples
|
|
||||||
|
|
||||||
### Content Recommendation System
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// User reads an article
|
|
||||||
const currentArticle = 'article-123'
|
|
||||||
|
|
||||||
// Find similar content
|
|
||||||
const recommendations = await neural.neighbors(currentArticle, 5)
|
|
||||||
|
|
||||||
// Group all content into topics
|
|
||||||
const topics = await neural.clusters()
|
|
||||||
|
|
||||||
// Find which topic this article belongs to
|
|
||||||
const currentTopic = topics.find(t =>
|
|
||||||
t.members.includes(currentArticle)
|
|
||||||
)
|
|
||||||
|
|
||||||
// Recommend from same topic first, then similar items
|
|
||||||
const sameTopicArticles = currentTopic.members
|
|
||||||
.filter(id => id !== currentArticle)
|
|
||||||
.slice(0, 3)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Customer Feedback Analysis
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
import { NounType } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
// Add feedback with metadata
|
|
||||||
const feedbackIds = []
|
|
||||||
for (const feedback of customerFeedback) {
|
|
||||||
const id = await brain.add({
|
|
||||||
data: feedback.text,
|
|
||||||
type: NounType.Document,
|
|
||||||
metadata: {
|
|
||||||
rating: feedback.rating,
|
|
||||||
date: feedback.date,
|
|
||||||
product: feedback.product
|
|
||||||
}
|
|
||||||
})
|
|
||||||
feedbackIds.push(id)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cluster to find themes
|
|
||||||
const neural = brain.neural()
|
|
||||||
const themes = await neural.clusters(feedbackIds)
|
|
||||||
|
|
||||||
// Analyze each theme
|
|
||||||
for (const theme of themes) {
|
|
||||||
// Get items using Promise.all with brain.get()
|
|
||||||
const items = await Promise.all(
|
|
||||||
theme.members.map(id => brain.get(id))
|
|
||||||
)
|
|
||||||
|
|
||||||
const avgRating = items.reduce((sum, item) =>
|
|
||||||
sum + (item?.metadata?.rating || 0), 0) / items.length
|
|
||||||
|
|
||||||
console.log(`Theme with ${theme.members.length} items`)
|
|
||||||
console.log(`Average rating: ${avgRating}`)
|
|
||||||
|
|
||||||
// Find representative feedback for this theme
|
|
||||||
const centroidId = theme.members[0] // Closest to center
|
|
||||||
const example = await brain.get(centroidId)
|
|
||||||
console.log(`Example: "${example?.data}"`)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Knowledge Base Organization
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
import { NounType } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
// Analyze existing knowledge base - use find() to get documents
|
|
||||||
const allDocs = await brain.find({ type: NounType.Document, limit: 1000 })
|
|
||||||
|
|
||||||
// Access neural API
|
|
||||||
const neural = brain.neural()
|
|
||||||
|
|
||||||
// Find duplicate or highly similar content
|
|
||||||
const duplicates = []
|
|
||||||
for (let i = 0; i < allDocs.length; i++) {
|
|
||||||
for (let j = i + 1; j < allDocs.length; j++) {
|
|
||||||
const similarity = await neural.similar(
|
|
||||||
allDocs[i].id,
|
|
||||||
allDocs[j].id
|
|
||||||
)
|
|
||||||
if (similarity > 0.95) {
|
|
||||||
duplicates.push([allDocs[i].id, allDocs[j].id])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Build topic hierarchy
|
|
||||||
const mainTopics = await neural.clusters({
|
|
||||||
maxClusters: 10,
|
|
||||||
algorithm: 'hierarchical'
|
|
||||||
})
|
|
||||||
|
|
||||||
// For each main topic, find subtopics
|
|
||||||
for (const topic of mainTopics) {
|
|
||||||
const subtopics = await neural.clusters(topic.members)
|
|
||||||
console.log(`Topic has ${subtopics.length} subtopics`)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Performance Tips
|
|
||||||
|
|
||||||
1. **Caching**: Neural API automatically caches results. Repeated calls with same parameters are instant.
|
|
||||||
|
|
||||||
2. **Batch Operations**: Process multiple items together rather than one at a time.
|
|
||||||
|
|
||||||
3. **Sampling**: For large datasets, use sampling:
|
|
||||||
```javascript
|
|
||||||
const clusters = await neural.clusters({
|
|
||||||
algorithm: 'sample',
|
|
||||||
sampleSize: 1000 // Only analyze 1000 items
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
4. **Async Processing**: All neural operations are async and non-blocking.
|
|
||||||
|
|
||||||
## Error Handling
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
try {
|
|
||||||
const similarity = await neural.similar('id1', 'id2')
|
|
||||||
} catch (error) {
|
|
||||||
// Handle errors
|
|
||||||
if (error.message.includes('not found')) {
|
|
||||||
console.log('One of the items does not exist')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Safe clustering with empty data
|
|
||||||
const clusters = await neural.clusters([])
|
|
||||||
// Returns empty array, doesn't throw
|
|
||||||
|
|
||||||
// Non-existent IDs return 0 similarity
|
|
||||||
const sim = await neural.similar('fake-id-1', 'fake-id-2')
|
|
||||||
// Returns 0
|
|
||||||
```
|
|
||||||
|
|
||||||
## Advanced Configuration
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Configure neural behavior at initialization
|
|
||||||
const brain = new Brainy({
|
|
||||||
neural: {
|
|
||||||
cacheSize: 1000, // Cache up to 1000 results
|
|
||||||
defaultAlgorithm: 'kmeans',
|
|
||||||
similarityMetric: 'cosine'
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## Next Steps
|
|
||||||
|
|
||||||
- Explore [Triple Intelligence](../architecture/triple-intelligence.md) for combined vector + graph + metadata queries
|
|
||||||
- Learn about [Augmentations](../augmentations/README.md) to extend Neural API
|
|
||||||
- See [API Reference](../api/README.md) for complete method documentation
|
|
||||||
|
|
@ -7,8 +7,8 @@ template: guide
|
||||||
order: 8
|
order: 8
|
||||||
description: Use the per-entity `_rev` counter and `update({ ifRev })` to coordinate concurrent writes safely. Covers the lock pattern, idempotent inserts with `ifAbsent`, and recovery on conflict.
|
description: Use the per-entity `_rev` counter and `update({ ifRev })` to coordinate concurrent writes safely. Covers the lock pattern, idempotent inserts with `ifAbsent`, and recovery on conflict.
|
||||||
next:
|
next:
|
||||||
|
- concepts/consistency-model
|
||||||
- guides/find-limits
|
- guides/find-limits
|
||||||
- api/README
|
|
||||||
---
|
---
|
||||||
|
|
||||||
# Optimistic concurrency with `_rev`
|
# Optimistic concurrency with `_rev`
|
||||||
|
|
@ -25,7 +25,7 @@ Brainy 7.31.0 adds a per-entity revision counter so multiple writers can coordin
|
||||||
| `add({ id, ifAbsent: true })` | By-ID idempotent insert. Returns the existing `id` if one is already present; no throw, no overwrite. |
|
| `add({ id, ifAbsent: true })` | By-ID idempotent insert. Returns the existing `id` if one is already present; no throw, no overwrite. |
|
||||||
| `addMany({ items, ifAbsent: true })` | Applies `ifAbsent` to every item. Per-item `ifAbsent` overrides the batch flag. |
|
| `addMany({ items, ifAbsent: true })` | Applies `ifAbsent` to every item. Per-item `ifAbsent` overrides the batch flag. |
|
||||||
|
|
||||||
`_rev` is independent of `brain.versions.*` (named snapshots), of `brain.fork()` / branches (COW), and of VFS file versioning. They all coexist; `_rev` is the per-write counter used for CAS.
|
`_rev` is the **per-entity** counter. Its store-wide counterpart is the generation counter behind the [Db API](../concepts/consistency-model.md): `brain.transact(ops, { ifAtGeneration })` is CAS over the whole store, `update({ ifRev })` (and `ifRev` on `transact()` update operations) is CAS over one entity.
|
||||||
|
|
||||||
## The lock pattern
|
## The lock pattern
|
||||||
|
|
||||||
|
|
@ -134,42 +134,49 @@ await brain.addIfMissing({ // ← not a real API
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
It's race-prone outside a transaction: two concurrent imports both see "not found," both insert, you get duplicates. Without a unique-index primitive (which Brainy doesn't have today), this pattern needs to live inside a transaction:
|
It's race-prone as a plain read-then-write: two concurrent imports both see "not found," both insert, you get duplicates. Without a unique-index primitive (which Brainy doesn't have today), close the race with whole-store CAS — read at a pinned generation, then commit only if nothing moved:
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
// The race-safe shape, available once brain.transact() ships in 8.0.
|
import { GenerationConflictError } from '@soulcraft/brainy'
|
||||||
await brain.transact(async tx => {
|
|
||||||
const existing = await tx.find({
|
async function addIfMissingByEmail(email: string, data: string) {
|
||||||
type: 'Person',
|
for (let attempt = 0; attempt < 5; attempt++) {
|
||||||
where: { email: 'x@y.com' },
|
const db = brain.now()
|
||||||
limit: 1
|
try {
|
||||||
})
|
const existing = await db.find({
|
||||||
if (existing.length === 0) {
|
type: NounType.Person,
|
||||||
await tx.add({ type: 'Person', data: '...', metadata: { email: 'x@y.com' } })
|
where: { email },
|
||||||
|
limit: 1
|
||||||
|
})
|
||||||
|
if (existing.length > 0) return existing[0].id
|
||||||
|
|
||||||
|
const committed = await brain.transact(
|
||||||
|
[{ op: 'add', type: NounType.Person, subtype: 'customer', data, metadata: { email } }],
|
||||||
|
{ ifAtGeneration: db.generation } // rejects if ANYTHING committed since the read
|
||||||
|
)
|
||||||
|
return committed.receipt!.ids[0]
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof GenerationConflictError) continue // world moved — re-read + retry
|
||||||
|
throw err
|
||||||
|
} finally {
|
||||||
|
await db.release()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
})
|
throw new Error('addIfMissingByEmail conflict after 5 attempts')
|
||||||
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
For 7.31.0, lean on `ifAbsent` when you control the ID, and accept the inherent dedup race when you don't. The 8.0 `brain.transact()` is the natural home for the attribute-based variant.
|
`ifAtGeneration` is deliberately coarse — *any* committed write invalidates it — so keep the retry bound. When you control the ID, `ifAbsent` stays the cheaper tool.
|
||||||
|
|
||||||
## How `_rev` interacts with the other versioning systems
|
## How `_rev` relates to generations
|
||||||
|
|
||||||
Brainy has several persistence primitives that all touch the word "version" in different ways. `_rev` is independent of every one of them:
|
Brainy 8.0 has exactly two write-coordination counters, at two granularities:
|
||||||
|
|
||||||
| System | What it tracks | When it advances |
|
| Counter | Scope | What it tracks | CAS surface | Conflict error |
|
||||||
|---|---|---|
|
|---|---|---|---|---|
|
||||||
| **`_rev`** (new in 7.31.0) | Per-entity write counter | Auto, on every successful `update()` |
|
| **`_rev`** | One entity | Per-entity write count, bumped on every successful update | `update({ ifRev })`, `{ op: 'update', ifRev }` in `transact()` | `RevisionConflictError` |
|
||||||
| **`brain.versions.save()`** | Named snapshots per entity | Explicit — you call `save()` with a tag |
|
| **Generation** | Whole store | One tick per committed `transact()` batch or single-operation write | `transact(ops, { ifAtGeneration })` | `GenerationConflictError` |
|
||||||
| **`brain.fork()` / branches** | Whole-brain copy-on-write | Explicit — you call `fork(name)` |
|
|
||||||
| **VFS file versioning** | Per-VFS-file snapshots (Document entity) | Same as `brain.versions.save()` |
|
|
||||||
|
|
||||||
If you're on a branch, each branch has its own copy of every entity (that's what COW means), so each branch has its own `_rev` per entity — same as every other field. Snapshots taken via `brain.versions.save()` capture the entity at that moment including its `_rev` at that time; the snapshot's own `version: number` is the snapshot index, distinct from `_rev`.
|
They compose: a `transact()` batch can carry per-entity `ifRev` checks *and* a whole-store `ifAtGeneration`; any failed check rejects the entire batch before anything is staged. Generations also power snapshots and time travel (`brain.now()`, `brain.asOf()`, `db.persist()`) — see the [consistency model](../concepts/consistency-model.md) and [Snapshots & Time Travel](./snapshots-and-time-travel.md).
|
||||||
|
|
||||||
## What's coming in 8.0
|
A snapshot or historical view captures each entity *including* its `_rev` at that moment, so reading the past and writing back with `ifRev` against the live state works exactly as you'd hope: the write fails if the entity moved since the state you copied from.
|
||||||
|
|
||||||
Brainy 8.0 ships a Datomic-style immutable `Db` API where `brain.transact(tx)` is the public multi-write atomicity primitive. Two things change for code written against 7.31.0's surface:
|
|
||||||
|
|
||||||
- `_rev` survives unchanged. It's part of the 8.0 entity shape; the auto-bump moves to the new `transact()` write path. Code that uses `update({ ifRev })` keeps working.
|
|
||||||
- A new `brain.transact(tx, { ifAtGeneration: prev.generation })` adds generation-based CAS for whole-transaction atomicity, layered on top of `_rev`. Per-entity for single-record patterns (the lock above), generation-based for "did the world move under me."
|
|
||||||
|
|
||||||
If you adopt `_rev` + `ifRev` in 7.31.0, no migration work is needed for 8.0.
|
|
||||||
|
|
|
||||||
|
|
@ -60,17 +60,17 @@ const nextId: string = await brain.add({
|
||||||
await brain.relate({
|
await brain.relate({
|
||||||
from: nextId,
|
from: nextId,
|
||||||
to: reactId,
|
to: reactId,
|
||||||
type: VerbType.BuiltOn
|
type: VerbType.DependsOn
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
## 5. Query with Triple Intelligence
|
## 5. Query with Triple Intelligence
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import type { FindResult } from '@soulcraft/brainy'
|
import type { Result } from '@soulcraft/brainy'
|
||||||
|
|
||||||
// All three search paradigms in one call
|
// All three search paradigms in one call
|
||||||
const results: FindResult[] = await brain.find({
|
const results: Result[] = await brain.find({
|
||||||
query: 'modern frontend frameworks', // Vector similarity search
|
query: 'modern frontend frameworks', // Vector similarity search
|
||||||
where: { year: { greaterThan: 2015 } }, // Metadata filtering
|
where: { year: { greaterThan: 2015 } }, // Metadata filtering
|
||||||
connected: { to: reactId, depth: 2 } // Graph traversal
|
connected: { to: reactId, depth: 2 } // Graph traversal
|
||||||
|
|
@ -108,4 +108,4 @@ await brain.find({ query: 'people who work at Anthropic' })
|
||||||
- [Triple Intelligence](/docs/concepts/triple-intelligence) — understand how the query engine works
|
- [Triple Intelligence](/docs/concepts/triple-intelligence) — understand how the query engine works
|
||||||
- [The Find System](/docs/guides/find-system) — advanced queries, operators, and graph traversal
|
- [The Find System](/docs/guides/find-system) — advanced queries, operators, and graph traversal
|
||||||
- [API Reference](/docs/api/reference) — complete method documentation
|
- [API Reference](/docs/api/reference) — complete method documentation
|
||||||
- [Storage Adapters](/docs/guides/storage-adapters) — S3, GCS, Azure, filesystem, OPFS
|
- [Storage Adapters](/docs/guides/storage-adapters) — filesystem, memory
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
# Schema Migrations
|
# Schema Migrations
|
||||||
|
|
||||||
Brainy includes a built-in migration system for transforming entity and verb metadata across storage versions. Migrations are pure functions that run once per storage instance, with automatic backup, resume support, and error tracking.
|
Brainy includes a built-in migration system for transforming entity and verb metadata across storage versions. Migrations are pure functions that run once per storage instance, with optional snapshot backup (`backupTo`), resume support, and error tracking.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -48,15 +48,13 @@ When `brain.init()` runs:
|
||||||
|
|
||||||
When `brain.migrate()` runs:
|
When `brain.migrate()` runs:
|
||||||
|
|
||||||
1. **Backup** — creates an instant COW branch (`pre-migration-7.17.0`) tagged with `system:backup` metadata. Rollback is possible by switching to this branch.
|
1. **Backup (optional)** — with `backupTo`, a hard-link snapshot of the current generation is persisted before any transform runs. Rollback is `brain.restore(backupPath, { confirm: true })`.
|
||||||
|
|
||||||
2. **Transform main branch** — iterates all nouns/verbs in paginated batches. For each entity, calls the `transform` function. If it returns a new object, saves it. If it returns `null`, skips. Vectors are never touched.
|
2. **Transform** — iterates all nouns/verbs in paginated batches. For each entity, calls the `transform` function. If it returns a new object, saves it. If it returns `null`, skips. Vectors are never touched.
|
||||||
|
|
||||||
3. **Transform other branches** — switches to each user branch, runs the same transforms. Inherited (already-migrated) entities return `null` and are skipped automatically.
|
3. **Save state** — records each completed migration ID so it never re-runs.
|
||||||
|
|
||||||
4. **Save state** — records each completed migration ID so it never re-runs.
|
4. **Rebuild indexes** — if any entities were modified, rebuilds the MetadataIndex.
|
||||||
|
|
||||||
5. **Rebuild indexes** — if any entities were modified, rebuilds the MetadataIndex.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -78,7 +76,7 @@ interface Migration {
|
||||||
|
|
||||||
- **Return a new object** to modify the entity's metadata.
|
- **Return a new object** to modify the entity's metadata.
|
||||||
- **Return `null`** to skip (no change needed).
|
- **Return `null`** to skip (no change needed).
|
||||||
- **Must be idempotent** — running the same transform twice on the same data should produce the same result (or return `null` the second time). This is required because branch iterations may re-encounter inherited entities.
|
- **Must be idempotent** — running the same transform twice on the same data should produce the same result (or return `null` the second time). This is required because interrupted runs resume and re-encounter already-migrated entities.
|
||||||
- **Must be pure** — no side effects, no async, no external state.
|
- **Must be pure** — no side effects, no async, no external state.
|
||||||
- Transforms only see metadata. Vectors, embeddings, and the `data` field stored inside metadata are available as properties on the metadata object.
|
- Transforms only see metadata. Vectors, embeddings, and the `data` field stored inside metadata are available as properties on the metadata object.
|
||||||
|
|
||||||
|
|
@ -110,9 +108,9 @@ const preview = await brain.migrate({ dryRun: true })
|
||||||
// preview.sampleChanges — up to 5 before/after samples
|
// preview.sampleChanges — up to 5 before/after samples
|
||||||
// preview.estimatedTime — rough time estimate string
|
// preview.estimatedTime — rough time estimate string
|
||||||
|
|
||||||
// Apply migrations
|
// Apply migrations (optionally with a pre-migration snapshot)
|
||||||
const result = await brain.migrate()
|
const result = await brain.migrate({ backupTo: '/backups/pre-migration' })
|
||||||
// result.backupBranch — name of COW backup branch, or null
|
// result.backupPath — snapshot path, or null when no backupTo was supplied
|
||||||
// result.migrationsApplied — array of migration IDs that ran
|
// result.migrationsApplied — array of migration IDs that ran
|
||||||
// result.entitiesProcessed — total entities scanned
|
// result.entitiesProcessed — total entities scanned
|
||||||
// result.entitiesModified — entities actually changed
|
// result.entitiesModified — entities actually changed
|
||||||
|
|
@ -164,20 +162,20 @@ const result = await brain.migrate({ maxErrors: 10000 })
|
||||||
|
|
||||||
## Backup and Rollback
|
## Backup and Rollback
|
||||||
|
|
||||||
Before modifying any data, `brain.migrate()` calls `brain.fork()` to create a COW snapshot. This is instant regardless of dataset size — it's a pointer copy, not a data copy.
|
Pass `backupTo` and `brain.migrate()` persists a snapshot of the current generation **before any transform runs**. On filesystem storage the snapshot is a hard-link farm — created without copying entity data, and immune to later writes (see [Snapshots & Time Travel](./snapshots-and-time-travel.md)):
|
||||||
|
|
||||||
The backup branch is named `pre-migration-{version}` and tagged with metadata:
|
|
||||||
- `type: 'system:backup'`
|
|
||||||
- `migrationVersion: '7.17.0'`
|
|
||||||
- `author: 'brainy-migration'`
|
|
||||||
|
|
||||||
To roll back, switch to the backup branch:
|
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
await brain.checkout('pre-migration-7.17.0')
|
const result = await brain.migrate({ backupTo: '/backups/pre-migration-8.0' })
|
||||||
|
console.log(result.backupPath) // '/backups/pre-migration-8.0' (null when no backupTo)
|
||||||
```
|
```
|
||||||
|
|
||||||
Old backup branches from previous migrations are cleaned up automatically before each new migration run.
|
To roll back, restore the snapshot wholesale:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
await brain.restore('/backups/pre-migration-8.0', { confirm: true })
|
||||||
|
```
|
||||||
|
|
||||||
|
Without `backupTo`, no backup is taken — transforms are idempotent (they return `null` when already applied), but a pre-migration snapshot is the cheap insurance for anything destructive.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -273,7 +271,7 @@ Progress is reported after each batch (batch size is determined by the storage a
|
||||||
|
|
||||||
## Storage Backend Compatibility
|
## Storage Backend Compatibility
|
||||||
|
|
||||||
Migrations work identically across all storage backends (Memory, FileSystem, S3, R2, GCS, OPFS). The system uses `BaseStorage` methods (`getNouns`, `saveNounMetadata`, `getVerbs`, `saveVerbMetadata`) which are implemented by every adapter.
|
Migrations work identically across all storage backends (Memory, FileSystem). The system uses `BaseStorage` methods (`getNouns`, `saveNounMetadata`, `getVerbs`, `saveVerbMetadata`) which are implemented by every adapter.
|
||||||
|
|
||||||
Batch size and rate limiting are automatically configured per adapter — no tuning required.
|
Batch size and rate limiting are automatically configured per adapter — no tuning required.
|
||||||
|
|
||||||
|
|
|
||||||
382
docs/guides/snapshots-and-time-travel.md
Normal file
382
docs/guides/snapshots-and-time-travel.md
Normal file
|
|
@ -0,0 +1,382 @@
|
||||||
|
---
|
||||||
|
title: Snapshots & Time Travel
|
||||||
|
slug: guides/snapshots-and-time-travel
|
||||||
|
public: true
|
||||||
|
category: guides
|
||||||
|
template: guide
|
||||||
|
order: 9
|
||||||
|
description: Recipes for the Db API — instant backups with persist(), restore, time-travel debugging with asOf(), range queries over history (diff, history, since, log windows), persist-before-migrate, what-if analysis with with(), and audit trails via transaction metadata.
|
||||||
|
next:
|
||||||
|
- concepts/consistency-model
|
||||||
|
- guides/optimistic-concurrency
|
||||||
|
---
|
||||||
|
|
||||||
|
# Snapshots & Time Travel
|
||||||
|
|
||||||
|
Brainy 8.0 treats the database as a **value**: `brain.now()` pins the
|
||||||
|
current state as an immutable `Db`, `brain.transact()` commits an atomic
|
||||||
|
batch and hands you the resulting value, `brain.asOf()` opens past state,
|
||||||
|
and `db.persist()` cuts a self-contained snapshot. This guide is the recipe
|
||||||
|
book. The precise guarantees behind every recipe live in the
|
||||||
|
[consistency model](../concepts/consistency-model.md).
|
||||||
|
|
||||||
|
## Instant backup
|
||||||
|
|
||||||
|
Pin the current state, persist it, release:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const db = brain.now()
|
||||||
|
try {
|
||||||
|
await db.persist('/backups/2026-06-11')
|
||||||
|
} finally {
|
||||||
|
await db.release()
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
On filesystem storage the snapshot is built from **hard links**: every data
|
||||||
|
file in Brainy is immutable-by-rename, so the snapshot is created without
|
||||||
|
copying entity data and shares disk space with the live store. Later writes
|
||||||
|
can never alter it — a rewrite swaps the inode, the snapshot keeps the old
|
||||||
|
bytes. Cross-device targets fall back to per-file byte copies, and
|
||||||
|
persisting an in-memory brain serializes it to the same directory layout —
|
||||||
|
a real, durable store.
|
||||||
|
|
||||||
|
Two things to know:
|
||||||
|
|
||||||
|
- `persist()` requires the view to still be the store's **latest**
|
||||||
|
generation. If something committed after your pin, it throws
|
||||||
|
`GenerationConflictError` instead of snapshotting the wrong state — pin
|
||||||
|
and persist before further writes, or retry with a fresh `brain.now()`.
|
||||||
|
- The target directory must be empty or absent.
|
||||||
|
|
||||||
|
For scheduled backups, this loop is the whole job:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const db = brain.now()
|
||||||
|
try {
|
||||||
|
await db.persist(`/backups/${new Date().toISOString().slice(0, 10)}`)
|
||||||
|
} finally {
|
||||||
|
await db.release()
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Restore
|
||||||
|
|
||||||
|
`restore()` replaces the store's **entire** current state from a snapshot —
|
||||||
|
entities, relationships, indexes, history. It is deliberately loud about it:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
await brain.restore('/backups/2026-06-11', { confirm: true })
|
||||||
|
```
|
||||||
|
|
||||||
|
- `{ confirm: true }` is mandatory — current state is destroyed.
|
||||||
|
- The snapshot is copied in (never linked), so it stays independent and can
|
||||||
|
be restored again later.
|
||||||
|
- All indexes are rebuilt from the restored records.
|
||||||
|
- The generation counter is floored at its pre-restore value, so generation
|
||||||
|
numbers you observed before the restore are never reissued.
|
||||||
|
- Live `Db` pins do not survive a restore — release them first.
|
||||||
|
|
||||||
|
## Open a snapshot read-only
|
||||||
|
|
||||||
|
You do not have to restore to look inside a snapshot. `Brainy.load()` opens
|
||||||
|
it as a self-contained read-only store with the **full query surface**,
|
||||||
|
including vector search:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const db = await Brainy.load('/backups/2026-06-11')
|
||||||
|
|
||||||
|
const hits = await db.find({ query: 'unpaid invoices from the spring campaign' })
|
||||||
|
const orders = await db.find({ type: NounType.Document, subtype: 'order' })
|
||||||
|
|
||||||
|
await db.release() // closes the underlying read-only instance
|
||||||
|
```
|
||||||
|
|
||||||
|
`brain.asOf('/backups/2026-06-11')` does the same from an existing brain.
|
||||||
|
This is also the 8.0 answer to "named branches": a branch is a name → path
|
||||||
|
mapping your application keeps, where each path is a persisted snapshot.
|
||||||
|
Need a writable copy? Restore the snapshot into a fresh data directory and
|
||||||
|
open a writer on it — instead of switching a shared store between branches
|
||||||
|
in place, every line of code always sees exactly the store it opened.
|
||||||
|
|
||||||
|
## Time-travel debugging
|
||||||
|
|
||||||
|
When production data looks wrong, query the past directly — by wall-clock
|
||||||
|
time or by generation:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// What did this order look like yesterday?
|
||||||
|
const yesterday = await brain.asOf(new Date(Date.now() - 86_400_000))
|
||||||
|
const before = await yesterday.get(orderId)
|
||||||
|
|
||||||
|
// Full queries work at any reachable generation — search, graph, filters:
|
||||||
|
const thenActive = await yesterday.find({
|
||||||
|
type: NounType.Document,
|
||||||
|
subtype: 'order',
|
||||||
|
where: { status: 'active' }
|
||||||
|
})
|
||||||
|
|
||||||
|
await yesterday.release()
|
||||||
|
```
|
||||||
|
|
||||||
|
Pin two points in time and diff them:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const before = await brain.asOf(1041)
|
||||||
|
const after = brain.now()
|
||||||
|
|
||||||
|
const changed = await after.since(before)
|
||||||
|
changed.nouns // entity ids touched by transactions in between
|
||||||
|
changed.verbs // relationship ids touched in between
|
||||||
|
|
||||||
|
await before.release()
|
||||||
|
await after.release()
|
||||||
|
```
|
||||||
|
|
||||||
|
Three things to remember:
|
||||||
|
|
||||||
|
- History granularity is per-write: EVERY write — `transact()` AND a
|
||||||
|
single-operation `add`/`update`/`remove`/`relate` — is its own immutable
|
||||||
|
generation, so a pin always freezes against later writes and every write is
|
||||||
|
individually addressable via `asOf()` (see the
|
||||||
|
[consistency model](../concepts/consistency-model.md)). Use `transact()` when
|
||||||
|
you want several operations to share ONE atomic generation.
|
||||||
|
- The first index-accelerated query (semantic search, traversal, cursors,
|
||||||
|
aggregation) at a historical generation builds an in-memory index
|
||||||
|
materialization — O(n at that generation), once per `Db`, freed on
|
||||||
|
`release()`. Metadata-level reads are free.
|
||||||
|
- Generations reclaimed by `compactHistory()` throw
|
||||||
|
`GenerationCompactedError` — persist anything you need to keep forever.
|
||||||
|
|
||||||
|
## Range queries over history
|
||||||
|
|
||||||
|
`asOf()` answers "what was the state AT a point". Four range verbs answer
|
||||||
|
"what happened BETWEEN two points" and "what is one entity's whole history".
|
||||||
|
They all build on the same generation records — no extra bookkeeping.
|
||||||
|
|
||||||
|
### `diff(a, b)` — what changed, classified
|
||||||
|
|
||||||
|
`since()` gives you the raw set of *touched* ids. `diff()` goes further: it
|
||||||
|
resolves each touched id at both endpoints and classifies it as **added**,
|
||||||
|
**removed**, or **modified** — split by entities (`nouns`) and relationships
|
||||||
|
(`verbs`). An id that was touched but ended up identical (changed then
|
||||||
|
reverted, or created and deleted within the interval) lands in **none** of the
|
||||||
|
buckets. Endpoints are a generation, a `Date`, or a `Db`, in either order:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const d = await brain.diff(1041, brain.generation())
|
||||||
|
|
||||||
|
d.added.nouns // entity ids created between the two states
|
||||||
|
d.removed.nouns // entity ids deleted
|
||||||
|
d.modified.nouns // entity ids whose stored value actually changed
|
||||||
|
d.added.verbs // …relationships, the same three ways
|
||||||
|
```
|
||||||
|
|
||||||
|
Orientation is `a → b`: `added` means "exists at `b`, not at `a`". The
|
||||||
|
comparison behind `modified` is key-order-insensitive, so a no-op re-write of
|
||||||
|
the same fields never shows up as a change.
|
||||||
|
|
||||||
|
### `history(id, range?)` — one entity, every version
|
||||||
|
|
||||||
|
`asOf()` is per-*generation*; `history()` is per-*entity*. It returns every
|
||||||
|
distinct version of one id over a range, oldest first — each `value` is the
|
||||||
|
materialized state at that version (and `null` marks a removal):
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const h = await brain.history(invoiceId)
|
||||||
|
|
||||||
|
for (const v of h.versions) {
|
||||||
|
console.log(v.generation, v.value?.metadata?.status ?? '(deleted)')
|
||||||
|
}
|
||||||
|
// 1041 'draft'
|
||||||
|
// 1043 'approved'
|
||||||
|
// 1050 'paid'
|
||||||
|
```
|
||||||
|
|
||||||
|
Every version ties to the trusted `asOf()` path — `v.value` equals
|
||||||
|
`(await brain.asOf(v.generation)).get(id)`. Pass `{ from, to }` (generation or
|
||||||
|
`Date`) to bound the range; a `from` below the compaction horizon is quietly
|
||||||
|
truncated to it rather than throwing (history is best-effort over surviving
|
||||||
|
records).
|
||||||
|
|
||||||
|
### `since()` and `transactionLog()` take ranges too
|
||||||
|
|
||||||
|
`since()` accepts a `Db`, a generation number, or a `Date` — all equivalent,
|
||||||
|
all an **exclusive** lower bound (`db.since(prior)` equals
|
||||||
|
`db.since(prior.generation)`):
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
await brain.now().since(1041) // ids changed after generation 1041
|
||||||
|
await brain.now().since(new Date(Date.now() - 3_600_000)) // …in the last hour
|
||||||
|
```
|
||||||
|
|
||||||
|
`transactionLog({ from, to })` windows the commit log **inclusively** on both
|
||||||
|
ends (a log window names the commits it spans — the deliberate contrast to
|
||||||
|
`since`'s exclusive lower bound); `limit` applies after the window, newest
|
||||||
|
first:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const window = await brain.transactionLog({ from: 1041, to: 1050 }) // commits 1041…1050
|
||||||
|
const recent = await brain.transactionLog({ from: lastHour, limit: 20 })
|
||||||
|
```
|
||||||
|
|
||||||
|
### Composing them
|
||||||
|
|
||||||
|
"Which orders changed in this window?" is `diff` ids intersected with an
|
||||||
|
`asOf` query — the two agree by construction:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const changed = await brain.diff(g1, g2)
|
||||||
|
const atG2 = await brain.asOf(g2)
|
||||||
|
const changedOrders = (await atG2.find({ type: NounType.Document, subtype: 'order' }))
|
||||||
|
.map(r => r.id)
|
||||||
|
.filter(id => changed.added.nouns.includes(id) || changed.modified.nouns.includes(id))
|
||||||
|
await atG2.release()
|
||||||
|
```
|
||||||
|
|
||||||
|
One contrast to keep straight: `diff` and `since` **throw**
|
||||||
|
`GenerationCompactedError` for a bound below the horizon, while `history`
|
||||||
|
**truncates** to the horizon — diffs must be exact, history is best-effort.
|
||||||
|
|
||||||
|
## Safe schema migration
|
||||||
|
|
||||||
|
`brain.migrate()` integrates with snapshots directly: pass `backupTo` and a
|
||||||
|
hard-link snapshot of the current generation is persisted **before any
|
||||||
|
transform runs**:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const result = await brain.migrate({ backupTo: '/backups/pre-migration-8.0' })
|
||||||
|
console.log(result.migrationsApplied, result.backupPath)
|
||||||
|
|
||||||
|
// If the migration went wrong, roll the whole store back:
|
||||||
|
await brain.restore('/backups/pre-migration-8.0', { confirm: true })
|
||||||
|
```
|
||||||
|
|
||||||
|
The same persist-before-mutate pattern works for any risky bulk operation,
|
||||||
|
not just migrations:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const pin = brain.now()
|
||||||
|
try {
|
||||||
|
await pin.persist('/backups/pre-bulk-edit')
|
||||||
|
} finally {
|
||||||
|
await pin.release()
|
||||||
|
}
|
||||||
|
await runRiskyBulkEdit(brain)
|
||||||
|
```
|
||||||
|
|
||||||
|
## What-if analysis
|
||||||
|
|
||||||
|
`db.with(ops)` applies a transaction **speculatively, in memory** — nothing
|
||||||
|
touches disk, the generation counter, or the indexes. Ask "what would the
|
||||||
|
store look like if…", then commit the same operations for real:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const ops = [
|
||||||
|
{ op: 'update', id: employeeId, metadata: { team: 'platform' } },
|
||||||
|
{ op: 'relate', from: employeeId, to: milestoneId, type: VerbType.ParticipatesIn, subtype: 'assignment' }
|
||||||
|
]
|
||||||
|
|
||||||
|
const base = brain.now()
|
||||||
|
const whatIf = await base.with(ops)
|
||||||
|
|
||||||
|
await whatIf.get(employeeId) // sees the change
|
||||||
|
await whatIf.find({ where: { team: 'platform' } }) // metadata finds work
|
||||||
|
await whatIf.related(employeeId) // overlay relations included
|
||||||
|
|
||||||
|
await whatIf.release()
|
||||||
|
await base.release()
|
||||||
|
|
||||||
|
// Looks right — make it real, atomically:
|
||||||
|
await brain.transact(ops)
|
||||||
|
```
|
||||||
|
|
||||||
|
**The boundary:** speculative entities carry no embeddings (`with()` never
|
||||||
|
invokes the embedder), so semantic search, traversal, cursors, aggregation,
|
||||||
|
and `persist()` throw `SpeculativeOverlayError` on overlay views instead of
|
||||||
|
returning silently incomplete results. `get()`, metadata-filter `find()`,
|
||||||
|
and filter-based `related()` are fully supported. Overlays chain — calling
|
||||||
|
`with()` on an overlay stacks another layer.
|
||||||
|
|
||||||
|
## Audit trails
|
||||||
|
|
||||||
|
`transact()` reifies transaction metadata: whatever you pass as `meta` is
|
||||||
|
recorded durably alongside the committed generation and timestamp, readable
|
||||||
|
via `brain.transactionLog()`:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
await brain.transact(
|
||||||
|
[{ op: 'update', id: invoiceId, metadata: { status: 'approved' } }],
|
||||||
|
{ meta: { author: 'approvals-service', actor: 'jane@example.com', reason: 'PO-7741' } }
|
||||||
|
)
|
||||||
|
|
||||||
|
const log = await brain.transactionLog({ limit: 20 }) // newest first
|
||||||
|
// [{ generation: 1042, timestamp: 1765432100000, meta: { author: 'approvals-service', ... } }]
|
||||||
|
```
|
||||||
|
|
||||||
|
Combine the log with `asOf()` to reconstruct exactly what any transaction
|
||||||
|
did:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const [entry] = await brain.transactionLog({ limit: 1 })
|
||||||
|
|
||||||
|
const after = await brain.asOf(entry.generation)
|
||||||
|
const before = await brain.asOf(entry.generation - 1)
|
||||||
|
|
||||||
|
const touched = await after.since(before)
|
||||||
|
for (const id of touched.nouns) {
|
||||||
|
console.log(id, await before.get(id), '→', await after.get(id))
|
||||||
|
}
|
||||||
|
|
||||||
|
await before.release()
|
||||||
|
await after.release()
|
||||||
|
```
|
||||||
|
|
||||||
|
For per-entity write coordination (rather than whole-store history), the
|
||||||
|
`_rev` counter and `ifRev` CAS remain the right tool — see
|
||||||
|
[optimistic concurrency](./optimistic-concurrency.md).
|
||||||
|
|
||||||
|
## Keeping history bounded
|
||||||
|
|
||||||
|
Under Model-B every write is a generation, so history can grow quickly —
|
||||||
|
Brainy auto-compacts on every `flush()`/`close()` under the **`retention`**
|
||||||
|
knob (configured on the constructor):
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// Zero-config: ADAPTIVE — keep as much history as free disk/RAM allows,
|
||||||
|
// reclaiming oldest-first under pressure. (This is the default.)
|
||||||
|
new Brainy({ /* retention unset */ })
|
||||||
|
|
||||||
|
// Unbounded — never reclaim history (opt in explicitly):
|
||||||
|
new Brainy({ retention: 'all' })
|
||||||
|
|
||||||
|
// Explicit CAPS — reclaim oldest-unpinned generations while ANY cap is exceeded:
|
||||||
|
new Brainy({ retention: { maxGenerations: 1000, maxAge: 7 * 86_400_000, maxBytes: 512 * 1024 ** 2 } })
|
||||||
|
```
|
||||||
|
|
||||||
|
Reclaim manually at any time (the same caps):
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
await brain.compactHistory({ maxGenerations: 100, maxAge: 7 * 24 * 60 * 60 * 1000 })
|
||||||
|
```
|
||||||
|
|
||||||
|
Compaction never breaks a pinned read — record-sets are reclaimed only when
|
||||||
|
no live `Db` could need them (live pins are ALWAYS exempt). Release views you
|
||||||
|
are done with (including the ones `transact()` returns), and `persist()` any
|
||||||
|
generation you want to keep beyond the retention window: snapshots are
|
||||||
|
self-contained and unaffected by compaction.
|
||||||
|
|
||||||
|
## From branches to values
|
||||||
|
|
||||||
|
If you used the pre-8.0 `fork`/`checkout`/`commit`/`versions` surface, every
|
||||||
|
use case maps to a sharper tool:
|
||||||
|
|
||||||
|
| Pre-8.0 habit | 8.0 recipe |
|
||||||
|
|---|---|
|
||||||
|
| `fork()` to experiment safely | `db.with(ops)` for speculation in memory; a restored snapshot in a fresh directory for a long-lived writable copy |
|
||||||
|
| `commit()` checkpoints | `transact(ops, { meta })` — every batch is an atomic, logged, time-travelable commit |
|
||||||
|
| `checkout()` to switch branches | Open the snapshot you want — `Brainy.load(path)` read-only, or restore into its own directory. No in-place switching: every handle always sees one unambiguous store. |
|
||||||
|
| `getHistory()` | `brain.transactionLog()` + `db.since(priorDb)` |
|
||||||
|
| `versions.save()` per-entity snapshots | A pinned `Db` or persisted snapshot captures *every* entity at that moment; `asOf()` reads any entity's past state |
|
||||||
|
| `versions.restore()` | `brain.restore(snapshot, { confirm: true })` for the whole store, or read the old entity via `asOf()` and write it back with `transact()` |
|
||||||
|
| Backup branches | `db.persist(path)` — instant, hard-link-shared, self-contained |
|
||||||
|
|
@ -5,197 +5,158 @@ public: true
|
||||||
category: guides
|
category: guides
|
||||||
template: guide
|
template: guide
|
||||||
order: 2
|
order: 2
|
||||||
description: "Six adapters for every deployment: in-memory, OPFS (browser), filesystem, S3, Google Cloud Storage, Azure Blob, and Cloudflare R2. All support copy-on-write branching."
|
description: "Two adapters cover every deployment: in-memory for tests + ephemeral workloads, filesystem for everything that needs to persist. Both share one on-disk contract, including generational history and snapshots. Cloud backup is operator tooling, not a built-in adapter."
|
||||||
next:
|
next:
|
||||||
- guides/plugins
|
- guides/plugins
|
||||||
- concepts/zero-config
|
- concepts/consistency-model
|
||||||
---
|
---
|
||||||
|
|
||||||
# Storage Adapters Guide
|
# Storage Adapters
|
||||||
|
|
||||||
## Overview
|
Brainy 8.0 ships **two storage adapters**:
|
||||||
Brainy supports 6 storage adapters for different deployment environments. All adapters implement the same StorageAdapter interface and support copy-on-write branching.
|
|
||||||
|
|
||||||
## Adapters
|
- **`FileSystemStorage`** — persistent on-disk storage. The default for any
|
||||||
|
deployment that needs to survive a restart. Runs on Node.js, Bun, and Deno.
|
||||||
|
- **`MemoryStorage`** — in-memory only. The right choice for tests, ephemeral
|
||||||
|
workloads, and short-lived demos.
|
||||||
|
|
||||||
### 1. MemoryStorage
|
Both implement the same `StorageAdapter` interface, support the full Db API
|
||||||
- **File**: `src/storage/adapters/memoryStorage.ts`
|
(generational history, snapshots, restore — see the
|
||||||
- **Use case**: Development, testing, prototyping
|
[consistency model](../concepts/consistency-model.md)), and use the same
|
||||||
- **Configuration**: None required
|
on-disk layout (memory's "disk" is a JS Map).
|
||||||
- **Persistence**: None (data lost on restart)
|
|
||||||
- **Batch config**: 1000 batch size, 0ms delay, 1000 concurrent ops, 100k ops/sec
|
|
||||||
|
|
||||||
```typescript
|
## Quick start
|
||||||
const brain = new Brainy({ storage: { type: 'memory' } })
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. FileSystemStorage
|
```ts
|
||||||
- **File**: `src/storage/adapters/fileSystemStorage.ts`
|
import { Brainy } from '@soulcraft/brainy'
|
||||||
- **Use case**: Node.js local persistence, single-server deployments
|
|
||||||
- **Configuration**: `basePath` (required), `readOnly` (optional)
|
|
||||||
- **Features**: zlib compression, atomic writes via temp files, UUID-based sharding
|
|
||||||
|
|
||||||
```typescript
|
// Filesystem (recommended for any persistent workload):
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: { type: 'filesystem', path: './brainy-data' }
|
storage: { type: 'filesystem', path: './brainy-data' }
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Memory (tests, ephemeral):
|
||||||
|
const brainMem = new Brainy({ storage: { type: 'memory' } })
|
||||||
|
|
||||||
|
// Auto-detect (filesystem on Node-like runtimes, memory in browsers):
|
||||||
|
const brainAuto = new Brainy({ storage: { type: 'auto' } })
|
||||||
```
|
```
|
||||||
|
|
||||||
### 3. S3CompatibleStorage
|
## When to use which
|
||||||
- **File**: `src/storage/adapters/s3CompatibleStorage.ts`
|
|
||||||
- **Use case**: AWS S3, MinIO, DigitalOcean Spaces, any S3-compatible service
|
|
||||||
- **Configuration**: `bucketName`, `region`, `accessKeyId`, `secretAccessKey`, `endpoint?` (for custom endpoints), `s3ForcePathStyle?`
|
|
||||||
- **Features**: Write buffering, request coalescing, throttling detection (429/503), progressive initialization
|
|
||||||
- **Batch config**: 1000 batch size, 150 concurrent ops, 5000 ops/sec burst
|
|
||||||
|
|
||||||
```typescript
|
| Use case | Adapter | Why |
|
||||||
const brain = new Brainy({
|
|---|---|---|
|
||||||
storage: {
|
| Production app | `filesystem` | Durable, snapshot-able, mmap-able |
|
||||||
type: 's3',
|
| Tests, CI | `memory` | No disk teardown; fast |
|
||||||
s3Storage: {
|
| Short-lived data pipeline | `memory` | No persistence needed |
|
||||||
bucketName: 'my-brainy-data',
|
| In-browser demo | `memory` | Filesystem unavailable in browsers |
|
||||||
region: 'us-east-1',
|
| Cloud deployment | `filesystem` on local disk + operator backup | See "Cloud backup" below |
|
||||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
|
||||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// For MinIO or other S3-compatible services:
|
## Cloud backup — operator tooling, not a built-in
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
Brainy 8.0 deliberately ships **no cloud storage adapters**. Cloud backup is
|
||||||
type: 's3',
|
handled at the operator layer with standard tooling, the same pattern every
|
||||||
s3Storage: {
|
production database uses (Postgres, SQLite, Redis):
|
||||||
bucketName: 'my-data',
|
|
||||||
region: 'us-east-1',
|
```bash
|
||||||
endpoint: 'http://localhost:9000',
|
# After a brainy flush, sync the on-disk artefact to your cloud of choice.
|
||||||
s3ForcePathStyle: true,
|
gsutil rsync -r /var/lib/brainy gs://my-backup-bucket/brainy/
|
||||||
accessKeyId: 'minio-key',
|
# or:
|
||||||
secretAccessKey: 'minio-secret'
|
aws s3 sync /var/lib/brainy s3://my-backup-bucket/brainy/
|
||||||
}
|
# or:
|
||||||
}
|
rclone sync /var/lib/brainy remote:brainy-backups/
|
||||||
})
|
# or:
|
||||||
|
azcopy sync /var/lib/brainy "https://account.blob.core.windows.net/brainy?sv=..."
|
||||||
```
|
```
|
||||||
|
|
||||||
### 4. R2Storage (Cloudflare)
|
Brainy's filesystem layout is sync-friendly:
|
||||||
- **File**: `src/storage/adapters/r2Storage.ts`
|
- Atomic writes (temp + rename) — readers never see torn files
|
||||||
- **Use case**: Zero egress fees, cost-effective cloud storage
|
- Per-shard files — `rsync`-style incremental sync works well
|
||||||
- **Configuration**: `bucketName`, `accountId`, `accessKeyId`, `secretAccessKey`, `cacheConfig?`
|
- Immutable generation records (`_generations/`) — append-only, cache-friendly
|
||||||
- **Features**: Zero egress fees, aggressive caching, write buffering, request coalescing
|
|
||||||
- **Batch config**: 1000 batch size, 150 concurrent ops, 6000 ops/sec
|
|
||||||
|
|
||||||
```typescript
|
For point-in-time backups, take a filesystem snapshot (ZFS, btrfs, LVM, EBS,
|
||||||
const brain = new Brainy({
|
etc.) or use `brain.now().persist(path)` to write a self-contained snapshot
|
||||||
storage: {
|
you can sync independently of the live brain — see
|
||||||
type: 'r2',
|
[Snapshots & Time Travel](./snapshots-and-time-travel.md).
|
||||||
r2Storage: {
|
|
||||||
accountId: process.env.CF_ACCOUNT_ID,
|
## Why no cloud adapters in 8.0?
|
||||||
bucketName: 'my-brainy-data',
|
|
||||||
accessKeyId: process.env.CF_ACCESS_KEY_ID,
|
Cloud storage adapters lived in Brainy 4.x-7.x. They were dropped in 8.0
|
||||||
secretAccessKey: process.env.CF_SECRET_ACCESS_KEY
|
because:
|
||||||
|
|
||||||
|
- Zero production consumers used them at scale — every known production
|
||||||
|
deployment ran on local filesystem.
|
||||||
|
- Cloud-storage HNSW / DiskANN doesn't perform — vector indexes need
|
||||||
|
low-latency random reads that S3 / GCS / R2 / Azure can't provide
|
||||||
|
consistently.
|
||||||
|
- Bundling cloud SDKs into the library cost ~3000-5000 LOC + 4-7 transitive
|
||||||
|
dependencies for a feature nobody used.
|
||||||
|
- Cloud backup via operator tooling is strictly more reliable than in-app
|
||||||
|
upload (better retry semantics, better observability, better cost control).
|
||||||
|
|
||||||
|
Brainy 8.0 is smaller, faster to install, and clearer about what it does.
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
```ts
|
||||||
|
// BrainyConfig['storage'] — either a config object or a pre-constructed adapter:
|
||||||
|
storage?:
|
||||||
|
| {
|
||||||
|
// The adapter type. Optional — a top-level `path` implies 'filesystem',
|
||||||
|
// so { path: '/data' } works without it. Defaults to 'auto'
|
||||||
|
// (filesystem on Node-like runtimes, memory otherwise).
|
||||||
|
type?: 'auto' | 'memory' | 'filesystem'
|
||||||
|
|
||||||
|
// CANONICAL directory for filesystem storage. The rest of the API already
|
||||||
|
// speaks `path` (persist(path), Brainy.load(path), asOf(path),
|
||||||
|
// restore(path)). Specifying it implies type: 'filesystem'. Passed through
|
||||||
|
// to storage factories, including plugin-provided ones.
|
||||||
|
path?: string
|
||||||
|
|
||||||
|
// REMOVED in 8.0 — passing `rootDirectory` THROWS. Use `path`.
|
||||||
|
rootDirectory?: string
|
||||||
|
|
||||||
|
// REMOVED in 8.0 — a nested `options.{path,rootDirectory}` THROWS. Use `path`.
|
||||||
|
options?: any
|
||||||
}
|
}
|
||||||
}
|
| StorageAdapter // e.g. storage: new MemoryStorage()
|
||||||
})
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### 5. GcsStorage (Google Cloud)
|
The canonical — and only — key is the top-level **`path`**. The pre-8.0 aliases
|
||||||
- **File**: `src/storage/adapters/gcsStorage.ts`
|
`rootDirectory`, `options.{path,rootDirectory}`, and
|
||||||
- **Use case**: Google Cloud ecosystem, Cloud Run deployments
|
`fileSystemStorage.{path,rootDirectory}` were **removed in 8.0**: passing one now
|
||||||
- **Authentication priority**:
|
throws with the exact rename (never a silent default that would misplace data on
|
||||||
1. Service Account Key File (`keyFilename`)
|
upgrade). Because `path` implies filesystem, `{ path: '/data' }` is a complete
|
||||||
2. Credentials Object (`credentials`)
|
config; the `type` is optional.
|
||||||
3. HMAC Keys (backward compat)
|
|
||||||
4. Application Default Credentials (automatic in Cloud Run)
|
|
||||||
- **Features**: Progressive initialization (fast cold starts <200ms), Autoclass lifecycle, bucket validation
|
|
||||||
- **Batch config**: 1000 batch size, 100 concurrent ops, 1000 ops/sec
|
|
||||||
|
|
||||||
```typescript
|
## Direct construction
|
||||||
// With explicit credentials
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 'gcs',
|
|
||||||
gcsStorage: {
|
|
||||||
bucketName: 'my-brainy-data',
|
|
||||||
keyFilename: './service-account.json'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// In Cloud Run (uses Application Default Credentials automatically)
|
If you want to skip the factory:
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
```ts
|
||||||
type: 'gcs',
|
import { FileSystemStorage, MemoryStorage } from '@soulcraft/brainy'
|
||||||
gcsStorage: {
|
|
||||||
bucketName: 'my-brainy-data'
|
const fsStorage = new FileSystemStorage('./brainy-data')
|
||||||
}
|
const memStorage = new MemoryStorage()
|
||||||
}
|
|
||||||
})
|
const brain = new Brainy({ storage: fsStorage })
|
||||||
```
|
```
|
||||||
|
|
||||||
**Progressive Initialization Modes:**
|
## Migration from 7.x cloud adapters
|
||||||
- `'strict'` (default locally): Full validation during init (100-500ms+)
|
|
||||||
- `'progressive'` (default in Cloud Run/Lambda): Fast init <200ms, background validation
|
|
||||||
- `'auto'`: Auto-detects environment
|
|
||||||
|
|
||||||
### 6. AzureBlobStorage
|
7.x consumers of `OPFSStorage`, `GcsStorage`, `R2Storage`, `S3CompatibleStorage`,
|
||||||
- **File**: `src/storage/adapters/azureBlobStorage.ts`
|
or `AzureBlobStorage` need to migrate to `FileSystemStorage` plus operator
|
||||||
- **Use case**: Azure ecosystem, enterprise deployments
|
backup tooling. The recipe:
|
||||||
- **Authentication priority**:
|
|
||||||
1. DefaultAzureCredential (Managed Identity - automatic in Azure)
|
|
||||||
2. Connection String
|
|
||||||
3. Account Name + Account Key
|
|
||||||
4. SAS Token
|
|
||||||
- **Features**: Progressive initialization, lifecycle management, container validation
|
|
||||||
- **Batch config**: 1000 batch size, 100 concurrent ops, 3000 ops/sec
|
|
||||||
|
|
||||||
```typescript
|
1. On the host running Brainy, mount a local disk (NVMe recommended). Cloud
|
||||||
// With connection string
|
providers all expose persistent local disks: GCP Persistent Disk, AWS EBS,
|
||||||
const brain = new Brainy({
|
Azure Managed Disks.
|
||||||
storage: {
|
2. Set `storage: { type: 'filesystem', path: '/mnt/brainy-data' }`.
|
||||||
type: 'azure',
|
3. Run your existing data import once into the new local store.
|
||||||
azureStorage: {
|
4. Set up an operator backup job using `gsutil` / `aws s3` / `rclone` /
|
||||||
connectionString: process.env.AZURE_STORAGE_CONNECTION_STRING,
|
`azcopy` on a cron — hourly or whatever your RPO requires. Point it at
|
||||||
containerName: 'brainy-data'
|
the brainy data dir.
|
||||||
}
|
5. For point-in-time backups, use filesystem snapshots or
|
||||||
}
|
`brain.now().persist(path)`.
|
||||||
})
|
|
||||||
|
|
||||||
// With Managed Identity (automatic in Azure App Service/Functions)
|
Same data, same APIs, no library-side cloud code.
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 'azure',
|
|
||||||
azureStorage: {
|
|
||||||
containerName: 'brainy-data'
|
|
||||||
// DefaultAzureCredential used automatically
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## Choosing an Adapter
|
|
||||||
|
|
||||||
| Scenario | Recommended Adapter |
|
|
||||||
|----------|-------------------|
|
|
||||||
| Development/Testing | MemoryStorage |
|
|
||||||
| Local persistence | FileSystemStorage |
|
|
||||||
| AWS deployment | S3CompatibleStorage |
|
|
||||||
| Google Cloud / Cloud Run | GcsStorage |
|
|
||||||
| Azure deployment | AzureBlobStorage |
|
|
||||||
| Cost-sensitive (high egress) | R2Storage |
|
|
||||||
| Self-hosted (MinIO) | S3CompatibleStorage |
|
|
||||||
|
|
||||||
## Common Configuration
|
|
||||||
|
|
||||||
All cloud adapters support:
|
|
||||||
- **Progressive initialization** for fast serverless cold starts
|
|
||||||
- **Read-only mode** (`readOnly: true`)
|
|
||||||
- **Cache configuration** (`cacheConfig: { hotCacheMaxSize, warmCacheTTL }`)
|
|
||||||
- **Copy-on-write branching** for git-style data management
|
|
||||||
|
|
||||||
## See Also
|
|
||||||
|
|
||||||
- [Cloud Deployment Guide](../deployment/CLOUD_DEPLOYMENT_GUIDE.md)
|
|
||||||
- [Cost Optimization: AWS S3](../operations/cost-optimization-aws-s3.md)
|
|
||||||
- [Cost Optimization: GCS](../operations/cost-optimization-gcs.md)
|
|
||||||
- [Cost Optimization: Azure](../operations/cost-optimization-azure.md)
|
|
||||||
- [Cost Optimization: R2](../operations/cost-optimization-cloudflare-r2.md)
|
|
||||||
|
|
|
||||||
|
|
@ -242,7 +242,7 @@ A realistic adoption sequence for a brain that started without these primitives:
|
||||||
```typescript
|
```typescript
|
||||||
import { Brainy, NounType } from '@soulcraft/brainy'
|
import { Brainy, NounType } from '@soulcraft/brainy'
|
||||||
|
|
||||||
const brain = new Brainy({ storage: { type: 'filesystem', options: { path: './brain-data' } } })
|
const brain = new Brainy({ storage: { type: 'filesystem', path: './brain-data' } })
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
||||||
// 1. Migrate any existing metadata.kind convention to the new top-level subtype
|
// 1. Migrate any existing metadata.kind convention to the new top-level subtype
|
||||||
|
|
@ -301,14 +301,14 @@ await brain.relate({
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Direct reports — fast path filter (column-store hit, not metadata fallback)
|
// Direct reports — fast path filter (column-store hit, not metadata fallback)
|
||||||
const direct = await brain.getRelations({
|
const direct = await brain.related({
|
||||||
from: ceoId,
|
from: ceoId,
|
||||||
type: VerbType.ReportsTo,
|
type: VerbType.ReportsTo,
|
||||||
subtype: 'direct'
|
subtype: 'direct'
|
||||||
})
|
})
|
||||||
|
|
||||||
// Set membership
|
// Set membership
|
||||||
const allReports = await brain.getRelations({
|
const allReports = await brain.related({
|
||||||
from: ceoId,
|
from: ceoId,
|
||||||
type: VerbType.ReportsTo,
|
type: VerbType.ReportsTo,
|
||||||
subtype: ['direct', 'dotted-line']
|
subtype: ['direct', 'dotted-line']
|
||||||
|
|
@ -552,8 +552,8 @@ Brainy 8.0 ships:
|
||||||
|
|
||||||
- `brain.relate({ ..., subtype: 'value' })` — write the field
|
- `brain.relate({ ..., subtype: 'value' })` — write the field
|
||||||
- `brain.updateRelation({ id, subtype, type?, weight?, ... })` — change a relationship
|
- `brain.updateRelation({ id, subtype, type?, weight?, ... })` — change a relationship
|
||||||
- `brain.getRelations({ subtype })` — filter (fast path)
|
- `brain.related({ subtype })` — filter (fast path)
|
||||||
- `brain.getRelations({ subtype: ['a', 'b'] })` — set membership
|
- `brain.related({ subtype: ['a', 'b'] })` — set membership
|
||||||
- `brain.find({ connected: { via, subtype, depth } })` — traversal filter
|
- `brain.find({ connected: { via, subtype, depth } })` — traversal filter
|
||||||
- `brain.counts.byRelationshipSubtype(verb, subtype?)` — O(1) counts
|
- `brain.counts.byRelationshipSubtype(verb, subtype?)` — O(1) counts
|
||||||
- `brain.counts.topRelationshipSubtypes(verb, n?)` — top N by count
|
- `brain.counts.topRelationshipSubtypes(verb, n?)` — top N by count
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,8 @@
|
||||||
|
|
||||||
Complete guide to integrating Brainy with Vue.js applications, covering Vue 3, Nuxt.js, Composition API, Options API, and advanced patterns.
|
Complete guide to integrating Brainy with Vue.js applications, covering Vue 3, Nuxt.js, Composition API, Options API, and advanced patterns.
|
||||||
|
|
||||||
|
> **Runtime**: Brainy 8.0 runs on Node.js 22+ and Bun, server-side only — it is not a browser library. In Vue apps, Brainy lives behind an HTTP endpoint (a Nuxt server route, or any Node/Bun backend), and your components call that endpoint. The composables, plugin, and store below are wrappers around `fetch`; the single Brainy instance is created once on the server (see [Nuxt Server Route](#nuxt-server-route)).
|
||||||
|
|
||||||
## 🚀 Quick Start
|
## 🚀 Quick Start
|
||||||
|
|
||||||
### Installation
|
### Installation
|
||||||
|
|
@ -15,34 +17,47 @@ npm install @soulcraft/brainy
|
||||||
|
|
||||||
### Basic Setup
|
### Basic Setup
|
||||||
|
|
||||||
|
The composable talks to a Brainy-backed endpoint (see [Nuxt Server Route](#nuxt-server-route)). It is always "ready" because there is no client-side initialization — the server owns the Brainy instance.
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// src/composables/useBrainy.js
|
// src/composables/useBrainy.js
|
||||||
import { ref, onMounted } from 'vue'
|
import { ref } from 'vue'
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
const brain = ref(null)
|
export function useBrainy(endpoint = '/api/brain') {
|
||||||
const isReady = ref(false)
|
const error = ref(null)
|
||||||
const error = ref(null)
|
|
||||||
|
|
||||||
export function useBrainy() {
|
const search = async (query, options = {}) => {
|
||||||
onMounted(async () => {
|
error.value = null
|
||||||
try {
|
try {
|
||||||
brain.value = new Brainy({
|
const res = await fetch(`${endpoint}/search`, {
|
||||||
storage: { type: 'opfs' } // Browser storage
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ query, options })
|
||||||
})
|
})
|
||||||
await brain.value.init()
|
if (!res.ok) throw new Error(`Search failed: ${res.status}`)
|
||||||
isReady.value = true
|
return (await res.json()).results
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
error.value = err.message
|
error.value = err.message
|
||||||
console.error('Brainy initialization failed:', err)
|
console.error('Brainy search failed:', err)
|
||||||
|
return []
|
||||||
}
|
}
|
||||||
})
|
|
||||||
|
|
||||||
return {
|
|
||||||
brain: readonly(brain),
|
|
||||||
isReady: readonly(isReady),
|
|
||||||
error: readonly(error)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const add = async (data, type, metadata) => {
|
||||||
|
const res = await fetch(`${endpoint}/add`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ data, type, metadata })
|
||||||
|
})
|
||||||
|
return (await res.json()).id
|
||||||
|
}
|
||||||
|
|
||||||
|
const stats = async () => {
|
||||||
|
const res = await fetch(`${endpoint}/stats`)
|
||||||
|
return await res.json()
|
||||||
|
}
|
||||||
|
|
||||||
|
return { search, add, stats, error: readonly(error) }
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -54,43 +69,36 @@ export function useBrainy() {
|
||||||
<!-- src/components/Search.vue -->
|
<!-- src/components/Search.vue -->
|
||||||
<template>
|
<template>
|
||||||
<div class="search-container">
|
<div class="search-container">
|
||||||
<div v-if="!isReady" class="loading">
|
<div class="search-input">
|
||||||
<div class="spinner"></div>
|
<input
|
||||||
<span>Initializing AI...</span>
|
v-model="query"
|
||||||
|
@input="handleSearch"
|
||||||
|
placeholder="Search..."
|
||||||
|
class="input"
|
||||||
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div v-else>
|
<div v-if="loading" class="loading">
|
||||||
<div class="search-input">
|
Searching...
|
||||||
<input
|
</div>
|
||||||
v-model="query"
|
|
||||||
@input="handleSearch"
|
<div v-else class="results">
|
||||||
placeholder="Search with AI..."
|
<div
|
||||||
class="input"
|
v-for="result in results"
|
||||||
/>
|
:key="result.id"
|
||||||
|
class="result-item"
|
||||||
|
>
|
||||||
|
<h3>{{ result.data }}</h3>
|
||||||
|
<div class="result-meta">
|
||||||
|
<span>Score: {{ (result.score * 100).toFixed(1) }}%</span>
|
||||||
|
<span v-if="result.metadata?.type">
|
||||||
|
Type: {{ result.metadata.type }}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div v-if="loading" class="loading">
|
<div v-if="query && !loading && results.length === 0" class="no-results">
|
||||||
Searching...
|
No results found for "{{ query }}"
|
||||||
</div>
|
|
||||||
|
|
||||||
<div v-else class="results">
|
|
||||||
<div
|
|
||||||
v-for="result in results"
|
|
||||||
:key="result.id"
|
|
||||||
class="result-item"
|
|
||||||
>
|
|
||||||
<h3>{{ result.data }}</h3>
|
|
||||||
<div class="result-meta">
|
|
||||||
<span>Score: {{ (result.score * 100).toFixed(1) }}%</span>
|
|
||||||
<span v-if="result.metadata?.type">
|
|
||||||
Type: {{ result.metadata.type }}
|
|
||||||
</span>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div v-if="query && !loading && results.length === 0" class="no-results">
|
|
||||||
No results found for "{{ query }}"
|
|
||||||
</div>
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -101,22 +109,21 @@ import { ref, watch } from 'vue'
|
||||||
import { useBrainy } from '../composables/useBrainy'
|
import { useBrainy } from '../composables/useBrainy'
|
||||||
import { debounce } from '../utils/debounce'
|
import { debounce } from '../utils/debounce'
|
||||||
|
|
||||||
const { brain, isReady } = useBrainy()
|
const { search: searchBrain } = useBrainy()
|
||||||
|
|
||||||
const query = ref('')
|
const query = ref('')
|
||||||
const results = ref([])
|
const results = ref([])
|
||||||
const loading = ref(false)
|
const loading = ref(false)
|
||||||
|
|
||||||
const search = async (searchQuery) => {
|
const search = async (searchQuery) => {
|
||||||
if (!isReady.value || !searchQuery.trim()) {
|
if (!searchQuery.trim()) {
|
||||||
results.value = []
|
results.value = []
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
loading.value = true
|
loading.value = true
|
||||||
try {
|
try {
|
||||||
const searchResults = await brain.value.find(searchQuery)
|
results.value = await searchBrain(searchQuery)
|
||||||
results.value = searchResults
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Search error:', error)
|
console.error('Search error:', error)
|
||||||
results.value = []
|
results.value = []
|
||||||
|
|
@ -228,11 +235,7 @@ watch(query, (newQuery) => {
|
||||||
<div class="data-manager">
|
<div class="data-manager">
|
||||||
<h2>Data Management</h2>
|
<h2>Data Management</h2>
|
||||||
|
|
||||||
<div v-if="!isReady" class="loading">
|
<div>
|
||||||
Initializing...
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div v-else>
|
|
||||||
<!-- Add Data Form -->
|
<!-- Add Data Form -->
|
||||||
<form @submit.prevent="addData" class="add-form">
|
<form @submit.prevent="addData" class="add-form">
|
||||||
<h3>Add New Data</h3>
|
<h3>Add New Data</h3>
|
||||||
|
|
@ -289,7 +292,7 @@ watch(query, (newQuery) => {
|
||||||
import { ref, reactive, onMounted } from 'vue'
|
import { ref, reactive, onMounted } from 'vue'
|
||||||
import { useBrainy } from '../composables/useBrainy'
|
import { useBrainy } from '../composables/useBrainy'
|
||||||
|
|
||||||
const { brain, isReady } = useBrainy()
|
const { add, stats: fetchStats } = useBrainy()
|
||||||
|
|
||||||
const newItem = reactive({
|
const newItem = reactive({
|
||||||
data: '',
|
data: '',
|
||||||
|
|
@ -299,18 +302,7 @@ const newItem = reactive({
|
||||||
|
|
||||||
const stats = ref(null)
|
const stats = ref(null)
|
||||||
|
|
||||||
onMounted(async () => {
|
onMounted(loadStats)
|
||||||
if (isReady.value) {
|
|
||||||
await loadStats()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Watch for brain readiness
|
|
||||||
watch(isReady, async (ready) => {
|
|
||||||
if (ready) {
|
|
||||||
await loadStats()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
const addData = async () => {
|
const addData = async () => {
|
||||||
try {
|
try {
|
||||||
|
|
@ -319,11 +311,7 @@ const addData = async () => {
|
||||||
tags: newItem.tags.split(',').map(tag => tag.trim()).filter(Boolean)
|
tags: newItem.tags.split(',').map(tag => tag.trim()).filter(Boolean)
|
||||||
}
|
}
|
||||||
|
|
||||||
await brain.value.add({
|
await add(newItem.data, newItem.type, metadata)
|
||||||
data: newItem.data,
|
|
||||||
type: newItem.type,
|
|
||||||
metadata
|
|
||||||
})
|
|
||||||
|
|
||||||
// Reset form
|
// Reset form
|
||||||
newItem.data = ''
|
newItem.data = ''
|
||||||
|
|
@ -339,7 +327,7 @@ const addData = async () => {
|
||||||
|
|
||||||
const loadStats = async () => {
|
const loadStats = async () => {
|
||||||
try {
|
try {
|
||||||
stats.value = await brain.value.stats()
|
stats.value = await fetchStats()
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Failed to load stats:', error)
|
console.error('Failed to load stats:', error)
|
||||||
}
|
}
|
||||||
|
|
@ -435,73 +423,55 @@ button:disabled {
|
||||||
<!-- src/components/SearchOptions.vue -->
|
<!-- src/components/SearchOptions.vue -->
|
||||||
<template>
|
<template>
|
||||||
<div class="search-container">
|
<div class="search-container">
|
||||||
<div v-if="!isReady" class="loading">
|
<input
|
||||||
Initializing AI...
|
v-model="query"
|
||||||
</div>
|
@input="handleSearch"
|
||||||
|
placeholder="Search..."
|
||||||
|
class="search-input"
|
||||||
|
/>
|
||||||
|
|
||||||
<div v-else>
|
<div v-if="loading" class="loading">Searching...</div>
|
||||||
<input
|
|
||||||
v-model="query"
|
|
||||||
@input="handleSearch"
|
|
||||||
placeholder="Search..."
|
|
||||||
class="search-input"
|
|
||||||
/>
|
|
||||||
|
|
||||||
<div v-if="loading" class="loading">Searching...</div>
|
<div class="results">
|
||||||
|
<div
|
||||||
<div class="results">
|
v-for="result in results"
|
||||||
<div
|
:key="result.id"
|
||||||
v-for="result in results"
|
class="result-item"
|
||||||
:key="result.id"
|
>
|
||||||
class="result-item"
|
<h3>{{ result.data }}</h3>
|
||||||
>
|
<p>Score: {{ (result.score * 100).toFixed(1) }}%</p>
|
||||||
<h3>{{ result.data }}</h3>
|
|
||||||
<p>Score: {{ (result.score * 100).toFixed(1) }}%</p>
|
|
||||||
</div>
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script>
|
<script>
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
import { debounce } from '../utils/debounce'
|
import { debounce } from '../utils/debounce'
|
||||||
|
|
||||||
export default {
|
export default {
|
||||||
name: 'SearchOptions',
|
name: 'SearchOptions',
|
||||||
data() {
|
data() {
|
||||||
return {
|
return {
|
||||||
brain: null,
|
|
||||||
isReady: false,
|
|
||||||
query: '',
|
query: '',
|
||||||
results: [],
|
results: [],
|
||||||
loading: false
|
loading: false
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
async mounted() {
|
|
||||||
await this.initBrain()
|
|
||||||
},
|
|
||||||
methods: {
|
methods: {
|
||||||
async initBrain() {
|
|
||||||
try {
|
|
||||||
this.brain = new Brainy({
|
|
||||||
storage: { type: 'opfs' }
|
|
||||||
})
|
|
||||||
await this.brain.init()
|
|
||||||
this.isReady = true
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Brain initialization failed:', error)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
async search(query) {
|
async search(query) {
|
||||||
if (!this.isReady || !query.trim()) {
|
if (!query.trim()) {
|
||||||
this.results = []
|
this.results = []
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
this.loading = true
|
this.loading = true
|
||||||
try {
|
try {
|
||||||
this.results = await this.brain.find(query)
|
const res = await fetch('/api/brain/search', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ query })
|
||||||
|
})
|
||||||
|
this.results = (await res.json()).results
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Search error:', error)
|
console.error('Search error:', error)
|
||||||
this.results = []
|
this.results = []
|
||||||
|
|
@ -520,10 +490,6 @@ export default {
|
||||||
this.loading = false
|
this.loading = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
|
||||||
beforeUnmount() {
|
|
||||||
// Cleanup if needed
|
|
||||||
this.brain = null
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
@ -533,31 +499,22 @@ export default {
|
||||||
|
|
||||||
### Global Brainy Plugin
|
### Global Brainy Plugin
|
||||||
|
|
||||||
|
The plugin exposes a global `$searchBrain` helper that calls your Brainy-backed endpoint. Brainy itself runs on the server (see [Nuxt Server Route](#nuxt-server-route)).
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// src/plugins/brainy.js
|
// src/plugins/brainy.js
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
export default {
|
export default {
|
||||||
install(app, options = {}) {
|
install(app, options = {}) {
|
||||||
const defaultOptions = {
|
const endpoint = options.endpoint ?? '/api/brain'
|
||||||
storage: { type: 'opfs' },
|
|
||||||
...options
|
|
||||||
}
|
|
||||||
|
|
||||||
const brain = new Brainy(defaultOptions)
|
// Add global search method (calls the server endpoint)
|
||||||
|
app.config.globalProperties.$searchBrain = async (query, searchOptions) => {
|
||||||
// Make brain available globally
|
const res = await fetch(`${endpoint}/search`, {
|
||||||
app.config.globalProperties.$brain = brain
|
method: 'POST',
|
||||||
app.provide('brain', brain)
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ query, options: searchOptions })
|
||||||
// Auto-initialize
|
})
|
||||||
brain.init().catch(error => {
|
return (await res.json()).results
|
||||||
console.error('Brainy plugin initialization failed:', error)
|
|
||||||
})
|
|
||||||
|
|
||||||
// Add global method
|
|
||||||
app.config.globalProperties.$searchBrain = async (query, options) => {
|
|
||||||
return await brain.find(query, options)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -574,7 +531,7 @@ import BrainyPlugin from './plugins/brainy'
|
||||||
const app = createApp(App)
|
const app = createApp(App)
|
||||||
|
|
||||||
app.use(BrainyPlugin, {
|
app.use(BrainyPlugin, {
|
||||||
storage: { type: 'opfs' }
|
endpoint: '/api/brain'
|
||||||
})
|
})
|
||||||
|
|
||||||
app.mount('#app')
|
app.mount('#app')
|
||||||
|
|
@ -611,24 +568,58 @@ export default {
|
||||||
|
|
||||||
## 🏰 Nuxt.js Integration
|
## 🏰 Nuxt.js Integration
|
||||||
|
|
||||||
### Nuxt Plugin
|
Nuxt's server engine (Nitro) is the natural home for Brainy: it runs on Node/Bun, so the Brainy instance lives in `server/`, and pages/components call its routes.
|
||||||
|
|
||||||
|
### Nuxt Server Route
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// plugins/brainy.client.js
|
// server/utils/brain.js (server-only — Nitro never bundles this into the client)
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
import { Brainy } from '@soulcraft/brainy'
|
||||||
|
|
||||||
export default defineNuxtPlugin(async () => {
|
let brainPromise
|
||||||
const brain = new Brainy({
|
|
||||||
storage: { type: 'opfs' }
|
|
||||||
})
|
|
||||||
|
|
||||||
await brain.init()
|
export function getBrain() {
|
||||||
|
if (!brainPromise) {
|
||||||
return {
|
brainPromise = (async () => {
|
||||||
provide: {
|
// new Brainy() auto-detects filesystem persistence on Node
|
||||||
brain
|
const brain = new Brainy()
|
||||||
}
|
await brain.init()
|
||||||
|
return brain
|
||||||
|
})()
|
||||||
}
|
}
|
||||||
|
return brainPromise
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
// server/api/brain/search.post.js
|
||||||
|
import { getBrain } from '../../utils/brain'
|
||||||
|
|
||||||
|
export default defineEventHandler(async (event) => {
|
||||||
|
const { query, options } = await readBody(event)
|
||||||
|
const brain = await getBrain()
|
||||||
|
return { results: await brain.find(query, options) }
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
// server/api/brain/add.post.js
|
||||||
|
import { getBrain } from '../../utils/brain'
|
||||||
|
|
||||||
|
export default defineEventHandler(async (event) => {
|
||||||
|
const { data, type, metadata } = await readBody(event)
|
||||||
|
const brain = await getBrain()
|
||||||
|
return { id: await brain.add({ data, type, metadata }) }
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
// server/api/brain/stats.get.js
|
||||||
|
import { getBrain } from '../../utils/brain'
|
||||||
|
|
||||||
|
export default defineEventHandler(async () => {
|
||||||
|
const brain = await getBrain()
|
||||||
|
return await brain.stats()
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -637,26 +628,25 @@ export default defineNuxtPlugin(async () => {
|
||||||
```javascript
|
```javascript
|
||||||
// composables/useBrainy.js
|
// composables/useBrainy.js
|
||||||
export const useBrainy = () => {
|
export const useBrainy = () => {
|
||||||
const { $brain } = useNuxtApp()
|
|
||||||
|
|
||||||
const search = async (query, options = {}) => {
|
const search = async (query, options = {}) => {
|
||||||
return await $brain.find(query, options)
|
return (await $fetch('/api/brain/search', {
|
||||||
|
method: 'POST',
|
||||||
|
body: { query, options }
|
||||||
|
})).results
|
||||||
}
|
}
|
||||||
|
|
||||||
const add = async (data, type, metadata) => {
|
const add = async (data, type, metadata) => {
|
||||||
return await $brain.add({ data, type, metadata })
|
return (await $fetch('/api/brain/add', {
|
||||||
|
method: 'POST',
|
||||||
|
body: { data, type, metadata }
|
||||||
|
})).id
|
||||||
}
|
}
|
||||||
|
|
||||||
const stats = async () => {
|
const stats = async () => {
|
||||||
return await $brain.stats()
|
return await $fetch('/api/brain/stats')
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return { search, add, stats }
|
||||||
brain: $brain,
|
|
||||||
search,
|
|
||||||
add,
|
|
||||||
stats
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -666,26 +656,20 @@ export const useBrainy = () => {
|
||||||
<!-- pages/search.vue -->
|
<!-- pages/search.vue -->
|
||||||
<template>
|
<template>
|
||||||
<div>
|
<div>
|
||||||
<h1>AI Search</h1>
|
<h1>Search</h1>
|
||||||
|
|
||||||
<div v-if="pending" class="loading">
|
<input
|
||||||
Initializing AI...
|
v-model="query"
|
||||||
</div>
|
@input="handleSearch"
|
||||||
|
placeholder="Search..."
|
||||||
|
/>
|
||||||
|
|
||||||
<div v-else>
|
<div v-if="searching" class="loading">Searching...</div>
|
||||||
<input
|
|
||||||
v-model="query"
|
|
||||||
@input="handleSearch"
|
|
||||||
placeholder="Search..."
|
|
||||||
/>
|
|
||||||
|
|
||||||
<div v-if="searching" class="loading">Searching...</div>
|
<div class="results">
|
||||||
|
<div v-for="result in results" :key="result.id" class="result">
|
||||||
<div class="results">
|
<h3>{{ result.data }}</h3>
|
||||||
<div v-for="result in results" :key="result.id" class="result">
|
<p>Score: {{ (result.score * 100).toFixed(1) }}%</p>
|
||||||
<h3>{{ result.data }}</h3>
|
|
||||||
<p>Score: {{ (result.score * 100).toFixed(1) }}%</p>
|
|
||||||
</div>
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -697,12 +681,6 @@ const { search } = useBrainy()
|
||||||
const query = ref('')
|
const query = ref('')
|
||||||
const results = ref([])
|
const results = ref([])
|
||||||
const searching = ref(false)
|
const searching = ref(false)
|
||||||
const pending = ref(true)
|
|
||||||
|
|
||||||
// Initialize
|
|
||||||
onMounted(() => {
|
|
||||||
pending.value = false
|
|
||||||
})
|
|
||||||
|
|
||||||
const handleSearch = debounce(async () => {
|
const handleSearch = debounce(async () => {
|
||||||
if (!query.value.trim()) {
|
if (!query.value.trim()) {
|
||||||
|
|
@ -722,82 +700,58 @@ const handleSearch = debounce(async () => {
|
||||||
</script>
|
</script>
|
||||||
```
|
```
|
||||||
|
|
||||||
### Nuxt Configuration
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// nuxt.config.ts
|
|
||||||
export default defineNuxtConfig({
|
|
||||||
ssr: false, // Disable SSR for browser-only features
|
|
||||||
|
|
||||||
// Or use client-side hydration
|
|
||||||
nitro: {
|
|
||||||
experimental: {
|
|
||||||
wasm: true
|
|
||||||
}
|
|
||||||
},
|
|
||||||
|
|
||||||
vite: {
|
|
||||||
define: {
|
|
||||||
global: 'globalThis'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🛠️ Advanced Patterns
|
## 🛠️ Advanced Patterns
|
||||||
|
|
||||||
### Global State Management with Pinia
|
### Global State Management with Pinia
|
||||||
|
|
||||||
|
The store caches results and stats client-side; all Brainy work happens behind the server endpoints.
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// stores/brainy.js
|
// stores/brainy.js
|
||||||
import { defineStore } from 'pinia'
|
import { defineStore } from 'pinia'
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
export const useBrainyStore = defineStore('brainy', () => {
|
export const useBrainyStore = defineStore('brainy', () => {
|
||||||
const brain = ref(null)
|
|
||||||
const isReady = ref(false)
|
|
||||||
const error = ref(null)
|
const error = ref(null)
|
||||||
const stats = ref(null)
|
const stats = ref(null)
|
||||||
|
|
||||||
const init = async (options = {}) => {
|
const search = async (query, options = {}) => {
|
||||||
|
error.value = null
|
||||||
try {
|
try {
|
||||||
brain.value = new Brainy(options)
|
const res = await fetch('/api/brain/search', {
|
||||||
await brain.value.init()
|
method: 'POST',
|
||||||
isReady.value = true
|
headers: { 'Content-Type': 'application/json' },
|
||||||
await loadStats()
|
body: JSON.stringify({ query, options })
|
||||||
|
})
|
||||||
|
return (await res.json()).results
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
error.value = err.message
|
error.value = err.message
|
||||||
console.error('Brainy initialization failed:', err)
|
throw err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const search = async (query, options = {}) => {
|
|
||||||
if (!isReady.value) throw new Error('Brain not ready')
|
|
||||||
return await brain.value.find(query, options)
|
|
||||||
}
|
|
||||||
|
|
||||||
const add = async (data, type, metadata) => {
|
const add = async (data, type, metadata) => {
|
||||||
if (!isReady.value) throw new Error('Brain not ready')
|
const res = await fetch('/api/brain/add', {
|
||||||
const id = await brain.value.add({ data, type, metadata })
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ data, type, metadata })
|
||||||
|
})
|
||||||
|
const { id } = await res.json()
|
||||||
await loadStats() // Refresh stats
|
await loadStats() // Refresh stats
|
||||||
return id
|
return id
|
||||||
}
|
}
|
||||||
|
|
||||||
const loadStats = async () => {
|
const loadStats = async () => {
|
||||||
if (!isReady.value) return
|
|
||||||
try {
|
try {
|
||||||
stats.value = await brain.value.stats()
|
const res = await fetch('/api/brain/stats')
|
||||||
|
stats.value = await res.json()
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('Failed to load stats:', err)
|
console.error('Failed to load stats:', err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
brain: readonly(brain),
|
|
||||||
isReady: readonly(isReady),
|
|
||||||
error: readonly(error),
|
error: readonly(error),
|
||||||
stats: readonly(stats),
|
stats: readonly(stats),
|
||||||
init,
|
|
||||||
search,
|
search,
|
||||||
add,
|
add,
|
||||||
loadStats
|
loadStats
|
||||||
|
|
@ -867,7 +821,7 @@ const showResults = ref(false)
|
||||||
const selectedIndex = ref(-1)
|
const selectedIndex = ref(-1)
|
||||||
|
|
||||||
const search = async (searchQuery) => {
|
const search = async (searchQuery) => {
|
||||||
if (!brainyStore.isReady || !searchQuery.trim()) {
|
if (!searchQuery.trim()) {
|
||||||
results.value = []
|
results.value = []
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
@ -1166,21 +1120,23 @@ onMounted(() => {
|
||||||
|
|
||||||
### Component Testing with Vitest
|
### Component Testing with Vitest
|
||||||
|
|
||||||
|
The component talks to the server over `fetch`, so mock the endpoint, not Brainy itself.
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// tests/components/Search.test.js
|
// tests/components/Search.test.js
|
||||||
import { mount } from '@vue/test-utils'
|
import { mount } from '@vue/test-utils'
|
||||||
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
||||||
import Search from '../src/components/Search.vue'
|
import Search from '../src/components/Search.vue'
|
||||||
|
|
||||||
// Mock Brainy
|
// Mock the server endpoint
|
||||||
vi.mock('@soulcraft/brainy', () => ({
|
beforeEach(() => {
|
||||||
Brainy: vi.fn().mockImplementation(() => ({
|
global.fetch = vi.fn().mockResolvedValue({
|
||||||
init: vi.fn().mockResolvedValue(undefined),
|
ok: true,
|
||||||
find: vi.fn().mockResolvedValue([
|
json: async () => ({
|
||||||
{ id: '1', data: 'Test result', score: 0.9 }
|
results: [{ id: '1', data: 'Test result', score: 0.9 }]
|
||||||
])
|
})
|
||||||
}))
|
})
|
||||||
}))
|
})
|
||||||
|
|
||||||
describe('Search Component', () => {
|
describe('Search Component', () => {
|
||||||
let wrapper
|
let wrapper
|
||||||
|
|
@ -1193,14 +1149,7 @@ describe('Search Component', () => {
|
||||||
expect(wrapper.find('input').exists()).toBe(true)
|
expect(wrapper.find('input').exists()).toBe(true)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('shows loading state initially', () => {
|
|
||||||
expect(wrapper.text()).toContain('Initializing AI')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('performs search when input changes', async () => {
|
it('performs search when input changes', async () => {
|
||||||
// Wait for brain to initialize
|
|
||||||
await wrapper.vm.$nextTick()
|
|
||||||
|
|
||||||
const input = wrapper.find('input')
|
const input = wrapper.find('input')
|
||||||
await input.setValue('test query')
|
await input.setValue('test query')
|
||||||
await input.trigger('input')
|
await input.trigger('input')
|
||||||
|
|
@ -1208,6 +1157,7 @@ describe('Search Component', () => {
|
||||||
// Wait for debounced search
|
// Wait for debounced search
|
||||||
await new Promise(resolve => setTimeout(resolve, 350))
|
await new Promise(resolve => setTimeout(resolve, 350))
|
||||||
|
|
||||||
|
expect(global.fetch).toHaveBeenCalled()
|
||||||
expect(wrapper.text()).toContain('Test result')
|
expect(wrapper.text()).toContain('Test result')
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
@ -1241,6 +1191,8 @@ test('search functionality works', async ({ page }) => {
|
||||||
|
|
||||||
### Bundle Optimization
|
### Bundle Optimization
|
||||||
|
|
||||||
|
Keep Brainy out of the client bundle — it belongs to the server build only. Mark it external for SSR so the bundler resolves it at runtime instead of inlining it.
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// vite.config.js
|
// vite.config.js
|
||||||
import { defineConfig } from 'vite'
|
import { defineConfig } from 'vite'
|
||||||
|
|
@ -1248,63 +1200,46 @@ import vue from '@vitejs/plugin-vue'
|
||||||
|
|
||||||
export default defineConfig({
|
export default defineConfig({
|
||||||
plugins: [vue()],
|
plugins: [vue()],
|
||||||
define: {
|
ssr: {
|
||||||
global: 'globalThis'
|
external: ['@soulcraft/brainy']
|
||||||
},
|
|
||||||
optimizeDeps: {
|
|
||||||
include: ['@soulcraft/brainy']
|
|
||||||
},
|
|
||||||
build: {
|
|
||||||
rollupOptions: {
|
|
||||||
output: {
|
|
||||||
manualChunks: {
|
|
||||||
'brainy': ['@soulcraft/brainy']
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
### Error Handling
|
### Error Handling
|
||||||
|
|
||||||
|
Wrap the endpoint call with retry/backoff so transient network failures don't surface to the user.
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// src/composables/useBrainyWithErrorHandling.js
|
// src/composables/useBrainyWithErrorHandling.js
|
||||||
import { ref, onMounted } from 'vue'
|
import { ref } from 'vue'
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
export function useBrainyWithErrorHandling() {
|
export function useBrainyWithErrorHandling(endpoint = '/api/brain') {
|
||||||
const brain = ref(null)
|
|
||||||
const isReady = ref(false)
|
|
||||||
const error = ref(null)
|
const error = ref(null)
|
||||||
const retryCount = ref(0)
|
|
||||||
|
|
||||||
const init = async () => {
|
const search = async (query, options = {}, maxRetries = 3) => {
|
||||||
try {
|
error.value = null
|
||||||
brain.value = new Brainy()
|
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||||
await brain.value.init()
|
try {
|
||||||
isReady.value = true
|
const res = await fetch(`${endpoint}/search`, {
|
||||||
error.value = null
|
method: 'POST',
|
||||||
retryCount.value = 0
|
headers: { 'Content-Type': 'application/json' },
|
||||||
} catch (err) {
|
body: JSON.stringify({ query, options })
|
||||||
error.value = err.message
|
})
|
||||||
console.error('Brainy initialization failed:', err)
|
if (!res.ok) throw new Error(`Search failed: ${res.status}`)
|
||||||
|
return (await res.json()).results
|
||||||
// Retry logic
|
} catch (err) {
|
||||||
if (retryCount.value < 3) {
|
error.value = err.message
|
||||||
retryCount.value++
|
if (attempt === maxRetries) throw err
|
||||||
setTimeout(init, 2000 * retryCount.value)
|
// Exponential backoff before retrying
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 2000 * (attempt + 1)))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
onMounted(init)
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
brain: readonly(brain),
|
|
||||||
isReady: readonly(isReady),
|
|
||||||
error: readonly(error),
|
error: readonly(error),
|
||||||
retry: init
|
search
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
@ -1315,6 +1250,13 @@ Here's a complete Vue 3 application structure:
|
||||||
|
|
||||||
```
|
```
|
||||||
my-brainy-vue-app/
|
my-brainy-vue-app/
|
||||||
|
├── server/ # Server-side (Node/Bun) — hosts Brainy
|
||||||
|
│ ├── utils/
|
||||||
|
│ │ └── brain.js # Shared Brainy instance (getBrain)
|
||||||
|
│ └── api/brain/
|
||||||
|
│ ├── search.post.js
|
||||||
|
│ ├── add.post.js
|
||||||
|
│ └── stats.get.js
|
||||||
├── src/
|
├── src/
|
||||||
│ ├── components/
|
│ ├── components/
|
||||||
│ │ ├── Search.vue
|
│ │ ├── Search.vue
|
||||||
|
|
@ -1336,7 +1278,7 @@ my-brainy-vue-app/
|
||||||
└── package.json
|
└── package.json
|
||||||
```
|
```
|
||||||
|
|
||||||
This provides a complete, production-ready Vue.js application with comprehensive Brainy integration.
|
This provides a complete, production-ready Vue.js application: Brainy runs on the server, and the client talks to it over HTTP.
|
||||||
|
|
||||||
## 📚 Next Steps
|
## 📚 Next Steps
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -219,7 +219,6 @@ Actual cache size: ~40GB (prevents waste on 128GB systems)
|
||||||
|
|
||||||
**Recommendations:**
|
**Recommendations:**
|
||||||
- ✅ Use FP32 model for maximum accuracy
|
- ✅ Use FP32 model for maximum accuracy
|
||||||
- ✅ Enable distributed storage (S3/GCS)
|
|
||||||
- ✅ Monitor fairness violations (HNSW shouldn't dominate cache)
|
- ✅ Monitor fairness violations (HNSW shouldn't dominate cache)
|
||||||
- ✅ Consider sharding beyond 50M entities
|
- ✅ Consider sharding beyond 50M entities
|
||||||
- ✅ Implement application-level caching for hot queries
|
- ✅ Implement application-level caching for hot queries
|
||||||
|
|
@ -452,7 +451,7 @@ const euWestBrain = new Brainy({
|
||||||
// Route queries based on user location
|
// Route queries based on user location
|
||||||
async function search(query, userRegion) {
|
async function search(query, userRegion) {
|
||||||
const brain = userRegion === 'US' ? usEastBrain : euWestBrain
|
const brain = userRegion === 'US' ? usEastBrain : euWestBrain
|
||||||
return await brain.search(query)
|
return await brain.find(query)
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -501,7 +500,7 @@ if (stats.fairness.fairnessViolation) {
|
||||||
```typescript
|
```typescript
|
||||||
// Track search latency
|
// Track search latency
|
||||||
console.time('search')
|
console.time('search')
|
||||||
const results = await brain.search('query')
|
const results = await brain.find('query')
|
||||||
console.timeEnd('search') // Target: <10ms for hot queries
|
console.timeEnd('search') // Target: <10ms for hot queries
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,138 +0,0 @@
|
||||||
# Cloud Run + Filestore (NFS) Deployment Guide
|
|
||||||
|
|
||||||
> Eliminate cloud storage rate limiting by using brainy's FileSystem adapter with a Filestore NFS mount on Cloud Run.
|
|
||||||
|
|
||||||
## Why Filestore?
|
|
||||||
|
|
||||||
Brainy's metadata index generates 26-40 file writes per `brain.add()` call. On GCS, each write is a cloud API call subject to rate limiting (~1 write/sec per object), causing HTTP 429 errors and 61-second latency for a single add operation.
|
|
||||||
|
|
||||||
With Filestore:
|
|
||||||
- **Local disk speed**: Each write is ~1ms (vs 50-300ms for cloud API)
|
|
||||||
- **No rate limits**: NFS has no per-object write throttling
|
|
||||||
- **Zero code changes**: Use brainy's existing `FileSystemStorage` adapter
|
|
||||||
- **Same Cloud Run deployment**: Just add a volume mount
|
|
||||||
|
|
||||||
## Prerequisites
|
|
||||||
|
|
||||||
- Google Cloud project with Filestore API enabled
|
|
||||||
- Cloud Run service (gen2 execution environment required for NFS)
|
|
||||||
- VPC connector or Direct VPC egress configured
|
|
||||||
|
|
||||||
## Step 1: Create a Filestore Instance
|
|
||||||
|
|
||||||
```bash
|
|
||||||
gcloud filestore instances create brainy-store \
|
|
||||||
--zone=us-central1-b \
|
|
||||||
--tier=BASIC_SSD \
|
|
||||||
--file-share=name=brainy_data,capacity=1TB \
|
|
||||||
--network=name=default
|
|
||||||
```
|
|
||||||
|
|
||||||
**Tier recommendations:**
|
|
||||||
- **BASIC_SSD** (~$370/month for 1 TiB): Good balance of performance and cost
|
|
||||||
- **BASIC_HDD** (~$204/month for 1 TiB): Lower cost, sufficient for moderate workloads
|
|
||||||
- **ENTERPRISE** or **ZONAL**: Higher IOPS for heavy workloads
|
|
||||||
|
|
||||||
Note the IP address from the output — you'll need it for the Cloud Run mount.
|
|
||||||
|
|
||||||
## Step 2: Configure Cloud Run NFS Volume Mount
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# service.yaml
|
|
||||||
apiVersion: serving.knative.dev/v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: my-brainy-service
|
|
||||||
spec:
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
annotations:
|
|
||||||
run.googleapis.com/execution-environment: gen2
|
|
||||||
run.googleapis.com/vpc-access-connector: projects/PROJECT/locations/REGION/connectors/CONNECTOR
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- image: gcr.io/PROJECT/my-brainy-app
|
|
||||||
volumeMounts:
|
|
||||||
- name: brainy-nfs
|
|
||||||
mountPath: /mnt/brainy
|
|
||||||
volumes:
|
|
||||||
- name: brainy-nfs
|
|
||||||
nfs:
|
|
||||||
server: FILESTORE_IP # e.g., 10.0.0.2
|
|
||||||
path: /brainy_data
|
|
||||||
```
|
|
||||||
|
|
||||||
Deploy:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
gcloud run services replace service.yaml --region=us-central1
|
|
||||||
```
|
|
||||||
|
|
||||||
## Step 3: Configure Brainy to Use FileSystem Adapter
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
import { FileSystemStorage } from '@soulcraft/brainy/storage'
|
|
||||||
|
|
||||||
const storage = new FileSystemStorage({
|
|
||||||
basePath: '/mnt/brainy/brain-data'
|
|
||||||
})
|
|
||||||
|
|
||||||
const brain = new Brainy({ storage })
|
|
||||||
await brain.init()
|
|
||||||
```
|
|
||||||
|
|
||||||
That's it. No GCS adapter, no rate limits, no retry logic needed.
|
|
||||||
|
|
||||||
## Caveats
|
|
||||||
|
|
||||||
### Single-Writer Recommendation
|
|
||||||
|
|
||||||
NFS does not provide strong file locking guarantees on Cloud Run. For best results:
|
|
||||||
|
|
||||||
- Use a **single Cloud Run instance** (max-instances=1) for write operations
|
|
||||||
- Multiple read-only instances can safely read from the same Filestore mount
|
|
||||||
- For multi-writer scenarios, use the GCS adapter with the write buffer (rate limit protection built in)
|
|
||||||
|
|
||||||
### Filestore Permissions
|
|
||||||
|
|
||||||
Cloud Run gen2 instances run as a specific service account. Ensure the Filestore instance allows access from your VPC network. No additional IAM permissions are needed — Filestore uses NFS network-level access.
|
|
||||||
|
|
||||||
### Cost Comparison
|
|
||||||
|
|
||||||
| Solution | Monthly Cost (1 TiB) | Write Latency | Rate Limits |
|
|
||||||
|----------|---------------------|---------------|-------------|
|
|
||||||
| GCS Standard | ~$20 storage + API ops | 50-300ms/write | 1 write/sec/object |
|
|
||||||
| GCS HNS | ~$20 storage + API ops | 50-300ms/write | 8,000 writes/sec |
|
|
||||||
| Filestore SSD | ~$370 fixed | ~1ms/write | None |
|
|
||||||
| Filestore HDD | ~$204 fixed | ~5ms/write | None |
|
|
||||||
|
|
||||||
Filestore costs more for storage but eliminates all rate limiting issues and provides significantly lower write latency.
|
|
||||||
|
|
||||||
### When to Choose Filestore vs GCS
|
|
||||||
|
|
||||||
**Choose Filestore when:**
|
|
||||||
- Write-heavy workloads (frequent `brain.add()`, chat applications)
|
|
||||||
- Low-latency requirements
|
|
||||||
- Single-writer architecture is acceptable
|
|
||||||
|
|
||||||
**Choose GCS (with write buffer) when:**
|
|
||||||
- Multi-instance deployments need shared storage
|
|
||||||
- Cost optimization for large datasets (lifecycle policies, Autoclass)
|
|
||||||
- Read-heavy workloads with infrequent writes
|
|
||||||
|
|
||||||
## Verification
|
|
||||||
|
|
||||||
After deploying, verify the mount works:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# In Cloud Run container
|
|
||||||
ls -la /mnt/brainy/
|
|
||||||
# Should show the Filestore share contents
|
|
||||||
|
|
||||||
# Test write performance
|
|
||||||
dd if=/dev/zero of=/mnt/brainy/test bs=1M count=100
|
|
||||||
# Expected: ~100MB/s+ for SSD tier
|
|
||||||
```
|
|
||||||
|
|
||||||
Then run your brainy application and confirm `brain.add()` completes without rate limit errors.
|
|
||||||
|
|
@ -1,399 +0,0 @@
|
||||||
# AWS S3 Cost Optimization Guide for Brainy
|
|
||||||
> **Cost Impact**: Reduce storage costs from $138k/year to $5.9k/year at 500TB scale (**96% savings**)
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
Brainy provides enterprise-grade cost optimization features for AWS S3 storage, enabling automatic tier transitions that dramatically reduce storage costs while maintaining performance.
|
|
||||||
|
|
||||||
## Cost Breakdown (Before Optimization)
|
|
||||||
|
|
||||||
### Standard S3 Storage Costs (500TB Dataset)
|
|
||||||
|
|
||||||
```
|
|
||||||
Storage: 500TB × $0.023/GB/month × 12 months = $138,000/year
|
|
||||||
Operations: ~$5,000/year (PUT, GET, LIST requests)
|
|
||||||
Total: $143,000/year
|
|
||||||
```
|
|
||||||
|
|
||||||
## S3 Storage Tiers
|
|
||||||
|
|
||||||
| Tier | Cost/GB/Month | Retrieval Fee | Use Case |
|
|
||||||
|------|---------------|---------------|----------|
|
|
||||||
| **Standard** | $0.023 | None | Frequently accessed |
|
|
||||||
| **Standard-IA** | $0.0125 | $0.01/GB | Infrequently accessed (30+ days) |
|
|
||||||
| **Intelligent-Tiering** | $0.023-0.00099 | None | Automatic optimization |
|
|
||||||
| **Glacier Instant** | $0.004 | $0.03/GB | Rare access, instant retrieval |
|
|
||||||
| **Glacier Flexible** | $0.0036 | $0.01/GB + time | Archive (minutes-hours retrieval) |
|
|
||||||
| **Glacier Deep Archive** | $0.00099 | $0.02/GB + time | Long-term archive (12 hours retrieval) |
|
|
||||||
|
|
||||||
## Strategy 1: Lifecycle Policies (Recommended)
|
|
||||||
|
|
||||||
### Setup: Automatic Tier Transitions
|
|
||||||
|
|
||||||
**Best for**: Predictable access patterns, batch workloads, archival data
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
import { S3CompatibleStorage } from '@soulcraft/brainy/storage'
|
|
||||||
|
|
||||||
// Initialize Brainy with S3 storage
|
|
||||||
const storage = new S3CompatibleStorage({
|
|
||||||
bucket: 'my-brainy-data',
|
|
||||||
region: 'us-east-1',
|
|
||||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
|
||||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
|
||||||
})
|
|
||||||
|
|
||||||
const brain = new Brainy({ storage })
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Set lifecycle policy for automatic archival
|
|
||||||
await storage.setLifecyclePolicy({
|
|
||||||
rules: [{
|
|
||||||
id: 'optimize-vectors',
|
|
||||||
prefix: 'entities/nouns/vectors/',
|
|
||||||
status: 'Enabled',
|
|
||||||
transitions: [
|
|
||||||
{ days: 30, storageClass: 'STANDARD_IA' }, // Infrequent Access after 30 days
|
|
||||||
{ days: 90, storageClass: 'GLACIER' }, // Glacier after 90 days
|
|
||||||
{ days: 365, storageClass: 'DEEP_ARCHIVE' } // Deep Archive after 1 year
|
|
||||||
]
|
|
||||||
}, {
|
|
||||||
id: 'optimize-metadata',
|
|
||||||
prefix: 'entities/nouns/metadata/',
|
|
||||||
status: 'Enabled',
|
|
||||||
transitions: [
|
|
||||||
{ days: 30, storageClass: 'STANDARD_IA' },
|
|
||||||
{ days: 180, storageClass: 'GLACIER' }
|
|
||||||
]
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
|
|
||||||
// Verify lifecycle policy
|
|
||||||
const policy = await storage.getLifecyclePolicy()
|
|
||||||
console.log('Lifecycle policy active:', policy.rules.length, 'rules')
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cost Calculation (500TB with Lifecycle Policy)
|
|
||||||
|
|
||||||
**Assumptions:**
|
|
||||||
- 40% of data accessed in last 30 days (Standard)
|
|
||||||
- 30% of data 30-90 days old (Standard-IA)
|
|
||||||
- 20% of data 90-365 days old (Glacier)
|
|
||||||
- 10% of data 365+ days old (Deep Archive)
|
|
||||||
|
|
||||||
```
|
|
||||||
Standard (200TB): 200TB × $0.023/GB × 12 = $55,200/year
|
|
||||||
Standard-IA (150TB): 150TB × $0.0125/GB × 12 = $22,500/year
|
|
||||||
Glacier (100TB): 100TB × $0.004/GB × 12 = $4,800/year
|
|
||||||
Deep Archive (50TB): 50TB × $0.00099/GB × 12 = $594/year
|
|
||||||
|
|
||||||
Total Storage Cost: $83,094/year (instead of $138,000)
|
|
||||||
Total with Operations: ~$88,000/year
|
|
||||||
Savings: $55,000/year (40%)
|
|
||||||
```
|
|
||||||
|
|
||||||
**But we can do better with Intelligent-Tiering...**
|
|
||||||
|
|
||||||
## Strategy 2: Intelligent-Tiering (Most Recommended)
|
|
||||||
|
|
||||||
### Setup: Automatic Access-Based Optimization
|
|
||||||
|
|
||||||
**Best for**: Unpredictable access patterns, mixed workloads, maximum automation
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Enable Intelligent-Tiering for automatic optimization
|
|
||||||
await storage.enableIntelligentTiering('entities/', 'brainy-auto-optimize')
|
|
||||||
|
|
||||||
// Benefits:
|
|
||||||
// - Automatically moves objects between tiers based on access patterns
|
|
||||||
// - No retrieval fees (unlike Glacier)
|
|
||||||
// - Transitions happen within 24-48 hours of last access
|
|
||||||
// - Supports Archive Access tier (90+ days) and Deep Archive Access tier (180+ days)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Intelligent-Tiering Tiers
|
|
||||||
|
|
||||||
Intelligent-Tiering automatically moves objects between:
|
|
||||||
|
|
||||||
1. **Frequent Access**: $0.023/GB/month (0-30 days)
|
|
||||||
2. **Infrequent Access**: $0.0125/GB/month (30-90 days)
|
|
||||||
3. **Archive Access**: $0.004/GB/month (90-180 days)
|
|
||||||
4. **Deep Archive Access**: $0.00099/GB/month (180+ days)
|
|
||||||
|
|
||||||
**Monitoring Fee**: $0.0025 per 1000 objects (minimal)
|
|
||||||
|
|
||||||
### Cost Calculation (500TB with Intelligent-Tiering)
|
|
||||||
|
|
||||||
**Realistic distribution after 1 year:**
|
|
||||||
- 15% Frequent Access (hot data)
|
|
||||||
- 20% Infrequent Access
|
|
||||||
- 35% Archive Access
|
|
||||||
- 30% Deep Archive Access
|
|
||||||
|
|
||||||
```
|
|
||||||
Frequent (75TB): 75TB × $0.023/GB × 12 = $20,700/year
|
|
||||||
Infrequent (100TB): 100TB × $0.0125/GB × 12 = $15,000/year
|
|
||||||
Archive (175TB): 175TB × $0.004/GB × 12 = $8,400/year
|
|
||||||
Deep Archive (150TB): 150TB × $0.00099/GB × 12 = $1,782/year
|
|
||||||
Monitoring: ~$300/year (minimal)
|
|
||||||
|
|
||||||
Total Storage Cost: $46,182/year
|
|
||||||
Total with Operations: ~$51,000/year
|
|
||||||
Savings vs Standard: $92,000/year (67%)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Strategy 3: Hybrid Approach (Maximum Savings)
|
|
||||||
|
|
||||||
### Setup: Lifecycle + Intelligent-Tiering
|
|
||||||
|
|
||||||
**Best for**: Maximum cost optimization with fine-grained control
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Enable Intelligent-Tiering for vectors (frequently searched)
|
|
||||||
await storage.enableIntelligentTiering('entities/nouns/vectors/', 'vectors-auto')
|
|
||||||
await storage.enableIntelligentTiering('entities/verbs/vectors/', 'verbs-auto')
|
|
||||||
|
|
||||||
// Set lifecycle policy for metadata (less frequently accessed)
|
|
||||||
await storage.setLifecyclePolicy({
|
|
||||||
rules: [{
|
|
||||||
id: 'archive-old-metadata',
|
|
||||||
prefix: 'entities/nouns/metadata/',
|
|
||||||
status: 'Enabled',
|
|
||||||
transitions: [
|
|
||||||
{ days: 30, storageClass: 'STANDARD_IA' },
|
|
||||||
{ days: 60, storageClass: 'GLACIER' },
|
|
||||||
{ days: 180, storageClass: 'DEEP_ARCHIVE' }
|
|
||||||
]
|
|
||||||
}, {
|
|
||||||
id: 'cleanup-old-system-data',
|
|
||||||
prefix: '_system/',
|
|
||||||
status: 'Enabled',
|
|
||||||
expiration: { days: 365 } // Delete old statistics
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cost Calculation (500TB Hybrid Approach)
|
|
||||||
|
|
||||||
**Vectors (300TB with Intelligent-Tiering):**
|
|
||||||
```
|
|
||||||
Frequent (45TB): 45TB × $0.023/GB × 12 = $12,420/year
|
|
||||||
Infrequent (60TB): 60TB × $0.0125/GB × 12 = $9,000/year
|
|
||||||
Archive (105TB): 105TB × $0.004/GB × 12 = $5,040/year
|
|
||||||
Deep Archive (90TB): 90TB × $0.00099/GB × 12 = $1,069/year
|
|
||||||
Subtotal: $27,529/year
|
|
||||||
```
|
|
||||||
|
|
||||||
**Metadata (200TB with Lifecycle Policy):**
|
|
||||||
```
|
|
||||||
Standard (60TB): 60TB × $0.023/GB × 12 = $16,560/year
|
|
||||||
Standard-IA (40TB): 40TB × $0.0125/GB × 12 = $6,000/year
|
|
||||||
Glacier (60TB): 60TB × $0.004/GB × 12 = $2,880/year
|
|
||||||
Deep Archive (40TB): 40TB × $0.00099/GB × 12 = $475/year
|
|
||||||
Subtotal: $25,915/year
|
|
||||||
```
|
|
||||||
|
|
||||||
**Total Cost: $53,444/year + operations (~$58,500/year total)**
|
|
||||||
**Savings vs Standard: $84,500/year (61%)**
|
|
||||||
|
|
||||||
## Strategy 4: Aggressive Archival (Maximum Savings)
|
|
||||||
|
|
||||||
### Setup: Fast Archival for Cold Data
|
|
||||||
|
|
||||||
**Best for**: Archival workloads, historical data, compliance
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
await storage.setLifecyclePolicy({
|
|
||||||
rules: [{
|
|
||||||
id: 'aggressive-archival',
|
|
||||||
prefix: 'entities/',
|
|
||||||
status: 'Enabled',
|
|
||||||
transitions: [
|
|
||||||
{ days: 14, storageClass: 'STANDARD_IA' }, // IA after 2 weeks
|
|
||||||
{ days: 30, storageClass: 'GLACIER' }, // Glacier after 1 month
|
|
||||||
{ days: 90, storageClass: 'DEEP_ARCHIVE' } // Deep Archive after 3 months
|
|
||||||
]
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cost Calculation (500TB Aggressive Archival)
|
|
||||||
|
|
||||||
**After 1 year:**
|
|
||||||
```
|
|
||||||
Standard (50TB): 50TB × $0.023/GB × 12 = $13,800/year
|
|
||||||
Standard-IA (50TB): 50TB × $0.0125/GB × 12 = $7,500/year
|
|
||||||
Glacier (100TB): 100TB × $0.004/GB × 12 = $4,800/year
|
|
||||||
Deep Archive (300TB): 300TB × $0.00099/GB × 12 = $3,564/year
|
|
||||||
|
|
||||||
Total Storage Cost: $29,664/year
|
|
||||||
Total with Operations: ~$34,000/year
|
|
||||||
Savings: $109,000/year (76%)
|
|
||||||
|
|
||||||
Note: Retrieval costs may be significant if archived data is accessed frequently
|
|
||||||
```
|
|
||||||
|
|
||||||
## Comparison Table: All Strategies
|
|
||||||
|
|
||||||
| Strategy | Annual Cost | Savings | Retrieval Speed | Best For |
|
|
||||||
|----------|-------------|---------|-----------------|----------|
|
|
||||||
| **No Optimization** | $143,000 | 0% | Instant | N/A |
|
|
||||||
| **Lifecycle Policy** | $88,000 | 38% | Varies | Predictable patterns |
|
|
||||||
| **Intelligent-Tiering** | $51,000 | 64% | Instant (no retrieval fees) | **Recommended** |
|
|
||||||
| **Hybrid Approach** | $58,500 | 59% | Instant for vectors | Fine-grained control |
|
|
||||||
| **Aggressive Archival** | $34,000 | 76% | Hours to 12 hours | Cold data, compliance |
|
|
||||||
|
|
||||||
## Batch Delete Operations
|
|
||||||
|
|
||||||
### Efficient Cleanup
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Batch delete (1000 objects per request)
|
|
||||||
const idsToDelete = [/* array of entity IDs */]
|
|
||||||
|
|
||||||
// Generate paths for both vector and metadata files
|
|
||||||
const paths = idsToDelete.flatMap(id => {
|
|
||||||
const shard = id.substring(0, 2)
|
|
||||||
return [
|
|
||||||
`entities/nouns/vectors/${shard}/${id}.json`,
|
|
||||||
`entities/nouns/metadata/${shard}/${id}.json`
|
|
||||||
]
|
|
||||||
})
|
|
||||||
|
|
||||||
// Batch delete (much faster and cheaper than individual deletes)
|
|
||||||
await storage.batchDelete(paths)
|
|
||||||
|
|
||||||
// Cost impact:
|
|
||||||
// - Individual deletes: 1M objects × $0.005 per 1000 = $5,000
|
|
||||||
// - Batch deletes: 1M/1000 × $0.005 = $5 (1000x cheaper!)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Monitoring and Optimization
|
|
||||||
|
|
||||||
### Get Current Lifecycle Policy
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const policy = await storage.getLifecyclePolicy()
|
|
||||||
console.log('Active rules:', policy.rules)
|
|
||||||
|
|
||||||
// Example output:
|
|
||||||
// {
|
|
||||||
// rules: [
|
|
||||||
// {
|
|
||||||
// id: 'optimize-vectors',
|
|
||||||
// prefix: 'entities/nouns/vectors/',
|
|
||||||
// status: 'Enabled',
|
|
||||||
// transitions: [...]
|
|
||||||
// }
|
|
||||||
// ]
|
|
||||||
// }
|
|
||||||
```
|
|
||||||
|
|
||||||
### Remove Lifecycle Policy (if needed)
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Remove all lifecycle rules
|
|
||||||
await storage.removeLifecyclePolicy()
|
|
||||||
```
|
|
||||||
|
|
||||||
### Check Intelligent-Tiering Configurations
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const configs = await storage.getIntelligentTieringConfigs()
|
|
||||||
console.log('Active configurations:', configs)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Disable Intelligent-Tiering
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
await storage.disableIntelligentTiering('brainy-auto-optimize')
|
|
||||||
```
|
|
||||||
|
|
||||||
## AWS Cost Explorer Analysis
|
|
||||||
|
|
||||||
### Track Your Savings
|
|
||||||
|
|
||||||
1. **Enable Cost Explorer** in AWS Console
|
|
||||||
2. **Group by Storage Class** to see tier distribution
|
|
||||||
3. **Set up Cost Anomaly Detection** for unexpected spikes
|
|
||||||
4. **Create Budget Alerts** for monthly storage costs
|
|
||||||
|
|
||||||
### Expected Metrics After 6 Months
|
|
||||||
|
|
||||||
```
|
|
||||||
Standard storage: 15-20% of total data
|
|
||||||
Standard-IA: 20-25%
|
|
||||||
Archive tiers: 55-65%
|
|
||||||
|
|
||||||
Monthly cost trend: Decreasing 5-10% per month as data ages into cheaper tiers
|
|
||||||
```
|
|
||||||
|
|
||||||
## Best Practices
|
|
||||||
|
|
||||||
1. ✅ **Start with Intelligent-Tiering** - No retrieval fees, automatic optimization
|
|
||||||
2. ✅ **Use batch operations** for deletions - 1000x cheaper than individual deletes
|
|
||||||
3. ✅ **Monitor storage class distribution** monthly via Cost Explorer
|
|
||||||
4. ✅ **Set lifecycle policies** for predictable archival (metadata, logs)
|
|
||||||
5. ✅ **Enable S3 Storage Lens** for detailed storage analytics
|
|
||||||
6. ✅ **Use S3 Select** for querying archived data without full retrieval
|
|
||||||
7. ✅ **Consider S3 Batch Operations** for large-scale tier changes
|
|
||||||
|
|
||||||
## Troubleshooting
|
|
||||||
|
|
||||||
### Issue: Data not transitioning to cheaper tiers
|
|
||||||
|
|
||||||
**Solution:**
|
|
||||||
```typescript
|
|
||||||
// Check if lifecycle policy is active
|
|
||||||
const policy = await storage.getLifecyclePolicy()
|
|
||||||
console.log('Policy status:', policy.rules.map(r => r.status))
|
|
||||||
|
|
||||||
// Ensure objects are old enough
|
|
||||||
// S3 requires objects to be at least 30 days old for IA transition
|
|
||||||
```
|
|
||||||
|
|
||||||
### Issue: High retrieval costs from Glacier
|
|
||||||
|
|
||||||
**Solution:**
|
|
||||||
```typescript
|
|
||||||
// Switch to Intelligent-Tiering (no retrieval fees)
|
|
||||||
await storage.disableIntelligentTiering('old-config')
|
|
||||||
await storage.enableIntelligentTiering('entities/', 'new-config')
|
|
||||||
|
|
||||||
// Or use Glacier Instant Retrieval instead of Glacier Flexible
|
|
||||||
```
|
|
||||||
|
|
||||||
### Issue: Unexpected monitoring fees
|
|
||||||
|
|
||||||
**Solution:**
|
|
||||||
- Intelligent-Tiering has $0.0025 per 1000 objects monitoring fee
|
|
||||||
- For 1 billion objects: $2,500/month monitoring
|
|
||||||
- If cost is high, use lifecycle policies instead (no monitoring fee)
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
**Recommended Strategy for Most Use Cases:**
|
|
||||||
- **Intelligent-Tiering** for vectors and frequently queried data
|
|
||||||
- **Lifecycle policies** for metadata and system files
|
|
||||||
- **Batch operations** for efficient cleanup
|
|
||||||
|
|
||||||
**Expected Savings:**
|
|
||||||
- **Year 1**: 40-50% reduction in storage costs
|
|
||||||
- **Year 2+**: 60-70% reduction as more data ages into archive tiers
|
|
||||||
- **Long-term**: 75-85% reduction for mature datasets
|
|
||||||
|
|
||||||
**500TB Example (Intelligent-Tiering):**
|
|
||||||
- Before: $143,000/year
|
|
||||||
- After: $51,000/year
|
|
||||||
- **Savings: $92,000/year (64%)**
|
|
||||||
|
|
||||||
**1PB Example (Intelligent-Tiering):**
|
|
||||||
- Before: $286,000/year
|
|
||||||
- After: $102,000/year
|
|
||||||
- **Savings: $184,000/year (64%)**
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Last Updated**: 2025-10-17
|
|
||||||
**Cloud Provider**: AWS S3
|
|
||||||
|
|
@ -1,552 +0,0 @@
|
||||||
# Azure Blob Storage Cost Optimization Guide for Brainy
|
|
||||||
> **Cost Impact**: Reduce storage costs from $107k/year to $5k/year at 500TB scale (**95% savings**)
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
Brainy provides enterprise-grade cost optimization features for Azure Blob Storage, including manual tier management, lifecycle policies, and batch operations.
|
|
||||||
|
|
||||||
## Cost Breakdown (Before Optimization)
|
|
||||||
|
|
||||||
### Hot Tier Azure Storage Costs (500TB Dataset)
|
|
||||||
|
|
||||||
```
|
|
||||||
Storage: 500TB × $0.0184/GB/month × 12 months = $107,520/year
|
|
||||||
Operations: ~$5,000/year (write/read operations)
|
|
||||||
Total: $112,520/year
|
|
||||||
```
|
|
||||||
|
|
||||||
## Azure Blob Storage Tiers
|
|
||||||
|
|
||||||
| Tier | Cost/GB/Month | Retrieval | Early Deletion | Use Case |
|
|
||||||
|------|---------------|-----------|----------------|----------|
|
|
||||||
| **Hot** | $0.0184 | None | None | Frequent access |
|
|
||||||
| **Cool** | $0.0115 | $0.01/GB | 30 days | Infrequent access |
|
|
||||||
| **Archive** | $0.00099 | $0.02/GB + rehydration time | 180 days | Long-term archive |
|
|
||||||
|
|
||||||
**Key Difference from AWS/GCS:**
|
|
||||||
- Azure has only 3 tiers (vs AWS 6 tiers, GCS 4 classes)
|
|
||||||
- Archive tier requires rehydration (hours to 15 hours) before access
|
|
||||||
- No "Intelligent-Tiering" equivalent - must use lifecycle policies or manual management
|
|
||||||
|
|
||||||
## Strategy 1: Manual Tier Management (Immediate Savings)
|
|
||||||
|
|
||||||
### Setup: Change Blob Tiers Manually
|
|
||||||
|
|
||||||
**Best for**: Quick wins, specific files, immediate cost reduction
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
import { AzureBlobStorage } from '@soulcraft/brainy/storage'
|
|
||||||
|
|
||||||
// Initialize Brainy with Azure storage
|
|
||||||
const storage = new AzureBlobStorage({
|
|
||||||
connectionString: process.env.AZURE_STORAGE_CONNECTION_STRING,
|
|
||||||
containerName: 'brainy-data'
|
|
||||||
})
|
|
||||||
|
|
||||||
const brain = new Brainy({ storage })
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Change tier for a single blob
|
|
||||||
await storage.changeBlobTier(
|
|
||||||
'entities/nouns/vectors/00/00123456-uuid.json',
|
|
||||||
'Cool'
|
|
||||||
)
|
|
||||||
|
|
||||||
// Batch tier changes (efficient for thousands of blobs)
|
|
||||||
const blobsToMove = [
|
|
||||||
'entities/nouns/vectors/01/...',
|
|
||||||
'entities/nouns/vectors/02/...',
|
|
||||||
// ... up to thousands of blobs
|
|
||||||
]
|
|
||||||
|
|
||||||
await storage.batchChangeTier(blobsToMove, 'Cool') // Hot → Cool
|
|
||||||
await storage.batchChangeTier(oldBlobs, 'Archive') // Cool → Archive
|
|
||||||
```
|
|
||||||
|
|
||||||
### Immediate Cost Impact
|
|
||||||
|
|
||||||
Moving 400TB from Hot to Cool:
|
|
||||||
```
|
|
||||||
Before (Hot): 400TB × $0.0184/GB × 12 = $86,016/year
|
|
||||||
After (Cool): 400TB × $0.0115/GB × 12 = $53,760/year
|
|
||||||
Savings: $32,256/year (37% savings on moved data)
|
|
||||||
```
|
|
||||||
|
|
||||||
Moving 100TB from Hot to Archive:
|
|
||||||
```
|
|
||||||
Before (Hot): 100TB × $0.0184/GB × 12 = $21,504/year
|
|
||||||
After (Archive): 100TB × $0.00099/GB × 12 = $1,158/year
|
|
||||||
Savings: $20,346/year (95% savings on moved data)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Strategy 2: Lifecycle Policies (Automated)
|
|
||||||
|
|
||||||
### Setup: Automatic Tier Transitions
|
|
||||||
|
|
||||||
**Best for**: Predictable patterns, automatic management
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Set lifecycle policy for automatic tier management
|
|
||||||
await storage.setLifecyclePolicy({
|
|
||||||
rules: [{
|
|
||||||
name: 'optimizeVectors',
|
|
||||||
enabled: true,
|
|
||||||
type: 'Lifecycle',
|
|
||||||
definition: {
|
|
||||||
filters: {
|
|
||||||
blobTypes: ['blockBlob'],
|
|
||||||
prefixMatch: ['entities/nouns/vectors/']
|
|
||||||
},
|
|
||||||
actions: {
|
|
||||||
baseBlob: {
|
|
||||||
tierToCool: { daysAfterModificationGreaterThan: 30 },
|
|
||||||
tierToArchive: { daysAfterModificationGreaterThan: 90 }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}, {
|
|
||||||
name: 'optimizeMetadata',
|
|
||||||
enabled: true,
|
|
||||||
type: 'Lifecycle',
|
|
||||||
definition: {
|
|
||||||
filters: {
|
|
||||||
blobTypes: ['blockBlob'],
|
|
||||||
prefixMatch: ['entities/nouns/metadata/']
|
|
||||||
},
|
|
||||||
actions: {
|
|
||||||
baseBlob: {
|
|
||||||
tierToCool: { daysAfterModificationGreaterThan: 30 },
|
|
||||||
tierToArchive: { daysAfterModificationGreaterThan: 180 }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}, {
|
|
||||||
name: 'cleanupOldSystemFiles',
|
|
||||||
enabled: true,
|
|
||||||
type: 'Lifecycle',
|
|
||||||
definition: {
|
|
||||||
filters: {
|
|
||||||
blobTypes: ['blockBlob'],
|
|
||||||
prefixMatch: ['_system/']
|
|
||||||
},
|
|
||||||
actions: {
|
|
||||||
baseBlob: {
|
|
||||||
delete: { daysAfterModificationGreaterThan: 365 }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
|
|
||||||
// Verify lifecycle policy
|
|
||||||
const policy = await storage.getLifecyclePolicy()
|
|
||||||
console.log('Lifecycle policy active:', policy.rules.length, 'rules')
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cost Calculation (500TB with Lifecycle Policy)
|
|
||||||
|
|
||||||
**Assumptions:**
|
|
||||||
- 30% of data in Hot tier (< 30 days old)
|
|
||||||
- 40% of data in Cool tier (30-90 days old)
|
|
||||||
- 30% of data in Archive tier (90+ days old)
|
|
||||||
|
|
||||||
```
|
|
||||||
Hot (150TB): 150TB × $0.0184/GB × 12 = $32,256/year
|
|
||||||
Cool (200TB): 200TB × $0.0115/GB × 12 = $26,880/year
|
|
||||||
Archive (150TB): 150TB × $0.00099/GB × 12 = $1,732/year
|
|
||||||
|
|
||||||
Total Storage Cost: $60,868/year
|
|
||||||
Total with Operations: ~$65,500/year
|
|
||||||
Savings: $47,000/year (42%)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Strategy 3: Aggressive Archival (Maximum Savings)
|
|
||||||
|
|
||||||
### Setup: Fast Archival for Cold Data
|
|
||||||
|
|
||||||
**Best for**: Archival workloads, compliance, historical data
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
await storage.setLifecyclePolicy({
|
|
||||||
rules: [{
|
|
||||||
name: 'aggressiveArchival',
|
|
||||||
enabled: true,
|
|
||||||
type: 'Lifecycle',
|
|
||||||
definition: {
|
|
||||||
filters: {
|
|
||||||
blobTypes: ['blockBlob'],
|
|
||||||
prefixMatch: ['entities/']
|
|
||||||
},
|
|
||||||
actions: {
|
|
||||||
baseBlob: {
|
|
||||||
tierToCool: { daysAfterModificationGreaterThan: 14 },
|
|
||||||
tierToArchive: { daysAfterModificationGreaterThan: 30 }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cost Calculation (500TB Aggressive Archival)
|
|
||||||
|
|
||||||
**After 6 months:**
|
|
||||||
```
|
|
||||||
Hot (50TB): 50TB × $0.0184/GB × 12 = $10,752/year
|
|
||||||
Cool (100TB): 100TB × $0.0115/GB × 12 = $13,440/year
|
|
||||||
Archive (350TB): 350TB × $0.00099/GB × 12 = $4,039/year
|
|
||||||
|
|
||||||
Total Storage Cost: $28,231/year
|
|
||||||
Total with Operations: ~$33,000/year
|
|
||||||
Savings: $79,500/year (71%)
|
|
||||||
|
|
||||||
Warning: Archive rehydration takes 1-15 hours
|
|
||||||
```
|
|
||||||
|
|
||||||
## Strategy 4: Hybrid Approach (Balanced)
|
|
||||||
|
|
||||||
### Setup: Different Policies for Different Data Types
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
await storage.setLifecyclePolicy({
|
|
||||||
rules: [{
|
|
||||||
// Vectors: Keep in Hot/Cool for search performance
|
|
||||||
name: 'vectors-moderate',
|
|
||||||
enabled: true,
|
|
||||||
type: 'Lifecycle',
|
|
||||||
definition: {
|
|
||||||
filters: {
|
|
||||||
blobTypes: ['blockBlob'],
|
|
||||||
prefixMatch: ['entities/nouns/vectors/', 'entities/verbs/vectors/']
|
|
||||||
},
|
|
||||||
actions: {
|
|
||||||
baseBlob: {
|
|
||||||
tierToCool: { daysAfterModificationGreaterThan: 60 }
|
|
||||||
// Don't archive vectors - keep searchable
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}, {
|
|
||||||
// Metadata: Aggressive archival
|
|
||||||
name: 'metadata-aggressive',
|
|
||||||
enabled: true,
|
|
||||||
type: 'Lifecycle',
|
|
||||||
definition: {
|
|
||||||
filters: {
|
|
||||||
blobTypes: ['blockBlob'],
|
|
||||||
prefixMatch: ['entities/nouns/metadata/', 'entities/verbs/metadata/']
|
|
||||||
},
|
|
||||||
actions: {
|
|
||||||
baseBlob: {
|
|
||||||
tierToCool: { daysAfterModificationGreaterThan: 30 },
|
|
||||||
tierToArchive: { daysAfterModificationGreaterThan: 90 }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cost Calculation (500TB Hybrid Approach)
|
|
||||||
|
|
||||||
**Vectors (300TB):**
|
|
||||||
```
|
|
||||||
Hot (90TB): 90TB × $0.0184/GB × 12 = $19,354/year
|
|
||||||
Cool (210TB): 210TB × $0.0115/GB × 12 = $28,980/year
|
|
||||||
Subtotal: $48,334/year
|
|
||||||
```
|
|
||||||
|
|
||||||
**Metadata (200TB):**
|
|
||||||
```
|
|
||||||
Hot (30TB): 30TB × $0.0184/GB × 12 = $6,451/year
|
|
||||||
Cool (70TB): 70TB × $0.0115/GB × 12 = $9,660/year
|
|
||||||
Archive (100TB): 100TB × $0.00099/GB × 12 = $1,158/year
|
|
||||||
Subtotal: $17,269/year
|
|
||||||
```
|
|
||||||
|
|
||||||
**Total Cost: $65,603/year + operations (~$70,500/year total)**
|
|
||||||
**Savings vs Hot: $42,000/year (37%)**
|
|
||||||
|
|
||||||
## Comparison Table: All Strategies
|
|
||||||
|
|
||||||
| Strategy | Annual Cost | Savings | Archive % | Best For |
|
|
||||||
|----------|-------------|---------|-----------|----------|
|
|
||||||
| **No Optimization** | $112,500 | 0% | 0% | N/A |
|
|
||||||
| **Manual Tier Mgmt** | $75,000 | 33% | 20% | Immediate savings |
|
|
||||||
| **Lifecycle Policy** | $65,500 | 42% | 30% | Automated management |
|
|
||||||
| **Hybrid Approach** | $70,500 | 37% | 20% | Balance performance/cost |
|
|
||||||
| **Aggressive Archival** | $33,000 | **71%** | 70% | Cold data, compliance |
|
|
||||||
|
|
||||||
## Archive Rehydration (Important!)
|
|
||||||
|
|
||||||
### Rehydrate from Archive Tier
|
|
||||||
|
|
||||||
**Required before accessing archived blobs:**
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Rehydrate blob from Archive to Hot (high priority)
|
|
||||||
await storage.rehydrateBlob(
|
|
||||||
'entities/nouns/vectors/00/00123456-uuid.json',
|
|
||||||
'High' // 'Standard' or 'High' priority
|
|
||||||
)
|
|
||||||
|
|
||||||
// Rehydration time:
|
|
||||||
// - High priority: 1 hour
|
|
||||||
// - Standard priority: up to 15 hours
|
|
||||||
|
|
||||||
// Check rehydration status
|
|
||||||
const metadata = await storage.getBlobMetadata(blobPath)
|
|
||||||
console.log('Archive status:', metadata.archiveStatus)
|
|
||||||
// Output: 'rehydrate-pending-to-hot', 'rehydrate-pending-to-cool', or undefined (done)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Batch Rehydration
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Rehydrate multiple blobs (useful for planned access)
|
|
||||||
const blobsToRehydrate = [/* array of blob paths */]
|
|
||||||
|
|
||||||
for (const blobPath of blobsToRehydrate) {
|
|
||||||
await storage.rehydrateBlob(blobPath, 'High')
|
|
||||||
}
|
|
||||||
|
|
||||||
// Wait for rehydration to complete (1-15 hours)
|
|
||||||
// Then access blobs normally
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cost Impact of Rehydration
|
|
||||||
|
|
||||||
```
|
|
||||||
Retrieval fee: $0.02/GB
|
|
||||||
High priority: Additional $0.10/GB
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
- 1GB blob (standard): $0.02 + wait 15 hours
|
|
||||||
- 1GB blob (high priority): $0.12 + wait 1 hour
|
|
||||||
- 1TB batch (high priority): $122.88 + wait 1 hour
|
|
||||||
```
|
|
||||||
|
|
||||||
## Batch Operations
|
|
||||||
|
|
||||||
### Efficient Bulk Deletions
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Batch delete (256 blobs per request)
|
|
||||||
const idsToDelete = [/* array of entity IDs */]
|
|
||||||
|
|
||||||
const paths = idsToDelete.flatMap(id => {
|
|
||||||
const shard = id.substring(0, 2)
|
|
||||||
return [
|
|
||||||
`entities/nouns/vectors/${shard}/${id}.json`,
|
|
||||||
`entities/nouns/metadata/${shard}/${id}.json`
|
|
||||||
]
|
|
||||||
})
|
|
||||||
|
|
||||||
// Batch delete via BlobBatchClient
|
|
||||||
await storage.batchDelete(paths)
|
|
||||||
|
|
||||||
// Cost impact:
|
|
||||||
// - Individual deletes: 1M operations × $0.0005 per 10k = $50
|
|
||||||
// - Batch deletes: 4k batches × $0.0005 = $2 (25x cheaper!)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Batch Tier Changes
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Change tier for thousands of blobs efficiently
|
|
||||||
const vectorPaths = [/* 10,000 blob paths */]
|
|
||||||
|
|
||||||
// Batch operation (256 blobs per batch)
|
|
||||||
await storage.batchChangeTier(vectorPaths, 'Cool')
|
|
||||||
|
|
||||||
// Much faster than individual changeBlobTier() calls
|
|
||||||
// Azure automatically batches internally for efficiency
|
|
||||||
```
|
|
||||||
|
|
||||||
## Monitoring and Management
|
|
||||||
|
|
||||||
### Get Current Lifecycle Policy
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const policy = await storage.getLifecyclePolicy()
|
|
||||||
console.log('Active rules:', policy.rules)
|
|
||||||
|
|
||||||
// Example output:
|
|
||||||
// {
|
|
||||||
// rules: [
|
|
||||||
// {
|
|
||||||
// name: 'optimizeVectors',
|
|
||||||
// enabled: true,
|
|
||||||
// type: 'Lifecycle',
|
|
||||||
// definition: {...}
|
|
||||||
// }
|
|
||||||
// ]
|
|
||||||
// }
|
|
||||||
```
|
|
||||||
|
|
||||||
### Remove Lifecycle Policy
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
await storage.removeLifecyclePolicy()
|
|
||||||
```
|
|
||||||
|
|
||||||
### Get Storage Status
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const status = await storage.getStorageStatus()
|
|
||||||
console.log('Storage type:', status.type)
|
|
||||||
console.log('Container:', status.details.container)
|
|
||||||
console.log('Account:', status.details.account)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Azure Portal Monitoring
|
|
||||||
|
|
||||||
### Track Your Savings
|
|
||||||
|
|
||||||
1. **Storage Account** → **Insights** → View tier distribution
|
|
||||||
2. **Cost Management** → **Cost Analysis** → Filter by storage account
|
|
||||||
3. **Monitoring** → **Metrics** → Track blob count by tier
|
|
||||||
4. **Lifecycle Management** → View policy execution logs
|
|
||||||
|
|
||||||
### Expected Metrics After 6 Months
|
|
||||||
|
|
||||||
```
|
|
||||||
Hot tier: 20-30% of total data
|
|
||||||
Cool tier: 40-50%
|
|
||||||
Archive tier: 20-40%
|
|
||||||
|
|
||||||
Monthly cost trend: Decreasing 5-8% per month as data transitions
|
|
||||||
```
|
|
||||||
|
|
||||||
## Best Practices
|
|
||||||
|
|
||||||
1. ✅ **Use lifecycle policies** for automatic management
|
|
||||||
2. ✅ **Archive cold data** aggressively (95% cost savings!)
|
|
||||||
3. ✅ **Use batch operations** for tier changes and deletions
|
|
||||||
4. ✅ **Plan rehydration** ahead of time (1-15 hour delay)
|
|
||||||
5. ✅ **Monitor early deletion charges** (Cool: 30 days, Archive: 180 days)
|
|
||||||
6. ✅ **Use Cool tier for infrequent access** (no rehydration needed)
|
|
||||||
7. ✅ **Consider ZRS or GRS** for critical data redundancy
|
|
||||||
|
|
||||||
## Storage Redundancy Options
|
|
||||||
|
|
||||||
| Option | Cost Multiplier | Copies | Availability | Use Case |
|
|
||||||
|--------|-----------------|--------|--------------|----------|
|
|
||||||
| **LRS** | 1x | 3 (same datacenter) | 11 nines | Cost-optimized |
|
|
||||||
| **ZRS** | 1.25x | 3 (different zones) | 12 nines | High availability |
|
|
||||||
| **GRS** | 2x | 6 (secondary region) | 16 nines | Disaster recovery |
|
|
||||||
| **RA-GRS** | 2.5x | 6 (read access) | 16 nines | Global read access |
|
|
||||||
|
|
||||||
**Recommendation for Brainy:**
|
|
||||||
- **Production**: GRS (geo-redundancy for disaster recovery)
|
|
||||||
- **Cost-optimized**: LRS (lowest cost, still very reliable)
|
|
||||||
|
|
||||||
## Troubleshooting
|
|
||||||
|
|
||||||
### Issue: Data not transitioning tiers
|
|
||||||
|
|
||||||
**Solution:**
|
|
||||||
```typescript
|
|
||||||
// Check lifecycle policy status
|
|
||||||
const policy = await storage.getLifecyclePolicy()
|
|
||||||
console.log('Policy rules:', policy.rules.map(r => ({
|
|
||||||
name: r.name,
|
|
||||||
enabled: r.enabled
|
|
||||||
})))
|
|
||||||
|
|
||||||
// Azure lifecycle policies run once per day
|
|
||||||
// Transitions may take 24-48 hours to execute
|
|
||||||
```
|
|
||||||
|
|
||||||
### Issue: Access denied on archived blob
|
|
||||||
|
|
||||||
**Solution:**
|
|
||||||
```typescript
|
|
||||||
// Archived blobs cannot be accessed directly
|
|
||||||
// Must rehydrate first:
|
|
||||||
await storage.rehydrateBlob(blobPath, 'High')
|
|
||||||
|
|
||||||
// Wait for rehydration (check status)
|
|
||||||
let status
|
|
||||||
do {
|
|
||||||
await new Promise(resolve => setTimeout(resolve, 60000)) // Wait 1 minute
|
|
||||||
const metadata = await storage.getBlobMetadata(blobPath)
|
|
||||||
status = metadata.archiveStatus
|
|
||||||
} while (status && status.includes('pending'))
|
|
||||||
|
|
||||||
// Now access the blob
|
|
||||||
const data = await storage.get(blobPath)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Issue: High early deletion charges
|
|
||||||
|
|
||||||
**Solution:**
|
|
||||||
- Cool tier: 30-day minimum storage
|
|
||||||
- Archive tier: 180-day minimum storage
|
|
||||||
- Early deletion incurs pro-rated charges
|
|
||||||
- Use lifecycle policies instead of manual changes to avoid early deletion fees
|
|
||||||
|
|
||||||
## Authentication
|
|
||||||
|
|
||||||
### Connection String (Simple)
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const storage = new AzureBlobStorage({
|
|
||||||
connectionString: process.env.AZURE_STORAGE_CONNECTION_STRING,
|
|
||||||
containerName: 'brainy-data'
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Account Key (Explicit)
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const storage = new AzureBlobStorage({
|
|
||||||
accountName: process.env.AZURE_STORAGE_ACCOUNT,
|
|
||||||
accountKey: process.env.AZURE_STORAGE_KEY,
|
|
||||||
containerName: 'brainy-data'
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### SAS Token (Granular Access)
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const storage = new AzureBlobStorage({
|
|
||||||
sasToken: process.env.AZURE_STORAGE_SAS_TOKEN,
|
|
||||||
accountName: process.env.AZURE_STORAGE_ACCOUNT,
|
|
||||||
containerName: 'brainy-data'
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
**Recommended Strategy for Most Use Cases:**
|
|
||||||
- **Lifecycle policies** for automatic tier management
|
|
||||||
- **Batch operations** for efficient tier changes and deletions
|
|
||||||
- **Cool tier** for infrequently accessed data (no rehydration delay)
|
|
||||||
- **Archive tier** for long-term storage (1-15 hour rehydration)
|
|
||||||
|
|
||||||
**Expected Savings:**
|
|
||||||
- **Year 1**: 40-50% reduction in storage costs
|
|
||||||
- **Year 2+**: 60-70% reduction as more data ages into Archive
|
|
||||||
- **Long-term**: 75-85% reduction for mature datasets
|
|
||||||
|
|
||||||
**500TB Example (Lifecycle Policy):**
|
|
||||||
- Before: $112,500/year
|
|
||||||
- After: $65,500/year
|
|
||||||
- **Savings: $47,000/year (42%)**
|
|
||||||
|
|
||||||
**500TB Example (Aggressive Archival):**
|
|
||||||
- Before: $112,500/year
|
|
||||||
- After: $33,000/year
|
|
||||||
- **Savings: $79,500/year (71%)**
|
|
||||||
|
|
||||||
**1PB Example (Lifecycle Policy):**
|
|
||||||
- Before: $225,000/year
|
|
||||||
- After: $131,000/year
|
|
||||||
- **Savings: $94,000/year (42%)**
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Last Updated**: 2025-10-17
|
|
||||||
**Cloud Provider**: Azure Blob Storage
|
|
||||||
|
|
@ -1,450 +0,0 @@
|
||||||
# Cloudflare R2 Cost Optimization Guide for Brainy
|
|
||||||
> **Cost Impact**: $0 egress fees + 96% storage savings = Lowest cloud storage costs
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
Cloudflare R2 is an S3-compatible object storage with **zero egress fees**, making it ideal for high-traffic applications. Brainy fully supports R2 with lifecycle policies and batch operations.
|
|
||||||
|
|
||||||
## Cost Breakdown
|
|
||||||
|
|
||||||
### R2 Pricing (As of 2025)
|
|
||||||
|
|
||||||
```
|
|
||||||
Storage: $0.015/GB/month
|
|
||||||
Class A Operations (write): $4.50 per million
|
|
||||||
Class B Operations (read): $0.36 per million
|
|
||||||
Egress: $0.00 (FREE!)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Comparison to AWS S3 (500TB Dataset)
|
|
||||||
|
|
||||||
**AWS S3 Standard:**
|
|
||||||
```
|
|
||||||
Storage: 500TB × $0.023/GB × 12 = $138,000/year
|
|
||||||
Egress (assume 100TB/month): 1.2PB × $0.09/GB = $108,000/year
|
|
||||||
Operations: $5,000/year
|
|
||||||
Total: $251,000/year
|
|
||||||
```
|
|
||||||
|
|
||||||
**Cloudflare R2 Standard:**
|
|
||||||
```
|
|
||||||
Storage: 500TB × $0.015/GB × 12 = $90,000/year
|
|
||||||
Egress: $0 (FREE!)
|
|
||||||
Operations: $5,000/year
|
|
||||||
Total: $95,000/year
|
|
||||||
Savings vs AWS: $156,000/year (62%)
|
|
||||||
```
|
|
||||||
|
|
||||||
**R2's Zero Egress Advantage:**
|
|
||||||
- **High-traffic apps**: Save $100k-$1M/year in egress fees
|
|
||||||
- **Video/media delivery**: No CDN egress costs
|
|
||||||
- **API responses**: Unlimited reads at no extra cost
|
|
||||||
|
|
||||||
## R2 Storage Classes (Coming Soon)
|
|
||||||
|
|
||||||
**Current State (2025):**
|
|
||||||
- R2 currently has only **one storage class** (Standard)
|
|
||||||
- No lifecycle policies or tier transitions yet
|
|
||||||
- Cloudflare plans to add infrequent access tiers
|
|
||||||
|
|
||||||
**When lifecycle features arrive:**
|
|
||||||
- Brainy is already prepared with `setLifecyclePolicy()` support
|
|
||||||
- Will work seamlessly once Cloudflare enables lifecycle management
|
|
||||||
|
|
||||||
## Strategy 1: Use R2 Standard (Current Best Practice)
|
|
||||||
|
|
||||||
### Setup: S3-Compatible API
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
import { S3CompatibleStorage } from '@soulcraft/brainy/storage'
|
|
||||||
|
|
||||||
// R2 uses S3-compatible API
|
|
||||||
const storage = new S3CompatibleStorage({
|
|
||||||
endpoint: `https://${process.env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com`,
|
|
||||||
bucket: 'my-brainy-data',
|
|
||||||
region: 'auto', // R2 uses 'auto' region
|
|
||||||
accessKeyId: process.env.R2_ACCESS_KEY_ID,
|
|
||||||
secretAccessKey: process.env.R2_SECRET_ACCESS_KEY
|
|
||||||
})
|
|
||||||
|
|
||||||
const brain = new Brainy({ storage })
|
|
||||||
await brain.init()
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cost Calculation (500TB on R2)
|
|
||||||
|
|
||||||
```
|
|
||||||
Storage: 500TB × $0.015/GB × 12 = $90,000/year
|
|
||||||
Class A ops (10M writes): $45/year
|
|
||||||
Class B ops (100M reads): $36/year
|
|
||||||
Egress: $0
|
|
||||||
|
|
||||||
Total: $90,081/year
|
|
||||||
```
|
|
||||||
|
|
||||||
**Compared to AWS S3 (with egress):**
|
|
||||||
- AWS: $251,000/year
|
|
||||||
- R2: $90,081/year
|
|
||||||
- **Savings: $160,919/year (64%)**
|
|
||||||
|
|
||||||
## Strategy 2: R2 + Workers (Edge Computing)
|
|
||||||
|
|
||||||
### Setup: Compute at the Edge
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Cloudflare Worker (runs at edge)
|
|
||||||
export default {
|
|
||||||
async fetch(request, env) {
|
|
||||||
// Initialize Brainy with R2
|
|
||||||
const storage = new S3CompatibleStorage({
|
|
||||||
endpoint: env.R2_ENDPOINT,
|
|
||||||
bucket: env.R2_BUCKET,
|
|
||||||
accessKeyId: env.R2_ACCESS_KEY,
|
|
||||||
secretAccessKey: env.R2_SECRET_KEY,
|
|
||||||
region: 'auto'
|
|
||||||
})
|
|
||||||
|
|
||||||
const brain = new Brainy({ storage })
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Process at edge (no origin server needed)
|
|
||||||
const results = await brain.search(request.query)
|
|
||||||
return new Response(JSON.stringify(results))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cost Calculation (Workers + R2)
|
|
||||||
|
|
||||||
```
|
|
||||||
R2 Storage (500TB): $90,000/year
|
|
||||||
Workers (10M requests/day):
|
|
||||||
- First 100k requests/day: FREE
|
|
||||||
- Additional 350M requests/month: $1,750/year
|
|
||||||
- CPU time (50ms avg): $5,000/year
|
|
||||||
|
|
||||||
Total: $96,750/year
|
|
||||||
|
|
||||||
vs Traditional Setup (AWS S3 + EC2 + CloudFront):
|
|
||||||
- S3: $138,000/year
|
|
||||||
- EC2 (t3.xlarge × 4): $24,000/year
|
|
||||||
- CloudFront egress: $50,000/year
|
|
||||||
- Load balancer: $3,000/year
|
|
||||||
Total: $215,000/year
|
|
||||||
|
|
||||||
Savings: $118,250/year (55%)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Strategy 3: Hybrid Multi-Cloud (R2 + S3)
|
|
||||||
|
|
||||||
### Setup: R2 for Hot Data, S3 for Archives
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Use R2 for frequently accessed data (zero egress)
|
|
||||||
const hotStorage = new S3CompatibleStorage({
|
|
||||||
endpoint: `https://${process.env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com`,
|
|
||||||
bucket: 'brainy-hot',
|
|
||||||
region: 'auto',
|
|
||||||
accessKeyId: process.env.R2_ACCESS_KEY_ID,
|
|
||||||
secretAccessKey: process.env.R2_SECRET_ACCESS_KEY
|
|
||||||
})
|
|
||||||
|
|
||||||
// Use AWS S3 with Intelligent-Tiering for cold data
|
|
||||||
const coldStorage = new S3CompatibleStorage({
|
|
||||||
endpoint: 's3.amazonaws.com',
|
|
||||||
bucket: 'brainy-archive',
|
|
||||||
region: 'us-east-1',
|
|
||||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
|
||||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
|
||||||
})
|
|
||||||
|
|
||||||
// Initialize separate Brainy instances or implement tiering logic
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cost Calculation (300TB R2 + 200TB S3 Archive)
|
|
||||||
|
|
||||||
```
|
|
||||||
R2 Hot Data (300TB):
|
|
||||||
Storage: 300TB × $0.015/GB × 12 = $54,000/year
|
|
||||||
Egress: $0
|
|
||||||
|
|
||||||
S3 Cold Data (200TB with lifecycle → Deep Archive):
|
|
||||||
Storage: 200TB × $0.00099/GB × 12 = $2,376/year
|
|
||||||
Ops: $1,000/year
|
|
||||||
|
|
||||||
Total: $57,376/year
|
|
||||||
|
|
||||||
vs All AWS S3:
|
|
||||||
- S3 storage + egress: $251,000/year
|
|
||||||
|
|
||||||
Savings: $193,624/year (77%)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Batch Operations
|
|
||||||
|
|
||||||
### Efficient Bulk Deletions
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// R2 supports S3 batch delete API
|
|
||||||
const idsToDelete = [/* array of entity IDs */]
|
|
||||||
|
|
||||||
const paths = idsToDelete.flatMap(id => {
|
|
||||||
const shard = id.substring(0, 2)
|
|
||||||
return [
|
|
||||||
`entities/nouns/vectors/${shard}/${id}.json`,
|
|
||||||
`entities/nouns/metadata/${shard}/${id}.json`
|
|
||||||
]
|
|
||||||
})
|
|
||||||
|
|
||||||
// Batch delete (1000 objects per request)
|
|
||||||
await storage.batchDelete(paths)
|
|
||||||
|
|
||||||
// Cost impact:
|
|
||||||
// - Individual deletes: 1M × $4.50/1M = $4.50
|
|
||||||
// - Batch deletes: 1k batches × $4.50/1M = $0.0045 (1000x cheaper!)
|
|
||||||
```
|
|
||||||
|
|
||||||
## R2 Advanced Features
|
|
||||||
|
|
||||||
### 1. R2 Custom Domains
|
|
||||||
|
|
||||||
**Free custom domains for R2 buckets:**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Configure custom domain in Cloudflare dashboard
|
|
||||||
# Then access via your domain
|
|
||||||
https://storage.yourdomain.com/entities/nouns/vectors/...
|
|
||||||
|
|
||||||
# Benefits:
|
|
||||||
# - No additional cost
|
|
||||||
# - Automatic SSL/TLS
|
|
||||||
# - Global CDN included
|
|
||||||
# - DDoS protection
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. R2 Event Notifications
|
|
||||||
|
|
||||||
**Trigger Workers on object events:**
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Worker triggered on R2 object upload
|
|
||||||
export default {
|
|
||||||
async fetch(request, env) {
|
|
||||||
// Process new objects automatically
|
|
||||||
// E.g., index new entities, generate thumbnails, etc.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cost: Only pay for Worker execution (no polling needed)
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. R2 Presigned URLs
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Generate presigned URL for direct browser uploads
|
|
||||||
const url = await storage.getPresignedUrl('upload-path', 3600) // 1 hour expiry
|
|
||||||
|
|
||||||
// Client uploads directly to R2 (no server bandwidth used)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Monitoring and Management
|
|
||||||
|
|
||||||
### Get Storage Status
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const status = await storage.getStorageStatus()
|
|
||||||
console.log('Storage type:', status.type) // 's3-compatible'
|
|
||||||
console.log('Bucket:', status.details.bucket)
|
|
||||||
console.log('Endpoint:', status.details.endpoint)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cloudflare Dashboard Monitoring
|
|
||||||
|
|
||||||
1. **R2 Dashboard** → View bucket metrics
|
|
||||||
2. **Analytics** → Track requests, storage, and bandwidth
|
|
||||||
3. **Workers Analytics** → Monitor edge compute usage
|
|
||||||
4. **Logs** → Real-time logs with Logpush
|
|
||||||
|
|
||||||
### Expected Metrics
|
|
||||||
|
|
||||||
```
|
|
||||||
Storage: Growing with your data
|
|
||||||
Class A ops: Writes (higher cost)
|
|
||||||
Class B ops: Reads (minimal cost)
|
|
||||||
Egress: Always $0 (R2's advantage)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Comparison Table: R2 vs Other Providers
|
|
||||||
|
|
||||||
| Feature | R2 | AWS S3 | GCS | Azure |
|
|
||||||
|---------|-----|--------|-----|-------|
|
|
||||||
| **Storage** | $0.015/GB | $0.023/GB | $0.020/GB | $0.0184/GB |
|
|
||||||
| **Egress** | **$0** | $0.09/GB | $0.12/GB | $0.087/GB |
|
|
||||||
| **Lifecycle Tiers** | Coming soon | 6 tiers | 4 classes | 3 tiers |
|
|
||||||
| **S3 API Compatible** | ✅ Yes | ✅ Native | ⚠️ Via interop | ⚠️ Via SDK |
|
|
||||||
| **CDN Included** | ✅ Yes | ❌ Extra cost | ❌ Extra cost | ❌ Extra cost |
|
|
||||||
| **Edge Compute** | ✅ Workers | ❌ Lambda@Edge | ❌ Cloud Functions | ❌ Functions |
|
|
||||||
|
|
||||||
## R2 Free Tier
|
|
||||||
|
|
||||||
**Generous free tier:**
|
|
||||||
```
|
|
||||||
Storage: 10 GB free per month
|
|
||||||
Class A ops: 1 million free per month
|
|
||||||
Class B ops: 10 million free per month
|
|
||||||
Egress: Unlimited (always free)
|
|
||||||
```
|
|
||||||
|
|
||||||
**Perfect for:**
|
|
||||||
- Development and testing
|
|
||||||
- Small applications (<10GB)
|
|
||||||
- Prototypes
|
|
||||||
|
|
||||||
## Best Practices
|
|
||||||
|
|
||||||
1. ✅ **Use R2 for high-egress workloads** - Zero egress fees
|
|
||||||
2. ✅ **Combine with Workers** - Edge compute included
|
|
||||||
3. ✅ **Use custom domains** - Free branded URLs
|
|
||||||
4. ✅ **Batch operations** for deletions - 1000x cheaper
|
|
||||||
5. ✅ **Use presigned URLs** - Direct client uploads
|
|
||||||
6. ✅ **Monitor with Analytics** - Built-in dashboarding
|
|
||||||
7. ✅ **Consider hybrid approach** - R2 hot + S3 archive cold
|
|
||||||
|
|
||||||
## Migration from S3 to R2
|
|
||||||
|
|
||||||
### Using rclone
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Install rclone
|
|
||||||
brew install rclone # or apt-get install rclone
|
|
||||||
|
|
||||||
# Configure S3 source
|
|
||||||
rclone config create s3-source s3 \
|
|
||||||
access_key_id=$AWS_ACCESS_KEY \
|
|
||||||
secret_access_key=$AWS_SECRET_KEY \
|
|
||||||
region=us-east-1
|
|
||||||
|
|
||||||
# Configure R2 destination
|
|
||||||
rclone config create r2-dest s3 \
|
|
||||||
access_key_id=$R2_ACCESS_KEY \
|
|
||||||
secret_access_key=$R2_SECRET_KEY \
|
|
||||||
endpoint=https://$R2_ACCOUNT_ID.r2.cloudflarestorage.com \
|
|
||||||
region=auto
|
|
||||||
|
|
||||||
# Copy data
|
|
||||||
rclone copy s3-source:my-bucket r2-dest:my-bucket --progress
|
|
||||||
|
|
||||||
# Verify
|
|
||||||
rclone check s3-source:my-bucket r2-dest:my-bucket
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cost of Migration
|
|
||||||
|
|
||||||
```
|
|
||||||
Data transfer out from S3: 500TB × $0.09/GB = $45,000
|
|
||||||
Data transfer into R2: $0 (ingress is free)
|
|
||||||
|
|
||||||
One-time migration cost: $45,000
|
|
||||||
|
|
||||||
Monthly savings after migration:
|
|
||||||
S3 storage + egress: $20,833/month
|
|
||||||
R2 storage: $7,500/month
|
|
||||||
Savings: $13,333/month
|
|
||||||
|
|
||||||
ROI: 3.4 months
|
|
||||||
```
|
|
||||||
|
|
||||||
## Future: R2 Lifecycle Policies (When Available)
|
|
||||||
|
|
||||||
### Prepared for Future Features
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Brainy is ready for R2 lifecycle features
|
|
||||||
await storage.setLifecyclePolicy({
|
|
||||||
rules: [{
|
|
||||||
id: 'archive-old-data',
|
|
||||||
prefix: 'entities/',
|
|
||||||
status: 'Enabled',
|
|
||||||
transitions: [
|
|
||||||
{ days: 30, storageClass: 'INFREQUENT_ACCESS' }, // When available
|
|
||||||
{ days: 90, storageClass: 'ARCHIVE' }
|
|
||||||
]
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
|
|
||||||
// Expected cost impact (when lifecycle is available):
|
|
||||||
// Standard: $0.015/GB
|
|
||||||
// Infrequent: ~$0.008/GB (estimated)
|
|
||||||
// Archive: ~$0.002/GB (estimated)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Troubleshooting
|
|
||||||
|
|
||||||
### Issue: Connection errors
|
|
||||||
|
|
||||||
**Solution:**
|
|
||||||
```typescript
|
|
||||||
// Ensure correct endpoint format
|
|
||||||
const storage = new S3CompatibleStorage({
|
|
||||||
endpoint: `https://${accountId}.r2.cloudflarestorage.com`,
|
|
||||||
// NOT: `https://r2.cloudflarestorage.com/${accountId}`
|
|
||||||
|
|
||||||
region: 'auto', // R2 requires 'auto'
|
|
||||||
forcePathStyle: false // R2 uses virtual-hosted-style
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Issue: High Class A operation costs
|
|
||||||
|
|
||||||
**Solution:**
|
|
||||||
- Use batch operations (writes are most expensive)
|
|
||||||
- Cache frequently written data
|
|
||||||
- Consolidate small writes into larger batches
|
|
||||||
- Consider Workers KV for high-frequency writes
|
|
||||||
|
|
||||||
### Issue: Need lifecycle management now
|
|
||||||
|
|
||||||
**Solution:**
|
|
||||||
- Manually move old data to S3 Deep Archive
|
|
||||||
- Use hybrid approach: R2 for hot, S3 for cold
|
|
||||||
- Wait for R2 lifecycle features (planned)
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
**R2 Advantages:**
|
|
||||||
- ✅ **Zero egress fees** - Unlimited reads at no cost
|
|
||||||
- ✅ **Lower storage costs** - $0.015/GB vs $0.023/GB (AWS)
|
|
||||||
- ✅ **S3-compatible API** - Drop-in replacement for S3
|
|
||||||
- ✅ **Global CDN included** - No additional CDN costs
|
|
||||||
- ✅ **Edge Workers** - Compute at the edge
|
|
||||||
- ✅ **Free custom domains** - Branded URLs
|
|
||||||
- ✅ **No minimums** - No minimum storage duration
|
|
||||||
|
|
||||||
**R2 Limitations (Current):**
|
|
||||||
- ⚠️ Single storage class (for now)
|
|
||||||
- ⚠️ No lifecycle policies yet (coming soon)
|
|
||||||
- ⚠️ Less mature than S3/GCS/Azure
|
|
||||||
|
|
||||||
**Recommended Use Cases:**
|
|
||||||
- 🎯 High-traffic APIs (zero egress fees!)
|
|
||||||
- 🎯 Video/media delivery (massive savings)
|
|
||||||
- 🎯 User-generated content
|
|
||||||
- 🎯 Web application assets
|
|
||||||
- 🎯 Hot data storage
|
|
||||||
|
|
||||||
**500TB Example (R2 vs AWS S3 with 100TB/month egress):**
|
|
||||||
- AWS S3: $251,000/year
|
|
||||||
- Cloudflare R2: $90,000/year
|
|
||||||
- **Savings: $161,000/year (64%)**
|
|
||||||
|
|
||||||
**1PB Example (R2 vs AWS S3 with 200TB/month egress):**
|
|
||||||
- AWS S3: $686,000/year
|
|
||||||
- Cloudflare R2: $180,000/year
|
|
||||||
- **Savings: $506,000/year (74%)**
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Last Updated**: 2025-10-17
|
|
||||||
**Cloud Provider**: Cloudflare R2
|
|
||||||
**Key Advantage**: **$0 egress fees forever**
|
|
||||||
|
|
@ -1,465 +0,0 @@
|
||||||
# Google Cloud Storage Cost Optimization Guide for Brainy
|
|
||||||
> **Cost Impact**: Reduce storage costs from $138k/year to $8.3k/year at 500TB scale (**94% savings**)
|
|
||||||
|
|
||||||
> **Disclaimer**: Cost savings percentages are PROJECTED based on published cloud provider pricing at 500TB scale. Actual savings depend on access patterns, data lifecycle, and workload characteristics. These are not measured benchmarks.
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
Brainy provides enterprise-grade cost optimization features for Google Cloud Storage, including lifecycle policies and Autoclass for automatic tier management.
|
|
||||||
|
|
||||||
## Cost Breakdown (Before Optimization)
|
|
||||||
|
|
||||||
### Standard GCS Storage Costs (500TB Dataset)
|
|
||||||
|
|
||||||
```
|
|
||||||
Storage: 500TB × $0.023/GB/month × 12 months = $138,000/year
|
|
||||||
Operations: ~$5,000/year (Class A/B operations)
|
|
||||||
Total: $143,000/year
|
|
||||||
```
|
|
||||||
|
|
||||||
## GCS Storage Classes
|
|
||||||
|
|
||||||
| Class | Cost/GB/Month | Retrieval Fee | Minimum Storage | Use Case |
|
|
||||||
|-------|---------------|---------------|-----------------|----------|
|
|
||||||
| **Standard** | $0.020 | None | None | Frequent access |
|
|
||||||
| **Nearline** | $0.010 | $0.01/GB | 30 days | Once per month |
|
|
||||||
| **Coldline** | $0.004 | $0.02/GB | 90 days | Once per quarter |
|
|
||||||
| **Archive** | $0.0012 | $0.05/GB | 365 days | Long-term archive |
|
|
||||||
|
|
||||||
## Strategy 1: Lifecycle Policies (Manual Control)
|
|
||||||
|
|
||||||
### Setup: Automatic Tier Transitions
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
import { GcsStorage } from '@soulcraft/brainy/storage'
|
|
||||||
|
|
||||||
// Initialize Brainy with GCS storage
|
|
||||||
const storage = new GcsStorage({
|
|
||||||
bucketName: 'my-brainy-data',
|
|
||||||
keyFilename: './service-account.json' // Or use ADC
|
|
||||||
})
|
|
||||||
|
|
||||||
const brain = new Brainy({ storage })
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Set lifecycle policy for automatic archival
|
|
||||||
await storage.setLifecyclePolicy({
|
|
||||||
rules: [{
|
|
||||||
condition: { age: 30 },
|
|
||||||
action: { type: 'SetStorageClass', storageClass: 'NEARLINE' }
|
|
||||||
}, {
|
|
||||||
condition: { age: 90 },
|
|
||||||
action: { type: 'SetStorageClass', storageClass: 'COLDLINE' }
|
|
||||||
}, {
|
|
||||||
condition: { age: 365 },
|
|
||||||
action: { type: 'SetStorageClass', storageClass: 'ARCHIVE' }
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
|
|
||||||
// Verify lifecycle policy
|
|
||||||
const policy = await storage.getLifecyclePolicy()
|
|
||||||
console.log('Lifecycle policy active:', policy.rules.length, 'rules')
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cost Calculation (500TB with Lifecycle Policy)
|
|
||||||
|
|
||||||
**Assumptions:**
|
|
||||||
- 40% of data accessed in last 30 days (Standard)
|
|
||||||
- 30% of data 30-90 days old (Nearline)
|
|
||||||
- 20% of data 90-365 days old (Coldline)
|
|
||||||
- 10% of data 365+ days old (Archive)
|
|
||||||
|
|
||||||
```
|
|
||||||
Standard (200TB): 200TB × $0.020/GB × 12 = $48,000/year
|
|
||||||
Nearline (150TB): 150TB × $0.010/GB × 12 = $18,000/year
|
|
||||||
Coldline (100TB): 100TB × $0.004/GB × 12 = $4,800/year
|
|
||||||
Archive (50TB): 50TB × $0.0012/GB × 12 = $720/year
|
|
||||||
|
|
||||||
Total Storage Cost: $71,520/year
|
|
||||||
Total with Operations: ~$76,500/year
|
|
||||||
Savings: $66,500/year (46%)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Strategy 2: Autoclass (Recommended)
|
|
||||||
|
|
||||||
### Setup: Automatic Class Optimization
|
|
||||||
|
|
||||||
**Best for**: Maximum automation, unpredictable access patterns
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Enable Autoclass for automatic tier management
|
|
||||||
await storage.enableAutoclass({
|
|
||||||
terminalStorageClass: 'ARCHIVE' // Optional: Set lowest tier
|
|
||||||
})
|
|
||||||
|
|
||||||
// Benefits:
|
|
||||||
// - Automatically moves objects between classes based on access patterns
|
|
||||||
// - No data retrieval delays (unlike AWS Glacier)
|
|
||||||
// - Transparent tier transitions within 24 hours
|
|
||||||
// - Supports all storage classes including Archive
|
|
||||||
// - No extra monitoring fees
|
|
||||||
```
|
|
||||||
|
|
||||||
### How Autoclass Works
|
|
||||||
|
|
||||||
1. **Initial Placement**: New objects start in Standard class
|
|
||||||
2. **Automatic Demotion**: Objects move to Nearline (30 days) → Coldline (90 days) → Archive (365 days)
|
|
||||||
3. **Automatic Promotion**: Accessed objects move back to Standard class
|
|
||||||
4. **Access-Pattern Learning**: Uses 90-day access history for optimization
|
|
||||||
|
|
||||||
### Cost Calculation (500TB with Autoclass)
|
|
||||||
|
|
||||||
**Realistic distribution after 1 year:**
|
|
||||||
- 10% Standard (hot data, frequently accessed)
|
|
||||||
- 15% Nearline (warm data)
|
|
||||||
- 35% Coldline (cool data)
|
|
||||||
- 40% Archive (cold data)
|
|
||||||
|
|
||||||
```
|
|
||||||
Standard (50TB): 50TB × $0.020/GB × 12 = $12,000/year
|
|
||||||
Nearline (75TB): 75TB × $0.010/GB × 12 = $9,000/year
|
|
||||||
Coldline (175TB): 175TB × $0.004/GB × 12 = $8,400/year
|
|
||||||
Archive (200TB): 200TB × $0.0012/GB × 12 = $2,880/year
|
|
||||||
|
|
||||||
Total Storage Cost: $32,280/year
|
|
||||||
Total with Operations: ~$37,000/year
|
|
||||||
Savings vs Standard: $106,000/year (74%)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Strategy 3: Hybrid Approach (Maximum Savings)
|
|
||||||
|
|
||||||
### Setup: Autoclass + Lifecycle Policies
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Enable Autoclass for vectors (frequently searched)
|
|
||||||
await storage.enableAutoclass({
|
|
||||||
terminalStorageClass: 'COLDLINE' // Don't archive vectors deeply
|
|
||||||
})
|
|
||||||
|
|
||||||
// Set lifecycle policy for metadata (less frequently accessed)
|
|
||||||
await storage.setLifecyclePolicy({
|
|
||||||
rules: [{
|
|
||||||
condition: { age: 30, matchesPrefix: ['entities/nouns/metadata/'] },
|
|
||||||
action: { type: 'SetStorageClass', storageClass: 'NEARLINE' }
|
|
||||||
}, {
|
|
||||||
condition: { age: 60, matchesPrefix: ['entities/nouns/metadata/'] },
|
|
||||||
action: { type: 'SetStorageClass', storageClass: 'COLDLINE' }
|
|
||||||
}, {
|
|
||||||
condition: { age: 180, matchesPrefix: ['entities/nouns/metadata/'] },
|
|
||||||
action: { type: 'SetStorageClass', storageClass: 'ARCHIVE' }
|
|
||||||
}, {
|
|
||||||
condition: { age: 730, matchesPrefix: ['_system/'] },
|
|
||||||
action: { type: 'Delete' } // Delete old system files after 2 years
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cost Calculation (500TB Hybrid Approach)
|
|
||||||
|
|
||||||
**Vectors (300TB with Autoclass):**
|
|
||||||
```
|
|
||||||
Standard (30TB): 30TB × $0.020/GB × 12 = $7,200/year
|
|
||||||
Nearline (45TB): 45TB × $0.010/GB × 12 = $5,400/year
|
|
||||||
Coldline (225TB): 225TB × $0.004/GB × 12 = $10,800/year
|
|
||||||
Subtotal: $23,400/year
|
|
||||||
```
|
|
||||||
|
|
||||||
**Metadata (200TB with Lifecycle Policy):**
|
|
||||||
```
|
|
||||||
Standard (30TB): 30TB × $0.020/GB × 12 = $7,200/year
|
|
||||||
Nearline (40TB): 40TB × $0.010/GB × 12 = $4,800/year
|
|
||||||
Coldline (80TB): 80TB × $0.004/GB × 12 = $3,840/year
|
|
||||||
Archive (50TB): 50TB × $0.0012/GB × 12 = $720/year
|
|
||||||
Subtotal: $16,560/year
|
|
||||||
```
|
|
||||||
|
|
||||||
**Total Cost: $39,960/year + operations (~$45,000/year total)**
|
|
||||||
**Savings vs Standard: $98,000/year (69%)**
|
|
||||||
|
|
||||||
## Strategy 4: Aggressive Archival (Maximum Savings)
|
|
||||||
|
|
||||||
### Setup: Fast Archival for Cold Data
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
await storage.setLifecyclePolicy({
|
|
||||||
rules: [{
|
|
||||||
condition: { age: 14 },
|
|
||||||
action: { type: 'SetStorageClass', storageClass: 'NEARLINE' }
|
|
||||||
}, {
|
|
||||||
condition: { age: 30 },
|
|
||||||
action: { type: 'SetStorageClass', storageClass: 'COLDLINE' }
|
|
||||||
}, {
|
|
||||||
condition: { age: 90 },
|
|
||||||
action: { type: 'SetStorageClass', storageClass: 'ARCHIVE' }
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
|
|
||||||
// Note: Archive class has 365-day minimum storage duration
|
|
||||||
// Early deletion incurs pro-rated charges for remaining days
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cost Calculation (500TB Aggressive Archival)
|
|
||||||
|
|
||||||
**After 1 year:**
|
|
||||||
```
|
|
||||||
Standard (25TB): 25TB × $0.020/GB × 12 = $6,000/year
|
|
||||||
Nearline (50TB): 50TB × $0.010/GB × 12 = $6,000/year
|
|
||||||
Coldline (75TB): 75TB × $0.004/GB × 12 = $3,600/year
|
|
||||||
Archive (350TB): 350TB × $0.0012/GB × 12 = $5,040/year
|
|
||||||
|
|
||||||
Total Storage Cost: $20,640/year
|
|
||||||
Total with Operations: ~$25,500/year
|
|
||||||
Savings: $117,500/year (82%)
|
|
||||||
|
|
||||||
Warning: High retrieval costs if archived data is accessed frequently
|
|
||||||
```
|
|
||||||
|
|
||||||
## Comparison Table: All Strategies
|
|
||||||
|
|
||||||
| Strategy | Annual Cost | Savings | Best For |
|
|
||||||
|----------|-------------|---------|----------|
|
|
||||||
| **No Optimization** | $143,000 | 0% | N/A |
|
|
||||||
| **Lifecycle Policy** | $76,500 | 46% | Predictable patterns |
|
|
||||||
| **Autoclass** | $37,000 | **74%** | **Recommended** |
|
|
||||||
| **Hybrid Approach** | $45,000 | 69% | Fine-grained control |
|
|
||||||
| **Aggressive Archival** | $25,500 | 82% | Cold data, compliance |
|
|
||||||
|
|
||||||
## Autoclass vs Lifecycle Policies
|
|
||||||
|
|
||||||
| Feature | Autoclass | Lifecycle Policies |
|
|
||||||
|---------|-----------|-------------------|
|
|
||||||
| **Automation** | Fully automatic | Rule-based |
|
|
||||||
| **Access-pattern learning** | Yes (90-day history) | No |
|
|
||||||
| **Promotion to Standard** | Automatic on access | Manual only |
|
|
||||||
| **Terminal class** | Configurable | Fixed by rules |
|
|
||||||
| **Complexity** | Single command | Multiple rules |
|
|
||||||
| **Cost** | Lower (smarter) | Moderate |
|
|
||||||
| **Best for** | Unpredictable patterns | Predictable patterns |
|
|
||||||
|
|
||||||
## Batch Delete Operations
|
|
||||||
|
|
||||||
### Efficient Cleanup
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Batch delete (100 objects per request for GCS)
|
|
||||||
const idsToDelete = [/* array of entity IDs */]
|
|
||||||
|
|
||||||
const paths = idsToDelete.flatMap(id => {
|
|
||||||
const shard = id.substring(0, 2)
|
|
||||||
return [
|
|
||||||
`entities/nouns/vectors/${shard}/${id}.json`,
|
|
||||||
`entities/nouns/metadata/${shard}/${id}.json`
|
|
||||||
]
|
|
||||||
})
|
|
||||||
|
|
||||||
// Batch delete
|
|
||||||
await storage.batchDelete(paths)
|
|
||||||
|
|
||||||
// Cost impact:
|
|
||||||
// - Individual deletes: 1M operations × $0.005 per 10k = $500
|
|
||||||
// - Batch deletes: 10k batches × $0.005 = $5 (100x cheaper!)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Monitoring and Management
|
|
||||||
|
|
||||||
### Check Autoclass Status
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const status = await storage.getAutoclassStatus()
|
|
||||||
console.log('Autoclass enabled:', status.enabled)
|
|
||||||
console.log('Terminal class:', status.terminalStorageClass)
|
|
||||||
|
|
||||||
// Example output:
|
|
||||||
// {
|
|
||||||
// enabled: true,
|
|
||||||
// terminalStorageClass: 'ARCHIVE',
|
|
||||||
// toggleTime: '2025-01-15T10:30:00Z'
|
|
||||||
// }
|
|
||||||
```
|
|
||||||
|
|
||||||
### Disable Autoclass
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Disable Autoclass (objects remain in current class)
|
|
||||||
await storage.disableAutoclass()
|
|
||||||
```
|
|
||||||
|
|
||||||
### Get Current Lifecycle Policy
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const policy = await storage.getLifecyclePolicy()
|
|
||||||
console.log('Active rules:', policy.rules)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Remove Lifecycle Policy
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
await storage.removeLifecyclePolicy()
|
|
||||||
```
|
|
||||||
|
|
||||||
## GCS Cloud Console Monitoring
|
|
||||||
|
|
||||||
### Track Your Savings
|
|
||||||
|
|
||||||
1. **Storage Browser** → View storage class distribution
|
|
||||||
2. **Monitoring** → Create custom dashboards for storage metrics
|
|
||||||
3. **Cloud Logging** → Track class transition events
|
|
||||||
4. **Cloud Billing Reports** → Compare storage costs month-over-month
|
|
||||||
|
|
||||||
### Expected Metrics After 6 Months (Autoclass)
|
|
||||||
|
|
||||||
```
|
|
||||||
Standard: 10-15% of total data
|
|
||||||
Nearline: 15-20%
|
|
||||||
Coldline: 30-40%
|
|
||||||
Archive: 30-45%
|
|
||||||
|
|
||||||
Monthly cost trend: Decreasing 8-12% per month as data ages into cheaper classes
|
|
||||||
```
|
|
||||||
|
|
||||||
## Best Practices
|
|
||||||
|
|
||||||
1. ✅ **Start with Autoclass** - Simplest and most effective
|
|
||||||
2. ✅ **Set terminal class to ARCHIVE** for maximum savings
|
|
||||||
3. ✅ **Use lifecycle policies for system files** - Predictable archival
|
|
||||||
4. ✅ **Monitor class distribution** monthly in Cloud Console
|
|
||||||
5. ✅ **Use batch operations** for deletions - 100x cheaper
|
|
||||||
6. ✅ **Enable Object Lifecycle Management logging** for auditing
|
|
||||||
7. ✅ **Consider Turbo Replication** for multi-region redundancy
|
|
||||||
|
|
||||||
## Troubleshooting
|
|
||||||
|
|
||||||
### Issue: Data not transitioning to cheaper classes
|
|
||||||
|
|
||||||
**Solution:**
|
|
||||||
```typescript
|
|
||||||
// Check Autoclass status
|
|
||||||
const status = await storage.getAutoclassStatus()
|
|
||||||
if (!status.enabled) {
|
|
||||||
await storage.enableAutoclass({ terminalStorageClass: 'ARCHIVE' })
|
|
||||||
}
|
|
||||||
|
|
||||||
// Autoclass requires 24-48 hours for initial transitions
|
|
||||||
```
|
|
||||||
|
|
||||||
### Issue: High retrieval costs
|
|
||||||
|
|
||||||
**Solution:**
|
|
||||||
- GCS has lower retrieval fees than AWS Glacier ($0.01-0.05/GB vs $0.01-0.20/GB)
|
|
||||||
- Autoclass automatically promotes frequently accessed objects to Standard
|
|
||||||
- Use Coldline for occasional access (better than Archive)
|
|
||||||
|
|
||||||
### Issue: Minimum storage duration charges
|
|
||||||
|
|
||||||
**Solution:**
|
|
||||||
- Nearline: 30-day minimum
|
|
||||||
- Coldline: 90-day minimum
|
|
||||||
- Archive: 365-day minimum
|
|
||||||
- Early deletion incurs pro-rated charges
|
|
||||||
- Use Autoclass to avoid manual class changes that might trigger early deletion fees
|
|
||||||
|
|
||||||
## ADC (Application Default Credentials) Setup
|
|
||||||
|
|
||||||
### Production Best Practice
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Use ADC instead of service account key file
|
|
||||||
const storage = new GcsStorage({
|
|
||||||
bucketName: 'my-brainy-data'
|
|
||||||
// No keyFilename needed - uses ADC automatically
|
|
||||||
})
|
|
||||||
|
|
||||||
// ADC authentication order:
|
|
||||||
// 1. GOOGLE_APPLICATION_CREDENTIALS environment variable
|
|
||||||
// 2. gcloud CLI credentials
|
|
||||||
// 3. Compute Engine/Cloud Run service account
|
|
||||||
```
|
|
||||||
|
|
||||||
### Set up ADC
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# For local development
|
|
||||||
gcloud auth application-default login
|
|
||||||
|
|
||||||
# For production (use service account)
|
|
||||||
export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account.json"
|
|
||||||
|
|
||||||
# For Cloud Run/GKE/Compute Engine (automatic)
|
|
||||||
# Service account is automatically available
|
|
||||||
```
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
**Recommended Strategy for Most Use Cases:**
|
|
||||||
- **Autoclass** for automatic optimization (simplest, most effective)
|
|
||||||
- **Lifecycle policies** for predictable archival (system files, logs)
|
|
||||||
- **Batch operations** for efficient cleanup
|
|
||||||
|
|
||||||
**Expected Savings:**
|
|
||||||
- **Year 1**: 50-60% reduction in storage costs
|
|
||||||
- **Year 2+**: 70-80% reduction as more data ages into archive classes
|
|
||||||
- **Long-term**: 85-90% reduction for mature datasets
|
|
||||||
|
|
||||||
**500TB Example (Autoclass):**
|
|
||||||
- Before: $143,000/year
|
|
||||||
- After: $37,000/year
|
|
||||||
- **Savings: $106,000/year (74%)**
|
|
||||||
|
|
||||||
**1PB Example (Autoclass):**
|
|
||||||
- Before: $286,000/year
|
|
||||||
- After: $74,000/year
|
|
||||||
- **Savings: $212,000/year (74%)**
|
|
||||||
|
|
||||||
**10PB Example (Autoclass):**
|
|
||||||
- Before: $2,860,000/year
|
|
||||||
- After: $740,000/year
|
|
||||||
- **Savings: $2,120,000/year (74%)**
|
|
||||||
|
|
||||||
## Strategy 5: HNS Buckets for Write-Heavy Workloads
|
|
||||||
|
|
||||||
### When to Use HNS (Hierarchical Namespace)
|
|
||||||
|
|
||||||
If you experience HTTP 429 rate limit errors during `brain.add()` operations (especially with many metadata fields), GCS Hierarchical Namespace buckets provide significantly higher write throughput with zero code changes.
|
|
||||||
|
|
||||||
### Why It Helps
|
|
||||||
|
|
||||||
Standard GCS buckets enforce ~1 write/sec per object path. Brainy's metadata index writes multiple chunk and sparse index files per `brain.add()` call, which can exceed these limits during burst writes.
|
|
||||||
|
|
||||||
HNS buckets provide:
|
|
||||||
- **8x higher initial QPS** (8,000 writes/sec vs 1,000 for standard buckets)
|
|
||||||
- **Better handling of hierarchical key patterns** (which brainy uses for sharded storage)
|
|
||||||
- **No code changes required** — create a new HNS-enabled bucket and point brainy at it
|
|
||||||
|
|
||||||
### Setup
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Create HNS-enabled bucket
|
|
||||||
gcloud storage buckets create gs://my-brainy-hns \
|
|
||||||
--location=us-central1 \
|
|
||||||
--uniform-bucket-level-access \
|
|
||||||
--enable-hierarchical-namespace
|
|
||||||
```
|
|
||||||
|
|
||||||
Then configure brainy to use the new bucket:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const storage = new GcsStorage({
|
|
||||||
bucketName: 'my-brainy-hns'
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Trade-offs
|
|
||||||
|
|
||||||
- HNS buckets do not support object versioning (irrelevant for brainy — uses its own COW versioning)
|
|
||||||
- HNS is available in most GCS regions
|
|
||||||
- Pricing is the same as standard buckets
|
|
||||||
|
|
||||||
### Alternative: Cloud Run with Filestore/NFS
|
|
||||||
|
|
||||||
For the highest write throughput with zero rate limiting, see the [Cloud Run Filestore Guide](cloud-run-filestore-guide.md) — mount a Filestore NFS volume and use brainy's FileSystem adapter instead of GCS.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Last Updated**: 2025-10-17
|
|
||||||
**Cloud Provider**: Google Cloud Storage
|
|
||||||
|
|
@ -1,3 +1,16 @@
|
||||||
|
---
|
||||||
|
title: Transactions & Atomicity
|
||||||
|
slug: guides/transactions
|
||||||
|
public: true
|
||||||
|
category: guides
|
||||||
|
template: guide
|
||||||
|
order: 10
|
||||||
|
description: How Brainy keeps every write atomic — automatic per-operation transactions with rollback, and brain.transact() for atomic multi-write batches with compare-and-swap.
|
||||||
|
next:
|
||||||
|
- concepts/consistency-model
|
||||||
|
- guides/optimistic-concurrency
|
||||||
|
---
|
||||||
|
|
||||||
# Transaction System
|
# Transaction System
|
||||||
|
|
||||||
**Status:** ✅ Production Ready
|
**Status:** ✅ Production Ready
|
||||||
|
|
@ -10,19 +23,19 @@ Brainy's transaction system provides **atomic operations** with automatic rollba
|
||||||
|
|
||||||
- **Atomicity**: All operations succeed or all rollback
|
- **Atomicity**: All operations succeed or all rollback
|
||||||
- **Consistency**: Indexes and storage remain consistent
|
- **Consistency**: Indexes and storage remain consistent
|
||||||
- **Automatic**: Transparently used by all `brain.add()`, `brain.update()`, `brain.delete()`, and `brain.relate()` operations
|
- **Automatic**: Transparently used by all `brain.add()`, `brain.update()`, `brain.remove()`, and `brain.relate()` operations
|
||||||
- **Compatible**: Works seamlessly with COW, sharding, type-aware storage, and distributed storage
|
- **Composable**: `brain.transact()` runs a multi-write batch through the same machinery as exactly one atomic commit
|
||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
```
|
```
|
||||||
User Code (brain.add(), brain.update(), etc.)
|
User Code (brain.add(), brain.update(), brain.transact(), etc.)
|
||||||
↓
|
↓
|
||||||
Transaction Manager (orchestration)
|
Transaction Manager (orchestration)
|
||||||
↓
|
↓
|
||||||
Operations (SaveNounMetadataOperation, SaveNounOperation, etc.)
|
Operations (SaveNounMetadataOperation, SaveNounOperation, etc.)
|
||||||
↓
|
↓
|
||||||
Storage Adapter (COW, sharding, type-aware routing)
|
Storage Adapter (sharding, ID-first routing)
|
||||||
```
|
```
|
||||||
|
|
||||||
### How It Works
|
### How It Works
|
||||||
|
|
@ -74,31 +87,41 @@ class SaveNounMetadataOperation {
|
||||||
|
|
||||||
## Compatibility with Advanced Features
|
## Compatibility with Advanced Features
|
||||||
|
|
||||||
### COW (Copy-on-Write)
|
### Multi-Write Batches: `brain.transact()`
|
||||||
|
|
||||||
✅ **Fully Compatible**
|
✅ **The 8.0 path for atomic multi-entity writes**
|
||||||
|
|
||||||
Transactions work transparently with COW branches:
|
Single-operation methods each commit their own transaction. When several
|
||||||
|
writes must succeed or fail **together**, use `brain.transact()` — a
|
||||||
|
declarative batch that commits as exactly one generation, with optional
|
||||||
|
whole-store compare-and-swap and durable transaction metadata:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Create branch
|
const db = await brain.transact([
|
||||||
await brain.cow.createBranch('feature-branch')
|
{ op: 'add', id: orderId, type: NounType.Document, subtype: 'order', data: 'Order #1042' },
|
||||||
await brain.cow.checkout('feature-branch')
|
{ op: 'update', id: customerId, metadata: { lastOrderAt: Date.now() }, ifRev: customer._rev },
|
||||||
|
{ op: 'relate', from: customerId, to: orderId, type: VerbType.Creates, subtype: 'purchase' }
|
||||||
|
], { meta: { author: 'order-service' } })
|
||||||
|
|
||||||
// Add entity (uses transaction on this branch)
|
db.receipt.ids // resolved id per operation, in input order
|
||||||
const id = await brain.add({
|
|
||||||
data: { name: 'Feature Entity' },
|
|
||||||
type: NounType.Thing
|
|
||||||
})
|
|
||||||
|
|
||||||
// On rollback: Branch remains clean, no partial commits
|
|
||||||
```
|
```
|
||||||
|
|
||||||
**How It Works:**
|
**How It Works:**
|
||||||
- Transactions use `StorageAdapter` interface
|
- The batch executes through the same TransactionManager as single
|
||||||
- COW operates at storage layer (refManager, blobStorage, commitLog)
|
operations, wrapped in the generational commit protocol: before-images are
|
||||||
- Branch isolation prevents cross-branch contamination
|
staged and fsynced first, and the atomic manifest rename is the commit
|
||||||
- Rollback = discard uncommitted changes (COW makes this trivial)
|
point — a crash anywhere before it rolls back to the exact
|
||||||
|
pre-transaction bytes.
|
||||||
|
- Per-entity `ifRev` and whole-store `ifAtGeneration` provide
|
||||||
|
compare-and-swap at two granularities; any conflict rejects the entire
|
||||||
|
batch before anything is staged.
|
||||||
|
- The returned `Db` is a pinned, snapshot-isolated view of the committed
|
||||||
|
state.
|
||||||
|
|
||||||
|
See the **[consistency model](concepts/consistency-model.md)** for the
|
||||||
|
full guarantees (snapshot isolation, time travel, snapshots) and
|
||||||
|
**[Snapshots & Time Travel](guides/snapshots-and-time-travel.md)** for
|
||||||
|
recipes.
|
||||||
|
|
||||||
### Sharding
|
### Sharding
|
||||||
|
|
||||||
|
|
@ -154,37 +177,27 @@ await brain.update({
|
||||||
- Storage layer uses O(1) ID-first path construction
|
- Storage layer uses O(1) ID-first path construction
|
||||||
- No type cache needed (removed in a previous version)
|
- No type cache needed (removed in a previous version)
|
||||||
- Type counters adjusted on commit/rollback
|
- Type counters adjusted on commit/rollback
|
||||||
- 40x faster on cloud storage (eliminates 42-type search)
|
- 40x faster path lookups (eliminates 42-type search)
|
||||||
|
|
||||||
### Distributed Storage
|
### Storage Adapter Interface
|
||||||
|
|
||||||
✅ **Fully Compatible**
|
✅ **Fully Compatible**
|
||||||
|
|
||||||
Transactions work with distributed/remote storage:
|
Transactions go through the `StorageAdapter` interface, so both shipped adapters (filesystem, memory) and any custom plugin adapter inherit the same atomicity guarantees:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Works with S3, Azure, GCS, etc.
|
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: {
|
storage: { type: 'filesystem', path: './data' }
|
||||||
type: 's3Compatible',
|
|
||||||
config: { /* S3 config */ }
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
|
|
||||||
// Transactions ensure atomicity at write coordinator level
|
await brain.add({ data: { name: 'Entity' }, type: NounType.Thing })
|
||||||
await brain.add({ data: { name: 'Remote Entity' }, type: NounType.Thing })
|
|
||||||
```
|
```
|
||||||
|
|
||||||
**How It Works:**
|
**How It Works:**
|
||||||
- Transactions operate through `StorageAdapter` interface
|
- Transactions operate through `StorageAdapter` interface
|
||||||
- Remote storage adapters implement same interface
|
- Custom adapters registered via the plugin system implement the same interface
|
||||||
- Atomicity guaranteed at write-coordinator level
|
- Atomicity guaranteed at the write-coordinator level
|
||||||
- Read-after-write consistency maintained
|
- Read-after-write consistency maintained inside a single Brainy process
|
||||||
|
|
||||||
**Design Philosophy:**
|
|
||||||
- **Single-node writes** (most common): Fully atomic ✅
|
|
||||||
- **Distributed reads + centralized writes**: Transactions on primary ✅
|
|
||||||
- **Multi-primary**: Transactions per-instance, cross-instance via coordinator ✅
|
|
||||||
|
|
||||||
## Examples
|
## Examples
|
||||||
|
|
||||||
|
|
@ -283,7 +296,7 @@ await brain.relate({
|
||||||
})
|
})
|
||||||
|
|
||||||
// Delete person (atomic - deletes entity + relationships)
|
// Delete person (atomic - deletes entity + relationships)
|
||||||
await brain.delete(personId)
|
await brain.remove(personId)
|
||||||
|
|
||||||
// If delete fails, both entity and relationships remain
|
// If delete fails, both entity and relationships remain
|
||||||
```
|
```
|
||||||
|
|
@ -316,37 +329,31 @@ try {
|
||||||
|
|
||||||
### Transaction Overhead
|
### Transaction Overhead
|
||||||
|
|
||||||
**MEASURED Performance Impact:**
|
**What a transaction costs:**
|
||||||
- Average overhead: ~2-5ms per transaction (measured: `tests/transaction/transaction.bench.ts`)
|
- A typical single-operation write wraps 2-8 operations (metadata + data + indexes) in one transaction
|
||||||
- Operations per transaction: 2-8 (metadata + data + indexes)
|
- The overhead is bookkeeping (operation objects + undo state), not extra I/O on the success path
|
||||||
- Rollback cost: ~1-3ms (restore previous state)
|
- Rollback cost is proportional to the operations already applied (each is undone in reverse order)
|
||||||
|
|
||||||
**Optimization:**
|
**Optimization:**
|
||||||
- Operations executed sequentially (not parallel) for consistency
|
- Operations executed sequentially (not parallel) for consistency
|
||||||
- Rollback only happens on failure (success path is fast)
|
- Rollback only happens on failure (success path is fast)
|
||||||
- Index updates batched within transaction
|
- Index updates batched within transaction
|
||||||
|
|
||||||
### Statistics and Monitoring
|
### Auditing Committed Batches
|
||||||
|
|
||||||
|
Every committed `brain.transact()` batch is recorded in the transaction
|
||||||
|
log, newest first:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Get transaction statistics
|
await brain.transact(ops, { meta: { author: 'import-job' } })
|
||||||
const stats = brain.transactionManager?.getStats()
|
|
||||||
console.log(stats)
|
const entries = await brain.transactionLog({ limit: 10 })
|
||||||
// {
|
// [{ generation: 1042, timestamp: 1765432100000, meta: { author: 'import-job' } }]
|
||||||
// totalTransactions: 1234,
|
|
||||||
// successfulTransactions: 1200,
|
|
||||||
// failedTransactions: 34,
|
|
||||||
// rollbacks: 34,
|
|
||||||
// averageOperationsPerTransaction: 4.2
|
|
||||||
// }
|
|
||||||
```
|
```
|
||||||
|
|
||||||
**Metrics Available:**
|
Single-operation writes advance the generation counter but do not append
|
||||||
- `totalTransactions`: Total number of transactions executed
|
log entries — see the [consistency model](concepts/consistency-model.md)
|
||||||
- `successfulTransactions`: Number of successful commits
|
for the history-granularity contract.
|
||||||
- `failedTransactions`: Number of rollbacks
|
|
||||||
- `rollbacks`: Total rollback count
|
|
||||||
- `averageOperationsPerTransaction`: Average operations per transaction
|
|
||||||
|
|
||||||
## Best Practices
|
## Best Practices
|
||||||
|
|
||||||
|
|
@ -356,7 +363,7 @@ console.log(stats)
|
||||||
// ✅ Recommended: Use Brainy's API (transactions automatic)
|
// ✅ Recommended: Use Brainy's API (transactions automatic)
|
||||||
await brain.add({ data, type })
|
await brain.add({ data, type })
|
||||||
await brain.update({ id, data })
|
await brain.update({ id, data })
|
||||||
await brain.delete(id)
|
await brain.remove(id)
|
||||||
|
|
||||||
// ❌ Avoid: Direct storage access bypasses transactions
|
// ❌ Avoid: Direct storage access bypasses transactions
|
||||||
await brain.storage.saveNoun(noun) // No transaction protection
|
await brain.storage.saveNoun(noun) // No transaction protection
|
||||||
|
|
@ -386,35 +393,34 @@ if (!isValidVector(vector, brain.dimension)) {
|
||||||
await brain.add({ data, type, vector })
|
await brain.add({ data, type, vector })
|
||||||
```
|
```
|
||||||
|
|
||||||
### 4. Monitor Transaction Statistics
|
### 4. Batch Related Writes with `transact()`
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// ✅ Recommended: Monitor in production
|
// ✅ Recommended: writes that must land together go in one batch
|
||||||
setInterval(() => {
|
await brain.transact([
|
||||||
const stats = brain.transactionManager?.getStats()
|
{ op: 'add', id: orderId, type: NounType.Document, subtype: 'order', data: 'Order #1042' },
|
||||||
if (stats) {
|
{ op: 'relate', from: customerId, to: orderId, type: VerbType.Creates, subtype: 'purchase' }
|
||||||
const failureRate = stats.failedTransactions / stats.totalTransactions
|
])
|
||||||
if (failureRate > 0.05) { // > 5% failure rate
|
|
||||||
console.warn('High transaction failure rate:', failureRate)
|
// ❌ Avoid: sequential single operations when partial application is unacceptable
|
||||||
}
|
const id = await brain.add({ ... }) // commits alone
|
||||||
}
|
await brain.relate({ ... }) // a crash here leaves the entity unlinked
|
||||||
}, 60000) // Check every minute
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### 5. Understand Atomicity Guarantees
|
### 5. Understand Atomicity Guarantees
|
||||||
|
|
||||||
**What Transactions GUARANTEE:**
|
**What Transactions GUARANTEE:**
|
||||||
- ✅ Atomicity within single Brainy instance
|
- ✅ Atomicity within a single Brainy process
|
||||||
- ✅ Consistent state across all indexes
|
- ✅ Consistent state across all indexes
|
||||||
- ✅ Automatic rollback on failure
|
- ✅ Automatic rollback on failure
|
||||||
- ✅ Works with all storage adapters (local, remote, COW, sharded)
|
- ✅ Works with all storage adapters (filesystem, memory, custom plugin adapters)
|
||||||
|
|
||||||
**What Transactions DON'T Provide:**
|
**What Transactions DON'T Provide:**
|
||||||
- ❌ Two-phase commit across multiple Brainy instances
|
- ❌ Two-phase commit across multiple Brainy instances
|
||||||
- ❌ Distributed locking across nodes
|
- ❌ Distributed locking across processes
|
||||||
- ❌ Cross-datacenter ACID guarantees
|
- ❌ Cross-datacenter ACID guarantees
|
||||||
|
|
||||||
**Design:** Transactions ensure atomicity at the **write coordinator level**. For multi-instance scenarios, use `DistributedCoordinator`.
|
**Design:** Transactions ensure atomicity at the **write-coordinator level** inside one process. Cross-instance coordination, if you need it, lives in your service layer.
|
||||||
|
|
||||||
## Testing Transactions
|
## Testing Transactions
|
||||||
|
|
||||||
|
|
@ -450,16 +456,18 @@ describe('Transaction Tests', () => {
|
||||||
### Integration Tests
|
### Integration Tests
|
||||||
|
|
||||||
See `tests/transaction/integration/` for comprehensive integration tests covering:
|
See `tests/transaction/integration/` for comprehensive integration tests covering:
|
||||||
- COW integration (`cow-transactions.test.ts`)
|
|
||||||
- Sharding integration (`sharding-transactions.test.ts`)
|
- Sharding integration (`sharding-transactions.test.ts`)
|
||||||
- TypeAware integration (`typeaware-transactions.test.ts`)
|
- Type-aware integration (`typeaware-transactions.test.ts`)
|
||||||
- Distributed storage integration (`distributed-transactions.test.ts`)
|
|
||||||
|
The atomicity guarantees of `brain.transact()` — including crash recovery
|
||||||
|
through the real recovery path — are proven in
|
||||||
|
`tests/integration/db-mvcc.test.ts`.
|
||||||
|
|
||||||
## Troubleshooting
|
## Troubleshooting
|
||||||
|
|
||||||
### High Rollback Rate
|
### High Rollback Rate
|
||||||
|
|
||||||
**Symptom:** `failedTransactions` / `totalTransactions` > 5%
|
**Symptom:** a high share of writes throw and roll back
|
||||||
|
|
||||||
**Possible Causes:**
|
**Possible Causes:**
|
||||||
1. Invalid vector dimensions
|
1. Invalid vector dimensions
|
||||||
|
|
@ -489,21 +497,6 @@ See `tests/transaction/integration/` for comprehensive integration tests coverin
|
||||||
- Disable unused indexes
|
- Disable unused indexes
|
||||||
- Use SSD storage
|
- Use SSD storage
|
||||||
|
|
||||||
### Transaction Statistics Missing
|
|
||||||
|
|
||||||
**Symptom:** `brain.transactionManager?.getStats()` returns `undefined`
|
|
||||||
|
|
||||||
**Cause:** TransactionManager not initialized
|
|
||||||
|
|
||||||
**Solution:**
|
|
||||||
```typescript
|
|
||||||
// Ensure Brainy is initialized
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Then access stats
|
|
||||||
const stats = brain.transactionManager?.getStats()
|
|
||||||
```
|
|
||||||
|
|
||||||
## Architecture Details
|
## Architecture Details
|
||||||
|
|
||||||
### Transaction Lifecycle
|
### Transaction Lifecycle
|
||||||
|
|
@ -552,32 +545,23 @@ interface StorageAdapter {
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
**Key Insight:** All storage adapters (filesystem, S3, Azure, GCS, memory) implement this interface. Transactions work with **any** storage adapter automatically.
|
**Key Insight:** Both shipped storage adapters (filesystem, memory) — and any custom plugin adapter — implement this interface. Transactions work with **any** storage adapter automatically.
|
||||||
|
|
||||||
## Additional Resources
|
## Additional Resources
|
||||||
|
|
||||||
- **Unit Tests:** `tests/transaction/transaction.test.ts` (36 passing tests)
|
- **Unit Tests:** `tests/transaction/Transaction.test.ts`, `tests/transaction/TransactionManager.test.ts`
|
||||||
- **Integration Tests:** `tests/transaction/integration/` (35 test scenarios)
|
- **Integration Tests:** `tests/transaction/integration/`
|
||||||
- **Compatibility Analysis:** `.strategy/TRANSACTION_COMPATIBILITY_ANALYSIS.md` (internal)
|
- **MVCC Proofs:** `tests/integration/db-mvcc.test.ts` (atomicity, CAS, crash recovery for `brain.transact()`)
|
||||||
- **Performance Benchmarks:** `tests/transaction/transaction.bench.ts`
|
- **Consistency Model:** [docs/concepts/consistency-model.md](concepts/consistency-model.md)
|
||||||
|
|
||||||
## Version History
|
|
||||||
|
|
||||||
- Initial transaction system release
|
|
||||||
- Atomic operations with rollback
|
|
||||||
- Compatible with COW, sharding, type-aware, distributed
|
|
||||||
- 36/36 unit tests passing
|
|
||||||
- 35 integration test scenarios
|
|
||||||
|
|
||||||
## Summary
|
## Summary
|
||||||
|
|
||||||
Brainy's transaction system provides **production-ready atomic operations** with automatic rollback. It works transparently with all Brainy APIs and is fully compatible with advanced features like COW, sharding, type-aware storage, and distributed storage.
|
Brainy's transaction system provides **production-ready atomic operations** with automatic rollback. Every single-operation write is transactional out of the box, and `brain.transact()` extends the same guarantee to multi-write batches — one atomic commit, with compare-and-swap and durable transaction metadata.
|
||||||
|
|
||||||
**Key Takeaways:**
|
**Key Takeaways:**
|
||||||
- ✅ **Automatic**: No manual transaction management needed
|
- ✅ **Automatic**: No manual transaction management needed for single operations
|
||||||
- ✅ **Atomic**: All operations succeed or all rollback
|
- ✅ **Atomic**: All operations succeed or all rollback — per operation and per `transact()` batch
|
||||||
- ✅ **Compatible**: Works with all storage adapters and features
|
- ✅ **Compatible**: Works with all storage adapters and features
|
||||||
- ✅ **Production-Ready**: Tested with 71 test scenarios (36 unit + 35 integration)
|
- ✅ **Coordinated**: Per-entity `ifRev` and whole-store `ifAtGeneration` CAS reject conflicting batches before anything is staged
|
||||||
- ✅ **Performant**: ~2-5ms overhead per transaction (measured)
|
|
||||||
|
|
||||||
Start using transactions today - they're already built into `brain.add()`, `brain.update()`, `brain.delete()`, and `brain.relate()`!
|
Start using transactions today - they're already built into `brain.add()`, `brain.update()`, `brain.remove()`, and `brain.relate()` — and reach for `brain.transact()` whenever several writes must land together.
|
||||||
|
|
|
||||||
|
|
@ -162,7 +162,7 @@ const brain = new Brainy({
|
||||||
|
|
||||||
3. **Test with exact match**
|
3. **Test with exact match**
|
||||||
```typescript
|
```typescript
|
||||||
const results = await brain.search("test content") // Exact text
|
const results = await brain.find("test content") // Exact text
|
||||||
console.log('Exact match results:', results)
|
console.log('Exact match results:', results)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -175,12 +175,13 @@ const brain = new Brainy({
|
||||||
1. **Add more context to queries**
|
1. **Add more context to queries**
|
||||||
```typescript
|
```typescript
|
||||||
// Instead of: "cat"
|
// Instead of: "cat"
|
||||||
const results = await brain.search("domestic cat animal pet")
|
const results = await brain.find("domestic cat animal pet")
|
||||||
```
|
```
|
||||||
|
|
||||||
2. **Use metadata filtering**
|
2. **Use metadata filtering**
|
||||||
```typescript
|
```typescript
|
||||||
const results = await brain.search("animals", {
|
const results = await brain.find({
|
||||||
|
query: "animals",
|
||||||
where: { category: "pets" },
|
where: { category: "pets" },
|
||||||
limit: 10
|
limit: 10
|
||||||
})
|
})
|
||||||
|
|
@ -219,13 +220,14 @@ const brain = new Brainy({
|
||||||
2. **Use appropriate limits**
|
2. **Use appropriate limits**
|
||||||
```typescript
|
```typescript
|
||||||
// Don't fetch more than needed
|
// Don't fetch more than needed
|
||||||
const results = await brain.search("query", { limit: 10 })
|
const results = await brain.find({ query: "query", limit: 10 })
|
||||||
```
|
```
|
||||||
|
|
||||||
3. **Consider metadata filtering first**
|
3. **Consider metadata filtering first**
|
||||||
```typescript
|
```typescript
|
||||||
// Filter by metadata first, then semantic search
|
// Filter by metadata first, then semantic search
|
||||||
const results = await brain.search("query", {
|
const results = await brain.find({
|
||||||
|
query: "query",
|
||||||
where: { category: "specific" }, // Reduces search space
|
where: { category: "specific" }, // Reduces search space
|
||||||
limit: 10
|
limit: 10
|
||||||
})
|
})
|
||||||
|
|
@ -364,7 +366,7 @@ try {
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
||||||
const id = await brain.add("health check", { nounType: 'content' })
|
const id = await brain.add("health check", { nounType: 'content' })
|
||||||
const results = await brain.search("health")
|
const results = await brain.find("health")
|
||||||
|
|
||||||
console.log('✅ Brainy is working correctly')
|
console.log('✅ Brainy is working correctly')
|
||||||
console.log(`Added item: ${id}`)
|
console.log(`Added item: ${id}`)
|
||||||
|
|
|
||||||
|
|
@ -380,7 +380,7 @@ displayAug.configure({
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Precompute display fields for better performance
|
// Precompute display fields for better performance
|
||||||
const entities = await brainy.search('*', { limit: 100 })
|
const entities = await brainy.find({ limit: 100 })
|
||||||
await displayAug.precomputeBatch(
|
await displayAug.precomputeBatch(
|
||||||
entities.map(e => ({ id: e.id, data: e.metadata }))
|
entities.map(e => ({ id: e.id, data: e.metadata }))
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -80,17 +80,16 @@ const brain = new Brainy({
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
// ✅ Pattern 2: Cloud storage (production)
|
// ✅ Pattern 2: Filesystem (production default)
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: {
|
storage: {
|
||||||
type: 's3',
|
type: 'filesystem',
|
||||||
bucket: 'my-vfs-data',
|
path: '/var/lib/brainy'
|
||||||
region: 'us-west-2'
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
// ✅ Pattern 3: Auto-detection (recommended)
|
// ✅ Pattern 3: Auto-detection (recommended)
|
||||||
const brain = new Brainy() // Automatically chooses best storage
|
const brain = new Brainy() // Picks filesystem on Node, memory in browser
|
||||||
```
|
```
|
||||||
|
|
||||||
## 🔍 Search Patterns
|
## 🔍 Search Patterns
|
||||||
|
|
@ -563,7 +562,7 @@ function vfsMiddleware() {
|
||||||
| ❌ **Avoid These Patterns** | ✅ **Use These Instead** |
|
| ❌ **Avoid These Patterns** | ✅ **Use These Instead** |
|
||||||
|---------------------------|------------------------|
|
|---------------------------|------------------------|
|
||||||
| Manual tree filtering | `vfs.getDirectChildren()` |
|
| Manual tree filtering | `vfs.getDirectChildren()` |
|
||||||
| Memory storage for files | Filesystem or cloud storage |
|
| Memory storage for files | Filesystem (snapshot off-site for backup) |
|
||||||
| Sequential file operations | Parallel processing with limits |
|
| Sequential file operations | Parallel processing with limits |
|
||||||
| Manual relationship tracking | Built-in `vfs.addRelationship()` |
|
| Manual relationship tracking | Built-in `vfs.addRelationship()` |
|
||||||
| Loading entire directories | Paginated/lazy loading |
|
| Loading entire directories | Paginated/lazy loading |
|
||||||
|
|
|
||||||
|
|
@ -289,7 +289,7 @@ export class SizeProjection extends BaseProjectionStrategy {
|
||||||
where: {
|
where: {
|
||||||
vfsType: 'file',
|
vfsType: 'file',
|
||||||
size: {
|
size: {
|
||||||
greaterEqual: min,
|
gte: min,
|
||||||
lessThan: max
|
lessThan: max
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
@ -305,7 +305,7 @@ export class SizeProjection extends BaseProjectionStrategy {
|
||||||
where: {
|
where: {
|
||||||
vfsType: 'file',
|
vfsType: 'file',
|
||||||
size: {
|
size: {
|
||||||
greaterEqual: min,
|
gte: min,
|
||||||
lessThan: max
|
lessThan: max
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
@ -359,7 +359,7 @@ export class StatusProjection extends BaseProjectionStrategy {
|
||||||
const results = await brain.find({
|
const results = await brain.find({
|
||||||
where: {
|
where: {
|
||||||
vfsType: 'file',
|
vfsType: 'file',
|
||||||
modified: { greaterEqual: oneDayAgo },
|
modified: { gte: oneDayAgo },
|
||||||
reviewStatus: { missing: true } // No review status set
|
reviewStatus: { missing: true } // No review status set
|
||||||
},
|
},
|
||||||
limit: 1000
|
limit: 1000
|
||||||
|
|
@ -387,7 +387,7 @@ export class StatusProjection extends BaseProjectionStrategy {
|
||||||
vfsType: 'file',
|
vfsType: 'file',
|
||||||
anyOf: [
|
anyOf: [
|
||||||
{ reviewStatus: { exists: true } },
|
{ reviewStatus: { exists: true } },
|
||||||
{ modified: { greaterEqual: Date.now() - 86400000 } }
|
{ modified: { gte: Date.now() - 86400000 } }
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
limit
|
limit
|
||||||
|
|
@ -415,7 +415,7 @@ Projection strategies use **Brainy Field Operators** (BFO), not MongoDB-style op
|
||||||
{ size: { $gte: 1000, $lte: 5000 } }
|
{ size: { $gte: 1000, $lte: 5000 } }
|
||||||
|
|
||||||
// ✅ BFO style (CORRECT)
|
// ✅ BFO style (CORRECT)
|
||||||
{ size: { greaterEqual: 1000, lessEqual: 5000 } }
|
{ size: { gte: 1000, lte: 5000 } }
|
||||||
```
|
```
|
||||||
|
|
||||||
### Logical Operators
|
### Logical Operators
|
||||||
|
|
@ -451,9 +451,9 @@ Projection strategies use **Brainy Field Operators** (BFO), not MongoDB-style op
|
||||||
// Comparison
|
// Comparison
|
||||||
{ field: value } // Exact match
|
{ field: value } // Exact match
|
||||||
{ field: { greaterThan: 10 } } // >
|
{ field: { greaterThan: 10 } } // >
|
||||||
{ field: { greaterEqual: 10 } } // >=
|
{ field: { gte: 10 } } // >=
|
||||||
{ field: { lessThan: 10 } } // <
|
{ field: { lessThan: 10 } } // <
|
||||||
{ field: { lessEqual: 10 } } // <=
|
{ field: { lte: 10 } } // <=
|
||||||
{ field: { not: value } } // !=
|
{ field: { not: value } } // !=
|
||||||
|
|
||||||
// Logical
|
// Logical
|
||||||
|
|
@ -485,7 +485,7 @@ All metadata fields are automatically indexed. Use direct equality or range quer
|
||||||
```typescript
|
```typescript
|
||||||
// ✅ Fast: Direct index lookup (O(log n))
|
// ✅ Fast: Direct index lookup (O(log n))
|
||||||
{ priority: 'high' }
|
{ priority: 'high' }
|
||||||
{ size: { greaterEqual: 1000 } }
|
{ size: { gte: 1000 } }
|
||||||
|
|
||||||
// ⚠️ Slower: Must scan results
|
// ⚠️ Slower: Must scan results
|
||||||
{ path: { matches: /complex-regex/ } }
|
{ path: { matches: /complex-regex/ } }
|
||||||
|
|
@ -668,7 +668,7 @@ async resolve(brain, vfs, period: string) {
|
||||||
|
|
||||||
const results = await brain.find({
|
const results = await brain.find({
|
||||||
where: {
|
where: {
|
||||||
modified: { greaterEqual: since }
|
modified: { gte: since }
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
return this.extractIds(results)
|
return this.extractIds(results)
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@
|
||||||
- **[VFS Core](VFS_CORE.md)** - Complete filesystem architecture and API
|
- **[VFS Core](VFS_CORE.md)** - Complete filesystem architecture and API
|
||||||
- **[Semantic VFS](SEMANTIC_VFS.md)** - Multi-dimensional file access (6 semantic dimensions)
|
- **[Semantic VFS](SEMANTIC_VFS.md)** - Multi-dimensional file access (6 semantic dimensions)
|
||||||
- **[Neural Extraction](NEURAL_EXTRACTION.md)** - AI-powered concept and entity extraction
|
- **[Neural Extraction](NEURAL_EXTRACTION.md)** - AI-powered concept and entity extraction
|
||||||
- **[Examples & Scenarios](VFS_EXAMPLES_SCENARIOS.md)** - Real-world use cases and code
|
- **[Common Patterns](COMMON_PATTERNS.md)** - Real-world use cases and code
|
||||||
- **[VFS API Guide](VFS_API_GUIDE.md)** - Complete API reference
|
- **[VFS API Guide](VFS_API_GUIDE.md)** - Complete API reference
|
||||||
|
|
||||||
## What is Brainy VFS?
|
## What is Brainy VFS?
|
||||||
|
|
@ -332,7 +332,7 @@ const urgent = await vfs.search('', {
|
||||||
|
|
||||||
## Advanced Features
|
## Advanced Features
|
||||||
|
|
||||||
> **Note:** See [VFS ROADMAP](./ROADMAP.md) for planned advanced features like version history, distributed filesystem, and more.
|
> **Note:** See [VFS ROADMAP](./ROADMAP.md) for planned advanced features like version history and more.
|
||||||
|
|
||||||
## Integration Possibilities
|
## Integration Possibilities
|
||||||
|
|
||||||
|
|
@ -351,7 +351,6 @@ Brainy VFS is designed for speed and scale:
|
||||||
**PROJECTED at larger scales (not yet tested):**
|
**PROJECTED at larger scales (not yet tested):**
|
||||||
- **Vector search** <100ms for millions of files (projected)
|
- **Vector search** <100ms for millions of files (projected)
|
||||||
- **Streaming support** for files of any size (architecture supports, see [limitations in ROADMAP](./ROADMAP.md))
|
- **Streaming support** for files of any size (architecture supports, see [limitations in ROADMAP](./ROADMAP.md))
|
||||||
- **Distributed sharding** for billions of files (architecture supports, not tested at scale)
|
|
||||||
|
|
||||||
See tests in `tests/vfs/` for actual measured performance.
|
See tests in `tests/vfs/` for actual measured performance.
|
||||||
|
|
||||||
|
|
@ -377,7 +376,7 @@ Brainy VFS fully leverages Brainy's revolutionary Triple Intelligence system:
|
||||||
| Manual organization | Self-organizing with intelligent fusion |
|
| Manual organization | Self-organizing with intelligent fusion |
|
||||||
| No relationships | Rich knowledge graph with traversal |
|
| No relationships | Rich knowledge graph with traversal |
|
||||||
| Static metadata | Dynamic, queryable metadata with field intelligence |
|
| Static metadata | Dynamic, queryable metadata with field intelligence |
|
||||||
| Single server | Distributed & federated |
|
| Manual scaling | Horizontal read scaling — many readers, one writer |
|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
|
|
@ -387,8 +386,7 @@ npm install @soulcraft/brainy
|
||||||
|
|
||||||
## Requirements
|
## Requirements
|
||||||
|
|
||||||
- Node.js 18+ (for server/desktop)
|
- Node.js 22+ or Bun (server-only)
|
||||||
- Modern browser (for web apps)
|
|
||||||
- Brainy 3.0+
|
- Brainy 3.0+
|
||||||
|
|
||||||
## API Reference
|
## API Reference
|
||||||
|
|
@ -438,10 +436,10 @@ The VFS is built with production scalability in mind:
|
||||||
- Parent-child relationship caching
|
- Parent-child relationship caching
|
||||||
- Hot path detection and optimization
|
- Hot path detection and optimization
|
||||||
|
|
||||||
2. **Distributed Architecture**
|
2. **Horizontal Read Scaling**
|
||||||
- Sharding by path prefix
|
- On-disk store partitioned into 256 hex directory buckets
|
||||||
- Read replicas for hot directories
|
- Many reader processes against one shared store
|
||||||
- CDN integration for static files
|
- Single writer keeps the store consistent
|
||||||
|
|
||||||
3. **Intelligent Indexing**
|
3. **Intelligent Indexing**
|
||||||
- Compound indexes on (parent, name)
|
- Compound indexes on (parent, name)
|
||||||
|
|
@ -589,49 +587,6 @@ vfs.on('file:added', async (path) => {
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
#### **5. Distributed Team Workspace**
|
|
||||||
|
|
||||||
Collaborative file management:
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Track file ownership and access
|
|
||||||
await vfs.writeFile('/projects/alpha/spec.md', content, {
|
|
||||||
metadata: {
|
|
||||||
owner: userId,
|
|
||||||
team: 'engineering',
|
|
||||||
permissions: {
|
|
||||||
[userId]: 'rw',
|
|
||||||
'team:engineering': 'r',
|
|
||||||
'others': '-'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Add collaborative features
|
|
||||||
await vfs.addTodo('/projects/alpha/spec.md', {
|
|
||||||
task: 'Review security section',
|
|
||||||
assignee: 'alice@company.com',
|
|
||||||
due: '2024-02-01',
|
|
||||||
priority: 'high'
|
|
||||||
})
|
|
||||||
|
|
||||||
// Track who's working on what
|
|
||||||
await vfs.setxattr('/projects/alpha/spec.md', 'locks', {
|
|
||||||
section3: {
|
|
||||||
user: 'bob@company.com',
|
|
||||||
since: Date.now()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Find all files assigned to a user
|
|
||||||
const assigned = await vfs.search('', {
|
|
||||||
where: {
|
|
||||||
'todos.assignee': 'alice@company.com',
|
|
||||||
'todos.status': 'pending'
|
|
||||||
}
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Monitoring & Operations (Planned)
|
### Monitoring & Operations (Planned)
|
||||||
|
|
||||||
> **Note:** Production monitoring features are planned for v1.1. See [ROADMAP](./ROADMAP.md).
|
> **Note:** Production monitoring features are planned for v1.1. See [ROADMAP](./ROADMAP.md).
|
||||||
|
|
@ -665,27 +620,11 @@ await vfs.init({
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
#### **Distributed Cluster**
|
|
||||||
```javascript
|
|
||||||
const vfs = new VirtualFileSystem()
|
|
||||||
await vfs.init({
|
|
||||||
distributed: true,
|
|
||||||
nodes: [
|
|
||||||
'vfs1.internal:8080',
|
|
||||||
'vfs2.internal:8080',
|
|
||||||
'vfs3.internal:8080'
|
|
||||||
],
|
|
||||||
replication: 3,
|
|
||||||
consistency: 'eventual'
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
## Roadmap & Future Features
|
## Roadmap & Future Features
|
||||||
|
|
||||||
See [VFS ROADMAP](./ROADMAP.md) for planned features including:
|
See [VFS ROADMAP](./ROADMAP.md) for planned features including:
|
||||||
- Enhanced streaming support (v1.1)
|
- Enhanced streaming support (v1.1)
|
||||||
- Version history (v1.2)
|
- Version history (v1.2)
|
||||||
- Distributed filesystem (v1.2)
|
|
||||||
- AI-powered automation (v2.0)
|
- AI-powered automation (v2.0)
|
||||||
- FUSE driver (v2.0 research)
|
- FUSE driver (v2.0 research)
|
||||||
- And more community-requested features
|
- And more community-requested features
|
||||||
|
|
|
||||||
|
|
@ -65,27 +65,6 @@ const diff = await vfs.diffVersions('/important-doc.md', v1, v2)
|
||||||
**Status:** Planned
|
**Status:** Planned
|
||||||
**Effort:** 4-5 weeks
|
**Effort:** 4-5 weeks
|
||||||
|
|
||||||
### Distributed Filesystem
|
|
||||||
Mount remote Brainy instances for federated file access.
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Planned API (not yet implemented)
|
|
||||||
await vfs.mount('/remote-team', {
|
|
||||||
type: 'brainy-remote',
|
|
||||||
url: 'https://team-brainy.example.com',
|
|
||||||
credentials: {...}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Federated search across mounted instances
|
|
||||||
const results = await vfs.search('project docs', { includeMounted: true })
|
|
||||||
|
|
||||||
// Sync directories
|
|
||||||
await vfs.sync('/local/docs', '/remote-team/docs')
|
|
||||||
```
|
|
||||||
|
|
||||||
**Status:** Planned
|
|
||||||
**Effort:** 6-8 weeks
|
|
||||||
|
|
||||||
### Backup & Recovery API
|
### Backup & Recovery API
|
||||||
Built-in backup and recovery operations.
|
Built-in backup and recovery operations.
|
||||||
|
|
||||||
|
|
@ -215,12 +194,6 @@ Vite integration with VFS for semantic module resolution.
|
||||||
|
|
||||||
## Far Future (v5.0+)
|
## Far Future (v5.0+)
|
||||||
|
|
||||||
### Automatic Node Discovery
|
|
||||||
Zero-config multi-node setup with automatic discovery via UDP broadcast, Kubernetes DNS, or cloud provider APIs.
|
|
||||||
|
|
||||||
### Automatic Failover
|
|
||||||
Health monitoring and automatic failover in distributed deployments.
|
|
||||||
|
|
||||||
### Cloud Provider Auto-Detection
|
### Cloud Provider Auto-Detection
|
||||||
```typescript
|
```typescript
|
||||||
// Far future concept
|
// Far future concept
|
||||||
|
|
|
||||||
|
|
@ -47,16 +47,18 @@ const authFiles = await vfs.readdir('/by-concept/authentication')
|
||||||
const aliceFiles = await vfs.readdir('/by-author/alice')
|
const aliceFiles = await vfs.readdir('/by-author/alice')
|
||||||
```
|
```
|
||||||
|
|
||||||
### 2. **Time Travel**
|
### 2. **Change Tracking by Date**
|
||||||
See your codebase as it existed at any point:
|
List the files that were modified on any given day:
|
||||||
```typescript
|
```typescript
|
||||||
// Code from March 15th
|
// Files that changed on March 15th
|
||||||
const snapshot = await vfs.readdir('/as-of/2024-03-15')
|
const changed = await vfs.readdir('/as-of/2024-03-15')
|
||||||
|
|
||||||
// Compare with today
|
// Everything under /src right now
|
||||||
const current = await vfs.readdir('/src')
|
const current = await vfs.readdir('/src')
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`/as-of/<date>` selects by *modification date* — it reads the files' current content, not historical versions. For true point-in-time queries over entity state, use the Db API (`brain.asOf(generation)`).
|
||||||
|
|
||||||
### 3. **Knowledge Graph Navigation**
|
### 3. **Knowledge Graph Navigation**
|
||||||
Navigate by semantic relationships:
|
Navigate by semantic relationships:
|
||||||
```typescript
|
```typescript
|
||||||
|
|
@ -118,13 +120,14 @@ await vfs.stat('/by-author/alice/config.ts')
|
||||||
### 4. By Time (Temporal) ✅ **Production**
|
### 4. By Time (Temporal) ✅ **Production**
|
||||||
```typescript
|
```typescript
|
||||||
await vfs.readdir('/as-of/2024-03-15')
|
await vfs.readdir('/as-of/2024-03-15')
|
||||||
// Files modified on March 15, 2024
|
// Files modified on March 15, 2024 (24-hour window)
|
||||||
|
|
||||||
await vfs.readFile('/as-of/2024-03-15/src/auth.ts')
|
await vfs.readFile('/as-of/2024-03-15/auth.ts')
|
||||||
// Read auth.ts as it existed that day
|
// Current content of auth.ts, addressed by modification date —
|
||||||
|
// the path only resolves if auth.ts was modified that day
|
||||||
```
|
```
|
||||||
|
|
||||||
**How it works:** Tracks `modified` timestamp. Uses B-tree range queries (`greaterEqual`/`lessEqual`) for O(log n) performance.
|
**How it works:** Tracks the `modified` timestamp on every file and runs a range query (`gte`/`lte`) over one 24-hour window for O(log n) performance. The VFS does not store historical file contents — `/as-of/` filters by *when a file last changed*; reads return the current bytes. For point-in-time state, use the Db API (`brain.asOf(generation)`).
|
||||||
|
|
||||||
**Status:** ✅ Fully implemented and tested at 10K file scale
|
**Status:** ✅ Fully implemented and tested at 10K file scale
|
||||||
|
|
||||||
|
|
@ -222,7 +225,7 @@ console.log(authFiles)
|
||||||
// ['login.ts', 'signup.ts', 'oauth.ts']
|
// ['login.ts', 'signup.ts', 'oauth.ts']
|
||||||
```
|
```
|
||||||
|
|
||||||
### Example 2: Time Travel
|
### Example 2: Changes by Day
|
||||||
```typescript
|
```typescript
|
||||||
// See what changed today
|
// See what changed today
|
||||||
const today = new Date().toISOString().split('T')[0]
|
const today = new Date().toISOString().split('T')[0]
|
||||||
|
|
@ -232,8 +235,8 @@ const todaysFiles = await vfs.readdir(`/as-of/${today}`)
|
||||||
const yesterday = new Date(Date.now() - 86400000).toISOString().split('T')[0]
|
const yesterday = new Date(Date.now() - 86400000).toISOString().split('T')[0]
|
||||||
const yesterdaysFiles = await vfs.readdir(`/as-of/${yesterday}`)
|
const yesterdaysFiles = await vfs.readdir(`/as-of/${yesterday}`)
|
||||||
|
|
||||||
const newFiles = todaysFiles.filter(f => !yesterdaysFiles.includes(f))
|
const onlyToday = todaysFiles.filter(f => !yesterdaysFiles.includes(f))
|
||||||
console.log('New files today:', newFiles)
|
console.log('Changed today (untouched yesterday):', onlyToday)
|
||||||
```
|
```
|
||||||
|
|
||||||
### Example 3: Graph Navigation
|
### Example 3: Graph Navigation
|
||||||
|
|
@ -397,9 +400,9 @@ Semantic VFS is built on Brainy's Triple Intelligence™:
|
||||||
### Real Implementations, Zero Mocks
|
### Real Implementations, Zero Mocks
|
||||||
|
|
||||||
Every component uses **production Brainy APIs**:
|
Every component uses **production Brainy APIs**:
|
||||||
- `brain.find()` - Real metadata queries (brainy.ts:580)
|
- `brain.find()` - Real metadata queries
|
||||||
- `brain.similar()` - Real HNSW search (brainy.ts:680)
|
- `brain.similar()` - Real HNSW search
|
||||||
- `brain.getRelations()` - Real graph traversal (brainy.ts:803)
|
- `brain.related()` - Real graph traversal
|
||||||
- `MetadataIndexManager` - Real B-tree indexes
|
- `MetadataIndexManager` - Real B-tree indexes
|
||||||
- `GraphAdjacencyIndex` - Real graph storage
|
- `GraphAdjacencyIndex` - Real graph storage
|
||||||
- `HNSW Index` - Real vector search
|
- `HNSW Index` - Real vector search
|
||||||
|
|
@ -432,25 +435,17 @@ await vfs.writeFile(path, code, {
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
### 3. Optimize for Your Scale
|
### 3. Combine Dimensions
|
||||||
```typescript
|
```typescript
|
||||||
// For < 100K files: Post-filtering is fine
|
// Find security files Alice changed on a given day
|
||||||
// For > 100K files: Use flattened indexes
|
// (each /as-of/<date> path covers exactly that one day)
|
||||||
|
|
||||||
// Force index refresh after bulk operations
|
|
||||||
await brain.storage.rebuildIndexes()
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4. Combine Dimensions
|
|
||||||
```typescript
|
|
||||||
// Find security files Alice worked on this week
|
|
||||||
const aliceFiles = await vfs.readdir('/by-author/alice')
|
const aliceFiles = await vfs.readdir('/by-author/alice')
|
||||||
const securityFiles = await vfs.readdir('/by-tag/security')
|
const securityFiles = await vfs.readdir('/by-tag/security')
|
||||||
const thisWeek = await vfs.readdir(`/as-of/${weekAgo}`)
|
const changedThatDay = await vfs.readdir('/as-of/2024-03-15')
|
||||||
|
|
||||||
const intersection = aliceFiles
|
const intersection = aliceFiles
|
||||||
.filter(f => securityFiles.includes(f))
|
.filter(f => securityFiles.includes(f))
|
||||||
.filter(f => thisWeek.includes(f))
|
.filter(f => changedThatDay.includes(f))
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
@ -469,14 +464,14 @@ console.log(entity.metadata.concepts)
|
||||||
|
|
||||||
### Slow Queries on Large Datasets
|
### Slow Queries on Large Datasets
|
||||||
```typescript
|
```typescript
|
||||||
// Check if indexes are built
|
// Check if indexes are built and populated
|
||||||
const stats = await brain.storage.getIndexStats()
|
const stats = await brain.getIndexStats()
|
||||||
console.log(stats)
|
console.log(stats)
|
||||||
|
|
||||||
// Rebuild if needed
|
|
||||||
await brain.storage.rebuildIndexes()
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
If an index looks empty or inconsistent, rebuild from raw storage with the CLI
|
||||||
|
(stop the live writer first): `brainy inspect repair <data-dir>`.
|
||||||
|
|
||||||
### Semantic Path Returns Empty
|
### Semantic Path Returns Empty
|
||||||
```typescript
|
```typescript
|
||||||
// Check if metadata exists
|
// Check if metadata exists
|
||||||
|
|
|
||||||
|
|
@ -183,7 +183,7 @@ console.log('VFS type:', entity.metadata.vfsType)
|
||||||
### 3. Check Relationships
|
### 3. Check Relationships
|
||||||
```javascript
|
```javascript
|
||||||
const parentId = await vfs.resolvePath('/directory')
|
const parentId = await vfs.resolvePath('/directory')
|
||||||
const relations = await brain.getRelations({
|
const relations = await brain.related({
|
||||||
from: parentId,
|
from: parentId,
|
||||||
type: VerbType.Contains
|
type: VerbType.Contains
|
||||||
})
|
})
|
||||||
|
|
@ -246,7 +246,7 @@ const tree = await vfs.getTreeStructure('/', {
|
||||||
If you encounter issues not covered here:
|
If you encounter issues not covered here:
|
||||||
|
|
||||||
1. Check the [VFS API Guide](./VFS_API_GUIDE.md)
|
1. Check the [VFS API Guide](./VFS_API_GUIDE.md)
|
||||||
2. Review [VFS Examples](./VFS_EXAMPLES_SCENARIOS.md)
|
2. Review [Common Patterns](./COMMON_PATTERNS.md)
|
||||||
3. Look at [test files](../../tests/vfs/) for working examples
|
3. Look at [test files](../../tests/vfs/) for working examples
|
||||||
4. Report issues at [GitHub Issues](https://github.com/soulcraftlabs/brainy/issues)
|
4. Report issues at [GitHub Issues](https://github.com/soulcraftlabs/brainy/issues)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,728 +0,0 @@
|
||||||
# VFS User Functions - Templates and Examples
|
|
||||||
|
|
||||||
This document provides template functions that you can implement for domain-specific needs. These functions combine VFS primitives to solve common problems.
|
|
||||||
|
|
||||||
## Table of Contents
|
|
||||||
1. [Code Analysis Functions](#code-analysis-functions)
|
|
||||||
2. [Export Format Functions](#export-format-functions)
|
|
||||||
3. [Project Management Functions](#project-management-functions)
|
|
||||||
4. [Creative Writing Functions](#creative-writing-functions)
|
|
||||||
5. [Game Development Functions](#game-development-functions)
|
|
||||||
|
|
||||||
## Code Analysis Functions
|
|
||||||
|
|
||||||
### Get Dependency Graph
|
|
||||||
```javascript
|
|
||||||
/**
|
|
||||||
* Build a dependency graph for JavaScript/TypeScript projects
|
|
||||||
*/
|
|
||||||
async function getDependencyGraph(vfs, srcPath) {
|
|
||||||
const files = await vfs.readdir(srcPath, { recursive: true })
|
|
||||||
const graph = {}
|
|
||||||
|
|
||||||
for (const file of files) {
|
|
||||||
const filePath = `${srcPath}/${file}`
|
|
||||||
|
|
||||||
// Only process JS/TS files
|
|
||||||
if (file.match(/\.(js|ts|jsx|tsx)$/)) {
|
|
||||||
const content = await vfs.readFile(filePath)
|
|
||||||
const text = content.toString()
|
|
||||||
|
|
||||||
// Parse imports (basic regex, use proper AST parser for production)
|
|
||||||
const imports = []
|
|
||||||
const importRegex = /import\s+.*?\s+from\s+['"](.+?)['"]/g
|
|
||||||
const requireRegex = /require\(['"](.+?)['"]\)/g
|
|
||||||
|
|
||||||
let match
|
|
||||||
while ((match = importRegex.exec(text)) !== null) {
|
|
||||||
imports.push(match[1])
|
|
||||||
}
|
|
||||||
while ((match = requireRegex.exec(text)) !== null) {
|
|
||||||
imports.push(match[1])
|
|
||||||
}
|
|
||||||
|
|
||||||
graph[filePath] = imports
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return graph
|
|
||||||
}
|
|
||||||
|
|
||||||
// Use it
|
|
||||||
const deps = await getDependencyGraph(vfs, '/src')
|
|
||||||
```
|
|
||||||
|
|
||||||
### Find Circular Dependencies
|
|
||||||
```javascript
|
|
||||||
/**
|
|
||||||
* Detect circular dependencies in your code
|
|
||||||
*/
|
|
||||||
async function findCircularDependencies(vfs, srcPath) {
|
|
||||||
const graph = await getDependencyGraph(vfs, srcPath)
|
|
||||||
const cycles = []
|
|
||||||
|
|
||||||
function detectCycle(node, visited = new Set(), stack = []) {
|
|
||||||
if (stack.includes(node)) {
|
|
||||||
const cycleStart = stack.indexOf(node)
|
|
||||||
cycles.push(stack.slice(cycleStart))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if (visited.has(node)) return
|
|
||||||
visited.add(node)
|
|
||||||
stack.push(node)
|
|
||||||
|
|
||||||
const dependencies = graph[node] || []
|
|
||||||
for (const dep of dependencies) {
|
|
||||||
// Resolve relative imports
|
|
||||||
const resolvedDep = resolvePath(node, dep)
|
|
||||||
if (graph[resolvedDep]) {
|
|
||||||
detectCycle(resolvedDep, visited, [...stack])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Object.keys(graph).forEach(node => detectCycle(node))
|
|
||||||
return cycles
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Find Untested Code
|
|
||||||
```javascript
|
|
||||||
/**
|
|
||||||
* Find source files without corresponding test files
|
|
||||||
*/
|
|
||||||
async function findUntestedCode(vfs, srcPath, testPath = null) {
|
|
||||||
testPath = testPath || srcPath.replace('/src', '/tests')
|
|
||||||
|
|
||||||
const sourceFiles = await vfs.readdir(srcPath, { recursive: true })
|
|
||||||
const testFiles = await vfs.readdir(testPath, { recursive: true }).catch(() => [])
|
|
||||||
|
|
||||||
const untestedFiles = []
|
|
||||||
|
|
||||||
for (const sourceFile of sourceFiles) {
|
|
||||||
if (!sourceFile.match(/\.(js|ts|jsx|tsx)$/)) continue
|
|
||||||
|
|
||||||
// Look for corresponding test file
|
|
||||||
const baseName = sourceFile.replace(/\.(js|ts|jsx|tsx)$/, '')
|
|
||||||
const hasTest = testFiles.some(testFile =>
|
|
||||||
testFile.includes(baseName) &&
|
|
||||||
testFile.match(/\.(test|spec)\.(js|ts|jsx|tsx)$/)
|
|
||||||
)
|
|
||||||
|
|
||||||
if (!hasTest) {
|
|
||||||
untestedFiles.push(`${srcPath}/${sourceFile}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return untestedFiles
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Find Similar Code (Duplicate Detection)
|
|
||||||
```javascript
|
|
||||||
/**
|
|
||||||
* Find potentially duplicate code using similarity scoring
|
|
||||||
*/
|
|
||||||
async function findSimilarCode(vfs, filePath, options = {}) {
|
|
||||||
const threshold = options.threshold || 0.8
|
|
||||||
const searchPath = options.searchPath || '/'
|
|
||||||
|
|
||||||
// Get the reference file content
|
|
||||||
const referenceContent = await vfs.readFile(filePath)
|
|
||||||
const referenceText = referenceContent.toString()
|
|
||||||
|
|
||||||
// Use VFS's semantic search
|
|
||||||
const similar = await vfs.findSimilar(filePath, {
|
|
||||||
limit: 10,
|
|
||||||
threshold
|
|
||||||
})
|
|
||||||
|
|
||||||
// Additionally, do structural comparison
|
|
||||||
const results = []
|
|
||||||
for (const match of similar) {
|
|
||||||
const matchContent = await vfs.readFile(match.path)
|
|
||||||
const matchText = matchContent.toString()
|
|
||||||
|
|
||||||
// Simple line-based similarity (use better algorithms in production)
|
|
||||||
const similarity = calculateSimilarity(referenceText, matchText)
|
|
||||||
|
|
||||||
if (similarity > threshold) {
|
|
||||||
results.push({
|
|
||||||
path: match.path,
|
|
||||||
similarity,
|
|
||||||
semanticScore: match.score
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return results.sort((a, b) => b.similarity - a.similarity)
|
|
||||||
}
|
|
||||||
|
|
||||||
function calculateSimilarity(text1, text2) {
|
|
||||||
// Simple Jaccard similarity on lines
|
|
||||||
const lines1 = new Set(text1.split('\n').map(l => l.trim()).filter(l => l))
|
|
||||||
const lines2 = new Set(text2.split('\n').map(l => l.trim()).filter(l => l))
|
|
||||||
|
|
||||||
const intersection = new Set([...lines1].filter(x => lines2.has(x)))
|
|
||||||
const union = new Set([...lines1, ...lines2])
|
|
||||||
|
|
||||||
return intersection.size / union.size
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Export Format Functions
|
|
||||||
|
|
||||||
### Export to EPUB (for novels)
|
|
||||||
```javascript
|
|
||||||
/**
|
|
||||||
* Export a directory of markdown files to EPUB format
|
|
||||||
*/
|
|
||||||
async function exportToEpub(vfs, path, metadata = {}) {
|
|
||||||
// First get the markdown export
|
|
||||||
const markdown = await vfs.exportToMarkdown(path)
|
|
||||||
|
|
||||||
// You'll need an EPUB library like epub-gen
|
|
||||||
const Epub = require('epub-gen')
|
|
||||||
|
|
||||||
// Convert markdown chapters to EPUB format
|
|
||||||
const chapters = []
|
|
||||||
const files = await vfs.readdir(path, { recursive: true })
|
|
||||||
|
|
||||||
for (const file of files.sort()) {
|
|
||||||
if (file.endsWith('.md')) {
|
|
||||||
const content = await vfs.readFile(`${path}/${file}`)
|
|
||||||
const title = file.replace('.md', '').replace(/-/g, ' ')
|
|
||||||
|
|
||||||
chapters.push({
|
|
||||||
title: title,
|
|
||||||
data: content.toString()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const options = {
|
|
||||||
title: metadata.title || 'My Book',
|
|
||||||
author: metadata.author || 'Author',
|
|
||||||
chapters: chapters
|
|
||||||
}
|
|
||||||
|
|
||||||
return new Epub(options)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Export to Static Site
|
|
||||||
```javascript
|
|
||||||
/**
|
|
||||||
* Export VFS content to static HTML site
|
|
||||||
*/
|
|
||||||
async function exportToStaticSite(vfs, sourcePath, options = {}) {
|
|
||||||
const json = await vfs.exportToJSON(sourcePath)
|
|
||||||
const html = []
|
|
||||||
|
|
||||||
html.push('<!DOCTYPE html>')
|
|
||||||
html.push('<html><head>')
|
|
||||||
html.push(`<title>${options.title || 'Documentation'}</title>`)
|
|
||||||
html.push('<style>/* Add your styles */</style>')
|
|
||||||
html.push('</head><body>')
|
|
||||||
|
|
||||||
function renderNode(node, name, depth = 0) {
|
|
||||||
const indent = ' '.repeat(depth)
|
|
||||||
|
|
||||||
if (node._meta?.type === 'file') {
|
|
||||||
html.push(`${indent}<article>`)
|
|
||||||
html.push(`${indent} <h${Math.min(depth + 2, 6)}>${name}</h${Math.min(depth + 2, 6)}>`)
|
|
||||||
|
|
||||||
if (typeof node._content === 'string') {
|
|
||||||
// Convert markdown to HTML if needed
|
|
||||||
html.push(`${indent} <pre>${escapeHtml(node._content)}</pre>`)
|
|
||||||
}
|
|
||||||
|
|
||||||
html.push(`${indent}</article>`)
|
|
||||||
} else if (node._meta?.type === 'directory') {
|
|
||||||
html.push(`${indent}<section>`)
|
|
||||||
html.push(`${indent} <h${Math.min(depth + 1, 6)}>${name}</h${Math.min(depth + 1, 6)}>`)
|
|
||||||
|
|
||||||
for (const [childName, childNode] of Object.entries(node)) {
|
|
||||||
if (!childName.startsWith('_')) {
|
|
||||||
renderNode(childNode, childName, depth + 1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
html.push(`${indent}</section>`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
renderNode(json, options.title || 'Root')
|
|
||||||
|
|
||||||
html.push('</body></html>')
|
|
||||||
return html.join('\n')
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Export to GraphQL Schema
|
|
||||||
```javascript
|
|
||||||
/**
|
|
||||||
* Generate GraphQL schema from VFS entities
|
|
||||||
*/
|
|
||||||
async function exportToGraphQLSchema(vfs) {
|
|
||||||
const entities = await vfs.listEntities()
|
|
||||||
const types = new Map()
|
|
||||||
|
|
||||||
// Group entities by type
|
|
||||||
for (const entity of entities) {
|
|
||||||
const type = entity.type || 'Unknown'
|
|
||||||
if (!types.has(type)) {
|
|
||||||
types.set(type, [])
|
|
||||||
}
|
|
||||||
types.get(type).push(entity)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate schema
|
|
||||||
let schema = 'type Query {\n'
|
|
||||||
|
|
||||||
for (const [typeName, entities] of types) {
|
|
||||||
schema += ` get${typeName}(id: ID!): ${typeName}\n`
|
|
||||||
schema += ` list${typeName}s: [${typeName}!]!\n`
|
|
||||||
}
|
|
||||||
|
|
||||||
schema += '}\n\n'
|
|
||||||
|
|
||||||
// Generate types
|
|
||||||
for (const [typeName, entities] of types) {
|
|
||||||
schema += `type ${typeName} {\n`
|
|
||||||
schema += ' id: ID!\n'
|
|
||||||
|
|
||||||
// Infer fields from first entity
|
|
||||||
if (entities.length > 0) {
|
|
||||||
const sample = entities[0]
|
|
||||||
for (const [key, value] of Object.entries(sample)) {
|
|
||||||
if (key !== 'id') {
|
|
||||||
const fieldType = inferGraphQLType(value)
|
|
||||||
schema += ` ${key}: ${fieldType}\n`
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
schema += '}\n\n'
|
|
||||||
}
|
|
||||||
|
|
||||||
return schema
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Project Management Functions
|
|
||||||
|
|
||||||
### Get Project Insights
|
|
||||||
```javascript
|
|
||||||
/**
|
|
||||||
* Analyze project for insights and patterns
|
|
||||||
*/
|
|
||||||
async function getProjectInsights(vfs, projectPath) {
|
|
||||||
const stats = await vfs.getProjectStats(projectPath)
|
|
||||||
const todos = await vfs.getAllTodos(projectPath)
|
|
||||||
const timeline = await vfs.getTimeline({ limit: 100 })
|
|
||||||
|
|
||||||
// Analyze activity patterns
|
|
||||||
const activityByDay = {}
|
|
||||||
const activityByUser = {}
|
|
||||||
const activityByFile = {}
|
|
||||||
|
|
||||||
for (const event of timeline) {
|
|
||||||
const day = event.timestamp.toISOString().split('T')[0]
|
|
||||||
activityByDay[day] = (activityByDay[day] || 0) + 1
|
|
||||||
|
|
||||||
const user = event.user || 'system'
|
|
||||||
activityByUser[user] = (activityByUser[user] || 0) + 1
|
|
||||||
|
|
||||||
activityByFile[event.path] = (activityByFile[event.path] || 0) + 1
|
|
||||||
}
|
|
||||||
|
|
||||||
// Find hotspots (most edited files)
|
|
||||||
const hotspots = Object.entries(activityByFile)
|
|
||||||
.sort(([,a], [,b]) => b - a)
|
|
||||||
.slice(0, 10)
|
|
||||||
.map(([path, count]) => ({ path, edits: count }))
|
|
||||||
|
|
||||||
// Todo analysis
|
|
||||||
const todosByPriority = {}
|
|
||||||
const todosByStatus = {}
|
|
||||||
|
|
||||||
for (const todo of todos) {
|
|
||||||
todosByPriority[todo.priority] = (todosByPriority[todo.priority] || 0) + 1
|
|
||||||
todosByStatus[todo.status] = (todosByStatus[todo.status] || 0) + 1
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
stats,
|
|
||||||
activity: {
|
|
||||||
byDay: activityByDay,
|
|
||||||
byUser: activityByUser,
|
|
||||||
hotspots
|
|
||||||
},
|
|
||||||
todos: {
|
|
||||||
total: todos.length,
|
|
||||||
byPriority: todosByPriority,
|
|
||||||
byStatus: todosByStatus,
|
|
||||||
highPriority: todos.filter(t => t.priority === 'high' && t.status === 'pending')
|
|
||||||
},
|
|
||||||
recommendations: generateRecommendations(stats, todos, hotspots)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function generateRecommendations(stats, todos, hotspots) {
|
|
||||||
const recommendations = []
|
|
||||||
|
|
||||||
if (stats.largestFile && stats.largestFile.size > 1024 * 1024) {
|
|
||||||
recommendations.push({
|
|
||||||
type: 'refactor',
|
|
||||||
message: `Consider splitting ${stats.largestFile.path} (${Math.round(stats.largestFile.size / 1024)}KB)`
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
if (todos.filter(t => t.priority === 'high' && t.status === 'pending').length > 5) {
|
|
||||||
recommendations.push({
|
|
||||||
type: 'priority',
|
|
||||||
message: 'You have many high-priority pending todos'
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
if (hotspots.length > 0 && hotspots[0].edits > 50) {
|
|
||||||
recommendations.push({
|
|
||||||
type: 'stability',
|
|
||||||
message: `${hotspots[0].path} changes frequently, consider stabilizing`
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return recommendations
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Generate Sprint Report
|
|
||||||
```javascript
|
|
||||||
/**
|
|
||||||
* Generate a report for the current sprint
|
|
||||||
*/
|
|
||||||
async function generateSprintReport(vfs, sprintStart, sprintEnd = new Date()) {
|
|
||||||
const timeline = await vfs.getTimeline({
|
|
||||||
from: sprintStart,
|
|
||||||
to: sprintEnd
|
|
||||||
})
|
|
||||||
|
|
||||||
const todos = await vfs.getAllTodos()
|
|
||||||
|
|
||||||
// Group work by user
|
|
||||||
const workByUser = {}
|
|
||||||
for (const event of timeline) {
|
|
||||||
const user = event.user || 'system'
|
|
||||||
if (!workByUser[user]) {
|
|
||||||
workByUser[user] = {
|
|
||||||
commits: 0,
|
|
||||||
filesModified: new Set(),
|
|
||||||
linesChanged: 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
workByUser[user].commits++
|
|
||||||
workByUser[user].filesModified.add(event.path)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Calculate completion rate
|
|
||||||
const completedTodos = todos.filter(t => t.status === 'completed').length
|
|
||||||
const totalTodos = todos.length
|
|
||||||
const completionRate = totalTodos > 0 ? (completedTodos / totalTodos * 100).toFixed(1) : 0
|
|
||||||
|
|
||||||
return {
|
|
||||||
period: {
|
|
||||||
start: sprintStart,
|
|
||||||
end: sprintEnd,
|
|
||||||
days: Math.ceil((sprintEnd - sprintStart) / (1000 * 60 * 60 * 24))
|
|
||||||
},
|
|
||||||
team: Object.entries(workByUser).map(([user, work]) => ({
|
|
||||||
user,
|
|
||||||
commits: work.commits,
|
|
||||||
filesModified: work.filesModified.size
|
|
||||||
})),
|
|
||||||
todos: {
|
|
||||||
completed: completedTodos,
|
|
||||||
total: totalTodos,
|
|
||||||
completionRate: `${completionRate}%`,
|
|
||||||
remaining: todos.filter(t => t.status === 'pending')
|
|
||||||
},
|
|
||||||
velocity: {
|
|
||||||
commitsPerDay: (timeline.length / 7).toFixed(1),
|
|
||||||
todosPerDay: (completedTodos / 7).toFixed(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Creative Writing Functions
|
|
||||||
|
|
||||||
### Track Character Arcs
|
|
||||||
```javascript
|
|
||||||
/**
|
|
||||||
* Track how characters evolve throughout a story
|
|
||||||
*/
|
|
||||||
async function trackCharacterArc(vfs, characterName, storyPath = '/') {
|
|
||||||
// Find the character entity
|
|
||||||
const entities = await vfs.searchEntities({
|
|
||||||
type: 'character',
|
|
||||||
name: characterName
|
|
||||||
})
|
|
||||||
|
|
||||||
if (entities.length === 0) {
|
|
||||||
throw new Error(`Character ${characterName} not found`)
|
|
||||||
}
|
|
||||||
|
|
||||||
const character = entities[0]
|
|
||||||
const occurrences = await vfs.findEntityOccurrences(character.id)
|
|
||||||
|
|
||||||
// Analyze each appearance
|
|
||||||
const arc = []
|
|
||||||
|
|
||||||
for (const occurrence of occurrences) {
|
|
||||||
const content = await vfs.readFile(occurrence.path)
|
|
||||||
const text = content.toString()
|
|
||||||
|
|
||||||
// Find mentions of the character (basic approach)
|
|
||||||
const mentions = text.split('\n').filter(line =>
|
|
||||||
line.toLowerCase().includes(characterName.toLowerCase())
|
|
||||||
)
|
|
||||||
|
|
||||||
arc.push({
|
|
||||||
chapter: occurrence.path,
|
|
||||||
mentions: mentions.length,
|
|
||||||
// Analyze emotional tone (simplified)
|
|
||||||
mood: analyzeMood(mentions),
|
|
||||||
// Extract key actions
|
|
||||||
actions: extractActions(mentions, characterName)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
character: character.metadata,
|
|
||||||
arc: arc,
|
|
||||||
summary: summarizeArc(arc)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function analyzeMood(mentions) {
|
|
||||||
const positiveWords = ['smiled', 'laughed', 'happy', 'joy', 'love', 'success']
|
|
||||||
const negativeWords = ['cried', 'angry', 'sad', 'fear', 'fail', 'death']
|
|
||||||
|
|
||||||
let positive = 0, negative = 0
|
|
||||||
|
|
||||||
for (const mention of mentions) {
|
|
||||||
const lower = mention.toLowerCase()
|
|
||||||
positive += positiveWords.filter(w => lower.includes(w)).length
|
|
||||||
negative += negativeWords.filter(w => lower.includes(w)).length
|
|
||||||
}
|
|
||||||
|
|
||||||
if (positive > negative) return 'positive'
|
|
||||||
if (negative > positive) return 'negative'
|
|
||||||
return 'neutral'
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Generate Story Bible
|
|
||||||
```javascript
|
|
||||||
/**
|
|
||||||
* Create a comprehensive reference for your story universe
|
|
||||||
*/
|
|
||||||
async function generateStoryBible(vfs, storyPath) {
|
|
||||||
const characters = await vfs.listEntities({ type: 'character' })
|
|
||||||
const locations = await vfs.listEntities({ type: 'location' })
|
|
||||||
const concepts = await vfs.findConcepts({ domain: 'narrative' })
|
|
||||||
|
|
||||||
const bible = {
|
|
||||||
title: 'Story Bible',
|
|
||||||
generated: new Date(),
|
|
||||||
characters: {},
|
|
||||||
locations: {},
|
|
||||||
plotThreads: {},
|
|
||||||
timeline: []
|
|
||||||
}
|
|
||||||
|
|
||||||
// Document characters
|
|
||||||
for (const char of characters) {
|
|
||||||
const occurrences = await vfs.findEntityOccurrences(char.id)
|
|
||||||
bible.characters[char.metadata.name] = {
|
|
||||||
...char.metadata,
|
|
||||||
appearances: occurrences.map(o => o.path),
|
|
||||||
relationships: await vfs.getEntityGraph(char.id, { depth: 1 })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Document locations
|
|
||||||
for (const loc of locations) {
|
|
||||||
bible.locations[loc.metadata.name] = {
|
|
||||||
...loc.metadata,
|
|
||||||
scenes: await vfs.findEntityOccurrences(loc.id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Plot threads from concepts
|
|
||||||
for (const concept of concepts) {
|
|
||||||
if (concept.type === 'plot') {
|
|
||||||
bible.plotThreads[concept.name] = {
|
|
||||||
description: concept.description,
|
|
||||||
keywords: concept.keywords,
|
|
||||||
chapters: await vfs.findByConcept(concept.name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate timeline
|
|
||||||
const events = await vfs.getTimeline({ limit: 1000 })
|
|
||||||
bible.timeline = events.map(e => ({
|
|
||||||
date: e.timestamp,
|
|
||||||
event: e.description,
|
|
||||||
chapter: e.path
|
|
||||||
}))
|
|
||||||
|
|
||||||
return bible
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Game Development Functions
|
|
||||||
|
|
||||||
### Validate Game Data
|
|
||||||
```javascript
|
|
||||||
/**
|
|
||||||
* Validate game configuration files for consistency
|
|
||||||
*/
|
|
||||||
async function validateGameData(vfs, gamePath) {
|
|
||||||
const errors = []
|
|
||||||
const warnings = []
|
|
||||||
|
|
||||||
// Load all game data
|
|
||||||
const gameData = await vfs.exportToJSON(gamePath)
|
|
||||||
|
|
||||||
// Check quest references
|
|
||||||
if (gameData.quests) {
|
|
||||||
for (const [questName, quest] of Object.entries(gameData.quests)) {
|
|
||||||
// Check NPC references
|
|
||||||
if (quest._content?.questGiver) {
|
|
||||||
const npcPath = `${gamePath}/npcs/${quest._content.questGiver}.json`
|
|
||||||
const exists = await vfs.exists(npcPath)
|
|
||||||
if (!exists) {
|
|
||||||
errors.push(`Quest ${questName} references missing NPC: ${quest._content.questGiver}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check item rewards
|
|
||||||
if (quest._content?.rewards?.items) {
|
|
||||||
for (const item of quest._content.rewards.items) {
|
|
||||||
const itemPath = `${gamePath}/items/${item}.json`
|
|
||||||
const exists = await vfs.exists(itemPath)
|
|
||||||
if (!exists) {
|
|
||||||
warnings.push(`Quest ${questName} rewards missing item: ${item}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check balance
|
|
||||||
if (gameData.items) {
|
|
||||||
const itemPowers = []
|
|
||||||
for (const [itemName, item] of Object.entries(gameData.items)) {
|
|
||||||
if (item._content?.stats) {
|
|
||||||
const totalPower = Object.values(item._content.stats)
|
|
||||||
.reduce((a, b) => a + b, 0)
|
|
||||||
itemPowers.push({ name: itemName, power: totalPower })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Find outliers
|
|
||||||
const avgPower = itemPowers.reduce((a, b) => a + b.power, 0) / itemPowers.length
|
|
||||||
const outliers = itemPowers.filter(i => Math.abs(i.power - avgPower) > avgPower * 2)
|
|
||||||
|
|
||||||
for (const outlier of outliers) {
|
|
||||||
warnings.push(`Item ${outlier.name} may be imbalanced (power: ${outlier.power}, avg: ${avgPower})`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return { errors, warnings, valid: errors.length === 0 }
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Generate Loot Tables
|
|
||||||
```javascript
|
|
||||||
/**
|
|
||||||
* Generate weighted loot tables from item definitions
|
|
||||||
*/
|
|
||||||
async function generateLootTables(vfs, itemsPath) {
|
|
||||||
const items = await vfs.exportToJSON(itemsPath)
|
|
||||||
const tables = {
|
|
||||||
common: [],
|
|
||||||
uncommon: [],
|
|
||||||
rare: [],
|
|
||||||
epic: [],
|
|
||||||
legendary: []
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const [itemName, item] of Object.entries(items)) {
|
|
||||||
if (item._meta?.type === 'file' && item._content?.rarity) {
|
|
||||||
const entry = {
|
|
||||||
name: itemName.replace('.json', ''),
|
|
||||||
weight: getWeight(item._content.rarity),
|
|
||||||
data: item._content
|
|
||||||
}
|
|
||||||
|
|
||||||
tables[item._content.rarity.toLowerCase()].push(entry)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Normalize weights
|
|
||||||
for (const table of Object.values(tables)) {
|
|
||||||
const totalWeight = table.reduce((a, b) => a + b.weight, 0)
|
|
||||||
for (const entry of table) {
|
|
||||||
entry.probability = (entry.weight / totalWeight * 100).toFixed(2) + '%'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return tables
|
|
||||||
}
|
|
||||||
|
|
||||||
function getWeight(rarity) {
|
|
||||||
const weights = {
|
|
||||||
common: 100,
|
|
||||||
uncommon: 50,
|
|
||||||
rare: 20,
|
|
||||||
epic: 5,
|
|
||||||
legendary: 1
|
|
||||||
}
|
|
||||||
return weights[rarity.toLowerCase()] || 10
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Using These Functions
|
|
||||||
|
|
||||||
All these functions are templates that you can customize for your specific needs. To use them:
|
|
||||||
|
|
||||||
1. Copy the function you need
|
|
||||||
2. Modify it for your specific requirements
|
|
||||||
3. Use it with your VFS instance:
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
// Initialize VFS
|
|
||||||
const brain = new Brainy()
|
|
||||||
await brain.init()
|
|
||||||
const vfs = brain.vfs()
|
|
||||||
await vfs.init()
|
|
||||||
|
|
||||||
// Use your custom function
|
|
||||||
const insights = await getProjectInsights(vfs, '/my-project')
|
|
||||||
console.log(insights.recommendations)
|
|
||||||
|
|
||||||
// Combine multiple functions
|
|
||||||
const deps = await getDependencyGraph(vfs, '/src')
|
|
||||||
const cycles = await findCircularDependencies(vfs, '/src')
|
|
||||||
const untested = await findUntestedCode(vfs, '/src', '/tests')
|
|
||||||
```
|
|
||||||
|
|
||||||
Remember: These are starting points. The power of VFS is that you can combine its primitives to build exactly what you need for your domain!
|
|
||||||
|
|
@ -632,7 +632,7 @@ VFS works with all built-in Brainy storage adapters:
|
||||||
// Memory (testing/development)
|
// Memory (testing/development)
|
||||||
const brain = new Brainy({ storage: { type: 'memory' } })
|
const brain = new Brainy({ storage: { type: 'memory' } })
|
||||||
|
|
||||||
// Filesystem (local development)
|
// Filesystem (production default)
|
||||||
const brain = new Brainy({
|
const brain = new Brainy({
|
||||||
storage: {
|
storage: {
|
||||||
type: 'filesystem',
|
type: 'filesystem',
|
||||||
|
|
@ -640,52 +640,12 @@ const brain = new Brainy({
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
// S3-compatible storage (production)
|
// Both work identically with VFS
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 's3',
|
|
||||||
bucket: 'my-bucket',
|
|
||||||
region: 'us-east-1'
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Cloudflare R2 (production)
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 'r2',
|
|
||||||
accountId: 'your-account-id',
|
|
||||||
bucket: 'my-bucket'
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Google Cloud Storage (production)
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 'gcs',
|
|
||||||
bucket: 'my-bucket',
|
|
||||||
projectId: 'my-project'
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Azure Blob Storage (production)
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: {
|
|
||||||
type: 'azure',
|
|
||||||
accountName: 'myaccount',
|
|
||||||
containerName: 'my-container'
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// OPFS (browser-native persistence)
|
|
||||||
const brain = new Brainy({ storage: { type: 'opfs' } })
|
|
||||||
|
|
||||||
// TypeAware storage (optimized for entity types)
|
|
||||||
const brain = new Brainy({ storage: { type: 'typeaware' } })
|
|
||||||
|
|
||||||
// All work identically with VFS
|
|
||||||
const vfs = new VirtualFileSystem(brain)
|
const vfs = new VirtualFileSystem(brain)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
For off-site backup of the filesystem artifact, snapshot `path` from your scheduler (`gsutil rsync`, `aws s3 sync`, `rclone`, `tar`).
|
||||||
|
|
||||||
**Custom Storage Adapters:** Redis, PostgreSQL, and other databases can be added via the [extension system](../api/EXTENSIBILITY.md). See `src/config/extensibleConfig.ts` for examples.
|
**Custom Storage Adapters:** Redis, PostgreSQL, and other databases can be added via the [extension system](../api/EXTENSIBILITY.md). See `src/config/extensibleConfig.ts` for examples.
|
||||||
|
|
||||||
### Best Practices
|
### Best Practices
|
||||||
|
|
|
||||||
|
|
@ -358,7 +358,6 @@ VFS scales to millions of files:
|
||||||
- O(1) path lookup with caching
|
- O(1) path lookup with caching
|
||||||
- Efficient graph traversal for directories
|
- Efficient graph traversal for directories
|
||||||
- Chunked storage for large files
|
- Chunked storage for large files
|
||||||
- Distributed storage backend support
|
|
||||||
- Vector search scales with HNSW index
|
- Vector search scales with HNSW index
|
||||||
|
|
||||||
## Method Availability
|
## Method Availability
|
||||||
|
|
|
||||||
|
|
@ -1,682 +0,0 @@
|
||||||
# VFS Examples and Scenarios
|
|
||||||
|
|
||||||
> **⚠️ DOCUMENTATION IN PROGRESS**: This document is being updated to reflect recent VFS architecture changes. Many examples below reference deprecated Knowledge Layer methods. Updated examples coming soon. For current VFS capabilities, see [VFS_CORE.md](./VFS_CORE.md) and [SEMANTIC_VFS.md](./SEMANTIC_VFS.md).
|
|
||||||
|
|
||||||
## Real-World Scenarios
|
|
||||||
|
|
||||||
This document demonstrates how VFS enables powerful real-world applications with semantic search, relationships, and AI-powered concept extraction.
|
|
||||||
|
|
||||||
### Legend
|
|
||||||
- ✅ **Real VFS methods** - Fully implemented and working
|
|
||||||
- 📝 **User functions** - Templates available in [USER_FUNCTIONS.md](./USER_FUNCTIONS.md)
|
|
||||||
- 🔮 **Future features** - Not yet available (AI augmentations)
|
|
||||||
|
|
||||||
**Note:** All ✅ marked methods are production-ready. For 📝 methods, see USER_FUNCTIONS.md for implementation templates.
|
|
||||||
|
|
||||||
## Scenario 1: Collaborative Novel Writing
|
|
||||||
|
|
||||||
Multiple authors working on a shared universe with recurring characters and locations.
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
async function novelWritingProject() {
|
|
||||||
const brain = new Brainy({ storage: { type: 'file', path: './novel-data' } })
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
const vfs = brain.vfs()
|
|
||||||
await vfs.init()
|
|
||||||
|
|
||||||
// Create project structure ✅
|
|
||||||
await vfs.mkdir('/novel')
|
|
||||||
await vfs.mkdir('/novel/chapters')
|
|
||||||
await vfs.mkdir('/novel/characters')
|
|
||||||
await vfs.mkdir('/novel/worldbuilding')
|
|
||||||
|
|
||||||
// Define main characters as files with rich metadata ✅
|
|
||||||
await vfs.writeFile('/novel/characters/elena-blackwood.md', `
|
|
||||||
# Elena Blackwood
|
|
||||||
|
|
||||||
A skilled detective with a mysterious past.
|
|
||||||
|
|
||||||
## Character Profile
|
|
||||||
- Age: 32
|
|
||||||
- Occupation: Private Investigator
|
|
||||||
- Skills: Deduction, combat, languages
|
|
||||||
- Personality: Determined, secretive, compassionate
|
|
||||||
`, {
|
|
||||||
metadata: {
|
|
||||||
characterType: 'protagonist',
|
|
||||||
tags: ['detective', 'mysterious', 'protagonist']
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
await vfs.writeFile('/novel/characters/marcus-void.md', `
|
|
||||||
# Marcus Void
|
|
||||||
description: 'A wealthy industrialist with dark secrets',
|
|
||||||
attributes: {
|
|
||||||
age: 45,
|
|
||||||
occupation: 'CEO of Void Industries',
|
|
||||||
traits: ['ruthless', 'charismatic', 'brilliant']
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Create location entities
|
|
||||||
const city = await vfs.createEntity({
|
|
||||||
name: 'Neo Tokyo',
|
|
||||||
type: 'location',
|
|
||||||
description: 'A sprawling cyberpunk metropolis',
|
|
||||||
attributes: {
|
|
||||||
population: '40 million',
|
|
||||||
districts: ['Shibuya-5', 'Crypto Quarter', 'Old Town'],
|
|
||||||
atmosphere: 'neon-lit, rain-soaked, vertical'
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Link entities ✅
|
|
||||||
await vfs.linkEntities(protagonist.id, city.id, 'lives_in')
|
|
||||||
await vfs.linkEntities(protagonist.id, antagonist.id, 'investigates')
|
|
||||||
|
|
||||||
// Write chapters with automatic entity tracking
|
|
||||||
await vfs.writeFile('/novel/chapters/chapter1.md', `
|
|
||||||
# Chapter 1: Rain in Neo Tokyo
|
|
||||||
|
|
||||||
Elena Blackwood stood at the edge of Shibuya-5, watching the rain cascade down
|
|
||||||
the neon-lit towers. The case file on Marcus Void burned in her pocket.
|
|
||||||
|
|
||||||
She had been tracking Void Industries for months, following a trail of
|
|
||||||
disappeared scientists and mysterious experiments. Tonight, she would finally
|
|
||||||
infiltrate the Crypto Quarter facility.
|
|
||||||
`)
|
|
||||||
|
|
||||||
// Multiple authors can work simultaneously ✅
|
|
||||||
vfs.setUser('author-alice')
|
|
||||||
await vfs.writeFile('/novel/chapters/chapter2.md', `
|
|
||||||
# Chapter 2: The Void Industries Tower
|
|
||||||
|
|
||||||
Marcus Void gazed down at Neo Tokyo from his penthouse office. Somewhere
|
|
||||||
in those rain-slicked streets, Elena Blackwood was hunting him...
|
|
||||||
`)
|
|
||||||
|
|
||||||
vfs.setUser('author-bob')
|
|
||||||
await vfs.appendFile('/novel/chapters/chapter2.md', `
|
|
||||||
|
|
||||||
He smiled. Let her come. The trap was already set.
|
|
||||||
`)
|
|
||||||
|
|
||||||
// Track character appearances across chapters ✅
|
|
||||||
const elenaAppearances = await vfs.findEntityOccurrences(protagonist.id)
|
|
||||||
console.log('Elena appears in:', elenaAppearances.map(f => f.path))
|
|
||||||
|
|
||||||
// Find all locations mentioned ✅
|
|
||||||
const locations = await vfs.listEntities({ type: 'location' })
|
|
||||||
|
|
||||||
// Generate character relationship graph ✅
|
|
||||||
const relationships = await vfs.getEntityGraph(protagonist.id, { depth: 2 })
|
|
||||||
|
|
||||||
// Track plot threads using concepts ✅
|
|
||||||
await vfs.createConcept({
|
|
||||||
name: 'The Void Conspiracy',
|
|
||||||
type: 'plot',
|
|
||||||
domain: 'narrative',
|
|
||||||
description: 'The main mystery involving disappeared scientists',
|
|
||||||
keywords: ['scientists', 'experiments', 'void industries', 'conspiracy']
|
|
||||||
})
|
|
||||||
|
|
||||||
// Find all chapters related to the conspiracy ✅
|
|
||||||
const conspiracyChapters = await vfs.findByConcept('The Void Conspiracy')
|
|
||||||
|
|
||||||
// Version control for revisions ✅
|
|
||||||
const chapterVersions = await vfs.getVersions('/novel/chapters/chapter1.md')
|
|
||||||
|
|
||||||
// Collaborative editing history ✅
|
|
||||||
const history = await vfs.getCollaborationHistory('/novel/chapters/chapter2.md')
|
|
||||||
console.log('Chapter 2 edited by:', history.map(h => h.user))
|
|
||||||
|
|
||||||
// Export for publishing ✅
|
|
||||||
const manuscript = await vfs.exportToMarkdown('/novel/chapters')
|
|
||||||
|
|
||||||
await vfs.close()
|
|
||||||
await brain.close()
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Scenario 2: Video Game Development
|
|
||||||
|
|
||||||
Game developers creating a complex RPG with quests, items, and NPCs.
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
async function gameDevProject() {
|
|
||||||
const brain = new Brainy({ storage: { type: 's3', bucket: 'game-assets' } })
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
const vfs = brain.vfs()
|
|
||||||
await vfs.init()
|
|
||||||
|
|
||||||
// Game project structure
|
|
||||||
await vfs.mkdir('/game')
|
|
||||||
await vfs.mkdir('/game/scripts')
|
|
||||||
await vfs.mkdir('/game/assets')
|
|
||||||
await vfs.mkdir('/game/quests')
|
|
||||||
await vfs.mkdir('/game/npcs')
|
|
||||||
await vfs.mkdir('/game/items')
|
|
||||||
|
|
||||||
// Define game systems as concepts
|
|
||||||
await vfs.createConcept({
|
|
||||||
name: 'Combat System',
|
|
||||||
type: 'system',
|
|
||||||
domain: 'gameplay',
|
|
||||||
keywords: ['damage', 'health', 'attacks', 'defense']
|
|
||||||
})
|
|
||||||
|
|
||||||
await vfs.createConcept({
|
|
||||||
name: 'Quest System',
|
|
||||||
type: 'system',
|
|
||||||
domain: 'gameplay',
|
|
||||||
keywords: ['objectives', 'rewards', 'dialogue', 'progression']
|
|
||||||
})
|
|
||||||
|
|
||||||
// Create NPC entities
|
|
||||||
const questGiver = await vfs.createEntity({
|
|
||||||
name: 'Elder Sage',
|
|
||||||
type: 'npc',
|
|
||||||
description: 'Wise old man who gives the main quest',
|
|
||||||
attributes: {
|
|
||||||
location: 'Village Square',
|
|
||||||
questsOffered: ['The Ancient Artifact', 'Lost Knowledge'],
|
|
||||||
dialogue: {
|
|
||||||
greeting: "Welcome, young adventurer...",
|
|
||||||
questStart: "I have a task of great importance..."
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Create item entities
|
|
||||||
const artifact = await vfs.createEntity({
|
|
||||||
name: 'Crystal of Power',
|
|
||||||
type: 'item',
|
|
||||||
description: 'A legendary artifact with immense magical power',
|
|
||||||
attributes: {
|
|
||||||
rarity: 'Legendary',
|
|
||||||
stats: { magic: 100, wisdom: 50 },
|
|
||||||
questItem: true,
|
|
||||||
lore: 'Forged in the age of dragons...'
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Write quest scripts
|
|
||||||
await vfs.writeFile('/game/quests/main_quest.json', JSON.stringify({
|
|
||||||
id: 'main_quest_01',
|
|
||||||
name: 'The Ancient Artifact',
|
|
||||||
description: 'Retrieve the Crystal of Power from the Dark Tower',
|
|
||||||
objectives: [
|
|
||||||
{ id: 'obj_1', description: 'Speak to the Elder Sage' },
|
|
||||||
{ id: 'obj_2', description: 'Travel to the Dark Tower' },
|
|
||||||
{ id: 'obj_3', description: 'Defeat the Guardian' },
|
|
||||||
{ id: 'obj_4', description: 'Retrieve the Crystal of Power' }
|
|
||||||
],
|
|
||||||
rewards: {
|
|
||||||
experience: 5000,
|
|
||||||
gold: 1000,
|
|
||||||
items: ['Crystal of Power']
|
|
||||||
}
|
|
||||||
}, null, 2))
|
|
||||||
|
|
||||||
// Link quest elements
|
|
||||||
await vfs.linkEntities(questGiver.id, 'main_quest_01', 'gives_quest')
|
|
||||||
await vfs.linkEntities('main_quest_01', artifact.id, 'rewards_item')
|
|
||||||
|
|
||||||
// Combat script with system tracking
|
|
||||||
await vfs.writeFile('/game/scripts/combat.js', `
|
|
||||||
class CombatSystem {
|
|
||||||
calculateDamage(attacker, defender) {
|
|
||||||
const baseDamage = attacker.stats.attack
|
|
||||||
const defense = defender.stats.defense
|
|
||||||
return Math.max(1, baseDamage - defense)
|
|
||||||
}
|
|
||||||
|
|
||||||
executeAttack(attacker, defender) {
|
|
||||||
const damage = this.calculateDamage(attacker, defender)
|
|
||||||
defender.health -= damage
|
|
||||||
return { damage, remaining: defender.health }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export default CombatSystem
|
|
||||||
`)
|
|
||||||
|
|
||||||
// Asset management ✅
|
|
||||||
await vfs.writeFile('/game/assets/sprites/elder_sage.png', spriteData)
|
|
||||||
await vfs.setMetadata('/game/assets/sprites/elder_sage.png', {
|
|
||||||
dimensions: '64x64',
|
|
||||||
animations: ['idle', 'talking'],
|
|
||||||
artist: 'Alice',
|
|
||||||
license: 'CC-BY-4.0'
|
|
||||||
})
|
|
||||||
|
|
||||||
// Track dependencies ✅
|
|
||||||
await vfs.addRelationship('/game/quests/main_quest.json', '/game/npcs/elder_sage.json', 'uses')
|
|
||||||
await vfs.addRelationship('/game/scripts/combat.js', '/game/systems/stats.js', 'imports')
|
|
||||||
|
|
||||||
// Find all content related to combat ✅
|
|
||||||
const combatFiles = await vfs.findByConcept('Combat System')
|
|
||||||
|
|
||||||
// Get all NPCs in a specific location ✅
|
|
||||||
const villageNPCs = await vfs.searchEntities({
|
|
||||||
type: 'npc',
|
|
||||||
where: { 'attributes.location': 'Village Square' }
|
|
||||||
})
|
|
||||||
|
|
||||||
// Track game balance changes ✅
|
|
||||||
const balanceHistory = await vfs.getHistory('/game/data/balance.json')
|
|
||||||
|
|
||||||
// Collaborative development tracking ✅
|
|
||||||
await vfs.addTodo('/game/quests/main_quest.json', {
|
|
||||||
task: 'Add voice dialogue triggers',
|
|
||||||
priority: 'medium',
|
|
||||||
status: 'pending',
|
|
||||||
assignee: 'audio-team'
|
|
||||||
})
|
|
||||||
|
|
||||||
// Export for build system ✅
|
|
||||||
const gameData = await vfs.exportToJSON('/game')
|
|
||||||
|
|
||||||
await vfs.close()
|
|
||||||
await brain.close()
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Scenario 3: Software Development Project
|
|
||||||
|
|
||||||
Building a web application with full project management.
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
async function softwareProject() {
|
|
||||||
const brain = new Brainy({ storage: { type: 'r2', bucket: 'project-files' } })
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
const vfs = brain.vfs()
|
|
||||||
await vfs.init()
|
|
||||||
|
|
||||||
// Import existing git repository ✅
|
|
||||||
await vfs.importFromGit('/local/repos/webapp', '/project')
|
|
||||||
|
|
||||||
// Define architectural concepts
|
|
||||||
await vfs.createConcept({
|
|
||||||
name: 'Authentication',
|
|
||||||
type: 'architecture',
|
|
||||||
domain: 'backend',
|
|
||||||
keywords: ['jwt', 'login', 'session', 'oauth'],
|
|
||||||
relatedConcepts: ['Security', 'User Management']
|
|
||||||
})
|
|
||||||
|
|
||||||
await vfs.createConcept({
|
|
||||||
name: 'State Management',
|
|
||||||
type: 'architecture',
|
|
||||||
domain: 'frontend',
|
|
||||||
keywords: ['redux', 'context', 'store', 'actions']
|
|
||||||
})
|
|
||||||
|
|
||||||
// Write source code
|
|
||||||
await vfs.writeFile('/project/src/auth/login.ts', `
|
|
||||||
import { User } from '../models/User'
|
|
||||||
import { generateJWT } from '../utils/jwt'
|
|
||||||
|
|
||||||
export async function login(email: string, password: string): Promise<string> {
|
|
||||||
const user = await User.findByEmail(email)
|
|
||||||
|
|
||||||
if (!user || !user.verifyPassword(password)) {
|
|
||||||
throw new Error('Invalid credentials')
|
|
||||||
}
|
|
||||||
|
|
||||||
return generateJWT(user.id)
|
|
||||||
}
|
|
||||||
`)
|
|
||||||
|
|
||||||
// Track imports and dependencies
|
|
||||||
await vfs.addRelationship('/project/src/auth/login.ts', '/project/src/models/User.ts', 'imports')
|
|
||||||
await vfs.addRelationship('/project/src/auth/login.ts', '/project/src/utils/jwt.ts', 'imports')
|
|
||||||
|
|
||||||
// Add inline TODOs
|
|
||||||
await vfs.addTodo('/project/src/auth/login.ts', {
|
|
||||||
task: 'Add rate limiting',
|
|
||||||
priority: 'high',
|
|
||||||
status: 'pending',
|
|
||||||
assignee: 'security-team',
|
|
||||||
due: '2025-02-01'
|
|
||||||
})
|
|
||||||
|
|
||||||
await vfs.addTodo('/project/src/auth/login.ts', {
|
|
||||||
task: 'Implement OAuth providers',
|
|
||||||
priority: 'medium',
|
|
||||||
status: 'in_progress',
|
|
||||||
assignee: 'backend-team'
|
|
||||||
})
|
|
||||||
|
|
||||||
// Configuration files
|
|
||||||
await vfs.writeFile('/project/config/database.json', JSON.stringify({
|
|
||||||
development: {
|
|
||||||
host: 'localhost',
|
|
||||||
port: 5432,
|
|
||||||
database: 'webapp_dev'
|
|
||||||
},
|
|
||||||
production: {
|
|
||||||
host: '${DB_HOST}',
|
|
||||||
port: 5432,
|
|
||||||
database: 'webapp_prod'
|
|
||||||
}
|
|
||||||
}, null, 2))
|
|
||||||
|
|
||||||
// Set security metadata
|
|
||||||
await vfs.setMetadata('/project/config/database.json', {
|
|
||||||
sensitive: true,
|
|
||||||
environment: 'all',
|
|
||||||
lastReviewed: '2025-01-15',
|
|
||||||
reviewer: 'security-team'
|
|
||||||
})
|
|
||||||
|
|
||||||
// Tests with relationship to source
|
|
||||||
await vfs.writeFile('/project/tests/auth/login.test.ts', `
|
|
||||||
import { login } from '../../src/auth/login'
|
|
||||||
|
|
||||||
describe('Authentication', () => {
|
|
||||||
test('valid login returns JWT', async () => {
|
|
||||||
const token = await login('user@example.com', 'password')
|
|
||||||
expect(token).toBeDefined()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
`)
|
|
||||||
|
|
||||||
await vfs.addRelationship('/project/tests/auth/login.test.ts', '/project/src/auth/login.ts', 'tests')
|
|
||||||
|
|
||||||
// Documentation
|
|
||||||
await vfs.writeFile('/project/docs/API.md', `
|
|
||||||
# API Documentation
|
|
||||||
|
|
||||||
## Authentication
|
|
||||||
|
|
||||||
### POST /api/login
|
|
||||||
Authenticates a user and returns a JWT token.
|
|
||||||
|
|
||||||
**Request:**
|
|
||||||
\`\`\`json
|
|
||||||
{
|
|
||||||
"email": "user@example.com",
|
|
||||||
"password": "secret"
|
|
||||||
}
|
|
||||||
\`\`\`
|
|
||||||
`)
|
|
||||||
|
|
||||||
// Find all files needing security review ✅
|
|
||||||
const securityFiles = await vfs.search('authentication password jwt oauth', {
|
|
||||||
path: '/project/src',
|
|
||||||
type: 'file'
|
|
||||||
})
|
|
||||||
|
|
||||||
// Get project insights 📝 (see USER_FUNCTIONS.md for getProjectInsights)
|
|
||||||
const insights = await getProjectInsights(vfs, '/project') // User function
|
|
||||||
console.log('Most modified files:', insights.hotspots)
|
|
||||||
console.log('Key concepts:', insights.concepts)
|
|
||||||
console.log('Team activity:', insights.contributors)
|
|
||||||
|
|
||||||
// Find circular dependencies 📝 (see USER_FUNCTIONS.md)
|
|
||||||
const circularDeps = await findCircularDependencies(vfs, '/project/src') // User function
|
|
||||||
|
|
||||||
// Get test coverage relationships 📝 (see USER_FUNCTIONS.md)
|
|
||||||
const untested = await findUntestedCode(vfs, '/project/src') // User function
|
|
||||||
|
|
||||||
// Track technical debt ✅
|
|
||||||
const todos = await vfs.getAllTodos('/project')
|
|
||||||
const highPriorityDebt = todos.filter(t => t.priority === 'high' && t.status === 'pending')
|
|
||||||
|
|
||||||
// Generate dependency graph 📝 (see USER_FUNCTIONS.md)
|
|
||||||
const depGraph = await getDependencyGraph(vfs, '/project/src') // User function
|
|
||||||
|
|
||||||
// Find similar code (potential refactoring) 📝 (see USER_FUNCTIONS.md)
|
|
||||||
const similarCode = await findSimilarCode(vfs, '/project/src/auth/login.ts', {
|
|
||||||
threshold: 0.8,
|
|
||||||
minLines: 10
|
|
||||||
}) // User function
|
|
||||||
|
|
||||||
// Export for CI/CD ✅ (Knowledge Layer provides wrapper)
|
|
||||||
await vfs.exportToGit('/project', '/tmp/build-output')
|
|
||||||
|
|
||||||
// Collaborative features ✅ / 🔮
|
|
||||||
vfs.setUser('developer-alice') // ✅ Real method
|
|
||||||
// const conflicts = await vfs.detectConflicts('/project/src/auth/login.ts') // 🔮 Future feature
|
|
||||||
|
|
||||||
await vfs.close()
|
|
||||||
await brain.close()
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Scenario 4: Multi-Project Knowledge Base
|
|
||||||
|
|
||||||
All projects in one Brainy instance, sharing concepts and cross-referencing.
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
async function unifiedKnowledgeBase() {
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: { type: 'file', path: './knowledge-base' }
|
|
||||||
})
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
const vfs = brain.vfs()
|
|
||||||
await vfs.init()
|
|
||||||
|
|
||||||
// Create separate project spaces
|
|
||||||
await vfs.mkdir('/novel')
|
|
||||||
await vfs.mkdir('/game')
|
|
||||||
await vfs.mkdir('/webapp')
|
|
||||||
|
|
||||||
// Shared universe - characters appear in both novel and game
|
|
||||||
const sharedCharacter = await vfs.createEntity({
|
|
||||||
name: 'Elena Blackwood',
|
|
||||||
type: 'character',
|
|
||||||
description: 'Protagonist appearing in multiple works'
|
|
||||||
})
|
|
||||||
|
|
||||||
// Novel chapter mentioning Elena
|
|
||||||
await vfs.writeFile('/novel/chapter1.md', `
|
|
||||||
Elena Blackwood stood at the edge of the digital void...
|
|
||||||
`)
|
|
||||||
|
|
||||||
// Game script featuring Elena as NPC
|
|
||||||
await vfs.writeFile('/game/npcs/elena.json', JSON.stringify({
|
|
||||||
name: 'Elena Blackwood',
|
|
||||||
type: 'ally',
|
|
||||||
dialogue: ['I\'ve seen this before in my world...']
|
|
||||||
}))
|
|
||||||
|
|
||||||
// Web app has Elena as example user
|
|
||||||
await vfs.writeFile('/webapp/seeds/users.json', JSON.stringify([{
|
|
||||||
name: 'Elena Blackwood',
|
|
||||||
email: 'elena@example.com',
|
|
||||||
role: 'detective'
|
|
||||||
}]))
|
|
||||||
|
|
||||||
// Cross-project entity tracking ✅
|
|
||||||
const elenaOccurrences = await vfs.findEntityOccurrences(sharedCharacter.id)
|
|
||||||
console.log('Elena appears across projects:', elenaOccurrences)
|
|
||||||
|
|
||||||
// Shared technical concepts
|
|
||||||
const authConcept = await vfs.createConcept({
|
|
||||||
name: 'Authentication',
|
|
||||||
type: 'technical',
|
|
||||||
domain: 'software'
|
|
||||||
})
|
|
||||||
|
|
||||||
// Find auth implementations across all projects ✅
|
|
||||||
const authImplementations = await vfs.findByConcept('Authentication')
|
|
||||||
// Returns: /webapp/src/auth.js, /game/scripts/player-auth.js, etc.
|
|
||||||
|
|
||||||
// Cross-project relationships ✅
|
|
||||||
await vfs.addRelationship('/novel/chapter1.md', '/game/story/intro.txt', 'inspires')
|
|
||||||
await vfs.addRelationship('/game/npcs/elena.json', '/novel/characters/elena.md', 'based_on')
|
|
||||||
|
|
||||||
// Universal search across all projects ✅
|
|
||||||
const results = await vfs.search('Elena Blackwood authentication', {
|
|
||||||
path: '/',
|
|
||||||
recursive: true
|
|
||||||
})
|
|
||||||
|
|
||||||
// Project statistics 📝 (see USER_FUNCTIONS.md for getProjectStats)
|
|
||||||
const novelStats = await getProjectStats(vfs, '/novel') // User function
|
|
||||||
const gameStats = await getProjectStats(vfs, '/game') // User function
|
|
||||||
const webappStats = await getProjectStats(vfs, '/webapp') // User function
|
|
||||||
|
|
||||||
console.log('Total files:', novelStats.fileCount + gameStats.fileCount + webappStats.fileCount)
|
|
||||||
console.log('Total size:', novelStats.totalSize + gameStats.totalSize + webappStats.totalSize)
|
|
||||||
|
|
||||||
// Knowledge graph visualization data 🔮 (future feature)
|
|
||||||
// const knowledgeGraph = await vfs.getGlobalKnowledgeGraph() // Not yet implemented
|
|
||||||
// Returns nodes (files, entities, concepts) and edges (relationships)
|
|
||||||
|
|
||||||
// Find connections between projects 🔮 (future feature)
|
|
||||||
// const crossProjectLinks = await vfs.findCrossProjectLinks() // Not yet implemented
|
|
||||||
|
|
||||||
// Unified timeline ✅
|
|
||||||
const timeline = await vfs.getTimeline({
|
|
||||||
from: '2025-01-01',
|
|
||||||
to: '2025-12-31'
|
|
||||||
})
|
|
||||||
|
|
||||||
await vfs.close()
|
|
||||||
await brain.close()
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Advanced Features
|
|
||||||
|
|
||||||
### Semantic Code Analysis 📝
|
|
||||||
|
|
||||||
These are user functions - see [USER_FUNCTIONS.md](./USER_FUNCTIONS.md) for implementation templates:
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Find security vulnerabilities (user function example)
|
|
||||||
const vulnerabilities = await findPatterns(vfs, [
|
|
||||||
'eval(',
|
|
||||||
'innerHTML =',
|
|
||||||
'SQL injection',
|
|
||||||
'hardcoded password'
|
|
||||||
])
|
|
||||||
|
|
||||||
// Find code smells (user function example)
|
|
||||||
const codeSmells = await analyzeCodeQuality(vfs, '/src', {
|
|
||||||
checkDuplication: true,
|
|
||||||
checkComplexity: true,
|
|
||||||
checkNaming: true
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### AI-Powered Features 🔮
|
|
||||||
|
|
||||||
**Note:** These features require AI integration and are not yet available.
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Future: Generate documentation
|
|
||||||
// const docs = await vfs.generateDocumentation('/src/auth/login.ts')
|
|
||||||
|
|
||||||
// Future: Suggest refactorings
|
|
||||||
// const refactorings = await vfs.suggestRefactorings('/src/utils.js')
|
|
||||||
|
|
||||||
// Future: Auto-complete code
|
|
||||||
// const completion = await vfs.completeCode('/src/api.ts', { line: 42, column: 10 })
|
|
||||||
```
|
|
||||||
|
|
||||||
### Migration and Backup 🔮
|
|
||||||
|
|
||||||
**Note:** These features are planned but not yet implemented.
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Future: Backup with full history
|
|
||||||
// await vfs.createBackup('/backup/2025-01-20.brainy')
|
|
||||||
|
|
||||||
// Future: Migrate between storage backends
|
|
||||||
// const migration = await vfs.migrate({
|
|
||||||
// from: { type: 'file', path: './old-data' },
|
|
||||||
// to: { type: 's3', bucket: 'new-bucket' }
|
|
||||||
// })
|
|
||||||
|
|
||||||
// Future: Incremental sync
|
|
||||||
// await vfs.sync('/local/path', '/vfs/path', {
|
|
||||||
// bidirectional: true,
|
|
||||||
// conflictStrategy: 'newest'
|
|
||||||
// })
|
|
||||||
```
|
|
||||||
|
|
||||||
### Performance at Scale
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Handle millions of files ✅
|
|
||||||
for (let i = 0; i < 1000000; i++) {
|
|
||||||
await vfs.writeFile(`/data/file${i}.txt`, `Content ${i}`)
|
|
||||||
// Uses chunking, compression, and efficient indexing
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fast parallel operations ✅
|
|
||||||
await Promise.all([
|
|
||||||
vfs.writeFile('/file1.txt', 'data1'),
|
|
||||||
vfs.writeFile('/file2.txt', 'data2'),
|
|
||||||
vfs.writeFile('/file3.txt', 'data3')
|
|
||||||
])
|
|
||||||
|
|
||||||
// Bulk write operations ✅
|
|
||||||
const files = [
|
|
||||||
{ path: '/data/file1.txt', content: 'Content 1' },
|
|
||||||
{ path: '/data/file2.txt', content: 'Content 2' },
|
|
||||||
// ... more files
|
|
||||||
]
|
|
||||||
await vfs.bulkWrite(files)
|
|
||||||
|
|
||||||
// Bulk imports 🔮 (future feature)
|
|
||||||
// await vfs.bulkImport('/massive/dataset', {
|
|
||||||
// parallel: 10,
|
|
||||||
// batchSize: 1000,
|
|
||||||
// progress: (count, total) => console.log(`${count}/${total}`)
|
|
||||||
// })
|
|
||||||
```
|
|
||||||
|
|
||||||
## Implementation Status
|
|
||||||
|
|
||||||
### ✅ Fully Implemented Features
|
|
||||||
|
|
||||||
All methods marked with ✅ are production-ready:
|
|
||||||
|
|
||||||
1. **Core VFS Operations**: mkdir, writeFile, readFile, appendFile, stat, readdir, etc.
|
|
||||||
2. **Entity System**: createEntity, linkEntities, findEntityOccurrences, updateEntity, getEntityGraph
|
|
||||||
3. **Concept System**: createConcept, findByConcept
|
|
||||||
4. **Knowledge Layer**: Event recording, semantic versioning, collaboration tracking
|
|
||||||
5. **Search**: Triple Intelligence (vector + field + graph)
|
|
||||||
6. **Git Integration**: importFromGit, exportToGit
|
|
||||||
7. **Export Formats**: exportToMarkdown, exportToJSON
|
|
||||||
8. **Bulk Operations**: bulkWrite for efficient batch processing
|
|
||||||
9. **Project Management**: todos, metadata, relationships
|
|
||||||
|
|
||||||
### 📝 User Functions
|
|
||||||
|
|
||||||
Methods marked with 📝 are domain-specific functions that you can implement using VFS primitives. See [USER_FUNCTIONS.md](./USER_FUNCTIONS.md) for ready-to-use templates:
|
|
||||||
|
|
||||||
- **Code Analysis**: getDependencyGraph, findCircularDependencies, findUntestedCode, findSimilarCode
|
|
||||||
- **Creative Writing**: trackCharacterArc, generateStoryBible
|
|
||||||
- **Game Development**: validateGameData, generateLootTables
|
|
||||||
- **Project Management**: getProjectInsights, generateSprintReport
|
|
||||||
- **Export Formats**: exportToEpub, exportToStaticSite
|
|
||||||
|
|
||||||
### 🔮 Future Features
|
|
||||||
|
|
||||||
Methods marked with 🔮 require AI integration or are planned for future releases:
|
|
||||||
|
|
||||||
- **AI-Powered**: generateDocumentation, suggestRefactorings, completeCode
|
|
||||||
- **Advanced Analysis**: detectConflicts, getGlobalKnowledgeGraph, findCrossProjectLinks
|
|
||||||
- **Migration Tools**: createBackup, migrate, sync, bulkImport
|
|
||||||
|
|
||||||
## Real Implementation Guarantees
|
|
||||||
|
|
||||||
- **No Mocks**: Every ✅ method is fully functional
|
|
||||||
- **Real Storage**: Uses Brainy entities with embeddings
|
|
||||||
- **Real Search**: Triple Intelligence combining vectors, fields, and graphs
|
|
||||||
- **Real Relationships**: Graph-based connections via brain.relate()
|
|
||||||
- **Production Ready**: Complete error handling, async/await, resource cleanup
|
|
||||||
|
|
||||||
The VFS + Knowledge Layer combination provides a solid foundation for intelligent applications. Use the ✅ methods directly, implement 📝 functions as needed for your domain, and stay tuned for 🔮 features.
|
|
||||||
|
|
@ -110,7 +110,8 @@ While using standard graph types, VFS adds domain-specific metadata for efficien
|
||||||
### 1. Cross-Domain Queries
|
### 1. Cross-Domain Queries
|
||||||
```javascript
|
```javascript
|
||||||
// Find all documents (including VFS files) about "machine learning"
|
// Find all documents (including VFS files) about "machine learning"
|
||||||
const results = await brain.search('machine learning', {
|
const results = await brain.find({
|
||||||
|
query: 'machine learning',
|
||||||
where: { type: NounType.Document }
|
where: { type: NounType.Document }
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
@ -118,13 +119,13 @@ const results = await brain.search('machine learning', {
|
||||||
### 2. Graph Traversal
|
### 2. Graph Traversal
|
||||||
```javascript
|
```javascript
|
||||||
// Find all entities contained in a directory
|
// Find all entities contained in a directory
|
||||||
const contained = await brain.getRelations({
|
const contained = await brain.related({
|
||||||
from: directoryId,
|
from: directoryId,
|
||||||
type: VerbType.Contains
|
type: VerbType.Contains
|
||||||
})
|
})
|
||||||
|
|
||||||
// Find what contains a file (parent directories)
|
// Find what contains a file (parent directories)
|
||||||
const parents = await brain.getRelations({
|
const parents = await brain.related({
|
||||||
to: fileId,
|
to: fileId,
|
||||||
type: VerbType.Contains
|
type: VerbType.Contains
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -142,26 +142,30 @@ await brain.init() // Required!
|
||||||
await brain.vfs.writeFile(...) // Now this works
|
await brain.vfs.writeFile(...) // Now this works
|
||||||
```
|
```
|
||||||
|
|
||||||
## Fork Support
|
## Snapshot Support
|
||||||
|
|
||||||
VFS works seamlessly with Brainy's Copy-on-Write (COW) fork feature:
|
VFS files are ordinary entities, so they participate fully in the 8.0 Db
|
||||||
|
API: a persisted snapshot contains every VFS file, and a snapshot opened
|
||||||
|
with `Brainy.load()` serves VFS reads at the snapshot's state:
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
const brain = new Brainy({ storage: { type: 'memory' } })
|
const brain = new Brainy({ storage: { type: 'filesystem', path: './data' } })
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
||||||
// Create files in parent
|
// Create files
|
||||||
await brain.vfs.writeFile('/config.json', '{"version": 1}')
|
await brain.vfs.writeFile('/config.json', '{"version": 1}')
|
||||||
|
|
||||||
// Fork inherits parent's files
|
// Snapshot the whole store (VFS files included)
|
||||||
const fork = await brain.fork('experiment')
|
const pin = brain.now()
|
||||||
const files = await fork.vfs.readdir('/') // Sees parent's config.json!
|
await pin.persist('/backups/with-vfs')
|
||||||
|
await pin.release()
|
||||||
|
|
||||||
// Fork modifications are isolated
|
// Later changes never touch the snapshot
|
||||||
await fork.vfs.writeFile('/test.txt', 'Fork only')
|
await brain.vfs.writeFile('/config.json', '{"version": 2}')
|
||||||
await brain.vfs.readdir('/') // Parent doesn't see test.txt
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
See [Snapshots & Time Travel](../guides/snapshots-and-time-travel.md).
|
||||||
|
|
||||||
## FAQ
|
## FAQ
|
||||||
|
|
||||||
### Q: Do I need to call `vfs.init()` anymore?
|
### Q: Do I need to call `vfs.init()` anymore?
|
||||||
|
|
@ -177,7 +181,7 @@ await brain.vfs.readdir('/') // Parent doesn't see test.txt
|
||||||
**A:** Yes, VFS configuration is passed through `brain.init()` config. The VFS-specific options are applied during auto-initialization.
|
**A:** Yes, VFS configuration is passed through `brain.init()` config. The VFS-specific options are applied during auto-initialization.
|
||||||
|
|
||||||
### Q: Does this work with all storage adapters?
|
### Q: Does this work with all storage adapters?
|
||||||
**A:** Yes! VFS auto-initialization works with all storage adapters: Memory, FileSystem, OPFS, S3, R2, Azure Blob, and Google Cloud Storage.
|
**A:** Yes! VFS auto-initialization works with both shipped adapters (FileSystem and Memory) and any plugin-provided storage adapter.
|
||||||
|
|
||||||
### Q: What if I need multiple VFS instances?
|
### Q: What if I need multiple VFS instances?
|
||||||
**A:** Each Brainy instance has its own VFS. Create multiple Brainy instances if you need multiple VFS instances.
|
**A:** Each Brainy instance has its own VFS. Create multiple Brainy instances if you need multiple VFS instances.
|
||||||
|
|
@ -187,4 +191,4 @@ await brain.vfs.readdir('/') // Parent doesn't see test.txt
|
||||||
- [VFS Quick Start](./QUICK_START.md) - 5-minute setup guide
|
- [VFS Quick Start](./QUICK_START.md) - 5-minute setup guide
|
||||||
- [VFS API Guide](./VFS_API_GUIDE.md) - Complete API reference
|
- [VFS API Guide](./VFS_API_GUIDE.md) - Complete API reference
|
||||||
- [Common Patterns](./COMMON_PATTERNS.md) - Best practices and patterns
|
- [Common Patterns](./COMMON_PATTERNS.md) - Best practices and patterns
|
||||||
- [Instant Fork](../features/instant-fork.md) - VFS + Copy-on-Write
|
- [Snapshots & Time Travel](../guides/snapshots-and-time-travel.md) - Backups and point-in-time reads
|
||||||
|
|
|
||||||
|
|
@ -1,233 +0,0 @@
|
||||||
/**
|
|
||||||
* Directory Import with Entity Extraction Caching Example
|
|
||||||
*
|
|
||||||
* Demonstrates:
|
|
||||||
* - Importing directories with progress tracking
|
|
||||||
* - Entity extraction caching for performance
|
|
||||||
* - Relationship detection with confidence scores
|
|
||||||
* - Cache statistics monitoring
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { Brainy, NounType, VerbType } from '../src/brainy.js'
|
|
||||||
import { DirectoryImporter } from '../src/vfs/importers/DirectoryImporter.js'
|
|
||||||
import { ProgressTracker, formatProgress } from '../src/types/progress.types.js'
|
|
||||||
import { detectRelationshipsWithConfidence } from '../src/neural/relationshipConfidence.js'
|
|
||||||
|
|
||||||
async function main() {
|
|
||||||
console.log('🧠 Brainy 3.21.0 - Directory Import with Caching Example\n')
|
|
||||||
|
|
||||||
// Initialize Brainy
|
|
||||||
const brain = new Brainy({ verbose: false })
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
console.log('✅ Brainy initialized\n')
|
|
||||||
|
|
||||||
// Example 1: Import directory with entity extraction caching
|
|
||||||
console.log('📁 Example 1: Import Directory with Caching\n')
|
|
||||||
|
|
||||||
const vfs = brain.vfs
|
|
||||||
const importer = new DirectoryImporter(vfs, brain)
|
|
||||||
|
|
||||||
// Progress tracking
|
|
||||||
const tracker = ProgressTracker.create(100)
|
|
||||||
tracker.start()
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Import with progress (using async generator)
|
|
||||||
console.log('Importing directory...')
|
|
||||||
|
|
||||||
let filesProcessed = 0
|
|
||||||
for await (const progress of importer.importStream('./examples', {
|
|
||||||
batchSize: 10,
|
|
||||||
recursive: true,
|
|
||||||
generateEmbeddings: true,
|
|
||||||
extractMetadata: true
|
|
||||||
})) {
|
|
||||||
if (progress.type === 'progress') {
|
|
||||||
filesProcessed = progress.processed
|
|
||||||
const trackedProgress = tracker.update(progress.processed, progress.current)
|
|
||||||
console.log(` ${formatProgress(trackedProgress)}`)
|
|
||||||
} else if (progress.type === 'complete') {
|
|
||||||
console.log(`\n✅ Import complete! Processed ${progress.processed} files\n`)
|
|
||||||
} else if (progress.type === 'error') {
|
|
||||||
console.error(`❌ Error: ${progress.error?.message}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
tracker.complete({ filesProcessed })
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Import failed:', error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Example 2: Entity extraction with caching
|
|
||||||
console.log('\n📝 Example 2: Entity Extraction with Caching\n')
|
|
||||||
|
|
||||||
const sampleText = `
|
|
||||||
John Smith created the user authentication system for the application.
|
|
||||||
The authentication system uses JWT tokens and bcrypt for password hashing.
|
|
||||||
Mary Johnson manages the backend team that maintains the system.
|
|
||||||
The system was built using Node.js and PostgreSQL database.
|
|
||||||
`
|
|
||||||
|
|
||||||
console.log('First extraction (cache miss):')
|
|
||||||
const startTime1 = Date.now()
|
|
||||||
const entities1 = await brain.neural.extractor.extract(sampleText, {
|
|
||||||
types: [NounType.Person, NounType.Service, NounType.Technology],
|
|
||||||
confidence: 0.7,
|
|
||||||
cache: {
|
|
||||||
enabled: true,
|
|
||||||
ttl: 7 * 24 * 60 * 60 * 1000, // 7 days
|
|
||||||
invalidateOn: 'hash'
|
|
||||||
}
|
|
||||||
})
|
|
||||||
const time1 = Date.now() - startTime1
|
|
||||||
console.log(` Extracted ${entities1.length} entities in ${time1}ms`)
|
|
||||||
console.log(` Entities: ${entities1.map(e => e.text).join(', ')}\n`)
|
|
||||||
|
|
||||||
console.log('Second extraction (cache hit):')
|
|
||||||
const startTime2 = Date.now()
|
|
||||||
const entities2 = await brain.neural.extractor.extract(sampleText, {
|
|
||||||
types: [NounType.Person, NounType.Service, NounType.Technology],
|
|
||||||
confidence: 0.7,
|
|
||||||
cache: {
|
|
||||||
enabled: true,
|
|
||||||
invalidateOn: 'hash'
|
|
||||||
}
|
|
||||||
})
|
|
||||||
const time2 = Date.now() - startTime2
|
|
||||||
console.log(` Extracted ${entities2.length} entities in ${time2}ms`)
|
|
||||||
console.log(` Speedup: ${Math.round(time1 / time2)}x faster!\n`)
|
|
||||||
|
|
||||||
// Show cache statistics
|
|
||||||
const cacheStats = brain.neural.extractor.getCacheStats()
|
|
||||||
console.log('📊 Cache Statistics:')
|
|
||||||
console.log(` Hits: ${cacheStats.hits}`)
|
|
||||||
console.log(` Misses: ${cacheStats.misses}`)
|
|
||||||
console.log(` Hit Rate: ${(cacheStats.hitRate * 100).toFixed(1)}%`)
|
|
||||||
console.log(` Total Entries: ${cacheStats.totalEntries}`)
|
|
||||||
console.log(` Avg Entities per Entry: ${cacheStats.averageEntitiesPerEntry}\n`)
|
|
||||||
|
|
||||||
// Example 3: Relationship detection with confidence
|
|
||||||
console.log('🔗 Example 3: Relationship Detection with Confidence\n')
|
|
||||||
|
|
||||||
const relationships = detectRelationshipsWithConfidence(
|
|
||||||
entities1,
|
|
||||||
sampleText,
|
|
||||||
{
|
|
||||||
minConfidence: 0.6,
|
|
||||||
maxDistance: 100,
|
|
||||||
useProximityBoost: true,
|
|
||||||
usePatternMatching: true,
|
|
||||||
useStructuralAnalysis: true
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
console.log(`Detected ${relationships.length} relationships:\n`)
|
|
||||||
for (const rel of relationships.slice(0, 5)) { // Show top 5
|
|
||||||
console.log(` ${rel.sourceEntity.text} --[${rel.verbType}]--> ${rel.targetEntity.text}`)
|
|
||||||
console.log(` Confidence: ${(rel.confidence * 100).toFixed(1)}%`)
|
|
||||||
console.log(` Evidence: ${rel.evidence.reasoning}`)
|
|
||||||
console.log(` Method: ${rel.evidence.method}`)
|
|
||||||
console.log(` Source: "${rel.evidence.sourceText?.substring(0, 60)}..."\n`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Example 4: Create relationships in graph with confidence
|
|
||||||
console.log('📊 Example 4: Creating Relationships in Graph\n')
|
|
||||||
|
|
||||||
const createdRelations = []
|
|
||||||
for (const rel of relationships.slice(0, 3)) { // Create top 3
|
|
||||||
try {
|
|
||||||
// Add entities to brain
|
|
||||||
const sourceId = await brain.add({
|
|
||||||
data: rel.sourceEntity.text,
|
|
||||||
type: rel.sourceEntity.type,
|
|
||||||
metadata: {
|
|
||||||
confidence: rel.sourceEntity.confidence,
|
|
||||||
extractedFrom: 'sample text'
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
const targetId = await brain.add({
|
|
||||||
data: rel.targetEntity.text,
|
|
||||||
type: rel.targetEntity.type,
|
|
||||||
metadata: {
|
|
||||||
confidence: rel.targetEntity.confidence,
|
|
||||||
extractedFrom: 'sample text'
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Create relationship with confidence
|
|
||||||
const relationId = await brain.relate({
|
|
||||||
from: sourceId,
|
|
||||||
to: targetId,
|
|
||||||
type: rel.verbType,
|
|
||||||
confidence: rel.confidence,
|
|
||||||
evidence: rel.evidence,
|
|
||||||
metadata: {
|
|
||||||
autoDetected: true,
|
|
||||||
detectedAt: new Date().toISOString()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
createdRelations.push(relationId)
|
|
||||||
console.log(` ✅ Created: ${rel.sourceEntity.text} → ${rel.targetEntity.text}`)
|
|
||||||
} catch (error) {
|
|
||||||
console.error(` ❌ Failed to create relationship:`, error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`\n✅ Created ${createdRelations.length} relationships in knowledge graph`)
|
|
||||||
|
|
||||||
// Example 5: Query relationships by confidence
|
|
||||||
console.log('\n🔍 Example 5: Query High-Confidence Relationships\n')
|
|
||||||
|
|
||||||
const allRelations = await brain.getRelations({
|
|
||||||
limit: 100
|
|
||||||
})
|
|
||||||
|
|
||||||
const highConfidence = allRelations.filter(r => (r.confidence || 0) >= 0.7)
|
|
||||||
console.log(`Found ${highConfidence.length} high-confidence relationships (≥70%):\n`)
|
|
||||||
|
|
||||||
for (const rel of highConfidence.slice(0, 5)) {
|
|
||||||
console.log(` ${rel.from} → ${rel.to} (${rel.type})`)
|
|
||||||
console.log(` Confidence: ${((rel.confidence || 0) * 100).toFixed(1)}%`)
|
|
||||||
if (rel.evidence) {
|
|
||||||
console.log(` Method: ${rel.evidence.method}`)
|
|
||||||
console.log(` Reasoning: ${rel.evidence.reasoning}\n`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Example 6: Cache management
|
|
||||||
console.log('🧹 Example 6: Cache Management\n')
|
|
||||||
|
|
||||||
console.log('Cache operations:')
|
|
||||||
|
|
||||||
// Cleanup expired entries
|
|
||||||
const cleaned = brain.neural.extractor.cleanupCache()
|
|
||||||
console.log(` Cleaned ${cleaned} expired entries`)
|
|
||||||
|
|
||||||
// Invalidate specific cache entry
|
|
||||||
const invalidated = brain.neural.extractor.invalidateCache('hash:abc123')
|
|
||||||
console.log(` Invalidated entry: ${invalidated}`)
|
|
||||||
|
|
||||||
// Get final stats
|
|
||||||
const finalStats = brain.neural.extractor.getCacheStats()
|
|
||||||
console.log(` Final cache size: ${finalStats.totalEntries} entries`)
|
|
||||||
console.log(` Memory used: ~${Math.round(finalStats.cacheSize / 1024)}KB`)
|
|
||||||
|
|
||||||
// Clear all cache (optional)
|
|
||||||
// brain.neural.extractor.clearCache()
|
|
||||||
// console.log(' Cleared entire cache')
|
|
||||||
|
|
||||||
console.log('\n✨ Example complete!')
|
|
||||||
console.log('\n📚 Key Takeaways:')
|
|
||||||
console.log(' • Entity extraction caching provides 10-100x speedup on repeated content')
|
|
||||||
console.log(' • Progress tracking gives real-time feedback for long operations')
|
|
||||||
console.log(' • Relationship confidence helps filter low-quality connections')
|
|
||||||
console.log(' • Evidence tracking makes relationships explainable and debuggable')
|
|
||||||
console.log(' • All features are opt-in and backward compatible')
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run example
|
|
||||||
main().catch(console.error)
|
|
||||||
|
|
@ -1,274 +0,0 @@
|
||||||
/**
|
|
||||||
* Instant Fork Usage Examples (v5.0.0)
|
|
||||||
*
|
|
||||||
* COW is ZERO-CONFIG in v5.0.0:
|
|
||||||
* - No setup needed
|
|
||||||
* - No configuration
|
|
||||||
* - Just works automatically
|
|
||||||
*
|
|
||||||
* This example shows how EASY and ELEGANT fork() is for developers.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { Brainy } from '@soulcraft/brainy'
|
|
||||||
|
|
||||||
// ========== Example 1: Basic Fork (Zero Config) ==========
|
|
||||||
|
|
||||||
async function basicFork() {
|
|
||||||
// Create Brainy (COW automatic!)
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: { adapter: 'memory' }
|
|
||||||
})
|
|
||||||
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Add some data
|
|
||||||
await brain.add({ noun: 'user', data: { name: 'Alice' } })
|
|
||||||
await brain.add({ noun: 'user', data: { name: 'Bob' } })
|
|
||||||
|
|
||||||
// Fork instantly (1-2 seconds, even with millions of entities!)
|
|
||||||
const experiment = await brain.fork('experiment')
|
|
||||||
|
|
||||||
// Make changes in fork (doesn't affect main)
|
|
||||||
await experiment.add({ noun: 'user', data: { name: 'Charlie' } })
|
|
||||||
|
|
||||||
// Main brain unchanged
|
|
||||||
console.log(await brain.find({ noun: 'user' })) // Alice, Bob
|
|
||||||
console.log(await experiment.find({ noun: 'user' })) // Alice, Bob, Charlie
|
|
||||||
|
|
||||||
// That's it! Zero config, pure elegance.
|
|
||||||
}
|
|
||||||
|
|
||||||
// ========== Example 2: Safe Experimentation ==========
|
|
||||||
|
|
||||||
async function safeExperimentation() {
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: { adapter: 'filesystem', path: './data' }
|
|
||||||
})
|
|
||||||
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Production data
|
|
||||||
const users = await brain.find({ noun: 'user' })
|
|
||||||
console.log(`Production: ${users.length} users`)
|
|
||||||
|
|
||||||
// Test a risky operation in fork
|
|
||||||
const test = await brain.fork('test-migration')
|
|
||||||
|
|
||||||
// Run migration on fork (safe!)
|
|
||||||
for (const user of await test.find({ noun: 'user' })) {
|
|
||||||
await test.update(user.id, {
|
|
||||||
email: user.data.email.toLowerCase() // Risky transformation
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test it
|
|
||||||
const results = await test.find({ noun: 'user' })
|
|
||||||
|
|
||||||
if (results.every(r => r.data.email === r.data.email.toLowerCase())) {
|
|
||||||
console.log('✅ Migration safe, apply to production')
|
|
||||||
// Apply to production...
|
|
||||||
} else {
|
|
||||||
console.log('❌ Migration failed, discard fork')
|
|
||||||
await test.destroy() // Discard failed experiment
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ========== Example 3: A/B Testing ==========
|
|
||||||
|
|
||||||
async function abTesting() {
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: { adapter: 's3', bucket: 'my-data' }
|
|
||||||
})
|
|
||||||
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Create variant A (control)
|
|
||||||
const variantA = await brain.fork('variant-a')
|
|
||||||
|
|
||||||
// Create variant B (test)
|
|
||||||
const variantB = await brain.fork('variant-b')
|
|
||||||
|
|
||||||
// Run different algorithms
|
|
||||||
await variantA.processWithAlgorithm('current')
|
|
||||||
await variantB.processWithAlgorithm('improved')
|
|
||||||
|
|
||||||
// Compare results
|
|
||||||
const metricsA = await variantA.getMetrics()
|
|
||||||
const metricsB = await variantB.getMetrics()
|
|
||||||
|
|
||||||
console.log('A:', metricsA.accuracy)
|
|
||||||
console.log('B:', metricsB.accuracy)
|
|
||||||
|
|
||||||
// Choose winner, apply to main
|
|
||||||
if (metricsB.accuracy > metricsA.accuracy) {
|
|
||||||
console.log('B wins! Deploying...')
|
|
||||||
// Merge B to main
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ========== Example 4: Time Travel (Enterprise) ==========
|
|
||||||
|
|
||||||
async function timeTravel() {
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: { adapter: 'memory' }
|
|
||||||
})
|
|
||||||
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Add data over time
|
|
||||||
await brain.add({ noun: 'doc', data: { version: 1 } })
|
|
||||||
await brain.commit({ message: 'Version 1' })
|
|
||||||
|
|
||||||
const yesterday = Date.now() - 86400000 // 24 hours ago
|
|
||||||
|
|
||||||
await brain.add({ noun: 'doc', data: { version: 2 } })
|
|
||||||
await brain.commit({ message: 'Version 2' })
|
|
||||||
|
|
||||||
// Query as of yesterday (time travel!)
|
|
||||||
const snapshot = await brain.asOf(yesterday)
|
|
||||||
|
|
||||||
const docs = await snapshot.find({ noun: 'doc' })
|
|
||||||
console.log(docs[0].data.version) // 1 (from yesterday!)
|
|
||||||
|
|
||||||
// Zero config, pure magic ✨
|
|
||||||
}
|
|
||||||
|
|
||||||
// ========== Example 5: Backup & Restore (Instant) ==========
|
|
||||||
|
|
||||||
async function instantBackup() {
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: { adapter: 'memory' }
|
|
||||||
})
|
|
||||||
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Work with production data
|
|
||||||
await brain.add({ noun: 'important', data: { value: 'critical' } })
|
|
||||||
|
|
||||||
// Instant backup (1-2 seconds!)
|
|
||||||
const backup = await brain.fork('backup-2024-01-01')
|
|
||||||
|
|
||||||
// Continue working
|
|
||||||
await brain.delete((await brain.find({ noun: 'important' }))[0].id)
|
|
||||||
|
|
||||||
// Oops! Need to restore
|
|
||||||
const restored = await brain.rollback(backup.getCurrentCommit())
|
|
||||||
|
|
||||||
// Data restored instantly!
|
|
||||||
console.log(await brain.find({ noun: 'important' })) // Back!
|
|
||||||
}
|
|
||||||
|
|
||||||
// ========== Example 6: Distributed Teams (Fork per Developer) ==========
|
|
||||||
|
|
||||||
async function distributedDevelopment() {
|
|
||||||
// Main production brain
|
|
||||||
const production = new Brainy({
|
|
||||||
storage: { adapter: 's3', bucket: 'production-data' }
|
|
||||||
})
|
|
||||||
|
|
||||||
await production.init()
|
|
||||||
|
|
||||||
// Alice's fork
|
|
||||||
const alice = await production.fork('alice-feature-x')
|
|
||||||
|
|
||||||
// Bob's fork
|
|
||||||
const bob = await production.fork('bob-feature-y')
|
|
||||||
|
|
||||||
// Both work independently (zero conflicts!)
|
|
||||||
await alice.add({ noun: 'feature', data: { name: 'X' } })
|
|
||||||
await bob.add({ noun: 'feature', data: { name: 'Y' } })
|
|
||||||
|
|
||||||
// Merge when ready
|
|
||||||
await production.merge(alice, { author: 'Alice' })
|
|
||||||
await production.merge(bob, { author: 'Bob' })
|
|
||||||
|
|
||||||
// Production has both features!
|
|
||||||
}
|
|
||||||
|
|
||||||
// ========== Example 7: VFS Snapshots ==========
|
|
||||||
|
|
||||||
async function vfsSnapshots() {
|
|
||||||
const brain = new Brainy({
|
|
||||||
storage: { adapter: 'memory' },
|
|
||||||
vfs: { enabled: true }
|
|
||||||
})
|
|
||||||
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Create file structure
|
|
||||||
await brain.vfs.writeFile('/project/README.md', '# My Project')
|
|
||||||
await brain.vfs.writeFile('/project/src/index.ts', 'console.log("v1")')
|
|
||||||
|
|
||||||
await brain.commit({ message: 'Initial project' })
|
|
||||||
|
|
||||||
// Fork for refactoring
|
|
||||||
const refactor = await brain.fork('refactor')
|
|
||||||
|
|
||||||
// Refactor code in fork
|
|
||||||
await refactor.vfs.writeFile('/project/src/index.ts', 'console.log("v2")')
|
|
||||||
await refactor.vfs.mkdir('/project/src/utils')
|
|
||||||
|
|
||||||
// Test refactor
|
|
||||||
// ...
|
|
||||||
|
|
||||||
// Switch to refactor branch if successful
|
|
||||||
// await brain.checkout('refactor')
|
|
||||||
}
|
|
||||||
|
|
||||||
// ========== The Key: It's All Zero Config! ==========
|
|
||||||
|
|
||||||
async function zeroConfigDemo() {
|
|
||||||
// This is ALL you need:
|
|
||||||
const brain = new Brainy({ storage: { adapter: 'memory' } })
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// COW is automatic:
|
|
||||||
// ✅ Compression: automatic (based on data type)
|
|
||||||
// ✅ Deduplication: automatic (content-addressable)
|
|
||||||
// ✅ Caching: automatic (LRU with memory limits)
|
|
||||||
// ✅ Reference counting: automatic (safe deletion)
|
|
||||||
// ✅ Garbage collection: automatic (optional manual trigger)
|
|
||||||
|
|
||||||
// Fork is instant:
|
|
||||||
const fork = await brain.fork() // < 2 seconds even at 1M entities
|
|
||||||
|
|
||||||
// That's it! No configuration, no complexity, pure elegance.
|
|
||||||
}
|
|
||||||
|
|
||||||
// ========== API Summary ==========
|
|
||||||
|
|
||||||
/*
|
|
||||||
DEVELOPER API (v5.0.0):
|
|
||||||
|
|
||||||
// Fork operations
|
|
||||||
brain.fork(branch?) → Create instant clone
|
|
||||||
brain.asOf(timestamp) → Time-travel query (Enterprise)
|
|
||||||
brain.rollback(commitHash) → Restore to commit (Enterprise)
|
|
||||||
brain.commit(options?) → Create commit (automatic)
|
|
||||||
|
|
||||||
// Branch operations
|
|
||||||
brain.listBranches() → List all branches
|
|
||||||
brain.checkout(branch) → Switch to branch
|
|
||||||
brain.deleteBranch(branch) → Delete a branch
|
|
||||||
|
|
||||||
// Time queries
|
|
||||||
brain.getHistory(limit?) → Get commit history
|
|
||||||
brain.findAtTime(timestamp) → Find commit at time
|
|
||||||
brain.getStats() → Get storage stats
|
|
||||||
|
|
||||||
// All existing Brainy APIs work the same:
|
|
||||||
brain.add()
|
|
||||||
brain.find()
|
|
||||||
brain.search()
|
|
||||||
brain.vfs.*
|
|
||||||
brain.query() // Triple Intelligence
|
|
||||||
|
|
||||||
ZERO CONFIG REQUIRED!
|
|
||||||
*/
|
|
||||||
|
|
||||||
// Run examples
|
|
||||||
if (require.main === module) {
|
|
||||||
basicFork()
|
|
||||||
.then(() => console.log('✅ All examples complete'))
|
|
||||||
.catch(console.error)
|
|
||||||
}
|
|
||||||
|
|
@ -1,25 +0,0 @@
|
||||||
{
|
|
||||||
"_name_or_path": "sentence-transformers/all-MiniLM-L6-v2",
|
|
||||||
"architectures": [
|
|
||||||
"BertModel"
|
|
||||||
],
|
|
||||||
"attention_probs_dropout_prob": 0.1,
|
|
||||||
"classifier_dropout": null,
|
|
||||||
"gradient_checkpointing": false,
|
|
||||||
"hidden_act": "gelu",
|
|
||||||
"hidden_dropout_prob": 0.1,
|
|
||||||
"hidden_size": 384,
|
|
||||||
"initializer_range": 0.02,
|
|
||||||
"intermediate_size": 1536,
|
|
||||||
"layer_norm_eps": 1e-12,
|
|
||||||
"max_position_embeddings": 512,
|
|
||||||
"model_type": "bert",
|
|
||||||
"num_attention_heads": 12,
|
|
||||||
"num_hidden_layers": 6,
|
|
||||||
"pad_token_id": 0,
|
|
||||||
"position_embedding_type": "absolute",
|
|
||||||
"transformers_version": "4.29.2",
|
|
||||||
"type_vocab_size": 2,
|
|
||||||
"use_cache": true,
|
|
||||||
"vocab_size": 30522
|
|
||||||
}
|
|
||||||
Binary file not shown.
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue