Initial commit

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,32 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: '[BUG] '
+labels: bug
+assignees: ''
+---
+
+## Bug Description
+A clear and concise description of what the bug is.
+
+## Reproduction Steps
+Steps to reproduce the behavior:
+1. Initialize BrainyData with '...'
+2. Call method '....'
+3. See error
+
+## Expected Behavior
+A clear and concise description of what you expected to happen.
+
+## Environment
+- Brainy version: [e.g. 0.9.4]
+- Environment: [e.g. Browser, Node.js, serverless]
+- Browser (if applicable): [e.g. Chrome, Safari]
+- Node.js version (if applicable): [e.g. 23.11.0]
+- Operating System: [e.g. Windows 10, macOS Monterey, Ubuntu 22.04]
+
+## Additional Context
+Add any other context about the problem here. If applicable, include code snippets, error messages, or screenshots.
+
+## Possible Solution
+If you have suggestions on how to fix the issue, please describe them here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,22 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: '[FEATURE] '
+labels: enhancement
+assignees: ''
+---
+
+## Problem Statement
+A clear and concise description of what problem this feature would solve. For example: "I'm always frustrated when [...]"
+
+## Proposed Solution
+A clear and concise description of what you want to happen.
+
+## Alternative Solutions
+A clear and concise description of any alternative solutions or features you've considered.
+
+## Use Case
+Describe a concrete use case that highlights the value of this feature.
+
+## Additional Context
+Add any other context, code examples, or references about the feature request here.

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,27 @@
+## Description
+Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context.
+
+Fixes # (issue)
+
+## Type of change
+Please delete options that are not relevant.
+
+- [ ] Bug fix (non-breaking change which fixes an issue)
+- [ ] New feature (non-breaking change which adds functionality)
+- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
+- [ ] Documentation update
+- [ ] Performance improvement
+- [ ] Code refactoring (no functional changes)
+
+## How Has This Been Tested?
+Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce.
+
+## Checklist:
+- [ ] My code follows the style guidelines of this project
+- [ ] I have performed a self-review of my own code
+- [ ] I have commented my code, particularly in hard-to-understand areas
+- [ ] I have made corresponding changes to the documentation
+- [ ] My changes generate no new warnings
+- [ ] I have added tests that prove my fix is effective or that my feature works
+- [ ] New and existing unit tests pass locally with my changes
+- [ ] Any dependent changes have been merged and published in downstream modules

.github/workflows/deploy-demo.yml

@@ -0,0 +1,68 @@
+name: Deploy Demo to GitHub Pages
+
+on:
+  push:
+    branches: [ main ]
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout 🛎️
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js 🔧
+        uses: actions/setup-node@v3
+        with:
+          node-version: '20'
+          cache: 'npm'
+
+      - name: Install dependencies 📦
+        run: npm install --legacy-peer-deps
+
+      - name: Build project 🏗️
+        run: |
+          npm run build
+          npm run build:browser
+
+      - name: Prepare deployment 📦
+        run: |
+          mkdir -p _site
+          mkdir -p _site/demo
+          mkdir -p _site/dist
+          cp index.html _site/
+          cp demo/index.html _site/demo/
+          cp -r dist/* _site/dist/
+          cp brainy.png _site/
+          # Copy dist directly to demo/dist for easier access
+          mkdir -p _site/demo/dist
+          cp -r dist/* _site/demo/dist/
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: _site
+
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.gitignore

@@ -0,0 +1,42 @@
+# Build output
+/tmp
+/out-tsc
+/dist
+/cloud-wrapper/dist
+
+# Dependencies
+/node_modules
+/cloud-wrapper/node_modules
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+/.pnp
+.pnp.js
+
+# Coverage directory
+/coverage
+
+# Environment files
+.env
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+# IDE files
+.vscode/
+.idea/
+*.iml
+*.iws
+*.ipr
+*.sublime-workspace
+*.sublime-project
+
+# OS files
+.DS_Store
+Thumbs.db
+
+# Data directories created by FileSystemStorage
+/brainy-data
+/custom-data
+/clean-history.sh

.npmignore

@@ -0,0 +1,40 @@
+# Exclude source maps
+*.map
+**/*.map
+
+# Development files
+node_modules/
+src/
+tests/
+examples/
+.github/
+.vscode/
+.idea/
+cloud-wrapper/
+scripts/
+
+# Configuration files
+.eslintrc
+.prettierrc
+tsconfig*.json
+rollup.config.js
+jest.config.js
+
+# Build artifacts
+emocoverage/
+.nyc_output/
+
+# Large files
+# Include the logo but exclude other PNGs
+!brainy.png
+*.png
+encoded-image.*
+README.demo.md
+scalingStrategy.md
+
+# Misc
+.DS_Store
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*

CODE_OF_CONDUCT.md

@@ -0,0 +1,128 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Project maintainers are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Project maintainers have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned with this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the project maintainers responsible for enforcement at
+conduct@soulcraft.com.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All project maintainers are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Project maintainers will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from project maintainers, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.

CONTRIBUTING.md

@@ -0,0 +1,86 @@
+<div align="center">
+<img src="./brainy.png" alt="Brainy Logo" width="200"/>
+
+# Contributing to Brainy
+
+</div>
+
+Thank you for your interest in contributing to Brainy! This document provides guidelines and instructions for
+contributing to the project.
+
+We welcome contributions of all kinds, including bug fixes, feature additions, documentation improvements, and more.
+By participating in this project, you agree to abide by our [Code of Conduct](CODE_OF_CONDUCT.md).
+
+## Commit Message Guidelines
+
+When contributing to this project, please write clear and descriptive commit messages that explain the purpose of your
+changes. Good commit messages help maintainers understand your contributions and make the review process smoother.
+
+### Best Practices
+
+- Keep the first line concise (ideally under 50 characters)
+- Use the imperative mood ("Add feature" not "Added feature")
+- Reference issues and pull requests where appropriate
+- When necessary, provide more detailed explanations in the commit body
+
+### Examples
+
+```
+Add vector normalization option
+Fix distance calculation in HNSW search
+Update API documentation
+Add support for IndexedDB storage
+Change API parameter order
+Simplify vector comparison logic
+Update build dependencies
+```
+
+## Pull Request Process
+
+1. Ensure your code follows the project's coding standards
+2. Update the documentation if necessary
+3. Use conventional commit messages in your PR
+4. Your PR will be reviewed by maintainers and merged if approved
+
+## Development Setup
+
+1. Fork and clone the repository
+2. Install dependencies: `npm install`
+3. Build the project: `npm run build`
+
+## Code Style
+
+This project uses ESLint and Prettier for code formatting and style checking. The configuration can be found in the `package.json` file. Please ensure your code follows these standards:
+
+- Use 2 spaces for indentation
+- Use single quotes for strings
+- No semicolons
+- Trailing commas are not used
+- Maximum line length is 80 characters
+
+You can check your code style by running:
+```bash
+npx eslint src/
+```
+
+## Branching Strategy
+
+- `main` - The main branch contains the latest stable release
+- `develop` - The development branch contains the latest development changes
+- Feature branches - Create a branch from `develop` for your feature or fix
+
+When working on a new feature or fix:
+1. Create a new branch from `develop` with a descriptive name (e.g., `feature/add-vector-normalization` or `fix/distance-calculation`)
+2. Make your changes in that branch
+3. Submit a pull request to merge your branch into `develop`
+
+## Issue Reporting
+
+Before submitting a new issue, please search existing issues to avoid duplicates.
+
+- For bugs, use the bug report template
+- For feature requests, use the feature request template
+- Be as detailed as possible in your description
+- Include code examples, error messages, and screenshots if applicable
+
+Thank you for contributing to Brainy!

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Soulcraft Research
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.demo.md

@@ -0,0 +1,59 @@
+# Running the Brainy Demo
+
+The Brainy interactive demo showcases the library's features in a web browser. Follow these steps to run it:
+
+## Prerequisites
+
+- Make sure you have Node.js installed (version 23.11.0 or higher)
+- Ensure the project is built (`npm run build:all`)
+
+## Running the Demo
+
+### Option 1: Using the npm script (recommended)
+
+Run the following command from the project root:
+
+```bash
+npm run demo
+```
+
+This will start an HTTP server and automatically open the demo in your default browser.
+
+### Option 2: Manual setup
+
+1. Start an HTTP server in the project root:
+
+```bash
+npx http-server
+```
+
+2. Open your browser and navigate to:
+   http://localhost:8080/demo/index.html
+
+## Troubleshooting
+
+If you see the error "Could not load Brainy library. Please ensure the project is built and served over HTTP", check the
+following:
+
+1. Make sure you've built the project with `npm run build:all`
+2. Ensure you're accessing the demo through HTTP (not by opening the file directly)
+3. Check your browser's console for additional error messages
+
+If issues persist, try clearing your browser cache or using a private/incognito window.
+
+## Build Process
+
+The Brainy library uses a two-step build process:
+
+1. `npm run build` - Compiles TypeScript files to JavaScript (used for Node.js environments)
+2. `npm run build:browser` - Creates a browser-compatible bundle using Rollup
+
+You can run both steps together with:
+
+```bash
+npm run build:all
+```
+
+The browser bundle is created from `src/unified.ts`, which provides environment detection and adapts to browser,
+Node.js, or serverless environments. This unified approach ensures that the library works correctly across all
+environments.

cli-wrapper.js

@@ -0,0 +1,56 @@
+#!/usr/bin/env node
+
+/**
+ * CLI Wrapper Script
+ *
+ * This script serves as a wrapper for the Brainy CLI, ensuring that command-line arguments
+ * are properly passed to the CLI when invoked through npm scripts.
+ */
+
+import { spawn } from 'child_process'
+import { fileURLToPath } from 'url'
+import { dirname, join } from 'path'
+import fs from 'fs'
+
+// Get the directory of the current module
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = dirname(__filename)
+
+// Path to the actual CLI script
+const cliPath = join(__dirname, 'dist', 'cli.js')
+
+// Check if the CLI script exists
+if (!fs.existsSync(cliPath)) {
+  console.error(`Error: CLI script not found at ${cliPath}`)
+  console.error('Make sure you have built the project with "npm run build"')
+  process.exit(1)
+}
+
+// Special handling for version flags
+if (process.argv.includes('--version') || process.argv.includes('-V')) {
+  // Read version directly from package.json to ensure it's always correct
+  try {
+    const packageJsonPath = join(__dirname, 'package.json')
+    const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'))
+    console.log(packageJson.version)
+    process.exit(0)
+  } catch (error) {
+    console.error('Error loading version information:', error.message)
+    process.exit(1)
+  }
+}
+
+// Forward all arguments to the CLI script
+const args = process.argv.slice(2)
+
+// Check if npm is passing --force flag
+// When npm runs with --force, it sets the npm_config_force environment variable
+if (process.env.npm_config_force === 'true' && args.includes('clear') && !args.includes('--force') && !args.includes('-f')) {
+  args.push('--force')
+}
+
+const cli = spawn('node', [cliPath, ...args], { stdio: 'inherit' })
+
+cli.on('close', (code) => {
+  process.exit(code)
+})

cloud-wrapper/.env.example

@@ -0,0 +1,24 @@
+# Server configuration
+PORT=3000
+
+# Storage configuration
+# Options: 'filesystem', 'memory', 's3'
+STORAGE_TYPE=filesystem
+STORAGE_ROOT_DIR=./data
+
+# S3 Storage configuration (only used if STORAGE_TYPE=s3)
+S3_BUCKET_NAME=your-bucket-name
+S3_ACCESS_KEY_ID=your-access-key
+S3_SECRET_ACCESS_KEY=your-secret-key
+S3_REGION=us-east-1
+# Optional: For custom S3-compatible services like MinIO, DigitalOcean Spaces, etc.
+# S3_ENDPOINT=https://your-custom-endpoint
+
+# Embedding configuration
+# Set to 'true' to use simple embedding (faster but less accurate)
+USE_SIMPLE_EMBEDDING=false
+
+# HNSW index configuration (optional)
+# HNSW_M=16                # Max connections per node
+# HNSW_EF_CONSTRUCTION=200 # Construction candidate list size
+# HNSW_EF_SEARCH=50        # Search candidate list size

cloud-wrapper/README.md

@@ -0,0 +1,341 @@
+<div align="center">
+<img src="../brainy.png" alt="Brainy Logo" width="200"/>
+
+# Brainy Cloud Wrapper
+</div>
+
+A standalone web service wrapper for the [Brainy](https://github.com/soulcraft/brainy) vector graph database. This wrapper allows you to deploy Brainy as a RESTful API service on various cloud platforms including AWS, Google Cloud, and Cloudflare.
+
+## Features
+
+- RESTful API for all Brainy operations
+- WebSocket API for real-time updates and subscriptions
+- Model Control Protocol (MCP) service for external model access
+- Support for multiple storage backends (Memory, FileSystem, S3)
+- Configurable via environment variables
+- Deployment scripts for AWS, Google Cloud, and Cloudflare
+- Secure by default with Helmet middleware
+- Cross-origin resource sharing (CORS) support
+
+## Prerequisites
+
+- Node.js 23.11.0 or higher
+- npm or yarn
+- For cloud deployments:
+  - AWS: AWS CLI installed and configured
+  - Google Cloud: Google Cloud SDK installed and configured
+  - Cloudflare: Wrangler CLI installed and configured
+
+## Installation
+
+1. Clone the repository:
+   ```bash
+   git clone https://github.com/soulcraft/brainy.git
+   cd brainy/cloud-wrapper
+   ```
+
+2. Install dependencies:
+   ```bash
+   npm install --legacy-peer-deps
+   ```
+
+3. Create a `.env` file based on the example:
+   ```bash
+   cp .env.example .env
+   ```
+
+4. Edit the `.env` file to configure your environment.
+
+## Configuration
+
+The cloud wrapper can be configured using environment variables. See the `.env.example` file for available options.
+
+### Storage Options
+
+- `STORAGE_TYPE`: The type of storage to use. Options:
+  - `memory`: In-memory storage (default for Cloudflare)
+  - `filesystem`: File system storage (default for local and AWS/GCP)
+  - `s3`: S3-compatible storage (AWS S3, MinIO, etc.)
+  - `r2`: Cloudflare R2 storage (Cloudflare only)
+
+### S3 Storage Configuration
+
+When using `STORAGE_TYPE=s3`, the following environment variables are required:
+
+- `S3_BUCKET_NAME`: The name of the S3 bucket
+- `S3_ACCESS_KEY_ID`: Your S3 access key ID
+- `S3_SECRET_ACCESS_KEY`: Your S3 secret access key
+- `S3_REGION`: The S3 region (default: `us-east-1`)
+- `S3_ENDPOINT` (optional): Custom endpoint for S3-compatible services
+
+### MCP Service Configuration
+
+The Model Control Protocol (MCP) service can be configured using the following environment variables:
+
+- `MCP_WS_PORT`: Port for the MCP WebSocket server (if not set, WebSocket server is disabled)
+- `MCP_REST_PORT`: Port for the MCP REST server (if not set, REST server is disabled)
+- `MCP_ENABLE_AUTH`: Enable authentication for MCP requests (`true` or `false`)
+- `MCP_API_KEYS`: Comma-separated list of API keys for authentication
+- `MCP_RATE_LIMIT_REQUESTS`: Maximum number of requests per time window
+- `MCP_RATE_LIMIT_WINDOW_MS`: Time window for rate limiting in milliseconds (default: `60000`)
+- `MCP_ENABLE_CORS`: Enable CORS for MCP REST server (`true` or `false`)
+
+## Local Development
+
+1. Build the project:
+   ```bash
+   npm run build
+   ```
+
+2. Start the development server:
+   ```bash
+   npm run dev
+   ```
+
+3. The API will be available at `http://localhost:3000`.
+
+## API Endpoints
+
+Brainy Cloud Wrapper provides REST API, WebSocket API, and Model Control Protocol (MCP) API for interacting with the database.
+
+### REST API
+
+#### Status
+
+- `GET /api/status`: Get database status
+
+#### Nouns (Entities)
+
+- `POST /api/nouns`: Add a new noun
+  - Body: `{ "text": "Your text", "metadata": { ... } }`
+- `GET /api/nouns/:id`: Get a noun by ID
+- `PUT /api/nouns/:id`: Update noun metadata
+  - Body: `{ "metadata": { ... } }`
+- `DELETE /api/nouns/:id`: Delete a noun
+
+#### Search
+
+- `POST /api/search`: Search for similar nouns
+  - Body: `{ "query": "Your search query", "limit": 10 }`
+
+#### Verbs (Relationships)
+
+- `POST /api/verbs`: Add a relationship between nouns
+  - Body: `{ "sourceId": "...", "targetId": "...", "metadata": { ... } }`
+- `GET /api/verbs`: Get all relationships
+- `GET /api/verbs/source/:id`: Get relationships by source
+- `GET /api/verbs/target/:id`: Get relationships by target
+- `DELETE /api/verbs/:id`: Delete a relationship
+
+#### Database Management
+
+- `DELETE /api/clear`: Clear all data
+
+### WebSocket API
+
+The WebSocket API provides real-time communication with the Brainy database. Connect to the WebSocket server at `ws://your-server:port`.
+
+#### Message Format
+
+All WebSocket messages follow this format:
+
+```json
+{
+  "type": "messageType",
+  "id": "unique-message-id",
+  "payload": {
+    // Message-specific data
+  }
+}
+```
+
+#### Available Message Types
+
+##### Status
+- Request: `{ "type": "status" }`
+- Response: `{ "type": "status", "id": "...", "payload": { /* database status */ } }`
+
+##### Nouns (Entities)
+- Add a noun:
+  - Request: `{ "type": "addNoun", "payload": { "text": "Your text", "metadata": { ... } } }`
+  - Response: `{ "type": "addNoun", "id": "...", "payload": { "id": "new-noun-id" } }`
+
+- Get a noun:
+  - Request: `{ "type": "getNoun", "payload": { "id": "noun-id" } }`
+  - Response: `{ "type": "getNoun", "id": "...", "payload": { /* noun data */ } }`
+
+- Update a noun:
+  - Request: `{ "type": "updateNoun", "payload": { "id": "noun-id", "metadata": { ... } } }`
+  - Response: `{ "type": "updateNoun", "id": "...", "payload": { "success": true } }`
+
+- Delete a noun:
+  - Request: `{ "type": "deleteNoun", "payload": { "id": "noun-id" } }`
+  - Response: `{ "type": "deleteNoun", "id": "...", "payload": { "success": true } }`
+
+##### Search
+- Search for similar nouns:
+  - Request: `{ "type": "search", "payload": { "query": "Your search query", "limit": 10 } }`
+  - Response: `{ "type": "search", "id": "...", "payload": [ /* search results */ ] }`
+
+##### Verbs (Relationships)
+- Add a verb:
+  - Request: `{ "type": "addVerb", "payload": { "sourceId": "...", "targetId": "...", "metadata": { ... } } }`
+  - Response: `{ "type": "addVerb", "id": "...", "payload": { "success": true } }`
+
+- Get all verbs:
+  - Request: `{ "type": "getVerbs" }`
+  - Response: `{ "type": "getVerbs", "id": "...", "payload": [ /* all verbs */ ] }`
+
+- Get verbs by source:
+  - Request: `{ "type": "getVerbsBySource", "payload": { "id": "source-id" } }`
+  - Response: `{ "type": "getVerbsBySource", "id": "...", "payload": [ /* verbs */ ] }`
+
+- Get verbs by target:
+  - Request: `{ "type": "getVerbsByTarget", "payload": { "id": "target-id" } }`
+  - Response: `{ "type": "getVerbsByTarget", "id": "...", "payload": [ /* verbs */ ] }`
+
+- Delete a verb:
+  - Request: `{ "type": "deleteVerb", "payload": { "id": "verb-id" } }`
+  - Response: `{ "type": "deleteVerb", "id": "...", "payload": { "success": true } }`
+
+##### Database Management
+- Clear all data:
+  - Request: `{ "type": "clear" }`
+  - Response: `{ "type": "clear", "id": "...", "payload": { "success": true } }`
+
+#### Real-time Subscriptions
+
+The WebSocket API supports subscribing to real-time updates:
+
+- Subscribe to updates:
+  - Request: `{ "type": "subscribe", "payload": { "type": "nouns" } }`
+  - Response: `{ "type": "subscribe", "id": "...", "payload": { "success": true, "type": "nouns" } }`
+
+- Unsubscribe from updates:
+  - Request: `{ "type": "unsubscribe", "payload": { "type": "nouns" } }`
+  - Response: `{ "type": "unsubscribe", "id": "...", "payload": { "success": true, "type": "nouns" } }`
+
+Available subscription types:
+- `nouns`: Updates about nouns (added, updated, deleted)
+- `verbs`: Updates about verbs (added, deleted)
+- `searchResults`: Updates about search results
+
+When subscribed, you'll receive messages when relevant events occur:
+
+```json
+{
+  "type": "subscribe",
+  "payload": {
+    "type": "nouns",
+    "data": {
+      "type": "added",
+      "id": "noun-id",
+      "data": { /* noun data */ }
+    }
+  }
+}
+```
+
+### MCP API
+
+The Model Control Protocol (MCP) API provides a standardized interface for external models to access Brainy data and use the augmentation pipeline as tools. The MCP API is available through both WebSocket and REST endpoints.
+
+#### MCP REST API Endpoints
+
+- `POST /mcp/data`: Access Brainy data (search, get, add, etc.)
+- `POST /mcp/tools`: Execute augmentation pipeline tools
+- `POST /mcp/system`: Get system information
+- `POST /mcp/auth`: Authenticate with the MCP service
+- `GET /mcp/tools`: Get available tools
+
+#### MCP WebSocket
+
+Connect to the MCP WebSocket server at `ws://your-server:MCP_WS_PORT` and send JSON messages in the MCP format:
+
+```json
+{
+  "type": "DATA_ACCESS",
+  "requestId": "unique-request-id",
+  "version": "1.0",
+  "operation": "search",
+  "parameters": {
+    "query": "Your search query",
+    "k": 5
+  }
+}
+```
+
+For detailed documentation on the MCP API, see the [MCP documentation](../src/mcp/README.md).
+
+## Cloud Deployment
+
+### AWS Lambda and API Gateway
+
+1. Configure AWS-specific environment variables:
+   ```
+   AWS_REGION=us-east-1
+   AWS_FUNCTION_NAME=brainy-cloud-service
+   AWS_API_GATEWAY_NAME=brainy-api
+   AWS_STAGE_NAME=prod
+   AWS_ACCOUNT_ID=your-account-id
+   ```
+
+2. Deploy to AWS:
+   ```bash
+   npm run deploy:aws
+   ```
+
+### Google Cloud Run
+
+1. Configure GCP-specific environment variables:
+   ```
+   GCP_PROJECT_ID=your-project-id
+   GCP_REGION=us-central1
+   GCP_SERVICE_NAME=brainy-cloud-service
+   GCP_IMAGE_NAME=brainy-cloud-service
+   GCP_MEMORY=512Mi
+   GCP_CPU=1
+   GCP_MAX_INSTANCES=10
+   GCP_MIN_INSTANCES=0
+   ```
+
+2. Deploy to Google Cloud:
+   ```bash
+   npm run deploy:gcp
+   ```
+
+### Cloudflare Workers
+
+1. Configure Cloudflare-specific environment variables:
+   ```
+   CF_ACCOUNT_ID=your-account-id
+   CF_WORKER_NAME=brainy-cloud-service
+   CF_KV_NAMESPACE=BRAINY_STORAGE
+   CF_R2_BUCKET=brainy-storage
+   ```
+
+2. Deploy to Cloudflare:
+   ```bash
+   npm run deploy:cloudflare
+   ```
+
+## Storage Considerations
+
+### AWS Lambda
+
+When deploying to AWS Lambda, it's recommended to use S3 storage for persistence. The filesystem storage option will work but data will be lost when the Lambda function is recycled.
+
+### Google Cloud Run
+
+For Google Cloud Run, you can use either filesystem storage (for ephemeral storage) or S3-compatible storage (like Google Cloud Storage with an S3 compatibility layer).
+
+### Cloudflare Workers
+
+Cloudflare Workers have limited storage options. The recommended approach is to use:
+- Cloudflare KV for small datasets
+- Cloudflare R2 for larger datasets
+- Memory storage for temporary data
+
+## License
+
+MIT

cloud-wrapper/package.json

@@ -0,0 +1,92 @@
+{
+  "name": "@soulcraft/brainy-cloud",
+  "version": "0.1.0",
+  "description": "Cloud deployment wrapper for Brainy vector graph database",
+  "main": "dist/index.js",
+  "type": "module",
+  "scripts": {
+    "build": "tsc",
+    "start": "node dist/index.js",
+    "dev": "nodemon --exec node --experimental-specifier-resolution=node --loader ts-node/esm src/index.ts",
+    "deploy:aws": "node scripts/deploy-aws.js",
+    "deploy:gcp": "node scripts/deploy-gcp.js",
+    "deploy:cloudflare": "node scripts/deploy-cloudflare.js"
+  },
+  "keywords": [
+    "brainy",
+    "vector-database",
+    "cloud",
+    "aws",
+    "google-cloud",
+    "cloudflare"
+  ],
+  "author": "David Snelling (david@soulcraft.com)",
+  "license": "MIT",
+  "dependencies": {
+    "@soulcraft/brainy": "^0.9.2",
+    "cors": "^2.8.5",
+    "dotenv": "^16.3.1",
+    "express": "^4.18.2",
+    "helmet": "^7.1.0",
+    "morgan": "^1.10.0",
+    "uuid": "^9.0.1",
+    "ws": "^8.16.0"
+  },
+  "devDependencies": {
+    "@types/cors": "^2.8.17",
+    "@types/express": "^4.17.21",
+    "@types/morgan": "^1.9.9",
+    "@types/node": "^20.10.5",
+    "@types/uuid": "^9.0.7",
+    "@types/ws": "^8.5.10",
+    "nodemon": "^3.0.2",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.3.3"
+  },
+  "engines": {
+    "node": ">=23.11.0"
+  },
+  "prettier": {
+    "arrowParens": "always",
+    "bracketSameLine": true,
+    "bracketSpacing": true,
+    "htmlWhitespaceSensitivity": "css",
+    "printWidth": 80,
+    "proseWrap": "preserve",
+    "semi": false,
+    "singleQuote": true,
+    "tabWidth": 2,
+    "trailingComma": "none",
+    "useTabs": false
+  },
+  "eslintConfig": {
+    "root": true,
+    "extends": [
+      "eslint:recommended",
+      "plugin:@typescript-eslint/recommended"
+    ],
+    "@typescript-eslint/no-explicit-any": "off",
+    "parser": "@typescript-eslint/parser",
+    "plugins": [
+      "@typescript-eslint"
+    ],
+    "rules": {
+      "no-unused-vars": "off",
+      "@typescript-eslint/no-unused-vars": [
+        "warn",
+        {
+          "args": "after-used",
+          "argsIgnorePattern": "^_"
+        }
+      ],
+      "semi": [
+        "error",
+        "never"
+      ],
+      "@typescript-eslint/semi": [
+        "error",
+        "never"
+      ]
+    }
+  }
+}

cloud-wrapper/scripts/deploy-aws.js

@@ -0,0 +1,205 @@
+#!/usr/bin/env node
+
+/**
+ * AWS Deployment Script for Brainy Cloud Wrapper
+ * 
+ * This script helps deploy the Brainy cloud wrapper to AWS Lambda and API Gateway.
+ * It uses the AWS SDK to create and configure the necessary resources.
+ * 
+ * Prerequisites:
+ * - AWS CLI installed and configured with appropriate credentials
+ * - Node.js 23.11.0 or higher
+ * - Brainy cloud wrapper built (npm run build)
+ */
+
+import { execSync } from 'child_process';
+import fs from 'fs';
+import path from 'path';
+import dotenv from 'dotenv';
+
+// Load environment variables
+dotenv.config();
+
+// Configuration
+const config = {
+  region: process.env.AWS_REGION || 'us-east-1',
+  functionName: process.env.AWS_FUNCTION_NAME || 'brainy-cloud-service',
+  s3Bucket: process.env.S3_BUCKET_NAME,
+  s3Key: process.env.S3_ACCESS_KEY_ID,
+  s3Secret: process.env.S3_SECRET_ACCESS_KEY,
+  apiGatewayName: process.env.AWS_API_GATEWAY_NAME || 'brainy-api',
+  stageName: process.env.AWS_STAGE_NAME || 'prod'
+};
+
+// Validate configuration
+if (!config.s3Bucket && process.env.STORAGE_TYPE === 's3') {
+  console.error('Error: S3 bucket name is required when using S3 storage');
+  process.exit(1);
+}
+
+// Create deployment package
+function createDeploymentPackage() {
+  console.log('Creating deployment package...');
+  
+  try {
+    // Create a temporary directory for the deployment package
+    if (!fs.existsSync('deploy')) {
+      fs.mkdirSync('deploy');
+    }
+    
+    // Copy necessary files
+    execSync('cp -r dist deploy/');
+    execSync('cp package.json deploy/');
+    execSync('cp .env deploy/');
+    
+    // Create a zip file
+    execSync('cd deploy && zip -r ../deployment.zip .');
+    
+    console.log('Deployment package created successfully');
+  } catch (error) {
+    console.error('Error creating deployment package:', error);
+    process.exit(1);
+  }
+}
+
+// Deploy to AWS Lambda
+function deployToLambda() {
+  console.log('Deploying to AWS Lambda...');
+  
+  try {
+    // Check if function exists
+    try {
+      execSync(`aws lambda get-function --function-name ${config.functionName} --region ${config.region}`);
+      
+      // Update existing function
+      execSync(`aws lambda update-function-code --function-name ${config.functionName} --zip-file fileb://deployment.zip --region ${config.region}`);
+      console.log(`Lambda function ${config.functionName} updated successfully`);
+    } catch (error) {
+      // Create new function
+      execSync(`aws lambda create-function --function-name ${config.functionName} --runtime nodejs20.x --handler dist/index.handler --zip-file fileb://deployment.zip --role arn:aws:iam::${process.env.AWS_ACCOUNT_ID}:role/lambda-basic-execution --region ${config.region}`);
+      console.log(`Lambda function ${config.functionName} created successfully`);
+    }
+    
+    // Configure environment variables
+    const envVars = {
+      Variables: {
+        NODE_ENV: 'production',
+        STORAGE_TYPE: process.env.STORAGE_TYPE || 'filesystem',
+        S3_BUCKET_NAME: config.s3Bucket,
+        S3_ACCESS_KEY_ID: config.s3Key,
+        S3_SECRET_ACCESS_KEY: config.s3Secret,
+        S3_REGION: config.region
+      }
+    };
+    
+    execSync(`aws lambda update-function-configuration --function-name ${config.functionName} --environment '${JSON.stringify(envVars)}' --region ${config.region}`);
+    console.log('Lambda function environment variables configured');
+    
+  } catch (error) {
+    console.error('Error deploying to AWS Lambda:', error);
+    process.exit(1);
+  }
+}
+
+// Create API Gateway
+function createApiGateway() {
+  console.log('Creating API Gateway...');
+  
+  try {
+    // Check if API Gateway exists
+    let apiId;
+    try {
+      const result = execSync(`aws apigateway get-rest-apis --region ${config.region}`).toString();
+      const apis = JSON.parse(result).items;
+      const api = apis.find(api => api.name === config.apiGatewayName);
+      
+      if (api) {
+        apiId = api.id;
+        console.log(`Using existing API Gateway: ${apiId}`);
+      } else {
+        throw new Error('API Gateway not found');
+      }
+    } catch (error) {
+      // Create new API Gateway
+      const result = execSync(`aws apigateway create-rest-api --name ${config.apiGatewayName} --region ${config.region}`).toString();
+      apiId = JSON.parse(result).id;
+      console.log(`API Gateway created: ${apiId}`);
+    }
+    
+    // Get root resource ID
+    const resourcesResult = execSync(`aws apigateway get-resources --rest-api-id ${apiId} --region ${config.region}`).toString();
+    const rootResourceId = JSON.parse(resourcesResult).items.find(resource => resource.path === '/').id;
+    
+    // Create proxy resource
+    let proxyResourceId;
+    try {
+      const proxyResource = JSON.parse(resourcesResult).items.find(resource => resource.path === '/{proxy+}');
+      if (proxyResource) {
+        proxyResourceId = proxyResource.id;
+      } else {
+        throw new Error('Proxy resource not found');
+      }
+    } catch (error) {
+      const proxyResult = execSync(`aws apigateway create-resource --rest-api-id ${apiId} --parent-id ${rootResourceId} --path-part {proxy+} --region ${config.region}`).toString();
+      proxyResourceId = JSON.parse(proxyResult).id;
+    }
+    
+    // Create ANY method
+    try {
+      execSync(`aws apigateway put-method --rest-api-id ${apiId} --resource-id ${proxyResourceId} --http-method ANY --authorization-type NONE --region ${config.region}`);
+    } catch (error) {
+      console.log('Method already exists, skipping...');
+    }
+    
+    // Create integration
+    try {
+      execSync(`aws apigateway put-integration --rest-api-id ${apiId} --resource-id ${proxyResourceId} --http-method ANY --type AWS_PROXY --integration-http-method POST --uri arn:aws:apigateway:${config.region}:lambda:path/2015-03-31/functions/arn:aws:lambda:${config.region}:${process.env.AWS_ACCOUNT_ID}:function:${config.functionName}/invocations --region ${config.region}`);
+    } catch (error) {
+      console.log('Integration already exists, skipping...');
+    }
+    
+    // Deploy API
+    execSync(`aws apigateway create-deployment --rest-api-id ${apiId} --stage-name ${config.stageName} --region ${config.region}`);
+    
+    console.log(`API Gateway deployed: https://${apiId}.execute-api.${config.region}.amazonaws.com/${config.stageName}`);
+    
+  } catch (error) {
+    console.error('Error creating API Gateway:', error);
+    process.exit(1);
+  }
+}
+
+// Clean up
+function cleanup() {
+  console.log('Cleaning up...');
+  
+  try {
+    execSync('rm -rf deploy');
+    execSync('rm -f deployment.zip');
+    
+    console.log('Cleanup completed');
+  } catch (error) {
+    console.error('Error during cleanup:', error);
+  }
+}
+
+// Main function
+async function main() {
+  try {
+    console.log('Starting AWS deployment...');
+    
+    createDeploymentPackage();
+    deployToLambda();
+    createApiGateway();
+    cleanup();
+    
+    console.log('Deployment completed successfully');
+  } catch (error) {
+    console.error('Deployment failed:', error);
+    cleanup();
+    process.exit(1);
+  }
+}
+
+// Run the script
+main();

cloud-wrapper/scripts/deploy-cloudflare.js

@@ -0,0 +1,245 @@
+#!/usr/bin/env node
+
+/**
+ * Cloudflare Deployment Script for Brainy Cloud Wrapper
+ * 
+ * This script helps deploy the Brainy cloud wrapper to Cloudflare Workers.
+ * It uses the Wrangler CLI to create and configure the necessary resources.
+ * 
+ * Prerequisites:
+ * - Wrangler CLI installed and configured with appropriate credentials
+ * - Node.js 23.11.0 or higher
+ * - Brainy cloud wrapper built (npm run build)
+ */
+
+import { execSync } from 'child_process';
+import fs from 'fs';
+import path from 'path';
+import dotenv from 'dotenv';
+
+// Load environment variables
+dotenv.config();
+
+// Configuration
+const config = {
+  workerName: process.env.CF_WORKER_NAME || 'brainy-cloud-service',
+  accountId: process.env.CF_ACCOUNT_ID,
+  kvNamespace: process.env.CF_KV_NAMESPACE || 'BRAINY_STORAGE',
+  r2Bucket: process.env.CF_R2_BUCKET || 'brainy-storage'
+};
+
+// Validate configuration
+if (!config.accountId) {
+  console.error('Error: CF_ACCOUNT_ID environment variable is required');
+  process.exit(1);
+}
+
+// Create wrangler.toml configuration
+function createWranglerConfig() {
+  console.log('Creating wrangler.toml configuration...');
+  
+  const wranglerContent = `
+name = "${config.workerName}"
+main = "dist/worker.js"
+compatibility_date = "2023-12-01"
+node_compat = true
+
+account_id = "${config.accountId}"
+
+[build]
+command = "npm run build"
+
+[vars]
+NODE_ENV = "production"
+STORAGE_TYPE = "${process.env.STORAGE_TYPE || 'memory'}"
+USE_SIMPLE_EMBEDDING = "${process.env.USE_SIMPLE_EMBEDDING || 'false'}"
+
+# KV Namespace for storage
+[[kv_namespaces]]
+binding = "BRAINY_KV"
+id = "${process.env.CF_KV_NAMESPACE_ID || 'create_kv_namespace_and_add_id_here'}"
+
+# R2 Bucket for storage (if using R2)
+[[r2_buckets]]
+binding = "BRAINY_R2"
+bucket_name = "${config.r2Bucket}"
+`;
+  
+  try {
+    fs.writeFileSync('wrangler.toml', wranglerContent);
+    console.log('wrangler.toml created successfully');
+  } catch (error) {
+    console.error('Error creating wrangler.toml:', error);
+    process.exit(1);
+  }
+}
+
+// Create Cloudflare Worker adapter
+function createWorkerAdapter() {
+  console.log('Creating Cloudflare Worker adapter...');
+  
+  const workerContent = `
+import { createServer } from '@cloudflare/workers-adapter';
+import app from './index.js';
+
+// Create a fetch handler for the worker
+export default {
+  async fetch(request, env, ctx) {
+    // Add environment variables to process.env
+    process.env = {
+      ...process.env,
+      ...env.vars,
+      CF_KV_NAMESPACE: env.BRAINY_KV,
+      CF_R2_BUCKET: env.BRAINY_R2
+    };
+
+    // Create a server adapter
+    const server = createServer(app);
+    
+    // Handle the request
+    return server.fetch(request, env, ctx);
+  }
+};
+`;
+  
+  try {
+    // Create dist directory if it doesn't exist
+    if (!fs.existsSync('dist')) {
+      fs.mkdirSync('dist');
+    }
+    
+    fs.writeFileSync('dist/worker.js', workerContent);
+    console.log('Worker adapter created successfully');
+  } catch (error) {
+    console.error('Error creating worker adapter:', error);
+    process.exit(1);
+  }
+}
+
+// Update package.json to include Cloudflare Workers dependencies
+function updatePackageJson() {
+  console.log('Updating package.json...');
+  
+  try {
+    const packageJson = JSON.parse(fs.readFileSync('package.json', 'utf8'));
+    
+    // Add Cloudflare Workers dependencies
+    packageJson.dependencies = {
+      ...packageJson.dependencies,
+      '@cloudflare/workers-adapter': '^1.1.0',
+      '@cloudflare/kv-asset-handler': '^0.3.0'
+    };
+    
+    // Add Cloudflare Workers scripts
+    packageJson.scripts = {
+      ...packageJson.scripts,
+      'deploy:cloudflare': 'wrangler deploy'
+    };
+    
+    fs.writeFileSync('package.json', JSON.stringify(packageJson, null, 2));
+    console.log('package.json updated successfully');
+    
+    // Install new dependencies
+    execSync('npm install --legacy-peer-deps');
+  } catch (error) {
+    console.error('Error updating package.json:', error);
+    process.exit(1);
+  }
+}
+
+// Create KV namespace if it doesn't exist
+function createKVNamespace() {
+  console.log('Creating KV namespace...');
+  
+  try {
+    // Check if KV namespace ID is provided
+    if (process.env.CF_KV_NAMESPACE_ID) {
+      console.log(`Using existing KV namespace: ${process.env.CF_KV_NAMESPACE_ID}`);
+      return;
+    }
+    
+    // Create KV namespace
+    const result = execSync(`wrangler kv:namespace create "${config.kvNamespace}"`).toString();
+    const match = result.match(/id = "([^"]+)"/);
+    
+    if (match && match[1]) {
+      const namespaceId = match[1];
+      console.log(`KV namespace created with ID: ${namespaceId}`);
+      
+      // Update wrangler.toml with the new namespace ID
+      let wranglerContent = fs.readFileSync('wrangler.toml', 'utf8');
+      wranglerContent = wranglerContent.replace(/id = "create_kv_namespace_and_add_id_here"/, `id = "${namespaceId}"`);
+      fs.writeFileSync('wrangler.toml', wranglerContent);
+    } else {
+      console.error('Failed to extract KV namespace ID from wrangler output');
+    }
+  } catch (error) {
+    console.error('Error creating KV namespace:', error);
+    console.log('You may need to create the KV namespace manually and update wrangler.toml');
+  }
+}
+
+// Create R2 bucket if it doesn't exist
+function createR2Bucket() {
+  console.log('Creating R2 bucket...');
+  
+  try {
+    // Only create R2 bucket if using R2 storage
+    if (process.env.STORAGE_TYPE !== 'r2') {
+      console.log('Skipping R2 bucket creation (not using R2 storage)');
+      return;
+    }
+    
+    // Create R2 bucket
+    execSync(`wrangler r2 bucket create ${config.r2Bucket}`);
+    console.log(`R2 bucket created: ${config.r2Bucket}`);
+  } catch (error) {
+    console.error('Error creating R2 bucket:', error);
+    console.log('You may need to create the R2 bucket manually');
+  }
+}
+
+// Deploy to Cloudflare Workers
+function deployToCloudflare() {
+  console.log('Deploying to Cloudflare Workers...');
+  
+  try {
+    execSync('wrangler deploy', { stdio: 'inherit' });
+    console.log('Deployed to Cloudflare Workers successfully');
+  } catch (error) {
+    console.error('Error deploying to Cloudflare Workers:', error);
+    process.exit(1);
+  }
+}
+
+// Clean up
+function cleanup() {
+  console.log('Cleaning up...');
+  
+  // No cleanup needed for now
+  console.log('Cleanup completed');
+}
+
+// Main function
+async function main() {
+  try {
+    console.log('Starting Cloudflare deployment...');
+    
+    createWranglerConfig();
+    createWorkerAdapter();
+    updatePackageJson();
+    createKVNamespace();
+    createR2Bucket();
+    deployToCloudflare();
+    cleanup();
+    
+    console.log('Deployment completed successfully');
+  } catch (error) {
+    console.error('Deployment failed:', error);
+    cleanup();
+    process.exit(1);
+  }
+}
+
+// Run the script
+main();

cloud-wrapper/scripts/deploy-gcp.js

@@ -0,0 +1,196 @@
+#!/usr/bin/env node
+
+/**
+ * Google Cloud Platform Deployment Script for Brainy Cloud Wrapper
+ * 
+ * This script helps deploy the Brainy cloud wrapper to Google Cloud Run.
+ * It uses the Google Cloud SDK to create and configure the necessary resources.
+ * 
+ * Prerequisites:
+ * - Google Cloud SDK installed and configured with appropriate credentials
+ * - Node.js 23.11.0 or higher
+ * - Brainy cloud wrapper built (npm run build)
+ * - Docker installed (for building container images)
+ */
+
+import { execSync } from 'child_process';
+import fs from 'fs';
+import path from 'path';
+import dotenv from 'dotenv';
+
+// Load environment variables
+dotenv.config();
+
+// Configuration
+const config = {
+  projectId: process.env.GCP_PROJECT_ID,
+  region: process.env.GCP_REGION || 'us-central1',
+  serviceName: process.env.GCP_SERVICE_NAME || 'brainy-cloud-service',
+  imageName: process.env.GCP_IMAGE_NAME || 'brainy-cloud-service',
+  memory: process.env.GCP_MEMORY || '512Mi',
+  cpu: process.env.GCP_CPU || '1',
+  maxInstances: process.env.GCP_MAX_INSTANCES || '10',
+  minInstances: process.env.GCP_MIN_INSTANCES || '0'
+};
+
+// Validate configuration
+if (!config.projectId) {
+  console.error('Error: GCP_PROJECT_ID environment variable is required');
+  process.exit(1);
+}
+
+// Create Dockerfile
+function createDockerfile() {
+  console.log('Creating Dockerfile...');
+  
+  const dockerfileContent = `
+FROM node:23-slim
+
+WORKDIR /app
+
+# Copy package.json and package-lock.json
+COPY package*.json ./
+
+# Install dependencies
+RUN npm install --production --legacy-peer-deps
+
+# Copy built application
+COPY dist/ ./dist/
+
+# Copy environment variables
+COPY .env ./
+
+# Expose port
+EXPOSE 8080
+
+# Start the application
+CMD ["node", "dist/index.js"]
+`;
+  
+  try {
+    fs.writeFileSync('Dockerfile', dockerfileContent);
+    console.log('Dockerfile created successfully');
+  } catch (error) {
+    console.error('Error creating Dockerfile:', error);
+    process.exit(1);
+  }
+}
+
+// Build and push Docker image
+function buildAndPushImage() {
+  console.log('Building and pushing Docker image...');
+  
+  try {
+    // Set Google Cloud project
+    execSync(`gcloud config set project ${config.projectId}`);
+    
+    // Build the Docker image
+    execSync(`docker build -t gcr.io/${config.projectId}/${config.imageName} .`);
+    
+    // Configure Docker to use gcloud as a credential helper
+    execSync('gcloud auth configure-docker');
+    
+    // Push the image to Google Container Registry
+    execSync(`docker push gcr.io/${config.projectId}/${config.imageName}`);
+    
+    console.log('Docker image built and pushed successfully');
+  } catch (error) {
+    console.error('Error building and pushing Docker image:', error);
+    process.exit(1);
+  }
+}
+
+// Deploy to Google Cloud Run
+function deployToCloudRun() {
+  console.log('Deploying to Google Cloud Run...');
+  
+  try {
+    // Create environment variables string
+    const envVars = [
+      `NODE_ENV=production`,
+      `PORT=8080`,
+      `STORAGE_TYPE=${process.env.STORAGE_TYPE || 'filesystem'}`
+    ];
+    
+    // Add S3 environment variables if using S3 storage
+    if (process.env.STORAGE_TYPE === 's3') {
+      envVars.push(`S3_BUCKET_NAME=${process.env.S3_BUCKET_NAME}`);
+      envVars.push(`S3_ACCESS_KEY_ID=${process.env.S3_ACCESS_KEY_ID}`);
+      envVars.push(`S3_SECRET_ACCESS_KEY=${process.env.S3_SECRET_ACCESS_KEY}`);
+      envVars.push(`S3_REGION=${process.env.S3_REGION || 'us-east-1'}`);
+      
+      if (process.env.S3_ENDPOINT) {
+        envVars.push(`S3_ENDPOINT=${process.env.S3_ENDPOINT}`);
+      }
+    }
+    
+    // Add embedding and HNSW configuration if provided
+    if (process.env.USE_SIMPLE_EMBEDDING) {
+      envVars.push(`USE_SIMPLE_EMBEDDING=${process.env.USE_SIMPLE_EMBEDDING}`);
+    }
+    
+    if (process.env.HNSW_M) {
+      envVars.push(`HNSW_M=${process.env.HNSW_M}`);
+      envVars.push(`HNSW_EF_CONSTRUCTION=${process.env.HNSW_EF_CONSTRUCTION || '200'}`);
+      envVars.push(`HNSW_EF_SEARCH=${process.env.HNSW_EF_SEARCH || '50'}`);
+    }
+    
+    // Deploy to Cloud Run
+    const envVarsString = envVars.map(env => `--set-env-vars="${env}"`).join(' ');
+    
+    execSync(`gcloud run deploy ${config.serviceName} \
+      --image=gcr.io/${config.projectId}/${config.imageName} \
+      --platform=managed \
+      --region=${config.region} \
+      --memory=${config.memory} \
+      --cpu=${config.cpu} \
+      --max-instances=${config.maxInstances} \
+      --min-instances=${config.minInstances} \
+      --allow-unauthenticated \
+      ${envVarsString}`);
+    
+    console.log('Deployed to Google Cloud Run successfully');
+    
+    // Get the service URL
+    const serviceUrl = execSync(`gcloud run services describe ${config.serviceName} --region=${config.region} --format="value(status.url)"`).toString().trim();
+    console.log(`Service URL: ${serviceUrl}`);
+  } catch (error) {
+    console.error('Error deploying to Google Cloud Run:', error);
+    process.exit(1);
+  }
+}
+
+// Clean up
+function cleanup() {
+  console.log('Cleaning up...');
+  
+  try {
+    // Remove Dockerfile
+    fs.unlinkSync('Dockerfile');
+    
+    console.log('Cleanup completed');
+  } catch (error) {
+    console.error('Error during cleanup:', error);
+  }
+}
+
+// Main function
+async function main() {
+  try {
+    console.log('Starting Google Cloud Platform deployment...');
+    
+    createDockerfile();
+    buildAndPushImage();
+    deployToCloudRun();
+    cleanup();
+    
+    console.log('Deployment completed successfully');
+  } catch (error) {
+    console.error('Deployment failed:', error);
+    cleanup();
+    process.exit(1);
+  }
+}
+
+// Run the script
+main();

cloud-wrapper/src/index.ts

@@ -0,0 +1,106 @@
+import express from 'express';
+import cors from 'cors';
+import helmet from 'helmet';
+import morgan from 'morgan';
+import dotenv from 'dotenv';
+import http from 'http';
+import { WebSocketServer } from 'ws';
+import { BrainyData } from '@soulcraft/brainy';
+import { setupRoutes } from './routes.js';
+import { initializeBrainy } from './services/brainyService.js';
+import { setupWebSocketHandlers } from './websocket.js';
+import { initializeMCPService } from './services/mcpService.js';
+
+// Load environment variables
+dotenv.config();
+
+// Initialize Express app
+const app = express();
+const port = process.env.PORT || 3000;
+
+// Middleware
+app.use(helmet()); // Security headers
+app.use(cors()); // Enable CORS
+app.use(morgan('combined')); // Logging
+app.use(express.json()); // Parse JSON bodies
+
+// Initialize Brainy
+let brainyInstance: BrainyData;
+
+// Health check endpoint
+app.get('/health', (req, res) => {
+  res.status(200).json({ status: 'ok' });
+});
+
+// Setup API routes
+const apiRouter = setupRoutes();
+app.use('/api', (req, res, next) => {
+  // Attach brainy instance to request
+  (req as any).brainy = brainyInstance;
+  next();
+}, apiRouter);
+
+// Start the server
+async function startServer() {
+  try {
+    // Initialize Brainy
+    brainyInstance = await initializeBrainy();
+    console.log('Brainy initialized successfully');
+
+    // Create HTTP server
+    const server = http.createServer(app);
+
+    // Create WebSocket server
+    const wss = new WebSocketServer({ server });
+
+    // Setup WebSocket handlers
+    setupWebSocketHandlers(wss, brainyInstance);
+    console.log('WebSocket server initialized');
+
+    // Initialize MCP service
+    const mcpWsPort = process.env.MCP_WS_PORT ? parseInt(process.env.MCP_WS_PORT, 10) : undefined;
+    const mcpRestPort = process.env.MCP_REST_PORT ? parseInt(process.env.MCP_REST_PORT, 10) : undefined;
+
+    if (mcpWsPort || mcpRestPort) {
+      initializeMCPService(brainyInstance, {
+        wsPort: mcpWsPort,
+        restPort: mcpRestPort,
+        enableAuth: process.env.MCP_ENABLE_AUTH === 'true',
+        apiKeys: process.env.MCP_API_KEYS ? process.env.MCP_API_KEYS.split(',') : undefined,
+        rateLimit: process.env.MCP_RATE_LIMIT_REQUESTS ? {
+          windowMs: parseInt(process.env.MCP_RATE_LIMIT_WINDOW_MS || '60000', 10),
+          maxRequests: parseInt(process.env.MCP_RATE_LIMIT_REQUESTS, 10)
+        } : undefined,
+        cors: process.env.MCP_ENABLE_CORS === 'true' ? {} : undefined
+      });
+      console.log('MCP service initialized');
+    }
+
+    // Start HTTP server
+    server.listen(port, () => {
+      console.log(`Server running on port ${port}`);
+      console.log(`WebSocket server running on ws://localhost:${port}`);
+    });
+  } catch (error) {
+    console.error('Failed to start server:', error);
+    process.exit(1);
+  }
+}
+
+// Handle graceful shutdown
+process.on('SIGTERM', async () => {
+  console.log('SIGTERM received, shutting down gracefully');
+  // Cleanup Brainy resources
+  await brainyInstance?.clear();
+  process.exit(0);
+});
+
+process.on('SIGINT', async () => {
+  console.log('SIGINT received, shutting down gracefully');
+  // Cleanup Brainy resources
+  await brainyInstance?.clear();
+  process.exit(0);
+});
+
+// Start the server
+startServer();

cloud-wrapper/src/routes.ts

@@ -0,0 +1,186 @@
+import { Router, Request, Response, NextFunction, RequestHandler } from 'express';
+import { BrainyData, NounType, VerbType } from '@soulcraft/brainy';
+
+// Define a custom request type that includes the Brainy instance
+interface BrainyRequest extends Request {
+  brainy: BrainyData;
+}
+
+// Helper function to handle async route handlers
+const asyncHandler = (fn: (req: BrainyRequest, res: Response, next: NextFunction) => Promise<any>) => 
+  (req: Request, res: Response, next: NextFunction) => {
+    Promise.resolve(fn(req as BrainyRequest, res, next)).catch(next);
+  };
+
+export function setupRoutes() {
+  const router = Router();
+
+  // Get database status
+  router.get('/status', asyncHandler(async (req: BrainyRequest, res: Response, next: NextFunction) => {
+    try {
+      const status = await req.brainy.status();
+      res.status(200).json(status);
+    } catch (error) {
+      console.error('Error getting status:', error);
+      res.status(500).json({ error: 'Failed to get database status' });
+    }
+  }));
+
+  // Add a noun (entity)
+  router.post('/nouns', asyncHandler(async (req: BrainyRequest, res: Response, next: NextFunction) => {
+    try {
+      const { text, metadata } = req.body;
+
+      if (!text) {
+        return res.status(400).json({ error: 'Text is required' });
+      }
+
+      const id = await req.brainy.add(text, metadata || {});
+      res.status(201).json({ id });
+    } catch (error) {
+      console.error('Error adding noun:', error);
+      res.status(500).json({ error: 'Failed to add noun' });
+    }
+  }));
+
+  // Get a noun by ID
+  router.get('/nouns/:id', asyncHandler(async (req: BrainyRequest, res: Response, next: NextFunction) => {
+    try {
+      const { id } = req.params;
+      const noun = await req.brainy.get(id);
+
+      if (!noun) {
+        return res.status(404).json({ error: 'Noun not found' });
+      }
+
+      res.status(200).json(noun);
+    } catch (error) {
+      console.error('Error getting noun:', error);
+      res.status(500).json({ error: 'Failed to get noun' });
+    }
+  }));
+
+  // Update noun metadata
+  router.put('/nouns/:id', asyncHandler(async (req: BrainyRequest, res: Response, next: NextFunction) => {
+    try {
+      const { id } = req.params;
+      const { metadata } = req.body;
+
+      if (!metadata) {
+        return res.status(400).json({ error: 'Metadata is required' });
+      }
+
+      await req.brainy.updateMetadata(id, metadata);
+      res.status(200).json({ success: true });
+    } catch (error) {
+      console.error('Error updating noun:', error);
+      res.status(500).json({ error: 'Failed to update noun' });
+    }
+  }));
+
+  // Delete a noun
+  router.delete('/nouns/:id', asyncHandler(async (req: BrainyRequest, res: Response, next: NextFunction) => {
+    try {
+      const { id } = req.params;
+      await req.brainy.delete(id);
+      res.status(200).json({ success: true });
+    } catch (error) {
+      console.error('Error deleting noun:', error);
+      res.status(500).json({ error: 'Failed to delete noun' });
+    }
+  }));
+
+  // Search for similar nouns
+  router.post('/search', asyncHandler(async (req: BrainyRequest, res: Response, next: NextFunction) => {
+    try {
+      const { query, limit = 10 } = req.body;
+
+      if (!query) {
+        return res.status(400).json({ error: 'Query is required' });
+      }
+
+      const results = await req.brainy.searchText(query, limit);
+      res.status(200).json(results);
+    } catch (error) {
+      console.error('Error searching:', error);
+      res.status(500).json({ error: 'Failed to search' });
+    }
+  }));
+
+  // Add a verb (relationship)
+  router.post('/verbs', asyncHandler(async (req: BrainyRequest, res: Response, next: NextFunction) => {
+    try {
+      const { sourceId, targetId, metadata } = req.body;
+
+      if (!sourceId || !targetId) {
+        return res.status(400).json({ error: 'Source ID and Target ID are required' });
+      }
+
+      await req.brainy.addVerb(sourceId, targetId, metadata || {});
+      res.status(201).json({ success: true });
+    } catch (error) {
+      console.error('Error adding verb:', error);
+      res.status(500).json({ error: 'Failed to add verb' });
+    }
+  }));
+
+  // Get all verbs
+  router.get('/verbs', asyncHandler(async (req: BrainyRequest, res: Response, next: NextFunction) => {
+    try {
+      const verbs = await req.brainy.getAllVerbs();
+      res.status(200).json(verbs);
+    } catch (error) {
+      console.error('Error getting verbs:', error);
+      res.status(500).json({ error: 'Failed to get verbs' });
+    }
+  }));
+
+  // Get verbs by source
+  router.get('/verbs/source/:id', asyncHandler(async (req: BrainyRequest, res: Response, next: NextFunction) => {
+    try {
+      const { id } = req.params;
+      const verbs = await req.brainy.getVerbsBySource(id);
+      res.status(200).json(verbs);
+    } catch (error) {
+      console.error('Error getting verbs by source:', error);
+      res.status(500).json({ error: 'Failed to get verbs by source' });
+    }
+  }));
+
+  // Get verbs by target
+  router.get('/verbs/target/:id', asyncHandler(async (req: BrainyRequest, res: Response, next: NextFunction) => {
+    try {
+      const { id } = req.params;
+      const verbs = await req.brainy.getVerbsByTarget(id);
+      res.status(200).json(verbs);
+    } catch (error) {
+      console.error('Error getting verbs by target:', error);
+      res.status(500).json({ error: 'Failed to get verbs by target' });
+    }
+  }));
+
+  // Delete a verb
+  router.delete('/verbs/:id', asyncHandler(async (req: BrainyRequest, res: Response, next: NextFunction) => {
+    try {
+      const { id } = req.params;
+      await req.brainy.deleteVerb(id);
+      res.status(200).json({ success: true });
+    } catch (error) {
+      console.error('Error deleting verb:', error);
+      res.status(500).json({ error: 'Failed to delete verb' });
+    }
+  }));
+
+  // Clear all data
+  router.delete('/clear', asyncHandler(async (req: BrainyRequest, res: Response, next: NextFunction) => {
+    try {
+      await req.brainy.clear();
+      res.status(200).json({ success: true });
+    } catch (error) {
+      console.error('Error clearing data:', error);
+      res.status(500).json({ error: 'Failed to clear data' });
+    }
+  }));
+
+  return router;
+}

cloud-wrapper/src/services/brainyService.ts

@@ -0,0 +1,97 @@
+import { BrainyData, BrainyDataConfig } from '@soulcraft/brainy'
+import dotenv from 'dotenv'
+
+// Load environment variables
+dotenv.config()
+
+/**
+ * Extended configuration interface that includes rootDirectory
+ */
+interface ExtendedBrainyDataConfig extends BrainyDataConfig {
+  storage: BrainyDataConfig['storage'] & {
+    rootDirectory?: string
+  }
+}
+
+/**
+ * Initialize the Brainy instance with appropriate configuration
+ * based on environment variables
+ */
+export async function initializeBrainy(): Promise<BrainyData> {
+  // Get storage configuration from environment variables
+  const storageType = process.env.STORAGE_TYPE || 'filesystem'
+
+  // Create configuration object
+  const config: ExtendedBrainyDataConfig = {
+    storage: {}
+  }
+
+  // Configure storage based on type
+  switch (storageType) {
+    case 's3':
+      // Configure S3 storage
+      if (process.env.S3_ENDPOINT) {
+        // Use customS3Storage for S3-compatible services with custom endpoints
+        config.storage.customS3Storage = {
+          bucketName: process.env.S3_BUCKET_NAME,
+          accessKeyId: process.env.S3_ACCESS_KEY_ID,
+          secretAccessKey: process.env.S3_SECRET_ACCESS_KEY,
+          region: process.env.S3_REGION || 'us-east-1',
+          endpoint: process.env.S3_ENDPOINT
+        }
+      } else {
+        // Use standard S3 storage for AWS S3
+        config.storage.s3Storage = {
+          bucketName: process.env.S3_BUCKET_NAME,
+          accessKeyId: process.env.S3_ACCESS_KEY_ID,
+          secretAccessKey: process.env.S3_SECRET_ACCESS_KEY,
+          region: process.env.S3_REGION || 'us-east-1'
+        }
+      }
+      break
+
+    case 'filesystem':
+      // Configure filesystem storage
+      config.storage.rootDirectory = process.env.STORAGE_ROOT_DIR || './data'
+      break
+
+    case 'memory':
+      // No additional configuration needed for memory storage
+      break
+
+    default:
+      // Default to filesystem storage
+      config.storage.rootDirectory = process.env.STORAGE_ROOT_DIR || './data'
+      break
+  }
+
+  // Note: Universal Sentence Encoder is now the only embedding option
+  // TensorFlow.js is required for embedding to work
+
+  // Configure HNSW index parameters if provided
+  if (process.env.HNSW_M) {
+    config.hnsw = {
+      M: parseInt(process.env.HNSW_M, 10),
+      efConstruction: process.env.HNSW_EF_CONSTRUCTION
+        ? parseInt(process.env.HNSW_EF_CONSTRUCTION, 10)
+        : 200,
+      efSearch: process.env.HNSW_EF_SEARCH
+        ? parseInt(process.env.HNSW_EF_SEARCH, 10)
+        : 50
+    }
+  }
+
+  // Create and initialize the Brainy instance
+  const brainy = new BrainyData(config)
+  await brainy.init()
+
+  return brainy
+}
+
+/**
+ * Get the current storage type being used by Brainy
+ */
+export async function getStorageType(brainy: BrainyData): Promise<string> {
+  const status = await brainy.status()
+  return status.type || 'unknown'
+}

cloud-wrapper/src/services/mcpService.ts

@@ -0,0 +1,204 @@
+import { WebSocketServer } from 'ws'
+import express from 'express'
+import cors from 'cors'
+import { BrainyData, BrainyMCPService, MCPRequestType } from '@soulcraft/brainy'
+import { v4 as uuidv4 } from 'uuid'
+
+/**
+ * Initialize the MCP service with WebSocket and REST API servers
+ */
+export function initializeMCPService(
+  brainy: BrainyData,
+  options: {
+    wsPort?: number
+    restPort?: number
+    enableAuth?: boolean
+    apiKeys?: string[]
+    rateLimit?: {
+      windowMs: number
+      maxRequests: number
+    }
+    cors?: any
+  }
+) {
+  // Create the MCP service
+  const mcpService = new BrainyMCPService(brainy, options)
+
+  // Start WebSocket server if port is provided
+  if (options.wsPort) {
+    startWebSocketServer(mcpService, options.wsPort)
+  }
+
+  // Start REST server if port is provided
+  if (options.restPort) {
+    startRESTServer(mcpService, options.restPort, options.cors)
+  }
+
+  return mcpService
+}
+
+/**
+ * Start a WebSocket server for the MCP service
+ */
+function startWebSocketServer(mcpService: BrainyMCPService, port: number) {
+  const wss = new WebSocketServer({ port })
+
+  wss.on('connection', (ws: any) => {
+    ws.on('message', async (message: string) => {
+      try {
+        const request = JSON.parse(message)
+
+        // Handle the request using the MCP service
+        const response = await mcpService.handleMCPRequest(request)
+
+        // Send the response
+        ws.send(JSON.stringify(response))
+      } catch (error) {
+        // Send error response
+        ws.send(
+          JSON.stringify({
+            success: false,
+            requestId: uuidv4(),
+            error: {
+              code: 'INTERNAL_ERROR',
+              message: error instanceof Error ? error.message : String(error)
+            }
+          })
+        )
+      }
+    })
+  })
+
+  console.log(`MCP WebSocket server started on port ${port}`)
+
+  return wss
+}
+
+/**
+ * Start a REST server for the MCP service
+ */
+function startRESTServer(
+  mcpService: BrainyMCPService,
+  port: number,
+  corsOptions?: any
+) {
+  const app = express()
+
+  // Parse JSON request bodies
+  app.use(express.json())
+
+  // Enable CORS if configured
+  if (corsOptions) {
+    app.use(cors(corsOptions))
+  }
+
+  // MCP endpoints
+  app.post('/mcp/data', async (req: any, res: any) => {
+    try {
+      const response = await mcpService.handleMCPRequest({
+        ...req.body,
+        type: 'DATA_ACCESS'
+      })
+
+      res.json(response)
+    } catch (error) {
+      res.status(500).json({
+        success: false,
+        requestId: uuidv4(),
+        error: {
+          code: 'INTERNAL_ERROR',
+          message: error instanceof Error ? error.message : String(error)
+        }
+      })
+    }
+  })
+
+  app.post('/mcp/tools', async (req: any, res: any) => {
+    try {
+      const response = await mcpService.handleMCPRequest({
+        ...req.body,
+        type: 'TOOL_EXECUTION'
+      })
+
+      res.json(response)
+    } catch (error) {
+      res.status(500).json({
+        success: false,
+        requestId: uuidv4(),
+        error: {
+          code: 'INTERNAL_ERROR',
+          message: error instanceof Error ? error.message : String(error)
+        }
+      })
+    }
+  })
+
+  app.post('/mcp/system', async (req: any, res: any) => {
+    try {
+      const response = await mcpService.handleMCPRequest({
+        ...req.body,
+        type: 'SYSTEM_INFO'
+      })
+
+      res.json(response)
+    } catch (error) {
+      res.status(500).json({
+        success: false,
+        requestId: uuidv4(),
+        error: {
+          code: 'INTERNAL_ERROR',
+          message: error instanceof Error ? error.message : String(error)
+        }
+      })
+    }
+  })
+
+  app.post('/mcp/auth', async (req: any, res: any) => {
+    try {
+      const response = await mcpService.handleMCPRequest({
+        ...req.body,
+        type: 'AUTHENTICATION'
+      })
+
+      res.json(response)
+    } catch (error) {
+      res.status(500).json({
+        success: false,
+        requestId: uuidv4(),
+        error: {
+          code: 'INTERNAL_ERROR',
+          message: error instanceof Error ? error.message : String(error)
+        }
+      })
+    }
+  })
+
+  // Get available tools
+  app.get('/mcp/tools', async (req: any, res: any) => {
+    try {
+      const response = await mcpService.handleMCPRequest({
+        type: MCPRequestType.SYSTEM_INFO,
+        requestId: uuidv4(),
+        version: '1.0'
+      })
+
+      res.json(response)
+    } catch (error) {
+      res.status(500).json({
+        success: false,
+        requestId: uuidv4(),
+        error: {
+          code: 'INTERNAL_ERROR',
+          message: error instanceof Error ? error.message : String(error)
+        }
+      })
+    }
+  })
+
+  // Start the server
+  const server = app.listen(port, () => {
+    console.log(`MCP REST API server started on port ${port}`)
+  })
+
+  return server
+}

cloud-wrapper/src/websocket.ts

@@ -0,0 +1,433 @@
+import { WebSocketServer, WebSocket } from 'ws';
+import { BrainyData } from '@soulcraft/brainy';
+import { v4 as uuidv4 } from 'uuid';
+
+// Define message types
+enum MessageType {
+  STATUS = 'status',
+  ADD_NOUN = 'addNoun',
+  GET_NOUN = 'getNoun',
+  UPDATE_NOUN = 'updateNoun',
+  DELETE_NOUN = 'deleteNoun',
+  SEARCH = 'search',
+  ADD_VERB = 'addVerb',
+  GET_VERBS = 'getVerbs',
+  GET_VERBS_BY_SOURCE = 'getVerbsBySource',
+  GET_VERBS_BY_TARGET = 'getVerbsByTarget',
+  DELETE_VERB = 'deleteVerb',
+  CLEAR = 'clear',
+  SUBSCRIBE = 'subscribe',
+  UNSUBSCRIBE = 'unsubscribe',
+  ERROR = 'error'
+}
+
+// Define subscription types
+enum SubscriptionType {
+  NOUNS = 'nouns',
+  VERBS = 'verbs',
+  SEARCH_RESULTS = 'searchResults'
+}
+
+// Define message interface
+interface WebSocketMessage {
+  type: MessageType;
+  id?: string;
+  payload?: any;
+}
+
+// Define client interface
+interface WebSocketClient {
+  id: string;
+  socket: WebSocket;
+  subscriptions: Set<SubscriptionType>;
+}
+
+// Store connected clients
+const clients: Map<string, WebSocketClient> = new Map();
+
+// Setup WebSocket handlers
+export function setupWebSocketHandlers(wss: WebSocketServer, brainy: BrainyData) {
+  // Handle new connections
+  wss.on('connection', (socket: WebSocket) => {
+    const clientId = uuidv4();
+    
+    // Create client object
+    const client: WebSocketClient = {
+      id: clientId,
+      socket,
+      subscriptions: new Set()
+    };
+    
+    // Store client
+    clients.set(clientId, client);
+    
+    console.log(`WebSocket client connected: ${clientId}`);
+    
+    // Send welcome message
+    sendMessage(socket, {
+      type: MessageType.STATUS,
+      id: uuidv4(),
+      payload: {
+        clientId,
+        message: 'Connected to Brainy WebSocket API',
+        status: 'connected'
+      }
+    });
+    
+    // Handle messages
+    socket.on('message', async (data: WebSocket.Data) => {
+      try {
+        const message: WebSocketMessage = JSON.parse(data.toString());
+        
+        // Ensure message has an ID
+        const messageId = message.id || uuidv4();
+        
+        console.log(`Received message: ${message.type} (${messageId})`);
+        
+        // Process message based on type
+        await processMessage(client, message, messageId, brainy);
+      } catch (error) {
+        console.error('Error processing WebSocket message:', error);
+        sendMessage(socket, {
+          type: MessageType.ERROR,
+          id: uuidv4(),
+          payload: {
+            message: 'Invalid message format',
+            error: (error as Error).message
+          }
+        });
+      }
+    });
+    
+    // Handle disconnection
+    socket.on('close', () => {
+      // Remove client
+      clients.delete(clientId);
+      console.log(`WebSocket client disconnected: ${clientId}`);
+    });
+  });
+}
+
+// Process incoming messages
+async function processMessage(
+  client: WebSocketClient,
+  message: WebSocketMessage,
+  messageId: string,
+  brainy: BrainyData
+) {
+  const { socket } = client;
+  
+  try {
+    switch (message.type) {
+      case MessageType.STATUS:
+        // Return database status
+        const status = await brainy.status();
+        sendMessage(socket, {
+          type: MessageType.STATUS,
+          id: messageId,
+          payload: status
+        });
+        break;
+        
+      case MessageType.ADD_NOUN:
+        // Add a noun
+        if (!message.payload?.text) {
+          throw new Error('Text is required');
+        }
+        
+        const nounId = await brainy.add(
+          message.payload.text,
+          message.payload.metadata || {}
+        );
+        
+        sendMessage(socket, {
+          type: MessageType.ADD_NOUN,
+          id: messageId,
+          payload: { id: nounId }
+        });
+        
+        // Notify subscribers
+        notifySubscribers(SubscriptionType.NOUNS, {
+          type: 'added',
+          id: nounId,
+          data: await brainy.get(nounId)
+        });
+        
+        break;
+        
+      case MessageType.GET_NOUN:
+        // Get a noun by ID
+        if (!message.payload?.id) {
+          throw new Error('Noun ID is required');
+        }
+        
+        const noun = await brainy.get(message.payload.id);
+        
+        if (!noun) {
+          throw new Error('Noun not found');
+        }
+        
+        sendMessage(socket, {
+          type: MessageType.GET_NOUN,
+          id: messageId,
+          payload: noun
+        });
+        break;
+        
+      case MessageType.UPDATE_NOUN:
+        // Update noun metadata
+        if (!message.payload?.id || !message.payload?.metadata) {
+          throw new Error('Noun ID and metadata are required');
+        }
+        
+        await brainy.updateMetadata(message.payload.id, message.payload.metadata);
+        
+        sendMessage(socket, {
+          type: MessageType.UPDATE_NOUN,
+          id: messageId,
+          payload: { success: true }
+        });
+        
+        // Notify subscribers
+        notifySubscribers(SubscriptionType.NOUNS, {
+          type: 'updated',
+          id: message.payload.id,
+          data: await brainy.get(message.payload.id)
+        });
+        
+        break;
+        
+      case MessageType.DELETE_NOUN:
+        // Delete a noun
+        if (!message.payload?.id) {
+          throw new Error('Noun ID is required');
+        }
+        
+        await brainy.delete(message.payload.id);
+        
+        sendMessage(socket, {
+          type: MessageType.DELETE_NOUN,
+          id: messageId,
+          payload: { success: true }
+        });
+        
+        // Notify subscribers
+        notifySubscribers(SubscriptionType.NOUNS, {
+          type: 'deleted',
+          id: message.payload.id
+        });
+        
+        break;
+        
+      case MessageType.SEARCH:
+        // Search for similar nouns
+        if (!message.payload?.query) {
+          throw new Error('Query is required');
+        }
+        
+        const limit = message.payload.limit || 10;
+        const results = await brainy.searchText(message.payload.query, limit);
+        
+        sendMessage(socket, {
+          type: MessageType.SEARCH,
+          id: messageId,
+          payload: results
+        });
+        
+        // Notify subscribers
+        notifySubscribers(SubscriptionType.SEARCH_RESULTS, {
+          type: 'search',
+          query: message.payload.query,
+          results
+        });
+        
+        break;
+        
+      case MessageType.ADD_VERB:
+        // Add a verb (relationship)
+        if (!message.payload?.sourceId || !message.payload?.targetId) {
+          throw new Error('Source ID and Target ID are required');
+        }
+        
+        await brainy.addVerb(
+          message.payload.sourceId,
+          message.payload.targetId,
+          message.payload.metadata || {}
+        );
+        
+        sendMessage(socket, {
+          type: MessageType.ADD_VERB,
+          id: messageId,
+          payload: { success: true }
+        });
+        
+        // Notify subscribers
+        notifySubscribers(SubscriptionType.VERBS, {
+          type: 'added',
+          sourceId: message.payload.sourceId,
+          targetId: message.payload.targetId,
+          metadata: message.payload.metadata
+        });
+        
+        break;
+        
+      case MessageType.GET_VERBS:
+        // Get all verbs
+        const verbs = await brainy.getAllVerbs();
+        
+        sendMessage(socket, {
+          type: MessageType.GET_VERBS,
+          id: messageId,
+          payload: verbs
+        });
+        break;
+        
+      case MessageType.GET_VERBS_BY_SOURCE:
+        // Get verbs by source
+        if (!message.payload?.id) {
+          throw new Error('Source ID is required');
+        }
+        
+        const sourceVerbs = await brainy.getVerbsBySource(message.payload.id);
+        
+        sendMessage(socket, {
+          type: MessageType.GET_VERBS_BY_SOURCE,
+          id: messageId,
+          payload: sourceVerbs
+        });
+        break;
+        
+      case MessageType.GET_VERBS_BY_TARGET:
+        // Get verbs by target
+        if (!message.payload?.id) {
+          throw new Error('Target ID is required');
+        }
+        
+        const targetVerbs = await brainy.getVerbsByTarget(message.payload.id);
+        
+        sendMessage(socket, {
+          type: MessageType.GET_VERBS_BY_TARGET,
+          id: messageId,
+          payload: targetVerbs
+        });
+        break;
+        
+      case MessageType.DELETE_VERB:
+        // Delete a verb
+        if (!message.payload?.id) {
+          throw new Error('Verb ID is required');
+        }
+        
+        await brainy.deleteVerb(message.payload.id);
+        
+        sendMessage(socket, {
+          type: MessageType.DELETE_VERB,
+          id: messageId,
+          payload: { success: true }
+        });
+        
+        // Notify subscribers
+        notifySubscribers(SubscriptionType.VERBS, {
+          type: 'deleted',
+          id: message.payload.id
+        });
+        
+        break;
+        
+      case MessageType.CLEAR:
+        // Clear all data
+        await brainy.clear();
+        
+        sendMessage(socket, {
+          type: MessageType.CLEAR,
+          id: messageId,
+          payload: { success: true }
+        });
+        
+        // Notify all subscribers
+        notifySubscribers(SubscriptionType.NOUNS, { type: 'cleared' });
+        notifySubscribers(SubscriptionType.VERBS, { type: 'cleared' });
+        
+        break;
+        
+      case MessageType.SUBSCRIBE:
+        // Subscribe to events
+        if (!message.payload?.type) {
+          throw new Error('Subscription type is required');
+        }
+        
+        const subscriptionType = message.payload.type as SubscriptionType;
+        
+        // Add subscription
+        client.subscriptions.add(subscriptionType);
+        
+        sendMessage(socket, {
+          type: MessageType.SUBSCRIBE,
+          id: messageId,
+          payload: {
+            success: true,
+            type: subscriptionType
+          }
+        });
+        break;
+        
+      case MessageType.UNSUBSCRIBE:
+        // Unsubscribe from events
+        if (!message.payload?.type) {
+          throw new Error('Subscription type is required');
+        }
+        
+        const unsubscribeType = message.payload.type as SubscriptionType;
+        
+        // Remove subscription
+        client.subscriptions.delete(unsubscribeType);
+        
+        sendMessage(socket, {
+          type: MessageType.UNSUBSCRIBE,
+          id: messageId,
+          payload: {
+            success: true,
+            type: unsubscribeType
+          }
+        });
+        break;
+        
+      default:
+        throw new Error(`Unknown message type: ${message.type}`);
+    }
+  } catch (error) {
+    console.error(`Error processing message ${message.type}:`, error);
+    
+    sendMessage(socket, {
+      type: MessageType.ERROR,
+      id: messageId,
+      payload: {
+        originalType: message.type,
+        message: 'Error processing message',
+        error: (error as Error).message
+      }
+    });
+  }
+}
+
+// Send a message to a client
+function sendMessage(socket: WebSocket, message: WebSocketMessage) {
+  if (socket.readyState === WebSocket.OPEN) {
+    socket.send(JSON.stringify(message));
+  }
+}
+
+// Notify subscribers of events
+function notifySubscribers(type: SubscriptionType, data: any) {
+  for (const client of clients.values()) {
+    if (client.subscriptions.has(type)) {
+      sendMessage(client.socket, {
+        type: MessageType.SUBSCRIBE,
+        payload: {
+          type,
+          data
+        }
+      });
+    }
+  }
+}

cloud-wrapper/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "esModuleInterop": true,
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "declaration": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}

index.html

@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>Brainy Interactive Demo - Redirecting...</title>
+  <link rel="icon" href="brainy.png" type="image/png">
+  <meta http-equiv="refresh" content="0;url=demo/index.html">
+  <script>
+    window.location.href = 'demo/index.html'
+  </script>
+</head>
+<body>
+<p>Redirecting to <a href="demo/index.html">Brainy Interactive Demo</a>...</p>
+<p>If you are not redirected automatically, please click the link above.</p>
+</body>
+</html>

package.json

@@ -0,0 +1,154 @@
+{
+  "name": "@soulcraft/brainy",
+  "version": "0.9.10",
+  "description": "A vector graph database using HNSW indexing with Origin Private File System storage",
+  "main": "dist/unified.js",
+  "module": "dist/unified.js",
+  "types": "dist/unified.d.ts",
+  "type": "module",
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "import": "./dist/unified.js",
+      "types": "./dist/unified.d.ts"
+    },
+    "./min": {
+      "import": "./dist/unified.min.js"
+    },
+    "./types/graphTypes": {
+      "import": "./dist/types/graphTypes.js",
+      "types": "./dist/types/graphTypes.d.ts"
+    },
+    "./types/augmentations": {
+      "import": "./dist/types/augmentations.js",
+      "types": "./dist/types/augmentations.d.ts"
+    }
+  },
+  "engines": {
+    "node": ">=23.11.0"
+  },
+  "scripts": {
+    "prebuild": "node scripts/generate-version.js",
+    "build": "BUILD_TYPE=unified rollup -c rollup.config.js",
+    "build:browser": "BUILD_TYPE=browser rollup -c rollup.config.js",
+    "start": "node dist/unified.js",
+    "demo": "npm run build && npm run build:browser && npx http-server -o /index.html",
+    "cli": "node ./cli-wrapper.js",
+    "version": "node scripts/generate-version.js",
+    "version:patch": "npm version patch",
+    "version:minor": "npm version minor",
+    "version:major": "npm version major",
+    "deploy": "npm run build && npm publish",
+    "deploy:cloud:aws": "cd cloud-wrapper && npm run build && npm run deploy:aws",
+    "deploy:cloud:gcp": "cd cloud-wrapper && npm run build && npm run deploy:gcp",
+    "deploy:cloud:cloudflare": "cd cloud-wrapper && npm run build && npm run deploy:cloudflare",
+    "deploy:cloud": "echo 'Please use one of the following commands to deploy to a specific cloud provider:' && echo '  npm run deploy:cloud:aws' && echo '  npm run deploy:cloud:gcp' && echo '  npm run deploy:cloud:cloudflare'",
+    "postinstall": "echo 'Note: If you encounter dependency conflicts with TensorFlow.js packages, please use: npm install --legacy-peer-deps'"
+  },
+  "bin": {
+    "brainy": "cli-wrapper.js"
+  },
+  "keywords": [
+    "vector-database",
+    "hnsw",
+    "opfs",
+    "origin-private-file-system",
+    "embeddings",
+    "graph-database",
+    "streaming-data"
+  ],
+  "author": "David Snelling (david@soulcraft.com)",
+  "license": "MIT",
+  "private": false,
+  "publishConfig": {
+    "access": "public"
+  },
+  "homepage": "https://github.com/soulcraft-research/brainy",
+  "bugs": {
+    "url": "https://github.com/soulcraft-research/brainy/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/soulcraft-research/brainy.git"
+  },
+  "files": [
+    "dist/**/*.js",
+    "dist/**/*.d.ts",
+    "dist/types/",
+    "LICENSE",
+    "README.md",
+    "cli-wrapper.js",
+    "brainy.png"
+  ],
+  "devDependencies": {
+    "@rollup/plugin-commonjs": "^25.0.7",
+    "@rollup/plugin-json": "^6.1.0",
+    "@rollup/plugin-node-resolve": "^15.2.3",
+    "@rollup/plugin-replace": "^5.0.5",
+    "@rollup/plugin-typescript": "^11.1.6",
+    "@types/node": "^20.4.5",
+    "@types/omelette": "^0.4.5",
+    "@types/uuid": "^10.0.0",
+    "@typescript-eslint/eslint-plugin": "^6.0.0",
+    "@typescript-eslint/parser": "^6.0.0",
+    "eslint": "^8.45.0",
+    "rollup": "^4.12.0",
+    "rollup-plugin-terser": "^7.0.2",
+    "tslib": "^2.8.1",
+    "typescript": "^5.1.6"
+  },
+  "dependencies": {
+    "@aws-sdk/client-s3": "^3.427.0",
+    "@tensorflow-models/universal-sentence-encoder": "^1.3.3",
+    "@tensorflow/tfjs": "^4.22.0",
+    "@tensorflow/tfjs-backend-cpu": "^4.22.0",
+    "@tensorflow/tfjs-converter": "^4.22.0",
+    "@tensorflow/tfjs-core": "^4.22.0",
+    "@tensorflow/tfjs-layers": "^4.22.0",
+    "buffer": "^6.0.3",
+    "commander": "^14.0.0",
+    "omelette": "^0.4.17",
+    "uuid": "^9.0.0"
+  },
+  "prettier": {
+    "arrowParens": "always",
+    "bracketSameLine": true,
+    "bracketSpacing": true,
+    "htmlWhitespaceSensitivity": "css",
+    "printWidth": 80,
+    "proseWrap": "preserve",
+    "semi": false,
+    "singleQuote": true,
+    "tabWidth": 2,
+    "trailingComma": "none",
+    "useTabs": false
+  },
+  "eslintConfig": {
+    "root": true,
+    "extends": [
+      "eslint:recommended",
+      "plugin:@typescript-eslint/recommended"
+    ],
+    "parser": "@typescript-eslint/parser",
+    "plugins": [
+      "@typescript-eslint"
+    ],
+    "rules": {
+      "@typescript-eslint/no-explicit-any": "off",
+      "no-unused-vars": "off",
+      "@typescript-eslint/no-unused-vars": [
+        "warn",
+        {
+          "args": "after-used",
+          "argsIgnorePattern": "^_"
+        }
+      ],
+      "semi": "off",
+      "@typescript-eslint/semi": [
+        "error",
+        "never"
+      ],
+      "no-extra-semi": "off"
+    }
+  }
+}

rollup.config.js

@@ -0,0 +1,179 @@
+import typescript from '@rollup/plugin-typescript'
+import resolve from '@rollup/plugin-node-resolve'
+import commonjs from '@rollup/plugin-commonjs'
+import json from '@rollup/plugin-json'
+import { terser } from 'rollup-plugin-terser'
+import replace from '@rollup/plugin-replace'
+
+// Custom plugin to provide empty shims for Node.js built-in modules in browser environments
+const nodeModuleShims = () => {
+  return {
+    name: 'node-module-shims',
+    resolveId(source) {
+      // List of Node.js built-in modules to shim
+      const nodeBuiltins = ['fs', 'path', 'util', 'child_process']
+
+      if (nodeBuiltins.includes(source)) {
+        // Return a virtual module ID for the shim
+        return `\0${source}-shim`
+      }
+      return null
+    },
+    load(id) {
+      // If this is one of our shims, return an empty module
+      if (id.startsWith('\0') && id.endsWith('-shim')) {
+        console.log(
+          `Providing empty shim for Node.js module: ${id.slice(1, -5)}`
+        )
+        return 'export default {}; export const promises = {};'
+      }
+      return null
+    }
+  }
+}
+
+// Custom plugin to provide Buffer polyfill for browser environments
+const bufferPolyfill = () => {
+  return {
+    name: 'buffer-polyfill',
+    // This will run before the bundle is generated
+    buildStart() {
+      console.log('Setting up Buffer polyfill for the bundle')
+    },
+    // This will run for each module
+    transform(code, id) {
+      // Only transform the entry point
+      if (id.includes('src/unified.ts')) {
+        // Add import for buffer at the top of the file
+        const bufferImport = `
+// Import Buffer polyfill
+import { Buffer as BufferPolyfill } from 'buffer';
+
+// Make Buffer available globally
+if (typeof window !== 'undefined' && typeof globalThis.Buffer === 'undefined') {
+  globalThis.Buffer = BufferPolyfill;
+}
+`;
+        return {
+          code: bufferImport + code,
+          map: { mappings: '' } // Provide an empty sourcemap to avoid warnings
+        };
+      }
+      return null; // Return null to let Rollup handle other files normally
+    }
+  }
+}
+
+// Custom plugin to fix 'this' references in specific files
+const fixThisReferences = () => {
+  return {
+    name: 'fix-this-references',
+    transform(code, id) {
+      // Only transform the specific files that have issues with 'this'
+      if (
+        id.includes(
+          '@tensorflow/tfjs-layers/dist/layers/convolutional_recurrent.js'
+        )
+      ) {
+        // Replace 'this' with 'globalThis' in the problematic code
+        return {
+          code: code.replace(/\bthis\b/g, 'globalThis'),
+          map: { mappings: '' } // Provide an empty sourcemap to avoid warnings
+        }
+      }
+      return null // Return null to let Rollup handle other files normally
+    }
+  }
+}
+
+// Get build type from environment variable or default to 'unified'
+const buildType = process.env.BUILD_TYPE || 'unified'
+
+// Configuration based on build type
+const config = {
+  unified: {
+    input: 'src/unified.ts',
+    outputPrefix: 'unified',
+    tsconfig: './tsconfig.unified.json',
+    declaration: true,
+    declarationMap: true,
+    intro: `
+// Buffer polyfill is now included via the buffer-polyfill plugin
+    `
+  },
+  browser: {
+    input: 'src/unified.ts',
+    outputPrefix: 'brainy',
+    tsconfig: './tsconfig.browser.json',
+    declaration: false,
+    declarationMap: false,
+    intro: `
+// Set global for compatibility
+var global = typeof window !== "undefined" ? window : this;
+// Buffer polyfill is now included via the buffer-polyfill plugin
+`
+  }
+}
+
+// Get configuration for the current build type
+const buildConfig = config[buildType]
+
+// Create the rollup configuration
+export default {
+  input: buildConfig.input,
+  output: [
+    {
+      dir: 'dist',
+      entryFileNames: `${buildConfig.outputPrefix}.js`,
+      format: 'es',
+      sourcemap: true,
+      inlineDynamicImports: true,
+      intro: buildConfig.intro
+    },
+    {
+      dir: 'dist',
+      entryFileNames: `${buildConfig.outputPrefix}.min.js`,
+      format: 'es',
+      sourcemap: true,
+      inlineDynamicImports: true,
+      intro: buildConfig.intro,
+      plugins: [terser()]
+    }
+  ],
+  plugins: [
+    // Add environment replacement for unified build
+    ...(buildType === 'unified'
+      ? [
+          replace({
+            preventAssignment: true,
+            'process.env.NODE_ENV': JSON.stringify('production')
+          })
+        ]
+      : []),
+    // Add our custom plugins
+    fixThisReferences(),
+    nodeModuleShims(),
+    bufferPolyfill(), // Add Buffer polyfill
+    resolve({
+      browser: true,
+      preferBuiltins: false
+    }),
+    commonjs({
+      transformMixedEsModules: true
+    }),
+    json(),
+    typescript({
+      tsconfig: buildConfig.tsconfig,
+      declaration: buildConfig.declaration,
+      declarationMap: buildConfig.declarationMap
+    })
+  ],
+  external: [
+    // Add any dependencies you want to exclude from the bundle
+    '@aws-sdk/client-s3',
+    '@smithy/util-stream',
+    '@smithy/node-http-handler',
+    '@aws-crypto/crc32c',
+    'node:stream/web'
+  ]
+}

scalingStrategy.md

@@ -0,0 +1,77 @@
+
+
+# HNSW and Large-Scale Data Management
+
+HNSW (Hierarchical Navigable Small World) is a graph-based algorithm for approximate nearest neighbor search that is designed to be efficient and scalable. However, when dealing with datasets that can't fit entirely in memory (like terabytes of data), there are important considerations and adaptations needed.
+
+## Standard HNSW Implementation Limitations
+
+Looking at the implementation in this project, the standard HNSW approach has some memory limitations:
+
+1. **In-Memory Index**: The core `HNSWIndex` class keeps all nodes and their connections in memory:
+   ```typescript
+   private nouns: Map<string, HNSWNoun> = new Map()
+   ```
+
+2. **Full Load During Initialization**: During initialization, all nodes are loaded from storage into memory:
+   ```typescript
+   // Load all nouns from storage
+   const nouns: HNSWNoun[] = await this.storage!.getAllNouns()
+   
+   // Clear the index and add all nouns
+   this.index.clear()
+   for (const noun of nouns) {
+     // Add to index
+     this.index.addItem({
+       id: noun.id,
+       vector: noun.vector
+     })
+   }
+   ```
+
+## Approaches for Terabyte-Scale Data
+
+For terabyte-scale data that can't fit in memory, several approaches can be used:
+
+### 1. Disk-Based HNSW
+
+Modified HNSW implementations can use disk-based storage with intelligent caching:
+
+- **Partial Loading**: Only load the most frequently accessed parts of the graph into memory
+- **Page-Based Access**: Organize the graph into pages that can be swapped in and out of memory
+- **Memory-Mapped Files**: Use memory-mapped files to let the OS handle paging
+
+### 2. Distributed HNSW
+
+For truly massive datasets, a distributed approach is necessary:
+
+- **Sharding**: Partition the vector space and distribute across multiple machines
+- **Hierarchical Search**: Use a coarse quantization layer to route queries to the right shard
+- **Federated Results**: Combine results from multiple shards
+
+### 3. Hybrid Solutions
+
+Practical implementations often combine multiple techniques:
+
+- **Quantization**: Reduce vector precision (e.g., from 32-bit to 8-bit) to fit more vectors in memory
+- **Product Quantization**: Compress vectors while maintaining search accuracy
+- **Two-Tier Architecture**: Use a small in-memory index to route to larger disk-based indices
+
+## Real-World Examples
+
+Several systems implement HNSW for large-scale data:
+
+- **Qdrant and Milvus**: Vector databases that support disk-based HNSW indices
+- **FAISS**: Facebook's similarity search library with HNSW implementation that supports GPU and distributed setups
+- **DiskANN**: Microsoft's disk-based approximate nearest neighbor search system
+
+## Conclusion
+
+While the basic HNSW algorithm requires the graph structure to be in memory for optimal performance, modified implementations can handle terabyte-scale data through:
+
+1. Disk-based storage with efficient caching
+2. Distributed architectures across multiple machines
+3. Vector compression techniques
+4. Hierarchical multi-tier approaches
+
+These adaptations allow HNSW to scale to massive datasets while maintaining reasonable query performance, though typically with some trade-offs in terms of search accuracy or latency compared to a fully in-memory implementation.

scripts/encode-image.js

@@ -0,0 +1,47 @@
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+// Get the directory of the current module
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+// Path to the image file
+const imagePath = path.join(__dirname, '..', 'brainy.png');
+
+// Read the image file
+const imageBuffer = fs.readFileSync(imagePath);
+
+// Convert the image to base64
+const base64Image = imageBuffer.toString('base64');
+
+// Get the MIME type based on file extension
+const getMimeType = (filePath) => {
+  const ext = path.extname(filePath).toLowerCase();
+  switch (ext) {
+    case '.png':
+      return 'image/png';
+    case '.jpg':
+    case '.jpeg':
+      return 'image/jpeg';
+    case '.gif':
+      return 'image/gif';
+    case '.svg':
+      return 'image/svg+xml';
+    default:
+      return 'application/octet-stream';
+  }
+};
+
+const mimeType = getMimeType(imagePath);
+
+// Create the data URL
+const dataUrl = `data:${mimeType};base64,${base64Image}`;
+
+// Output the complete HTML img tag
+const imgTag = `<img src="${dataUrl}" alt="Brainy Logo" width="200"/>`;
+
+// Write to a file instead of console.log to avoid truncation
+fs.writeFileSync(path.join(__dirname, '..', 'encoded-image.html'), imgTag);
+
+console.log('Base64 encoded image has been saved to encoded-image.html');

scripts/generate-version.js

@@ -0,0 +1,107 @@
+#!/usr/bin/env node
+
+/**
+ * Generate Version Script
+ *
+ * This script generates a version.js file that exports the version from package.json.
+ * This allows the CLI to access the version without having to read package.json directly,
+ * which can be problematic when the package is installed globally.
+ *
+ * It also updates the version in the README.md file to ensure it stays in sync with package.json.
+ */
+
+import fs from 'fs'
+import path from 'path'
+import { fileURLToPath } from 'url'
+
+// Get the directory of the current module
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+
+// Path to the root directory
+const rootDir = path.join(__dirname, '..')
+
+// Path to package.json
+const packageJsonPath = path.join(rootDir, 'package.json')
+
+// Path to the output directory
+const outputDir = path.join(rootDir, 'src', 'utils')
+
+// Path to the output file
+const outputFile = path.join(outputDir, 'version.ts')
+
+// Path to README.md
+const readmePath = path.join(rootDir, 'README.md')
+
+// Read package.json
+const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'))
+const version = packageJson.version
+
+// Create the output directory if it doesn't exist
+if (!fs.existsSync(outputDir)) {
+  fs.mkdirSync(outputDir, { recursive: true })
+}
+
+// Generate the version.ts file
+const content = `/**
+ * This file is auto-generated during the build process.
+ * Do not modify this file directly.
+ */
+
+export const VERSION = '${version}';
+`
+
+// Write the file
+fs.writeFileSync(outputFile, content)
+
+console.log(`Generated version.ts with version ${version}`)
+
+// Update README.md with the current version, Node.js version, and TypeScript version
+try {
+  let readmeContent = fs.readFileSync(readmePath, 'utf8')
+
+  // Get Node.js version requirement from package.json
+  const nodeVersion = packageJson.engines.node.replace('>=', '')
+
+  // Get TypeScript version from package.json devDependencies
+  const typescriptVersion = packageJson.devDependencies.typescript.replace('^', '')
+
+  // Update npm badge - using a more flexible approach with explicit version
+  const npmBadgeRegex = /\[\!\[npm\].*?\]\(https:\/\/www\.npmjs\.com\/package\/@soulcraft\/brainy\)/g
+  if (npmBadgeRegex.test(readmeContent)) {
+    readmeContent = readmeContent.replace(
+      npmBadgeRegex,
+      `[![npm](https://img.shields.io/badge/npm-v${version}-blue.svg)](https://www.npmjs.com/package/@soulcraft/brainy)`
+    )
+  } else {
+    console.log('Warning: Could not find npm badge in README.md')
+  }
+
+  // Update Node.js badge - using a more flexible approach
+  const nodeBadgeRegex = /\[\!\[Node\.js\].*?\]\(https:\/\/nodejs\.org\/\)/g
+  if (nodeBadgeRegex.test(readmeContent)) {
+    readmeContent = readmeContent.replace(
+      nodeBadgeRegex,
+      `[![Node.js](https://img.shields.io/badge/node-%3E%3D${nodeVersion}-brightgreen.svg)](https://nodejs.org/)`
+    )
+  } else {
+    console.log('Warning: Could not find Node.js badge in README.md')
+  }
+
+  // Update TypeScript badge - using a more flexible approach
+  const tsBadgeRegex = /\[\!\[TypeScript\].*?\]\(https:\/\/www\.typescriptlang\.org\/\)/g
+  if (tsBadgeRegex.test(readmeContent)) {
+    readmeContent = readmeContent.replace(
+      tsBadgeRegex,
+      `[![TypeScript](https://img.shields.io/badge/TypeScript-${typescriptVersion}-blue.svg)](https://www.typescriptlang.org/)`
+    )
+  } else {
+    console.log('Warning: Could not find TypeScript badge in README.md')
+  }
+
+  // Write the updated README back to disk
+  fs.writeFileSync(readmePath, readmeContent)
+  console.log(`Updated README.md with npm version ${version}, Node.js version ${nodeVersion}, and TypeScript version ${typescriptVersion}`)
+} catch (error) {
+  console.error('Error updating README.md:', error)
+}

scripts/update-readme.js

@@ -0,0 +1,26 @@
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+// Get the directory of the current module
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+// Paths to the files
+const readmePath = path.join(__dirname, '..', 'README.md');
+const encodedImagePath = path.join(__dirname, '..', 'encoded-image.html');
+
+// Read the files
+const readmeContent = fs.readFileSync(readmePath, 'utf8');
+const encodedImageTag = fs.readFileSync(encodedImagePath, 'utf8');
+
+// Replace the image tag in the README
+const updatedReadme = readmeContent.replace(
+  /<img src="brainy\.png" alt="Brainy Logo" width="200"\/>/,
+  encodedImageTag
+);
+
+// Write the updated README
+fs.writeFileSync(readmePath, updatedReadme);
+
+console.log('README.md has been updated with the base64-encoded image.');

src/augmentationFactory.ts

@@ -0,0 +1,628 @@
+/**
+ * Augmentation Factory
+ *
+ * This module provides a simplified factory for creating augmentations with minimal boilerplate.
+ * It reduces the complexity of creating and using augmentations by providing a fluent API
+ * and handling common patterns automatically.
+ */
+
+import {
+  IAugmentation,
+  AugmentationType,
+  AugmentationResponse,
+  ISenseAugmentation,
+  IConduitAugmentation,
+  ICognitionAugmentation,
+  IMemoryAugmentation,
+  IPerceptionAugmentation,
+  IDialogAugmentation,
+  IActivationAugmentation,
+  IWebSocketSupport,
+  WebSocketConnection
+} from './types/augmentations.js'
+import { registerAugmentation } from './augmentationRegistry.js'
+
+/**
+ * Options for creating an augmentation
+ */
+export interface AugmentationOptions {
+  name: string
+  description?: string
+  enabled?: boolean
+  autoRegister?: boolean
+  autoInitialize?: boolean
+}
+
+/**
+ * Base class for all augmentations created with the factory
+ * Handles common functionality like initialization, shutdown, and status
+ */
+class BaseAugmentation implements IAugmentation {
+  readonly name: string
+  readonly description: string
+  enabled: boolean = true
+  protected isInitialized: boolean = false
+
+  constructor(options: AugmentationOptions) {
+    this.name = options.name
+    this.description = options.description || `${options.name} augmentation`
+    this.enabled = options.enabled !== false
+  }
+
+  async initialize(): Promise<void> {
+    if (this.isInitialized) return
+    this.isInitialized = true
+  }
+
+  async shutDown(): Promise<void> {
+    this.isInitialized = false
+  }
+
+  async getStatus(): Promise<'active' | 'inactive' | 'error'> {
+    return this.isInitialized ? 'active' : 'inactive'
+  }
+
+  protected async ensureInitialized(): Promise<void> {
+    if (!this.isInitialized) {
+      await this.initialize()
+    }
+  }
+}
+
+/**
+ * Factory for creating sense augmentations
+ */
+export function createSenseAugmentation(
+  options: AugmentationOptions & {
+    processRawData?: (
+      rawData: Buffer | string,
+      dataType: string
+    ) =>
+      | Promise<AugmentationResponse<{ nouns: string[]; verbs: string[] }>>
+      | AugmentationResponse<{
+          nouns: string[]
+          verbs: string[]
+        }>
+    listenToFeed?: (
+      feedUrl: string,
+      callback: (data: { nouns: string[]; verbs: string[] }) => void
+    ) => Promise<void>
+  }
+): ISenseAugmentation {
+  const augmentation = new BaseAugmentation(options) as unknown as ISenseAugmentation
+
+  // Implement the sense augmentation methods
+  augmentation.processRawData = async (
+    rawData: Buffer | string,
+    dataType: string
+  ) => {
+    await augmentation.ensureInitialized()
+
+    if (options.processRawData) {
+      const result = options.processRawData(rawData, dataType)
+      return result instanceof Promise ? await result : result
+    }
+
+    return {
+      success: false,
+      data: { nouns: [], verbs: [] },
+      error: 'processRawData not implemented'
+    }
+  }
+
+  augmentation.listenToFeed = async (
+    feedUrl: string,
+    callback: (data: { nouns: string[]; verbs: string[] }) => void
+  ) => {
+    await augmentation.ensureInitialized()
+
+    if (options.listenToFeed) {
+      return options.listenToFeed(feedUrl, callback)
+    }
+
+    throw new Error('listenToFeed not implemented')
+  }
+
+  // Auto-register if requested
+  if (options.autoRegister) {
+    registerAugmentation(augmentation)
+
+    // Auto-initialize if requested
+    if (options.autoInitialize) {
+      augmentation.initialize().catch((error) => {
+        console.error(
+          `Failed to initialize augmentation ${augmentation.name}:`,
+          error
+        )
+      })
+    }
+  }
+
+  return augmentation
+}
+
+/**
+ * Factory for creating conduit augmentations
+ */
+export function createConduitAugmentation(
+  options: AugmentationOptions & {
+    establishConnection?: (
+      targetSystemId: string,
+      config: Record<string, unknown>
+    ) =>
+      | Promise<AugmentationResponse<WebSocketConnection>>
+      | AugmentationResponse<WebSocketConnection>
+    readData?: (
+      query: Record<string, unknown>,
+      options?: Record<string, unknown>
+    ) => Promise<AugmentationResponse<unknown>> | AugmentationResponse<unknown>
+    writeData?: (
+      data: Record<string, unknown>,
+      options?: Record<string, unknown>
+    ) => Promise<AugmentationResponse<unknown>> | AugmentationResponse<unknown>
+    monitorStream?: (
+      streamId: string,
+      callback: (data: unknown) => void
+    ) => Promise<void>
+  }
+): IConduitAugmentation {
+  const augmentation = new BaseAugmentation(options) as unknown as IConduitAugmentation
+
+  // Implement the conduit augmentation methods
+  augmentation.establishConnection = async (
+    targetSystemId: string,
+    config: Record<string, unknown>
+  ) => {
+    await augmentation.ensureInitialized()
+
+    if (options.establishConnection) {
+      const result = options.establishConnection(targetSystemId, config)
+      return result instanceof Promise ? await result : result
+    }
+
+    return {
+      success: false,
+      data: null as any,
+      error: 'establishConnection not implemented'
+    }
+  }
+
+  augmentation.readData = async (
+    query: Record<string, unknown>,
+    opts?: Record<string, unknown>
+  ) => {
+    await augmentation.ensureInitialized()
+
+    if (options.readData) {
+      const result = options.readData(query, opts)
+      return result instanceof Promise ? await result : result
+    }
+
+    return {
+      success: false,
+      data: null,
+      error: 'readData not implemented'
+    }
+  }
+
+  augmentation.writeData = async (
+    data: Record<string, unknown>,
+    opts?: Record<string, unknown>
+  ) => {
+    await augmentation.ensureInitialized()
+
+    if (options.writeData) {
+      const result = options.writeData(data, opts)
+      return result instanceof Promise ? await result : result
+    }
+
+    return {
+      success: false,
+      data: null,
+      error: 'writeData not implemented'
+    }
+  }
+
+  augmentation.monitorStream = async (
+    streamId: string,
+    callback: (data: unknown) => void
+  ) => {
+    await augmentation.ensureInitialized()
+
+    if (options.monitorStream) {
+      return options.monitorStream(streamId, callback)
+    }
+
+    throw new Error('monitorStream not implemented')
+  }
+
+  // Auto-register if requested
+  if (options.autoRegister) {
+    registerAugmentation(augmentation)
+
+    // Auto-initialize if requested
+    if (options.autoInitialize) {
+      augmentation.initialize().catch((error) => {
+        console.error(
+          `Failed to initialize augmentation ${augmentation.name}:`,
+          error
+        )
+      })
+    }
+  }
+
+  return augmentation
+}
+
+/**
+ * Factory for creating memory augmentations
+ */
+export function createMemoryAugmentation(
+  options: AugmentationOptions & {
+    storeData?: (
+      key: string,
+      data: unknown,
+      options?: Record<string, unknown>
+    ) => Promise<AugmentationResponse<boolean>> | AugmentationResponse<boolean>
+    retrieveData?: (
+      key: string,
+      options?: Record<string, unknown>
+    ) => Promise<AugmentationResponse<unknown>> | AugmentationResponse<unknown>
+    updateData?: (
+      key: string,
+      data: unknown,
+      options?: Record<string, unknown>
+    ) => Promise<AugmentationResponse<boolean>> | AugmentationResponse<boolean>
+    deleteData?: (
+      key: string,
+      options?: Record<string, unknown>
+    ) => Promise<AugmentationResponse<boolean>> | AugmentationResponse<boolean>
+    listDataKeys?: (
+      pattern?: string,
+      options?: Record<string, unknown>
+    ) =>
+      | Promise<AugmentationResponse<string[]>>
+      | AugmentationResponse<string[]>
+    search?: (
+      query: unknown,
+      k?: number,
+      options?: Record<string, unknown>
+    ) =>
+      | Promise<
+          AugmentationResponse<
+            Array<{ id: string; score: number; data: unknown }>
+          >
+        >
+      | AugmentationResponse<
+          Array<{ id: string; score: number; data: unknown }>
+        >
+  }
+): IMemoryAugmentation {
+  const augmentation = new BaseAugmentation(options) as unknown as IMemoryAugmentation
+
+  // Implement the memory augmentation methods
+  augmentation.storeData = async (
+    key: string,
+    data: unknown,
+    opts?: Record<string, unknown>
+  ) => {
+    await augmentation.ensureInitialized()
+
+    if (options.storeData) {
+      const result = options.storeData(key, data, opts)
+      return result instanceof Promise ? await result : result
+    }
+
+    return {
+      success: false,
+      data: false,
+      error: 'storeData not implemented'
+    }
+  }
+
+  augmentation.retrieveData = async (
+    key: string,
+    opts?: Record<string, unknown>
+  ) => {
+    await augmentation.ensureInitialized()
+
+    if (options.retrieveData) {
+      const result = options.retrieveData(key, opts)
+      return result instanceof Promise ? await result : result
+    }
+
+    return {
+      success: false,
+      data: null,
+      error: 'retrieveData not implemented'
+    }
+  }
+
+  augmentation.updateData = async (
+    key: string,
+    data: unknown,
+    opts?: Record<string, unknown>
+  ) => {
+    await augmentation.ensureInitialized()
+
+    if (options.updateData) {
+      const result = options.updateData(key, data, opts)
+      return result instanceof Promise ? await result : result
+    }
+
+    return {
+      success: false,
+      data: false,
+      error: 'updateData not implemented'
+    }
+  }
+
+  augmentation.deleteData = async (
+    key: string,
+    opts?: Record<string, unknown>
+  ) => {
+    await augmentation.ensureInitialized()
+
+    if (options.deleteData) {
+      const result = options.deleteData(key, opts)
+      return result instanceof Promise ? await result : result
+    }
+
+    return {
+      success: false,
+      data: false,
+      error: 'deleteData not implemented'
+    }
+  }
+
+  augmentation.listDataKeys = async (
+    pattern?: string,
+    opts?: Record<string, unknown>
+  ) => {
+    await augmentation.ensureInitialized()
+
+    if (options.listDataKeys) {
+      const result = options.listDataKeys(pattern, opts)
+      return result instanceof Promise ? await result : result
+    }
+
+    return {
+      success: false,
+      data: [],
+      error: 'listDataKeys not implemented'
+    }
+  }
+
+  augmentation.search = async (
+    query: unknown,
+    k?: number,
+    opts?: Record<string, unknown>
+  ) => {
+    await augmentation.ensureInitialized()
+
+    if (options.search) {
+      const result = options.search(query, k, opts)
+      return result instanceof Promise ? await result : result
+    }
+
+    return {
+      success: false,
+      data: [],
+      error: 'search not implemented'
+    }
+  }
+
+  // Auto-register if requested
+  if (options.autoRegister) {
+    registerAugmentation(augmentation)
+
+    // Auto-initialize if requested
+    if (options.autoInitialize) {
+      augmentation.initialize().catch((error) => {
+        console.error(
+          `Failed to initialize augmentation ${augmentation.name}:`,
+          error
+        )
+      })
+    }
+  }
+
+  return augmentation
+}
+
+/**
+ * Factory for creating WebSocket-enabled augmentations
+ * This can be combined with other augmentation factories to create WebSocket-enabled versions
+ */
+export function addWebSocketSupport<T extends IAugmentation>(
+  augmentation: T,
+  options: {
+    connectWebSocket?: (
+      url: string,
+      protocols?: string | string[]
+    ) => Promise<WebSocketConnection>
+    sendWebSocketMessage?: (
+      connectionId: string,
+      data: unknown
+    ) => Promise<void>
+    onWebSocketMessage?: (
+      connectionId: string,
+      callback: (data: unknown) => void
+    ) => Promise<void>
+    offWebSocketMessage?: (
+      connectionId: string,
+      callback: (data: unknown) => void
+    ) => Promise<void>
+    closeWebSocket?: (
+      connectionId: string,
+      code?: number,
+      reason?: string
+    ) => Promise<void>
+  }
+): T & IWebSocketSupport {
+  const wsAugmentation = augmentation as T & IWebSocketSupport
+
+  // Add WebSocket methods
+  wsAugmentation.connectWebSocket = async (
+    url: string,
+    protocols?: string | string[]
+  ) => {
+    await (augmentation as any).ensureInitialized?.()
+
+    if (options.connectWebSocket) {
+      return options.connectWebSocket(url, protocols)
+    }
+
+    throw new Error('connectWebSocket not implemented')
+  }
+
+  wsAugmentation.sendWebSocketMessage = async (
+    connectionId: string,
+    data: unknown
+  ) => {
+    await (augmentation as any).ensureInitialized?.()
+
+    if (options.sendWebSocketMessage) {
+      return options.sendWebSocketMessage(connectionId, data)
+    }
+
+    throw new Error('sendWebSocketMessage not implemented')
+  }
+
+  wsAugmentation.onWebSocketMessage = async (
+    connectionId: string,
+    callback: (data: unknown) => void
+  ) => {
+    await (augmentation as any).ensureInitialized?.()
+
+    if (options.onWebSocketMessage) {
+      return options.onWebSocketMessage(connectionId, callback)
+    }
+
+    throw new Error('onWebSocketMessage not implemented')
+  }
+
+  wsAugmentation.offWebSocketMessage = async (
+    connectionId: string,
+    callback: (data: unknown) => void
+  ) => {
+    await (augmentation as any).ensureInitialized?.()
+
+    if (options.offWebSocketMessage) {
+      return options.offWebSocketMessage(connectionId, callback)
+    }
+
+    throw new Error('offWebSocketMessage not implemented')
+  }
+
+  wsAugmentation.closeWebSocket = async (
+    connectionId: string,
+    code?: number,
+    reason?: string
+  ) => {
+    await (augmentation as any).ensureInitialized?.()
+
+    if (options.closeWebSocket) {
+      return options.closeWebSocket(connectionId, code, reason)
+    }
+
+    throw new Error('closeWebSocket not implemented')
+  }
+
+  return wsAugmentation
+}
+
+/**
+ * Simplified function to execute an augmentation method with automatic error handling
+ * This provides a more concise way to execute augmentation methods compared to the full pipeline
+ */
+export async function executeAugmentation<T, R>(
+  augmentation: IAugmentation,
+  method: string,
+  ...args: any[]
+): Promise<AugmentationResponse<R>> {
+  try {
+    if (!augmentation.enabled) {
+      return {
+        success: false,
+        data: null as any,
+        error: `Augmentation ${augmentation.name} is disabled`
+      }
+    }
+
+    if (typeof (augmentation as any)[method] !== 'function') {
+      return {
+        success: false,
+        data: null as any,
+        error: `Method ${method} not found on augmentation ${augmentation.name}`
+      }
+    }
+
+    const result = await (augmentation as any)[method](...args)
+    return result
+  } catch (error) {
+    console.error(`Error executing ${method} on ${augmentation.name}:`, error)
+    return {
+      success: false,
+      data: null as any,
+      error: error instanceof Error ? error.message : String(error)
+    }
+  }
+}
+
+/**
+ * Dynamically load augmentations from a module at runtime
+ * This allows for lazy-loading augmentations when needed instead of at build time
+ */
+export async function loadAugmentationModule(
+  modulePromise: Promise<any>,
+  options: {
+    autoRegister?: boolean
+    autoInitialize?: boolean
+  } = {}
+): Promise<IAugmentation[]> {
+  try {
+    const module = await modulePromise
+    const augmentations: IAugmentation[] = []
+
+    // Extract augmentations from the module
+    for (const key in module) {
+      const exported = module[key]
+
+      // Skip non-objects and null
+      if (!exported || typeof exported !== 'object') {
+        continue
+      }
+
+      // Check if it's an augmentation
+      if (
+        typeof exported.name === 'string' &&
+        typeof exported.initialize === 'function' &&
+        typeof exported.shutDown === 'function' &&
+        typeof exported.getStatus === 'function'
+      ) {
+        augmentations.push(exported)
+
+        // Auto-register if requested
+        if (options.autoRegister) {
+          registerAugmentation(exported)
+
+          // Auto-initialize if requested
+          if (options.autoInitialize) {
+            exported.initialize().catch((error: Error) => {
+              console.error(
+                `Failed to initialize augmentation ${exported.name}:`,
+                error
+              )
+            })
+          }
+        }
+      }
+    }
+
+    return augmentations
+  } catch (error) {
+    console.error('Error loading augmentation module:', error)
+    return []
+  }
+}

src/augmentationPipeline.ts

@@ -0,0 +1,709 @@
+/**
+ * Augmentation Event Pipeline
+ *
+ * This module provides a pipeline for managing and executing multiple augmentations
+ * of each type. It allows registering multiple augmentations and executing them
+ * in sequence or in parallel.
+ */
+
+import {
+  BrainyAugmentations,
+  IAugmentation,
+  IWebSocketSupport,
+  AugmentationResponse,
+  AugmentationType
+} from './types/augmentations.js'
+import { isThreadingAvailable, isBrowser, isNode } from './utils/environment.js'
+import { executeInThread } from './utils/workerUtils.js'
+
+/**
+ * Type definitions for the augmentation registry
+ */
+type AugmentationRegistry = {
+  sense: BrainyAugmentations.ISenseAugmentation[];
+  conduit: BrainyAugmentations.IConduitAugmentation[];
+  cognition: BrainyAugmentations.ICognitionAugmentation[];
+  memory: BrainyAugmentations.IMemoryAugmentation[];
+  perception: BrainyAugmentations.IPerceptionAugmentation[];
+  dialog: BrainyAugmentations.IDialogAugmentation[];
+  activation: BrainyAugmentations.IActivationAugmentation[];
+  webSocket: IWebSocketSupport[];
+}
+
+/**
+ * Execution mode for the pipeline
+ */
+export enum ExecutionMode {
+  SEQUENTIAL = 'sequential',
+  PARALLEL = 'parallel',
+  FIRST_SUCCESS = 'firstSuccess',
+  FIRST_RESULT = 'firstResult',
+  THREADED = 'threaded'  // Execute in separate threads when available
+}
+
+/**
+ * Options for pipeline execution
+ */
+export interface PipelineOptions {
+  mode?: ExecutionMode;
+  timeout?: number;
+  stopOnError?: boolean;
+  forceThreading?: boolean;  // Force threading even if not in THREADED mode
+  disableThreading?: boolean;  // Disable threading even if in THREADED mode
+}
+
+/**
+ * Default pipeline options
+ */
+const DEFAULT_PIPELINE_OPTIONS: PipelineOptions = {
+  mode: ExecutionMode.SEQUENTIAL,
+  timeout: 30000,
+  stopOnError: false,
+  forceThreading: false,
+  disableThreading: false
+}
+
+/**
+ * AugmentationPipeline class
+ *
+ * Manages multiple augmentations of each type and provides methods to execute them.
+ */
+export class AugmentationPipeline {
+  private registry: AugmentationRegistry = {
+    sense: [],
+    conduit: [],
+    cognition: [],
+    memory: [],
+    perception: [],
+    dialog: [],
+    activation: [],
+    webSocket: []
+  }
+
+  /**
+   * Register an augmentation with the pipeline
+   *
+   * @param augmentation The augmentation to register
+   * @returns The pipeline instance for chaining
+   */
+  public register<T extends IAugmentation>(augmentation: T): AugmentationPipeline {
+    let registered = false
+
+    // Check for specific augmentation types
+    if (this.isAugmentationType<BrainyAugmentations.ISenseAugmentation>(
+      augmentation,
+      'processRawData',
+      'listenToFeed'
+    )) {
+      this.registry.sense.push(augmentation)
+      registered = true
+    } else if (this.isAugmentationType<BrainyAugmentations.IConduitAugmentation>(
+      augmentation,
+      'establishConnection',
+      'readData',
+      'writeData',
+      'monitorStream'
+    )) {
+      this.registry.conduit.push(augmentation)
+      registered = true
+    } else if (this.isAugmentationType<BrainyAugmentations.ICognitionAugmentation>(
+      augmentation,
+      'reason',
+      'infer',
+      'executeLogic'
+    )) {
+      this.registry.cognition.push(augmentation)
+      registered = true
+    } else if (this.isAugmentationType<BrainyAugmentations.IMemoryAugmentation>(
+      augmentation,
+      'storeData',
+      'retrieveData',
+      'updateData',
+      'deleteData',
+      'listDataKeys'
+    )) {
+      this.registry.memory.push(augmentation)
+      registered = true
+    } else if (this.isAugmentationType<BrainyAugmentations.IPerceptionAugmentation>(
+      augmentation,
+      'interpret',
+      'organize',
+      'generateVisualization'
+    )) {
+      this.registry.perception.push(augmentation)
+      registered = true
+    } else if (this.isAugmentationType<BrainyAugmentations.IDialogAugmentation>(
+      augmentation,
+      'processUserInput',
+      'generateResponse',
+      'manageContext'
+    )) {
+      this.registry.dialog.push(augmentation)
+      registered = true
+    } else if (this.isAugmentationType<BrainyAugmentations.IActivationAugmentation>(
+      augmentation,
+      'triggerAction',
+      'generateOutput',
+      'interactExternal'
+    )) {
+      this.registry.activation.push(augmentation)
+      registered = true
+    }
+
+    // Check if the augmentation supports WebSocket
+    if (this.isAugmentationType<IWebSocketSupport>(
+      augmentation,
+      'connectWebSocket',
+      'sendWebSocketMessage',
+      'onWebSocketMessage',
+      'closeWebSocket'
+    )) {
+      this.registry.webSocket.push(augmentation as IWebSocketSupport)
+      registered = true
+    }
+
+    // If the augmentation wasn't registered as any known type, throw an error
+    if (!registered) {
+      throw new Error(`Unknown augmentation type: ${augmentation.name}`)
+    }
+
+    return this
+  }
+
+  /**
+   * Unregister an augmentation from the pipeline
+   *
+   * @param augmentationName The name of the augmentation to unregister
+   * @returns The pipeline instance for chaining
+   */
+  public unregister(augmentationName: string): AugmentationPipeline {
+    let found = false
+
+    // Remove from all registries
+    for (const type in this.registry) {
+      const typedRegistry = this.registry[type as keyof AugmentationRegistry]
+      const index = typedRegistry.findIndex(aug => aug.name === augmentationName)
+
+      if (index !== -1) {
+        typedRegistry.splice(index, 1)
+        found = true
+      }
+    }
+
+    return this
+  }
+
+  /**
+   * Initialize all registered augmentations
+   *
+   * @returns A promise that resolves when all augmentations are initialized
+   */
+  public async initialize(): Promise<void> {
+    const allAugmentations = this.getAllAugmentations()
+
+    await Promise.all(
+      allAugmentations.map(augmentation =>
+        augmentation.initialize().catch(error => {
+          console.error(`Failed to initialize augmentation ${augmentation.name}:`, error)
+        })
+      )
+    )
+  }
+
+  /**
+   * Shut down all registered augmentations
+   *
+   * @returns A promise that resolves when all augmentations are shut down
+   */
+  public async shutDown(): Promise<void> {
+    const allAugmentations = this.getAllAugmentations()
+
+    await Promise.all(
+      allAugmentations.map(augmentation =>
+        augmentation.shutDown().catch(error => {
+          console.error(`Failed to shut down augmentation ${augmentation.name}:`, error)
+        })
+      )
+    )
+  }
+
+  /**
+   * Execute a sense pipeline
+   *
+   * @param method The method to execute on each sense augmentation
+   * @param args The arguments to pass to the method
+   * @param options The pipeline execution options
+   * @returns A promise that resolves with the results from all augmentations
+   */
+  public async executeSensePipeline<
+    M extends keyof BrainyAugmentations.ISenseAugmentation & string,
+    R extends BrainyAugmentations.ISenseAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.ISenseAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.ISenseAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const opts = { ...DEFAULT_PIPELINE_OPTIONS, ...options }
+    return this.executeTypedPipeline<BrainyAugmentations.ISenseAugmentation, M, R>(
+      this.registry.sense,
+      method,
+      args,
+      opts
+    )
+  }
+
+  /**
+   * Execute a conduit pipeline
+   *
+   * @param method The method to execute on each conduit augmentation
+   * @param args The arguments to pass to the method
+   * @param options The pipeline execution options
+   * @returns A promise that resolves with the results from all augmentations
+   */
+  public async executeConduitPipeline<
+    M extends keyof BrainyAugmentations.IConduitAugmentation & string,
+    R extends BrainyAugmentations.IConduitAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.IConduitAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.IConduitAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const opts = { ...DEFAULT_PIPELINE_OPTIONS, ...options }
+    return this.executeTypedPipeline<BrainyAugmentations.IConduitAugmentation, M, R>(
+      this.registry.conduit,
+      method,
+      args,
+      opts
+    )
+  }
+
+  /**
+   * Execute a cognition pipeline
+   *
+   * @param method The method to execute on each cognition augmentation
+   * @param args The arguments to pass to the method
+   * @param options The pipeline execution options
+   * @returns A promise that resolves with the results from all augmentations
+   */
+  public async executeCognitionPipeline<
+    M extends keyof BrainyAugmentations.ICognitionAugmentation & string,
+    R extends BrainyAugmentations.ICognitionAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.ICognitionAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.ICognitionAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const opts = { ...DEFAULT_PIPELINE_OPTIONS, ...options }
+    return this.executeTypedPipeline<BrainyAugmentations.ICognitionAugmentation, M, R>(
+      this.registry.cognition,
+      method,
+      args,
+      opts
+    )
+  }
+
+  /**
+   * Execute a memory pipeline
+   *
+   * @param method The method to execute on each memory augmentation
+   * @param args The arguments to pass to the method
+   * @param options The pipeline execution options
+   * @returns A promise that resolves with the results from all augmentations
+   */
+  public async executeMemoryPipeline<
+    M extends keyof BrainyAugmentations.IMemoryAugmentation & string,
+    R extends BrainyAugmentations.IMemoryAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.IMemoryAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.IMemoryAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const opts = { ...DEFAULT_PIPELINE_OPTIONS, ...options }
+    return this.executeTypedPipeline<BrainyAugmentations.IMemoryAugmentation, M, R>(
+      this.registry.memory,
+      method,
+      args,
+      opts
+    )
+  }
+
+  /**
+   * Execute a perception pipeline
+   *
+   * @param method The method to execute on each perception augmentation
+   * @param args The arguments to pass to the method
+   * @param options The pipeline execution options
+   * @returns A promise that resolves with the results from all augmentations
+   */
+  public async executePerceptionPipeline<
+    M extends keyof BrainyAugmentations.IPerceptionAugmentation & string,
+    R extends BrainyAugmentations.IPerceptionAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.IPerceptionAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.IPerceptionAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const opts = { ...DEFAULT_PIPELINE_OPTIONS, ...options }
+    return this.executeTypedPipeline<BrainyAugmentations.IPerceptionAugmentation, M, R>(
+      this.registry.perception,
+      method,
+      args,
+      opts
+    )
+  }
+
+  /**
+   * Execute a dialog pipeline
+   *
+   * @param method The method to execute on each dialog augmentation
+   * @param args The arguments to pass to the method
+   * @param options The pipeline execution options
+   * @returns A promise that resolves with the results from all augmentations
+   */
+  public async executeDialogPipeline<
+    M extends keyof BrainyAugmentations.IDialogAugmentation & string,
+    R extends BrainyAugmentations.IDialogAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.IDialogAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.IDialogAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const opts = { ...DEFAULT_PIPELINE_OPTIONS, ...options }
+    return this.executeTypedPipeline<BrainyAugmentations.IDialogAugmentation, M, R>(
+      this.registry.dialog,
+      method,
+      args,
+      opts
+    )
+  }
+
+  /**
+   * Execute an activation pipeline
+   *
+   * @param method The method to execute on each activation augmentation
+   * @param args The arguments to pass to the method
+   * @param options The pipeline execution options
+   * @returns A promise that resolves with the results from all augmentations
+   */
+  public async executeActivationPipeline<
+    M extends keyof BrainyAugmentations.IActivationAugmentation & string,
+    R extends BrainyAugmentations.IActivationAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.IActivationAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.IActivationAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const opts = { ...DEFAULT_PIPELINE_OPTIONS, ...options }
+    return this.executeTypedPipeline<BrainyAugmentations.IActivationAugmentation, M, R>(
+      this.registry.activation,
+      method,
+      args,
+      opts
+    )
+  }
+
+  /**
+   * Get all registered augmentations
+   *
+   * @returns An array of all registered augmentations
+   */
+  public getAllAugmentations(): IAugmentation[] {
+    // Create a Set to avoid duplicates (an augmentation might be in multiple registries)
+    const allAugmentations = new Set<IAugmentation>([
+      ...this.registry.sense,
+      ...this.registry.conduit,
+      ...this.registry.cognition,
+      ...this.registry.memory,
+      ...this.registry.perception,
+      ...this.registry.dialog,
+      ...this.registry.activation,
+      ...this.registry.webSocket
+    ])
+
+    // Convert back to array
+    return Array.from(allAugmentations)
+  }
+
+  /**
+   * Get all augmentations of a specific type
+   *
+   * @param type The type of augmentation to get
+   * @returns An array of all augmentations of the specified type
+   */
+  public getAugmentationsByType(type: AugmentationType): IAugmentation[] {
+    switch (type) {
+      case AugmentationType.SENSE:
+        return [...this.registry.sense]
+      case AugmentationType.CONDUIT:
+        return [...this.registry.conduit]
+      case AugmentationType.COGNITION:
+        return [...this.registry.cognition]
+      case AugmentationType.MEMORY:
+        return [...this.registry.memory]
+      case AugmentationType.PERCEPTION:
+        return [...this.registry.perception]
+      case AugmentationType.DIALOG:
+        return [...this.registry.dialog]
+      case AugmentationType.ACTIVATION:
+        return [...this.registry.activation]
+      case AugmentationType.WEBSOCKET:
+        return [...this.registry.webSocket]
+      default:
+        return []
+    }
+  }
+
+  /**
+   * Get all available augmentation types
+   *
+   * @returns An array of all augmentation types that have at least one registered augmentation
+   */
+  public getAvailableAugmentationTypes(): AugmentationType[] {
+    const availableTypes: AugmentationType[] = []
+
+    if (this.registry.sense.length > 0) availableTypes.push(AugmentationType.SENSE)
+    if (this.registry.conduit.length > 0) availableTypes.push(AugmentationType.CONDUIT)
+    if (this.registry.cognition.length > 0) availableTypes.push(AugmentationType.COGNITION)
+    if (this.registry.memory.length > 0) availableTypes.push(AugmentationType.MEMORY)
+    if (this.registry.perception.length > 0) availableTypes.push(AugmentationType.PERCEPTION)
+    if (this.registry.dialog.length > 0) availableTypes.push(AugmentationType.DIALOG)
+    if (this.registry.activation.length > 0) availableTypes.push(AugmentationType.ACTIVATION)
+    if (this.registry.webSocket.length > 0) availableTypes.push(AugmentationType.WEBSOCKET)
+
+    return availableTypes
+  }
+
+  /**
+   * Get all WebSocket-supporting augmentations
+   *
+   * @returns An array of all augmentations that support WebSocket connections
+   */
+  public getWebSocketAugmentations(): IWebSocketSupport[] {
+    return [...this.registry.webSocket]
+  }
+
+  /**
+   * Check if an augmentation is of a specific type
+   *
+   * @param augmentation The augmentation to check
+   * @param methods The methods that should be present on the augmentation
+   * @returns True if the augmentation is of the specified type
+   */
+  private isAugmentationType<T extends IAugmentation>(
+    augmentation: IAugmentation,
+    ...methods: (keyof T)[]
+  ): augmentation is T {
+    // First check that the augmentation has all the required base methods
+    const baseMethodsExist = [
+      'initialize',
+      'shutDown',
+      'getStatus'
+    ].every(method => typeof (augmentation as any)[method] === 'function')
+
+    if (!baseMethodsExist) {
+      return false
+    }
+
+    // Then check that it has all the specific methods for this type
+    return methods.every(method => typeof (augmentation as any)[method] === 'function')
+  }
+
+  /**
+   * Determines if threading should be used based on options and environment
+   * 
+   * @param options The pipeline options
+   * @returns True if threading should be used, false otherwise
+   */
+  private shouldUseThreading(options: PipelineOptions): boolean {
+    // If threading is explicitly disabled, don't use it
+    if (options.disableThreading) {
+      return false;
+    }
+
+    // If threading is explicitly forced, use it if available
+    if (options.forceThreading) {
+      return isThreadingAvailable();
+    }
+
+    // If in THREADED mode, use threading if available
+    if (options.mode === ExecutionMode.THREADED) {
+      return isThreadingAvailable();
+    }
+
+    // Otherwise, don't use threading
+    return false;
+  }
+
+  /**
+   * Execute a pipeline for a specific augmentation type
+   *
+   * @param augmentations The augmentations to execute
+   * @param method The method to execute on each augmentation
+   * @param args The arguments to pass to the method
+   * @param options The pipeline execution options
+   * @returns A promise that resolves with the results from all augmentations
+   */
+  private async executeTypedPipeline<
+    T extends IAugmentation,
+    M extends keyof T & string,
+    R extends T[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    augmentations: T[],
+    method: M & (T[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<T[M], (...args: any[]) => any>>,
+    options: PipelineOptions
+  ): Promise<Promise<{
+    success: boolean
+    data: R
+    error?: string
+  }>[]> {
+    // Filter out disabled augmentations
+    const enabledAugmentations = augmentations.filter(aug => aug.enabled !== false)
+
+    if (enabledAugmentations.length === 0) {
+      return []
+    }
+
+    // Create a function to execute the method on an augmentation
+    const executeMethod = async (augmentation: T): Promise<{
+      success: boolean
+      data: R
+      error?: string
+    }> => {
+      try {
+        // Create a timeout promise if a timeout is specified
+        const timeoutPromise = options.timeout
+          ? new Promise<{
+            success: boolean
+            data: R
+            error?: string
+          }>((_, reject) => {
+            setTimeout(() => {
+              reject(new Error(`Timeout executing ${String(method)} on ${augmentation.name}`))
+            }, options.timeout)
+          })
+          : null
+
+        // Check if threading should be used
+        const useThreading = this.shouldUseThreading(options);
+
+        // Execute the method on the augmentation, using threading if appropriate
+        let methodPromise: Promise<AugmentationResponse<R>>;
+
+        if (useThreading) {
+          // Execute in a separate thread
+          try {
+            // Create a function that can be serialized and executed in a worker
+            const workerFn = (...workerArgs: any[]) => {
+              // This function will be stringified and executed in the worker
+              // It needs to be self-contained
+              const augFn = augmentation[method as string] as Function;
+              return augFn.apply(augmentation, workerArgs);
+            };
+
+            methodPromise = executeInThread<AugmentationResponse<R>>(workerFn.toString(), args);
+          } catch (threadError) {
+            console.warn(`Failed to execute in thread, falling back to main thread: ${threadError}`);
+            // Fall back to executing in the main thread
+            methodPromise = Promise.resolve((augmentation[method] as Function)(...args) as AugmentationResponse<R>);
+          }
+        } else {
+          // Execute in the main thread
+          methodPromise = Promise.resolve((augmentation[method] as Function)(...args) as AugmentationResponse<R>);
+        }
+
+        // Race the method promise against the timeout promise if a timeout is specified
+        const result = timeoutPromise
+          ? await Promise.race([methodPromise, timeoutPromise])
+          : await methodPromise
+
+        return result
+      } catch (error) {
+        console.error(`Error executing ${String(method)} on ${augmentation.name}:`, error)
+        return {
+          success: false,
+          data: null as unknown as R,
+          error: error instanceof Error ? error.message : String(error)
+        }
+      }
+    }
+
+    // Execute the pipeline based on the specified mode
+    switch (options.mode) {
+      case ExecutionMode.PARALLEL:
+        // Execute all augmentations in parallel
+        return enabledAugmentations.map(executeMethod)
+
+      case ExecutionMode.THREADED:
+        // Execute all augmentations in parallel with threading enabled
+        // Force threading for this mode
+        const threadedOptions = { ...options, forceThreading: true };
+
+        // Create a new executeMethod function that uses the threaded options
+        const executeMethodThreaded = async (augmentation: T) => {
+          // Save the original options
+          const originalOptions = options;
+
+          // Set the options to the threaded options
+          options = threadedOptions;
+
+          // Execute the method
+          const result = await executeMethod(augmentation);
+
+          // Restore the original options
+          options = originalOptions;
+
+          return result;
+        };
+
+        return enabledAugmentations.map(executeMethodThreaded);
+
+      case ExecutionMode.FIRST_SUCCESS:
+        // Execute augmentations sequentially until one succeeds
+        for (const augmentation of enabledAugmentations) {
+          const resultPromise = executeMethod(augmentation)
+          const result = await resultPromise
+          if (result.success) {
+            return [resultPromise]
+          }
+        }
+        return []
+
+      case ExecutionMode.FIRST_RESULT:
+        // Execute augmentations sequentially until one returns a result
+        for (const augmentation of enabledAugmentations) {
+          const resultPromise = executeMethod(augmentation)
+          const result = await resultPromise
+          if (result.success && result.data) {
+            return [resultPromise]
+          }
+        }
+        return []
+
+      case ExecutionMode.SEQUENTIAL:
+      default:
+        // Execute augmentations sequentially
+        const results: Promise<{
+          success: boolean
+          data: R
+          error?: string
+        }>[] = []
+        for (const augmentation of enabledAugmentations) {
+          const resultPromise = executeMethod(augmentation)
+          results.push(resultPromise)
+
+          // Check if we need to stop on error
+          if (options.stopOnError) {
+            const result = await resultPromise
+            if (!result.success) {
+              break
+            }
+          }
+        }
+        return results
+    }
+  }
+}
+
+// Create and export a default instance of the pipeline
+export const augmentationPipeline = new AugmentationPipeline()

src/augmentationRegistry.ts

@@ -0,0 +1,122 @@
+/**
+ * Augmentation Registry
+ *
+ * This module provides a registry for augmentations that are loaded at build time.
+ * It replaces the dynamic loading mechanism in pluginLoader.ts.
+ */
+
+import { IPipeline } from './types/pipelineTypes.js'
+import { AugmentationType, IAugmentation } from './types/augmentations.js'
+
+// Forward declaration of the pipeline instance to avoid circular dependency
+// The actual pipeline will be provided when initializeAugmentationPipeline is called
+let defaultPipeline: IPipeline | null = null
+
+/**
+ * Sets the default pipeline instance
+ * This function should be called from pipeline.ts after the pipeline is created
+ */
+export function setDefaultPipeline(pipeline: IPipeline): void {
+  defaultPipeline = pipeline
+}
+
+/**
+ * Registry of all available augmentations
+ */
+export const availableAugmentations: IAugmentation[] = []
+
+/**
+ * Registers an augmentation with the registry
+ *
+ * @param augmentation The augmentation to register
+ * @returns The augmentation that was registered
+ */
+export function registerAugmentation<T extends IAugmentation>(augmentation: T): T {
+  // Set enabled to true by default if not specified
+  if (augmentation.enabled === undefined) {
+    augmentation.enabled = true
+  }
+
+  // Add to the registry
+  availableAugmentations.push(augmentation)
+
+  return augmentation
+}
+
+/**
+ * Initializes the augmentation pipeline with all registered augmentations
+ *
+ * @param pipeline Optional custom pipeline to use instead of the default
+ * @returns The pipeline that was initialized
+ * @throws Error if no pipeline is provided and the default pipeline hasn't been set
+ */
+export function initializeAugmentationPipeline(
+  pipelineInstance?: IPipeline
+): IPipeline {
+  // Use the provided pipeline or fall back to the default
+  const pipeline = pipelineInstance || defaultPipeline
+
+  if (!pipeline) {
+    throw new Error('No pipeline provided and default pipeline not set. Call setDefaultPipeline first.')
+  }
+
+  // Register all augmentations with the pipeline
+  for (const augmentation of availableAugmentations) {
+    if (augmentation.enabled) {
+      pipeline.register(augmentation)
+    }
+  }
+
+  return pipeline
+}
+
+/**
+ * Enables or disables an augmentation by name
+ *
+ * @param name The name of the augmentation to enable/disable
+ * @param enabled Whether to enable or disable the augmentation
+ * @returns True if the augmentation was found and updated, false otherwise
+ */
+export function setAugmentationEnabled(name: string, enabled: boolean): boolean {
+  const augmentation = availableAugmentations.find(aug => aug.name === name)
+
+  if (augmentation) {
+    augmentation.enabled = enabled
+    return true
+  }
+
+  return false
+}
+
+/**
+ * Gets all augmentations of a specific type
+ *
+ * @param type The type of augmentation to get
+ * @returns An array of all augmentations of the specified type
+ */
+export function getAugmentationsByType(type: AugmentationType): IAugmentation[] {
+  return availableAugmentations.filter(aug => {
+    // Check if the augmentation is of the specified type
+    // This is a simplified check and may need to be updated based on how types are determined
+    switch (type) {
+      case AugmentationType.SENSE:
+        return 'processRawData' in aug && 'listenToFeed' in aug
+      case AugmentationType.CONDUIT:
+        return 'establishConnection' in aug && 'readData' in aug && 'writeData' in aug
+      case AugmentationType.COGNITION:
+        return 'reason' in aug && 'infer' in aug && 'executeLogic' in aug
+      case AugmentationType.MEMORY:
+        return 'storeData' in aug && 'retrieveData' in aug && 'updateData' in aug
+      case AugmentationType.PERCEPTION:
+        return 'interpret' in aug && 'organize' in aug && 'generateVisualization' in aug
+      case AugmentationType.DIALOG:
+        return 'processUserInput' in aug && 'generateResponse' in aug && 'manageContext' in aug
+      case AugmentationType.ACTIVATION:
+        return 'triggerAction' in aug && 'generateOutput' in aug && 'interactExternal' in aug
+      case AugmentationType.WEBSOCKET:
+        return 'connectWebSocket' in aug && 'sendWebSocketMessage' in aug && 'onWebSocketMessage' in aug
+      default:
+        return false
+    }
+  })
+}

src/augmentationRegistryLoader.ts

@@ -0,0 +1,291 @@
+/**
+ * Augmentation Registry Loader
+ *
+ * This module provides functionality for loading augmentation registrations
+ * at build time. It's designed to be used with build tools like webpack or rollup
+ * to automatically discover and register augmentations.
+ */
+
+import { IAugmentation } from './types/augmentations.js'
+import { registerAugmentation } from './augmentationRegistry.js'
+
+/**
+ * Options for the augmentation registry loader
+ */
+export interface AugmentationRegistryLoaderOptions {
+  /**
+   * Whether to automatically initialize the augmentations after loading
+   * @default false
+   */
+  autoInitialize?: boolean;
+
+  /**
+   * Whether to log debug information during loading
+   * @default false
+   */
+  debug?: boolean;
+}
+
+/**
+ * Default options for the augmentation registry loader
+ */
+const DEFAULT_OPTIONS: AugmentationRegistryLoaderOptions = {
+  autoInitialize: false,
+  debug: false
+}
+
+/**
+ * Result of loading augmentations
+ */
+export interface AugmentationLoadResult {
+  /**
+   * The augmentations that were loaded
+   */
+  augmentations: IAugmentation[];
+
+  /**
+   * Any errors that occurred during loading
+   */
+  errors: Error[];
+}
+
+/**
+ * Loads augmentations from the specified modules
+ *
+ * This function is designed to be used with build tools like webpack or rollup
+ * to automatically discover and register augmentations.
+ *
+ * @param modules An object containing modules with augmentations to register
+ * @param options Options for the loader
+ * @returns A promise that resolves with the result of loading the augmentations
+ *
+ * @example
+ * ```typescript
+ * // webpack.config.js
+ * const { AugmentationRegistryPlugin } = require('brainy/dist/webpack');
+ *
+ * module.exports = {
+ *   // ... other webpack config
+ *   plugins: [
+ *     new AugmentationRegistryPlugin({
+ *       // Pattern to match files containing augmentations
+ *       pattern: /augmentation\.js$/,
+ *       // Options for the loader
+ *       options: {
+ *         autoInitialize: true,
+ *         debug: true
+ *       }
+ *     })
+ *   ]
+ * };
+ * ```
+ */
+export async function loadAugmentationsFromModules(
+  modules: Record<string, any>,
+  options: AugmentationRegistryLoaderOptions = {}
+): Promise<AugmentationLoadResult> {
+  const opts = { ...DEFAULT_OPTIONS, ...options }
+  const result: AugmentationLoadResult = {
+    augmentations: [],
+    errors: []
+  }
+
+  if (opts.debug) {
+    console.log(`[AugmentationRegistryLoader] Loading augmentations from ${Object.keys(modules).length} modules`)
+  }
+
+  // Process each module
+  for (const [modulePath, module] of Object.entries(modules)) {
+    try {
+      if (opts.debug) {
+        console.log(`[AugmentationRegistryLoader] Processing module: ${modulePath}`)
+      }
+
+      // Extract augmentations from the module
+      const augmentations = extractAugmentationsFromModule(module)
+
+      if (augmentations.length === 0) {
+        if (opts.debug) {
+          console.log(`[AugmentationRegistryLoader] No augmentations found in module: ${modulePath}`)
+        }
+        continue
+      }
+
+      // Register each augmentation
+      for (const augmentation of augmentations) {
+        try {
+          const registered = registerAugmentation(augmentation)
+          result.augmentations.push(registered)
+
+          if (opts.debug) {
+            console.log(`[AugmentationRegistryLoader] Registered augmentation: ${registered.name}`)
+          }
+        } catch (error) {
+          const err = error instanceof Error ? error : new Error(String(error))
+          result.errors.push(err)
+
+          if (opts.debug) {
+            console.error(`[AugmentationRegistryLoader] Failed to register augmentation: ${err.message}`)
+          }
+        }
+      }
+    } catch (error) {
+      const err = error instanceof Error ? error : new Error(String(error))
+      result.errors.push(err)
+
+      if (opts.debug) {
+        console.error(`[AugmentationRegistryLoader] Error processing module ${modulePath}: ${err.message}`)
+      }
+    }
+  }
+
+  if (opts.debug) {
+    console.log(`[AugmentationRegistryLoader] Loaded ${result.augmentations.length} augmentations with ${result.errors.length} errors`)
+  }
+
+  return result
+}
+
+/**
+ * Extracts augmentations from a module
+ *
+ * @param module The module to extract augmentations from
+ * @returns An array of augmentations found in the module
+ */
+function extractAugmentationsFromModule(module: any): IAugmentation[] {
+  const augmentations: IAugmentation[] = []
+
+  // If the module itself is an augmentation, add it
+  if (isAugmentation(module)) {
+    augmentations.push(module)
+  }
+
+  // Check for exported augmentations
+  if (module && typeof module === 'object') {
+    for (const key of Object.keys(module)) {
+      const exported = module[key]
+
+      // Skip non-objects and null
+      if (!exported || typeof exported !== 'object') {
+        continue
+      }
+
+      // If the exported value is an augmentation, add it
+      if (isAugmentation(exported)) {
+        augmentations.push(exported)
+      }
+
+      // If the exported value is an array of augmentations, add them
+      if (Array.isArray(exported) && exported.every(isAugmentation)) {
+        augmentations.push(...exported)
+      }
+    }
+  }
+
+  return augmentations
+}
+
+/**
+ * Checks if an object is an augmentation
+ *
+ * @param obj The object to check
+ * @returns True if the object is an augmentation
+ */
+function isAugmentation(obj: any): obj is IAugmentation {
+  return (
+    obj &&
+    typeof obj === 'object' &&
+    typeof obj.name === 'string' &&
+    typeof obj.initialize === 'function' &&
+    typeof obj.shutDown === 'function' &&
+    typeof obj.getStatus === 'function'
+  )
+}
+
+/**
+ * Creates a webpack plugin for automatically loading augmentations
+ *
+ * @param options Options for the plugin
+ * @returns A webpack plugin
+ *
+ * @example
+ * ```typescript
+ * // webpack.config.js
+ * const { createAugmentationRegistryPlugin } = require('brainy/dist/webpack');
+ *
+ * module.exports = {
+ *   // ... other webpack config
+ *   plugins: [
+ *     createAugmentationRegistryPlugin({
+ *       pattern: /augmentation\.js$/,
+ *       options: {
+ *         autoInitialize: true,
+ *         debug: true
+ *       }
+ *     })
+ *   ]
+ * };
+ * ```
+ */
+export function createAugmentationRegistryPlugin(options: {
+  /**
+   * Pattern to match files containing augmentations
+   */
+  pattern: RegExp;
+
+  /**
+   * Options for the loader
+   */
+  options?: AugmentationRegistryLoaderOptions;
+}) {
+  // This is just a placeholder - the actual implementation would depend on the build tool
+  return {
+    name: 'AugmentationRegistryPlugin',
+    pattern: options.pattern,
+    options: options.options || {}
+  }
+}
+
+/**
+ * Creates a rollup plugin for automatically loading augmentations
+ *
+ * @param options Options for the plugin
+ * @returns A rollup plugin
+ *
+ * @example
+ * ```typescript
+ * // rollup.config.js
+ * import { createAugmentationRegistryRollupPlugin } from 'brainy/dist/rollup';
+ *
+ * export default {
+ *   // ... other rollup config
+ *   plugins: [
+ *     createAugmentationRegistryRollupPlugin({
+ *       pattern: /augmentation\.js$/,
+ *       options: {
+ *         autoInitialize: true,
+ *         debug: true
+ *       }
+ *     })
+ *   ]
+ * };
+ * ```
+ */
+export function createAugmentationRegistryRollupPlugin(options: {
+  /**
+   * Pattern to match files containing augmentations
+   */
+  pattern: RegExp;
+
+  /**
+   * Options for the loader
+   */
+  options?: AugmentationRegistryLoaderOptions;
+}) {
+  // This is just a placeholder - the actual implementation would depend on the build tool
+  return {
+    name: 'augmentation-registry-rollup-plugin',
+    pattern: options.pattern,
+    options: options.options || {}
+  }
+}

src/augmentations/README.md

@@ -0,0 +1,251 @@
+<div align="center">
+<img src="../../brainy.png" alt="Brainy Logo" width="200"/>
+
+# Brainy Augmentations
+
+</div>
+
+This directory contains the augmentation implementations for Brainy. Augmentations are pluggable components that extend
+Brainy's functionality in various ways.
+
+## Available Augmentations
+
+### Conduit Augmentations
+
+Conduit augmentations provide data synchronization between Brainy instances.
+
+#### WebSocketConduitAugmentation
+
+A conduit augmentation that syncs Brainy instances using WebSockets. This is used for syncing between browsers and
+servers, or between servers.
+
+```javascript
+import { createConduitAugmentation, augmentationPipeline } from '@soulcraft/brainy'
+
+// Create a WebSocket conduit augmentation
+const wsConduit = await createConduitAugmentation('websocket', 'my-websocket-sync')
+
+// Register the augmentation with the pipeline
+augmentationPipeline.register(wsConduit)
+
+// Connect to another Brainy instance
+const connectionResult = await wsConduit.establishConnection(
+  'wss://your-websocket-server.com/brainy-sync',
+  { protocols: 'brainy-sync' }
+)
+```
+
+#### WebRTCConduitAugmentation
+
+A conduit augmentation that syncs Brainy instances using WebRTC. This is used for direct peer-to-peer syncing between
+browsers.
+
+```javascript
+import { createConduitAugmentation, augmentationPipeline } from '@soulcraft/brainy'
+
+// Create a WebRTC conduit augmentation
+const webrtcConduit = await createConduitAugmentation('webrtc', 'my-webrtc-sync')
+
+// Register the augmentation with the pipeline
+augmentationPipeline.register(webrtcConduit)
+
+// Connect to a peer
+const connectionResult = await webrtcConduit.establishConnection(
+  'peer-id-to-connect-to',
+  {
+    signalServerUrl: 'wss://your-signal-server.com',
+    localPeerId: 'my-peer-id',
+    iceServers: [{ urls: 'stun:stun.l.google.com:19302' }]
+  }
+)
+```
+
+#### ServerSearchConduitAugmentation
+
+A specialized conduit augmentation that provides functionality for searching a server-hosted Brainy instance and storing
+results locally. This allows you to:
+
+- Search a server-hosted Brainy instance from a browser
+- Store the search results in a local Brainy instance
+- Perform further searches against the local instance without needing to query the server again
+- Add data to both local and server instances
+
+```javascript
+import {
+  ServerSearchConduitAugmentation,
+  createServerSearchAugmentations,
+  augmentationPipeline
+} from '@soulcraft/brainy'
+
+// Using the factory function (recommended)
+const { conduit, activation, connection } = await createServerSearchAugmentations(
+  'wss://your-brainy-server.com/ws',
+  { protocols: 'brainy-sync' }
+)
+
+// Register the augmentations with the pipeline
+augmentationPipeline.register(conduit)
+augmentationPipeline.register(activation)
+
+// Search the server and store results locally
+const serverSearchResult = await conduit.searchServer(
+  connection.connectionId,
+  'your search query',
+  5 // limit
+)
+
+// Search the local instance
+const localSearchResult = await conduit.searchLocal('your search query', 5)
+
+// Perform a combined search (local first, then server if needed)
+const combinedSearchResult = await conduit.searchCombined(
+  connection.connectionId,
+  'your search query',
+  5
+)
+
+// Add data to both local and server
+const addResult = await conduit.addToBoth(
+  connection.connectionId,
+  'Text to add',
+  { /* metadata */ }
+)
+```
+
+### Activation Augmentations
+
+Activation augmentations dictate how Brainy initiates actions, responses, or data manipulations.
+
+#### ServerSearchActivationAugmentation
+
+An activation augmentation that provides actions for server search functionality. This works in conjunction with the
+ServerSearchConduitAugmentation to provide a complete solution for browser-server search.
+
+```javascript
+import {
+  ServerSearchActivationAugmentation,
+  createServerSearchAugmentations,
+  augmentationPipeline
+} from '@soulcraft/brainy'
+
+// Using the factory function (recommended)
+const { conduit, activation, connection } = await createServerSearchAugmentations(
+  'wss://your-brainy-server.com/ws',
+  { protocols: 'brainy-sync' }
+)
+
+// Register the augmentations with the pipeline
+augmentationPipeline.register(conduit)
+augmentationPipeline.register(activation)
+
+// Use the activation augmentation to search the server
+const serverSearchAction = activation.triggerAction('searchServer', {
+  connectionId: connection.connectionId,
+  query: 'your search query',
+  limit: 5
+})
+
+if (serverSearchAction.success) {
+  // The data property contains a promise that will resolve to the search results
+  const serverSearchResult = await serverSearchAction.data
+  console.log('Server search results:', serverSearchResult)
+}
+
+// Other available actions:
+// - 'connectToServer': Connect to a server
+// - 'searchLocal': Search the local instance
+// - 'searchCombined': Search both local and server
+// - 'addToBoth': Add data to both local and server
+```
+
+## Using the Augmentation Pipeline
+
+The augmentation pipeline provides a way to execute augmentations based on their type.
+
+```javascript
+import { augmentationPipeline } from '@soulcraft/brainy'
+
+// Execute a conduit augmentation
+const conduitResults = await augmentationPipeline.executeConduitPipeline(
+  'methodName',
+  [arg1, arg2, ...],
+  { /* options */ }
+)
+
+// Execute an activation augmentation
+const activationResults = await augmentationPipeline.executeActivationPipeline(
+  'methodName',
+  [arg1, arg2, ...],
+  { /* options */ }
+)
+```
+
+## Creating Custom Augmentations
+
+To create a custom augmentation, implement one of the augmentation interfaces:
+
+- `ISenseAugmentation`: For processing raw data
+- `IConduitAugmentation`: For data synchronization
+- `ICognitionAugmentation`: For reasoning and inference
+- `IMemoryAugmentation`: For data storage
+- `IPerceptionAugmentation`: For data interpretation and visualization
+- `IDialogAugmentation`: For natural language processing
+- `IActivationAugmentation`: For triggering actions
+
+Example:
+
+```javascript
+import { AugmentationType, IActivationAugmentation } from '@soulcraft/brainy'
+
+class MyCustomActivation implements IActivationAugmentation {
+  readonly
+  name = 'my-custom-activation'
+  readonly
+  description = 'My custom activation augmentation'
+  enabled = true
+
+  getType(): AugmentationType {
+    return AugmentationType.ACTIVATION
+  }
+
+  async initialize(): Promise<void> {
+    // Initialization code
+  }
+
+  async shutDown(): Promise<void> {
+    // Cleanup code
+  }
+
+  async getStatus(): Promise<'active' | 'inactive' | 'error'> {
+    return 'active'
+  }
+
+  triggerAction(actionName: string, parameters
+
+?:
+
+  Record<string, unknown>
+
+):
+
+  AugmentationResponse<unknown> {
+    // Implementation
+  }
+
+  generateOutput(knowledgeId: string, format: string): AugmentationResponse<string | Record<string, unknown
+
+>> {
+  // Implementation
+}
+
+interactExternal(systemId
+:
+string, payload
+:
+Record < string, unknown >
+):
+AugmentationResponse < unknown > {
+  // Implementation
+}
+}
+```

src/augmentations/memoryAugmentations.ts

@@ -0,0 +1,333 @@
+import {
+  AugmentationType,
+  IMemoryAugmentation,
+  AugmentationResponse
+} from '../types/augmentations.js'
+import { StorageAdapter, Vector } from '../coreTypes.js'
+import { MemoryStorage } from '../storage/opfsStorage.js'
+import { FileSystemStorage } from '../storage/fileSystemStorage.js'
+import { OPFSStorage } from '../storage/opfsStorage.js'
+import { cosineDistance } from '../utils/distance.js'
+
+/**
+ * Base class for memory augmentations that wrap a StorageAdapter
+ */
+abstract class BaseMemoryAugmentation implements IMemoryAugmentation {
+  readonly name: string
+  readonly description: string = 'Base memory augmentation'
+  enabled: boolean = true
+  protected storage: StorageAdapter
+  protected isInitialized = false
+
+  constructor(name: string, storage: StorageAdapter) {
+    this.name = name
+    this.storage = storage
+  }
+
+  async initialize(): Promise<void> {
+    if (this.isInitialized) {
+      return
+    }
+
+    try {
+      await this.storage.init()
+      this.isInitialized = true
+    } catch (error) {
+      console.error(`Failed to initialize ${this.name}:`, error)
+      throw new Error(`Failed to initialize ${this.name}: ${error}`)
+    }
+  }
+
+  async shutDown(): Promise<void> {
+    this.isInitialized = false
+  }
+
+  async getStatus(): Promise<'active' | 'inactive' | 'error'> {
+    return this.isInitialized ? 'active' : 'inactive'
+  }
+
+  async storeData(
+    key: string,
+    data: unknown,
+    options?: Record<string, unknown>
+  ): Promise<AugmentationResponse<boolean>> {
+    await this.ensureInitialized()
+
+    try {
+      await this.storage.saveMetadata(key, data)
+      return { success: true, data: true }
+    } catch (error) {
+      console.error(`Failed to store data for key ${key}:`, error)
+      return {
+        success: false,
+        data: false,
+        error: `Failed to store data: ${error}`
+      }
+    }
+  }
+
+  async retrieveData(
+    key: string,
+    options?: Record<string, unknown>
+  ): Promise<AugmentationResponse<unknown>> {
+    await this.ensureInitialized()
+
+    try {
+      const data = await this.storage.getMetadata(key)
+      return {
+        success: true,
+        data
+      }
+    } catch (error) {
+      console.error(`Failed to retrieve data for key ${key}:`, error)
+      return {
+        success: false,
+        data: null,
+        error: `Failed to retrieve data: ${error}`
+      }
+    }
+  }
+
+  async updateData(
+    key: string,
+    data: unknown,
+    options?: Record<string, unknown>
+  ): Promise<AugmentationResponse<boolean>> {
+    await this.ensureInitialized()
+
+    try {
+      await this.storage.saveMetadata(key, data)
+      return { success: true, data: true }
+    } catch (error) {
+      console.error(`Failed to update data for key ${key}:`, error)
+      return {
+        success: false,
+        data: false,
+        error: `Failed to update data: ${error}`
+      }
+    }
+  }
+
+  async deleteData(
+    key: string,
+    options?: Record<string, unknown>
+  ): Promise<AugmentationResponse<boolean>> {
+    await this.ensureInitialized()
+
+    try {
+      // There's no direct deleteMetadata method, so we save null
+      await this.storage.saveMetadata(key, null)
+      return { success: true, data: true }
+    } catch (error) {
+      console.error(`Failed to delete data for key ${key}:`, error)
+      return {
+        success: false,
+        data: false,
+        error: `Failed to delete data: ${error}`
+      }
+    }
+  }
+
+  async listDataKeys(
+    pattern?: string,
+    options?: Record<string, unknown>
+  ): Promise<AugmentationResponse<string[]>> {
+    // This is a limitation of the current StorageAdapter interface
+    // It doesn't provide a way to list all metadata keys
+    // We could implement this in the future by extending the StorageAdapter interface
+    return {
+      success: false,
+      data: [],
+      error: 'listDataKeys is not supported by this storage adapter'
+    }
+  }
+
+  /**
+   * Searches for data in the storage using vector similarity.
+   * Implements the findNearest functionality by calculating distances client-side.
+   * @param query The query vector or data to search for
+   * @param k Number of results to return (default: 10)
+   * @param options Optional search options
+   */
+  async search(
+    query: unknown,
+    k: number = 10,
+    options?: Record<string, unknown>
+  ): Promise<AugmentationResponse<Array<{
+    id: string;
+    score: number;
+    data: unknown;
+  }>>> {
+    await this.ensureInitialized()
+
+    try {
+      // Check if query is a vector
+      let queryVector: Vector
+
+      if (Array.isArray(query) && query.every(item => typeof item === 'number')) {
+        queryVector = query as Vector
+      } else {
+        // If query is not a vector, we can't perform vector search
+        return {
+          success: false,
+          data: [],
+          error: 'Query must be a vector (array of numbers) for vector search'
+        }
+      }
+
+      // Get all nodes from storage
+      const nodes = await this.storage.getAllNouns()
+
+      // Calculate distances and prepare results
+      const results: Array<{
+        id: string;
+        score: number;
+        data: unknown;
+      }> = []
+
+      for (const node of nodes) {
+        // Skip nodes that don't have a vector
+        if (!node.vector || !Array.isArray(node.vector)) {
+          continue
+        }
+
+        // Get metadata for the node
+        const metadata = await this.storage.getMetadata(node.id)
+
+        // Calculate distance between query vector and node vector
+        const distance = cosineDistance(queryVector, node.vector)
+
+        // Convert distance to similarity score (1 - distance for cosine)
+        // This way higher scores are better (more similar)
+        const score = 1 - distance
+
+        results.push({
+          id: node.id,
+          score,
+          data: metadata
+        })
+      }
+
+      // Sort results by score (descending) and take top k
+      results.sort((a, b) => b.score - a.score)
+      const topResults = results.slice(0, k)
+
+      return {
+        success: true,
+        data: topResults
+      }
+    } catch (error) {
+      console.error(`Failed to search in storage:`, error)
+      return {
+        success: false,
+        data: [],
+        error: `Failed to search in storage: ${error}`
+      }
+    }
+  }
+
+  protected async ensureInitialized(): Promise<void> {
+    if (!this.isInitialized) {
+      await this.initialize()
+    }
+  }
+}
+
+/**
+ * Memory augmentation that uses in-memory storage
+ */
+export class MemoryStorageAugmentation extends BaseMemoryAugmentation {
+  readonly description = 'Memory augmentation that stores data in memory'
+  enabled = true
+
+  constructor(name: string) {
+    super(name, new MemoryStorage())
+  }
+
+  getType(): AugmentationType {
+    return AugmentationType.MEMORY
+  }
+}
+
+/**
+ * Memory augmentation that uses file system storage
+ */
+export class FileSystemStorageAugmentation extends BaseMemoryAugmentation {
+  readonly description = 'Memory augmentation that stores data in the file system'
+  enabled = true
+
+  constructor(name: string, rootDirectory?: string) {
+    super(name, new FileSystemStorage(rootDirectory))
+  }
+
+  getType(): AugmentationType {
+    return AugmentationType.MEMORY
+  }
+}
+
+/**
+ * Memory augmentation that uses OPFS (Origin Private File System) storage
+ */
+export class OPFSStorageAugmentation extends BaseMemoryAugmentation {
+  readonly description = 'Memory augmentation that stores data in the Origin Private File System'
+  enabled = true
+
+  constructor(name: string) {
+    super(name, new OPFSStorage())
+  }
+
+  getType(): AugmentationType {
+    return AugmentationType.MEMORY
+  }
+}
+
+/**
+ * Factory function to create the appropriate memory augmentation based on the environment
+ */
+export async function createMemoryAugmentation(
+  name: string,
+  options: {
+    storageType?: 'memory' | 'filesystem' | 'opfs'
+    rootDirectory?: string
+    requestPersistentStorage?: boolean
+  } = {}
+): Promise<IMemoryAugmentation> {
+  // If a specific storage type is requested, use that
+  if (options.storageType) {
+    switch (options.storageType) {
+      case 'memory':
+        return new MemoryStorageAugmentation(name)
+      case 'filesystem':
+        return new FileSystemStorageAugmentation(name, options.rootDirectory)
+      case 'opfs':
+        return new OPFSStorageAugmentation(name)
+    }
+  }
+
+  // Otherwise, select based on environment
+  // Use the global isNode variable from the environment detection
+  const isNodeEnv = globalThis.__ENV__?.isNode || (
+    typeof process !== 'undefined' &&
+    process.versions != null &&
+    process.versions.node != null
+  )
+
+  if (isNodeEnv) {
+    // In Node.js, use FileSystemStorage
+    return new FileSystemStorageAugmentation(name, options.rootDirectory)
+  } else {
+    // In browser, try OPFS first
+    const opfsStorage = new OPFSStorage()
+
+    if (opfsStorage.isOPFSAvailable()) {
+      // Request persistent storage if specified
+      if (options.requestPersistentStorage) {
+        await opfsStorage.requestPersistentStorage()
+      }
+      return new OPFSStorageAugmentation(name)
+    } else {
+      // Fall back to memory storage
+      return new MemoryStorageAugmentation(name)
+    }
+  }
+}

src/augmentations/serverSearchAugmentations.ts

@@ -0,0 +1,665 @@
+/**
+ * Server Search Augmentations
+ *
+ * This file implements conduit and activation augmentations for browser-server search functionality.
+ * It allows Brainy to search a server-hosted instance and store results locally.
+ */
+
+import {
+  AugmentationType,
+  IConduitAugmentation,
+  IActivationAugmentation,
+  IWebSocketSupport,
+  AugmentationResponse,
+  WebSocketConnection
+} from '../types/augmentations.js'
+import { WebSocketConduitAugmentation } from './conduitAugmentations.js'
+import { v4 as uuidv4 } from 'uuid'
+import { BrainyDataInterface } from '../types/brainyDataInterface.js'
+
+/**
+ * ServerSearchConduitAugmentation
+ *
+ * A specialized conduit augmentation that provides functionality for searching
+ * a server-hosted Brainy instance and storing results locally.
+ */
+export class ServerSearchConduitAugmentation extends WebSocketConduitAugmentation {
+  private localDb: BrainyDataInterface | null = null
+
+  constructor(name: string = 'server-search-conduit') {
+    super(name)
+    // this.description = 'Conduit augmentation for server-hosted Brainy search'
+  }
+
+  /**
+   * Initialize the augmentation
+   */
+  async initialize(): Promise<void> {
+    if (this.isInitialized) {
+      return
+    }
+
+    try {
+      // Initialize the base conduit
+      await super.initialize()
+
+      // Local DB must be set before initialization
+      if (!this.localDb) {
+        throw new Error('Local database not set. Call setLocalDb before initializing.')
+      }
+
+      this.isInitialized = true
+    } catch (error) {
+      console.error(`Failed to initialize ${this.name}:`, error)
+      throw new Error(`Failed to initialize ${this.name}: ${error}`)
+    }
+  }
+
+  /**
+   * Set the local Brainy instance
+   * @param db The Brainy instance to use for local storage
+   */
+  setLocalDb(db: BrainyDataInterface): void {
+    this.localDb = db
+  }
+
+  /**
+   * Get the local Brainy instance
+   * @returns The local Brainy instance
+   */
+  getLocalDb(): BrainyDataInterface | null {
+    return this.localDb
+  }
+
+  /**
+   * Search the server-hosted Brainy instance and store results locally
+   * @param connectionId The ID of the established connection
+   * @param query The search query
+   * @param limit Maximum number of results to return
+   * @returns Search results
+   */
+  async searchServer(
+    connectionId: string,
+    query: string,
+    limit: number = 10
+  ): Promise<AugmentationResponse<unknown>> {
+    await this.ensureInitialized()
+
+    try {
+      // Create a search request
+      const readResult = await this.readData({
+        connectionId,
+        query: {
+          type: 'search',
+          query,
+          limit
+        }
+      })
+
+      if (readResult.success && readResult.data) {
+        const searchResults = readResult.data as any[]
+
+        // Store the results in the local Brainy instance
+        if (this.localDb) {
+          for (const result of searchResults) {
+            // Check if the noun already exists in the local database
+            const existingNoun = await this.localDb.get(result.id)
+
+            if (!existingNoun) {
+              // Add the noun to the local database
+              await this.localDb.add(result.vector, result.metadata)
+            }
+          }
+        }
+
+        return {
+          success: true,
+          data: searchResults
+        }
+      } else {
+        return {
+          success: false,
+          data: null,
+          error: readResult.error || 'Unknown error searching server'
+        }
+      }
+    } catch (error) {
+      console.error('Error searching server:', error)
+      return {
+        success: false,
+        data: null,
+        error: `Error searching server: ${error}`
+      }
+    }
+  }
+
+  /**
+   * Search the local Brainy instance
+   * @param query The search query
+   * @param limit Maximum number of results to return
+   * @returns Search results
+   */
+  async searchLocal(
+    query: string,
+    limit: number = 10
+  ): Promise<AugmentationResponse<unknown>> {
+    await this.ensureInitialized()
+
+    try {
+      if (!this.localDb) {
+        return {
+          success: false,
+          data: null,
+          error: 'Local database not initialized'
+        }
+      }
+
+      const results = await this.localDb.searchText(query, limit)
+
+      return {
+        success: true,
+        data: results
+      }
+    } catch (error) {
+      console.error('Error searching local database:', error)
+      return {
+        success: false,
+        data: null,
+        error: `Error searching local database: ${error}`
+      }
+    }
+  }
+
+  /**
+   * Search both server and local instances, combine results, and store server results locally
+   * @param connectionId The ID of the established connection
+   * @param query The search query
+   * @param limit Maximum number of results to return
+   * @returns Combined search results
+   */
+  async searchCombined(
+    connectionId: string,
+    query: string,
+    limit: number = 10
+  ): Promise<AugmentationResponse<unknown>> {
+    await this.ensureInitialized()
+
+    try {
+      // Search local first
+      const localSearchResult = await this.searchLocal(query, limit)
+
+      if (!localSearchResult.success) {
+        return localSearchResult
+      }
+
+      const localResults = localSearchResult.data as any[]
+
+      // If we have enough local results, return them
+      if (localResults.length >= limit) {
+        return localSearchResult
+      }
+
+      // Otherwise, search server for additional results
+      const serverSearchResult = await this.searchServer(
+        connectionId,
+        query,
+        limit - localResults.length
+      )
+
+      if (!serverSearchResult.success) {
+        // If server search fails, return local results
+        return localSearchResult
+      }
+
+      const serverResults = serverSearchResult.data as any[]
+
+      // Combine results, removing duplicates
+      const combinedResults = [...localResults]
+      const localIds = new Set(localResults.map(r => r.id))
+
+      for (const result of serverResults) {
+        if (!localIds.has(result.id)) {
+          combinedResults.push(result)
+        }
+      }
+
+      return {
+        success: true,
+        data: combinedResults
+      }
+    } catch (error) {
+      console.error('Error performing combined search:', error)
+      return {
+        success: false,
+        data: null,
+        error: `Error performing combined search: ${error}`
+      }
+    }
+  }
+
+  /**
+   * Add data to both local and server instances
+   * @param connectionId The ID of the established connection
+   * @param data Text or vector to add
+   * @param metadata Metadata for the data
+   * @returns ID of the added data
+   */
+  async addToBoth(
+    connectionId: string,
+    data: string | any[],
+    metadata: any = {}
+  ): Promise<AugmentationResponse<string>> {
+    await this.ensureInitialized()
+
+    try {
+      if (!this.localDb) {
+        return {
+          success: false,
+          data: '',
+          error: 'Local database not initialized'
+        }
+      }
+
+      // Add to local first
+      const id = await this.localDb.add(data, metadata)
+
+      // Get the vector and metadata
+      const noun = await this.localDb.get(id) as import('../coreTypes.js').VectorDocument<unknown>
+
+      if (!noun) {
+        return {
+          success: false,
+          data: '',
+          error: 'Failed to retrieve newly created noun'
+        }
+      }
+
+      // Add to server
+      const writeResult = await this.writeData({
+        connectionId,
+        data: {
+          type: 'addNoun',
+          vector: noun.vector,
+          metadata: noun.metadata
+        }
+      })
+
+      if (!writeResult.success) {
+        return {
+          success: true,
+          data: id,
+          error: `Added locally but failed to add to server: ${writeResult.error}`
+        }
+      }
+
+      return {
+        success: true,
+        data: id
+      }
+    } catch (error) {
+      console.error('Error adding data to both:', error)
+      return {
+        success: false,
+        data: '',
+        error: `Error adding data to both: ${error}`
+      }
+    }
+  }
+}
+
+/**
+ * ServerSearchActivationAugmentation
+ *
+ * An activation augmentation that provides actions for server search functionality.
+ */
+export class ServerSearchActivationAugmentation implements IActivationAugmentation {
+  readonly name: string
+  readonly description: string
+  enabled: boolean = true
+  private isInitialized = false
+  private conduitAugmentation: ServerSearchConduitAugmentation | null = null
+  private connections: Map<string, WebSocketConnection> = new Map()
+
+  constructor(name: string = 'server-search-activation') {
+    this.name = name
+    this.description = 'Activation augmentation for server-hosted Brainy search'
+  }
+
+  getType(): AugmentationType {
+    return AugmentationType.ACTIVATION
+  }
+
+  /**
+   * Initialize the augmentation
+   */
+  async initialize(): Promise<void> {
+    if (this.isInitialized) {
+      return
+    }
+
+    this.isInitialized = true
+  }
+
+  /**
+   * Shut down the augmentation
+   */
+  async shutDown(): Promise<void> {
+    this.isInitialized = false
+  }
+
+  /**
+   * Get the status of the augmentation
+   */
+  async getStatus(): Promise<'active' | 'inactive' | 'error'> {
+    return this.isInitialized ? 'active' : 'inactive'
+  }
+
+  /**
+   * Set the conduit augmentation to use for server search
+   * @param conduit The ServerSearchConduitAugmentation to use
+   */
+  setConduitAugmentation(conduit: ServerSearchConduitAugmentation): void {
+    this.conduitAugmentation = conduit
+  }
+
+  /**
+   * Store a connection for later use
+   * @param connectionId The ID to use for the connection
+   * @param connection The WebSocket connection
+   */
+  storeConnection(connectionId: string, connection: WebSocketConnection): void {
+    this.connections.set(connectionId, connection)
+  }
+
+  /**
+   * Get a stored connection
+   * @param connectionId The ID of the connection to retrieve
+   * @returns The WebSocket connection
+   */
+  getConnection(connectionId: string): WebSocketConnection | undefined {
+    return this.connections.get(connectionId)
+  }
+
+  /**
+   * Trigger an action based on a processed command or internal state
+   * @param actionName The name of the action to trigger
+   * @param parameters Optional parameters for the action
+   */
+  triggerAction(
+    actionName: string,
+    parameters?: Record<string, unknown>
+  ): AugmentationResponse<unknown> {
+    if (!this.conduitAugmentation) {
+      return {
+        success: false,
+        data: null,
+        error: 'Conduit augmentation not set'
+      }
+    }
+
+    // Handle different actions
+    switch (actionName) {
+      case 'connectToServer':
+        return this.handleConnectToServer(parameters || {})
+      case 'searchServer':
+        return this.handleSearchServer(parameters || {})
+      case 'searchLocal':
+        return this.handleSearchLocal(parameters || {})
+      case 'searchCombined':
+        return this.handleSearchCombined(parameters || {})
+      case 'addToBoth':
+        return this.handleAddToBoth(parameters || {})
+      default:
+        return {
+          success: false,
+          data: null,
+          error: `Unknown action: ${actionName}`
+        }
+    }
+  }
+
+  /**
+   * Handle the connectToServer action
+   * @param parameters Action parameters
+   */
+  private handleConnectToServer(
+    parameters: Record<string, unknown>
+  ): AugmentationResponse<unknown> {
+    const serverUrl = parameters.serverUrl as string
+    const protocols = parameters.protocols as string | string[] | undefined
+
+    if (!serverUrl) {
+      return {
+        success: false,
+        data: null,
+        error: 'serverUrl parameter is required'
+      }
+    }
+
+    // Return a promise that will be resolved when the connection is established
+    return {
+      success: true,
+      data: this.conduitAugmentation!.establishConnection(serverUrl, {
+        protocols
+      })
+    }
+  }
+
+  /**
+   * Handle the searchServer action
+   * @param parameters Action parameters
+   */
+  private handleSearchServer(
+    parameters: Record<string, unknown>
+  ): AugmentationResponse<unknown> {
+    const connectionId = parameters.connectionId as string
+    const query = parameters.query as string
+    const limit = parameters.limit as number || 10
+
+    if (!connectionId) {
+      return {
+        success: false,
+        data: null,
+        error: 'connectionId parameter is required'
+      }
+    }
+
+    if (!query) {
+      return {
+        success: false,
+        data: null,
+        error: 'query parameter is required'
+      }
+    }
+
+    // Return a promise that will be resolved when the search is complete
+    return {
+      success: true,
+      data: this.conduitAugmentation!.searchServer(connectionId, query, limit)
+    }
+  }
+
+  /**
+   * Handle the searchLocal action
+   * @param parameters Action parameters
+   */
+  private handleSearchLocal(
+    parameters: Record<string, unknown>
+  ): AugmentationResponse<unknown> {
+    const query = parameters.query as string
+    const limit = parameters.limit as number || 10
+
+    if (!query) {
+      return {
+        success: false,
+        data: null,
+        error: 'query parameter is required'
+      }
+    }
+
+    // Return a promise that will be resolved when the search is complete
+    return {
+      success: true,
+      data: this.conduitAugmentation!.searchLocal(query, limit)
+    }
+  }
+
+  /**
+   * Handle the searchCombined action
+   * @param parameters Action parameters
+   */
+  private handleSearchCombined(
+    parameters: Record<string, unknown>
+  ): AugmentationResponse<unknown> {
+    const connectionId = parameters.connectionId as string
+    const query = parameters.query as string
+    const limit = parameters.limit as number || 10
+
+    if (!connectionId) {
+      return {
+        success: false,
+        data: null,
+        error: 'connectionId parameter is required'
+      }
+    }
+
+    if (!query) {
+      return {
+        success: false,
+        data: null,
+        error: 'query parameter is required'
+      }
+    }
+
+    // Return a promise that will be resolved when the search is complete
+    return {
+      success: true,
+      data: this.conduitAugmentation!.searchCombined(connectionId, query, limit)
+    }
+  }
+
+  /**
+   * Handle the addToBoth action
+   * @param parameters Action parameters
+   */
+  private handleAddToBoth(
+    parameters: Record<string, unknown>
+  ): AugmentationResponse<unknown> {
+    const connectionId = parameters.connectionId as string
+    const data = parameters.data
+    const metadata = parameters.metadata || {}
+
+    if (!connectionId) {
+      return {
+        success: false,
+        data: null,
+        error: 'connectionId parameter is required'
+      }
+    }
+
+    if (!data) {
+      return {
+        success: false,
+        data: null,
+        error: 'data parameter is required'
+      }
+    }
+
+    // Return a promise that will be resolved when the add is complete
+    return {
+      success: true,
+      data: this.conduitAugmentation!.addToBoth(connectionId, data as any, metadata as any)
+    }
+  }
+
+  /**
+   * Generates an expressive output or response from Brainy
+   * @param knowledgeId The identifier of the knowledge to express
+   * @param format The desired output format (e.g., 'text', 'json')
+   */
+  generateOutput(
+    knowledgeId: string,
+    format: string
+  ): AugmentationResponse<string | Record<string, unknown>> {
+    // This method is not used for server search functionality
+    return {
+      success: false,
+      data: '',
+      error: 'generateOutput is not implemented for ServerSearchActivationAugmentation'
+    }
+  }
+
+  /**
+   * Interacts with an external system or API
+   * @param systemId The identifier of the external system
+   * @param payload The data to send to the external system
+   */
+  interactExternal(
+    systemId: string,
+    payload: Record<string, unknown>
+  ): AugmentationResponse<unknown> {
+    // This method is not used for server search functionality
+    return {
+      success: false,
+      data: null,
+      error: 'interactExternal is not implemented for ServerSearchActivationAugmentation'
+    }
+  }
+}
+
+/**
+ * Factory function to create server search augmentations
+ * @param serverUrl The URL of the server to connect to
+ * @param options Additional options
+ * @returns An object containing the created augmentations
+ */
+export async function createServerSearchAugmentations(
+  serverUrl: string,
+  options: {
+    conduitName?: string,
+    activationName?: string,
+    protocols?: string | string[],
+    localDb?: BrainyDataInterface
+  } = {}
+): Promise<{
+  conduit: ServerSearchConduitAugmentation,
+  activation: ServerSearchActivationAugmentation,
+  connection: WebSocketConnection
+}> {
+  // Create the conduit augmentation
+  const conduit = new ServerSearchConduitAugmentation(options.conduitName)
+  await conduit.initialize()
+
+  // Set the local database if provided
+  if (options.localDb) {
+    conduit.setLocalDb(options.localDb)
+  }
+
+  // Create the activation augmentation
+  const activation = new ServerSearchActivationAugmentation(options.activationName)
+  await activation.initialize()
+
+  // Link the augmentations
+  activation.setConduitAugmentation(conduit)
+
+  // Connect to the server
+  const connectionResult = await conduit.establishConnection(
+    serverUrl,
+    { protocols: options.protocols }
+  )
+
+  if (!connectionResult.success || !connectionResult.data) {
+    throw new Error(`Failed to connect to server: ${connectionResult.error}`)
+  }
+
+  const connection = connectionResult.data
+
+  // Store the connection in the activation augmentation
+  activation.storeConnection(connection.connectionId, connection)
+
+  return {
+    conduit,
+    activation,
+    connection
+  }
+}

src/coreTypes.ts

@@ -0,0 +1,163 @@
+/**
+ * Type definitions for the Soulcraft Brainy
+ */
+
+/**
+ * Vector representation - an array of numbers
+ */
+export type Vector = number[]
+
+/**
+ * A document with a vector embedding and optional metadata
+ */
+export interface VectorDocument<T = any> {
+  id: string
+  vector: Vector
+  metadata?: T
+}
+
+/**
+ * Search result with similarity score
+ */
+export interface SearchResult<T = any> {
+  id: string
+  score: number
+  vector: Vector
+  metadata?: T
+}
+
+/**
+ * Distance function for comparing vectors
+ */
+export type DistanceFunction = (a: Vector, b: Vector) => number
+
+/**
+ * Embedding function for converting data to vectors
+ */
+export type EmbeddingFunction = (data: any) => Promise<Vector>
+
+/**
+ * Embedding model interface
+ */
+export interface EmbeddingModel {
+  /**
+   * Initialize the embedding model
+   */
+  init(): Promise<void>
+
+  /**
+   * Embed data into a vector
+   */
+  embed(data: any): Promise<Vector>
+
+  /**
+   * Dispose of the model resources
+   */
+  dispose(): Promise<void>
+}
+
+/**
+ * HNSW graph noun
+ */
+export interface HNSWNoun {
+  id: string
+  vector: Vector
+  connections: Map<number, Set<string>> // level -> set of connected noun ids
+}
+
+/**
+ * Verb representing a relationship between nouns
+ * Extends HNSWNoun to allow verbs to be first-class entities in the data model
+ */
+export interface GraphVerb extends HNSWNoun {
+  sourceId: string // ID of the source noun
+  targetId: string // ID of the target noun
+  type?: string // Optional type of the relationship
+  weight?: number // Optional weight of the relationship
+  metadata?: any // Optional metadata for the verb
+
+  // Additional properties used in the codebase
+  source?: string // Alias for sourceId
+  target?: string // Alias for targetId
+  verb?: string // Alias for type
+  data?: Record<string, any> // Additional flexible data storage
+  embedding?: Vector // Vector representation of the relationship
+}
+
+/**
+ * HNSW index configuration
+ */
+export interface HNSWConfig {
+  M: number // Maximum number of connections per noun
+  efConstruction: number // Size of the dynamic candidate list during construction
+  efSearch: number // Size of the dynamic candidate list during search
+  ml: number // Maximum level
+}
+
+/**
+ * Storage interface for persistence
+ */
+export interface StorageAdapter {
+  init(): Promise<void>
+
+  saveNoun(noun: HNSWNoun): Promise<void>
+
+  getNoun(id: string): Promise<HNSWNoun | null>
+
+  getAllNouns(): Promise<HNSWNoun[]>
+
+  /**
+   * Get nouns by noun type
+   * @param nounType The noun type to filter by
+   * @returns Promise that resolves to an array of nouns of the specified noun type
+   */
+  getNounsByNounType(nounType: string): Promise<HNSWNoun[]>
+
+  deleteNoun(id: string): Promise<void>
+
+  saveVerb(verb: GraphVerb): Promise<void>
+
+  getVerb(id: string): Promise<GraphVerb | null>
+
+  getAllVerbs(): Promise<GraphVerb[]>
+
+  getVerbsBySource(sourceId: string): Promise<GraphVerb[]>
+
+  getVerbsByTarget(targetId: string): Promise<GraphVerb[]>
+
+  getVerbsByType(type: string): Promise<GraphVerb[]>
+
+  deleteVerb(id: string): Promise<void>
+
+  saveMetadata(id: string, metadata: any): Promise<void>
+
+  getMetadata(id: string): Promise<any | null>
+
+  clear(): Promise<void>
+
+  /**
+   * Get information about storage usage and capacity
+   * @returns Promise that resolves to an object containing storage status information
+   */
+  getStorageStatus(): Promise<{
+    /**
+     * The type of storage being used (e.g., 'filesystem', 'opfs', 'memory')
+     */
+    type: string
+
+    /**
+     * The amount of storage being used in bytes
+     */
+    used: number
+
+    /**
+     * The total amount of storage available in bytes, or null if unknown
+     */
+    quota: number | null
+
+    /**
+     * Additional storage-specific information
+     */
+    details?: Record<string, any>
+  }>
+}

src/examples/basicUsage.ts

@@ -0,0 +1,163 @@
+/**
+ * Basic usage example for the Soulcraft Brainy database
+ */
+
+import { BrainyData } from '../brainyData.js'
+
+// Example data - word embeddings
+const wordEmbeddings = {
+  cat: [0.2, 0.3, 0.4, 0.1],
+  dog: [0.3, 0.2, 0.4, 0.2],
+  fish: [0.1, 0.1, 0.8, 0.2],
+  bird: [0.1, 0.4, 0.2, 0.5],
+  tiger: [0.3, 0.4, 0.3, 0.1],
+  lion: [0.4, 0.3, 0.2, 0.1],
+  shark: [0.2, 0.1, 0.7, 0.3],
+  eagle: [0.2, 0.5, 0.1, 0.4]
+}
+
+// Example metadata
+const metadata = {
+  cat: { type: 'mammal', domesticated: true },
+  dog: { type: 'mammal', domesticated: true },
+  fish: { type: 'fish', domesticated: false },
+  bird: { type: 'bird', domesticated: false },
+  tiger: { type: 'mammal', domesticated: false },
+  lion: { type: 'mammal', domesticated: false },
+  shark: { type: 'fish', domesticated: false },
+  eagle: { type: 'bird', domesticated: false }
+}
+
+/**
+ * Run the example
+ */
+async function runExample() {
+  console.log('Initializing vector database...')
+
+  // Create a new vector database
+  const db = new BrainyData()
+  await db.init()
+
+  console.log('Adding vectors to the database...')
+
+  // Add vectors to the database
+  const ids: Record<string, string> = {}
+  const metadata: Record<string, { type: string; domesticated: boolean }> = {
+    cat: { type: 'mammal', domesticated: true },
+    dog: { type: 'mammal', domesticated: true },
+    fish: { type: 'fish', domesticated: false },
+    bird: { type: 'bird', domesticated: false },
+    tiger: { type: 'mammal', domesticated: false },
+    lion: { type: 'mammal', domesticated: false },
+    shark: { type: 'fish', domesticated: false },
+    eagle: { type: 'bird', domesticated: false }
+  }
+  for (const [word, vector] of Object.entries(wordEmbeddings)) {
+    ids[word] = await db.add(vector, metadata[word])
+
+    console.log(`Added "${word}" with ID: ${ids[word]}`)
+  }
+
+  console.log('\nDatabase size:', db.size())
+
+  // Search for similar vectors
+  console.log('\nSearching for vectors similar to "cat"...')
+  const catResults = await db.search(wordEmbeddings['cat'], 3)
+  console.log('Results:')
+  for (const result of catResults) {
+    const word =
+      Object.entries(ids).find(([_, id]) => id === result.id)?.[0] || 'unknown'
+    console.log(
+      `- ${word} (score: ${result.score.toFixed(4)}, metadata:`,
+      result.metadata,
+      ')'
+    )
+  }
+
+  // Search for similar vectors
+  console.log('\nSearching for vectors similar to "fish"...')
+  const fishResults = await db.search(wordEmbeddings['fish'], 3)
+  console.log('Results:')
+  for (const result of fishResults) {
+    const word =
+      Object.entries(ids).find(([_, id]) => id === result.id)?.[0] || 'unknown'
+    console.log(
+      `- ${word} (score: ${result.score.toFixed(4)}, metadata:`,
+      result.metadata,
+      ')'
+    )
+  }
+
+  // Update metadata
+  console.log('\nUpdating metadata for "bird"...')
+  await db.updateMetadata(ids['bird'], {
+    ...metadata['bird'],
+    notes: 'Can fly'
+  })
+
+  // Get the updated document
+  const birdDoc = await db.get(ids['bird'])
+  console.log('Updated bird document:', birdDoc)
+
+  // Delete a vector
+  console.log('\nDeleting "shark"...')
+  await db.delete(ids['shark'])
+  console.log('Database size after deletion:', db.size())
+
+  // Search again to verify shark is gone
+  console.log('\nSearching for vectors similar to "fish" after deletion...')
+  const fishResultsAfterDeletion = await db.search(wordEmbeddings['fish'], 3)
+  console.log('Results:')
+  for (const result of fishResultsAfterDeletion) {
+    const word =
+      Object.entries(ids).find(([_, id]) => id === result.id)?.[0] || 'unknown'
+    console.log(
+      `- ${word} (score: ${result.score.toFixed(4)}, metadata:`,
+      result.metadata,
+      ')'
+    )
+  }
+
+  console.log('\nExample completed successfully!')
+}
+
+// Check if we're in a browser or Node.js environment
+if (typeof window !== 'undefined') {
+  // Browser environment
+  document.addEventListener('DOMContentLoaded', () => {
+    const button = document.createElement('button')
+    button.textContent = 'Run BrainyData Example'
+    button.addEventListener('click', async () => {
+      const output = document.createElement('pre')
+      document.body.appendChild(output)
+
+      // Redirect console.log to the output element
+      const originalLog = console.log
+      console.log = (...args) => {
+        originalLog(...args)
+        output.textContent +=
+          args
+            .map((arg) =>
+              typeof arg === 'object' ? JSON.stringify(arg, null, 2) : arg
+            )
+            .join(' ') + '\n'
+      }
+
+      try {
+        await runExample()
+      } catch (error) {
+        console.error('Error running example:', error)
+      }
+
+      // Restore console.log
+      console.log = originalLog
+    })
+
+    document.body.appendChild(button)
+  })
+} else {
+  // Node.js environment
+  runExample().catch((error) => {
+    console.error('Error running example:', error)
+  })
+}

src/global.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Global type declarations for Brainy
+ */
+
+// Extend the globalThis interface to include the __ENV__ property
+declare global {
+  var __ENV__: {
+    isBrowser: boolean;
+    isNode: string | false;
+    isServerless: boolean;
+  };
+}
+
+// This export is needed to make this file a module
+export {};

src/hnsw/hnswIndex.ts

@@ -0,0 +1,704 @@
+/**
+ * HNSW (Hierarchical Navigable Small World) Index implementation
+ * Based on the paper: "Efficient and robust approximate nearest neighbor search using Hierarchical Navigable Small World graphs"
+ */
+
+import { DistanceFunction, HNSWConfig, HNSWNoun, Vector, VectorDocument } from '../coreTypes.js'
+import { euclideanDistance } from '../utils/index.js'
+import { executeInThread } from '../utils/workerUtils.js'
+
+// Default HNSW parameters
+const DEFAULT_CONFIG: HNSWConfig = {
+  M: 16, // Max number of connections per noun
+  efConstruction: 200, // Size of a dynamic candidate list during construction
+  efSearch: 50, // Size of a dynamic candidate list during search
+  ml: 16 // Max level
+}
+
+export class HNSWIndex {
+  private nouns: Map<string, HNSWNoun> = new Map()
+  private entryPointId: string | null = null
+  private maxLevel = 0
+  private config: HNSWConfig
+  private distanceFunction: DistanceFunction
+  private dimension: number | null = null
+  private useParallelization: boolean = true // Whether to use parallelization for performance-critical operations
+
+  constructor(
+    config: Partial<HNSWConfig> = {},
+    distanceFunction: DistanceFunction = euclideanDistance,
+    options: { useParallelization?: boolean } = {}
+  ) {
+    this.config = { ...DEFAULT_CONFIG, ...config }
+    this.distanceFunction = distanceFunction
+    this.useParallelization = options.useParallelization !== undefined ? options.useParallelization : true
+  }
+
+  /**
+   * Set whether to use parallelization for performance-critical operations
+   */
+  public setUseParallelization(useParallelization: boolean): void {
+    this.useParallelization = useParallelization
+  }
+
+  /**
+   * Get whether parallelization is enabled
+   */
+  public getUseParallelization(): boolean {
+    return this.useParallelization
+  }
+
+  /**
+   * Calculate distances between a query vector and multiple vectors in parallel
+   * This is used to optimize performance for search operations
+   * @param queryVector The query vector
+   * @param vectors Array of vectors to compare against
+   * @returns Array of distances
+   */
+  private async calculateDistancesInParallel(
+    queryVector: Vector,
+    vectors: Array<{ id: string; vector: Vector }>
+  ): Promise<Array<{ id: string; distance: number }>> {
+    // If parallelization is disabled or there are very few vectors, use sequential processing
+    if (!this.useParallelization || vectors.length < 10) {
+      return vectors.map(item => ({
+        id: item.id,
+        distance: this.distanceFunction(queryVector, item.vector)
+      }))
+    }
+
+    // Function to be executed in a worker thread
+    const distanceCalculator = (
+      args: {
+        queryVector: Vector,
+        vectors: Array<{ id: string; vector: Vector }>,
+        distanceFnString: string
+      }
+    ) => {
+      const { queryVector, vectors, distanceFnString } = args;
+
+      // Recreate the distance function from its string representation
+      const distanceFunction = new Function('return ' + distanceFnString)() as DistanceFunction
+
+      // Calculate distances for all items
+      return vectors.map(item => ({
+        id: item.id,
+        distance: distanceFunction(queryVector, item.vector)
+      }))
+    }
+
+    try {
+      // Convert the distance function to a string for serialization
+      const distanceFnString = this.distanceFunction.toString()
+
+      // Execute the distance calculation in a separate thread
+      return await executeInThread<Array<{ id: string; distance: number }>>(
+        distanceCalculator.toString(),
+        { queryVector, vectors, distanceFnString }
+      )
+    } catch (error) {
+      console.error('Error in parallel distance calculation, falling back to sequential:', error)
+      // Fall back to sequential processing if parallel execution fails
+      return vectors.map(item => ({
+        id: item.id,
+        distance: this.distanceFunction(queryVector, item.vector)
+      }))
+    }
+  }
+
+  /**
+   * Add a vector to the index
+   */
+  public async addItem(item: VectorDocument): Promise<string> {
+    // Check if item is defined
+    if (!item) {
+      throw new Error('Item is undefined or null')
+    }
+
+    const { id, vector } = item
+
+    // Check if vector is defined
+    if (!vector) {
+      throw new Error('Vector is undefined or null')
+    }
+
+    // Set dimension on first insert
+    if (this.dimension === null) {
+      this.dimension = vector.length
+    } else if (vector.length !== this.dimension) {
+      throw new Error(
+        `Vector dimension mismatch: expected ${this.dimension}, got ${vector.length}`
+      )
+    }
+
+    // Generate random level for this noun
+    const nounLevel = this.getRandomLevel()
+
+    // Create new noun
+    const noun: HNSWNoun = {
+      id,
+      vector,
+      connections: new Map()
+    }
+
+    // Initialize empty connection sets for each level
+    for (let level = 0; level <= nounLevel; level++) {
+      noun.connections.set(level, new Set<string>())
+    }
+
+    // If this is the first noun, make it the entry point
+    if (this.nouns.size === 0) {
+      this.entryPointId = id
+      this.maxLevel = nounLevel
+      this.nouns.set(id, noun)
+      return id
+    }
+
+    // Find entry point
+    if (!this.entryPointId) {
+      console.error('Entry point ID is null')
+      // If there's no entry point, this is the first noun, so we should have returned earlier
+      // This is a safety check
+      this.entryPointId = id
+      this.maxLevel = nounLevel
+      this.nouns.set(id, noun)
+      return id
+    }
+
+    const entryPoint = this.nouns.get(this.entryPointId)
+    if (!entryPoint) {
+      console.error(`Entry point with ID ${this.entryPointId} not found`)
+      // If the entry point doesn't exist, treat this as the first noun
+      this.entryPointId = id
+      this.maxLevel = nounLevel
+      this.nouns.set(id, noun)
+      return id
+    }
+
+    let currObj = entryPoint
+    let currDist = this.distanceFunction(vector, entryPoint.vector)
+
+    // Traverse the graph from top to bottom to find the closest noun
+    for (let level = this.maxLevel; level > nounLevel; level--) {
+      let changed = true
+      while (changed) {
+        changed = false
+
+        // Check all neighbors at current level
+        const connections = currObj.connections.get(level) || new Set<string>()
+
+        for (const neighborId of connections) {
+          const neighbor = this.nouns.get(neighborId)
+          if (!neighbor) {
+            // Skip neighbors that don't exist (expected during rapid additions/deletions)
+            continue
+          }
+          const distToNeighbor = this.distanceFunction(vector, neighbor.vector)
+
+          if (distToNeighbor < currDist) {
+            currDist = distToNeighbor
+            currObj = neighbor
+            changed = true
+          }
+        }
+      }
+    }
+
+    // For each level from nounLevel down to 0
+    for (let level = Math.min(nounLevel, this.maxLevel); level >= 0; level--) {
+      // Find ef nearest elements using greedy search
+      const nearestNouns = await this.searchLayer(
+        vector,
+        currObj,
+        this.config.efConstruction,
+        level
+      )
+
+      // Select M nearest neighbors
+      const neighbors = this.selectNeighbors(
+        vector,
+        nearestNouns,
+        this.config.M
+      )
+
+      // Add bidirectional connections
+      for (const [neighborId, _] of neighbors) {
+        const neighbor = this.nouns.get(neighborId)
+        if (!neighbor) {
+          // Skip neighbors that don't exist (expected during rapid additions/deletions)
+          continue
+        }
+
+        noun.connections.get(level)!.add(neighborId)
+
+        // Add reverse connection
+        if (!neighbor.connections.has(level)) {
+          neighbor.connections.set(level, new Set<string>())
+        }
+        neighbor.connections.get(level)!.add(id)
+
+        // Ensure neighbor doesn't have too many connections
+        if (neighbor.connections.get(level)!.size > this.config.M) {
+          this.pruneConnections(neighbor, level)
+        }
+      }
+
+      // Update entry point for the next level
+      if (nearestNouns.size > 0) {
+        const [nearestId, nearestDist] = [...nearestNouns][0]
+        if (nearestDist < currDist) {
+          currDist = nearestDist
+          const nearestNoun = this.nouns.get(nearestId)
+          if (!nearestNoun) {
+            console.error(`Nearest noun with ID ${nearestId} not found in addItem`)
+            // Keep the current object as is
+          } else {
+            currObj = nearestNoun
+          }
+        }
+      }
+    }
+
+    // Update max level and entry point if needed
+    if (nounLevel > this.maxLevel) {
+      this.maxLevel = nounLevel
+      this.entryPointId = id
+    }
+
+    // Add noun to the index
+    this.nouns.set(id, noun)
+    return id
+  }
+
+  /**
+   * Search for nearest neighbors
+   */
+  public async search(queryVector: Vector, k: number = 10): Promise<Array<[string, number]>> {
+    if (this.nouns.size === 0) {
+      return []
+    }
+
+    // Check if query vector is defined
+    if (!queryVector) {
+      throw new Error('Query vector is undefined or null')
+    }
+
+    if (this.dimension !== null && queryVector.length !== this.dimension) {
+      throw new Error(
+        `Query vector dimension mismatch: expected ${this.dimension}, got ${queryVector.length}`
+      )
+    }
+
+    // Start from the entry point
+    if (!this.entryPointId) {
+      console.error('Entry point ID is null')
+      return []
+    }
+
+    const entryPoint = this.nouns.get(this.entryPointId)
+    if (!entryPoint) {
+      console.error(`Entry point with ID ${this.entryPointId} not found`)
+      return []
+    }
+
+    let currObj = entryPoint
+    let currDist = this.distanceFunction(queryVector, currObj.vector)
+
+    // Traverse the graph from top to bottom to find the closest noun
+    for (let level = this.maxLevel; level > 0; level--) {
+      let changed = true
+      while (changed) {
+        changed = false
+
+        // Check all neighbors at current level
+        const connections = currObj.connections.get(level) || new Set<string>()
+
+        // If we have enough connections, use parallel distance calculation
+        if (this.useParallelization && connections.size >= 10) {
+          // Prepare vectors for parallel calculation
+          const vectors: Array<{ id: string; vector: Vector }> = []
+          for (const neighborId of connections) {
+            const neighbor = this.nouns.get(neighborId)
+            if (!neighbor) continue
+            vectors.push({ id: neighborId, vector: neighbor.vector })
+          }
+
+          // Calculate distances in parallel
+          const distances = await this.calculateDistancesInParallel(queryVector, vectors)
+
+          // Find the closest neighbor
+          for (const { id, distance } of distances) {
+            if (distance < currDist) {
+              currDist = distance
+              const neighbor = this.nouns.get(id)
+              if (neighbor) {
+                currObj = neighbor
+                changed = true
+              }
+            }
+          }
+        } else {
+          // Use sequential processing for small number of connections
+          for (const neighborId of connections) {
+            const neighbor = this.nouns.get(neighborId)
+            if (!neighbor) {
+              // Skip neighbors that don't exist (expected during rapid additions/deletions)
+              continue
+            }
+            const distToNeighbor = this.distanceFunction(
+              queryVector,
+              neighbor.vector
+            )
+
+            if (distToNeighbor < currDist) {
+              currDist = distToNeighbor
+              currObj = neighbor
+              changed = true
+            }
+          }
+        }
+      }
+    }
+
+    // Search at level 0 with ef = k
+    const nearestNouns = await this.searchLayer(
+      queryVector,
+      currObj,
+      Math.max(this.config.efSearch, k),
+      0
+    )
+
+    // Convert to array and sort by distance
+    return [...nearestNouns].slice(0, k)
+  }
+
+  /**
+   * Remove an item from the index
+   */
+  public removeItem(id: string): boolean {
+    if (!this.nouns.has(id)) {
+      return false
+    }
+
+    const noun = this.nouns.get(id)!
+
+    // Remove connections to this noun from all neighbors
+    for (const [level, connections] of noun.connections.entries()) {
+      for (const neighborId of connections) {
+        const neighbor = this.nouns.get(neighborId)
+        if (!neighbor) {
+          // Skip neighbors that don't exist (expected during rapid additions/deletions)
+          continue
+        }
+        if (neighbor.connections.has(level)) {
+          neighbor.connections.get(level)!.delete(id)
+
+          // Prune connections after removing this noun to ensure consistency
+          this.pruneConnections(neighbor, level)
+        }
+      }
+    }
+
+    // Also check all other nouns for references to this noun and remove them
+    for (const [nounId, otherNoun] of this.nouns.entries()) {
+      if (nounId === id) continue // Skip the noun being removed
+
+      for (const [level, connections] of otherNoun.connections.entries()) {
+        if (connections.has(id)) {
+          connections.delete(id)
+
+          // Prune connections after removing this reference
+          this.pruneConnections(otherNoun, level)
+        }
+      }
+    }
+
+    // Remove the noun
+    this.nouns.delete(id)
+
+    // If we removed the entry point, find a new one
+    if (this.entryPointId === id) {
+      if (this.nouns.size === 0) {
+        this.entryPointId = null
+        this.maxLevel = 0
+      } else {
+        // Find the noun with the highest level
+        let maxLevel = 0
+        let newEntryPointId = null
+
+        for (const [nounId, noun] of this.nouns.entries()) {
+          if (noun.connections.size === 0) continue // Skip nouns with no connections
+
+          const nounLevel = Math.max(...noun.connections.keys())
+          if (nounLevel >= maxLevel) {
+            maxLevel = nounLevel
+            newEntryPointId = nounId
+          }
+        }
+
+        this.entryPointId = newEntryPointId
+        this.maxLevel = maxLevel
+      }
+    }
+
+    return true
+  }
+
+  /**
+   * Get all nouns in the index
+   */
+  public getNouns(): Map<string, HNSWNoun> {
+    return new Map(this.nouns)
+  }
+
+
+  /**
+   * Clear the index
+   */
+  public clear(): void {
+    this.nouns.clear()
+    this.entryPointId = null
+    this.maxLevel = 0
+  }
+
+  /**
+   * Get the size of the index
+   */
+  public size(): number {
+    return this.nouns.size
+  }
+
+  /**
+   * Get the distance function used by the index
+   */
+  public getDistanceFunction(): DistanceFunction {
+    return this.distanceFunction
+  }
+
+  /**
+   * Get the entry point ID
+   */
+  public getEntryPointId(): string | null {
+    return this.entryPointId
+  }
+
+  /**
+   * Get the maximum level
+   */
+  public getMaxLevel(): number {
+    return this.maxLevel
+  }
+
+  /**
+   * Get the dimension
+   */
+  public getDimension(): number | null {
+    return this.dimension
+  }
+
+  /**
+   * Get the configuration
+   */
+  public getConfig(): HNSWConfig {
+    return { ...this.config }
+  }
+
+  /**
+   * Search within a specific layer
+   * Returns a map of noun IDs to distances, sorted by distance
+   */
+  private async searchLayer(
+    queryVector: Vector,
+    entryPoint: HNSWNoun,
+    ef: number,
+    level: number
+  ): Promise<Map<string, number>> {
+    // Set of visited nouns
+    const visited = new Set<string>([entryPoint.id])
+
+    // Priority queue of candidates (closest first)
+    const candidates = new Map<string, number>()
+    candidates.set(
+      entryPoint.id,
+      this.distanceFunction(queryVector, entryPoint.vector)
+    )
+
+    // Priority queue of nearest neighbors found so far (closest first)
+    const nearest = new Map<string, number>()
+    nearest.set(
+      entryPoint.id,
+      this.distanceFunction(queryVector, entryPoint.vector)
+    )
+
+    // While there are candidates to explore
+    while (candidates.size > 0) {
+      // Get closest candidate
+      const [closestId, closestDist] = [...candidates][0]
+      candidates.delete(closestId)
+
+      // If this candidate is farther than the farthest in our result set, we're done
+      const farthestInNearest = [...nearest][nearest.size - 1]
+      if (nearest.size >= ef && closestDist > farthestInNearest[1]) {
+        break
+      }
+
+      // Explore neighbors of the closest candidate
+      const noun = this.nouns.get(closestId)
+      if (!noun) {
+        console.error(`Noun with ID ${closestId} not found in searchLayer`)
+        continue
+      }
+      const connections = noun.connections.get(level) || new Set<string>()
+
+      // If we have enough connections and parallelization is enabled, use parallel distance calculation
+      if (this.useParallelization && connections.size >= 10) {
+        // Collect unvisited neighbors
+        const unvisitedNeighbors: Array<{ id: string; vector: Vector }> = []
+        for (const neighborId of connections) {
+          if (!visited.has(neighborId)) {
+            visited.add(neighborId)
+            const neighbor = this.nouns.get(neighborId)
+            if (!neighbor) continue
+            unvisitedNeighbors.push({ id: neighborId, vector: neighbor.vector })
+          }
+        }
+
+        if (unvisitedNeighbors.length > 0) {
+          // Calculate distances in parallel
+          const distances = await this.calculateDistancesInParallel(queryVector, unvisitedNeighbors)
+
+          // Process the results
+          for (const { id, distance } of distances) {
+            // If we haven't found ef nearest neighbors yet, or this neighbor is closer than the farthest one we've found
+            if (nearest.size < ef || distance < farthestInNearest[1]) {
+              candidates.set(id, distance)
+              nearest.set(id, distance)
+
+              // If we have more than ef neighbors, remove the farthest one
+              if (nearest.size > ef) {
+                const sortedNearest = [...nearest].sort((a, b) => a[1] - b[1])
+                nearest.clear()
+                for (let i = 0; i < ef; i++) {
+                  nearest.set(sortedNearest[i][0], sortedNearest[i][1])
+                }
+              }
+            }
+          }
+        }
+      } else {
+        // Use sequential processing for small number of connections
+        for (const neighborId of connections) {
+          if (!visited.has(neighborId)) {
+            visited.add(neighborId)
+
+            const neighbor = this.nouns.get(neighborId)
+            if (!neighbor) {
+              // Skip neighbors that don't exist (expected during rapid additions/deletions)
+              continue
+            }
+            const distToNeighbor = this.distanceFunction(
+              queryVector,
+              neighbor.vector
+            )
+
+            // If we haven't found ef nearest neighbors yet, or this neighbor is closer than the farthest one we've found
+            if (nearest.size < ef || distToNeighbor < farthestInNearest[1]) {
+              candidates.set(neighborId, distToNeighbor)
+              nearest.set(neighborId, distToNeighbor)
+
+              // If we have more than ef neighbors, remove the farthest one
+              if (nearest.size > ef) {
+                const sortedNearest = [...nearest].sort((a, b) => a[1] - b[1])
+                nearest.clear()
+                for (let i = 0; i < ef; i++) {
+                  nearest.set(sortedNearest[i][0], sortedNearest[i][1])
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+
+    // Sort nearest by distance
+    return new Map([...nearest].sort((a, b) => a[1] - b[1]))
+  }
+
+  /**
+   * Select M nearest neighbors from the candidate set
+   */
+  private selectNeighbors(
+    queryVector: Vector,
+    candidates: Map<string, number>,
+    M: number
+  ): Map<string, number> {
+    if (candidates.size <= M) {
+      return candidates
+    }
+
+    // Simple heuristic: just take the M closest
+    const sortedCandidates = [...candidates].sort((a, b) => a[1] - b[1])
+    const result = new Map<string, number>()
+
+    for (let i = 0; i < Math.min(M, sortedCandidates.length); i++) {
+      result.set(sortedCandidates[i][0], sortedCandidates[i][1])
+    }
+
+    return result
+  }
+
+  /**
+   * Ensure a noun doesn't have too many connections at a given level
+   */
+  private pruneConnections(noun: HNSWNoun, level: number): void {
+    const connections = noun.connections.get(level)!
+    if (connections.size <= this.config.M) {
+      return
+    }
+
+    // Calculate distances to all neighbors
+    const distances = new Map<string, number>()
+    const validNeighborIds = new Set<string>()
+
+    for (const neighborId of connections) {
+      const neighbor = this.nouns.get(neighborId)
+      if (!neighbor) {
+        // Skip neighbors that don't exist (expected during rapid additions/deletions)
+        continue
+      }
+
+      // Only add valid neighbors to the distances map
+      distances.set(
+        neighborId,
+        this.distanceFunction(noun.vector, neighbor.vector)
+      )
+      validNeighborIds.add(neighborId)
+    }
+
+    // Only proceed if we have valid neighbors
+    if (distances.size === 0) {
+      // If no valid neighbors, clear connections at this level
+      noun.connections.set(level, new Set())
+      return
+    }
+
+    // Select M closest neighbors from valid ones
+    const selectedNeighbors = this.selectNeighbors(
+      noun.vector,
+      distances,
+      this.config.M
+    )
+
+    // Update connections with only valid neighbors
+    noun.connections.set(level, new Set(selectedNeighbors.keys()))
+  }
+
+  /**
+   * Generate a random level for a new noun
+   * Uses the same distribution as in the original HNSW paper
+   */
+  private getRandomLevel(): number {
+    const r = Math.random()
+    return Math.floor(-Math.log(r) * (1.0 / Math.log(this.config.M)))
+  }
+}

src/hnsw/hnswIndexOptimized.ts

@@ -0,0 +1,586 @@
+/**
+ * Optimized HNSW (Hierarchical Navigable Small World) Index implementation
+ * Extends the base HNSW implementation with support for large datasets
+ * Uses product quantization for dimensionality reduction and disk-based storage when needed
+ */
+
+import {
+  DistanceFunction,
+  HNSWConfig,
+  HNSWNoun,
+  Vector,
+  VectorDocument
+} from '../coreTypes.js'
+import { HNSWIndex } from './hnswIndex.js'
+import { StorageAdapter } from '../coreTypes.js'
+
+// Configuration for the optimized HNSW index
+export interface HNSWOptimizedConfig extends HNSWConfig {
+  // Memory threshold in bytes - when exceeded, will use disk-based approach
+  memoryThreshold?: number
+
+  // Product quantization settings
+  productQuantization?: {
+    // Whether to use product quantization
+    enabled: boolean
+    // Number of subvectors to split the vector into
+    numSubvectors?: number
+    // Number of centroids per subvector
+    numCentroids?: number
+  }
+
+  // Whether to use disk-based storage for the index
+  useDiskBasedIndex?: boolean
+}
+
+// Default configuration for the optimized HNSW index
+const DEFAULT_OPTIMIZED_CONFIG: HNSWOptimizedConfig = {
+  M: 16,
+  efConstruction: 200,
+  efSearch: 50,
+  ml: 16,
+  memoryThreshold: 1024 * 1024 * 1024, // 1GB default threshold
+  productQuantization: {
+    enabled: false,
+    numSubvectors: 16,
+    numCentroids: 256
+  },
+  useDiskBasedIndex: false
+}
+
+/**
+ * Product Quantization implementation
+ * Reduces vector dimensionality by splitting vectors into subvectors
+ * and quantizing each subvector to the nearest centroid
+ */
+class ProductQuantizer {
+  private numSubvectors: number
+  private numCentroids: number
+  private centroids: Vector[][] = []
+  private subvectorSize: number = 0
+  private initialized: boolean = false
+  private dimension: number = 0
+
+  constructor(numSubvectors: number = 16, numCentroids: number = 256) {
+    this.numSubvectors = numSubvectors
+    this.numCentroids = numCentroids
+  }
+
+  /**
+   * Initialize the product quantizer with training data
+   * @param vectors Training vectors to use for learning centroids
+   */
+  public train(vectors: Vector[]): void {
+    if (vectors.length === 0) {
+      throw new Error('Cannot train product quantizer with empty vector set')
+    }
+
+    this.dimension = vectors[0].length
+    this.subvectorSize = Math.ceil(this.dimension / this.numSubvectors)
+
+    // Initialize centroids for each subvector
+    for (let i = 0; i < this.numSubvectors; i++) {
+      // Extract subvectors from training data
+      const subvectors: Vector[] = vectors.map((vector) => {
+        const start = i * this.subvectorSize
+        const end = Math.min(start + this.subvectorSize, this.dimension)
+        return vector.slice(start, end)
+      })
+
+      // Initialize centroids for this subvector using k-means++
+      this.centroids[i] = this.kMeansPlusPlus(subvectors, this.numCentroids)
+    }
+
+    this.initialized = true
+  }
+
+  /**
+   * Quantize a vector using product quantization
+   * @param vector Vector to quantize
+   * @returns Array of centroid indices, one for each subvector
+   */
+  public quantize(vector: Vector): number[] {
+    if (!this.initialized) {
+      throw new Error('Product quantizer not initialized. Call train() first.')
+    }
+
+    if (vector.length !== this.dimension) {
+      throw new Error(
+        `Vector dimension mismatch: expected ${this.dimension}, got ${vector.length}`
+      )
+    }
+
+    const codes: number[] = []
+
+    // Quantize each subvector
+    for (let i = 0; i < this.numSubvectors; i++) {
+      const start = i * this.subvectorSize
+      const end = Math.min(start + this.subvectorSize, this.dimension)
+      const subvector = vector.slice(start, end)
+
+      // Find nearest centroid
+      let minDist = Number.MAX_VALUE
+      let nearestCentroidIndex = 0
+
+      for (let j = 0; j < this.centroids[i].length; j++) {
+        const centroid = this.centroids[i][j]
+        const dist = this.euclideanDistanceSquared(subvector, centroid)
+
+        if (dist < minDist) {
+          minDist = dist
+          nearestCentroidIndex = j
+        }
+      }
+
+      codes.push(nearestCentroidIndex)
+    }
+
+    return codes
+  }
+
+  /**
+   * Reconstruct a vector from its quantized representation
+   * @param codes Array of centroid indices
+   * @returns Reconstructed vector
+   */
+  public reconstruct(codes: number[]): Vector {
+    if (!this.initialized) {
+      throw new Error('Product quantizer not initialized. Call train() first.')
+    }
+
+    if (codes.length !== this.numSubvectors) {
+      throw new Error(
+        `Code length mismatch: expected ${this.numSubvectors}, got ${codes.length}`
+      )
+    }
+
+    const reconstructed: Vector = []
+
+    // Reconstruct each subvector
+    for (let i = 0; i < this.numSubvectors; i++) {
+      const centroidIndex = codes[i]
+      const centroid = this.centroids[i][centroidIndex]
+
+      // Add centroid components to reconstructed vector
+      for (const component of centroid) {
+        reconstructed.push(component)
+      }
+    }
+
+    // Trim to original dimension if needed
+    return reconstructed.slice(0, this.dimension)
+  }
+
+  /**
+   * Compute squared Euclidean distance between two vectors
+   * @param a First vector
+   * @param b Second vector
+   * @returns Squared Euclidean distance
+   */
+  private euclideanDistanceSquared(a: Vector, b: Vector): number {
+    let sum = 0
+    const length = Math.min(a.length, b.length)
+
+    for (let i = 0; i < length; i++) {
+      const diff = a[i] - b[i]
+      sum += diff * diff
+    }
+
+    return sum
+  }
+
+  /**
+   * Implement k-means++ algorithm to initialize centroids
+   * @param vectors Vectors to cluster
+   * @param k Number of clusters
+   * @returns Array of centroids
+   */
+  private kMeansPlusPlus(vectors: Vector[], k: number): Vector[] {
+    if (vectors.length < k) {
+      // If we have fewer vectors than centroids, use the vectors as centroids
+      return [...vectors]
+    }
+
+    const centroids: Vector[] = []
+
+    // Choose first centroid randomly
+    const firstIndex = Math.floor(Math.random() * vectors.length)
+    centroids.push([...vectors[firstIndex]])
+
+    // Choose remaining centroids
+    for (let i = 1; i < k; i++) {
+      // Compute distances to nearest centroid for each vector
+      const distances: number[] = vectors.map((vector) => {
+        let minDist = Number.MAX_VALUE
+
+        for (const centroid of centroids) {
+          const dist = this.euclideanDistanceSquared(vector, centroid)
+          minDist = Math.min(minDist, dist)
+        }
+
+        return minDist
+      })
+
+      // Compute sum of distances
+      const distSum = distances.reduce((sum, dist) => sum + dist, 0)
+
+      // Choose next centroid with probability proportional to distance
+      let r = Math.random() * distSum
+      let nextIndex = 0
+
+      for (let j = 0; j < distances.length; j++) {
+        r -= distances[j]
+        if (r <= 0) {
+          nextIndex = j
+          break
+        }
+      }
+
+      centroids.push([...vectors[nextIndex]])
+    }
+
+    return centroids
+  }
+
+  /**
+   * Get the centroids for each subvector
+   * @returns Array of centroid arrays
+   */
+  public getCentroids(): Vector[][] {
+    return this.centroids
+  }
+
+  /**
+   * Set the centroids for each subvector
+   * @param centroids Array of centroid arrays
+   */
+  public setCentroids(centroids: Vector[][]): void {
+    this.centroids = centroids
+    this.numSubvectors = centroids.length
+    this.numCentroids = centroids[0].length
+    this.initialized = true
+  }
+
+  /**
+   * Get the dimension of the vectors
+   * @returns Dimension
+   */
+  public getDimension(): number {
+    return this.dimension
+  }
+
+  /**
+   * Set the dimension of the vectors
+   * @param dimension Dimension
+   */
+  public setDimension(dimension: number): void {
+    this.dimension = dimension
+    this.subvectorSize = Math.ceil(dimension / this.numSubvectors)
+  }
+}
+
+/**
+ * Optimized HNSW Index implementation
+ * Extends the base HNSW implementation with support for large datasets
+ * Uses product quantization for dimensionality reduction and disk-based storage when needed
+ */
+export class HNSWIndexOptimized extends HNSWIndex {
+  private optimizedConfig: HNSWOptimizedConfig
+  private productQuantizer: ProductQuantizer | null = null
+  private storage: StorageAdapter | null = null
+  private useDiskBasedIndex: boolean = false
+  private useProductQuantization: boolean = false
+  private quantizedVectors: Map<string, number[]> = new Map()
+  private memoryUsage: number = 0
+  private vectorCount: number = 0
+
+  constructor(
+    config: Partial<HNSWOptimizedConfig> = {},
+    distanceFunction: DistanceFunction,
+    storage: StorageAdapter | null = null
+  ) {
+    // Initialize base HNSW index with standard config
+    super(config, distanceFunction)
+
+    // Set optimized config
+    this.optimizedConfig = { ...DEFAULT_OPTIMIZED_CONFIG, ...config }
+
+    // Set storage adapter
+    this.storage = storage
+
+    // Initialize product quantizer if enabled
+    if (this.optimizedConfig.productQuantization?.enabled) {
+      this.useProductQuantization = true
+      this.productQuantizer = new ProductQuantizer(
+        this.optimizedConfig.productQuantization.numSubvectors,
+        this.optimizedConfig.productQuantization.numCentroids
+      )
+    }
+
+    // Set disk-based index flag
+    this.useDiskBasedIndex = this.optimizedConfig.useDiskBasedIndex || false
+  }
+
+  /**
+   * Add a vector to the index
+   * Uses product quantization if enabled and memory threshold is exceeded
+   */
+  public override async addItem(item: VectorDocument): Promise<string> {
+    // Check if item is defined
+    if (!item) {
+      throw new Error('Item is undefined or null')
+    }
+
+    const { id, vector } = item
+
+    // Check if vector is defined
+    if (!vector) {
+      throw new Error('Vector is undefined or null')
+    }
+
+    // Estimate memory usage for this vector
+    const vectorMemory = vector.length * 8 // 8 bytes per number (Float64)
+    const connectionsMemory = this.optimizedConfig.M * this.optimizedConfig.ml * 16 // Estimate for connections
+    const totalMemory = vectorMemory + connectionsMemory
+
+    // Update memory usage estimate
+    this.memoryUsage += totalMemory
+    this.vectorCount++
+
+    // Check if we should switch to product quantization
+    if (
+      this.useProductQuantization &&
+      this.memoryUsage > this.optimizedConfig.memoryThreshold! &&
+      this.productQuantizer &&
+      !this.productQuantizer.getDimension()
+    ) {
+      // Initialize product quantizer with existing vectors
+      this.initializeProductQuantizer()
+    }
+
+    // If product quantization is active, quantize the vector
+    if (
+      this.useProductQuantization &&
+      this.productQuantizer &&
+      this.productQuantizer.getDimension() > 0
+    ) {
+      // Quantize the vector
+      const codes = this.productQuantizer.quantize(vector)
+
+      // Store the quantized vector
+      this.quantizedVectors.set(id, codes)
+
+      // Reconstruct the vector for indexing
+      const reconstructedVector = this.productQuantizer.reconstruct(codes)
+
+      // Add the reconstructed vector to the index
+      return await super.addItem({ id, vector: reconstructedVector })
+    }
+
+    // If disk-based index is active and storage is available, store the vector
+    if (this.useDiskBasedIndex && this.storage) {
+      // Create a noun object
+      const noun: HNSWNoun = {
+        id,
+        vector,
+        connections: new Map()
+      }
+
+      // Store the noun
+      this.storage.saveNoun(noun).catch((error) => {
+        console.error(`Failed to save noun ${id} to storage:`, error)
+      })
+    }
+
+    // Add the vector to the in-memory index
+    return await super.addItem(item)
+  }
+
+  /**
+   * Search for nearest neighbors
+   * Uses product quantization if enabled
+   */
+  public override async search(
+    queryVector: Vector,
+    k: number = 10
+  ): Promise<Array<[string, number]>> {
+    // Check if query vector is defined
+    if (!queryVector) {
+      throw new Error('Query vector is undefined or null')
+    }
+
+    // If product quantization is active, quantize the query vector
+    if (
+      this.useProductQuantization &&
+      this.productQuantizer &&
+      this.productQuantizer.getDimension() > 0
+    ) {
+      // Quantize the query vector
+      const codes = this.productQuantizer.quantize(queryVector)
+
+      // Reconstruct the query vector
+      const reconstructedVector = this.productQuantizer.reconstruct(codes)
+
+      // Search with the reconstructed vector
+      return await super.search(reconstructedVector, k)
+    }
+
+    // Otherwise, use the standard search
+    return await super.search(queryVector, k)
+  }
+
+  /**
+   * Remove an item from the index
+   */
+  public override removeItem(id: string): boolean {
+    // If product quantization is active, remove the quantized vector
+    if (this.useProductQuantization) {
+      this.quantizedVectors.delete(id)
+    }
+
+    // If disk-based index is active and storage is available, remove the vector from storage
+    if (this.useDiskBasedIndex && this.storage) {
+      this.storage.deleteNoun(id).catch((error) => {
+        console.error(`Failed to delete noun ${id} from storage:`, error)
+      })
+    }
+
+    // Update memory usage estimate
+    if (this.vectorCount > 0) {
+      this.memoryUsage = Math.max(
+        0,
+        this.memoryUsage - this.memoryUsage / this.vectorCount
+      )
+      this.vectorCount--
+    }
+
+    // Remove the item from the in-memory index
+    return super.removeItem(id)
+  }
+
+  /**
+   * Clear the index
+   */
+  public override clear(): void {
+    // Clear product quantization data
+    if (this.useProductQuantization) {
+      this.quantizedVectors.clear()
+      this.productQuantizer = new ProductQuantizer(
+        this.optimizedConfig.productQuantization!.numSubvectors,
+        this.optimizedConfig.productQuantization!.numCentroids
+      )
+    }
+
+    // Reset memory usage
+    this.memoryUsage = 0
+    this.vectorCount = 0
+
+    // Clear the in-memory index
+    super.clear()
+  }
+
+  /**
+   * Initialize product quantizer with existing vectors
+   */
+  private initializeProductQuantizer(): void {
+    if (!this.productQuantizer) {
+      return
+    }
+
+    // Get all vectors from the index
+    const nouns = super.getNouns()
+    const vectors: Vector[] = []
+
+    // Extract vectors
+    for (const [_, noun] of nouns) {
+      vectors.push(noun.vector)
+    }
+
+    // Train the product quantizer
+    if (vectors.length > 0) {
+      this.productQuantizer.train(vectors)
+
+      // Quantize all existing vectors
+      for (const [id, noun] of nouns) {
+        const codes = this.productQuantizer.quantize(noun.vector)
+        this.quantizedVectors.set(id, codes)
+      }
+
+      console.log(
+        `Initialized product quantizer with ${vectors.length} vectors`
+      )
+    }
+  }
+
+  /**
+   * Get the product quantizer
+   * @returns Product quantizer or null if not enabled
+   */
+  public getProductQuantizer(): ProductQuantizer | null {
+    return this.productQuantizer
+  }
+
+  /**
+   * Get the optimized configuration
+   * @returns Optimized configuration
+   */
+  public getOptimizedConfig(): HNSWOptimizedConfig {
+    return { ...this.optimizedConfig }
+  }
+
+  /**
+   * Get the estimated memory usage
+   * @returns Estimated memory usage in bytes
+   */
+  public getMemoryUsage(): number {
+    return this.memoryUsage
+  }
+
+  /**
+   * Set the storage adapter
+   * @param storage Storage adapter
+   */
+  public setStorage(storage: StorageAdapter): void {
+    this.storage = storage
+  }
+
+  /**
+   * Get the storage adapter
+   * @returns Storage adapter or null if not set
+   */
+  public getStorage(): StorageAdapter | null {
+    return this.storage
+  }
+
+  /**
+   * Set whether to use disk-based index
+   * @param useDiskBasedIndex Whether to use disk-based index
+   */
+  public setUseDiskBasedIndex(useDiskBasedIndex: boolean): void {
+    this.useDiskBasedIndex = useDiskBasedIndex
+  }
+
+  /**
+   * Get whether disk-based index is used
+   * @returns Whether disk-based index is used
+   */
+  public getUseDiskBasedIndex(): boolean {
+    return this.useDiskBasedIndex
+  }
+
+  /**
+   * Set whether to use product quantization
+   * @param useProductQuantization Whether to use product quantization
+   */
+  public setUseProductQuantization(useProductQuantization: boolean): void {
+    this.useProductQuantization = useProductQuantization
+  }
+
+  /**
+   * Get whether product quantization is used
+   * @returns Whether product quantization is used
+   */
+  public getUseProductQuantization(): boolean {
+    return this.useProductQuantization
+  }
+}

src/index.ts

@@ -0,0 +1,351 @@
+/**
+ * OPFS BrainyData
+ * A vector database using HNSW indexing with Origin Private File System storage
+ */
+
+// Export main BrainyData class and related types
+import { BrainyData, BrainyDataConfig } from './brainyData.js'
+
+export { BrainyData }
+export type { BrainyDataConfig }
+
+// Export distance functions for convenience
+import {
+  euclideanDistance,
+  cosineDistance,
+  manhattanDistance,
+  dotProductDistance
+} from './utils/index.js'
+
+export {
+  euclideanDistance,
+  cosineDistance,
+  manhattanDistance,
+  dotProductDistance
+}
+
+// Export embedding functionality
+import {
+  UniversalSentenceEncoder,
+  createEmbeddingFunction,
+  createTensorFlowEmbeddingFunction,
+  createThreadedEmbeddingFunction,
+  defaultEmbeddingFunction
+} from './utils/embedding.js'
+
+export {
+  UniversalSentenceEncoder,
+  createEmbeddingFunction,
+  createTensorFlowEmbeddingFunction,
+  createThreadedEmbeddingFunction,
+  defaultEmbeddingFunction
+}
+
+// Export storage adapters
+import {
+  OPFSStorage,
+  MemoryStorage,
+  createStorage
+} from './storage/opfsStorage.js'
+import { FileSystemStorage } from './storage/fileSystemStorage.js'
+import { R2Storage, S3CompatibleStorage } from './storage/s3CompatibleStorage.js'
+
+export {
+  OPFSStorage,
+  MemoryStorage,
+  FileSystemStorage,
+  R2Storage,
+  S3CompatibleStorage,
+  createStorage
+}
+
+// Export unified pipeline
+import {
+  Pipeline,
+  pipeline,
+  augmentationPipeline,
+  ExecutionMode,
+  PipelineOptions,
+  PipelineResult,
+  executeStreamlined,
+  executeByType,
+  executeSingle,
+  processStaticData,
+  processStreamingData,
+  createPipeline,
+  createStreamingPipeline,
+  StreamlinedExecutionMode,
+  StreamlinedPipelineOptions,
+  StreamlinedPipelineResult
+} from './pipeline.js'
+
+// Export sequential pipeline (for backward compatibility)
+import {
+  SequentialPipeline,
+  sequentialPipeline,
+  SequentialPipelineOptions
+} from './sequentialPipeline.js'
+
+// Export augmentation factory
+import {
+  createSenseAugmentation,
+  addWebSocketSupport,
+  executeAugmentation,
+  loadAugmentationModule,
+  AugmentationOptions
+} from './augmentationFactory.js'
+
+export {
+  // Unified pipeline exports
+  Pipeline,
+  pipeline,
+  augmentationPipeline,
+  ExecutionMode,
+  SequentialPipeline,
+  sequentialPipeline,
+
+  // Streamlined pipeline exports (now part of unified pipeline)
+  executeStreamlined,
+  executeByType,
+  executeSingle,
+  processStaticData,
+  processStreamingData,
+  createPipeline,
+  createStreamingPipeline,
+  StreamlinedExecutionMode,
+
+  // Augmentation factory exports
+  createSenseAugmentation,
+  addWebSocketSupport,
+  executeAugmentation,
+  loadAugmentationModule
+}
+export type { 
+  PipelineOptions, 
+  PipelineResult, 
+  SequentialPipelineOptions,
+  StreamlinedPipelineOptions,
+  StreamlinedPipelineResult,
+  AugmentationOptions
+}
+
+// Export augmentation registry for build-time loading
+import {
+  availableAugmentations,
+  registerAugmentation,
+  initializeAugmentationPipeline,
+  setAugmentationEnabled,
+  getAugmentationsByType
+} from './augmentationRegistry.js'
+
+export {
+  availableAugmentations,
+  registerAugmentation,
+  initializeAugmentationPipeline,
+  setAugmentationEnabled,
+  getAugmentationsByType
+}
+
+// Export augmentation registry loader for build tools
+import {
+  loadAugmentationsFromModules,
+  createAugmentationRegistryPlugin,
+  createAugmentationRegistryRollupPlugin
+} from './augmentationRegistryLoader.js'
+import type {
+  AugmentationRegistryLoaderOptions,
+  AugmentationLoadResult
+} from './augmentationRegistryLoader.js'
+
+export {
+  loadAugmentationsFromModules,
+  createAugmentationRegistryPlugin,
+  createAugmentationRegistryRollupPlugin
+}
+export type {
+  AugmentationRegistryLoaderOptions,
+  AugmentationLoadResult
+}
+
+// Export augmentation implementations
+import {
+  MemoryStorageAugmentation,
+  FileSystemStorageAugmentation,
+  OPFSStorageAugmentation,
+  createMemoryAugmentation
+} from './augmentations/memoryAugmentations.js'
+import {
+  WebSocketConduitAugmentation,
+  WebRTCConduitAugmentation,
+  createConduitAugmentation
+} from './augmentations/conduitAugmentations.js'
+import {
+  ServerSearchConduitAugmentation,
+  ServerSearchActivationAugmentation,
+  createServerSearchAugmentations
+} from './augmentations/serverSearchAugmentations.js'
+
+// Non-LLM exports
+export {
+  MemoryStorageAugmentation,
+  FileSystemStorageAugmentation,
+  OPFSStorageAugmentation,
+  createMemoryAugmentation,
+  WebSocketConduitAugmentation,
+  WebRTCConduitAugmentation,
+  createConduitAugmentation,
+  ServerSearchConduitAugmentation,
+  ServerSearchActivationAugmentation,
+  createServerSearchAugmentations
+}
+
+// LLM augmentations are optional and not imported by default
+// They can be imported directly from their module if needed:
+// import { LLMCognitionAugmentation, LLMActivationAugmentation, createLLMAugmentations } from './augmentations/llmAugmentations.js'
+
+
+// Export types
+import type {
+  Vector,
+  VectorDocument,
+  SearchResult,
+  DistanceFunction,
+  EmbeddingFunction,
+  EmbeddingModel,
+  HNSWNoun,
+  GraphVerb,
+  HNSWConfig,
+  StorageAdapter
+} from './coreTypes.js'
+
+// Export HNSW index and optimized version
+import { HNSWIndex } from './hnsw/hnswIndex.js'
+import { HNSWIndexOptimized, HNSWOptimizedConfig } from './hnsw/hnswIndexOptimized.js'
+
+export {
+  HNSWIndex,
+  HNSWIndexOptimized
+}
+
+export type {
+  Vector,
+  VectorDocument,
+  SearchResult,
+  DistanceFunction,
+  EmbeddingFunction,
+  EmbeddingModel,
+  HNSWNoun,
+  GraphVerb,
+  HNSWConfig,
+  HNSWOptimizedConfig,
+  StorageAdapter
+}
+
+// Export augmentation types
+import type {
+  IAugmentation,
+  AugmentationResponse,
+  IWebSocketSupport,
+  ISenseAugmentation,
+  IConduitAugmentation,
+  ICognitionAugmentation,
+  IMemoryAugmentation,
+  IPerceptionAugmentation,
+  IDialogAugmentation,
+  IActivationAugmentation
+} from './types/augmentations.js'
+import { AugmentationType, BrainyAugmentations } from './types/augmentations.js'
+
+export type {
+  IAugmentation,
+  AugmentationResponse,
+  IWebSocketSupport
+}
+export {
+  AugmentationType,
+  BrainyAugmentations,
+  ISenseAugmentation,
+  IConduitAugmentation,
+  ICognitionAugmentation,
+  IMemoryAugmentation,
+  IPerceptionAugmentation,
+  IDialogAugmentation,
+  IActivationAugmentation
+}
+
+// Export combined WebSocket augmentation interfaces
+export type {
+  IWebSocketCognitionAugmentation,
+  IWebSocketSenseAugmentation,
+  IWebSocketPerceptionAugmentation,
+  IWebSocketActivationAugmentation,
+  IWebSocketDialogAugmentation,
+  IWebSocketConduitAugmentation,
+  IWebSocketMemoryAugmentation
+} from './types/augmentations.js'
+
+// Export graph types
+import type {
+  GraphNoun,
+  EmbeddedGraphVerb,
+  Person,
+  Place,
+  Thing,
+  Event,
+  Concept,
+  Content
+} from './types/graphTypes.js'
+import { NounType, VerbType } from './types/graphTypes.js'
+
+export type {
+  GraphNoun,
+  EmbeddedGraphVerb,
+  Person,
+  Place,
+  Thing,
+  Event,
+  Concept,
+  Content
+}
+export { NounType, VerbType }
+
+// Export MCP (Model Control Protocol) components
+import {
+  BrainyMCPAdapter,
+  MCPAugmentationToolset,
+  BrainyMCPService
+} from './mcp/index.js' // Import from mcp/index.js
+import {
+  MCPRequest,
+  MCPResponse,
+  MCPDataAccessRequest,
+  MCPToolExecutionRequest,
+  MCPSystemInfoRequest,
+  MCPAuthenticationRequest,
+  MCPRequestType,
+  MCPServiceOptions,
+  MCPTool,
+  MCP_VERSION
+} from './types/mcpTypes.js'
+
+export {
+  // MCP classes
+  BrainyMCPAdapter,
+  MCPAugmentationToolset,
+  BrainyMCPService,
+
+  // MCP types
+  MCPRequestType,
+  MCP_VERSION
+}
+
+export type {
+  MCPRequest,
+  MCPResponse,
+  MCPDataAccessRequest,
+  MCPToolExecutionRequest,
+  MCPSystemInfoRequest,
+  MCPAuthenticationRequest,
+  MCPServiceOptions,
+  MCPTool
+}

src/mcp/README.md

@@ -0,0 +1,180 @@
+# Model Control Protocol (MCP) for Brainy
+
+This document provides information about the Model Control Protocol (MCP) implementation in Brainy, which allows external models to access Brainy data and use the augmentation pipeline as tools.
+
+## Components
+
+The MCP implementation consists of three main components:
+
+1. **BrainyMCPAdapter**: Provides access to Brainy data through MCP
+2. **MCPAugmentationToolset**: Exposes the augmentation pipeline as tools
+3. **BrainyMCPService**: Integrates the adapter and toolset, providing WebSocket and REST server implementations for external model access
+
+## Environment Compatibility
+
+### BrainyMCPAdapter
+
+The `BrainyMCPAdapter` has no environment-specific dependencies and can run in any environment where Brainy itself runs, including:
+
+- Browser environments
+- Node.js environments
+- Server environments
+
+### MCPAugmentationToolset
+
+The `MCPAugmentationToolset` also has no environment-specific dependencies and can run in any environment where Brainy itself runs, including:
+
+- Browser environments
+- Node.js environments
+- Server environments
+
+### BrainyMCPService
+
+The `BrainyMCPService` has been refactored to separate the core functionality from the Node.js-specific server functionality:
+
+1. **Core Functionality**: The core request handling functionality (`handleMCPRequest`) can run in any environment where Brainy itself runs. This is what remains in the main Brainy package.
+
+2. **Server Functionality**: The WebSocket and REST server functionality has been moved to the cloud-wrapper project to avoid including Node.js-specific dependencies in the browser bundle:
+   - `ws` for WebSocket server
+   - `express` for REST API
+   - `cors` for Cross-Origin Resource Sharing
+
+This separation ensures that the browser bundle remains lightweight and doesn't include unnecessary Node.js-specific dependencies. In browser or other environments, you can still use the core functionality through the `handleMCPRequest` method.
+
+## Usage
+
+### In Any Environment (Browser, Node.js, Server)
+
+```typescript
+import { BrainyData, BrainyMCPAdapter, MCPAugmentationToolset } from '@soulcraft/brainy'
+
+// Create a BrainyData instance
+const brainyData = new BrainyData()
+await brainyData.init()
+
+// Create an MCP adapter
+const adapter = new BrainyMCPAdapter(brainyData)
+
+// Create a toolset
+const toolset = new MCPAugmentationToolset()
+
+// Use the adapter to access Brainy data
+const response = await adapter.handleRequest({
+  type: 'data_access',
+  operation: 'search',
+  requestId: adapter.generateRequestId(),
+  version: '1.0.0',
+  parameters: {
+    query: 'example query',
+    k: 5
+  }
+})
+
+// Use the toolset to execute augmentation pipeline tools
+const toolResponse = await toolset.handleRequest({
+  type: 'tool_execution',
+  toolName: 'brainy_memory_storeData',
+  requestId: toolset.generateRequestId(),
+  version: '1.0.0',
+  parameters: {
+    args: ['key1', { some: 'data' }]
+  }
+})
+```
+
+### In Node.js Environment with Server Functionality
+
+To use the MCP service with WebSocket and REST server functionality, you should use the cloud-wrapper project:
+
+```typescript
+import { BrainyData } from '@soulcraft/brainy'
+import { initializeBrainy } from './services/brainyService.js'
+import { initializeMCPService } from './services/mcpService.js'
+
+// Initialize Brainy
+const brainyData = await initializeBrainy()
+
+// Initialize MCP service with WebSocket and REST server functionality
+const mcpService = initializeMCPService(brainyData, {
+  wsPort: 8080,
+  restPort: 3000,
+  enableAuth: true,
+  apiKeys: ['your-api-key'],
+  rateLimit: {
+    maxRequests: 100,
+    windowMs: 60000 // 1 minute
+  },
+  cors: {
+    origin: '*',
+    credentials: true
+  }
+})
+```
+
+Alternatively, you can configure the MCP service using environment variables in the cloud-wrapper:
+
+```
+# MCP configuration
+MCP_WS_PORT=8080
+MCP_REST_PORT=3000
+MCP_ENABLE_AUTH=true
+MCP_API_KEYS=your-api-key,another-key
+MCP_RATE_LIMIT_REQUESTS=100
+MCP_RATE_LIMIT_WINDOW_MS=60000
+MCP_ENABLE_CORS=true
+```
+
+### In Browser Environment (Core Functionality Only)
+
+```typescript
+import { BrainyData, BrainyMCPService } from '@soulcraft/brainy'
+
+// Create a BrainyData instance
+const brainyData = new BrainyData()
+await brainyData.init()
+
+// Create an MCP service (server functionality will be disabled in browser)
+const mcpService = new BrainyMCPService(brainyData)
+
+// Use the core functionality
+const response = await mcpService.handleMCPRequest({
+  type: 'data_access',
+  operation: 'search',
+  requestId: mcpService.generateRequestId(),
+  version: '1.0.0',
+  parameters: {
+    query: 'example query',
+    k: 5
+  }
+})
+```
+
+## Cloud Wrapper Integration
+
+The MCP service's server functionality has been integrated directly into the cloud-wrapper project. The cloud-wrapper automatically initializes the MCP service if the appropriate environment variables are set:
+
+```
+# MCP configuration
+MCP_WS_PORT=8080
+MCP_REST_PORT=3000
+MCP_ENABLE_AUTH=true
+MCP_API_KEYS=your-api-key,another-key
+MCP_RATE_LIMIT_REQUESTS=100
+MCP_RATE_LIMIT_WINDOW_MS=60000
+MCP_ENABLE_CORS=true
+```
+
+You can deploy the cloud wrapper to various cloud platforms using the npm scripts from the root directory:
+
+```bash
+# Deploy to AWS Lambda and API Gateway
+npm run deploy:cloud:aws
+
+# Deploy to Google Cloud Run
+npm run deploy:cloud:gcp
+
+# Deploy to Cloudflare Workers
+npm run deploy:cloud:cloudflare
+```
+
+The cloud wrapper is specifically designed for server environments and includes additional features like logging, security headers, and deployment scripts for various cloud providers. See the [Cloud Wrapper README](../../cloud-wrapper/README.md) for detailed configuration instructions and API documentation.

src/mcp/brainyMCPAdapter.ts

@@ -0,0 +1,203 @@
+/**
+ * BrainyMCPAdapter
+ * 
+ * This class provides an adapter for accessing Brainy data through the Model Control Protocol (MCP).
+ * It wraps a BrainyData instance and exposes methods for getting vectors, searching similar items,
+ * and getting relationships.
+ */
+
+import { v4 as uuidv4 } from 'uuid'
+import { BrainyDataInterface } from '../types/brainyDataInterface.js'
+import { 
+  MCPRequest, 
+  MCPResponse, 
+  MCPDataAccessRequest,
+  MCPRequestType,
+  MCP_VERSION
+} from '../types/mcpTypes.js'
+
+export class BrainyMCPAdapter {
+  private brainyData: BrainyDataInterface
+
+  /**
+   * Creates a new BrainyMCPAdapter
+   * @param brainyData The BrainyData instance to wrap
+   */
+  constructor(brainyData: BrainyDataInterface) {
+    this.brainyData = brainyData
+  }
+
+  /**
+   * Handles an MCP data access request
+   * @param request The MCP request
+   * @returns An MCP response
+   */
+  async handleRequest(request: MCPDataAccessRequest): Promise<MCPResponse> {
+    try {
+      switch (request.operation) {
+        case 'get':
+          return await this.handleGetRequest(request)
+        case 'search':
+          return await this.handleSearchRequest(request)
+        case 'add':
+          return await this.handleAddRequest(request)
+        case 'getRelationships':
+          return await this.handleGetRelationshipsRequest(request)
+        default:
+          return this.createErrorResponse(
+            request.requestId,
+            'UNSUPPORTED_OPERATION',
+            `Operation ${request.operation} is not supported`
+          )
+      }
+    } catch (error) {
+      return this.createErrorResponse(
+        request.requestId,
+        'INTERNAL_ERROR',
+        error instanceof Error ? error.message : String(error)
+      )
+    }
+  }
+
+  /**
+   * Handles a get request
+   * @param request The MCP request
+   * @returns An MCP response
+   */
+  private async handleGetRequest(request: MCPDataAccessRequest): Promise<MCPResponse> {
+    const { id } = request.parameters
+
+    if (!id) {
+      return this.createErrorResponse(
+        request.requestId,
+        'MISSING_PARAMETER',
+        'Parameter "id" is required'
+      )
+    }
+
+    const noun = await this.brainyData.get(id)
+    
+    if (!noun) {
+      return this.createErrorResponse(
+        request.requestId,
+        'NOT_FOUND',
+        `No noun found with id ${id}`
+      )
+    }
+
+    return this.createSuccessResponse(request.requestId, noun)
+  }
+
+  /**
+   * Handles a search request
+   * @param request The MCP request
+   * @returns An MCP response
+   */
+  private async handleSearchRequest(request: MCPDataAccessRequest): Promise<MCPResponse> {
+    const { query, k = 10 } = request.parameters
+
+    if (!query) {
+      return this.createErrorResponse(
+        request.requestId,
+        'MISSING_PARAMETER',
+        'Parameter "query" is required'
+      )
+    }
+
+    const results = await this.brainyData.searchText(query, k)
+    return this.createSuccessResponse(request.requestId, results)
+  }
+
+  /**
+   * Handles an add request
+   * @param request The MCP request
+   * @returns An MCP response
+   */
+  private async handleAddRequest(request: MCPDataAccessRequest): Promise<MCPResponse> {
+    const { text, metadata } = request.parameters
+
+    if (!text) {
+      return this.createErrorResponse(
+        request.requestId,
+        'MISSING_PARAMETER',
+        'Parameter "text" is required'
+      )
+    }
+
+    const id = await this.brainyData.add(text, metadata)
+    return this.createSuccessResponse(request.requestId, { id })
+  }
+
+  /**
+   * Handles a getRelationships request
+   * @param request The MCP request
+   * @returns An MCP response
+   */
+  private async handleGetRelationshipsRequest(request: MCPDataAccessRequest): Promise<MCPResponse> {
+    const { id } = request.parameters
+
+    if (!id) {
+      return this.createErrorResponse(
+        request.requestId,
+        'MISSING_PARAMETER',
+        'Parameter "id" is required'
+      )
+    }
+
+    // This is a simplified implementation - in a real implementation, we would
+    // need to check if these methods exist on the BrainyDataInterface
+    const outgoing = await (this.brainyData as any).getVerbsBySource?.(id) || []
+    const incoming = await (this.brainyData as any).getVerbsByTarget?.(id) || []
+
+    return this.createSuccessResponse(request.requestId, { outgoing, incoming })
+  }
+
+  /**
+   * Creates a success response
+   * @param requestId The request ID
+   * @param data The response data
+   * @returns An MCP response
+   */
+  private createSuccessResponse(requestId: string, data: any): MCPResponse {
+    return {
+      success: true,
+      requestId,
+      version: MCP_VERSION,
+      data
+    }
+  }
+
+  /**
+   * Creates an error response
+   * @param requestId The request ID
+   * @param code The error code
+   * @param message The error message
+   * @param details Optional error details
+   * @returns An MCP response
+   */
+  private createErrorResponse(
+    requestId: string,
+    code: string,
+    message: string,
+    details?: any
+  ): MCPResponse {
+    return {
+      success: false,
+      requestId,
+      version: MCP_VERSION,
+      error: {
+        code,
+        message,
+        details
+      }
+    }
+  }
+
+  /**
+   * Creates a new request ID
+   * @returns A new UUID
+   */
+  generateRequestId(): string {
+    return uuidv4()
+  }
+}

src/mcp/brainyMCPService.ts

@@ -0,0 +1,347 @@
+/**
+ * BrainyMCPService
+ *
+ * This class provides a unified service for accessing Brainy data and augmentations
+ * through the Model Control Protocol (MCP). It integrates the BrainyMCPAdapter and
+ * MCPAugmentationToolset classes and provides WebSocket and REST server implementations
+ * for external model access.
+ */
+
+import { v4 as uuidv4 } from 'uuid'
+import { BrainyDataInterface } from '../types/brainyDataInterface.js'
+import {
+  MCPRequest,
+  MCPResponse,
+  MCPDataAccessRequest,
+  MCPToolExecutionRequest,
+  MCPSystemInfoRequest,
+  MCPAuthenticationRequest,
+  MCPRequestType,
+  MCPServiceOptions,
+  MCP_VERSION,
+  MCPTool
+} from '../types/mcpTypes.js'
+import { BrainyMCPAdapter } from './brainyMCPAdapter.js'
+import { MCPAugmentationToolset } from './mcpAugmentationToolset.js'
+import { isBrowser, isNode } from '../utils/environment.js'
+
+export class BrainyMCPService {
+  private dataAdapter: BrainyMCPAdapter
+  private toolset: MCPAugmentationToolset
+  private options: MCPServiceOptions
+  private authTokens: Map<string, { userId: string; expires: number }>
+  private rateLimits: Map<string, { count: number; resetTime: number }>
+
+  /**
+   * Creates a new BrainyMCPService
+   * @param brainyData The BrainyData instance to wrap
+   * @param options Configuration options for the service
+   */
+  constructor(
+    brainyData: BrainyDataInterface,
+    options: MCPServiceOptions = {}
+  ) {
+    this.dataAdapter = new BrainyMCPAdapter(brainyData)
+    this.toolset = new MCPAugmentationToolset()
+    this.options = options
+    this.authTokens = new Map()
+    this.rateLimits = new Map()
+  }
+
+  /**
+   * Handles an MCP request
+   * @param request The MCP request
+   * @returns An MCP response
+   */
+  async handleRequest(request: MCPRequest): Promise<MCPResponse> {
+    try {
+      switch (request.type) {
+        case MCPRequestType.DATA_ACCESS:
+          return await this.dataAdapter.handleRequest(
+            request as MCPDataAccessRequest
+          )
+
+        case MCPRequestType.TOOL_EXECUTION:
+          return await this.toolset.handleRequest(
+            request as MCPToolExecutionRequest
+          )
+
+        case MCPRequestType.SYSTEM_INFO:
+          return await this.handleSystemInfoRequest(
+            request as MCPSystemInfoRequest
+          )
+
+        case MCPRequestType.AUTHENTICATION:
+          return await this.handleAuthenticationRequest(
+            request as MCPAuthenticationRequest
+          )
+
+        default:
+          return this.createErrorResponse(
+            request.requestId,
+            'UNSUPPORTED_REQUEST_TYPE',
+            `Request type ${request.type} is not supported`
+          )
+      }
+    } catch (error) {
+      return this.createErrorResponse(
+        request.requestId,
+        'INTERNAL_ERROR',
+        error instanceof Error ? error.message : String(error)
+      )
+    }
+  }
+
+  /**
+   * Handles a system info request
+   * @param request The MCP request
+   * @returns An MCP response
+   */
+  private async handleSystemInfoRequest(
+    request: MCPSystemInfoRequest
+  ): Promise<MCPResponse> {
+    try {
+      switch (request.infoType) {
+        case 'status':
+          return this.createSuccessResponse(request.requestId, {
+            status: 'active',
+            version: MCP_VERSION,
+            environment: isBrowser() ? 'browser' : isNode() ? 'node' : 'unknown'
+          })
+
+        case 'availableTools':
+          const tools: MCPTool[] = await this.toolset.getAvailableTools()
+          return this.createSuccessResponse(request.requestId, tools)
+
+        case 'version':
+          return this.createSuccessResponse(request.requestId, {
+            version: MCP_VERSION
+          })
+
+        default:
+          return this.createErrorResponse(
+            request.requestId,
+            'UNSUPPORTED_INFO_TYPE',
+            `Info type ${request.infoType} is not supported`
+          )
+      }
+    } catch (error) {
+      return this.createErrorResponse(
+        request.requestId,
+        'INTERNAL_ERROR',
+        error instanceof Error ? error.message : String(error)
+      )
+    }
+  }
+
+  /**
+   * Handles an authentication request
+   * @param request The MCP request
+   * @returns An MCP response
+   */
+  private async handleAuthenticationRequest(
+    request: MCPAuthenticationRequest
+  ): Promise<MCPResponse> {
+    try {
+      if (!this.options.enableAuth) {
+        return this.createSuccessResponse(request.requestId, {
+          authenticated: true,
+          message: 'Authentication is not enabled'
+        })
+      }
+
+      const { credentials } = request
+
+      // Check API key authentication
+      if (
+        credentials.apiKey &&
+        this.options.apiKeys?.includes(credentials.apiKey)
+      ) {
+        const token = this.generateAuthToken('api-user')
+        return this.createSuccessResponse(request.requestId, {
+          authenticated: true,
+          token
+        })
+      }
+
+      // Check username/password authentication
+      // This is a placeholder - in a real implementation, you would check against a database
+      if (
+        credentials.username === 'admin' &&
+        credentials.password === 'password'
+      ) {
+        const token = this.generateAuthToken(credentials.username)
+        return this.createSuccessResponse(request.requestId, {
+          authenticated: true,
+          token
+        })
+      }
+
+      return this.createErrorResponse(
+        request.requestId,
+        'INVALID_CREDENTIALS',
+        'Invalid credentials'
+      )
+    } catch (error) {
+      return this.createErrorResponse(
+        request.requestId,
+        'INTERNAL_ERROR',
+        error instanceof Error ? error.message : String(error)
+      )
+    }
+  }
+
+  /**
+   * Checks if a request is valid
+   * @param request The request to check
+   * @returns Whether the request is valid
+   */
+  private isValidRequest(request: any): boolean {
+    return (
+      request &&
+      typeof request === 'object' &&
+      request.type &&
+      request.requestId &&
+      request.version
+    )
+  }
+
+  /**
+   * Checks if a request is authenticated
+   * @param request The request to check
+   * @returns Whether the request is authenticated
+   */
+  private isAuthenticated(request: MCPRequest): boolean {
+    if (!this.options.enableAuth) {
+      return true
+    }
+
+    return request.authToken ? this.isValidToken(request.authToken) : false
+  }
+
+  /**
+   * Checks if a token is valid
+   * @param token The token to check
+   * @returns Whether the token is valid
+   */
+  private isValidToken(token: string): boolean {
+    const tokenInfo = this.authTokens.get(token)
+    if (!tokenInfo) {
+      return false
+    }
+
+    if (tokenInfo.expires < Date.now()) {
+      this.authTokens.delete(token)
+      return false
+    }
+
+    return true
+  }
+
+  /**
+   * Generates an authentication token
+   * @param userId The user ID to associate with the token
+   * @returns The generated token
+   */
+  private generateAuthToken(userId: string): string {
+    const token = uuidv4()
+    const expires = Date.now() + 24 * 60 * 60 * 1000 // 24 hours
+
+    this.authTokens.set(token, { userId, expires })
+
+    return token
+  }
+
+  /**
+   * Checks if a client has exceeded the rate limit
+   * @param clientId The client ID to check
+   * @returns Whether the client is within the rate limit
+   */
+  private checkRateLimit(clientId: string): boolean {
+    if (!this.options.rateLimit) {
+      return true
+    }
+
+    const now = Date.now()
+    const limit = this.rateLimits.get(clientId)
+
+    if (!limit) {
+      this.rateLimits.set(clientId, {
+        count: 1,
+        resetTime: now + this.options.rateLimit.windowMs
+      })
+      return true
+    }
+
+    if (limit.resetTime < now) {
+      limit.count = 1
+      limit.resetTime = now + this.options.rateLimit.windowMs
+      return true
+    }
+
+    if (limit.count >= this.options.rateLimit.maxRequests) {
+      return false
+    }
+
+    limit.count++
+    return true
+  }
+
+  /**
+   * Creates a success response
+   * @param requestId The request ID
+   * @param data The response data
+   * @returns An MCP response
+   */
+  private createSuccessResponse(requestId: string, data: any): MCPResponse {
+    return {
+      success: true,
+      requestId,
+      version: MCP_VERSION,
+      data
+    }
+  }
+
+  /**
+   * Creates an error response
+   * @param requestId The request ID
+   * @param code The error code
+   * @param message The error message
+   * @param details Optional error details
+   * @returns An MCP response
+   */
+  private createErrorResponse(
+    requestId: string,
+    code: string,
+    message: string,
+    details?: any
+  ): MCPResponse {
+    return {
+      success: false,
+      requestId,
+      version: MCP_VERSION,
+      error: {
+        code,
+        message,
+        details
+      }
+    }
+  }
+
+  /**
+   * Creates a new request ID
+   * @returns A new UUID
+   */
+  generateRequestId(): string {
+    return uuidv4()
+  }
+
+  /**
+   * Handles an MCP request directly (for in-process models)
+   * @param request The MCP request
+   * @returns An MCP response
+   */
+  async handleMCPRequest(request: MCPRequest): Promise<MCPResponse> {
+    return await this.handleRequest(request)
+  }
+}

src/mcp/index.ts

@@ -0,0 +1,19 @@
+/**
+ * Model Control Protocol (MCP) for Brainy
+ * 
+ * This module provides a Model Control Protocol (MCP) implementation for Brainy,
+ * allowing external models to access Brainy data and use the augmentation pipeline as tools.
+ */
+
+// Import and re-export the MCP components
+import { BrainyMCPAdapter } from './brainyMCPAdapter.js'
+import { MCPAugmentationToolset } from './mcpAugmentationToolset.js'
+import { BrainyMCPService } from './brainyMCPService.js'
+
+// Export the MCP components
+export { BrainyMCPAdapter }
+export { MCPAugmentationToolset }
+export { BrainyMCPService }
+
+// Export the MCP types
+export * from '../types/mcpTypes.js'

src/mcp/mcpAugmentationToolset.ts

@@ -0,0 +1,225 @@
+/**
+ * MCPAugmentationToolset
+ * 
+ * This class exposes the Brainy augmentation pipeline as tools through the Model Control Protocol (MCP).
+ * It provides methods for getting available tools and executing tools.
+ */
+
+import { v4 as uuidv4 } from 'uuid'
+import { 
+  MCPResponse, 
+  MCPToolExecutionRequest,
+  MCPTool,
+  MCP_VERSION
+} from '../types/mcpTypes.js'
+import { AugmentationType } from '../types/augmentations.js'
+
+// Import the augmentation pipeline
+import { augmentationPipeline } from '../augmentationPipeline.js'
+
+export class MCPAugmentationToolset {
+  /**
+   * Creates a new MCPAugmentationToolset
+   */
+  constructor() {
+    // No initialization needed
+  }
+
+  /**
+   * Handles an MCP tool execution request
+   * @param request The MCP request
+   * @returns An MCP response
+   */
+  async handleRequest(request: MCPToolExecutionRequest): Promise<MCPResponse> {
+    try {
+      const { toolName, parameters } = request
+
+      // Extract the augmentation type and method from the tool name
+      // Tool names are in the format: brainy_{augmentationType}_{method}
+      const parts = toolName.split('_')
+      
+      if (parts.length < 3 || parts[0] !== 'brainy') {
+        return this.createErrorResponse(
+          request.requestId,
+          'INVALID_TOOL',
+          `Invalid tool name: ${toolName}. Tool names should be in the format: brainy_{augmentationType}_{method}`
+        )
+      }
+
+      const augmentationType = parts[1]
+      const method = parts.slice(2).join('_')
+
+      // Validate the augmentation type
+      if (!this.isValidAugmentationType(augmentationType)) {
+        return this.createErrorResponse(
+          request.requestId,
+          'INVALID_AUGMENTATION_TYPE',
+          `Invalid augmentation type: ${augmentationType}`
+        )
+      }
+
+      // Execute the appropriate pipeline based on the augmentation type
+      const result = await this.executePipeline(augmentationType, method, parameters)
+      
+      return this.createSuccessResponse(request.requestId, result)
+    } catch (error) {
+      return this.createErrorResponse(
+        request.requestId,
+        'INTERNAL_ERROR',
+        error instanceof Error ? error.message : String(error)
+      )
+    }
+  }
+
+  /**
+   * Gets all available tools
+   * @returns An array of MCP tools
+   */
+  async getAvailableTools(): Promise<MCPTool[]> {
+    const tools: MCPTool[] = []
+    
+    // Get all available augmentation types
+    const augmentationTypes = augmentationPipeline.getAvailableAugmentationTypes()
+    
+    for (const type of augmentationTypes) {
+      // Get all augmentations of this type
+      const augmentations = augmentationPipeline.getAugmentationsByType(type)
+      
+      for (const augmentation of augmentations) {
+        // Get all methods of this augmentation (excluding private methods and base methods)
+        const methods = Object.getOwnPropertyNames(Object.getPrototypeOf(augmentation))
+          .filter(method => 
+            !method.startsWith('_') && 
+            method !== 'constructor' &&
+            method !== 'initialize' &&
+            method !== 'shutDown' &&
+            method !== 'getStatus' &&
+            typeof augmentation[method] === 'function'
+          )
+        
+        // Create a tool for each method
+        for (const method of methods) {
+          tools.push(this.createToolDefinition(type, augmentation.name, method))
+        }
+      }
+    }
+    
+    return tools
+  }
+
+  /**
+   * Creates a tool definition
+   * @param type The augmentation type
+   * @param augmentationName The augmentation name
+   * @param method The method name
+   * @returns An MCP tool definition
+   */
+  private createToolDefinition(type: string, augmentationName: string, method: string): MCPTool {
+    return {
+      name: `brainy_${type}_${method}`,
+      description: `Access to Brainy's ${type} augmentation '${augmentationName}' method '${method}'`,
+      parameters: {
+        type: 'object',
+        properties: {
+          args: {
+            type: 'array',
+            description: `Arguments for the ${method} method`
+          },
+          options: {
+            type: 'object',
+            description: 'Optional execution options'
+          }
+        },
+        required: ['args']
+      }
+    }
+  }
+
+  /**
+   * Executes the appropriate pipeline based on the augmentation type
+   * @param type The augmentation type
+   * @param method The method to execute
+   * @param parameters The parameters for the method
+   * @returns The result of the pipeline execution
+   */
+  private async executePipeline(type: string, method: string, parameters: any): Promise<any> {
+    const { args = [], options = {} } = parameters
+    
+    switch (type) {
+      case AugmentationType.SENSE:
+        return await augmentationPipeline.executeSensePipeline(method, args, options)
+      case AugmentationType.CONDUIT:
+        return await augmentationPipeline.executeConduitPipeline(method, args, options)
+      case AugmentationType.COGNITION:
+        return await augmentationPipeline.executeCognitionPipeline(method, args, options)
+      case AugmentationType.MEMORY:
+        return await augmentationPipeline.executeMemoryPipeline(method, args, options)
+      case AugmentationType.PERCEPTION:
+        return await augmentationPipeline.executePerceptionPipeline(method, args, options)
+      case AugmentationType.DIALOG:
+        return await augmentationPipeline.executeDialogPipeline(method, args, options)
+      case AugmentationType.ACTIVATION:
+        return await augmentationPipeline.executeActivationPipeline(method, args, options)
+      default:
+        throw new Error(`Unsupported augmentation type: ${type}`)
+    }
+  }
+
+  /**
+   * Checks if an augmentation type is valid
+   * @param type The augmentation type to check
+   * @returns Whether the augmentation type is valid
+   */
+  private isValidAugmentationType(type: string): boolean {
+    return Object.values(AugmentationType).includes(type as AugmentationType)
+  }
+
+  /**
+   * Creates a success response
+   * @param requestId The request ID
+   * @param data The response data
+   * @returns An MCP response
+   */
+  private createSuccessResponse(requestId: string, data: any): MCPResponse {
+    return {
+      success: true,
+      requestId,
+      version: MCP_VERSION,
+      data
+    }
+  }
+
+  /**
+   * Creates an error response
+   * @param requestId The request ID
+   * @param code The error code
+   * @param message The error message
+   * @param details Optional error details
+   * @returns An MCP response
+   */
+  private createErrorResponse(
+    requestId: string,
+    code: string,
+    message: string,
+    details?: any
+  ): MCPResponse {
+    return {
+      success: false,
+      requestId,
+      version: MCP_VERSION,
+      error: {
+        code,
+        message,
+        details
+      }
+    }
+  }
+
+  /**
+   * Creates a new request ID
+   * @returns A new UUID
+   */
+  generateRequestId(): string {
+    return uuidv4()
+  }
+}

src/pipeline.ts

@@ -0,0 +1,919 @@
+/**
+ * Unified Pipeline
+ *
+ * This module combines the functionality of the primary augmentation pipeline and the streamlined pipeline
+ * into a single, unified pipeline system. It provides both the registry functionality of the primary pipeline
+ * and the simplified execution API of the streamlined pipeline.
+ */
+
+import {
+  IAugmentation,
+  AugmentationType,
+  AugmentationResponse,
+  IWebSocketSupport,
+  BrainyAugmentations
+} from './types/augmentations.js'
+import { AugmentationRegistry, IPipeline } from './types/pipelineTypes.js'
+import { isThreadingAvailable } from './utils/environment.js'
+import { executeInThread } from './utils/workerUtils.js'
+import { executeAugmentation } from './augmentationFactory.js'
+import { setDefaultPipeline } from './augmentationRegistry.js'
+
+/**
+ * Execution mode for the pipeline
+ */
+export enum ExecutionMode {
+  SEQUENTIAL = 'sequential',
+  PARALLEL = 'parallel',
+  FIRST_SUCCESS = 'firstSuccess',
+  FIRST_RESULT = 'firstResult',
+  THREADED = 'threaded'  // Execute in separate threads when available
+}
+
+/**
+ * Options for pipeline execution
+ */
+export interface PipelineOptions {
+  mode?: ExecutionMode;
+  timeout?: number;
+  stopOnError?: boolean;
+  forceThreading?: boolean;  // Force threading even if not in THREADED mode
+  disableThreading?: boolean;  // Disable threading even if in THREADED mode
+}
+
+/**
+ * Default pipeline options
+ */
+const DEFAULT_PIPELINE_OPTIONS: PipelineOptions = {
+  mode: ExecutionMode.SEQUENTIAL,
+  timeout: 30000,
+  stopOnError: false,
+  forceThreading: false,
+  disableThreading: false
+}
+
+/**
+ * Result of a pipeline execution
+ */
+export interface PipelineResult<T> {
+  results: AugmentationResponse<T>[];
+  errors: Error[];
+  successful: AugmentationResponse<T>[];
+}
+
+/**
+ * Pipeline class
+ *
+ * Manages multiple augmentations of each type and provides methods to execute them.
+ * Implements the IPipeline interface to avoid circular dependencies.
+ */
+export class Pipeline implements IPipeline {
+  private registry: AugmentationRegistry = {
+    sense: [],
+    conduit: [],
+    cognition: [],
+    memory: [],
+    perception: [],
+    dialog: [],
+    activation: [],
+    webSocket: []
+  }
+
+  /**
+   * Register an augmentation with the pipeline
+   *
+   * @param augmentation The augmentation to register
+   * @returns The pipeline instance for chaining
+   */
+  public register<T extends IAugmentation>(augmentation: T): Pipeline {
+    let registered = false
+
+    // Check for specific augmentation types
+    if (this.isAugmentationType<BrainyAugmentations.ISenseAugmentation>(
+      augmentation,
+      'processRawData',
+      'listenToFeed'
+    )) {
+      this.registry.sense.push(augmentation)
+      registered = true
+    } else if (this.isAugmentationType<BrainyAugmentations.IConduitAugmentation>(
+      augmentation,
+      'establishConnection',
+      'readData',
+      'writeData',
+      'monitorStream'
+    )) {
+      this.registry.conduit.push(augmentation)
+      registered = true
+    } else if (this.isAugmentationType<BrainyAugmentations.ICognitionAugmentation>(
+      augmentation,
+      'reason',
+      'infer',
+      'executeLogic'
+    )) {
+      this.registry.cognition.push(augmentation)
+      registered = true
+    } else if (this.isAugmentationType<BrainyAugmentations.IMemoryAugmentation>(
+      augmentation,
+      'storeData',
+      'retrieveData',
+      'updateData',
+      'deleteData',
+      'listDataKeys'
+    )) {
+      this.registry.memory.push(augmentation)
+      registered = true
+    } else if (this.isAugmentationType<BrainyAugmentations.IPerceptionAugmentation>(
+      augmentation,
+      'interpret',
+      'organize',
+      'generateVisualization'
+    )) {
+      this.registry.perception.push(augmentation)
+      registered = true
+    } else if (this.isAugmentationType<BrainyAugmentations.IDialogAugmentation>(
+      augmentation,
+      'processUserInput',
+      'generateResponse',
+      'manageContext'
+    )) {
+      this.registry.dialog.push(augmentation)
+      registered = true
+    } else if (this.isAugmentationType<BrainyAugmentations.IActivationAugmentation>(
+      augmentation,
+      'triggerAction',
+      'generateOutput',
+      'interactExternal'
+    )) {
+      this.registry.activation.push(augmentation)
+      registered = true
+    }
+
+    // Check if the augmentation supports WebSocket
+    if (this.isAugmentationType<IWebSocketSupport>(
+      augmentation,
+      'connectWebSocket',
+      'sendWebSocketMessage',
+      'onWebSocketMessage',
+      'closeWebSocket'
+    )) {
+      this.registry.webSocket.push(augmentation as IWebSocketSupport)
+      registered = true
+    }
+
+    // If the augmentation wasn't registered as any known type, throw an error
+    if (!registered) {
+      throw new Error(`Unknown augmentation type: ${augmentation.name}`)
+    }
+
+    return this
+  }
+
+  /**
+   * Unregister an augmentation from the pipeline
+   *
+   * @param augmentationName The name of the augmentation to unregister
+   * @returns The pipeline instance for chaining
+   */
+  public unregister(augmentationName: string): Pipeline {
+    let found = false
+
+    // Remove from all registries
+    for (const type in this.registry) {
+      const typedRegistry = this.registry[type as keyof AugmentationRegistry]
+      const index = typedRegistry.findIndex(aug => aug.name === augmentationName)
+
+      if (index !== -1) {
+        typedRegistry.splice(index, 1)
+        found = true
+      }
+    }
+
+    return this
+  }
+
+  /**
+   * Initialize all registered augmentations
+   *
+   * @returns A promise that resolves when all augmentations are initialized
+   */
+  public async initialize(): Promise<void> {
+    const allAugmentations = this.getAllAugmentations()
+
+    await Promise.all(
+      allAugmentations.map(augmentation =>
+        augmentation.initialize().catch(error => {
+          console.error(`Failed to initialize augmentation ${augmentation.name}:`, error)
+        })
+      )
+    )
+  }
+
+  /**
+   * Shut down all registered augmentations
+   *
+   * @returns A promise that resolves when all augmentations are shut down
+   */
+  public async shutDown(): Promise<void> {
+    const allAugmentations = this.getAllAugmentations()
+
+    await Promise.all(
+      allAugmentations.map(augmentation =>
+        augmentation.shutDown().catch(error => {
+          console.error(`Failed to shut down augmentation ${augmentation.name}:`, error)
+        })
+      )
+    )
+  }
+
+  /**
+   * Get all registered augmentations
+   *
+   * @returns An array of all registered augmentations
+   */
+  public getAllAugmentations(): IAugmentation[] {
+    // Create a Set to avoid duplicates (an augmentation might be in multiple registries)
+    const allAugmentations = new Set<IAugmentation>([
+      ...this.registry.sense,
+      ...this.registry.conduit,
+      ...this.registry.cognition,
+      ...this.registry.memory,
+      ...this.registry.perception,
+      ...this.registry.dialog,
+      ...this.registry.activation,
+      ...this.registry.webSocket
+    ])
+
+    // Convert back to array
+    return Array.from(allAugmentations)
+  }
+
+  /**
+   * Get all augmentations of a specific type
+   *
+   * @param type The type of augmentation to get
+   * @returns An array of all augmentations of the specified type
+   */
+  public getAugmentationsByType(type: AugmentationType): IAugmentation[] {
+    switch (type) {
+      case AugmentationType.SENSE:
+        return [...this.registry.sense]
+      case AugmentationType.CONDUIT:
+        return [...this.registry.conduit]
+      case AugmentationType.COGNITION:
+        return [...this.registry.cognition]
+      case AugmentationType.MEMORY:
+        return [...this.registry.memory]
+      case AugmentationType.PERCEPTION:
+        return [...this.registry.perception]
+      case AugmentationType.DIALOG:
+        return [...this.registry.dialog]
+      case AugmentationType.ACTIVATION:
+        return [...this.registry.activation]
+      case AugmentationType.WEBSOCKET:
+        return [...this.registry.webSocket]
+      default:
+        return []
+    }
+  }
+
+  /**
+   * Get all available augmentation types
+   *
+   * @returns An array of all augmentation types that have at least one registered augmentation
+   */
+  public getAvailableAugmentationTypes(): AugmentationType[] {
+    const availableTypes: AugmentationType[] = []
+
+    if (this.registry.sense.length > 0) availableTypes.push(AugmentationType.SENSE)
+    if (this.registry.conduit.length > 0) availableTypes.push(AugmentationType.CONDUIT)
+    if (this.registry.cognition.length > 0) availableTypes.push(AugmentationType.COGNITION)
+    if (this.registry.memory.length > 0) availableTypes.push(AugmentationType.MEMORY)
+    if (this.registry.perception.length > 0) availableTypes.push(AugmentationType.PERCEPTION)
+    if (this.registry.dialog.length > 0) availableTypes.push(AugmentationType.DIALOG)
+    if (this.registry.activation.length > 0) availableTypes.push(AugmentationType.ACTIVATION)
+    if (this.registry.webSocket.length > 0) availableTypes.push(AugmentationType.WEBSOCKET)
+
+    return availableTypes
+  }
+
+  /**
+   * Get all WebSocket-supporting augmentations
+   *
+   * @returns An array of all augmentations that support WebSocket connections
+   */
+  public getWebSocketAugmentations(): IWebSocketSupport[] {
+    return [...this.registry.webSocket]
+  }
+
+  /**
+   * Check if an augmentation is of a specific type
+   *
+   * @param augmentation The augmentation to check
+   * @param methods The methods that should be present on the augmentation
+   * @returns True if the augmentation is of the specified type
+   */
+  private isAugmentationType<T extends IAugmentation>(
+    augmentation: IAugmentation,
+    ...methods: (keyof T)[]
+  ): augmentation is T {
+    // First check that the augmentation has all the required base methods
+    const baseMethodsExist = [
+      'initialize',
+      'shutDown',
+      'getStatus'
+    ].every(method => typeof (augmentation as any)[method] === 'function')
+
+    if (!baseMethodsExist) {
+      return false
+    }
+
+    // Then check that it has all the specific methods for this type
+    return methods.every(method => typeof (augmentation as any)[method] === 'function')
+  }
+
+  /**
+   * Determines if threading should be used based on options and environment
+   * 
+   * @param options The pipeline options
+   * @returns True if threading should be used, false otherwise
+   */
+  private shouldUseThreading(options: PipelineOptions): boolean {
+    // If threading is explicitly disabled, don't use it
+    if (options.disableThreading) {
+      return false
+    }
+
+    // If threading is explicitly forced, use it if available
+    if (options.forceThreading) {
+      return isThreadingAvailable()
+    }
+
+    // If in THREADED mode, use threading if available
+    if (options.mode === ExecutionMode.THREADED) {
+      return isThreadingAvailable()
+    }
+
+    // Otherwise, don't use threading
+    return false
+  }
+
+  /**
+   * Executes a method on multiple augmentations using the specified execution mode
+   * 
+   * @param augmentations The augmentations to execute the method on
+   * @param method The method to execute
+   * @param args The arguments to pass to the method
+   * @param options Options for the execution
+   * @returns A promise that resolves with the results
+   */
+  public async execute<T>(
+    augmentations: IAugmentation[],
+    method: string,
+    args: any[] = [],
+    options: PipelineOptions = {}
+  ): Promise<PipelineResult<T>> {
+    const opts = { ...DEFAULT_PIPELINE_OPTIONS, ...options }
+    const enabledAugmentations = augmentations.filter(aug => aug.enabled !== false)
+
+    if (enabledAugmentations.length === 0) {
+      return { results: [], errors: [], successful: [] }
+    }
+
+    const result: PipelineResult<T> = {
+      results: [],
+      errors: [],
+      successful: []
+    }
+
+    // Create a function to execute with timeout
+    const executeWithTimeout = async (
+      augmentation: IAugmentation
+    ): Promise<AugmentationResponse<T>> => {
+      try {
+        // Create a timeout promise if a timeout is specified
+        if (opts.timeout) {
+          const timeoutPromise = new Promise<never>((_, reject) => {
+            setTimeout(() => {
+              reject(
+                new Error(`Timeout executing ${method} on ${augmentation.name}`)
+              )
+            }, opts.timeout)
+          })
+
+          // Check if threading should be used
+          const useThreading = this.shouldUseThreading(opts)
+
+          // Execute the method on the augmentation, using threading if appropriate
+          let methodPromise: Promise<AugmentationResponse<T>>
+
+          if (useThreading) {
+            // Execute in a separate thread
+            try {
+              // Create a function that can be serialized and executed in a worker
+              const workerFn = (...workerArgs: any[]) => {
+                // This function will be stringified and executed in the worker
+                // It needs to be self-contained
+                const augFn = augmentation[method as string] as Function
+                return augFn.apply(augmentation, workerArgs)
+              }
+
+              methodPromise = executeInThread<AugmentationResponse<T>>(workerFn.toString(), args)
+            } catch (threadError) {
+              console.warn(`Failed to execute in thread, falling back to main thread: ${threadError}`)
+              // Fall back to executing in the main thread
+              methodPromise = Promise.resolve((augmentation[method] as Function)(...args) as AugmentationResponse<T>)
+            }
+          } else {
+            // Execute in the main thread
+            methodPromise = Promise.resolve((augmentation[method] as Function)(...args) as AugmentationResponse<T>)
+          }
+
+          // Race the method promise against the timeout promise
+          return await Promise.race([methodPromise, timeoutPromise])
+        } else {
+          // No timeout, just execute the method
+          return await executeAugmentation<any, T>(augmentation, method, ...args)
+        }
+      } catch (error) {
+        result.errors.push(
+          error instanceof Error ? error : new Error(String(error))
+        )
+        return {
+          success: false,
+          data: null as any,
+          error: error instanceof Error ? error.message : String(error)
+        }
+      }
+    }
+
+    // Execute based on the specified mode
+    switch (opts.mode) {
+      case ExecutionMode.PARALLEL:
+      case ExecutionMode.THREADED:
+        // Execute all augmentations in parallel
+        result.results = await Promise.all(
+          enabledAugmentations.map((aug) => executeWithTimeout(aug))
+        )
+        break
+
+      case ExecutionMode.FIRST_SUCCESS:
+        // Execute augmentations sequentially until one succeeds
+        for (const augmentation of enabledAugmentations) {
+          const response = await executeWithTimeout(augmentation)
+          result.results.push(response)
+
+          if (response.success) {
+            break
+          }
+        }
+        break
+
+      case ExecutionMode.FIRST_RESULT:
+        // Execute augmentations sequentially until one returns a non-null result
+        for (const augmentation of enabledAugmentations) {
+          const response = await executeWithTimeout(augmentation)
+          result.results.push(response)
+
+          if (
+            response.success &&
+            response.data !== null &&
+            response.data !== undefined
+          ) {
+            break
+          }
+        }
+        break
+
+      case ExecutionMode.SEQUENTIAL:
+      default:
+        // Execute augmentations sequentially
+        for (const augmentation of enabledAugmentations) {
+          const response = await executeWithTimeout(augmentation)
+          result.results.push(response)
+
+          // Check if we need to stop on error
+          if (opts.stopOnError && !response.success) {
+            break
+          }
+        }
+        break
+    }
+
+    // Filter successful results
+    result.successful = result.results.filter((r) => r.success)
+
+    return result
+  }
+
+  /**
+   * Executes a method on augmentations of a specific type
+   * 
+   * @param type The type of augmentations to execute the method on
+   * @param method The method to execute
+   * @param args The arguments to pass to the method
+   * @param options Options for the execution
+   * @returns A promise that resolves with the results
+   */
+  public async executeByType<T>(
+    type: AugmentationType,
+    method: string,
+    args: any[] = [],
+    options: PipelineOptions = {}
+  ): Promise<PipelineResult<T>> {
+    const augmentations = this.getAugmentationsByType(type)
+    return this.execute<T>(augmentations, method, args, options)
+  }
+
+  /**
+   * Executes a method on a single augmentation with automatic error handling
+   * 
+   * @param augmentation The augmentation to execute the method on
+   * @param method The method to execute
+   * @param args The arguments to pass to the method
+   * @returns A promise that resolves with the result
+   */
+  public async executeSingle<T>(
+    augmentation: IAugmentation,
+    method: string,
+    ...args: any[]
+  ): Promise<AugmentationResponse<T>> {
+    return executeAugmentation<any, T>(augmentation, method, ...args)
+  }
+
+  /**
+   * Process static data through a pipeline of augmentations
+   * 
+   * @param data The data to process
+   * @param pipeline An array of processing steps, each with an augmentation, method, and optional args transformer
+   * @param options Options for the execution
+   * @returns A promise that resolves with the final result
+   */
+  public async processStaticData<T, R = any>(
+    data: T,
+    pipeline: Array<{
+      augmentation: IAugmentation
+      method: string
+      transformArgs?: (data: any, prevResult?: any) => any[]
+    }>,
+    options: PipelineOptions = {}
+  ): Promise<AugmentationResponse<R>> {
+    let currentData = data
+    let prevResult: any = undefined
+
+    for (const step of pipeline) {
+      // Transform args if a transformer is provided, otherwise use the current data as the only arg
+      const args = step.transformArgs
+        ? step.transformArgs(currentData, prevResult)
+        : [currentData]
+
+      // Execute the method
+      const result = await this.executeSingle<any>(
+        step.augmentation,
+        step.method,
+        ...args
+      )
+
+      // If the step failed, return the error
+      if (!result.success) {
+        return result as AugmentationResponse<R>
+      }
+
+      // Update the current data for the next step
+      currentData = result.data
+      prevResult = result
+    }
+
+    // Return the final result
+    return prevResult as AugmentationResponse<R>
+  }
+
+  /**
+   * Process streaming data through a pipeline of augmentations
+   * 
+   * @param source The source augmentation that provides the data stream
+   * @param sourceMethod The method on the source augmentation that provides the data stream
+   * @param sourceArgs The arguments to pass to the source method
+   * @param pipeline An array of processing steps, each with an augmentation, method, and optional args transformer
+   * @param callback Function to call with the results of processing each data item
+   * @param options Options for the execution
+   * @returns A promise that resolves when the pipeline is set up
+   */
+  public async processStreamingData<T>(
+    source: IAugmentation,
+    sourceMethod: string,
+    sourceArgs: any[],
+    pipeline: Array<{
+      augmentation: IAugmentation
+      method: string
+      transformArgs?: (data: any, prevResult?: any) => any[]
+    }>,
+    callback: (result: AugmentationResponse<T>) => void,
+    options: PipelineOptions = {}
+  ): Promise<void> {
+    // Create a chain of processors
+    const processData = async (data: any) => {
+      let currentData = data
+      let prevResult: any = undefined
+
+      for (const step of pipeline) {
+        // Transform args if a transformer is provided, otherwise use the current data as the only arg
+        const args = step.transformArgs
+          ? step.transformArgs(currentData, prevResult)
+          : [currentData]
+
+        // Execute the method
+        const result = await this.executeSingle<any>(
+          step.augmentation,
+          step.method,
+          ...args
+        )
+
+        // If the step failed, return the error
+        if (!result.success) {
+          callback(result as AugmentationResponse<T>)
+          return
+        }
+
+        // Update the current data for the next step
+        currentData = result.data
+        prevResult = result
+      }
+
+      // Call the callback with the final result
+      callback(prevResult as AugmentationResponse<T>)
+    }
+
+    // The last argument to the source method should be a callback that receives the data
+    const dataCallback = (data: any) => {
+      processData(data).catch((error) => {
+        console.error('Error processing streaming data:', error)
+        callback({
+          success: false,
+          data: null as any,
+          error: error instanceof Error ? error.message : String(error)
+        })
+      })
+    }
+
+    // Execute the source method with the provided args and the data callback
+    await this.executeSingle(source, sourceMethod, ...sourceArgs, dataCallback)
+  }
+
+  /**
+   * Create a reusable pipeline for processing data
+   * 
+   * @param pipeline An array of processing steps
+   * @param options Options for the execution
+   * @returns A function that processes data through the pipeline
+   */
+  public createPipeline<T, R>(
+    pipeline: Array<{
+      augmentation: IAugmentation
+      method: string
+      transformArgs?: (data: any, prevResult?: any) => any[]
+    }>,
+    options: PipelineOptions = {}
+  ): (data: T) => Promise<AugmentationResponse<R>> {
+    return (data: T) => this.processStaticData<T, R>(data, pipeline, options)
+  }
+
+  /**
+   * Create a reusable streaming pipeline
+   * 
+   * @param source The source augmentation
+   * @param sourceMethod The method on the source augmentation
+   * @param pipeline An array of processing steps
+   * @param options Options for the execution
+   * @returns A function that sets up the streaming pipeline
+   */
+  public createStreamingPipeline<T, R>(
+    source: IAugmentation,
+    sourceMethod: string,
+    pipeline: Array<{
+      augmentation: IAugmentation
+      method: string
+      transformArgs?: (data: any, prevResult?: any) => any[]
+    }>,
+    options: PipelineOptions = {}
+  ): (
+    sourceArgs: any[],
+    callback: (result: AugmentationResponse<R>) => void
+  ) => Promise<void> {
+    return (
+      sourceArgs: any[],
+      callback: (result: AugmentationResponse<R>) => void
+    ) =>
+      this.processStreamingData<R>(
+        source,
+        sourceMethod,
+        sourceArgs,
+        pipeline,
+        callback,
+        options
+      )
+  }
+
+  // Legacy methods for backward compatibility
+
+  /**
+   * Execute a sense pipeline (legacy method)
+   */
+  public async executeSensePipeline<
+    M extends keyof BrainyAugmentations.ISenseAugmentation & string,
+    R extends BrainyAugmentations.ISenseAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.ISenseAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.ISenseAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const result = await this.executeByType<R>(AugmentationType.SENSE, method, args, options)
+    return result.results.map(r => Promise.resolve(r))
+  }
+
+  /**
+   * Execute a conduit pipeline (legacy method)
+   */
+  public async executeConduitPipeline<
+    M extends keyof BrainyAugmentations.IConduitAugmentation & string,
+    R extends BrainyAugmentations.IConduitAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.IConduitAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.IConduitAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const result = await this.executeByType<R>(AugmentationType.CONDUIT, method, args, options)
+    return result.results.map(r => Promise.resolve(r))
+  }
+
+  /**
+   * Execute a cognition pipeline (legacy method)
+   */
+  public async executeCognitionPipeline<
+    M extends keyof BrainyAugmentations.ICognitionAugmentation & string,
+    R extends BrainyAugmentations.ICognitionAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.ICognitionAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.ICognitionAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const result = await this.executeByType<R>(AugmentationType.COGNITION, method, args, options)
+    return result.results.map(r => Promise.resolve(r))
+  }
+
+  /**
+   * Execute a memory pipeline (legacy method)
+   */
+  public async executeMemoryPipeline<
+    M extends keyof BrainyAugmentations.IMemoryAugmentation & string,
+    R extends BrainyAugmentations.IMemoryAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.IMemoryAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.IMemoryAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const result = await this.executeByType<R>(AugmentationType.MEMORY, method, args, options)
+    return result.results.map(r => Promise.resolve(r))
+  }
+
+  /**
+   * Execute a perception pipeline (legacy method)
+   */
+  public async executePerceptionPipeline<
+    M extends keyof BrainyAugmentations.IPerceptionAugmentation & string,
+    R extends BrainyAugmentations.IPerceptionAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.IPerceptionAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.IPerceptionAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const result = await this.executeByType<R>(AugmentationType.PERCEPTION, method, args, options)
+    return result.results.map(r => Promise.resolve(r))
+  }
+
+  /**
+   * Execute a dialog pipeline (legacy method)
+   */
+  public async executeDialogPipeline<
+    M extends keyof BrainyAugmentations.IDialogAugmentation & string,
+    R extends BrainyAugmentations.IDialogAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.IDialogAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.IDialogAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const result = await this.executeByType<R>(AugmentationType.DIALOG, method, args, options)
+    return result.results.map(r => Promise.resolve(r))
+  }
+
+  /**
+   * Execute an activation pipeline (legacy method)
+   */
+  public async executeActivationPipeline<
+    M extends keyof BrainyAugmentations.IActivationAugmentation & string,
+    R extends BrainyAugmentations.IActivationAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.IActivationAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.IActivationAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const result = await this.executeByType<R>(AugmentationType.ACTIVATION, method, args, options)
+    return result.results.map(r => Promise.resolve(r))
+  }
+}
+
+// Create and export a default instance of the pipeline
+export const pipeline = new Pipeline()
+
+// Set the default pipeline instance for the augmentation registry
+// This breaks the circular dependency between pipeline.ts and augmentationRegistry.ts
+setDefaultPipeline(pipeline)
+
+// Re-export the legacy pipeline for backward compatibility
+export const augmentationPipeline = pipeline
+
+// Re-export the streamlined execution functions for backward compatibility
+export const executeStreamlined = <T>(
+  augmentations: IAugmentation[],
+  method: string,
+  args: any[] = [],
+  options: PipelineOptions = {}
+): Promise<PipelineResult<T>> => {
+  return pipeline.execute<T>(augmentations, method, args, options)
+}
+
+export const executeByType = <T>(
+  type: AugmentationType,
+  method: string,
+  args: any[] = [],
+  options: PipelineOptions = {}
+): Promise<PipelineResult<T>> => {
+  return pipeline.executeByType<T>(type, method, args, options)
+}
+
+export const executeSingle = <T>(
+  augmentation: IAugmentation,
+  method: string,
+  ...args: any[]
+): Promise<AugmentationResponse<T>> => {
+  return pipeline.executeSingle<T>(augmentation, method, ...args)
+}
+
+export const processStaticData = <T, R = any>(
+  data: T,
+  pipelineSteps: Array<{
+    augmentation: IAugmentation
+    method: string
+    transformArgs?: (data: any, prevResult?: any) => any[]
+  }>,
+  options: PipelineOptions = {}
+): Promise<AugmentationResponse<R>> => {
+  return pipeline.processStaticData<T, R>(data, pipelineSteps, options)
+}
+
+export const processStreamingData = <T>(
+  source: IAugmentation,
+  sourceMethod: string,
+  sourceArgs: any[],
+  pipelineSteps: Array<{
+    augmentation: IAugmentation
+    method: string
+    transformArgs?: (data: any, prevResult?: any) => any[]
+  }>,
+  callback: (result: AugmentationResponse<T>) => void,
+  options: PipelineOptions = {}
+): Promise<void> => {
+  return pipeline.processStreamingData<T>(source, sourceMethod, sourceArgs, pipelineSteps, callback, options)
+}
+
+export const createPipeline = <T, R>(
+  pipelineSteps: Array<{
+    augmentation: IAugmentation
+    method: string
+    transformArgs?: (data: any, prevResult?: any) => any[]
+  }>,
+  options: PipelineOptions = {}
+): (data: T) => Promise<AugmentationResponse<R>> => {
+  return pipeline.createPipeline<T, R>(pipelineSteps, options)
+}
+
+export const createStreamingPipeline = <T, R>(
+  source: IAugmentation,
+  sourceMethod: string,
+  pipelineSteps: Array<{
+    augmentation: IAugmentation
+    method: string
+    transformArgs?: (data: any, prevResult?: any) => any[]
+  }>,
+  options: PipelineOptions = {}
+): (
+  sourceArgs: any[],
+  callback: (result: AugmentationResponse<R>) => void
+) => Promise<void> => {
+  return pipeline.createStreamingPipeline<T, R>(source, sourceMethod, pipelineSteps, options)
+}
+
+// For backward compatibility with StreamlinedExecutionMode
+export const StreamlinedExecutionMode = ExecutionMode
+export type StreamlinedPipelineOptions = PipelineOptions
+export type StreamlinedPipelineResult<T> = PipelineResult<T>

src/sequentialPipeline.ts

@@ -0,0 +1,572 @@
+/**
+ * Sequential Augmentation Pipeline
+ *
+ * This module provides a pipeline for executing augmentations in a specific sequence:
+ * ISense -> IMemory -> ICognition -> IConduit -> IActivation -> IPerception
+ *
+ * It supports high-performance streaming data from WebSockets without blocking.
+ * Optimized for Node.js 23.11+ using native WebStreams API.
+ */
+
+import {
+  AugmentationType,
+  IAugmentation,
+  IWebSocketSupport,
+  ISenseAugmentation,
+  IMemoryAugmentation,
+  ICognitionAugmentation,
+  IConduitAugmentation,
+  IActivationAugmentation,
+  IPerceptionAugmentation,
+  AugmentationResponse,
+  WebSocketConnection
+} from './types/augmentations.js'
+import { BrainyData } from './brainyData.js'
+import { augmentationPipeline } from './augmentationPipeline.js'
+// Use the browser's built-in WebStreams API or Node.js native WebStreams API
+// This approach ensures compatibility with both environments
+let TransformStream: any, ReadableStream: any, WritableStream: any;
+
+// Function to initialize the stream classes
+const initializeStreamClasses = () => {
+  // Try to use the browser's built-in WebStreams API first
+  if (typeof globalThis.TransformStream !== 'undefined' &&
+      typeof globalThis.ReadableStream !== 'undefined' &&
+      typeof globalThis.WritableStream !== 'undefined') {
+    TransformStream = globalThis.TransformStream;
+    ReadableStream = globalThis.ReadableStream;
+    WritableStream = globalThis.WritableStream;
+    return Promise.resolve();
+  } else {
+    // In Node.js environment, try to import from node:stream/web
+    // This will be executed in Node.js but not in browsers
+    return import('node:stream/web')
+      .then(streamWebModule => {
+        TransformStream = streamWebModule.TransformStream;
+        ReadableStream = streamWebModule.ReadableStream;
+        WritableStream = streamWebModule.WritableStream;
+      })
+      .catch(error => {
+        console.error('Failed to import WebStreams API:', error);
+        // Provide fallback implementations or throw a more helpful error
+        throw new Error('WebStreams API is not available in this environment. Please use a modern browser or Node.js 18+.');
+      });
+  }
+};
+
+// Initialize immediately but don't block module execution
+const streamClassesPromise = initializeStreamClasses();
+
+/**
+ * Options for sequential pipeline execution
+ */
+export interface SequentialPipelineOptions {
+  /**
+   * Timeout for each augmentation execution in milliseconds
+   */
+  timeout?: number;
+
+  /**
+   * Whether to stop execution if an error occurs
+   */
+  stopOnError?: boolean;
+
+  /**
+   * BrainyData instance to use for storage
+   */
+  brainyData?: BrainyData;
+}
+
+/**
+ * Default pipeline options
+ */
+const DEFAULT_SEQUENTIAL_PIPELINE_OPTIONS: SequentialPipelineOptions = {
+  timeout: 30000,
+  stopOnError: false
+}
+
+/**
+ * Result of a pipeline execution
+ */
+export interface PipelineResult<T> {
+  /**
+   * Whether the pipeline execution was successful
+   */
+  success: boolean;
+
+  /**
+   * The data returned by the pipeline
+   */
+  data: T;
+
+  /**
+   * Error message if the pipeline execution failed
+   */
+  error?: string;
+
+  /**
+   * Results from each stage of the pipeline
+   */
+  stageResults: {
+    sense?: AugmentationResponse<unknown>;
+    memory?: AugmentationResponse<unknown>;
+    cognition?: AugmentationResponse<unknown>;
+    conduit?: AugmentationResponse<unknown>;
+    activation?: AugmentationResponse<unknown>;
+    perception?: AugmentationResponse<unknown>;
+  }
+}
+
+/**
+ * SequentialPipeline class
+ *
+ * Executes augmentations in a specific sequence:
+ * ISense -> IMemory -> ICognition -> IConduit -> IActivation -> IPerception
+ */
+export class SequentialPipeline {
+  private brainyData: BrainyData;
+
+  /**
+   * Create a new sequential pipeline
+   * 
+   * @param options Options for the pipeline
+   */
+  constructor(options: SequentialPipelineOptions = {}) {
+    this.brainyData = options.brainyData || new BrainyData();
+  }
+
+  /**
+   * Ensure stream classes are initialized
+   * @private
+   */
+  private async ensureStreamClassesInitialized(): Promise<void> {
+    await streamClassesPromise;
+  }
+
+  /**
+   * Initialize the pipeline
+   * 
+   * @returns A promise that resolves when initialization is complete
+   */
+  public async initialize(): Promise<void> {
+    // Initialize stream classes and BrainyData in parallel
+    await Promise.all([
+      this.ensureStreamClassesInitialized(),
+      this.brainyData.init()
+    ]);
+  }
+
+  /**
+   * Process data through the sequential pipeline
+   * 
+   * @param rawData The raw data to process
+   * @param dataType The type of data (e.g., 'text', 'image', 'audio')
+   * @param options Options for pipeline execution
+   * @returns A promise that resolves with the pipeline result
+   */
+  public async processData(
+    rawData: Buffer | string,
+    dataType: string,
+    options: SequentialPipelineOptions = {}
+  ): Promise<PipelineResult<unknown>> {
+    const opts = { ...DEFAULT_SEQUENTIAL_PIPELINE_OPTIONS, ...options };
+    const result: PipelineResult<unknown> = {
+      success: true,
+      data: null,
+      stageResults: {}
+    };
+
+    try {
+      // Step 1: Process raw data with ISense augmentations
+      const senseResults = await augmentationPipeline.executeSensePipeline(
+        'processRawData',
+        [rawData, dataType],
+        { timeout: opts.timeout, stopOnError: opts.stopOnError }
+      );
+
+      // Get the first successful result
+      let senseResult: AugmentationResponse<{ nouns: string[], verbs: string[] }> | null = null;
+      for (const resultPromise of senseResults) {
+        const res = await resultPromise;
+        if (res.success) {
+          senseResult = res;
+          break;
+        }
+      }
+
+      if (!senseResult || !senseResult.success) {
+        return {
+          success: false,
+          data: null,
+          error: 'Failed to process raw data with ISense augmentations',
+          stageResults: { sense: senseResult || { success: false, data: null, error: 'No sense augmentations available' } }
+        };
+      }
+
+      result.stageResults.sense = senseResult;
+
+      // Step 2: Store data in BrainyData using IMemory augmentations
+      const memoryAugmentations = augmentationPipeline.getAugmentationsByType(AugmentationType.MEMORY) as IMemoryAugmentation[];
+
+      if (memoryAugmentations.length === 0) {
+        return {
+          success: false,
+          data: null,
+          error: 'No memory augmentations available',
+          stageResults: result.stageResults
+        };
+      }
+
+      // Use the first available memory augmentation
+      const memoryAugmentation = memoryAugmentations[0];
+
+      // Generate a key for the data
+      const dataKey = `data_${Date.now()}_${Math.random().toString(36).substring(2, 15)}`;
+
+      // Store the data
+      const memoryResult = await memoryAugmentation.storeData(
+        dataKey,
+        {
+          rawData,
+          dataType,
+          nouns: senseResult.data.nouns,
+          verbs: senseResult.data.verbs,
+          timestamp: Date.now()
+        }
+      );
+
+      if (!memoryResult.success) {
+        return {
+          success: false,
+          data: null,
+          error: `Failed to store data: ${memoryResult.error}`,
+          stageResults: { ...result.stageResults, memory: memoryResult }
+        };
+      }
+
+      result.stageResults.memory = memoryResult;
+
+      // Step 3: Trigger ICognition augmentations to analyze the data
+      const cognitionResults = await augmentationPipeline.executeCognitionPipeline(
+        'reason',
+        [`Analyze data with key ${dataKey}`, { dataKey }],
+        { timeout: opts.timeout, stopOnError: opts.stopOnError }
+      );
+
+      // Get the first successful result
+      let cognitionResult: AugmentationResponse<{ inference: string, confidence: number }> | null = null;
+      for (const resultPromise of cognitionResults) {
+        const res = await resultPromise;
+        if (res.success) {
+          cognitionResult = res;
+          break;
+        }
+      }
+
+      if (cognitionResult) {
+        result.stageResults.cognition = cognitionResult;
+      }
+
+      // Step 4: Send notifications to IConduit augmentations
+      const conduitResults = await augmentationPipeline.executeConduitPipeline(
+        'writeData',
+        [{ dataKey, nouns: senseResult.data.nouns, verbs: senseResult.data.verbs }],
+        { timeout: opts.timeout, stopOnError: opts.stopOnError }
+      );
+
+      // Get the first successful result
+      let conduitResult: AugmentationResponse<unknown> | null = null;
+      for (const resultPromise of conduitResults) {
+        const res = await resultPromise;
+        if (res.success) {
+          conduitResult = res;
+          break;
+        }
+      }
+
+      if (conduitResult) {
+        result.stageResults.conduit = conduitResult;
+      }
+
+      // Step 5: Send notifications to IActivation augmentations
+      const activationResults = await augmentationPipeline.executeActivationPipeline(
+        'triggerAction',
+        ['dataProcessed', { dataKey }],
+        { timeout: opts.timeout, stopOnError: opts.stopOnError }
+      );
+
+      // Get the first successful result
+      let activationResult: AugmentationResponse<unknown> | null = null;
+      for (const resultPromise of activationResults) {
+        const res = await resultPromise;
+        if (res.success) {
+          activationResult = res;
+          break;
+        }
+      }
+
+      if (activationResult) {
+        result.stageResults.activation = activationResult;
+      }
+
+      // Step 6: Send notifications to IPerception augmentations
+      const perceptionResults = await augmentationPipeline.executePerceptionPipeline(
+        'interpret',
+        [senseResult.data.nouns, senseResult.data.verbs, { dataKey }],
+        { timeout: opts.timeout, stopOnError: opts.stopOnError }
+      );
+
+      // Get the first successful result
+      let perceptionResult: AugmentationResponse<Record<string, unknown>> | null = null;
+      for (const resultPromise of perceptionResults) {
+        const res = await resultPromise;
+        if (res.success) {
+          perceptionResult = res;
+          break;
+        }
+      }
+
+      if (perceptionResult) {
+        result.stageResults.perception = perceptionResult;
+        result.data = perceptionResult.data;
+      } else {
+        // If no perception result, use the cognition result as the final data
+        result.data = cognitionResult ? cognitionResult.data : { dataKey };
+      }
+
+      return result;
+    } catch (error: unknown) {
+      return {
+        success: false,
+        data: null,
+        error: `Pipeline execution failed: ${error}`,
+        stageResults: result.stageResults
+      };
+    }
+  }
+
+  /**
+   * Process WebSocket data through the sequential pipeline
+   * 
+   * @param connection The WebSocket connection
+   * @param dataType The type of data (e.g., 'text', 'image', 'audio')
+   * @param options Options for pipeline execution
+   * @returns A function to handle incoming WebSocket messages
+   */
+  public async createWebSocketHandler(
+    connection: WebSocketConnection,
+    dataType: string,
+    options: SequentialPipelineOptions = {}
+  ): Promise<(data: unknown) => void> {
+    // Ensure stream classes are initialized
+    await this.ensureStreamClassesInitialized();
+
+    // Create a transform stream for processing data
+    const transformStream = new TransformStream({
+      transform: async (chunk: unknown, controller: TransformStreamDefaultController) => {
+        try {
+          const data = typeof chunk === 'string' ? chunk : JSON.stringify(chunk);
+          const result = await this.processData(data, dataType, options);
+          if (result.success) {
+            controller.enqueue(result);
+          } else {
+            console.warn('Pipeline processing failed:', result.error);
+          }
+        } catch (error: unknown) {
+          console.error('Error in transform stream:', error);
+        }
+      }
+    });
+
+    // Create a writable stream that will be the sink for our data
+    const writableStream = new WritableStream({
+      write: async (result: PipelineResult<unknown>) => {
+        // Handle the processed result if needed
+        if (connection.send && typeof connection.send === 'function') {
+          try {
+            // Only send back results if the connection supports it
+            await connection.send(JSON.stringify(result));
+          } catch (error: unknown) {
+            console.error('Error sending result back to WebSocket:', error);
+          }
+        }
+      }
+    });
+
+    // Connect the transform stream to the writable stream
+    transformStream.readable.pipeTo(writableStream).catch((error: Error) => {
+      console.error('Error in pipeline stream:', error);
+    });
+
+    // Return a function that writes to the transform stream
+    return (data: unknown) => {
+      try {
+        // Write to the transform stream's writable side
+        const writer = transformStream.writable.getWriter();
+        writer.write(data).catch((error: Error) => {
+          console.error('Error writing to stream:', error);
+        }).finally(() => {
+          writer.releaseLock();
+        });
+      } catch (error: unknown) {
+        console.error('Error getting writer for transform stream:', error);
+      }
+    };
+  }
+
+  /**
+   * Set up a WebSocket connection to process data through the pipeline
+   * 
+   * @param url The WebSocket URL to connect to
+   * @param dataType The type of data (e.g., 'text', 'image', 'audio')
+   * @param options Options for pipeline execution
+   * @returns A promise that resolves with the WebSocket connection and associated streams
+   */
+  public async setupWebSocketPipeline(
+    url: string,
+    dataType: string,
+    options: SequentialPipelineOptions = {}
+  ): Promise<WebSocketConnection & { 
+    readableStream?: ReadableStream<unknown>,
+    writableStream?: WritableStream<unknown>
+  }> {
+    // Ensure stream classes are initialized
+    await this.ensureStreamClassesInitialized();
+
+    // Get WebSocket-supporting augmentations
+    const webSocketAugmentations = augmentationPipeline.getWebSocketAugmentations();
+
+    if (webSocketAugmentations.length === 0) {
+      throw new Error('No WebSocket-supporting augmentations available');
+    }
+
+    // Use the first available WebSocket augmentation
+    const webSocketAugmentation = webSocketAugmentations[0];
+
+    // Connect to the WebSocket
+    const connection = await webSocketAugmentation.connectWebSocket(url);
+
+    // Create a readable stream from the WebSocket messages
+    const readableStream = new ReadableStream({
+      start: (controller: ReadableStreamDefaultController) => {
+        // Define a message handler that writes to the stream
+        const messageHandler = (event: { data: unknown }) => {
+          try {
+            const data = typeof event.data === 'string' 
+              ? event.data 
+              : event.data instanceof Blob 
+                ? new Promise(resolve => {
+                    const reader = new FileReader();
+                    reader.onload = () => resolve(reader.result as string);
+                    reader.readAsText(event.data as Blob);
+                  })
+                : JSON.stringify(event.data);
+
+            // Handle both string data and promises
+            if (data instanceof Promise) {
+              data.then(resolvedData => {
+                controller.enqueue(resolvedData);
+              }).catch((error: Error) => {
+                console.error('Error processing blob data:', error);
+              });
+            } else {
+              controller.enqueue(data);
+            }
+          } catch (error: unknown) {
+            console.error('Error processing WebSocket message:', error);
+          }
+        };
+
+        // Create a wrapper function that adapts the event-based handler to the data-based callback
+        const messageHandlerWrapper = (data: unknown) => {
+          messageHandler({ data });
+        };
+
+        // Store both handlers for later cleanup
+        connection._streamMessageHandler = messageHandler;
+        connection._messageHandlerWrapper = messageHandlerWrapper;
+
+        webSocketAugmentation.onWebSocketMessage(
+          connection.connectionId, 
+          messageHandlerWrapper
+        ).catch((error: Error) => {
+          console.error('Error registering WebSocket message handler:', error);
+          controller.error(error);
+        });
+      },
+      cancel: () => {
+        // Clean up the message handler when the stream is cancelled
+        if (connection._messageHandlerWrapper) {
+          webSocketAugmentation.offWebSocketMessage(
+            connection.connectionId, 
+            connection._messageHandlerWrapper
+          ).catch((error: Error) => {
+            console.error('Error removing WebSocket message handler:', error);
+          });
+          delete connection._streamMessageHandler;
+          delete connection._messageHandlerWrapper;
+        }
+      }
+    });
+
+    // Create a handler for processing the data
+    const handlerPromise = this.createWebSocketHandler(connection, dataType, options);
+
+    // Create a writable stream that sends data to the WebSocket
+    const writableStream = new WritableStream({
+      write: async (chunk: unknown) => {
+        if (connection.send && typeof connection.send === 'function') {
+          try {
+            const data = typeof chunk === 'string' ? chunk : JSON.stringify(chunk);
+            await connection.send(data);
+          } catch (error: unknown) {
+            console.error('Error sending data to WebSocket:', error);
+            throw error;
+          }
+        } else {
+          throw new Error('WebSocket connection does not support sending data');
+        }
+      },
+      close: () => {
+        // Close the WebSocket connection when the stream is closed
+        if (connection.close && typeof connection.close === 'function') {
+          connection.close().catch((error: Error) => {
+            console.error('Error closing WebSocket connection:', error);
+          });
+        }
+      }
+    });
+
+    // Pipe the readable stream through our processing pipeline
+    readableStream
+      .pipeThrough(new TransformStream({
+        transform: async (chunk: unknown, controller: TransformStreamDefaultController) => {
+          // Process each chunk through our handler
+          const handler = await handlerPromise;
+          handler(chunk);
+          // Pass through the original data
+          controller.enqueue(chunk);
+        }
+      }))
+      .pipeTo(new WritableStream({
+        write: () => {},
+        abort: (error: Error) => {
+          console.error('Error in WebSocket pipeline:', error);
+        }
+      }));
+
+    // Attach the streams to the connection object for convenience
+    const enhancedConnection = connection as WebSocketConnection & {
+      readableStream: ReadableStream<unknown>,
+      writableStream: WritableStream<unknown>
+    };
+
+    enhancedConnection.readableStream = readableStream;
+    enhancedConnection.writableStream = writableStream;
+
+    return enhancedConnection;
+  }
+}
+
+// Create and export a default instance of the sequential pipeline
+export const sequentialPipeline = new SequentialPipeline();

src/storage/fileSystemStorage.ts

@@ -0,0 +1,931 @@
+import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js'
+
+// We'll dynamically import Node.js built-in modules
+let fs: any
+let path: any
+
+// Constants for directory and file names
+const ROOT_DIR = 'brainy-data'
+const NOUNS_DIR = 'nouns'
+const VERBS_DIR = 'verbs'
+const METADATA_DIR = 'metadata'
+
+// Constants for noun type directories
+const PERSON_DIR = 'person'
+const PLACE_DIR = 'place'
+const THING_DIR = 'thing'
+const EVENT_DIR = 'event'
+const CONCEPT_DIR = 'concept'
+const CONTENT_DIR = 'content'
+const DEFAULT_DIR = 'default' // For nodes without a noun type
+
+/**
+ * File system storage adapter for Node.js environments
+ */
+export class FileSystemStorage implements StorageAdapter {
+  private rootDir: string
+  private nounsDir: string
+  private verbsDir: string
+  private metadataDir: string
+  private personDir: string
+  private placeDir: string
+  private thingDir: string
+  private eventDir: string
+  private conceptDir: string
+  private contentDir: string
+  private defaultDir: string
+  private isInitialized = false
+
+  constructor(rootDirectory?: string) {
+    // We'll set the paths in the init method after dynamically importing the modules
+    this.rootDir = rootDirectory || ''
+    this.nounsDir = ''
+    this.verbsDir = ''
+    this.metadataDir = ''
+    this.personDir = ''
+    this.placeDir = ''
+    this.thingDir = ''
+    this.eventDir = ''
+    this.conceptDir = ''
+    this.contentDir = ''
+    this.defaultDir = ''
+  }
+
+  /**
+   * Initialize the storage adapter
+   */
+  public async init(): Promise<void> {
+    if (this.isInitialized) {
+      return
+    }
+
+    try {
+      // Dynamically import Node.js built-in modules
+      try {
+        // Import the modules
+        const fsModule = await import('fs')
+        const pathModule = await import('path')
+
+        // Assign to our module-level variables
+        fs = fsModule.default || fsModule
+        path = pathModule.default || pathModule
+
+        // Now set up the directory paths
+        const rootDir = this.rootDir || process.cwd()
+        this.rootDir = path.resolve(rootDir, ROOT_DIR)
+        this.nounsDir = path.join(this.rootDir, NOUNS_DIR)
+        this.verbsDir = path.join(this.rootDir, VERBS_DIR)
+        this.metadataDir = path.join(this.rootDir, METADATA_DIR)
+
+        // Set up noun type directory paths
+        this.personDir = path.join(this.nounsDir, PERSON_DIR)
+        this.placeDir = path.join(this.nounsDir, PLACE_DIR)
+        this.thingDir = path.join(this.nounsDir, THING_DIR)
+        this.eventDir = path.join(this.nounsDir, EVENT_DIR)
+        this.conceptDir = path.join(this.nounsDir, CONCEPT_DIR)
+        this.contentDir = path.join(this.nounsDir, CONTENT_DIR)
+        this.defaultDir = path.join(this.nounsDir, DEFAULT_DIR)
+      } catch (importError) {
+        throw new Error(`Failed to import Node.js modules: ${importError}. This adapter requires a Node.js environment.`)
+      }
+
+      // Create directories if they don't exist
+      await this.ensureDirectoryExists(this.rootDir)
+      await this.ensureDirectoryExists(this.nounsDir)
+      await this.ensureDirectoryExists(this.verbsDir)
+      await this.ensureDirectoryExists(this.metadataDir)
+
+      // Create noun type directories
+      await this.ensureDirectoryExists(this.personDir)
+      await this.ensureDirectoryExists(this.placeDir)
+      await this.ensureDirectoryExists(this.thingDir)
+      await this.ensureDirectoryExists(this.eventDir)
+      await this.ensureDirectoryExists(this.conceptDir)
+      await this.ensureDirectoryExists(this.contentDir)
+      await this.ensureDirectoryExists(this.defaultDir)
+
+      this.isInitialized = true
+    } catch (error) {
+      console.error('Failed to initialize file system storage:', error)
+      throw new Error(`Failed to initialize file system storage: ${error}`)
+    }
+  }
+
+  /**
+   * Save a node to storage
+   */
+  public async saveNoun(noun: HNSWNoun): Promise<void> {
+    await this.ensureInitialized()
+
+    try {
+      // Convert connections Map to a serializable format
+      const serializableNode = {
+        ...noun,
+        connections: this.mapToObject(noun.connections, (set) => Array.from(set as Set<string>))
+      }
+
+      // Get the appropriate directory based on the node's metadata
+      const nodeDir = await this.getNodeDirectory(noun.id)
+
+      const filePath = path.join(nodeDir, `${noun.id}.json`)
+      await fs.promises.writeFile(
+        filePath,
+        JSON.stringify(serializableNode, null, 2),
+        'utf8'
+      )
+    } catch (error) {
+      console.error(`Failed to save node ${noun.id}:`, error)
+      throw new Error(`Failed to save node ${noun.id}: ${error}`)
+    }
+  }
+
+  /**
+   * Get a node from storage
+   */
+  public async getNoun(id: string): Promise<HNSWNoun | null> {
+    await this.ensureInitialized()
+
+    try {
+      // Get the appropriate directory based on the node's metadata
+      const nodeDir = await this.getNodeDirectory(id)
+
+      const filePath = path.join(nodeDir, `${id}.json`)
+
+      // Check if a file exists
+      try {
+        await fs.promises.access(filePath)
+      } catch {
+        // If the file doesn't exist in the expected directory, try the default directory
+        if (nodeDir !== this.defaultDir) {
+          const defaultFilePath = path.join(this.defaultDir, `${id}.json`)
+          try {
+            await fs.promises.access(defaultFilePath)
+            // If found in default directory, use that path
+            const data = await fs.promises.readFile(defaultFilePath, 'utf8')
+            const parsedNode = JSON.parse(data)
+
+            // Convert serialized connections back to Map<number, Set<string>>
+            const connections = new Map<number, Set<string>>()
+            for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
+              connections.set(Number(level), new Set(nodeIds as string[]))
+            }
+
+            return {
+              id: parsedNode.id,
+              vector: parsedNode.vector,
+              connections
+            }
+          } catch {
+            // If not found in default directory either, try all noun type directories
+            const directories = [
+              this.personDir,
+              this.placeDir,
+              this.thingDir,
+              this.eventDir,
+              this.conceptDir,
+              this.contentDir
+            ]
+
+            for (const dir of directories) {
+              if (dir === nodeDir) continue // Skip the already checked directory
+
+              const dirFilePath = path.join(dir, `${id}.json`)
+              try {
+                await fs.promises.access(dirFilePath)
+                // If found in this directory, use that path
+                const data = await fs.promises.readFile(dirFilePath, 'utf8')
+                const parsedNode = JSON.parse(data)
+
+                // Convert serialized connections back to Map<number, Set<string>>
+                const connections = new Map<number, Set<string>>()
+                for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
+                  connections.set(Number(level), new Set(nodeIds as string[]))
+                }
+
+                return {
+                  id: parsedNode.id,
+                  vector: parsedNode.vector,
+                  connections
+                }
+              } catch {
+                // Continue to the next directory
+              }
+            }
+
+            return null // File doesn't exist in any directory
+          }
+        }
+        return null // File doesn't exist
+      }
+
+      const data = await fs.promises.readFile(filePath, 'utf8')
+      const parsedNode = JSON.parse(data)
+
+      // Convert serialized connections back to Map<number, Set<string>>
+      const connections = new Map<number, Set<string>>()
+      for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
+        connections.set(Number(level), new Set(nodeIds as string[]))
+      }
+
+      return {
+        id: parsedNode.id,
+        vector: parsedNode.vector,
+        connections
+      }
+    } catch (error) {
+      console.error(`Failed to get node ${id}:`, error)
+      return null
+    }
+  }
+
+  /**
+   * Get nodes by noun type
+   * @param nounType The noun type to filter by
+   * @returns Promise that resolves to an array of nodes of the specified noun type
+   */
+  public async getNounsByNounType(nounType: string): Promise<HNSWNoun[]> {
+    await this.ensureInitialized()
+
+    try {
+      // Determine the directory based on the noun type
+      let dir: string
+      switch (nounType) {
+        case 'person':
+          dir = this.personDir
+          break
+        case 'place':
+          dir = this.placeDir
+          break
+        case 'thing':
+          dir = this.thingDir
+          break
+        case 'event':
+          dir = this.eventDir
+          break
+        case 'concept':
+          dir = this.conceptDir
+          break
+        case 'content':
+          dir = this.contentDir
+          break
+        default:
+          dir = this.defaultDir
+      }
+
+      const nodes: HNSWNoun[] = []
+
+      try {
+        const files = await fs.promises.readdir(dir)
+        const nodePromises = files
+          .filter((file: string) => file.endsWith('.json'))
+          .map((file: string) => {
+            // Use the file path directly instead of getNode to avoid redundant searches
+            return this.readNodeFromFile(path.join(dir, file))
+          })
+
+        const dirNodes = await Promise.all(nodePromises)
+        nodes.push(...dirNodes.filter((node): node is HNSWNoun => node !== null))
+      } catch (dirError) {
+        // If directory doesn't exist or can't be read, log a warning
+        console.warn(`Could not read directory for noun type ${nounType}:`, dirError)
+      }
+
+      return nodes
+    } catch (error) {
+      console.error(`Failed to get nodes for noun type ${nounType}:`, error)
+      throw new Error(`Failed to get nodes for noun type ${nounType}: ${error}`)
+    }
+  }
+
+  /**
+   * Get all nodes from storage
+   */
+  public async getAllNouns(): Promise<HNSWNoun[]> {
+    await this.ensureInitialized()
+
+    try {
+      // Get all noun types
+      const nounTypes = [
+        'person',
+        'place',
+        'thing',
+        'event',
+        'concept',
+        'content',
+        'default'
+      ]
+
+      // Run searches in parallel for all noun types
+      const nodePromises = nounTypes.map(nounType => this.getNounsByNounType(nounType))
+      const nodeArrays = await Promise.all(nodePromises)
+
+      // Combine all results
+      const allNodes: HNSWNoun[] = []
+      for (const nodes of nodeArrays) {
+        allNodes.push(...nodes)
+      }
+
+      return allNodes
+    } catch (error) {
+      console.error('Failed to get all nodes:', error)
+      throw new Error(`Failed to get all nodes: ${error}`)
+    }
+  }
+
+  /**
+   * Read a node from a file
+   */
+  private async readNodeFromFile(filePath: string): Promise<HNSWNoun | null> {
+    try {
+      const data = await fs.promises.readFile(filePath, 'utf8')
+      const parsedNode = JSON.parse(data)
+
+      // Convert serialized connections back to Map<number, Set<string>>
+      const connections = new Map<number, Set<string>>()
+      for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
+        connections.set(Number(level), new Set(nodeIds as string[]))
+      }
+
+      return {
+        id: parsedNode.id,
+        vector: parsedNode.vector,
+        connections
+      }
+    } catch (error) {
+      console.error(`Failed to read node from file ${filePath}:`, error)
+      return null
+    }
+  }
+
+  /**
+   * Delete a node from storage
+   */
+  public async deleteNoun(id: string): Promise<void> {
+    await this.ensureInitialized()
+
+    try {
+      // Get the appropriate directory based on the node's metadata
+      const nodeDir = await this.getNodeDirectory(id)
+
+      const filePath = path.join(nodeDir, `${id}.json`)
+
+      // Check if a file exists before attempting to delete
+      try {
+        await fs.promises.access(filePath)
+        await fs.promises.unlink(filePath)
+        return // File found and deleted
+      } catch {
+        // If the file doesn't exist in the expected directory, try the default directory
+        if (nodeDir !== this.defaultDir) {
+          const defaultFilePath = path.join(this.defaultDir, `${id}.json`)
+          try {
+            await fs.promises.access(defaultFilePath)
+            await fs.promises.unlink(defaultFilePath)
+            return // File found and deleted
+          } catch {
+            // If not found in default directory either, try all noun type directories
+            const directories = [
+              this.personDir,
+              this.placeDir,
+              this.thingDir,
+              this.eventDir,
+              this.conceptDir,
+              this.contentDir
+            ]
+
+            for (const dir of directories) {
+              if (dir === nodeDir) continue // Skip the already checked directory
+
+              const dirFilePath = path.join(dir, `${id}.json`)
+              try {
+                await fs.promises.access(dirFilePath)
+                await fs.promises.unlink(dirFilePath)
+                return // File found and deleted
+              } catch {
+                // Continue to the next directory
+              }
+            }
+
+            return // File doesn't exist in any directory, nothing to delete
+          }
+        }
+        return // File doesn't exist, nothing to delete
+      }
+    } catch (error) {
+      console.error(`Failed to delete node ${id}:`, error)
+      throw new Error(`Failed to delete node ${id}: ${error}`)
+    }
+  }
+
+  /**
+   * Save an edge to storage
+   */
+  public async saveVerb(verb: GraphVerb): Promise<void> {
+    await this.ensureInitialized()
+
+    try {
+      // Convert connections Map to a serializable format
+      const serializableEdge = {
+        ...verb,
+        connections: this.mapToObject(verb.connections, (set) => Array.from(set as Set<string>))
+      }
+
+      const filePath = path.join(this.verbsDir, `${verb.id}.json`)
+      await fs.promises.writeFile(
+        filePath,
+        JSON.stringify(serializableEdge, null, 2),
+        'utf8'
+      )
+    } catch (error) {
+      console.error(`Failed to save edge ${verb.id}:`, error)
+      throw new Error(`Failed to save edge ${verb.id}: ${error}`)
+    }
+  }
+
+  /**
+   * Get an edge from storage
+   */
+  public async getVerb(id: string): Promise<GraphVerb | null> {
+    await this.ensureInitialized()
+
+    try {
+      const filePath = path.join(this.verbsDir, `${id}.json`)
+
+      // Check if a file exists
+      try {
+        await fs.promises.access(filePath)
+      } catch {
+        return null // File doesn't exist
+      }
+
+      const data = await fs.promises.readFile(filePath, 'utf8')
+      const parsedEdge = JSON.parse(data)
+
+      // Convert serialized connections back to Map<number, Set<string>>
+      const connections = new Map<number, Set<string>>()
+      for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
+        connections.set(Number(level), new Set(nodeIds as string[]))
+      }
+
+      return {
+        id: parsedEdge.id,
+        vector: parsedEdge.vector,
+        connections,
+        sourceId: parsedEdge.sourceId,
+        targetId: parsedEdge.targetId,
+        type: parsedEdge.type,
+        weight: parsedEdge.weight,
+        metadata: parsedEdge.metadata
+      }
+    } catch (error) {
+      console.error(`Failed to get edge ${id}:`, error)
+      return null
+    }
+  }
+
+  /**
+   * Get all edges from storage
+   */
+  public async getAllVerbs(): Promise<GraphVerb[]> {
+    await this.ensureInitialized()
+
+    try {
+      const files = await fs.promises.readdir(this.verbsDir)
+      const edgePromises = files
+        .filter((file: string) => file.endsWith('.json'))
+        .map((file: string) => {
+          const id = path.basename(file, '.json')
+          return this.getVerb(id)
+        })
+
+      const edges = await Promise.all(edgePromises)
+      return edges.filter((edge): edge is GraphVerb => edge !== null)
+    } catch (error) {
+      console.error('Failed to get all edges:', error)
+      throw new Error(`Failed to get all edges: ${error}`)
+    }
+  }
+
+  /**
+   * Get edges by source node ID
+   */
+  public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> {
+    await this.ensureInitialized()
+
+    try {
+      const allEdges = await this.getAllVerbs()
+      return allEdges.filter(edge => edge.sourceId === sourceId)
+    } catch (error) {
+      console.error(`Failed to get edges by source ${sourceId}:`, error)
+      throw new Error(`Failed to get edges by source ${sourceId}: ${error}`)
+    }
+  }
+
+  /**
+   * Get edges by target node ID
+   */
+  public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> {
+    await this.ensureInitialized()
+
+    try {
+      const allEdges = await this.getAllVerbs()
+      return allEdges.filter(edge => edge.targetId === targetId)
+    } catch (error) {
+      console.error(`Failed to get edges by target ${targetId}:`, error)
+      throw new Error(`Failed to get edges by target ${targetId}: ${error}`)
+    }
+  }
+
+  /**
+   * Get edges by type
+   */
+  public async getVerbsByType(type: string): Promise<GraphVerb[]> {
+    await this.ensureInitialized()
+
+    try {
+      const allEdges = await this.getAllVerbs()
+      return allEdges.filter(edge => edge.type === type)
+    } catch (error) {
+      console.error(`Failed to get edges by type ${type}:`, error)
+      throw new Error(`Failed to get edges by type ${type}: ${error}`)
+    }
+  }
+
+  /**
+   * Delete an edge from storage
+   */
+  public async deleteVerb(id: string): Promise<void> {
+    await this.ensureInitialized()
+
+    try {
+      const filePath = path.join(this.verbsDir, `${id}.json`)
+
+      // Check if a file exists before attempting to delete
+      try {
+        await fs.promises.access(filePath)
+      } catch {
+        return // File doesn't exist, nothing to delete
+      }
+
+      await fs.promises.unlink(filePath)
+    } catch (error) {
+      console.error(`Failed to delete edge ${id}:`, error)
+      throw new Error(`Failed to delete edge ${id}: ${error}`)
+    }
+  }
+
+  /**
+   * Save metadata to storage
+   */
+  public async saveMetadata(id: string, metadata: any): Promise<void> {
+    await this.ensureInitialized()
+
+    try {
+      const filePath = path.join(this.metadataDir, `${id}.json`)
+      await fs.promises.writeFile(
+        filePath,
+        JSON.stringify(metadata, null, 2),
+        'utf8'
+      )
+    } catch (error) {
+      console.error(`Failed to save metadata for ${id}:`, error)
+      throw new Error(`Failed to save metadata for ${id}: ${error}`)
+    }
+  }
+
+  /**
+   * Get metadata from storage
+   */
+  public async getMetadata(id: string): Promise<any | null> {
+    await this.ensureInitialized()
+
+    try {
+      const filePath = path.join(this.metadataDir, `${id}.json`)
+
+      // Check if a file exists
+      try {
+        await fs.promises.access(filePath)
+      } catch {
+        return null // File doesn't exist
+      }
+
+      const data = await fs.promises.readFile(filePath, 'utf8')
+      return JSON.parse(data)
+    } catch (error) {
+      console.error(`Failed to get metadata for ${id}:`, error)
+      return null
+    }
+  }
+
+  /**
+   * Clear all data from storage
+   */
+  public async clear(): Promise<void> {
+    await this.ensureInitialized()
+
+    try {
+      // Delete and recreate the nodes, edges, and metadata directories
+      await this.deleteDirectory(this.nounsDir)
+      await this.deleteDirectory(this.verbsDir)
+      await this.deleteDirectory(this.metadataDir)
+
+      await this.ensureDirectoryExists(this.nounsDir)
+      await this.ensureDirectoryExists(this.verbsDir)
+      await this.ensureDirectoryExists(this.metadataDir)
+
+      // Create noun type directories
+      await this.ensureDirectoryExists(this.personDir)
+      await this.ensureDirectoryExists(this.placeDir)
+      await this.ensureDirectoryExists(this.thingDir)
+      await this.ensureDirectoryExists(this.eventDir)
+      await this.ensureDirectoryExists(this.conceptDir)
+      await this.ensureDirectoryExists(this.contentDir)
+      await this.ensureDirectoryExists(this.defaultDir)
+    } catch (error) {
+      console.error('Failed to clear storage:', error)
+      throw new Error(`Failed to clear storage: ${error}`)
+    }
+  }
+
+  /**
+   * Ensure the storage adapter is initialized
+   */
+  private async ensureInitialized(): Promise<void> {
+    if (!this.isInitialized) {
+      await this.init()
+    }
+  }
+
+  /**
+   * Ensure a directory exists, creating it if necessary
+   */
+  private async ensureDirectoryExists(dirPath: string): Promise<void> {
+    try {
+      await fs.promises.access(dirPath)
+    } catch {
+      // Directory doesn't exist, create it
+      await fs.promises.mkdir(dirPath, { recursive: true })
+    }
+  }
+
+  /**
+   * Delete a directory and all its contents recursively
+   */
+  private async deleteDirectory(dirPath: string): Promise<void> {
+    try {
+      const files = await fs.promises.readdir(dirPath)
+
+      for (const file of files) {
+        const filePath = path.join(dirPath, file)
+        const stats = await fs.promises.stat(filePath)
+
+        if (stats.isDirectory()) {
+          // Recursively delete subdirectories
+          await this.deleteDirectory(filePath)
+        } else {
+          // Delete files
+          await fs.promises.unlink(filePath)
+        }
+      }
+
+      // After all contents are deleted, remove the directory itself
+      await fs.promises.rmdir(dirPath)
+    } catch (error) {
+      // If the directory doesn't exist, that's fine
+      if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
+        throw error
+      }
+    }
+  }
+
+  /**
+   * Count the number of JSON files in a directory
+   */
+  private async countFilesInDirectory(dirPath: string): Promise<number> {
+    try {
+      const files = await fs.promises.readdir(dirPath)
+      return files.filter((file: string) => file.endsWith('.json')).length
+    } catch (error) {
+      // If the directory doesn't exist, return 0
+      if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+        return 0
+      }
+      throw error
+    }
+  }
+
+  /**
+   * Convert a Map to a plain object for serialization
+   */
+  private mapToObject<K extends string | number, V>(
+    map: Map<K, V>,
+    valueTransformer: (value: V) => any = (v) => v
+  ): Record<string, any> {
+    const obj: Record<string, any> = {}
+    for (const [key, value] of map.entries()) {
+      obj[key.toString()] = valueTransformer(value)
+    }
+    return obj
+  }
+
+  /**
+   * Get the appropriate directory for a node based on its metadata
+   */
+  private async getNodeDirectory(id: string): Promise<string> {
+    try {
+      // Try to get the metadata for the node
+      const metadata = await this.getMetadata(id)
+
+      // If metadata exists and has a noun field, use the corresponding directory
+      if (metadata && metadata.noun) {
+        switch (metadata.noun) {
+          case 'person':
+            return this.personDir
+          case 'place':
+            return this.placeDir
+          case 'thing':
+            return this.thingDir
+          case 'event':
+            return this.eventDir
+          case 'concept':
+            return this.conceptDir
+          case 'content':
+            return this.contentDir
+          default:
+            return this.defaultDir
+        }
+      }
+
+      // If no metadata or no noun field, use the default directory
+      return this.defaultDir
+    } catch (error) {
+      // If there's an error getting the metadata, use the default directory
+      return this.defaultDir
+    }
+  }
+
+  /**
+   * Get information about storage usage and capacity
+   */
+  public async getStorageStatus(): Promise<{
+    type: string;
+    used: number;
+    quota: number | null;
+    details?: Record<string, any>;
+  }> {
+    await this.ensureInitialized()
+
+    try {
+      // Calculate the total size of all files in the storage directories
+      let totalSize = 0
+
+      // Helper function to calculate directory size
+      const calculateDirSize = async (dirPath: string): Promise<number> => {
+        let size = 0
+        try {
+          const files = await fs.promises.readdir(dirPath)
+
+          for (const file of files) {
+            const filePath = path.join(dirPath, file)
+            const stats = await fs.promises.stat(filePath)
+
+            if (stats.isDirectory()) {
+              size += await calculateDirSize(filePath)
+            } else {
+              size += stats.size
+            }
+          }
+        } catch (error) {
+          console.warn(`Error calculating size for ${dirPath}:`, error)
+        }
+        return size
+      }
+
+      // Calculate size for each directory
+      const nodesDirSize = await calculateDirSize(this.nounsDir)
+      const edgesDirSize = await calculateDirSize(this.verbsDir)
+      const metadataDirSize = await calculateDirSize(this.metadataDir)
+
+      // Calculate sizes of noun type directories
+      const personDirSize = await calculateDirSize(this.personDir)
+      const placeDirSize = await calculateDirSize(this.placeDir)
+      const thingDirSize = await calculateDirSize(this.thingDir)
+      const eventDirSize = await calculateDirSize(this.eventDir)
+      const conceptDirSize = await calculateDirSize(this.conceptDir)
+      const contentDirSize = await calculateDirSize(this.contentDir)
+      const defaultDirSize = await calculateDirSize(this.defaultDir)
+
+      // Note: The noun type directories are subdirectories of the nodes directory,
+      // so their sizes are already included in nodesDirSize.
+      // We don't need to add them again to avoid double counting.
+      totalSize = nodesDirSize + edgesDirSize + metadataDirSize
+
+      // Get filesystem information
+      let quota = null
+      let details: {
+        nounTypes?: {
+          person: { size: number; count: number };
+          place: { size: number; count: number };
+          thing: { size: number; count: number };
+          event: { size: number; count: number };
+          concept: { size: number; count: number };
+          content: { size: number; count: number };
+          default: { size: number; count: number };
+        };
+        availableSpace?: number;
+        totalSpace?: number;
+        freePercentage?: number;
+      } = {
+        nounTypes: {
+          person: {
+            size: personDirSize,
+            count: await this.countFilesInDirectory(this.personDir)
+          },
+          place: {
+            size: placeDirSize,
+            count: await this.countFilesInDirectory(this.placeDir)
+          },
+          thing: {
+            size: thingDirSize,
+            count: await this.countFilesInDirectory(this.thingDir)
+          },
+          event: {
+            size: eventDirSize,
+            count: await this.countFilesInDirectory(this.eventDir)
+          },
+          concept: {
+            size: conceptDirSize,
+            count: await this.countFilesInDirectory(this.conceptDir)
+          },
+          content: {
+            size: contentDirSize,
+            count: await this.countFilesInDirectory(this.contentDir)
+          },
+          default: {
+            size: defaultDirSize,
+            count: await this.countFilesInDirectory(this.defaultDir)
+          }
+        }
+      }
+
+      try {
+        // Try to get disk space information
+        const stats = await fs.promises.statfs(this.rootDir)
+        if (stats) {
+          const availableSpace = stats.bavail * stats.bsize
+          const totalSpace = stats.blocks * stats.bsize
+
+          quota = totalSpace
+          details = {
+            availableSpace,
+            totalSpace,
+            freePercentage: (availableSpace / totalSpace) * 100
+          }
+        }
+      } catch (error) {
+        console.warn('Unable to get filesystem stats:', error)
+        // If statfs is not available, try to use df command on Unix-like systems
+        try {
+          const { exec } = await import('child_process')
+          const util = await import('util')
+          const execPromise = util.promisify(exec)
+
+          const { stdout } = await execPromise(`df -k "${this.rootDir}"`)
+          const lines = stdout.trim().split('\n')
+          if (lines.length > 1) {
+            const parts = lines[1].split(/\s+/)
+            if (parts.length >= 4) {
+              const totalKB = parseInt(parts[1], 10)
+              const usedKB = parseInt(parts[2], 10)
+              const availableKB = parseInt(parts[3], 10)
+
+              quota = totalKB * 1024
+              details = {
+                availableSpace: availableKB * 1024,
+                totalSpace: totalKB * 1024,
+                freePercentage: (availableKB / totalKB) * 100
+              }
+            }
+          }
+        } catch (dfError) {
+          console.warn('Unable to get disk space using df command:', dfError)
+        }
+      }
+
+      return {
+        type: 'filesystem',
+        used: totalSize,
+        quota,
+        details
+      }
+    } catch (error) {
+      console.error('Failed to get storage status:', error)
+      return {
+        type: 'filesystem',
+        used: 0,
+        quota: null,
+        details: { error: String(error) }
+      }
+    }
+  }
+}

src/types/augmentations.ts

@@ -0,0 +1,413 @@
+/** Common types for the augmentation system */
+
+/**
+ * Enum representing all types of augmentations available in the Brainy system.
+ */
+export enum AugmentationType {
+  SENSE = 'sense',
+  CONDUIT = 'conduit',
+  COGNITION = 'cognition',
+  MEMORY = 'memory',
+  PERCEPTION = 'perception',
+  DIALOG = 'dialog',
+  ACTIVATION = 'activation',
+  WEBSOCKET = 'webSocket'
+}
+
+export type WebSocketConnection = {
+  connectionId: string
+  url: string
+  status: 'connected' | 'disconnected' | 'error'
+  send?: (data: string | ArrayBufferLike | Blob | ArrayBufferView) => Promise<void>
+  close?: () => Promise<void>
+  _streamMessageHandler?: (event: { data: unknown }) => void
+  _messageHandlerWrapper?: (data: unknown) => void
+}
+
+type DataCallback<T> = (data: T) => void
+export type AugmentationResponse<T> = {
+  success: boolean
+  data: T
+  error?: string
+}
+
+/**
+ * Base interface for all Brainy augmentations.
+ * All augmentations must implement these core properties.
+ */
+export interface IAugmentation {
+  /** A unique identifier for the augmentation (e.g., "my-reasoner-v1") */
+  readonly name: string
+  /** A human-readable description of the augmentation's purpose */
+  readonly description: string
+  /** Whether this augmentation is enabled */
+  enabled: boolean
+
+  /**
+   * Initializes the augmentation. This method is called when Brainy starts up.
+   * @returns A Promise that resolves when initialization is complete
+   */
+  initialize(): Promise<void>
+
+  shutDown(): Promise<void>
+
+  getStatus(): Promise<'active' | 'inactive' | 'error'>
+
+  // Allow string indexing for dynamic method access
+  [key: string]: any;
+}
+
+/**
+ * Interface for WebSocket support.
+ * Augmentations that implement this interface can communicate via WebSockets.
+ */
+export interface IWebSocketSupport extends IAugmentation {
+  /**
+   * Establishes a WebSocket connection.
+   * @param url The WebSocket server URL to connect to
+   * @param protocols Optional subprotocols
+   * @returns A Promise resolving to a connection handle or status
+   */
+  connectWebSocket(url: string, protocols?: string | string[]): Promise<WebSocketConnection>
+
+  /**
+   * Sends data through an established WebSocket connection.
+   * @param connectionId The identifier of the established connection
+   * @param data The data to send (will be serialized if not a string)
+   */
+  sendWebSocketMessage(connectionId: string, data: unknown): Promise<void>
+
+  /**
+   * Registers a callback for incoming WebSocket messages.
+   * @param connectionId The identifier of the established connection
+   * @param callback The function to call when a message is received
+   */
+  onWebSocketMessage(connectionId: string, callback: DataCallback<unknown>): Promise<void>
+
+  /**
+   * Removes a callback for incoming WebSocket messages.
+   * @param connectionId The identifier of the established connection
+   * @param callback The function to remove from the callbacks
+   */
+  offWebSocketMessage(connectionId: string, callback: DataCallback<unknown>): Promise<void>
+
+  /**
+   * Closes an established WebSocket connection.
+   * @param connectionId The identifier of the established connection
+   * @param code Optional close code
+   * @param reason Optional close reason
+   */
+  closeWebSocket(connectionId: string, code?: number, reason?: string): Promise<void>
+}
+
+export namespace BrainyAugmentations {
+  /**
+   * Interface for Senses augmentations.
+   * These augmentations ingest and process raw, unstructured data into nouns and verbs.
+   */
+  export interface ISenseAugmentation extends IAugmentation {
+    /**
+     * Processes raw input data into structured nouns and verbs.
+     * @param rawData The raw, unstructured data (e.g., text, image buffer, audio stream)
+     * @param dataType The type of raw data (e.g., 'text', 'image', 'audio')
+     */
+    processRawData(rawData: Buffer | string, dataType: string): Promise<AugmentationResponse<{
+      nouns: string[]
+      verbs: string[]
+    }>>
+
+    /**
+     * Registers a listener for real-time data feeds.
+     * @param feedUrl The URL or identifier of the real-time feed
+     * @param callback A function to call with processed data
+     */
+    listenToFeed(
+      feedUrl: string,
+      callback: DataCallback<{ nouns: string[]; verbs: string[] }>
+    ): Promise<void>
+  }
+
+  /**
+   * Interface for Conduits augmentations.
+   * These augmentations establish and manage high-bandwidth, dedicated channels for structured, programmatic two-way data exchange.
+   */
+  export interface IConduitAugmentation extends IAugmentation {
+    /**
+     * Establishes a connection for programmatic data exchange.
+     * @param targetSystemId The identifier of the external system to connect to
+     * @param config Configuration details for the connection (e.g., API keys, endpoints)
+     */
+    establishConnection(
+      targetSystemId: string,
+      config: Record<string, unknown>
+    ): Promise<AugmentationResponse<WebSocketConnection>>
+
+    /**
+     * Reads structured data directly from Brainy's knowledge graph.
+     * @param query A structured query (e.g., graph query language, object path)
+     * @param options Optional query options (e.g., depth, filters)
+     */
+    readData(
+      query: Record<string, unknown>,
+      options?: Record<string, unknown>
+    ): Promise<AugmentationResponse<unknown>>
+
+    /**
+     * Writes or updates structured data directly into Brainy's knowledge graph.
+     * @param data The structured data to write/update
+     * @param options Optional write options (e.g., merge, overwrite)
+     */
+    writeData(
+      data: Record<string, unknown>,
+      options?: Record<string, unknown>
+    ): Promise<AugmentationResponse<unknown>>
+
+    /**
+     * Monitors a specific data stream or event within Brainy for external systems.
+     * @param streamId The identifier of the data stream or event
+     * @param callback A function to call when new data/events occur
+     */
+    monitorStream(streamId: string, callback: DataCallback<unknown>): Promise<void>
+  }
+
+  /**
+   * Interface for Cognitions augmentations.
+   * These augmentations enable advanced reasoning, inference, and logical operations.
+   */
+  export interface ICognitionAugmentation extends IAugmentation {
+    /**
+     * Performs a reasoning operation based on current knowledge.
+     * @param query The specific reasoning task or question
+     * @param context Optional additional context for the reasoning
+     */
+    reason(query: string, context?: Record<string, unknown>): AugmentationResponse<{
+      inference: string
+      confidence: number
+    }>
+
+    /**
+     * Infers relationships or new facts from existing data.
+     * @param dataSubset A subset of data to infer from
+     */
+    infer(dataSubset: Record<string, unknown>): AugmentationResponse<Record<string, unknown>>
+
+    /**
+     * Executes a logical operation or rule set.
+     * @param ruleId The identifier of the rule or logic to apply
+     * @param input Data to apply the logic to
+     */
+    executeLogic(ruleId: string, input: Record<string, unknown>): AugmentationResponse<boolean>
+  }
+
+  /**
+   * Interface for Memory augmentations.
+   * These augmentations provide storage capabilities for data in different formats (e.g., fileSystem, in-memory).
+   */
+  export interface IMemoryAugmentation extends IAugmentation {
+    /**
+     * Stores data in the memory system.
+     * @param key The unique identifier for the data
+     * @param data The data to store
+     * @param options Optional storage options (e.g., expiration, format)
+     */
+    storeData(
+      key: string,
+      data: unknown,
+      options?: Record<string, unknown>
+    ): Promise<AugmentationResponse<boolean>>
+
+    /**
+     * Retrieves data from the memory system.
+     * @param key The unique identifier for the data
+     * @param options Optional retrieval options (e.g., format, version)
+     */
+    retrieveData(
+      key: string,
+      options?: Record<string, unknown>
+    ): Promise<AugmentationResponse<unknown>>
+
+    /**
+     * Updates existing data in the memory system.
+     * @param key The unique identifier for the data
+     * @param data The updated data
+     * @param options Optional update options (e.g., merge, overwrite)
+     */
+    updateData(
+      key: string,
+      data: unknown,
+      options?: Record<string, unknown>
+    ): Promise<AugmentationResponse<boolean>>
+
+    /**
+     * Deletes data from the memory system.
+     * @param key The unique identifier for the data
+     * @param options Optional deletion options
+     */
+    deleteData(
+      key: string,
+      options?: Record<string, unknown>
+    ): Promise<AugmentationResponse<boolean>>
+
+    /**
+     * Lists available data keys in the memory system.
+     * @param pattern Optional pattern to filter keys (e.g., prefix, regex)
+     * @param options Optional listing options (e.g., limit, offset)
+     */
+    listDataKeys(
+      pattern?: string,
+      options?: Record<string, unknown>
+    ): Promise<AugmentationResponse<string[]>>
+
+    /**
+     * Searches for data in the memory system using vector similarity.
+     * @param query The query vector or data to search for
+     * @param k Number of results to return
+     * @param options Optional search options
+     */
+    search(
+      query: unknown,
+      k?: number,
+      options?: Record<string, unknown>
+    ): Promise<AugmentationResponse<Array<{
+      id: string;
+      score: number;
+      data: unknown;
+    }>>>
+  }
+
+  /**
+   * Interface for Perceptions augmentations.
+   * These augmentations interpret, contextualize, and visualize identified nouns and verbs.
+   */
+  export interface IPerceptionAugmentation extends IAugmentation {
+    /**
+     * Interprets and contextualizes processed nouns and verbs.
+     * @param nouns The list of identified nouns
+     * @param verbs The list of identified verbs
+     * @param context Optional additional context for interpretation
+     */
+    interpret(
+      nouns: string[],
+      verbs: string[],
+      context?: Record<string, unknown>
+    ): AugmentationResponse<Record<string, unknown>>
+
+    /**
+     * Organizes and filters information.
+     * @param data The data to organize (e.g., interpreted perceptions)
+     * @param criteria Optional criteria for filtering/prioritization
+     */
+    organize(
+      data: Record<string, unknown>,
+      criteria?: Record<string, unknown>
+    ): AugmentationResponse<Record<string, unknown>>
+
+    /**
+     * Generates a visualization based on the provided data.
+     * @param data The data to visualize (e.g., interpreted patterns)
+     * @param visualizationType The desired type of visualization (e.g., 'graph', 'chart')
+     */
+    generateVisualization(
+      data: Record<string, unknown>,
+      visualizationType: string
+    ): AugmentationResponse<string | Buffer | Record<string, unknown>>
+  }
+
+  /**
+   * Interface for Dialogs augmentations.
+   * These augmentations facilitate natural language understanding and generation for conversational interaction.
+   */
+  export interface IDialogAugmentation extends IAugmentation {
+    /**
+     * Processes a user's natural language input (query).
+     * @param naturalLanguageQuery The raw text query from the user
+     * @param sessionId An optional session ID for conversational context
+     */
+    processUserInput(naturalLanguageQuery: string, sessionId?: string): AugmentationResponse<{
+      intent: string
+      nouns: string[]
+      verbs: string[]
+      context: Record<string, unknown>
+    }>
+
+    /**
+     * Generates a natural language response based on Brainy's knowledge and interpreted input.
+     * @param interpretedInput The output from `processUserInput` or similar
+     * @param knowledgeContext Relevant knowledge retrieved from Brainy
+     * @param sessionId An optional session ID for conversational context
+     */
+    generateResponse(
+      interpretedInput: Record<string, unknown>,
+      knowledgeContext: Record<string, unknown>,
+      sessionId?: string
+    ): AugmentationResponse<string>
+
+    /**
+     * Manages and updates conversational context.
+     * @param sessionId The session ID
+     * @param contextUpdate The data to update the context with
+     */
+    manageContext(sessionId: string, contextUpdate: Record<string, unknown>): Promise<void>
+  }
+
+  /**
+   * Interface for Activations augmentations.
+   * These augmentations dictate how Brainy initiates actions, responses, or data manipulations.
+   */
+  export interface IActivationAugmentation extends IAugmentation {
+    /**
+     * Triggers an action based on a processed command or internal state.
+     * @param actionName The name of the action to trigger
+     * @param parameters Optional parameters for the action
+     */
+    triggerAction(
+      actionName: string,
+      parameters?: Record<string, unknown>
+    ): AugmentationResponse<unknown>
+
+    /**
+     * Generates an expressive output or response from Brainy.
+     * @param knowledgeId The identifier of the knowledge to express
+     * @param format The desired output format (e.g., 'text', 'json')
+     */
+    generateOutput(knowledgeId: string, format: string): AugmentationResponse<string | Record<string, unknown>>
+
+    /**
+     * Interacts with an external system or API.
+     * @param systemId The identifier of the external system
+     * @param payload The data to send to the external system
+     */
+    interactExternal(systemId: string, payload: Record<string, unknown>): AugmentationResponse<unknown>
+  }
+}
+
+/** Direct exports of augmentation interfaces for easier imports */
+export interface ISenseAugmentation extends BrainyAugmentations.ISenseAugmentation {
+}
+
+export interface IConduitAugmentation extends BrainyAugmentations.IConduitAugmentation {
+}
+
+export interface ICognitionAugmentation extends BrainyAugmentations.ICognitionAugmentation {
+}
+
+export interface IMemoryAugmentation extends BrainyAugmentations.IMemoryAugmentation {
+}
+
+export interface IPerceptionAugmentation extends BrainyAugmentations.IPerceptionAugmentation {
+}
+
+export interface IDialogAugmentation extends BrainyAugmentations.IDialogAugmentation {
+}
+
+export interface IActivationAugmentation extends BrainyAugmentations.IActivationAugmentation {
+}
+
+/** WebSocket-enabled augmentation interfaces */
+export type IWebSocketSenseAugmentation = BrainyAugmentations.ISenseAugmentation & IWebSocketSupport
+export type IWebSocketConduitAugmentation = BrainyAugmentations.IConduitAugmentation & IWebSocketSupport
+export type IWebSocketCognitionAugmentation = BrainyAugmentations.ICognitionAugmentation & IWebSocketSupport
+export type IWebSocketMemoryAugmentation = BrainyAugmentations.IMemoryAugmentation & IWebSocketSupport
+export type IWebSocketPerceptionAugmentation = BrainyAugmentations.IPerceptionAugmentation & IWebSocketSupport
+export type IWebSocketDialogAugmentation = BrainyAugmentations.IDialogAugmentation & IWebSocketSupport
+export type IWebSocketActivationAugmentation = BrainyAugmentations.IActivationAugmentation & IWebSocketSupport

src/types/brainyDataInterface.ts

@@ -0,0 +1,55 @@
+/**
+ * BrainyDataInterface
+ * 
+ * This interface defines the methods from BrainyData that are used by serverSearchAugmentations.ts.
+ * It's used to break the circular dependency between brainyData.ts and serverSearchAugmentations.ts.
+ */
+
+import { Vector } from '../coreTypes.js'
+
+export interface BrainyDataInterface<T = unknown> {
+  /**
+   * Initialize the database
+   */
+  init(): Promise<void>
+
+  /**
+   * Get a noun by ID
+   * @param id The ID of the noun to get
+   */
+  get(id: string): Promise<unknown>
+
+  /**
+   * Add a vector or data to the database
+   * @param vectorOrData Vector or data to add
+   * @param metadata Optional metadata to associate with the vector
+   * @returns The ID of the added vector
+   */
+  add(vectorOrData: Vector | unknown, metadata?: T): Promise<string>
+
+  /**
+   * Search for text in the database
+   * @param text The text to search for
+   * @param limit Maximum number of results to return
+   * @returns Search results
+   */
+  searchText(text: string, limit?: number): Promise<unknown[]>
+
+  /**
+   * Create a relationship between two entities
+   * @param sourceId The ID of the source entity
+   * @param targetId The ID of the target entity
+   * @param relationType The type of relationship
+   * @param metadata Optional metadata about the relationship
+   * @returns The ID of the created relationship
+   */
+  relate(sourceId: string, targetId: string, relationType: string, metadata?: unknown): Promise<string>
+
+  /**
+   * Find entities similar to a given entity ID
+   * @param id ID of the entity to find similar entities for
+   * @param options Additional options
+   * @returns Array of search results with similarity scores
+   */
+  findSimilar(id: string, options?: { limit?: number }): Promise<unknown[]>
+}

src/types/fileSystemTypes.ts

@@ -0,0 +1,14 @@
+/**
+ * Type declarations for the File System Access API
+ * Extends the FileSystemDirectoryHandle interface to include the [Symbol.asyncIterator] method
+ */
+
+// Extend the FileSystemDirectoryHandle interface
+interface FileSystemDirectoryHandle {
+  [Symbol.asyncIterator](): AsyncIterableIterator<[string, FileSystemHandle]>;
+  keys(): AsyncIterableIterator<string>;
+  entries(): AsyncIterableIterator<[string, FileSystemHandle]>;
+}
+
+// Export something to make this a module
+export const fileSystemTypesLoaded = true;

src/types/graphTypes.ts

@@ -0,0 +1,150 @@
+// Common metadata types
+/**
+ * Represents a high-precision timestamp with seconds and nanoseconds
+ * Used for tracking creation and update times of graph elements
+ */
+interface Timestamp {
+  seconds: number
+  nanoseconds: number
+}
+
+/**
+ * Metadata about the creator/source of a graph noun
+ * Tracks which augmentation and model created the element
+ */
+interface CreatorMetadata {
+  augmentation: string // Name of the augmentation that created this element
+  version: string // Version of the augmentation
+}
+
+/**
+ * Base interface for nodes (nouns) in the graph
+ * Represents entities like people, places, things, etc.
+ */
+export interface GraphNoun {
+  id: string // Unique identifier for the noun
+  createdBy: CreatorMetadata // Information about what created this noun
+  noun: NounType // Type classification of the noun
+  createdAt: Timestamp // When the noun was created
+  updatedAt: Timestamp // When the noun was last updated
+  label?: string // Optional descriptive label
+  data?: Record<string, any> // Additional flexible data storage
+  embeddedVerbs?: EmbeddedGraphVerb[] // Optional embedded relationships
+  embedding?: number[] // Vector representation of the noun
+}
+
+/**
+ * Base interface for edges (verbs) in the graph
+ * Represents relationships between nouns
+ */
+export interface GraphVerb {
+  id: string // Unique identifier for the verb
+  source: string // ID of the source noun
+  target: string // ID of the target noun
+  label?: string // Optional descriptive label
+  verb: VerbType // Type of relationship
+  createdAt: Timestamp // When the verb was created
+  updatedAt: Timestamp // When the verb was last updated
+  data?: Record<string, any> // Additional flexible data storage
+  embedding?: number[] // Vector representation of the relationship
+  confidence?: number // Confidence score (0-1)
+  weight?: number // Strength/importance of the relationship
+}
+
+/**
+ * Version of GraphVerb for embedded relationships
+ * Used when the source is implicit from the parent document
+ */
+export type EmbeddedGraphVerb = Omit<GraphVerb, 'source'>
+
+// Proper Noun interfaces - extend GraphNoun with specific noun types
+
+/**
+ * Represents a person entity in the graph
+ */
+export interface Person extends GraphNoun {
+  noun: typeof NounType.Person
+}
+
+/**
+ * Represents a physical location in the graph
+ */
+export interface Place extends GraphNoun {
+  noun: typeof NounType.Place
+}
+
+/**
+ * Represents a physical or virtual object in the graph
+ */
+export interface Thing extends GraphNoun {
+  noun: typeof NounType.Thing
+}
+
+/**
+ * Represents an event or occurrence in the graph
+ */
+export interface Event extends GraphNoun {
+  noun: typeof NounType.Event
+}
+
+/**
+ * Represents an abstract concept or idea in the graph
+ */
+export interface Concept extends GraphNoun {
+  noun: typeof NounType.Concept
+}
+
+export interface Group extends GraphNoun {
+  noun: typeof NounType.Group
+}
+
+export interface List extends GraphNoun {
+  noun: typeof NounType.List
+}
+
+/**
+ * Represents content (text, media, etc.) in the graph
+ */
+export interface Content extends GraphNoun {
+  noun: typeof NounType.Content
+}
+
+/**
+ * Defines valid noun types for graph entities
+ * Used for categorizing different types of nodes
+ */
+
+export const NounType = {
+  Person: 'person', // Person entities
+  Place: 'place', // Physical locations
+  Thing: 'thing', // Physical or virtual objects
+  Event: 'event', // Events or occurrences
+  Concept: 'concept', // Abstract concepts or ideas
+  Content: 'content', // Content items
+  Group: 'group', // Groups of related entities
+  List: 'list', // Ordered collections of entities
+  Category: 'category' // Categories for content items including tags
+} as const
+export type NounType = (typeof NounType)[keyof typeof NounType]
+
+/**
+ * Defines valid verb types for relationships
+ * Used for categorizing different types of connections
+ */
+export const VerbType = {
+  AttributedTo: 'attributedTo', // Indicates attribution or authorship
+  Controls: 'controls', // Indicates control or ownership
+  Created: 'created', // Indicates creation or authorship
+  Earned: 'earned', // Indicates achievement or acquisition
+  Owns: 'owns', // Indicates ownership
+  MemberOf: 'memberOf', // Indicates membership or affiliation
+  RelatedTo: 'relatedTo', // Indicates family relationship
+  WorksWith: 'worksWith', // Indicates professional relationship
+  FriendOf: 'friendOf', // Indicates friendship
+  ReportsTo: 'reportsTo', // Indicates reporting relationship
+  Supervises: 'supervises', // Indicates supervisory relationship
+  Mentors: 'mentors', // Indicates mentorship relationship
+  Follows: 'follows', // Indicates the following relationship
+  Likes: 'likes' // Indicates liking relationship
+} as const
+export type VerbType = (typeof VerbType)[keyof typeof VerbType]

src/types/mcpTypes.ts

@@ -0,0 +1,149 @@
+/**
+ * Model Control Protocol (MCP) Types
+ * 
+ * This file defines the types and interfaces for the Model Control Protocol (MCP)
+ * implementation in Brainy. MCP allows external models to access Brainy data and
+ * use the augmentation pipeline as tools.
+ */
+
+/**
+ * MCP version information
+ */
+export const MCP_VERSION = '1.0.0'
+
+/**
+ * MCP request types
+ */
+export enum MCPRequestType {
+  DATA_ACCESS = 'data_access',
+  TOOL_EXECUTION = 'tool_execution',
+  SYSTEM_INFO = 'system_info',
+  AUTHENTICATION = 'authentication'
+}
+
+/**
+ * Base interface for all MCP requests
+ */
+export interface MCPRequest {
+  /** The type of request */
+  type: MCPRequestType
+  /** Request ID for tracking and correlation */
+  requestId: string
+  /** API version */
+  version: string
+  /** Authentication token (if required) */
+  authToken?: string
+}
+
+/**
+ * Interface for data access requests
+ */
+export interface MCPDataAccessRequest extends MCPRequest {
+  type: MCPRequestType.DATA_ACCESS
+  /** The data access operation to perform */
+  operation: 'get' | 'search' | 'add' | 'getRelationships'
+  /** Parameters for the operation */
+  parameters: Record<string, any>
+}
+
+/**
+ * Interface for tool execution requests
+ */
+export interface MCPToolExecutionRequest extends MCPRequest {
+  type: MCPRequestType.TOOL_EXECUTION
+  /** The name of the tool to execute */
+  toolName: string
+  /** Parameters for the tool */
+  parameters: Record<string, any>
+}
+
+/**
+ * Interface for system info requests
+ */
+export interface MCPSystemInfoRequest extends MCPRequest {
+  type: MCPRequestType.SYSTEM_INFO
+  /** The type of information to retrieve */
+  infoType: 'status' | 'availableTools' | 'version'
+}
+
+/**
+ * Interface for authentication requests
+ */
+export interface MCPAuthenticationRequest extends MCPRequest {
+  type: MCPRequestType.AUTHENTICATION
+  /** The authentication credentials */
+  credentials: {
+    apiKey?: string
+    username?: string
+    password?: string
+  }
+}
+
+/**
+ * Base interface for all MCP responses
+ */
+export interface MCPResponse {
+  /** Whether the request was successful */
+  success: boolean
+  /** The request ID from the original request */
+  requestId: string
+  /** API version */
+  version: string
+  /** Response data (if successful) */
+  data?: any
+  /** Error information (if unsuccessful) */
+  error?: {
+    code: string
+    message: string
+    details?: any
+  }
+}
+
+/**
+ * Interface for MCP tool definitions
+ */
+export interface MCPTool {
+  /** The name of the tool */
+  name: string
+  /** A description of what the tool does */
+  description: string
+  /** The parameters the tool accepts */
+  parameters: {
+    type: 'object'
+    properties: Record<string, {
+      type: string
+      description: string
+      enum?: string[]
+      required?: boolean
+    }>
+    required: string[]
+  }
+}
+
+/**
+ * Configuration options for MCP services
+ */
+export interface MCPServiceOptions {
+  /** Port for the WebSocket server */
+  wsPort?: number
+  /** Port for the REST server */
+  restPort?: number
+  /** Whether to enable authentication */
+  enableAuth?: boolean
+  /** API keys for authentication */
+  apiKeys?: string[]
+  /** Rate limiting configuration */
+  rateLimit?: {
+    /** Maximum number of requests per window */
+    maxRequests: number
+    /** Time window in milliseconds */
+    windowMs: number
+  }
+  /** CORS configuration for REST API */
+  cors?: {
+    /** Allowed origins */
+    origin: string | string[]
+    /** Whether to allow credentials */
+    credentials: boolean
+  }
+}

src/types/pipelineTypes.ts

@@ -0,0 +1,33 @@
+/**
+ * Pipeline Types
+ *
+ * This module provides shared types for the pipeline system to avoid circular dependencies.
+ */
+
+import {
+  BrainyAugmentations,
+  IWebSocketSupport,
+  IAugmentation
+} from './augmentations.js'
+
+/**
+ * Type definitions for the augmentation registry
+ */
+export type AugmentationRegistry = {
+  sense: BrainyAugmentations.ISenseAugmentation[];
+  conduit: BrainyAugmentations.IConduitAugmentation[];
+  cognition: BrainyAugmentations.ICognitionAugmentation[];
+  memory: BrainyAugmentations.IMemoryAugmentation[];
+  perception: BrainyAugmentations.IPerceptionAugmentation[];
+  dialog: BrainyAugmentations.IDialogAugmentation[];
+  activation: BrainyAugmentations.IActivationAugmentation[];
+  webSocket: IWebSocketSupport[];
+}
+
+/**
+ * Interface for the Pipeline class
+ * This is used to break circular dependencies between pipeline.ts and augmentationRegistry.ts
+ */
+export interface IPipeline {
+  register<T extends IAugmentation>(augmentation: T): IPipeline;
+}

src/types/tensorflowTypes.ts

@@ -0,0 +1,14 @@
+// Type declarations for TensorFlow.js models
+
+// This file is a placeholder for TensorFlow.js model types
+// We're using type assertions in the code instead of module declarations
+
+// Define some basic types that might be useful
+export interface TensorflowModel {
+  load(): Promise<any>;
+  embed(data: string[]): any;
+  dispose(): void;
+}
+
+// Export a dummy constant to make this a proper module
+export const tensorflowModelsLoaded = true;

src/unified.ts

@@ -0,0 +1,36 @@
+/**
+ * Unified entry point for Brainy
+ * This file exports everything from index.ts
+ * Environment detection is handled here and made available to all components
+ */
+
+// Export environment information
+export const environment = {
+  isBrowser: typeof window !== 'undefined',
+  isNode:
+    typeof process !== 'undefined' && process.versions && process.versions.node,
+  isServerless:
+    typeof window === 'undefined' &&
+    (typeof process === 'undefined' ||
+      !process.versions ||
+      !process.versions.node)
+}
+
+// Make environment information available globally
+if (typeof globalThis !== 'undefined') {
+  globalThis.__ENV__ = environment
+}
+
+// Log the detected environment
+console.log(
+  `Brainy running in ${
+    environment.isBrowser
+      ? 'browser'
+      : environment.isNode
+        ? 'Node.js'
+        : 'serverless/unknown'
+  } environment`
+)
+
+// Re-export everything from index.ts
+export * from './index.js'

src/utils/distance.ts

@@ -0,0 +1,86 @@
+/**
+ * Distance functions for vector similarity calculations
+ * Optimized for Node.js 23.11+ using enhanced array methods
+ */
+
+import { DistanceFunction, Vector } from '../coreTypes.js'
+
+/**
+ * Calculates the Euclidean distance between two vectors
+ * Lower values indicate higher similarity
+ * Optimized using array methods for Node.js 23.11+
+ */
+export const euclideanDistance: DistanceFunction = (a: Vector, b: Vector): number => {
+  if (a.length !== b.length) {
+    throw new Error('Vectors must have the same dimensions')
+  }
+
+  // Use array.reduce for better performance in Node.js 23.11+
+  const sum = a.reduce((acc, val, i) => {
+    const diff = val - b[i]
+    return acc + (diff * diff)
+  }, 0)
+
+  return Math.sqrt(sum)
+}
+
+/**
+ * Calculates the cosine distance between two vectors
+ * Lower values indicate higher similarity
+ * Range: 0 (identical) to 2 (opposite)
+ * Optimized using array methods for Node.js 23.11+
+ */
+export const cosineDistance: DistanceFunction = (a: Vector, b: Vector): number => {
+  if (a.length !== b.length) {
+    throw new Error('Vectors must have the same dimensions')
+  }
+
+  // Use array.reduce to calculate all values in a single pass
+  const { dotProduct, normA, normB } = a.reduce((acc, val, i) => {
+    return {
+      dotProduct: acc.dotProduct + (val * b[i]),
+      normA: acc.normA + (val * val),
+      normB: acc.normB + (b[i] * b[i])
+    };
+  }, { dotProduct: 0, normA: 0, normB: 0 });
+
+  if (normA === 0 || normB === 0) {
+    return 2 // Maximum distance for zero vectors
+  }
+
+  const similarity = dotProduct / (Math.sqrt(normA) * Math.sqrt(normB))
+  // Convert cosine similarity (-1 to 1) to distance (0 to 2)
+  return 1 - similarity
+}
+
+/**
+ * Calculates the Manhattan (L1) distance between two vectors
+ * Lower values indicate higher similarity
+ * Optimized using array methods for Node.js 23.11+
+ */
+export const manhattanDistance: DistanceFunction = (a: Vector, b: Vector): number => {
+  if (a.length !== b.length) {
+    throw new Error('Vectors must have the same dimensions')
+  }
+
+  // Use array.reduce for better performance in Node.js 23.11+
+  return a.reduce((sum, val, i) => sum + Math.abs(val - b[i]), 0)
+}
+
+/**
+ * Calculates the dot product similarity between two vectors
+ * Higher values indicate higher similarity
+ * Converted to a distance metric (lower is better)
+ * Optimized using array methods for Node.js 23.11+
+ */
+export const dotProductDistance: DistanceFunction = (a: Vector, b: Vector): number => {
+  if (a.length !== b.length) {
+    throw new Error('Vectors must have the same dimensions')
+  }
+
+  // Use array.reduce for better performance in Node.js 23.11+
+  const dotProduct = a.reduce((sum, val, i) => sum + (val * b[i]), 0)
+
+  // Convert to a distance metric (lower is better)
+  return -dotProduct
+}

src/utils/embedding.ts

@@ -0,0 +1,293 @@
+/**
+ * Embedding functions for converting data to vectors
+ */
+
+import { EmbeddingFunction, EmbeddingModel, Vector } from '../coreTypes.js'
+import { executeInThread } from './workerUtils.js'
+
+/**
+ * TensorFlow Universal Sentence Encoder embedding model
+ * This model provides high-quality text embeddings using TensorFlow.js
+ * The required TensorFlow.js dependencies are automatically installed with this package
+ */
+export class UniversalSentenceEncoder implements EmbeddingModel {
+  private model: any = null
+  private initialized = false
+  private tf: any = null
+  private use: any = null
+
+  /**
+   * Initialize the embedding model
+   */
+  public async init(): Promise<void> {
+    try {
+      // Save original console.warn
+      const originalWarn = console.warn
+
+      // Override console.warn to suppress TensorFlow.js Node.js backend message
+      console.warn = function (message?: any, ...optionalParams: any[]) {
+        if (
+          message &&
+          typeof message === 'string' &&
+          message.includes(
+            'Hi, looks like you are running TensorFlow.js in Node.js'
+          )
+        ) {
+          return // Suppress the specific warning
+        }
+        originalWarn(message, ...optionalParams)
+      }
+
+      // Define EPSILON flag before TensorFlow.js is loaded
+      // This prevents the "Cannot evaluate flag 'EPSILON': no evaluation function found" error
+      if (typeof window !== 'undefined') {
+        ;(window as any).EPSILON = 1e-7
+        // Define the flag with an evaluation function for TensorFlow.js
+        ;(window as any).ENV = (window as any).ENV || {}
+        ;(window as any).ENV.flagRegistry =
+          (window as any).ENV.flagRegistry || {}
+        ;(window as any).ENV.flagRegistry.EPSILON = {
+          evaluationFn: () => 1e-7
+        }
+      } else if (typeof global !== 'undefined') {
+        ;(global as any).EPSILON = 1e-7
+        // Define the flag with an evaluation function for TensorFlow.js
+        ;(global as any).ENV = (global as any).ENV || {}
+        ;(global as any).ENV.flagRegistry =
+          (global as any).ENV.flagRegistry || {}
+        ;(global as any).ENV.flagRegistry.EPSILON = {
+          evaluationFn: () => 1e-7
+        }
+      }
+
+      // Dynamically import TensorFlow.js and Universal Sentence Encoder
+      // Use type assertions to tell TypeScript these modules exist
+      this.tf = await import('@tensorflow/tfjs')
+      this.use = await import('@tensorflow-models/universal-sentence-encoder')
+
+      // Load the model
+      this.model = await this.use.load()
+      this.initialized = true
+
+      // Restore original console.warn
+      console.warn = originalWarn
+    } catch (error) {
+      console.error('Failed to initialize Universal Sentence Encoder:', error)
+      throw new Error(
+        `Failed to initialize Universal Sentence Encoder: ${error}`
+      )
+    }
+  }
+
+  /**
+   * Embed text into a vector using Universal Sentence Encoder
+   * @param data Text to embed
+   */
+  public async embed(data: string | string[]): Promise<Vector> {
+    if (!this.initialized) {
+      await this.init()
+    }
+
+    try {
+      // Handle different input types
+      let textToEmbed: string[]
+      if (typeof data === 'string') {
+        // Handle empty string case
+        if (data.trim() === '') {
+          // Return a zero vector of appropriate dimension (512 is the default for USE)
+          return new Array(512).fill(0)
+        }
+        textToEmbed = [data]
+      } else if (
+        Array.isArray(data) &&
+        data.every((item) => typeof item === 'string')
+      ) {
+        // Handle empty array or array with empty strings
+        if (data.length === 0 || data.every((item) => item.trim() === '')) {
+          return new Array(512).fill(0)
+        }
+        // Filter out empty strings
+        textToEmbed = data.filter((item) => item.trim() !== '')
+        if (textToEmbed.length === 0) {
+          return new Array(512).fill(0)
+        }
+      } else {
+        throw new Error(
+          'UniversalSentenceEncoder only supports string or string[] data'
+        )
+      }
+
+      // Get embeddings
+      const embeddings = await this.model.embed(textToEmbed)
+
+      // Convert to array and return the first embedding
+      const embeddingArray = await embeddings.array()
+      return embeddingArray[0]
+    } catch (error) {
+      console.error(
+        'Failed to embed text with Universal Sentence Encoder:',
+        error
+      )
+      throw new Error(
+        `Failed to embed text with Universal Sentence Encoder: ${error}`
+      )
+    }
+  }
+
+  /**
+   * Dispose of the model resources
+   */
+  public async dispose(): Promise<void> {
+    if (this.model && this.tf) {
+      try {
+        // Dispose of the model and tensors
+        this.model.dispose()
+        this.tf.disposeVariables()
+        this.initialized = false
+      } catch (error) {
+        console.error('Failed to dispose Universal Sentence Encoder:', error)
+      }
+    }
+    return Promise.resolve()
+  }
+}
+
+/**
+ * Create an embedding function from an embedding model
+ * @param model Embedding model to use
+ */
+export function createEmbeddingFunction(
+  model: EmbeddingModel
+): EmbeddingFunction {
+  return async (data: any): Promise<Vector> => {
+    return await model.embed(data)
+  }
+}
+
+/**
+ * Creates a TensorFlow-based Universal Sentence Encoder embedding function
+ * This is the required embedding function for all text embeddings
+ */
+export function createTensorFlowEmbeddingFunction(): EmbeddingFunction {
+  // Create a single shared instance of the model
+  const model = new UniversalSentenceEncoder()
+  let modelInitialized = false
+
+  return async (data: any): Promise<Vector> => {
+    try {
+      // Initialize the model if it hasn't been initialized yet
+      if (!modelInitialized) {
+        await model.init()
+        modelInitialized = true
+      }
+
+      return await model.embed(data)
+    } catch (error) {
+      console.error('Failed to use TensorFlow embedding:', error)
+      throw new Error(
+        `Universal Sentence Encoder is required but failed: ${error}`
+      )
+    }
+  }
+}
+
+/**
+ * Creates a TensorFlow-based Universal Sentence Encoder embedding function that runs in a separate thread
+ * This provides better performance for CPU-intensive embedding operations
+ * @param options Configuration options
+ * @returns An embedding function that runs in a separate thread
+ */
+export function createThreadedEmbeddingFunction(
+  options: { fallbackToMain?: boolean } = {}
+): EmbeddingFunction {
+  // Create a standard embedding function to use as fallback
+  const standardEmbedding = createTensorFlowEmbeddingFunction()
+
+  // Flag to track if we've fallen back to main thread
+  let useFallback = false
+
+  return async (data: any): Promise<Vector> => {
+    // If we've already determined that threading doesn't work, use the fallback
+    if (useFallback) {
+      return standardEmbedding(data)
+    }
+
+    try {
+      // Function to be executed in a worker thread
+      // This must be a regular function (not async) to avoid Promise cloning issues
+      const embedInWorker = (inputData: any) => {
+        // Return a plain object with the input data
+        // All async operations will be performed inside the worker
+        return { data: inputData }
+      }
+
+      // Worker implementation function that will be stringified and run in the worker
+      const workerImplementation = async ({ data }: { data: any }) => {
+        // We need to dynamically import TensorFlow.js and USE in the worker
+        const tf = await import('@tensorflow/tfjs')
+        const use = await import('@tensorflow-models/universal-sentence-encoder')
+
+        // Load the model
+        const model = await use.load()
+
+        // Handle different input types
+        let textToEmbed: string[]
+        if (typeof data === 'string') {
+          if (data.trim() === '') {
+            return new Array(512).fill(0)
+          }
+          textToEmbed = [data]
+        } else if (
+          Array.isArray(data) &&
+          data.every((item) => typeof item === 'string')
+        ) {
+          if (data.length === 0 || data.every((item) => item.trim() === '')) {
+            return new Array(512).fill(0)
+          }
+          textToEmbed = data.filter((item) => item.trim() !== '')
+          if (textToEmbed.length === 0) {
+            return new Array(512).fill(0)
+          }
+        } else {
+          throw new Error(
+            'UniversalSentenceEncoder only supports string or string[] data'
+          )
+        }
+
+        // Get embeddings
+        const embeddings = await model.embed(textToEmbed)
+
+        // Convert to array and return the first embedding
+        const embeddingArray = await embeddings.array()
+
+        // Dispose of the tensor to free memory
+        embeddings.dispose()
+
+        return embeddingArray[0]
+      }
+
+      // Execute the embedding function in a separate thread
+      // Pass the worker implementation as a string to avoid Promise cloning issues
+      return await executeInThread<Vector>(workerImplementation.toString(), embedInWorker(data))
+    } catch (error) {
+      // If threading fails and fallback is enabled, use the standard embedding function
+      if (options.fallbackToMain) {
+        console.warn('Threaded embedding failed, falling back to main thread:', error)
+        useFallback = true
+        return standardEmbedding(data)
+      }
+
+      // Otherwise, propagate the error
+      throw new Error(`Threaded embedding failed: ${error}`)
+    }
+  }
+}
+
+/**
+ * Default embedding function
+ * Uses UniversalSentenceEncoder for all text embeddings
+ * TensorFlow.js is required for this to work
+ * Uses threading when available for better performance
+ */
+export const defaultEmbeddingFunction: EmbeddingFunction =
+  createThreadedEmbeddingFunction({ fallbackToMain: true })

src/utils/environment.ts

@@ -0,0 +1,57 @@
+/**
+ * Utility functions for environment detection
+ */
+
+/**
+ * Check if code is running in a browser environment
+ */
+export function isBrowser(): boolean {
+  return typeof window !== 'undefined' && typeof document !== 'undefined';
+}
+
+/**
+ * Check if code is running in a Node.js environment
+ */
+export function isNode(): boolean {
+  return typeof process !== 'undefined' && 
+    process.versions != null && 
+    process.versions.node != null;
+}
+
+/**
+ * Check if code is running in a Web Worker environment
+ */
+export function isWebWorker(): boolean {
+  return typeof self === 'object' && 
+    self.constructor && 
+    self.constructor.name === 'DedicatedWorkerGlobalScope';
+}
+
+/**
+ * Check if Web Workers are available in the current environment
+ */
+export function areWebWorkersAvailable(): boolean {
+  return isBrowser() && typeof Worker !== 'undefined';
+}
+
+/**
+ * Check if Worker Threads are available in the current environment (Node.js)
+ */
+export function areWorkerThreadsAvailable(): boolean {
+  if (!isNode()) return false;
+  
+  try {
+    // Dynamic import to avoid errors in browser environments
+    require('worker_threads');
+    return true;
+  } catch (e) {
+    return false;
+  }
+}
+
+/**
+ * Determine if threading is available in the current environment
+ */
+export function isThreadingAvailable(): boolean {
+  return areWebWorkersAvailable() || areWorkerThreadsAvailable();
+}

src/utils/index.ts

@@ -0,0 +1,2 @@
+export * from './distance.js'
+export * from './embedding.js'

src/utils/version.ts

@@ -0,0 +1,6 @@
+/**
+ * This file is auto-generated during the build process.
+ * Do not modify this file directly.
+ */
+
+export const VERSION = '0.9.10';

src/utils/workerUtils.ts

@@ -0,0 +1,145 @@
+/**
+ * Utility functions for working with Web Workers and Worker Threads
+ */
+
+import { isNode, isBrowser } from './environment.js'
+
+/**
+ * Execute a function in a Web Worker (browser environment)
+ *
+ * @param fnString The function to execute as a string
+ * @param args The arguments to pass to the function
+ * @returns A promise that resolves with the result of the function
+ */
+export function executeInWebWorker<T>(fnString: string, args: any): Promise<T> {
+  return new Promise((resolve, reject) => {
+    // Create a blob URL for the worker script
+    const workerScript = `
+      self.onmessage = async function(e) {
+        try {
+          const fn = ${fnString};
+          const result = await fn(e.data);
+          self.postMessage({ success: true, data: result });
+        } catch (error) {
+          self.postMessage({ 
+            success: false, 
+            error: error instanceof Error ? error.message : String(error) 
+          });
+        }
+      };
+    `
+
+    const blob = new Blob([workerScript], { type: 'application/javascript' })
+    const blobURL = URL.createObjectURL(blob)
+
+    // Create a worker
+    const worker = new Worker(blobURL)
+
+    // Set up message handling
+    worker.onmessage = function (
+      e: MessageEvent<{ success: boolean; data: T; error?: string }>
+    ) {
+      URL.revokeObjectURL(blobURL) // Clean up
+      worker.terminate() // Terminate the worker
+
+      if (e.data.success) {
+        resolve(e.data.data)
+      } else {
+        reject(new Error(e.data.error))
+      }
+    }
+
+    worker.onerror = function (error: ErrorEvent) {
+      URL.revokeObjectURL(blobURL) // Clean up
+      worker.terminate() // Terminate the worker
+      reject(error)
+    }
+
+    // Start the worker
+    worker.postMessage(args)
+  })
+}
+
+/**
+ * Execute a function in a Worker Thread (Node.js environment)
+ *
+ * @param fnString The function to execute as a string
+ * @param args The arguments to pass to the function
+ * @returns A promise that resolves with the result of the function
+ */
+export function executeInWorkerThread<T>(
+  fnString: string,
+  args: any
+): Promise<T> {
+  return new Promise((resolve, reject) => {
+    try {
+      // Dynamic import to avoid errors in browser environments
+      const { Worker } = require('worker_threads')
+
+      // Create a worker script
+      const workerScript = `
+        const { parentPort } = require('worker_threads');
+
+        parentPort.once('message', async (data) => {
+          try {
+            const fn = ${fnString};
+            const result = await fn(data);
+            parentPort.postMessage({ success: true, data: result });
+          } catch (error) {
+            parentPort.postMessage({ 
+              success: false, 
+              error: error instanceof Error ? error.message : String(error) 
+            });
+          }
+        });
+      `
+
+      // Create a worker
+      const worker = new Worker(workerScript, { eval: true })
+
+      // Set up message handling
+      worker.on(
+        'message',
+        (data: { success: boolean; data: T; error?: string }) => {
+          worker.terminate() // Terminate the worker
+
+          if (data.success) {
+            resolve(data.data)
+          } else {
+            reject(new Error(data.error))
+          }
+        }
+      )
+
+      worker.on('error', (error: Error) => {
+        worker.terminate() // Terminate the worker
+        reject(error)
+      })
+
+      // Start the worker
+      worker.postMessage(args)
+    } catch (error) {
+      reject(error)
+    }
+  })
+}
+
+/**
+ * Execute a function in a separate thread based on the environment
+ *
+ * @param fnString The function to execute as a string
+ * @param args The arguments to pass to the function
+ * @returns A promise that resolves with the result of the function
+ */
+export function executeInThread<T>(fnString: string, args: any): Promise<T> {
+  if (isBrowser()) {
+    return executeInWebWorker<T>(fnString, args)
+  } else if (isNode()) {
+    return executeInWorkerThread<T>(fnString, args)
+  } else {
+    // Fall back to executing in the main thread
+    // Parse the function from string and execute it
+    const fn = new Function('return ' + fnString)()
+    return Promise.resolve(fn(args) as T)
+  }
+}

tsconfig.browser.json

@@ -0,0 +1,30 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "ESNext",
+    "moduleResolution": "Bundler",
+    "esModuleInterop": true,
+    "strict": true,
+    "declaration": false,
+    "outDir": "./dist",
+    "rootDir": ".",
+    "lib": [
+      "DOM",
+      "ESNext"
+    ],
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "noEmit": false,
+    "preserveConstEnums": true,
+    "sourceMap": true
+  },
+  "include": [
+    "src/**/*",
+    "demo/**/*"
+  ],
+  "exclude": [
+    "node_modules",
+    "dist",
+    "**/*.test.ts"
+  ]
+}

tsconfig.json

@@ -0,0 +1,29 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "esModuleInterop": true,
+    "strict": true,
+    "declaration": true,
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "lib": [
+      "DOM",
+      "ESNext"
+    ],
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "noEmit": false,
+    "preserveConstEnums": true,
+    "sourceMap": true
+  },
+  "include": [
+    "src/**/*"
+  ],
+  "exclude": [
+    "node_modules",
+    "dist",
+    "**/*.test.ts"
+  ]
+}

tsconfig.unified.json

@@ -0,0 +1,29 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "esModuleInterop": true,
+    "strict": true,
+    "declaration": true,
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "lib": [
+      "DOM",
+      "ESNext"
+    ],
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "noEmit": false,
+    "preserveConstEnums": true,
+    "sourceMap": true
+  },
+  "include": [
+    "src/**/*"
+  ],
+  "exclude": [
+    "node_modules",
+    "dist",
+    "**/*.test.ts"
+  ]
+}

verify-package-size.js

@@ -0,0 +1,135 @@
+#!/usr/bin/env node
+
+import fs from 'fs'
+import path from 'path'
+import { execSync } from 'child_process'
+import { fileURLToPath } from 'url'
+
+// Get the current directory
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+
+// Function to get the size of a file in MB
+function getFileSizeInMB(filePath) {
+  const stats = fs.statSync(filePath)
+  return stats.size / (1024 * 1024)
+}
+
+// Function to check if a file should be included in the package
+function shouldIncludeFile(filePath, npmignorePatterns, includePatterns) {
+  const relativePath = path.relative('.', filePath)
+
+  // Check if the file matches any npmignore pattern
+  for (const pattern of npmignorePatterns) {
+    if (pattern.test(relativePath)) {
+      return false
+    }
+  }
+
+  // If we have explicit include patterns, check if the file matches any
+  if (includePatterns.length > 0) {
+    for (const pattern of includePatterns) {
+      if (pattern.test(relativePath)) {
+        return true
+      }
+    }
+    return false
+  }
+
+  return true
+}
+
+// Parse .npmignore file
+function parseNpmignore() {
+  const patterns = []
+  if (fs.existsSync('.npmignore')) {
+    const content = fs.readFileSync('.npmignore', 'utf8')
+    const lines = content.split('\n')
+
+    for (const line of lines) {
+      const trimmedLine = line.trim()
+      if (trimmedLine && !trimmedLine.startsWith('#')) {
+        // Convert glob pattern to regex
+        let pattern = trimmedLine
+          .replace(/\./g, '\\.')
+          .replace(/\*/g, '.*')
+          .replace(/\?/g, '.')
+
+        // Handle directory patterns
+        if (pattern.endsWith('/')) {
+          pattern = `${pattern}.*`
+        }
+
+        patterns.push(new RegExp(`^${pattern}$`))
+      }
+    }
+  }
+  return patterns
+}
+
+// Parse package.json files array
+function parsePackageFiles() {
+  const patterns = []
+  const packageJson = JSON.parse(fs.readFileSync('package.json', 'utf8'))
+
+  if (packageJson.files && Array.isArray(packageJson.files)) {
+    for (const pattern of packageJson.files) {
+      // Convert glob pattern to regex
+      let regexPattern = pattern
+        .replace(/\./g, '\\.')
+        .replace(/\*/g, '.*')
+        .replace(/\?/g, '.')
+
+      // Handle directory patterns
+      if (regexPattern.endsWith('/')) {
+        regexPattern = `${regexPattern}.*`
+      }
+
+      patterns.push(new RegExp(`^${regexPattern}$`))
+    }
+  }
+
+  return patterns
+}
+
+// Calculate the total size of files that would be included in the package
+function calculatePackageSize() {
+  const npmignorePatterns = parseNpmignore()
+  const includePatterns = parsePackageFiles()
+
+  let totalSize = 0
+  let includedFiles = []
+
+  function processDirectory(dirPath) {
+    const entries = fs.readdirSync(dirPath, { withFileTypes: true })
+
+    for (const entry of entries) {
+      const fullPath = path.join(dirPath, entry.name)
+
+      if (entry.isDirectory()) {
+        processDirectory(fullPath)
+      } else if (entry.isFile()) {
+        if (shouldIncludeFile(fullPath, npmignorePatterns, includePatterns)) {
+          const sizeInMB = getFileSizeInMB(fullPath)
+          totalSize += sizeInMB
+          includedFiles.push({ path: fullPath, size: sizeInMB })
+        }
+      }
+    }
+  }
+
+  processDirectory('.')
+
+  // Sort files by size (largest first)
+  includedFiles.sort((a, b) => b.size - a.size)
+
+  console.log('Estimated package size: ' + totalSize.toFixed(2) + ' MB')
+  console.log('\nLargest files:')
+  for (let i = 0; i < Math.min(10, includedFiles.length); i++) {
+    console.log(
+      `${includedFiles[i].path}: ${includedFiles[i].size.toFixed(2)} MB`
+    )
+  }
+}
+
+calculatePackageSize()