docs: consolidate and archive redundant documentation

- Archived 13 API design iterations to docs/api-design-archive/
- Consolidated augmentation docs to docs/augmentations-archive/
- Maintained ONE definitive API doc at docs/api/README.md
- Cleaned up documentation structure for 2.0 release
- Preserved all historical documents for reference
This commit is contained in:
David Snelling 2025-08-25 10:15:38 -07:00
parent ef8f35ec2a
commit 994276f09f
35 changed files with 444 additions and 239 deletions

View file

@ -0,0 +1,549 @@
# 🌐 Brainy API Exposure Architecture
## 📡 Current State: Built-in MCP Support
Brainy **already has** Model Context Protocol (MCP) support built-in:
```typescript
// Already exists in Brainy!
import { BrainyMCPService } from 'brainy/mcp'
const brain = new BrainyData()
const mcpService = new BrainyMCPService(brain)
// Handles MCP requests
const response = await mcpService.handleRequest({
type: 'data_access',
operation: 'search',
parameters: { query: 'find documents' }
})
```
### What's Already Built:
- **BrainyMCPAdapter** - Exposes data operations
- **MCPAugmentationToolset** - Exposes augmentations as MCP tools
- **BrainyMCPService** - Unified service layer
- **ServerSearchConduitAugmentation** - Connect to remote Brainy instances
## 🚀 The Missing Piece: API Server Augmentation
What's **NOT** built yet is a unified API server that exposes REST, WebSocket, and MCP over network. This SHOULD be an augmentation!
```typescript
/**
* Universal API Server Augmentation
* Exposes Brainy through REST, WebSocket, MCP, and GraphQL
*/
export class APIServerAugmentation extends BaseAugmentation {
readonly name = 'api-server'
readonly timing = 'after' as const
readonly operations = ['all'] as const // Monitor all operations
readonly priority = 5 // Low priority, runs last
private httpServer?: any
private wsServer?: any
private mcpService?: BrainyMCPService
private clients = new Set<any>()
private apiKeys = new Map<string, any>()
protected async onInitialize(): Promise<void> {
const config = this.context.config.apiServer || {}
if (!config.enabled) {
this.log('API Server disabled in config')
return
}
// Initialize MCP service
this.mcpService = new BrainyMCPService(this.context.brain, {
enableAuth: config.requireAuth
})
// Start servers based on environment
if (typeof process !== 'undefined' && process.versions?.node) {
await this.startNodeServers(config)
} else if (typeof Deno !== 'undefined') {
await this.startDenoServer(config)
} else if (typeof self !== 'undefined') {
await this.startServiceWorker(config)
}
}
private async startNodeServers(config: any) {
const express = await import('express')
const { WebSocketServer } = await import('ws')
const cors = await import('cors')
const app = express.default()
// Middleware
app.use(cors.default(config.cors))
app.use(express.json())
app.use(this.authMiddleware.bind(this))
app.use(this.rateLimitMiddleware.bind(this))
// REST API Routes
this.setupRESTRoutes(app)
// Start HTTP server
this.httpServer = app.listen(config.port || 3000, () => {
this.log(`REST API listening on port ${config.port || 3000}`)
})
// WebSocket server for real-time
this.wsServer = new WebSocketServer({
server: this.httpServer,
path: '/ws'
})
this.setupWebSocketServer()
// MCP over WebSocket
this.setupMCPWebSocket()
}
private setupRESTRoutes(app: any) {
// Health check
app.get('/health', (req: any, res: any) => {
res.json({ status: 'healthy', version: '2.0.0' })
})
// Search endpoint
app.post('/api/search', async (req: any, res: any) => {
try {
const { query, limit = 10, options = {} } = req.body
const results = await this.context.brain.search(query, limit, options)
res.json({ success: true, results })
} catch (error) {
res.status(500).json({
success: false,
error: error.message
})
}
})
// Add data endpoint
app.post('/api/add', async (req: any, res: any) => {
try {
const { content, metadata } = req.body
const id = await this.context.brain.add(content, metadata)
res.json({ success: true, id })
} catch (error) {
res.status(500).json({
success: false,
error: error.message
})
}
})
// Get endpoint
app.get('/api/get/:id', async (req: any, res: any) => {
try {
const data = await this.context.brain.get(req.params.id)
res.json({ success: true, data })
} catch (error) {
res.status(404).json({
success: false,
error: 'Not found'
})
}
})
// Delete endpoint
app.delete('/api/delete/:id', async (req: any, res: any) => {
try {
await this.context.brain.delete(req.params.id)
res.json({ success: true })
} catch (error) {
res.status(500).json({
success: false,
error: error.message
})
}
})
// Relate endpoint
app.post('/api/relate', async (req: any, res: any) => {
try {
const { source, target, verb, metadata } = req.body
await this.context.brain.relate(source, target, verb, metadata)
res.json({ success: true })
} catch (error) {
res.status(500).json({
success: false,
error: error.message
})
}
})
// Find endpoint (complex queries)
app.post('/api/find', async (req: any, res: any) => {
try {
const results = await this.context.brain.find(req.body)
res.json({ success: true, results })
} catch (error) {
res.status(500).json({
success: false,
error: error.message
})
}
})
// Cluster endpoint
app.post('/api/cluster', async (req: any, res: any) => {
try {
const { algorithm = 'kmeans', options = {} } = req.body
const clusters = await this.context.brain.cluster(algorithm, options)
res.json({ success: true, clusters })
} catch (error) {
res.status(500).json({
success: false,
error: error.message
})
}
})
// MCP endpoint (for non-WebSocket MCP)
app.post('/api/mcp', async (req: any, res: any) => {
try {
const response = await this.mcpService.handleRequest(req.body)
res.json(response)
} catch (error) {
res.status(500).json({
success: false,
error: error.message
})
}
})
// GraphQL endpoint (optional)
if (this.context.config.apiServer?.enableGraphQL) {
this.setupGraphQL(app)
}
}
private setupWebSocketServer() {
this.wsServer.on('connection', (ws: any) => {
this.clients.add(ws)
ws.on('message', async (message: string) => {
try {
const msg = JSON.parse(message)
await this.handleWebSocketMessage(msg, ws)
} catch (error) {
ws.send(JSON.stringify({
type: 'error',
error: error.message
}))
}
})
ws.on('close', () => {
this.clients.delete(ws)
})
})
}
private async handleWebSocketMessage(msg: any, ws: any) {
switch (msg.type) {
case 'subscribe':
// Subscribe to operations
ws.subscriptions = msg.operations || ['all']
ws.send(JSON.stringify({
type: 'subscribed',
operations: ws.subscriptions
}))
break
case 'search':
const results = await this.context.brain.search(msg.query, msg.limit)
ws.send(JSON.stringify({
type: 'searchResults',
results
}))
break
case 'mcp':
// Handle MCP over WebSocket
const response = await this.mcpService.handleRequest(msg.request)
ws.send(JSON.stringify({
type: 'mcpResponse',
response
}))
break
}
}
private setupMCPWebSocket() {
// Dedicated MCP WebSocket endpoint
const { WebSocketServer } = require('ws')
const mcpWs = new WebSocketServer({
port: (this.context.config.apiServer?.mcpPort || 3001),
path: '/mcp'
})
mcpWs.on('connection', (ws: any) => {
ws.on('message', async (message: string) => {
try {
const request = JSON.parse(message)
const response = await this.mcpService.handleRequest(request)
ws.send(JSON.stringify(response))
} catch (error) {
ws.send(JSON.stringify({
error: error.message,
type: 'error'
}))
}
})
})
this.log(`MCP WebSocket listening on port ${this.context.config.apiServer?.mcpPort || 3001}`)
}
// Broadcast changes to all connected clients
async execute<T>(operation: string, params: any, next: () => Promise<T>): Promise<T> {
const result = await next()
// Broadcast to WebSocket clients
if (this.clients.size > 0) {
const message = JSON.stringify({
type: 'operation',
operation,
params: this.sanitizeParams(params),
timestamp: Date.now()
})
for (const client of this.clients) {
if (client.subscriptions?.includes('all') ||
client.subscriptions?.includes(operation)) {
client.send(message)
}
}
}
return result
}
private authMiddleware(req: any, res: any, next: any) {
if (!this.context.config.apiServer?.requireAuth) {
return next()
}
const apiKey = req.headers['x-api-key']
if (!apiKey || !this.apiKeys.has(apiKey)) {
return res.status(401).json({ error: 'Unauthorized' })
}
req.user = this.apiKeys.get(apiKey)
next()
}
private rateLimitMiddleware(req: any, res: any, next: any) {
// Simple rate limiting
const ip = req.ip
const limit = this.context.config.apiServer?.rateLimit || 100
// Implementation details...
next()
}
private sanitizeParams(params: any) {
// Remove sensitive data before broadcasting
const safe = { ...params }
delete safe.apiKey
delete safe.password
return safe
}
protected async onShutdown() {
// Close all connections
for (const client of this.clients) {
client.close()
}
// Close servers
if (this.httpServer) {
await new Promise(resolve => this.httpServer.close(resolve))
}
if (this.wsServer) {
this.wsServer.close()
}
}
}
```
## 🎯 Usage: Deploy Brainy as a Server
```typescript
import { BrainyData } from 'brainy'
import { APIServerAugmentation } from 'brainy/augmentations'
const brain = new BrainyData({
apiServer: {
enabled: true,
port: 3000,
mcpPort: 3001,
requireAuth: true,
rateLimit: 100,
cors: { origin: '*' },
enableGraphQL: false
}
})
// Register the API server augmentation
brain.augmentations.register(new APIServerAugmentation())
await brain.init()
console.log('Brainy API Server running!')
console.log('REST API: http://localhost:3000')
console.log('WebSocket: ws://localhost:3000/ws')
console.log('MCP: ws://localhost:3001/mcp')
```
## 🔌 Client Usage
### REST API
```javascript
// Search
const response = await fetch('http://localhost:3000/api/search', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'X-API-Key': 'your-key'
},
body: JSON.stringify({
query: 'find documents about AI',
limit: 10
})
})
const { results } = await response.json()
```
### WebSocket (Real-time)
```javascript
const ws = new WebSocket('ws://localhost:3000/ws')
ws.onopen = () => {
// Subscribe to operations
ws.send(JSON.stringify({
type: 'subscribe',
operations: ['add', 'delete', 'relate']
}))
}
ws.onmessage = (event) => {
const msg = JSON.parse(event.data)
console.log('Operation:', msg.operation, msg.params)
}
```
### MCP (AI Agents)
```javascript
const mcpWs = new WebSocket('ws://localhost:3001/mcp')
mcpWs.send(JSON.stringify({
type: 'data_access',
operation: 'search',
requestId: '123',
parameters: { query: 'test' }
}))
mcpWs.onmessage = (event) => {
const response = JSON.parse(event.data)
console.log('MCP Response:', response)
}
```
## 🏗️ Architecture Benefits
### Why as an Augmentation?
1. **Optional** - Not everyone needs a server
2. **Configurable** - Easy to enable/disable
3. **Extensible** - Add custom endpoints
4. **Integrated** - Hooks into all operations
5. **Real-time** - Broadcasts changes automatically
### Security Features
- **API Key Authentication**
- **Rate Limiting**
- **CORS Configuration**
- **Parameter Sanitization**
- **SSL/TLS Support** (with proper certs)
## 🌍 Deployment Options
### Local Development
```bash
npm install brainy
node server.js # Your server file with APIServerAugmentation
```
### Docker Container
```dockerfile
FROM node:18-alpine
WORKDIR /app
COPY package*.json ./
RUN npm install brainy
COPY server.js .
EXPOSE 3000 3001
CMD ["node", "server.js"]
```
### Cloud Deployment
Deploy to any Node.js hosting:
- Vercel Edge Functions
- Cloudflare Workers (with adapter)
- AWS Lambda (with adapter)
- Google Cloud Run
- Traditional VPS
### Browser Service Worker
```javascript
// In browser, use Service Worker for local API
if ('serviceWorker' in navigator) {
// APIServerAugmentation can create a Service Worker
// that intercepts fetch() calls and handles them locally
}
```
## 🎯 The Complete Picture
```
┌─────────────────────────────────────────────┐
│ Client Applications │
├─────────────┬───────────┬──────────────────┤
│ REST │ WebSocket │ MCP │
└──────┬──────┴─────┬─────┴──────┬───────────┘
│ │ │
▼ ▼ ▼
┌─────────────────────────────────────────────┐
│ APIServerAugmentation │
│ (Unified API exposure as augmentation) │
└─────────────────┬───────────────────────────┘
┌─────────────────────────────────────────────┐
│ BrainyData Core │
│ (with all augmentations in pipeline) │
└─────────────────────────────────────────────┘
┌─────────────────────────────────────────────┐
│ Storage Layer │
│ (FileSystem, S3, OPFS, Memory) │
└─────────────────────────────────────────────┘
```
## 🚀 Summary
1. **MCP is built-in** - Already in Brainy core
2. **API Server should be an augmentation** - Optional, configurable
3. **Exposes everything** - REST, WebSocket, MCP, GraphQL
4. **Real-time by default** - Broadcasts all operations
5. **Secure** - Auth, rate limiting, CORS
6. **Deploy anywhere** - Node, Deno, Browser, Cloud
The beauty is that the API server is just another augmentation - it hooks into the pipeline like everything else and exposes Brainy's capabilities to the world!

View file

@ -0,0 +1,774 @@
# 🚀 Real-World Augmentation Examples
## 1. 💬 Chat Interface Augmentation
**"Talk to your data through natural language"**
```typescript
import { BaseAugmentation } from './brainyAugmentation.js'
export class ChatInterfaceAugmentation extends BaseAugmentation {
readonly name = 'chat-interface'
readonly timing = 'after' as const // Process after operations
readonly operations = ['search', 'add', 'delete'] as const
readonly priority = 30 // Medium priority
private chatHistory: Array<{role: string, content: string}> = []
private llmClient: any // User's chosen LLM
protected async onInitialize(): Promise<void> {
// User provides their own LLM
this.llmClient = this.context.config.llmClient || null
if (!this.llmClient) {
this.log('Chat augmentation needs LLM client in config')
}
}
async execute<T>(operation: string, params: any, next: () => Promise<T>): Promise<T> {
// If params include natural language query
if (params.chatQuery) {
// Convert natural language to Brainy operations
const intent = await this.parseIntent(params.chatQuery)
// Transform params based on intent
if (intent.type === 'search') {
params.query = intent.query
params.k = intent.limit || 10
} else if (intent.type === 'add') {
params.content = intent.content
params.metadata = { ...params.metadata, source: 'chat' }
}
// Store in chat history
this.chatHistory.push({
role: 'user',
content: params.chatQuery
})
}
// Execute the operation
const result = await next()
// Generate conversational response
if (params.chatQuery && this.llmClient) {
const response = await this.generateResponse(operation, result)
this.chatHistory.push({
role: 'assistant',
content: response
})
// Enhance result with chat response
return {
...result,
chatResponse: response,
chatHistory: this.chatHistory
} as T
}
return result
}
private async parseIntent(query: string) {
// Use Brainy's NLP patterns + LLM to understand intent
const prompt = `Parse this query into a Brainy operation:
Query: ${query}
Return JSON with:
- type: 'search' | 'add' | 'delete' | 'relate'
- query: search terms or content
- filters: any metadata filters
- limit: number of results`
const response = await this.llmClient.complete(prompt)
return JSON.parse(response)
}
private async generateResponse(operation: string, result: any) {
const prompt = `Generate a friendly response for this operation:
Operation: ${operation}
Result: ${JSON.stringify(result).slice(0, 500)}
Chat History: ${JSON.stringify(this.chatHistory.slice(-3))}
Be conversational and helpful.`
return await this.llmClient.complete(prompt)
}
}
// Usage:
const brain = new BrainyData({
augmentations: [
new ChatInterfaceAugmentation()
],
llmClient: openai // Bring your own LLM
})
// Now you can chat!
const result = await brain.search({
chatQuery: "Show me all documents about project roadmap from last week"
})
console.log(result.chatResponse) // "I found 5 documents about the project roadmap..."
```
## 2. 🤖 MCP Agent Memory Augmentation
**"Provide persistent memory for AI agents through MCP"**
```typescript
import { BaseAugmentation } from './brainyAugmentation.js'
import { Server } from '@modelcontextprotocol/sdk'
export class MCPAgentMemoryAugmentation extends BaseAugmentation {
readonly name = 'mcp-agent-memory'
readonly timing = 'around' as const // Wrap operations
readonly operations = ['all'] as const // Monitor everything
readonly priority = 70 // High priority
private mcpServer: Server
private agentSessions: Map<string, any> = new Map()
protected async onInitialize(): Promise<void> {
// Initialize MCP server
this.mcpServer = new Server({
name: 'brainy-memory',
version: '1.0.0'
})
// Register MCP tools for agents
this.mcpServer.setRequestHandler('tools/list', () => ({
tools: [
{
name: 'remember',
description: 'Store information in long-term memory',
inputSchema: {
type: 'object',
properties: {
content: { type: 'string' },
category: { type: 'string' },
importance: { type: 'number' }
}
}
},
{
name: 'recall',
description: 'Retrieve information from memory',
inputSchema: {
type: 'object',
properties: {
query: { type: 'string' },
category: { type: 'string' },
limit: { type: 'number' }
}
}
},
{
name: 'forget',
description: 'Remove information from memory',
inputSchema: {
type: 'object',
properties: {
query: { type: 'string' },
category: { type: 'string' }
}
}
}
]
}))
// Handle tool calls from agents
this.mcpServer.setRequestHandler('tools/call', async (request) => {
const { name, arguments: args } = request.params
switch (name) {
case 'remember':
return await this.rememberForAgent(args)
case 'recall':
return await this.recallForAgent(args)
case 'forget':
return await this.forgetForAgent(args)
default:
throw new Error(`Unknown tool: ${name}`)
}
})
// Start MCP server
await this.mcpServer.connect(process.stdin, process.stdout)
this.log('MCP Agent Memory server started')
}
async execute<T>(operation: string, params: any, next: () => Promise<T>): Promise<T> {
// Extract agent context if present
const agentId = params.metadata?._agentId || 'default'
const sessionId = params.metadata?._sessionId
// Track agent operations
if (agentId && sessionId) {
if (!this.agentSessions.has(sessionId)) {
this.agentSessions.set(sessionId, {
agentId,
startTime: Date.now(),
operations: []
})
}
const session = this.agentSessions.get(sessionId)
session.operations.push({
operation,
params: { ...params },
timestamp: Date.now()
})
}
// Execute with agent context
const result = await next()
// Auto-remember important operations
if (operation === 'add' && agentId) {
await this.autoRemember(agentId, params, result)
}
return result
}
private async rememberForAgent(args: any) {
// Store in Brainy with agent-specific metadata
const id = await this.context.brain.add(args.content, {
_agentMemory: true,
_agentId: args.agentId || 'default',
category: args.category,
importance: args.importance || 0.5,
timestamp: new Date().toISOString()
})
return {
content: [
{
type: 'text',
text: `Remembered with ID: ${id}`
}
]
}
}
private async recallForAgent(args: any) {
// Search agent's memories
const results = await this.context.brain.search(args.query, args.limit || 10, {
where: {
_agentMemory: true,
_agentId: args.agentId || 'default',
category: args.category
}
})
return {
content: [
{
type: 'text',
text: JSON.stringify(results, null, 2)
}
]
}
}
private async forgetForAgent(args: any) {
// Remove specific memories
const results = await this.context.brain.find({
where: {
_agentMemory: true,
_agentId: args.agentId || 'default',
category: args.category
}
})
for (const item of results) {
await this.context.brain.delete(item.id)
}
return {
content: [
{
type: 'text',
text: `Forgot ${results.length} memories`
}
]
}
}
private async autoRemember(agentId: string, params: any, result: any) {
// Automatically remember important information
if (params.metadata?.important) {
await this.context.brain.add(params.content, {
...params.metadata,
_agentMemory: true,
_agentId: agentId,
_autoRemembered: true,
_originalOperation: 'add',
_resultId: result
})
}
}
}
// Usage:
const brain = new BrainyData({
augmentations: [
new MCPAgentMemoryAugmentation()
]
})
// Now AI agents can use Brainy as memory through MCP!
// Agents connect via MCP and use remember/recall/forget tools
```
## 3. 🌐 API Server Augmentation
**"Expose Brainy through REST, WebSocket, and MCP APIs"**
```typescript
import { BaseAugmentation } from './brainyAugmentation.js'
import { BrainyMCPService } from '../mcp/brainyMCPService.js'
export class APIServerAugmentation extends BaseAugmentation {
readonly name = 'api-server'
readonly timing = 'after' as const
readonly operations = ['all'] as ('all')[]
readonly priority = 5 // Low priority, runs after other augmentations
private httpServer: any
private wsServer: any
private mcpService: BrainyMCPService
protected async onInitialize(): Promise<void> {
// Initialize MCP service
this.mcpService = new BrainyMCPService(this.context.brain)
// Start HTTP server with REST endpoints
await this.startHTTPServer()
// Start WebSocket server for real-time
await this.startWebSocketServer()
this.log(`API Server running on port ${this.config.port || 3000}`)
}
async execute<T>(operation: string, params: any, next: () => Promise<T>): Promise<T> {
const result = await next()
// Broadcast operation to WebSocket clients
this.broadcast({
type: 'operation',
operation,
params: this.sanitizeParams(params),
timestamp: Date.now()
})
return result
}
private async startHTTPServer() {
// REST endpoints: /api/search, /api/add, /api/get/:id, etc.
// MCP endpoint: /api/mcp
// Health check: /health
}
private async startWebSocketServer() {
// WebSocket for real-time subscriptions
// Clients can subscribe to specific operations
}
}
// Usage:
const brain = new BrainyData()
brain.augmentations.register(new APIServerAugmentation({ port: 3000 }))
await brain.init()
// Now access Brainy via:
// - REST: http://localhost:3000/api/*
// - WebSocket: ws://localhost:3000/ws
// - MCP: http://localhost:3000/api/mcp
```
## 4. 📊 Graph Visualization Augmentation
**"Real-time graph visualization with clustering"**
```typescript
import { BaseAugmentation } from './brainyAugmentation.js'
import { WebSocketServer } from 'ws'
export class GraphVisualizationAugmentation extends BaseAugmentation {
readonly name = 'graph-visualization'
readonly timing = 'after' as const
readonly operations = ['all'] as const // Monitor all changes
readonly priority = 20
private wsServer: WebSocketServer
private graphState: {
nodes: Map<string, any>
edges: Map<string, any>
clusters: Map<string, Set<string>>
}
private clients: Set<any> = new Set()
protected async onInitialize(): Promise<void> {
// Initialize WebSocket server for real-time updates
this.wsServer = new WebSocketServer({
port: this.context.config.visualizationPort || 8080
})
this.graphState = {
nodes: new Map(),
edges: new Map(),
clusters: new Map()
}
// Load initial graph state
await this.loadGraphState()
// Handle client connections
this.wsServer.on('connection', (ws) => {
this.clients.add(ws)
// Send initial state
ws.send(JSON.stringify({
type: 'init',
data: this.serializeGraphState()
}))
// Handle client messages
ws.on('message', async (message) => {
const msg = JSON.parse(message.toString())
await this.handleClientMessage(msg, ws)
})
ws.on('close', () => {
this.clients.delete(ws)
})
})
// Start clustering in background
this.startClusteringWorker()
this.log('Graph visualization server started on port ' +
(this.context.config.visualizationPort || 8080))
}
async execute<T>(operation: string, params: any, next: () => Promise<T>): Promise<T> {
const result = await next()
// Update graph state based on operation
switch (operation) {
case 'add':
case 'addNoun':
await this.handleNodeAdded(result, params)
break
case 'relate':
case 'addVerb':
await this.handleEdgeAdded(params)
break
case 'delete':
await this.handleNodeDeleted(params)
break
case 'search':
await this.handleSearchPerformed(params, result)
break
}
return result
}
private async handleNodeAdded(id: string, data: any) {
// Add node to graph
const node = {
id,
label: data.content?.slice(0, 50) || id,
type: data.metadata?.type || 'default',
metadata: data.metadata,
position: this.calculatePosition(id),
clusterId: null
}
this.graphState.nodes.set(id, node)
// Broadcast to clients
this.broadcast({
type: 'nodeAdded',
data: node
})
// Trigger re-clustering
this.scheduleReClustering()
}
private async handleEdgeAdded(params: any) {
const edge = {
id: `${params.source}-${params.verb}-${params.target}`,
source: params.source,
target: params.target,
label: params.verb,
weight: params.weight || 1
}
this.graphState.edges.set(edge.id, edge)
this.broadcast({
type: 'edgeAdded',
data: edge
})
}
private async handleSearchPerformed(params: any, results: any) {
// Highlight search results in visualization
const highlightNodes = results.map((r: any) => r.id)
this.broadcast({
type: 'highlight',
data: {
nodes: highlightNodes,
query: params.query,
duration: 5000 // Highlight for 5 seconds
}
})
}
private async loadGraphState() {
// Load all nodes (nouns)
const nouns = await this.context.brain.getAllNouns()
for (const noun of nouns) {
this.graphState.nodes.set(noun.id, {
id: noun.id,
label: noun.content?.slice(0, 50) || noun.id,
type: noun.type,
metadata: noun.metadata,
position: this.calculatePosition(noun.id)
})
}
// Load all edges (verbs/relationships)
const verbs = await this.context.brain.getAllVerbs()
for (const verb of verbs) {
this.graphState.edges.set(verb.id, {
id: verb.id,
source: verb.source,
target: verb.target,
label: verb.type,
weight: verb.weight
})
}
// Initial clustering
await this.performClustering()
}
private async performClustering() {
// Use Brainy's clustering capabilities
const clusteringResult = await this.context.brain.cluster({
algorithm: 'hierarchical',
threshold: 0.7
})
// Update cluster state
this.graphState.clusters.clear()
for (const [clusterId, nodeIds] of Object.entries(clusteringResult)) {
this.graphState.clusters.set(clusterId, new Set(nodeIds as string[]))
// Update nodes with cluster IDs
for (const nodeId of nodeIds as string[]) {
const node = this.graphState.nodes.get(nodeId)
if (node) {
node.clusterId = clusterId
}
}
}
// Broadcast cluster update
this.broadcast({
type: 'clustersUpdated',
data: this.serializeClusters()
})
}
private startClusteringWorker() {
// Re-cluster periodically or when graph changes significantly
setInterval(async () => {
if (this.graphState.nodes.size > 0) {
await this.performClustering()
}
}, 30000) // Every 30 seconds
}
private scheduleReClustering = (() => {
let timeout: NodeJS.Timeout
return () => {
clearTimeout(timeout)
timeout = setTimeout(() => this.performClustering(), 5000)
}
})()
private calculatePosition(id: string) {
// Simple force-directed layout position
const hash = id.split('').reduce((a, b) => {
a = ((a << 5) - a) + b.charCodeAt(0)
return a & a
}, 0)
return {
x: (hash % 1000) - 500,
y: ((hash * 7) % 1000) - 500
}
}
private broadcast(message: any) {
const data = JSON.stringify(message)
for (const client of this.clients) {
client.send(data)
}
}
private async handleClientMessage(msg: any, ws: any) {
switch (msg.type) {
case 'requestClustering':
await this.performClustering()
break
case 'search':
const results = await this.context.brain.search(msg.query)
ws.send(JSON.stringify({
type: 'searchResults',
data: results
}))
break
case 'getNodeDetails':
const node = await this.context.brain.get(msg.nodeId)
ws.send(JSON.stringify({
type: 'nodeDetails',
data: node
}))
break
case 'expandNode':
const connections = await this.context.brain.getConnections(msg.nodeId)
ws.send(JSON.stringify({
type: 'nodeConnections',
data: connections
}))
break
}
}
private serializeGraphState() {
return {
nodes: Array.from(this.graphState.nodes.values()),
edges: Array.from(this.graphState.edges.values()),
clusters: this.serializeClusters()
}
}
private serializeClusters() {
const clusters: any = {}
for (const [id, nodeIds] of this.graphState.clusters) {
clusters[id] = Array.from(nodeIds)
}
return clusters
}
protected async onShutdown() {
this.wsServer.close()
this.clients.clear()
}
}
// Usage:
const brain = new BrainyData({
augmentations: [
new GraphVisualizationAugmentation()
],
visualizationPort: 8080
})
// Now connect a web-based graph viz tool to ws://localhost:8080
// It receives real-time updates as data changes!
```
## 4. 🌐 Multi-Agent Team Coordination
**"Multiple AI agents sharing knowledge and coordinating tasks"**
```typescript
export class TeamCoordinationAugmentation extends BaseAugmentation {
readonly name = 'team-coordination'
readonly timing = 'around' as const
readonly operations = ['all'] as const
readonly priority = 85
private agents: Map<string, AgentState> = new Map()
private tasks: Map<string, Task> = new Map()
private sharedMemory: Map<string, any> = new Map()
async execute<T>(operation: string, params: any, next: () => Promise<T>): Promise<T> {
const agentId = params.metadata?._agentId
if (agentId) {
// Track agent activity
this.updateAgentState(agentId, operation, params)
// Check if operation needs coordination
if (await this.needsCoordination(operation, params)) {
return await this.coordinatedExecute(agentId, operation, params, next)
}
}
return next()
}
private async coordinatedExecute<T>(
agentId: string,
operation: string,
params: any,
next: () => Promise<T>
): Promise<T> {
// Acquire distributed lock
const lockId = await this.acquireLock(operation, params)
try {
// Check shared memory for related work
const relatedWork = await this.findRelatedWork(params)
if (relatedWork) {
params.metadata._relatedWork = relatedWork
}
// Execute with team context
const result = await next()
// Update shared memory
await this.updateSharedMemory(agentId, operation, params, result)
// Notify other agents
await this.notifyTeam(agentId, operation, result)
return result
} finally {
await this.releaseLock(lockId)
}
}
}
```
## 🎯 Key Patterns
All these augmentations follow the same pattern:
1. **Extend BaseAugmentation**
2. **Define timing & operations**
3. **Initialize resources** in `onInitialize()`
4. **Intercept operations** in `execute()`
5. **Clean up** in `onShutdown()`
They can:
- **Add APIs** (REST, WebSocket, MCP)
- **Transform data** (chat queries → operations)
- **Coordinate agents** (distributed locking, shared memory)
- **Visualize in real-time** (WebSocket broadcasts)
- **Integrate any service** (LLMs, databases, APIs)
The beauty is they all use the **same simple interface** but achieve vastly different goals!

View file

@ -0,0 +1,306 @@
# 🔄 How Augmentations Hook Into Brainy
## The Complete Pipeline Architecture
```
User Code → BrainyData Method → Augmentation Pipeline → Storage/Operations
↑ ↓
└────── Augmentations Execute Here ──┘
```
## 🎯 How Augmentations Register & Execute
### 1. **Registration During Initialization**
```typescript
// In BrainyData constructor/init
class BrainyData {
private augmentations = new AugmentationRegistry()
async init() {
// Register built-in augmentations in priority order
this.augmentations.register(new WALAugmentation()) // Priority: 100
this.augmentations.register(new EntityRegistryAugmentation()) // Priority: 90
this.augmentations.register(new NeuralImportAugmentation()) // Priority: 80
this.augmentations.register(new BatchProcessingAugmentation()) // Priority: 50
// Initialize all with context
const context: AugmentationContext = {
brain: this,
storage: this.storage,
config: this.config,
log: (msg, level) => console.log(msg)
}
await this.augmentations.initialize(context)
}
}
```
### 2. **Execution Through Method Interception**
Every BrainyData operation wraps its core logic with augmentation execution:
```typescript
// Example: The add() method
async add(content: string, metadata?: any): Promise<string> {
// Augmentations wrap the core operation
return this.augmentations.execute(
'add', // Operation name
{ content, metadata }, // Parameters
async () => { // Core operation
// Actual add logic here
const id = generateId()
await this.storage.set(id, { content, metadata })
return id
}
)
}
```
### 3. **The Execution Chain**
```typescript
// In AugmentationRegistry
async execute<T>(operation: string, params: any, mainOperation: () => Promise<T>): Promise<T> {
// 1. Filter augmentations that should run for this operation
const applicable = this.augmentations.filter(aug =>
aug.shouldExecute(operation, params)
)
// 2. Sort by priority (already sorted during registration)
// Priority 100 runs first, then 90, 80, etc.
// 3. Create middleware chain
let index = 0
const executeNext = async (): Promise<T> => {
if (index >= applicable.length) {
// All augmentations processed, run main operation
return mainOperation()
}
const augmentation = applicable[index++]
// Each augmentation decides what to do with the operation
return augmentation.execute(operation, params, executeNext)
}
return executeNext()
}
```
## 🎭 The Four Timing Modes in Action
### **`timing: 'before'`** - Pre-processing
```typescript
class NeuralImportAugmentation {
timing = 'before'
async execute(op, params, next) {
// Analyze data BEFORE storage
const analysis = await this.analyzeWithAI(params.content)
params.metadata._neural = analysis
// Continue with enhanced params
return next()
}
}
```
### **`timing: 'after'`** - Post-processing
```typescript
class NotionSynapse {
timing = 'after'
async execute(op, params, next) {
// Let operation complete first
const result = await next()
// Then sync to Notion
await this.syncToNotion(op, params, result)
return result
}
}
```
### **`timing: 'around'`** - Wrapping
```typescript
class WALAugmentation {
timing = 'around'
async execute(op, params, next) {
// Write to WAL before
await this.wal.write({ op, params, timestamp: Date.now() })
try {
// Execute operation
const result = await next()
// Mark as committed
await this.wal.commit()
return result
} catch (error) {
// Rollback on failure
await this.wal.rollback()
throw error
}
}
}
```
### **`timing: 'replace'`** - Complete replacement
```typescript
class S3StorageAugmentation {
timing = 'replace'
async execute(op, params, next) {
if (op === 'storage.get') {
// Don't call next() - completely replace
return await this.s3.getObject(params.key)
}
// For other operations, pass through
return next()
}
}
```
## 📊 Real Example: How `brain.add()` Works
```typescript
// User calls:
await brain.add("John is a developer", { type: "person" })
// This triggers the chain:
1. BrainyData.add() calls augmentations.execute('add', params, coreLogic)
2. AugmentationRegistry filters applicable augmentations:
- WALAugmentation (priority: 100, operations: ['all'])
- EntityRegistryAugmentation (priority: 90, operations: ['add'])
- NeuralImportAugmentation (priority: 80, operations: ['add'])
- BatchProcessingAugmentation (priority: 50, operations: ['add'])
3. Execution chain (highest priority first):
WALAugmentation.execute() {
await wal.write(operation) // Log to WAL
const result = await next() // Call next in chain
await wal.commit() // Commit WAL
return result
}
EntityRegistryAugmentation.execute() {
const hash = computeHash(params.content)
if (registry.has(hash)) {
return registry.get(hash) // Return existing ID
}
const result = await next() // Continue chain
registry.set(hash, result) // Register new entity
return result
}
NeuralImportAugmentation.execute() {
const analysis = await analyzeWithAI(params)
params.metadata._neural = analysis // Add AI insights
return next() // Continue with enhanced data
}
BatchProcessingAugmentation.execute() {
batch.add(params) // Add to batch
if (batch.isFull()) {
await batch.flush() // Process batch if full
}
return next() // Continue
}
Core add() logic {
// Finally, the actual storage operation
const id = generateId()
await storage.set(id, params)
await index.add(id, vector)
return id
}
```
## 🔌 Dynamic Registration
Augmentations can be registered at any time:
```typescript
// During initialization
brain.augmentations.register(new CustomAugmentation())
// Or later, dynamically
const synapse = new NotionSynapse({ apiKey: 'xxx' })
brain.augmentations.register(synapse)
// From Brain Cloud marketplace
import { EmotionalIntelligence } from '@brain-cloud/empathy'
brain.augmentations.register(new EmotionalIntelligence())
```
## 🎯 Operation Targeting
Augmentations declare which operations they care about:
```typescript
class SearchOptimizer {
operations = ['search', 'searchText', 'findSimilar'] // Only search ops
}
class GlobalLogger {
operations = ['all'] // Every operation
}
class StorageReplacer {
operations = ['storage'] // Storage operations only
}
```
## 🔍 On-Demand Execution
Some augmentations can be triggered manually:
```typescript
// Get specific augmentation
const neuralImport = brain.augmentations.get('neural-import')
// Use its public API directly
const analysis = await neuralImport.getNeuralAnalysis(data, 'json')
// Or trigger through operations
await brain.add(data) // Automatically uses neural import if registered
```
## 📈 Priority System
```
100: Critical Infrastructure (WAL, Transactions)
90: Data Integrity (Entity Registry, Deduplication)
80: Data Processing (Neural Import, Transformation)
50: Performance (Batching, Caching)
10: Features (Scoring, Analytics)
1: Monitoring (Logging, Metrics)
```
## 🌊 The Flow
1. **User Action**`brain.add()`, `brain.search()`, etc.
2. **Method Wraps** → Core logic wrapped with `augmentations.execute()`
3. **Filter** → Find augmentations for this operation
4. **Sort** → Order by priority
5. **Chain** → Each augmentation calls next() or not
6. **Core** → Eventually hits actual implementation
7. **Unwind** → Results flow back through chain
8. **Return** → Enhanced result to user
## 💡 Key Insights
1. **Everything is interceptable** - All operations go through the pipeline
2. **Augmentations compose** - They stack like middleware
3. **Priority matters** - Higher priority runs first
4. **Timing is flexible** - before/after/around/replace covers all needs
5. **Simple but powerful** - One interface, infinite possibilities
This is why the single `BrainyAugmentation` interface works for EVERYTHING - it's just middleware with superpowers! 🚀

View file

@ -0,0 +1,288 @@
# Simple Guide: Creating Augmentations
## The One Interface That Rules Them All
**EVERY augmentation is a `BrainyAugmentation`:**
```typescript
interface BrainyAugmentation {
name: string // Unique name
timing: 'before' | 'after' | 'around' | 'replace' // When to run
operations: string[] // What to intercept
priority: number // Order (higher = first)
initialize(context): Promise<void> // Setup
execute(op, params, next): Promise<T> // Do work
shutdown?(): Promise<void> // Cleanup (optional)
}
```
That's it! Every augmentation implements this interface.
## Creating Different Types of Augmentations
### 1. Basic Feature Augmentation
**Use Case:** Add logging, caching, validation, etc.
```typescript
import { BaseAugmentation } from 'brainy'
export class LoggingAugmentation extends BaseAugmentation {
name = 'logging'
timing = 'around' // Wrap operations
operations = ['add', 'delete'] // What to log
priority = 10 // Low priority
async execute(op, params, next) {
console.log(`Starting ${op}`)
const result = await next()
console.log(`Completed ${op}`)
return result
}
}
// Usage
brain.augmentations.register(new LoggingAugmentation())
```
### 2. Storage Augmentation
**Use Case:** Provide a storage backend (special: has `provideStorage()` method)
```typescript
import { StorageAugmentation } from 'brainy'
export class RedisStorageAugmentation extends StorageAugmentation {
constructor(config) {
super('redis-storage') // Pass name to parent
this.config = config
}
// Special method for storage only!
async provideStorage() {
return new RedisAdapter(this.config)
}
}
// Usage (BEFORE init!)
brain.augmentations.register(new RedisStorageAugmentation({
host: 'localhost',
port: 6379
}))
await brain.init() // Will use Redis!
```
### 3. Data Processing Augmentation
**Use Case:** Transform or validate data before storage
```typescript
export class ValidationAugmentation extends BaseAugmentation {
name = 'validator'
timing = 'before' // Run before operation
operations = ['add'] // Validate on add
priority = 50
async execute(op, params, next) {
// Validate data
if (!params.data || !params.data.title) {
throw new Error('Title is required')
}
// Add timestamp
params.data.createdAt = new Date()
// Continue with modified params
return next()
}
}
```
### 4. External System Augmentation (Synapse)
**Use Case:** Sync with external systems like Notion, Slack, etc.
```typescript
export class NotionSyncAugmentation extends BaseAugmentation {
name = 'notion-sync'
timing = 'after' // Sync after local operation
operations = ['add', 'update', 'delete']
priority = 30
private notion: NotionClient
async initialize(context) {
await super.initialize(context)
this.notion = new NotionClient(this.apiKey)
}
async execute(op, params, next) {
// Do local operation first
const result = await next()
// Then sync to Notion
if (op === 'add') {
await this.notion.createPage({
title: params.data.title,
content: params.data.content
})
}
return result
}
}
```
### 5. Performance Optimization Augmentation
**Use Case:** Add caching, batching, deduplication
```typescript
export class CacheAugmentation extends BaseAugmentation {
name = 'smart-cache'
timing = 'around' // Wrap to check cache
operations = ['search'] // Cache searches only
priority = 60
private cache = new Map()
async execute(op, params, next) {
const key = JSON.stringify(params)
// Check cache
if (this.cache.has(key)) {
this.log('Cache hit!')
return this.cache.get(key)
}
// Miss - execute and cache
const result = await next()
this.cache.set(key, result)
// Clear old entries if too many
if (this.cache.size > 1000) {
const firstKey = this.cache.keys().next().value
this.cache.delete(firstKey)
}
return result
}
}
```
## Quick Reference: When to Use Each Timing
| Timing | Use For | Example |
|--------|---------|---------|
| `before` | Validation, transformation | Check required fields |
| `after` | Logging, syncing, analytics | Send to external API |
| `around` | Caching, error handling, timing | Wrap with try/catch |
| `replace` | Complete replacement | Storage backends |
## Quick Reference: Common Operations
| Operation | Description |
|-----------|-------------|
| `'add'` | Adding data to brain |
| `'search'` | Searching/querying |
| `'update'` | Updating existing data |
| `'delete'` | Removing data |
| `'storage'` | Storage resolution (special) |
| `'all'` | Intercept everything |
## The Context Object
Every augmentation gets this during `initialize()`:
```typescript
{
brain: BrainyData, // The brain instance
storage: StorageAdapter, // Storage backend
config: BrainyDataConfig, // Configuration
log: (msg, level) => void // Logger
}
```
## Priority Guidelines
| Priority | Use For |
|----------|---------|
| 100 | Storage (critical infrastructure) |
| 80-99 | System operations (WAL, connections) |
| 50-79 | Performance (caching, batching) |
| 20-49 | Features (validation, transformation) |
| 1-19 | Logging, analytics |
## Complete Working Example
Here's a full augmentation that adds word count to all documents:
```typescript
import { BaseAugmentation } from 'brainy'
export class WordCountAugmentation extends BaseAugmentation {
name = 'word-counter'
timing = 'before'
operations = ['add', 'update']
priority = 40
async execute(operation, params, next) {
// Add word count to metadata
if (params.data && params.data.content) {
const wordCount = params.data.content.split(/\s+/).length
params.metadata = params.metadata || {}
params.metadata.wordCount = wordCount
this.log(`Added word count: ${wordCount}`)
}
// Continue with enhanced params
return next()
}
async initialize(context) {
await super.initialize(context)
this.log('Word counter ready!')
}
}
// Usage
const brain = new BrainyData()
brain.augmentations.register(new WordCountAugmentation())
await brain.init()
// Now all adds include word count
await brain.add('Hello world', {
content: 'This is a test document with nine words here'
})
// Automatically adds: metadata.wordCount = 9
```
## Key Points to Remember
1. **All augmentations are `BrainyAugmentation`** - One interface
2. **Storage augmentations** add `provideStorage()` method
3. **Register before `init()`** for storage, anytime for others
4. **Use `BaseAugmentation`** for convenience (has helpers)
5. **`next()` is crucial** - Always call it (unless `replace`)
6. **Order matters** - Use priority to control execution order
## Testing Your Augmentation
```typescript
describe('MyAugmentation', () => {
it('should enhance data', async () => {
const brain = new BrainyData()
brain.augmentations.register(new MyAugmentation())
await brain.init()
await brain.add('test', { data: 'test' })
const result = await brain.search('test')
expect(result[0].metadata.enhanced).toBe(true)
})
})
```
That's it! Augmentations are simple middleware that intercept operations. Pick your timing, operations, and priority, then implement `execute()`!

View file

@ -0,0 +1,292 @@
# 🧠 The Complete Brainy Architecture Vision
## 🎯 The Genius: Everything is an Augmentation
```
🧠 BRAINY CORE
┌────────┴────────┐
│ Augmentations │
│ Pipeline │
└────────┬────────┘
┌────────────────────┼────────────────────┐
│ │ │
Data Processing External API Exposure
Augmentations Connections Augmentations
│ │ │
NeuralImport Synapses APIServer
EntityRegistry (Notion,etc) (REST/WS)
BatchProcessing │ MCPServer
IntelligentScoring │ GraphQLServer
│ │ │
└────────────────────┴────────────────────┘
Storage Layer
(FS, S3, OPFS, Memory)
```
## 🔄 How It All Works Together
### 1. **Core Pipeline**
Every operation flows through the augmentation pipeline:
```typescript
User Action → BrainyData Method → Augmentation Pipeline → Storage
All Augmentations Execute Here
```
### 2. **Augmentation Categories (All Using Same Interface!)**
#### 🧬 **Data Processing** (timing: 'before')
- **NeuralImport** - AI understands data before storage
- **EntityRegistry** - Deduplicates entities
- **BatchProcessing** - Optimizes bulk operations
#### 🌐 **External Connections** (timing: 'after')
- **Synapses** - Sync with Notion, Salesforce, etc.
- **WebSocketBroadcast** - Real-time updates to clients
- **TeamCoordination** - Multi-agent synchronization
#### 📡 **API Exposure** (timing: 'after' or separate process)
- **APIServerAugmentation** - REST/WebSocket/MCP server
- **GraphQLAugmentation** - GraphQL endpoint
- **ServiceWorkerAugmentation** - Browser local API
#### 💾 **Storage Backends** (timing: 'replace')
- **S3StorageAugmentation** - Use S3 instead of local
- **RedisAugmentation** - Use Redis for caching
- **PostgresAugmentation** - Use Postgres for persistence
#### 🛡️ **Infrastructure** (timing: 'around')
- **WALAugmentation** - Write-ahead logging
- **TransactionAugmentation** - ACID transactions
- **CacheAugmentation** - Multi-level caching
## 🌟 The Beautiful Simplicity
### One Interface Rules All
```typescript
interface BrainyAugmentation {
name: string
timing: 'before' | 'after' | 'around' | 'replace'
operations: string[]
priority: number
initialize(context): Promise<void>
execute(operation, params, next): Promise<any>
shutdown?(): Promise<void>
}
```
This single interface can:
- **Process data** with AI
- **Connect** to any external service
- **Expose** APIs (REST, WebSocket, MCP, GraphQL)
- **Replace** storage backends
- **Add** infrastructure (WAL, transactions, caching)
- **Coordinate** distributed systems
- **Visualize** data in real-time
- Literally **ANYTHING**
## 🏗️ Real-World Deployment Architecture
### Scenario 1: Local Development
```typescript
const brain = new BrainyData({
augmentations: [
new NeuralImportAugmentation(), // AI processing
new EntityRegistryAugmentation(), // Deduplication
new WALAugmentation() // Durability
]
})
```
### Scenario 2: Production Server
```typescript
const brain = new BrainyData({
augmentations: [
// Infrastructure
new WALAugmentation(),
new ConnectionPoolAugmentation(),
new RequestDeduplicatorAugmentation(),
// Data Processing
new NeuralImportAugmentation(),
new EntityRegistryAugmentation(),
new BatchProcessingAugmentation(),
// External Connections
new NotionSynapse({ apiKey: 'xxx' }),
new SlackSynapse({ token: 'xxx' }),
// API Exposure
new APIServerAugmentation({ port: 3000 }),
new MCPServerAugmentation({ port: 3001 }),
// Monitoring
new MetricsAugmentation(),
new LoggingAugmentation()
]
})
```
### Scenario 3: Distributed AI Agent System
```typescript
const brain = new BrainyData({
augmentations: [
// Agent Coordination
new TeamCoordinationAugmentation(),
new DistributedLockAugmentation(),
new SharedMemoryAugmentation(),
// Agent Memory
new MCPAgentMemoryAugmentation(),
new ConversationHistoryAugmentation(),
// Real-time Communication
new WebSocketBroadcastAugmentation(),
new PubSubAugmentation(),
// Visualization
new GraphVisualizationAugmentation()
]
})
```
## 🔌 How API Exposure Works
The **APIServerAugmentation** is special - it can run in two modes:
### Mode 1: Embedded (Same Process)
```typescript
brain.augmentations.register(new APIServerAugmentation())
// API server runs in same process, hooks into pipeline
```
### Mode 2: Standalone (Separate Process)
```typescript
// server.js - separate file
import { BrainyData } from 'brainy'
import { APIServerAugmentation } from 'brainy/augmentations'
const brain = new BrainyData()
const apiServer = new APIServerAugmentation()
// Can also run as standalone server connecting to remote Brainy
apiServer.connectToRemoteBrainy('ws://brainy-host:8080')
apiServer.listen(3000)
```
## 🎭 The Four Timing Modes in Practice
### System Startup Sequence
```
1. INITIALIZE Phase
└─> All augmentations initialize (storage, connections, servers)
2. OPERATION Phase (for each operation)
├─> 'before' augmentations (NeuralImport, Validation)
├─> 'around' augmentations start (WAL, Transactions)
├─> 'replace' augmentations (if any, skip core)
├─> Core operation (or replaced operation)
├─> 'around' augmentations complete (Commit/Rollback)
└─> 'after' augmentations (Sync, Broadcast, Log)
3. SHUTDOWN Phase
└─> All augmentations cleanup (close connections, flush buffers)
```
## 🌍 Deployment Patterns
### Pattern 1: Monolithic
Everything in one process:
```
┌─────────────────────────────┐
│ Single Node.js Process │
│ ┌───────────────────────┐ │
│ │ BrainyData Core │ │
│ ├───────────────────────┤ │
│ │ All Augmentations │ │
│ ├───────────────────────┤ │
│ │ API Server │ │
│ └───────────────────────┘ │
└─────────────────────────────┘
```
### Pattern 2: Microservices
Distributed across services:
```
┌──────────────┐ ┌──────────────┐ ┌──────────────┐
│ Brainy Core │────▶│ API Gateway │────▶│ Clients │
└──────┬───────┘ └──────────────┘ └──────────────┘
├──────────────┐
▼ ▼
┌──────────────┐ ┌──────────────┐
│ Synapses │ │ AI Agents │
│ Service │ │ Service │
└──────────────┘ └──────────────┘
```
### Pattern 3: Edge Computing
Brainy at the edge:
```
┌─────────────────────────────────────┐
│ CloudFlare Worker │
│ ┌───────────────────────────────┐ │
│ │ Brainy (Memory Storage) │ │
│ │ + API Server Augmentation │ │
│ └───────────────────────────────┘ │
└─────────────────────────────────────┘
┌───────────────┐
│ S3 Storage │
└───────────────┘
```
## 🚀 The Power of Composition
Any combination works because everything uses the same interface:
```typescript
// Local AI Assistant
[NeuralImport, ChatInterface, LocalStorage]
// Production API
[WAL, S3Storage, APIServer, RateLimiting]
// Multi-Agent System
[TeamCoordination, MCPServer, GraphVisualization]
// Data Pipeline
[KafkaConsumer, NeuralImport, PostgresStorage]
// Real-time Analytics
[StreamProcessing, Clustering, WebSocketBroadcast]
```
## 🎯 Key Insights
1. **No Special Cases** - Everything is an augmentation
2. **Complete Flexibility** - Mix and match any combination
3. **Environment Agnostic** - Works in browser, Node, Deno, edge
4. **Protocol Agnostic** - REST, WebSocket, MCP, GraphQL, gRPC
5. **Storage Agnostic** - Local, S3, Redis, Postgres, anything
6. **Infinitely Extensible** - Just add more augmentations
## 🧠 The Philosophy
> "Make everything an augmentation, and the system becomes infinitely flexible while remaining dead simple."
This is why Brainy can be:
- A local embedded database
- A distributed knowledge graph
- An AI agent memory system
- A real-time collaboration platform
- A data pipeline processor
- All of the above simultaneously
**One interface. Infinite possibilities. That's the Brainy way.** 🚀

View file

@ -0,0 +1,18 @@
# Augmentations Documentation Archive
This directory contains historical augmentation documentation from the Brainy 2.0 development process.
## Current Documentation
The definitive augmentation documentation is now located at:
- **`/docs/augmentations/README.md`** - Main augmentation guide
- **`/docs/architecture/augmentations.md`** - Architecture details
## Archived Documents
These documents represent the evolution of the augmentation system design:
- Various implementation approaches
- Pipeline architecture exploration
- Storage augmentation patterns
- Example implementations
## Note
These documents are archived to preserve development history while maintaining a clean documentation structure.

View file

@ -0,0 +1,276 @@
# Storage Augmentations Guide
## Overview
Brainy uses a unified augmentation system for storage backends. This guide explains the difference between built-in storage augmentations and how to create custom ones.
## Built-in Storage Augmentations
These wrap existing, battle-tested storage adapters from `/storage/adapters/`:
| Augmentation | Underlying Adapter | Environment | Description |
|--------------|-------------------|-------------|-------------|
| `MemoryStorageAugmentation` | `MemoryStorage` | Universal | Fast in-memory storage (not persistent) |
| `FileSystemStorageAugmentation` | `FileSystemStorage` | Node.js | Persistent file-based storage |
| `OPFSStorageAugmentation` | `OPFSStorage` | Browser | Browser persistent storage |
| `S3StorageAugmentation` | `S3CompatibleStorage` | Universal | Amazon S3 with throttling & caching |
| `R2StorageAugmentation` | `R2Storage` | Universal | Cloudflare R2 storage |
| `GCSStorageAugmentation` | `S3CompatibleStorage` | Universal | Google Cloud Storage |
### Architecture of Built-in Storage
```
BrainyData
StorageAugmentation (thin wrapper)
StorageAdapter (actual implementation in /storage/adapters/)
Actual Storage (filesystem, S3, memory, etc.)
```
### Why This Design?
1. **Preserve existing code** - Storage adapters have years of bug fixes
2. **Complex features intact** - S3 throttling, caching, retry logic preserved
3. **Minimal wrapper** - Augmentations are just 20-30 lines
4. **Zero feature loss** - All 30+ StorageAdapter methods work unchanged
## Using Built-in Storage
### 1. Zero-Config (Auto-Selection)
```typescript
const brain = new BrainyData()
await brain.init()
// Automatically selects:
// - Node.js → FileSystemStorage
// - Browser → OPFSStorage (or Memory fallback)
```
### 2. Configuration-Based
```typescript
const brain = new BrainyData({
storage: {
s3Storage: {
bucketName: 'my-bucket',
accessKeyId: 'xxx',
secretAccessKey: 'yyy'
}
}
})
```
### 3. Augmentation Override
```typescript
const brain = new BrainyData()
brain.augmentations.register(new S3StorageAugmentation({
bucketName: 'my-bucket',
region: 'us-east-1',
accessKeyId: 'xxx',
secretAccessKey: 'yyy'
}))
await brain.init()
```
## Creating Custom Storage Augmentations
Custom storage augmentations can either:
1. Wrap an existing adapter (like built-ins do)
2. Implement the StorageAdapter interface directly
### Option 1: Wrapping an Existing Adapter
```typescript
import { StorageAugmentation } from 'brainy'
import { CustomAdapter } from './my-custom-adapter'
export class CustomStorageAugmentation extends StorageAugmentation {
private config: CustomConfig
constructor(config: CustomConfig) {
super('custom-storage')
this.config = config
}
async provideStorage(): Promise<StorageAdapter> {
// Create and return your adapter
const adapter = new CustomAdapter(this.config)
return adapter
}
}
```
### Option 2: Self-Contained Implementation
```typescript
import { StorageAugmentation, StorageAdapter } from 'brainy'
import Redis from 'ioredis'
export class RedisStorageAugmentation extends StorageAugmentation {
private redis: Redis
constructor(config: RedisConfig) {
super('redis-storage')
this.redis = new Redis(config)
}
async provideStorage(): Promise<StorageAdapter> {
// Return an object implementing StorageAdapter
return {
async init() {
await this.redis.ping()
},
async saveNoun(noun) {
await this.redis.set(
`noun:${noun.id}`,
JSON.stringify(noun)
)
},
async getNoun(id) {
const data = await this.redis.get(`noun:${id}`)
return data ? JSON.parse(data) : null
},
async deleteNoun(id) {
await this.redis.del(`noun:${id}`)
},
// ... implement all 30+ required methods
// See StorageAdapter interface in coreTypes.ts
}
}
}
```
## StorageAdapter Interface Requirements
Your custom storage must implement these core methods:
```typescript
interface StorageAdapter {
// Initialization
init(): Promise<void>
// Noun operations
saveNoun(noun: HNSWNoun): Promise<void>
getNoun(id: string): Promise<HNSWNoun | null>
deleteNoun(id: string): Promise<void>
getNounsByNounType(type: string): Promise<HNSWNoun[]>
// Verb operations
saveVerb(verb: HNSWVerb): Promise<void>
getVerb(id: string): Promise<HNSWVerb | null>
deleteVerb(id: string): Promise<void>
getVerbsBySource(sourceId: string): Promise<HNSWVerb[]>
getVerbsByTarget(targetId: string): Promise<HNSWVerb[]>
// Metadata operations
saveMetadata(id: string, metadata: any): Promise<void>
getMetadata(id: string): Promise<any | null>
saveVerbMetadata(id: string, metadata: any): Promise<void>
getVerbMetadata(id: string): Promise<any | null>
// Pagination
getNouns(options?: PaginationOptions): Promise<PaginatedResult>
getVerbs(options?: PaginationOptions): Promise<PaginatedResult>
// Statistics
getStatistics(): Promise<StatisticsData | null>
saveStatistics(stats: StatisticsData): Promise<void>
incrementStatistic(type: string, service: string): Promise<void>
// Utility
clear(): Promise<void>
getStorageStatus(): Promise<StorageStatus>
// ... plus ~10 more methods
}
```
## Publishing to Brain Cloud (Future)
Custom storage augmentations can be published to the Brain Cloud marketplace:
```json
// package.json
{
"name": "@brain-cloud/redis-storage",
"version": "1.0.0",
"brainy": {
"type": "augmentation",
"category": "storage",
"implements": "StorageAdapter"
}
}
```
Users will be able to install via:
```bash
brainy augment install redis-storage
```
## Best Practices
1. **Use existing adapters when possible** - They're well-tested
2. **Implement all methods** - StorageAdapter has 30+ required methods
3. **Handle errors gracefully** - Storage is critical infrastructure
4. **Include connection pooling** - For network-based storage
5. **Add retry logic** - Network operations can fail
6. **Implement caching** - Reduce latency for hot data
7. **Track statistics** - Use BaseStorageAdapter if possible
8. **Document configuration** - Make it easy for users
## Examples in the Wild
### MongoDB Storage (Community)
```typescript
class MongoStorageAugmentation extends StorageAugmentation {
async provideStorage() {
const client = new MongoClient(this.uri)
const db = client.db('brainy')
return {
async saveNoun(noun) {
await db.collection('nouns').replaceOne(
{ _id: noun.id },
noun,
{ upsert: true }
)
},
// ... full implementation
}
}
}
```
### PostgreSQL Storage (Premium)
```typescript
class PostgreSQLStorageAugmentation extends StorageAugmentation {
async provideStorage() {
const pool = new Pool(this.config)
return {
async saveNoun(noun) {
await pool.query(
'INSERT INTO nouns (id, data) VALUES ($1, $2) ON CONFLICT (id) DO UPDATE SET data = $2',
[noun.id, JSON.stringify(noun)]
)
},
// ... full implementation
}
}
}
```
## Summary
- **Built-in augmentations** wrap existing adapters (thin layer)
- **Custom augmentations** can wrap OR implement directly
- **Storage adapters** in `/storage/adapters/` are for core only
- **Premium storage** comes as self-contained augmentations
- **Everything uses** the same StorageAdapter interface
- **Zero-config** still works perfectly
This design provides maximum flexibility while preserving all existing functionality!

View file

@ -0,0 +1,239 @@
# 🧠 Unified Augmentation System
## The Single Interface That Rules Them All
Brainy uses ONE elegant interface for ALL augmentations:
```typescript
interface BrainyAugmentation {
name: string
timing: 'before' | 'after' | 'around' | 'replace'
operations: string[]
priority: number
initialize(context): Promise<void>
execute<T>(operation, params, next): Promise<T>
shutdown?(): Promise<void>
}
```
## Why This Works for EVERYTHING
### 🎭 The Four Timing Modes
1. **`before`**: Pre-process data
- Data validation
- Authentication checks
- Input transformation
2. **`after`**: Post-process results
- Logging
- Analytics
- Cache updates
3. **`around`**: Wrap operations (middleware)
- Error handling
- Performance monitoring
- Transaction management
4. **`replace`**: Complete replacement
- Alternative storage backends
- Mock implementations
- Custom algorithms
### 🎯 Operation Targeting
Augmentations can target:
- Specific operations: `['add', 'search']`
- All operations: `['all']`
- Pattern matching: Operations containing certain strings
### 🔄 The Execute Chain
```typescript
async execute<T>(operation, params, next): Promise<T> {
// Before logic
console.log(`Starting ${operation}`)
// Call next (or don't!)
const result = await next()
// After logic
console.log(`Completed ${operation}`)
return result
}
```
## 📦 Categories of Augmentations
While using the same interface, augmentations naturally fall into categories:
### 1. **Data Processing**
```typescript
class NeuralImportAugmentation {
timing = 'before'
operations = ['add', 'addNoun']
async execute(op, params, next) {
// Analyze data with AI
const enhanced = await this.processWithAI(params)
// Continue with enhanced data
return next(enhanced)
}
}
```
### 2. **External Connections (Synapses)**
```typescript
class NotionSynapse {
timing = 'after'
operations = ['add', 'update', 'delete']
async initialize(context) {
await this.connectToNotion()
}
async execute(op, params, next) {
const result = await next()
// Sync to Notion after local operation
await this.syncToNotion(op, params)
return result
}
}
```
### 3. **Storage Backends**
```typescript
class S3StorageAugmentation {
timing = 'replace'
operations = ['storage']
async execute(op, params, next) {
// Don't call next() - replace entirely
return await this.s3Client.store(params)
}
}
```
### 4. **Real-time Communication**
```typescript
class WebSocketBroadcast {
timing = 'after'
operations = ['all']
async initialize(context) {
this.ws = new WebSocket(url)
}
async execute(op, params, next) {
const result = await next()
// Broadcast changes
this.ws.send({ op, params, result })
return result
}
}
```
### 5. **AI Agent Coordination**
```typescript
class TeamMemoryAugmentation {
timing = 'around'
operations = ['add', 'search']
async execute(op, params, next) {
// Acquire distributed lock
await this.acquireLock(op)
try {
// Synchronize with team
const teamData = await this.syncWithTeam(params)
const result = await next(teamData)
// Broadcast result to team
await this.broadcastToTeam(result)
return result
} finally {
await this.releaseLock(op)
}
}
}
```
### 6. **Analytics & Prediction**
```typescript
class PredictiveAnalytics {
timing = 'after'
operations = ['search']
async execute(op, params, next) {
const results = await next()
// Analyze search patterns
this.recordPattern(params, results)
// Add predictions
results.predictions = await this.predict(params)
return results
}
}
```
## 🔌 How Augmentations Connect
```typescript
// In BrainyData initialization
const brain = new BrainyData({
augmentations: [
new NeuralImportAugmentation(),
new NotionSynapse({ apiKey: 'xxx' }),
new TeamMemoryAugmentation(),
new PredictiveAnalytics()
]
})
// Or dynamically
brain.augmentations.register(new CustomAugmentation())
```
## 🎯 Priority System
```typescript
// Execution order (highest first)
100: Critical (WAL, Storage)
50: Performance (Cache, Dedup)
10: Features (Scoring, Analytics)
1: Optional (Logging)
```
## 🌍 Brain Cloud Integration
All augmentations (free, community, premium) use this SAME interface:
```typescript
// From Brain Cloud marketplace
import { EmotionalIntelligence } from '@brainy-cloud/empathy'
const empathy = new EmotionalIntelligence()
// It's just a BrainyAugmentation!
brain.augmentations.register(empathy)
```
## 💡 Why This Design Wins
1. **Simplicity**: One interface to learn
2. **Flexibility**: Can do literally anything
3. **Composability**: Stack augmentations like middleware
4. **Extensibility**: Easy to add new augmentations
5. **Marketplace Ready**: All augmentations compatible
## 🚀 The Future is Unified
No more complex type hierarchies. No more ISenseAugmentation, IConduitAugmentation, etc.
Just one beautiful, simple interface that can:
- Process data with AI
- Connect to any platform
- Coordinate AI teams
- Provide predictive analytics
- Add empathy to AI
- Store anywhere
- Communicate in real-time
- And literally anything else you can imagine
**One interface. Infinite possibilities. That's the Brainy way.** 🧠✨