From b5bc9000cf3f3e98d10479c2d19b400ac70deb1e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 20 Jan 2026 16:21:11 -0800 Subject: [PATCH] feat: Integration Hub for external tool connectivity - Add native config option: `new Brainy({ integrations: true })` - OData integration for Excel Power Query and Power BI - Google Sheets integration with Apps Script - Server-Sent Events (SSE) for real-time streaming - Webhooks for push notifications - Zero-config with sensible defaults - Full tree-shaking when disabled --- integrations/README.md | 350 ++++++++ integrations/google-sheets/Code.gs | 451 ++++++++++ integrations/google-sheets/README.md | 142 ++++ integrations/google-sheets/Sidebar.html | 421 ++++++++++ integrations/google-sheets/appsscript.json | 11 + src/brainy.ts | 68 +- src/index.ts | 83 ++ src/integrations/core/EventBus.ts | 313 +++++++ src/integrations/core/IntegrationBase.ts | 425 ++++++++++ src/integrations/core/IntegrationHub.ts | 368 +++++++++ src/integrations/core/IntegrationLoader.ts | 279 +++++++ src/integrations/core/TabularExporter.ts | 574 +++++++++++++ src/integrations/core/index.ts | 64 ++ src/integrations/core/types.ts | 261 ++++++ src/integrations/index.ts | 138 ++++ src/integrations/odata/EdmxGenerator.ts | 238 ++++++ src/integrations/odata/ODataIntegration.ts | 710 ++++++++++++++++ src/integrations/odata/ODataQueryParser.ts | 650 +++++++++++++++ src/integrations/odata/index.ts | 24 + .../sheets/GoogleSheetsIntegration.ts | 771 ++++++++++++++++++ src/integrations/sheets/index.ts | 12 + src/integrations/sse/SSEIntegration.ts | 449 ++++++++++ src/integrations/sse/index.ts | 12 + .../webhooks/WebhookIntegration.ts | 548 +++++++++++++ src/integrations/webhooks/index.ts | 12 + src/types/brainy.types.ts | 43 + tests/integrations/core.test.ts | 427 ++++++++++ tests/integrations/odata.test.ts | 343 ++++++++ 28 files changed, 8186 insertions(+), 1 deletion(-) create mode 100644 integrations/README.md create mode 100644 integrations/google-sheets/Code.gs create mode 100644 integrations/google-sheets/README.md create mode 100644 integrations/google-sheets/Sidebar.html create mode 100644 integrations/google-sheets/appsscript.json create mode 100644 src/integrations/core/EventBus.ts create mode 100644 src/integrations/core/IntegrationBase.ts create mode 100644 src/integrations/core/IntegrationHub.ts create mode 100644 src/integrations/core/IntegrationLoader.ts create mode 100644 src/integrations/core/TabularExporter.ts create mode 100644 src/integrations/core/index.ts create mode 100644 src/integrations/core/types.ts create mode 100644 src/integrations/index.ts create mode 100644 src/integrations/odata/EdmxGenerator.ts create mode 100644 src/integrations/odata/ODataIntegration.ts create mode 100644 src/integrations/odata/ODataQueryParser.ts create mode 100644 src/integrations/odata/index.ts create mode 100644 src/integrations/sheets/GoogleSheetsIntegration.ts create mode 100644 src/integrations/sheets/index.ts create mode 100644 src/integrations/sse/SSEIntegration.ts create mode 100644 src/integrations/sse/index.ts create mode 100644 src/integrations/webhooks/WebhookIntegration.ts create mode 100644 src/integrations/webhooks/index.ts create mode 100644 tests/integrations/core.test.ts create mode 100644 tests/integrations/odata.test.ts diff --git a/integrations/README.md b/integrations/README.md new file mode 100644 index 00000000..aa3d795b --- /dev/null +++ b/integrations/README.md @@ -0,0 +1,350 @@ +# Brainy Integrations + +Connect Brainy to spreadsheets, BI tools, and external systems with zero configuration. + +## Quick Start + +```typescript +import { Brainy } from '@soulcraft/brainy' + +const brain = new Brainy({ integrations: true }) +await brain.init() + +// That's it! Your endpoints are ready: +console.log(brain.hub.endpoints) +// { odata: '/odata', sheets: '/sheets', sse: '/events', webhooks: '/webhooks' } +``` + +## Supported Tools + +| Tool | Protocol | How to Connect | +|------|----------|----------------| +| Excel | OData | Data → Get Data → From OData Feed → `http://your-server/odata` | +| Power BI | OData | Get Data → OData Feed → `http://your-server/odata` | +| Tableau | OData | Connect → OData → `http://your-server/odata` | +| Google Sheets | REST | Install Apps Script add-on (see below) | +| Any SSE Client | SSE | `new EventSource('http://your-server/events')` | + +--- + +## Excel (Power Query) + +### Connect in 3 clicks: + +1. **Data** tab → **Get Data** → **From Other Sources** → **From OData Feed** +2. Enter URL: `http://your-server/odata` +3. Click **OK** → **Load** + +### Query Options + +Excel Power Query supports OData query parameters: + +``` +/odata/Entities?$filter=Type eq 'person'&$top=100 +/odata/Entities?$select=Id,Type,Metadata_name +/odata/Entities?$orderby=CreatedAt desc +/odata/Entities?$search=machine learning +``` + +### Refresh Data + +- Manual: **Data** → **Refresh All** +- Automatic: **Query Properties** → Set refresh interval + +--- + +## Power BI + +### Connect: + +1. **Get Data** → **OData Feed** +2. Enter URL: `http://your-server/odata` +3. Select tables: `Entities`, `Relationships` +4. Click **Load** + +### Advanced + +Power BI DirectQuery is supported for real-time data. + +--- + +## Tableau + +### Connect: + +1. **Connect** → **To a Server** → **OData** +2. Enter: `http://your-server/odata` +3. Drag tables to canvas + +--- + +## Google Sheets + +### Setup (5 minutes): + +1. Open your Google Sheet +2. **Extensions** → **Apps Script** +3. Delete existing code +4. Copy `integrations/google-sheets/Code.gs` into the editor +5. Create new file `Sidebar.html`, paste from `integrations/google-sheets/Sidebar.html` +6. **Project Settings** → **Script properties** → Add: + - `BRAINY_URL` = `http://your-server` +7. Save and refresh spreadsheet + +### Custom Functions: + +``` +=BRAINY_QUERY("type:person", 100) // Query entities +=BRAINY_GET("entity-id") // Get one entity +=BRAINY_SIMILAR("machine learning", 5) // Semantic search +=BRAINY_RELATIONS("entity-id", "from") // Get relationships +``` + +### Sidebar: + +**Extensions** → **Brainy** → **Open Sidebar** + +- Search and insert results +- Add new entities +- Sync selected range to Brainy + +--- + +## Real-Time Streaming (SSE) + +### JavaScript: + +```javascript +const source = new EventSource('http://your-server/events') + +source.onmessage = (event) => { + const data = JSON.parse(event.data) + console.log('Change:', data) +} + +// Filter by type +const filtered = new EventSource('http://your-server/events?types=noun&operations=create,update') +``` + +### Query Parameters: + +- `types` - Entity types: `noun`, `verb`, `vfs` +- `operations` - Operations: `create`, `update`, `delete` +- `nounTypes` - Filter by noun type: `person`, `document`, etc. + +--- + +## Webhooks + +Push events to external URLs: + +```typescript +const brain = new Brainy({ integrations: true }) +await brain.init() + +// Register a webhook +await brain.hub.webhooks?.register({ + url: 'https://your-app.com/webhook', + events: { entityTypes: ['noun'], operations: ['create', 'update'] }, + secret: 'your-hmac-secret' // optional, for signature verification +}) +``` + +### Webhook Payload: + +```json +{ + "events": [ + { + "id": "event-123", + "type": "noun", + "operation": "create", + "entityId": "entity-456", + "timestamp": 1704067200000 + } + ], + "deliveredAt": 1704067201000 +} +``` + +### Signature Verification: + +Webhooks include `X-Brainy-Signature` header with HMAC-SHA256 signature. + +--- + +## Server Examples + +### Minimal (in-memory): + +```typescript +import { Brainy } from '@soulcraft/brainy' + +const brain = new Brainy({ integrations: true }) +await brain.init() + +// Add some data +await brain.add({ type: 'Person', metadata: { name: 'Alice' } }) + +// Get connection instructions +console.log(brain.hub.getInstructions()) +``` + +### Express: + +```typescript +import express from 'express' +import { Brainy } from '@soulcraft/brainy' + +const app = express() +const brain = new Brainy({ + storage: { type: 'filesystem', options: { basePath: './data' } }, + integrations: true +}) +await brain.init() + +app.use(express.json()) + +// Route all integrations +app.all(['/odata/*', '/sheets/*', '/events/*'], async (req, res) => { + const response = await brain.hub.handleRequest({ + method: req.method, + path: req.path, + query: req.query as Record, + headers: req.headers as Record, + body: req.body + }) + + res.status(response.status).set(response.headers) + if (response.body) res.json(response.body) + else res.end() +}) + +app.listen(3000, () => { + console.log('Brainy server ready!') + console.log('Excel/Power BI: http://localhost:3000/odata') + console.log('Google Sheets: http://localhost:3000/sheets') + console.log('SSE Stream: http://localhost:3000/events') +}) +``` + +### Hono (Cloudflare Workers): + +```typescript +import { Hono } from 'hono' +import { Brainy } from '@soulcraft/brainy' + +const app = new Hono() + +const brain = new Brainy({ + storage: { type: 'memory' }, + integrations: true +}) +await brain.init() + +app.all('/odata/*', async (c) => { + const response = await brain.hub.handleRequest({ + method: c.req.method, + path: c.req.path, + query: Object.fromEntries(new URL(c.req.url).searchParams), + headers: Object.fromEntries(c.req.raw.headers), + body: await c.req.json().catch(() => undefined) + }) + return c.json(response.body, response.status) +}) + +export default app +``` + +--- + +## Configuration + +### Custom Base Path: + +```typescript +const brain = new Brainy({ + integrations: { basePath: '/api/v1' } +}) +await brain.init() + +// Endpoints become: +// /api/v1/odata +// /api/v1/sheets +// /api/v1/events +``` + +### Select Integrations: + +```typescript +// Only OData and Sheets +const brain = new Brainy({ + integrations: { enable: ['odata', 'sheets'] } +}) +await brain.init() +``` + +### Per-Integration Config: + +```typescript +const brain = new Brainy({ + integrations: { + config: { + odata: { basePath: '/data' }, + sse: { heartbeatInterval: 15000 } + } + } +}) +await brain.init() +``` + +--- + +## Troubleshooting + +### Excel: "Can't connect to OData feed" +- Ensure server is running and accessible +- Check firewall settings +- Try opening URL in browser first + +### Google Sheets: "Brainy URL not configured" +- Add `BRAINY_URL` in Apps Script → Project Settings → Script properties + +### Power BI: "Invalid OData" +- Ensure `$metadata` endpoint returns valid XML +- Try: `http://your-server/odata/$metadata` + +### SSE: "Connection dropped" +- Check network/proxy timeout settings +- The integration sends heartbeats every 30 seconds by default + +--- + +## API Reference + +### OData Endpoints + +``` +GET /odata/$metadata - Schema (XML) +GET /odata/Entities - List entities +GET /odata/Entities('id') - Get entity +GET /odata/Relationships - List relationships +POST /odata/Entities - Create entity +``` + +### Sheets Endpoints + +``` +GET /sheets/query?q=...&limit=100 - Query entities +GET /sheets/entity/:id - Get entity +GET /sheets/similar?text=...&k=10 - Semantic search +POST /sheets/add - Add entity +``` + +### SSE Endpoint + +``` +GET /events - All events +GET /events?types=noun - Filter by type +GET /events?operations=create,update - Filter by operation +``` diff --git a/integrations/google-sheets/Code.gs b/integrations/google-sheets/Code.gs new file mode 100644 index 00000000..898f19a2 --- /dev/null +++ b/integrations/google-sheets/Code.gs @@ -0,0 +1,451 @@ +/** + * Brainy Google Sheets Add-on + * + * Custom functions and sidebar for two-way sync with Brainy. + * + * Setup: + * 1. Open Script Editor: Extensions → Apps Script + * 2. Paste this code into Code.gs + * 3. Set your Brainy URL: Tools → Script properties → Add BRAINY_URL + * 4. Refresh your spreadsheet + * + * Custom Functions: + * - =BRAINY_QUERY(query, limit) - Query entities + * - =BRAINY_GET(id) - Get entity by ID + * - =BRAINY_SIMILAR(text, limit) - Semantic search + * - =BRAINY_RELATIONS(entityId, direction) - Get relationships + * - =BRAINY_TYPES() - List available types + * + * For full documentation: https://github.com/soulcraft/brainy + */ + +// Configuration +const CONFIG_BRAINY_URL = 'BRAINY_URL'; +const CONFIG_API_KEY = 'BRAINY_API_KEY'; +const DEFAULT_LIMIT = 100; + +/** + * Get Brainy URL from script properties + */ +function getBrainyUrl() { + const props = PropertiesService.getScriptProperties(); + const url = props.getProperty(CONFIG_BRAINY_URL); + if (!url) { + throw new Error('Brainy URL not configured. Go to Extensions → Apps Script → Project Settings → Script Properties and add BRAINY_URL'); + } + return url.replace(/\/$/, ''); // Remove trailing slash +} + +/** + * Get API key (optional) + */ +function getApiKey() { + const props = PropertiesService.getScriptProperties(); + return props.getProperty(CONFIG_API_KEY) || ''; +} + +/** + * Make authenticated request to Brainy + */ +function brainyFetch(endpoint, options = {}) { + const baseUrl = getBrainyUrl(); + const apiKey = getApiKey(); + + const url = `${baseUrl}/sheets${endpoint}`; + + const fetchOptions = { + method: options.method || 'GET', + contentType: 'application/json', + muteHttpExceptions: true, + ...options + }; + + if (apiKey) { + fetchOptions.headers = { + ...fetchOptions.headers, + 'Authorization': `Bearer ${apiKey}` + }; + } + + if (options.payload) { + fetchOptions.payload = JSON.stringify(options.payload); + } + + const response = UrlFetchApp.fetch(url, fetchOptions); + const code = response.getResponseCode(); + const text = response.getContentText(); + + if (code >= 400) { + const error = JSON.parse(text); + throw new Error(error.error || `HTTP ${code}`); + } + + return JSON.parse(text); +} + +// ============= Custom Functions ============= + +/** + * Query Brainy entities + * + * @param {string} query Semantic search query or type filter (e.g., "type:person") + * @param {number} limit Maximum results (default: 100) + * @return {Array} Results as rows with headers + * @customfunction + */ +function BRAINY_QUERY(query, limit) { + query = query || ''; + limit = limit || DEFAULT_LIMIT; + + const params = new URLSearchParams(); + params.append('limit', limit); + + if (query) { + // Check if it's a type filter + if (query.toLowerCase().startsWith('type:')) { + params.append('type', query.substring(5).trim()); + } else { + params.append('q', query); + } + } + + const result = brainyFetch(`/query?${params.toString()}`); + + if (!result.rows || result.rows.length === 0) { + return [['No results found']]; + } + + return [result.headers, ...result.rows]; +} + +/** + * Get a single entity by ID + * + * @param {string} id Entity ID + * @return {Array} Entity as rows with headers + * @customfunction + */ +function BRAINY_GET(id) { + if (!id) { + return [['Error: ID required']]; + } + + const result = brainyFetch(`/entity/${id}`); + + if (!result.rows || result.rows.length === 0) { + return [['Entity not found']]; + } + + return [result.headers, ...result.rows]; +} + +/** + * Semantic similarity search + * + * @param {string} text Text to find similar entities to + * @param {number} limit Maximum results (default: 10) + * @return {Array} Similar entities as rows with headers + * @customfunction + */ +function BRAINY_SIMILAR(text, limit) { + if (!text) { + return [['Error: Search text required']]; + } + + limit = limit || 10; + + const params = new URLSearchParams(); + params.append('q', text); + params.append('limit', limit); + + const result = brainyFetch(`/similar?${params.toString()}`); + + if (!result.rows || result.rows.length === 0) { + return [['No similar entities found']]; + } + + return [result.headers, ...result.rows]; +} + +/** + * Get relationships for an entity + * + * @param {string} entityId Entity ID to get relationships for + * @param {string} direction Direction: "from", "to", or "both" (default: "from") + * @param {number} limit Maximum results (default: 100) + * @return {Array} Relationships as rows with headers + * @customfunction + */ +function BRAINY_RELATIONS(entityId, direction, limit) { + if (!entityId) { + return [['Error: Entity ID required']]; + } + + direction = direction || 'from'; + limit = limit || DEFAULT_LIMIT; + + const params = new URLSearchParams(); + params.append('limit', limit); + + if (direction === 'from' || direction === 'both') { + params.append('from', entityId); + } + if (direction === 'to' || direction === 'both') { + params.append('to', entityId); + } + + const result = brainyFetch(`/relations?${params.toString()}`); + + if (!result.rows || result.rows.length === 0) { + return [['No relationships found']]; + } + + return [result.headers, ...result.rows]; +} + +/** + * List available entity types + * + * @return {Array} List of noun types + * @customfunction + */ +function BRAINY_TYPES() { + const result = brainyFetch('/schema'); + return [['Type'], ...result.nounTypes.map(t => [t])]; +} + +/** + * List available relationship types + * + * @return {Array} List of verb types + * @customfunction + */ +function BRAINY_RELATION_TYPES() { + const result = brainyFetch('/schema'); + return [['Type'], ...result.verbTypes.map(t => [t])]; +} + +// ============= Menu & Sidebar ============= + +/** + * Add menu on spreadsheet open + */ +function onOpen() { + const ui = SpreadsheetApp.getUi(); + ui.createMenu('Brainy') + .addItem('Open Sidebar', 'showSidebar') + .addSeparator() + .addItem('Configure Connection', 'showConfig') + .addItem('Test Connection', 'testConnection') + .addToUi(); +} + +/** + * Show the Brainy sidebar + */ +function showSidebar() { + const html = HtmlService.createHtmlOutputFromFile('Sidebar') + .setTitle('Brainy') + .setWidth(300); + SpreadsheetApp.getUi().showSidebar(html); +} + +/** + * Show configuration dialog + */ +function showConfig() { + const ui = SpreadsheetApp.getUi(); + const props = PropertiesService.getScriptProperties(); + const currentUrl = props.getProperty(CONFIG_BRAINY_URL) || ''; + + const response = ui.prompt( + 'Configure Brainy Connection', + `Enter your Brainy server URL (current: ${currentUrl || 'not set'}):`, + ui.ButtonSet.OK_CANCEL + ); + + if (response.getSelectedButton() === ui.Button.OK) { + const url = response.getResponseText().trim(); + if (url) { + props.setProperty(CONFIG_BRAINY_URL, url); + ui.alert('Configuration saved! URL: ' + url); + } + } +} + +/** + * Test the connection + */ +function testConnection() { + const ui = SpreadsheetApp.getUi(); + try { + const result = brainyFetch('/health'); + ui.alert('Connection successful!\n\nStatus: ' + result.status + '\nUptime: ' + Math.round(result.uptime / 1000) + 's'); + } catch (e) { + ui.alert('Connection failed!\n\n' + e.message); + } +} + +// ============= Sidebar Functions ============= + +/** + * Get connection status for sidebar + */ +function getConnectionStatus() { + try { + const result = brainyFetch('/health'); + return { + connected: true, + url: getBrainyUrl(), + status: result.status, + uptime: result.uptime + }; + } catch (e) { + return { + connected: false, + error: e.message + }; + } +} + +/** + * Query entities from sidebar + */ +function sidebarQuery(query, type, limit) { + const params = new URLSearchParams(); + params.append('limit', limit || 50); + + if (query) params.append('q', query); + if (type) params.append('type', type); + + return brainyFetch(`/query?${params.toString()}`); +} + +/** + * Add entity from sidebar + */ +function sidebarAddEntity(type, data, metadata) { + return brainyFetch('/add', { + method: 'POST', + payload: { type, data, metadata } + }); +} + +/** + * Update entity from sidebar + */ +function sidebarUpdateEntity(id, data, metadata) { + return brainyFetch('/update', { + method: 'POST', + payload: { id, data, metadata, merge: true } + }); +} + +/** + * Delete entity from sidebar + */ +function sidebarDeleteEntity(id) { + return brainyFetch('/delete', { + method: 'POST', + payload: { id } + }); +} + +/** + * Get schema for sidebar dropdowns + */ +function sidebarGetSchema() { + return brainyFetch('/schema'); +} + +/** + * Insert query result into sheet at selection + */ +function insertQueryResult(query, type, limit) { + const sheet = SpreadsheetApp.getActiveSheet(); + const selection = sheet.getActiveCell(); + + const result = sidebarQuery(query, type, limit); + + if (!result.rows || result.rows.length === 0) { + selection.setValue('No results'); + return; + } + + const data = [result.headers, ...result.rows]; + const range = sheet.getRange( + selection.getRow(), + selection.getColumn(), + data.length, + data[0].length + ); + + range.setValues(data); + + return { + inserted: result.rows.length, + columns: result.headers.length + }; +} + +/** + * Sync selected range to Brainy + */ +function syncRangeToBrainy(type) { + const sheet = SpreadsheetApp.getActiveSheet(); + const range = sheet.getActiveRange(); + const values = range.getValues(); + + if (values.length < 2) { + throw new Error('Need at least header row and one data row'); + } + + const headers = values[0]; + const operations = []; + + // Find ID column + const idCol = headers.findIndex(h => h.toLowerCase() === 'id'); + + for (let i = 1; i < values.length; i++) { + const row = values[i]; + const metadata = {}; + + for (let j = 0; j < headers.length; j++) { + if (j !== idCol && row[j]) { + metadata[headers[j]] = row[j]; + } + } + + if (idCol >= 0 && row[idCol]) { + // Update existing + operations.push({ + action: 'update', + id: row[idCol], + metadata + }); + } else { + // Add new + operations.push({ + action: 'add', + type: type, + metadata + }); + } + } + + const result = brainyFetch('/batch', { + method: 'POST', + payload: { operations } + }); + + // Update IDs in sheet for new entities + if (idCol >= 0) { + for (let i = 0; i < result.results.length; i++) { + if (result.results[i].action === 'add' && result.results[i].id) { + sheet.getRange(range.getRow() + 1 + i, range.getColumn() + idCol).setValue(result.results[i].id); + } + } + } + + return result; +} diff --git a/integrations/google-sheets/README.md b/integrations/google-sheets/README.md new file mode 100644 index 00000000..b2b0af3a --- /dev/null +++ b/integrations/google-sheets/README.md @@ -0,0 +1,142 @@ +# Brainy Google Sheets Add-on + +Two-way sync between Brainy and Google Sheets. + +## Quick Setup + +1. Open your Google Sheet +2. Go to **Extensions → Apps Script** +3. Delete any existing code +4. Copy and paste `Code.gs` into the editor +5. Create a new HTML file called `Sidebar.html` and paste the content +6. Go to **Project Settings** (gear icon) → **Script properties** +7. Add property: `BRAINY_URL` = your Brainy server URL (e.g., `http://localhost:3000`) +8. Save and refresh your spreadsheet + +## Custom Functions + +Use these directly in cells: + +### `=BRAINY_QUERY(query, limit)` +Query entities using semantic search or type filter. + +``` +=BRAINY_QUERY("machine learning", 10) +=BRAINY_QUERY("type:person", 100) +``` + +### `=BRAINY_GET(id)` +Get a single entity by ID. + +``` +=BRAINY_GET("entity-uuid-here") +``` + +### `=BRAINY_SIMILAR(text, limit)` +Find semantically similar entities. + +``` +=BRAINY_SIMILAR("artificial intelligence research", 5) +``` + +### `=BRAINY_RELATIONS(entityId, direction)` +Get relationships for an entity. + +``` +=BRAINY_RELATIONS("entity-id", "from") +=BRAINY_RELATIONS("entity-id", "to") +=BRAINY_RELATIONS("entity-id", "both") +``` + +### `=BRAINY_TYPES()` +List all available entity types. + +### `=BRAINY_RELATION_TYPES()` +List all available relationship types. + +## Sidebar + +Open via **Brainy → Open Sidebar** menu: + +- **Search**: Query entities and insert results into the sheet +- **Add**: Create new entities with a form +- **Sync**: Sync selected range to Brainy (add or update) + +## Authentication + +If your Brainy server requires authentication: + +1. Go to **Project Settings → Script properties** +2. Add property: `BRAINY_API_KEY` = your API key + +## Real-Time Sync + +For real-time updates, enable SSE in your Brainy server: + +```javascript +brain.augmentations.register(new SSEIntegration()) +``` + +The sidebar will automatically show changes as they happen. + +## Troubleshooting + +### "Brainy URL not configured" +Add the `BRAINY_URL` script property in Apps Script settings. + +### "Connection failed" +- Check that your Brainy server is running +- Ensure the URL is correct (include port if needed) +- For local development, use ngrok or similar to expose localhost + +### Custom functions not appearing +- Refresh the spreadsheet +- Wait a few seconds (Apps Script can be slow to register) +- Check the Apps Script execution log for errors + +## Server Setup + +The simplest way to enable all integrations: + +```javascript +import { Brainy } from '@soulcraft/brainy' + +const brain = new Brainy({ integrations: true }) +await brain.init() + +console.log(brain.hub.endpoints) +// { odata: '/odata', sheets: '/sheets', sse: '/events', webhooks: '/webhooks' } +``` + +With Express: + +```javascript +import express from 'express' +import { Brainy } from '@soulcraft/brainy' + +const app = express() +const brain = new Brainy({ integrations: true }) +await brain.init() + +// Route all integration requests +app.use(express.json()) +app.all(['/odata/*', '/sheets/*', '/events/*'], async (req, res) => { + const response = await brain.hub.handleRequest({ + method: req.method, + path: req.path, + query: req.query, + headers: req.headers, + body: req.body + }) + + if (response.isSSE) { + // Handle SSE stream + res.setHeader('Content-Type', 'text/event-stream') + // ... streaming logic + } else { + res.status(response.status).set(response.headers).json(response.body) + } +}) + +app.listen(3000) +``` diff --git a/integrations/google-sheets/Sidebar.html b/integrations/google-sheets/Sidebar.html new file mode 100644 index 00000000..5e453d7f --- /dev/null +++ b/integrations/google-sheets/Sidebar.html @@ -0,0 +1,421 @@ + + + + + + + +
+

🧠 Brainy

+
+
+ +
+ +
+
Search
+
Add
+
Sync
+
+ + +
+
+ + +
+ + +
+
+
+
Enter a search query
+
+
+ + +
+
+
Entity Type
+ +
+
+
Name
+ +
+
+
Description
+ +
+
+
Additional Fields (JSON)
+ +
+ +
+ + +
+
+

Select a range with entity data and sync it to Brainy.

+

+ First row should be headers. Include an "Id" column for updates. +

+
+
+
Type for New Entities
+ +
+ +
+ + + + diff --git a/integrations/google-sheets/appsscript.json b/integrations/google-sheets/appsscript.json new file mode 100644 index 00000000..dd4e5313 --- /dev/null +++ b/integrations/google-sheets/appsscript.json @@ -0,0 +1,11 @@ +{ + "timeZone": "America/New_York", + "dependencies": {}, + "exceptionLogging": "STACKDRIVER", + "runtimeVersion": "V8", + "oauthScopes": [ + "https://www.googleapis.com/auth/spreadsheets.currentonly", + "https://www.googleapis.com/auth/script.container.ui", + "https://www.googleapis.com/auth/script.external_request" + ] +} diff --git a/src/brainy.ts b/src/brainy.ts index 3c978102..816faf9d 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -84,6 +84,7 @@ import { } from './types/brainy.types.js' import { NounType, VerbType } from './types/graphTypes.js' import { BrainyInterface } from './types/brainyInterface.js' +import type { IntegrationHub } from './integrations/core/IntegrationHub.js' /** * The main Brainy class - Clean, Beautiful, Powerful @@ -129,6 +130,7 @@ export class Brainy implements BrainyInterface { private _versions?: VersioningAPI private _vfs?: VirtualFileSystem private _vfsInitialized = false // v7.3.0: Track VFS init completion separately + private _hub?: IntegrationHub // v7.4.0: Integration Hub for external tools // State private initialized = false @@ -322,6 +324,22 @@ export class Brainy implements BrainyInterface { console.log('✅ Embedding engine ready') } + // v7.4.0: Integration Hub initialization + // Creates the hub when integrations are enabled in config + // Uses dynamic import for tree-shaking when integrations are disabled + if (this.config.integrations) { + const hubConfig = this.config.integrations === true + ? { enable: 'all' as const } + : this.config.integrations + + const { IntegrationHub } = await import('./integrations/core/IntegrationHub.js') + this._hub = await IntegrationHub.create(this, { + basePath: hubConfig.basePath, + enable: hubConfig.enable, + config: hubConfig.config as any // Type flexibility for user config + }) + } + // v7.3.0: Resolve ready Promise - consumers awaiting brain.ready will now proceed if (this._readyResolve) { this._readyResolve() @@ -3852,6 +3870,52 @@ export class Brainy implements BrainyInterface { return this._vfs } + /** + * Integration Hub for external tools (Excel, Power BI, Google Sheets) + * + * Provides HTTP endpoints that external tools can connect to: + * - OData API for Excel Power Query, Power BI, Tableau + * - REST API for Google Sheets custom functions + * - SSE streaming for real-time dashboards + * - Webhooks for push notifications + * + * Only available when `integrations: true` is set in config. + * + * @example Basic usage + * ```typescript + * const brain = new Brainy({ integrations: true }) + * await brain.init() + * + * // Get endpoint URLs + * console.log(brain.hub.endpoints) + * // { odata: '/odata', sheets: '/sheets', sse: '/events', webhooks: '/webhooks' } + * + * // Handle requests (use with Express, Hono, etc.) + * app.all('/odata/*', async (req, res) => { + * const response = await brain.hub.handleRequest({ + * method: req.method, + * path: req.path, + * query: req.query, + * headers: req.headers, + * body: req.body + * }) + * res.status(response.status).set(response.headers).json(response.body) + * }) + * ``` + * + * @since v7.4.0 + * @throws Error if integrations are not enabled in config + */ + get hub(): IntegrationHub { + if (!this._hub) { + throw new Error( + 'Integration Hub not enabled. Set integrations: true in config:\n' + + 'new Brainy({ integrations: true })' + ) + } + return this._hub + } + /** * Data Management API - backup, restore, import, export */ @@ -5692,7 +5756,9 @@ export class Brainy implements BrainyInterface { // HNSW persistence mode (v6.2.8) - undefined = smart default in setupIndex hnswPersistMode: config?.hnswPersistMode ?? undefined as any, // Embedding initialization (v7.1.2) - false = lazy init on first embed() - eagerEmbeddings: config?.eagerEmbeddings ?? false + eagerEmbeddings: config?.eagerEmbeddings ?? false, + // Integration Hub (v7.4.0) - undefined/false = disabled + integrations: config?.integrations ?? undefined as any } } diff --git a/src/index.ts b/src/index.ts index a8d0e9e8..5fd6e269 100644 --- a/src/index.ts +++ b/src/index.ts @@ -542,3 +542,86 @@ export type { MCPServiceOptions, MCPTool } + +// ============= Integration Hub (v7.4.0) ============= +// Connect Brainy to Excel, Power BI, Google Sheets, and more +// Enable with: new Brainy({ integrations: true }) + +// Hub class (used internally by brain.hub, also available for advanced use) +export { + IntegrationHub, + createIntegrationHub +} from './integrations/index.js' + +export type { + IntegrationHubConfig, + IntegrationRequest, + IntegrationResponse +} from './integrations/index.js' + +// Re-export IntegrationsConfig from types (for TypeScript users) +export type { IntegrationsConfig } from './types/brainy.types.js' + +// Core infrastructure +export { + EventBus, + TabularExporter, + IntegrationBase, + IntegrationLoader, + createIntegrationLoader, + detectEnvironment, + INTEGRATION_CATALOG +} from './integrations/index.js' + +// Integration types +export type { + BrainyEvent, + EventFilter, + EventHandler, + EventSubscription, + TabularRow, + RelationTabularRow, + TabularExporterConfig, + IntegrationConfig, + IntegrationHealthStatus, + HTTPIntegration, + StreamingIntegration, + IntegrationType, + RuntimeEnvironment, + IntegrationInfo, + IntegrationLoaderConfig, + ODataQueryOptions, + WebhookRegistration, + WebhookDeliveryResult +} from './integrations/index.js' + +// Concrete integrations +export { + GoogleSheetsIntegration, + ODataIntegration, + SSEIntegration, + WebhookIntegration +} from './integrations/index.js' + +export type { + GoogleSheetsConfig, + ODataConfig, + SSEConfig, + WebhookConfig +} from './integrations/index.js' + +// OData utilities (advanced) +export { + parseODataQuery, + parseFilter, + parseOrderBy, + parseSelect, + odataToFindParams, + applyFilter, + applySelect, + applyOrderBy, + applyPagination, + generateEdmx, + generateMetadataJson, + generateServiceDocument +} from './integrations/index.js' diff --git a/src/integrations/core/EventBus.ts b/src/integrations/core/EventBus.ts new file mode 100644 index 00000000..9c9f9acd --- /dev/null +++ b/src/integrations/core/EventBus.ts @@ -0,0 +1,313 @@ +/** + * Integration Hub - Event Bus + * + * Central event emitter for real-time change propagation. + * Enables integrations to react to Brainy data changes. + */ + +import { + BrainyEvent, + EventFilter, + EventHandler, + EventSubscription +} from './types.js' +import { Entity, Relation } from '../../types/brainy.types.js' +import { NounType, VerbType } from '../../types/graphTypes.js' + +/** + * Central event bus for real-time Brainy events + * + * Features: + * - Pub/sub pattern for event distribution + * - Filtering by entity type, operation, noun/verb types + * - Sequence IDs for ordering and resumption + * - Optional event buffering for batch processing + * - Memory-efficient circular buffer for replay + * + * @example + * ```typescript + * const eventBus = new EventBus() + * + * // Subscribe to all noun creates + * eventBus.subscribe( + * { entityTypes: ['noun'], operations: ['create'] }, + * (event) => console.log('New entity:', event.entityId) + * ) + * + * // Emit event + * eventBus.emit({ + * entityType: 'noun', + * operation: 'create', + * entityId: 'entity-123', + * nounType: NounType.Person + * }) + * ``` + */ +export class EventBus { + private subscriptions: Map< + string, + { filter: EventFilter; handler: EventHandler } + > = new Map() + private sequenceCounter: bigint = 0n + private eventBuffer: BrainyEvent[] = [] + private bufferSize: number + private subscriptionIdCounter = 0 + + /** + * Create a new EventBus + * + * @param options Configuration options + * @param options.bufferSize Size of replay buffer (default: 1000) + */ + constructor(options: { bufferSize?: number } = {}) { + this.bufferSize = options.bufferSize ?? 1000 + } + + /** + * Subscribe to events matching a filter + * + * @param filter Event filter criteria + * @param handler Function to call when matching events occur + * @returns Subscription that can be used to unsubscribe + */ + subscribe(filter: EventFilter, handler: EventHandler): EventSubscription { + const id = `sub-${++this.subscriptionIdCounter}` + + this.subscriptions.set(id, { filter, handler }) + + // If filter has 'since', replay buffered events + if (filter.since !== undefined) { + this.replayEvents(filter, handler) + } + + return { + id, + unsubscribe: () => { + this.subscriptions.delete(id) + } + } + } + + /** + * Emit an event to all matching subscribers + * + * @param partialEvent Event data (id, timestamp, sequenceId auto-generated) + */ + emit( + partialEvent: Omit + ): BrainyEvent { + const event: BrainyEvent = { + ...partialEvent, + id: this.generateEventId(), + timestamp: Date.now(), + sequenceId: ++this.sequenceCounter + } + + // Add to buffer + this.addToBuffer(event) + + // Dispatch to matching subscribers + this.dispatch(event) + + return event + } + + /** + * Emit a noun event + */ + emitNoun( + operation: 'create' | 'update' | 'delete', + entityId: string, + nounType: NounType, + options?: { service?: string; data?: Entity } + ): BrainyEvent { + return this.emit({ + entityType: 'noun', + operation, + entityId, + nounType, + service: options?.service, + data: options?.data + }) + } + + /** + * Emit a verb/relation event + */ + emitVerb( + operation: 'create' | 'update' | 'delete', + entityId: string, + verbType: VerbType, + options?: { service?: string; data?: Relation } + ): BrainyEvent { + return this.emit({ + entityType: 'verb', + operation, + entityId, + verbType, + service: options?.service, + data: options?.data + }) + } + + /** + * Emit a VFS event + */ + emitVFS( + operation: 'create' | 'update' | 'delete', + entityId: string, + options?: { service?: string; data?: Entity } + ): BrainyEvent { + return this.emit({ + entityType: 'vfs', + operation, + entityId, + service: options?.service, + data: options?.data + }) + } + + /** + * Get current sequence ID for resumption + */ + getCurrentSequenceId(): bigint { + return this.sequenceCounter + } + + /** + * Get events since a sequence ID (from buffer) + */ + getEventsSince(sequenceId: bigint): BrainyEvent[] { + return this.eventBuffer.filter((event) => event.sequenceId > sequenceId) + } + + /** + * Get subscription count + */ + getSubscriptionCount(): number { + return this.subscriptions.size + } + + /** + * Clear all subscriptions + */ + clear(): void { + this.subscriptions.clear() + this.eventBuffer = [] + this.sequenceCounter = 0n + } + + /** + * Check if an event matches a filter + */ + private matchesFilter(event: BrainyEvent, filter: EventFilter): boolean { + // Check entity types + if ( + filter.entityTypes && + filter.entityTypes.length > 0 && + !filter.entityTypes.includes(event.entityType) + ) { + return false + } + + // Check operations + if ( + filter.operations && + filter.operations.length > 0 && + !filter.operations.includes(event.operation) + ) { + return false + } + + // Check noun types + if ( + filter.nounTypes && + filter.nounTypes.length > 0 && + event.nounType && + !filter.nounTypes.includes(event.nounType) + ) { + return false + } + + // Check verb types + if ( + filter.verbTypes && + filter.verbTypes.length > 0 && + event.verbType && + !filter.verbTypes.includes(event.verbType) + ) { + return false + } + + // Check service + if (filter.service && event.service !== filter.service) { + return false + } + + // Check sequence ID + if (filter.since !== undefined && event.sequenceId <= filter.since) { + return false + } + + return true + } + + /** + * Dispatch event to matching subscribers + */ + private async dispatch(event: BrainyEvent): Promise { + const promises: Promise[] = [] + + for (const [_, subscription] of this.subscriptions) { + if (this.matchesFilter(event, subscription.filter)) { + const result = subscription.handler(event) + if (result instanceof Promise) { + promises.push(result) + } + } + } + + // Wait for all async handlers (fire and forget for sync handlers) + if (promises.length > 0) { + await Promise.allSettled(promises) + } + } + + /** + * Replay buffered events to a new subscriber + */ + private async replayEvents( + filter: EventFilter, + handler: EventHandler + ): Promise { + const eventsToReplay = this.eventBuffer.filter((event) => + this.matchesFilter(event, filter) + ) + + for (const event of eventsToReplay) { + const result = handler(event) + if (result instanceof Promise) { + await result + } + } + } + + /** + * Add event to circular buffer + */ + private addToBuffer(event: BrainyEvent): void { + this.eventBuffer.push(event) + + // Maintain buffer size + while (this.eventBuffer.length > this.bufferSize) { + this.eventBuffer.shift() + } + } + + /** + * Generate unique event ID + */ + private generateEventId(): string { + return `evt-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 9)}` + } +} diff --git a/src/integrations/core/IntegrationBase.ts b/src/integrations/core/IntegrationBase.ts new file mode 100644 index 00000000..9617892f --- /dev/null +++ b/src/integrations/core/IntegrationBase.ts @@ -0,0 +1,425 @@ +/** + * Integration Hub - Integration Base Class + * + * Base class for all integration augmentations. Provides common functionality + * for event subscriptions, tabular export, and lifecycle management. + */ + +import { + BaseAugmentation, + AugmentationContext +} from '../../augmentations/brainyAugmentation.js' +import { AugmentationManifest } from '../../augmentations/manifest.js' +import { EventBus } from './EventBus.js' +import { TabularExporter } from './TabularExporter.js' +import { + EventFilter, + EventHandler, + EventSubscription, + IntegrationConfig, + IntegrationHealthStatus, + TabularExporterConfig +} from './types.js' +import { Entity, Relation, FindParams } from '../../types/brainy.types.js' + +/** + * Base class for all integration augmentations + * + * Provides: + * - Shared EventBus for real-time updates + * - TabularExporter for entity-to-row conversion + * - Common lifecycle methods (start/stop) + * - Health monitoring + * - Standard configuration patterns + * + * @example + * ```typescript + * class MySQLIntegration extends IntegrationBase { + * readonly name = 'sql' + * readonly category = 'integration' + * + * protected async onStart(): Promise { + * // Start SQL server + * } + * + * protected async onStop(): Promise { + * // Stop SQL server + * } + * } + * ``` + */ +export abstract class IntegrationBase extends BaseAugmentation { + // Augmentation interface implementation + readonly timing = 'after' as const + readonly metadata = 'readonly' as const + readonly operations: ('all')[] = ['all'] + readonly priority = 5 // Low priority - runs after main operations + category: 'internal' | 'core' | 'premium' | 'community' | 'external' = 'core' + + // Shared infrastructure + protected eventBus: EventBus + protected exporter: TabularExporter + + // Integration state + protected isRunning = false + protected startedAt?: number + protected requestCount = 0 + protected errorCount = 0 + protected lastError?: string + + // Event subscriptions managed by this integration + private managedSubscriptions: EventSubscription[] = [] + + /** + * Create a new integration + * + * @param config Integration configuration + * @param exporterConfig Optional TabularExporter configuration + */ + constructor( + config?: IntegrationConfig, + exporterConfig?: TabularExporterConfig + ) { + super(config) + this.eventBus = new EventBus() + this.exporter = new TabularExporter(exporterConfig) + } + + /** + * Integration name (must be unique) + */ + abstract readonly name: string + + /** + * Start the integration (implement in subclass) + */ + protected abstract onStart(): Promise + + /** + * Stop the integration (implement in subclass) + */ + protected abstract onStop(): Promise + + // BaseAugmentation lifecycle integration + + protected async onInitialize(): Promise { + // Auto-start if enabled + if (this.config.enabled !== false) { + await this.start() + } + } + + protected async onShutdown(): Promise { + await this.stop() + } + + /** + * Execute augmentation (intercept operations to emit events) + */ + async execute( + operation: string, + params: any, + next: () => Promise + ): Promise { + // Execute main operation first + const result = await next() + + // Emit events for data-changing operations + if (this.isRunning) { + this.emitOperationEvent(operation, params, result) + } + + return result + } + + // Public API + + /** + * Start the integration + */ + async start(): Promise { + if (this.isRunning) { + return + } + + this.log(`Starting ${this.name} integration...`) + + try { + await this.onStart() + this.isRunning = true + this.startedAt = Date.now() + this.log(`${this.name} integration started`) + } catch (error: any) { + this.lastError = error.message + this.log(`Failed to start ${this.name}: ${error.message}`, 'error') + throw error + } + } + + /** + * Stop the integration + */ + async stop(): Promise { + if (!this.isRunning) { + return + } + + this.log(`Stopping ${this.name} integration...`) + + try { + // Unsubscribe all managed subscriptions + for (const sub of this.managedSubscriptions) { + sub.unsubscribe() + } + this.managedSubscriptions = [] + + await this.onStop() + this.isRunning = false + this.log(`${this.name} integration stopped`) + } catch (error: any) { + this.lastError = error.message + this.log(`Error stopping ${this.name}: ${error.message}`, 'error') + throw error + } + } + + /** + * Check if integration is running + */ + running(): boolean { + return this.isRunning + } + + /** + * Get health status + */ + health(): IntegrationHealthStatus { + return { + name: this.name, + status: this.isRunning + ? this.errorCount > 10 + ? 'degraded' + : 'healthy' + : 'stopped', + message: this.isRunning ? 'Running' : 'Stopped', + uptimeMs: this.startedAt ? Date.now() - this.startedAt : undefined, + requestCount: this.requestCount, + errorCount: this.errorCount, + lastError: this.lastError, + checkedAt: Date.now() + } + } + + /** + * Get the shared EventBus + */ + getEventBus(): EventBus { + return this.eventBus + } + + /** + * Get the TabularExporter + */ + getExporter(): TabularExporter { + return this.exporter + } + + // Protected helpers for subclasses + + /** + * Subscribe to Brainy events (auto-unsubscribed on stop) + */ + protected subscribeToChanges( + filter: EventFilter, + handler: EventHandler + ): EventSubscription { + const subscription = this.eventBus.subscribe(filter, handler) + this.managedSubscriptions.push(subscription) + return subscription + } + + /** + * Query entities using Brainy find() + */ + protected async queryEntities(params: FindParams): Promise { + if (!this.context) { + throw new Error('Integration not initialized') + } + + const results = await this.context.brain.find(params) + return results.map((r: any) => r.entity) + } + + /** + * Query relations using Brainy getRelations() + */ + protected async queryRelations(params?: { + from?: string + to?: string + type?: any + limit?: number + offset?: number + }): Promise { + if (!this.context) { + throw new Error('Integration not initialized') + } + + return this.context.brain.getRelations(params) + } + + /** + * Get a single entity by ID + */ + protected async getEntity(id: string): Promise { + if (!this.context) { + throw new Error('Integration not initialized') + } + + return this.context.brain.get(id) + } + + /** + * Export entities to tabular format + */ + protected exportEntities(entities: Entity[]): any[] { + return this.exporter.entitiesToRows(entities) + } + + /** + * Export relations to tabular format + */ + protected exportRelations(relations: Relation[]): any[] { + return this.exporter.relationsToRows(relations) + } + + /** + * Record a successful request + */ + protected recordRequest(): void { + this.requestCount++ + } + + /** + * Record an error + */ + protected recordError(error: Error | string): void { + this.errorCount++ + this.lastError = typeof error === 'string' ? error : error.message + } + + /** + * Get manifest for this integration + * Subclasses should override this + */ + getManifest(): AugmentationManifest { + return { + id: this.name, + name: this.name, + version: '1.0.0', + description: `${this.name} integration`, + category: 'integration', + status: 'stable', + configSchema: { + type: 'object', + properties: { + enabled: { + type: 'boolean', + default: true, + description: 'Whether the integration is enabled' + } + } + }, + configDefaults: { + enabled: true + } + } + } + + // Private helpers + + /** + * Emit events based on operation type + */ + private emitOperationEvent(operation: string, params: any, result: any): void { + // Map operations to event types + const opMap: Record< + string, + { entityType: 'noun' | 'verb' | 'vfs'; op: 'create' | 'update' | 'delete' } + > = { + add: { entityType: 'noun', op: 'create' }, + addNoun: { entityType: 'noun', op: 'create' }, + update: { entityType: 'noun', op: 'update' }, + delete: { entityType: 'noun', op: 'delete' }, + relate: { entityType: 'verb', op: 'create' }, + addVerb: { entityType: 'verb', op: 'create' }, + unrelate: { entityType: 'verb', op: 'delete' }, + deleteVerb: { entityType: 'verb', op: 'delete' } + } + + const mapping = opMap[operation] + if (!mapping) { + return + } + + // Extract entity ID from result or params + let entityId = result?.id || params?.id + if (!entityId && Array.isArray(result)) { + // Batch operation - emit for each + for (const item of result) { + if (item?.id) { + this.eventBus.emit({ + entityType: mapping.entityType, + operation: mapping.op, + entityId: item.id, + nounType: params?.type || item?.type, + verbType: params?.type || item?.type, + service: params?.service || item?.service, + data: item + }) + } + } + return + } + + if (entityId) { + this.eventBus.emit({ + entityType: mapping.entityType, + operation: mapping.op, + entityId, + nounType: params?.type, + verbType: params?.type, + service: params?.service, + data: result + }) + } + } +} + +/** + * Interface for integrations that expose HTTP endpoints + */ +export interface HTTPIntegration { + /** Port the server is listening on */ + port: number + + /** Base path for routes */ + basePath: string + + /** Get registered routes */ + getRoutes(): Array<{ + method: string + path: string + description: string + }> +} + +/** + * Interface for integrations that support streaming + */ +export interface StreamingIntegration { + /** Subscribe to real-time events via callback */ + stream( + filter: EventFilter, + callback: (event: any) => void + ): { close: () => void } +} diff --git a/src/integrations/core/IntegrationHub.ts b/src/integrations/core/IntegrationHub.ts new file mode 100644 index 00000000..3fee573f --- /dev/null +++ b/src/integrations/core/IntegrationHub.ts @@ -0,0 +1,368 @@ +/** + * Integration Hub - Zero-Config Integration Manager + * + * The simplest way to enable external tool integrations. + * One line of code, all integrations ready. + * + * @example + * ```typescript + * // Zero-config: Enable all integrations + * const hub = await IntegrationHub.create(brain) + * + * // Get your endpoints + * console.log(hub.endpoints) + * // { + * // odata: '/odata', → Excel, Power BI, Tableau + * // sheets: '/sheets', → Google Sheets + * // sse: '/events', → Real-time streaming + * // webhooks: '/webhooks' → Push notifications + * // } + * ``` + */ + +import { IntegrationBase } from './IntegrationBase.js' +import { IntegrationLoader, IntegrationType, INTEGRATION_CATALOG } from './IntegrationLoader.js' +import { IntegrationConfig, IntegrationHealthStatus } from './types.js' +import { ODataIntegration } from '../odata/ODataIntegration.js' +import { GoogleSheetsIntegration } from '../sheets/GoogleSheetsIntegration.js' +import { SSEIntegration } from '../sse/SSEIntegration.js' +import { WebhookIntegration } from '../webhooks/WebhookIntegration.js' + +/** + * Integration Hub configuration + */ +export interface IntegrationHubConfig { + /** Base path for all endpoints (default: '') */ + basePath?: string + + /** Which integrations to enable (default: all) */ + enable?: IntegrationType[] | 'all' + + /** Per-integration config overrides */ + config?: { + odata?: IntegrationConfig & { basePath?: string } + sheets?: IntegrationConfig & { basePath?: string } + sse?: IntegrationConfig & { basePath?: string } + webhooks?: IntegrationConfig & { basePath?: string } + } +} + +/** + * HTTP request for integration routing + */ +export interface IntegrationRequest { + method: string + path: string + query?: Record + headers?: Record + body?: any +} + +/** + * HTTP response from integration + */ +export interface IntegrationResponse { + status: number + headers: Record + body: any + isSSE?: boolean +} + +/** + * Integration Hub - Zero-Configuration Integration Manager + * + * Provides instant access to: + * - OData API (Excel Power Query, Power BI, Tableau) + * - Google Sheets API (two-way sync) + * - SSE streaming (real-time dashboards) + * - Webhooks (push notifications) + * + * All integrations work in any environment with zero external dependencies. + */ +export class IntegrationHub { + private integrations: Map = new Map() + private config: Required + private _endpoints: Record = {} as any + + /** + * Create and initialize the Integration Hub + * + * @example + * ```typescript + * // All integrations, default paths + * const hub = await IntegrationHub.create(brain) + * + * // Custom base path + * const hub = await IntegrationHub.create(brain, { basePath: '/api/v1' }) + * + * // Only specific integrations + * const hub = await IntegrationHub.create(brain, { enable: ['odata', 'sheets'] }) + * ``` + */ + static async create(brain: any, config?: IntegrationHubConfig): Promise { + const hub = new IntegrationHub(config) + await hub.initialize(brain) + return hub + } + + private constructor(config?: IntegrationHubConfig) { + this.config = { + basePath: config?.basePath ?? '', + enable: config?.enable ?? 'all', + config: config?.config ?? {} + } + } + + private async initialize(brain: any): Promise { + const toEnable = this.config.enable === 'all' + ? (['odata', 'sheets', 'sse', 'webhooks'] as IntegrationType[]) + : this.config.enable + + // Create context for integration initialization + const context = { + brain, + storage: brain.getStorage?.() || null, + config: {}, + log: (message: string, level?: string) => { + // Silent logging - integrations handle their own logging + } + } + + for (const type of toEnable) { + const integration = await this.createIntegration(type) + if (integration) { + // Initialize the integration with context (BaseAugmentation pattern) + await integration.initialize(context) + this.integrations.set(type, integration) + this._endpoints[type] = this.getBasePath(type) + } + } + } + + private async createIntegration(type: IntegrationType): Promise { + const basePath = this.config.basePath + const cfg = this.config.config?.[type] + + switch (type) { + case 'odata': + return new ODataIntegration({ + ...cfg, + basePath: cfg?.basePath ?? `${basePath}/odata` + }) + case 'sheets': + return new GoogleSheetsIntegration({ + ...cfg, + basePath: cfg?.basePath ?? `${basePath}/sheets` + }) + case 'sse': + return new SSEIntegration({ + ...cfg, + basePath: cfg?.basePath ?? `${basePath}/events` + }) + case 'webhooks': + return new WebhookIntegration(cfg) + default: + return null + } + } + + private getBasePath(type: IntegrationType): string { + const basePath = this.config.basePath + const cfg = this.config.config?.[type] as any + + switch (type) { + case 'odata': + return cfg?.basePath ?? `${basePath}/odata` + case 'sheets': + return cfg?.basePath ?? `${basePath}/sheets` + case 'sse': + return cfg?.basePath ?? `${basePath}/events` + case 'webhooks': + return `${basePath}/webhooks` + default: + return '' + } + } + + /** + * Get all endpoint paths + */ + get endpoints(): Record { + return { ...this._endpoints } + } + + /** + * Handle an HTTP request and route to the appropriate integration + * + * @example + * ```typescript + * // Express middleware + * app.use('/api', async (req, res) => { + * const response = await hub.handleRequest({ + * method: req.method, + * path: req.path, + * query: req.query, + * headers: req.headers, + * body: req.body + * }) + * + * if (response.isSSE) { + * // Handle SSE stream + * } else { + * res.status(response.status).set(response.headers).json(response.body) + * } + * }) + * ``` + */ + async handleRequest(request: IntegrationRequest): Promise { + const { path } = request + + // Route to appropriate integration based on path + for (const [type, basePath] of Object.entries(this._endpoints)) { + if (path.startsWith(basePath) || path === basePath) { + const integration = this.integrations.get(type as IntegrationType) + if (integration && 'handleRequest' in integration) { + const handler = integration as any + const relativePath = path.slice(basePath.length) || '/' + + return handler.handleRequest({ + ...request, + path: relativePath + }) + } + } + } + + return { + status: 404, + headers: { 'Content-Type': 'application/json' }, + body: { error: 'Not found', path } + } + } + + /** + * Get a specific integration + */ + get(type: IntegrationType): T | undefined { + return this.integrations.get(type) as T | undefined + } + + /** + * Get the OData integration + */ + get odata(): ODataIntegration | undefined { + return this.integrations.get('odata') as ODataIntegration + } + + /** + * Get the Google Sheets integration + */ + get sheets(): GoogleSheetsIntegration | undefined { + return this.integrations.get('sheets') as GoogleSheetsIntegration + } + + /** + * Get the SSE integration + */ + get sse(): SSEIntegration | undefined { + return this.integrations.get('sse') as SSEIntegration + } + + /** + * Get the Webhook integration + */ + get webhooks(): WebhookIntegration | undefined { + return this.integrations.get('webhooks') as WebhookIntegration + } + + /** + * Check if an integration is enabled + */ + has(type: IntegrationType): boolean { + return this.integrations.has(type) + } + + /** + * Get health status of all integrations + */ + health(): Record { + const result: Record = {} + + for (const [type, integration] of this.integrations) { + result[type] = integration.health() + } + + return result as Record + } + + /** + * Stop all integrations + */ + async stop(): Promise { + for (const integration of this.integrations.values()) { + await integration.stop() + } + } + + /** + * Get connection instructions for each tool + */ + getInstructions(): Record { + const base = this.config.basePath || 'http://localhost:3000' + + return { + excel: ` +Excel Power Query: +1. Data → Get Data → From Other Sources → From OData Feed +2. Enter URL: ${base}/odata +3. Click OK, then Load + `.trim(), + + powerbi: ` +Power BI: +1. Get Data → OData Feed +2. Enter URL: ${base}/odata +3. Click OK, then Load + `.trim(), + + googleSheets: ` +Google Sheets: +1. Install the Brainy add-on from Google Workspace Marketplace +2. Open sidebar: Extensions → Brainy → Open +3. Connect to: ${base}/sheets +4. Use custom functions: =BRAINY_QUERY("type:Person", 100) + `.trim(), + + realtime: ` +Real-time Streaming (SSE): +const source = new EventSource('${base}/events') +source.onmessage = (event) => console.log(JSON.parse(event.data)) + `.trim(), + + webhooks: ` +Webhooks: +POST ${base}/webhooks/register +{ + "url": "https://your-server.com/webhook", + "events": { "entityTypes": ["noun"], "operations": ["create", "update"] } +} + `.trim() + } + } +} + +/** + * Create an Integration Hub with zero configuration + * + * @example + * ```typescript + * const hub = await createIntegrationHub(brain) + * console.log(hub.endpoints) + * ``` + */ +export async function createIntegrationHub( + brain: any, + config?: IntegrationHubConfig +): Promise { + return IntegrationHub.create(brain, config) +} diff --git a/src/integrations/core/IntegrationLoader.ts b/src/integrations/core/IntegrationLoader.ts new file mode 100644 index 00000000..86a44df0 --- /dev/null +++ b/src/integrations/core/IntegrationLoader.ts @@ -0,0 +1,279 @@ +/** + * Integration Hub - Integration Loader + * + * Lazy-loads integrations with environment detection. + * Zero external dependencies - works everywhere. + */ + +import { IntegrationBase } from './IntegrationBase.js' +import { IntegrationConfig } from './types.js' + +/** + * Supported integration types + */ +export type IntegrationType = + | 'odata' // Excel Power Query, Power BI, Tableau + | 'sheets' // Google Sheets two-way sync + | 'sse' // Server-Sent Events streaming + | 'webhooks' // Push notifications to external URLs + +/** + * Runtime environment + */ +export type RuntimeEnvironment = + | 'node' + | 'browser' + | 'deno' + | 'cloudflare' + | 'bun' + +/** + * Integration metadata + */ +export interface IntegrationInfo { + id: IntegrationType + name: string + description: string + environments: RuntimeEnvironment[] + tools: string[] // What tools this works with +} + +/** + * Integration catalog - all built-in, zero dependencies + */ +export const INTEGRATION_CATALOG: Record = { + odata: { + id: 'odata', + name: 'OData 4.0 API', + description: 'REST API for spreadsheets and BI tools', + environments: ['node', 'browser', 'deno', 'cloudflare', 'bun'], + tools: ['Excel Power Query', 'Power BI', 'Tableau', 'Qlik', 'SAP'] + }, + sheets: { + id: 'sheets', + name: 'Google Sheets', + description: 'Two-way sync with Google Sheets', + environments: ['node', 'browser', 'deno', 'cloudflare', 'bun'], + tools: ['Google Sheets', 'Apps Script'] + }, + sse: { + id: 'sse', + name: 'Real-time Streaming', + description: 'Server-Sent Events for live updates', + environments: ['node', 'browser', 'deno', 'cloudflare', 'bun'], + tools: ['Dashboards', 'Live UIs', 'Monitoring'] + }, + webhooks: { + id: 'webhooks', + name: 'Webhooks', + description: 'Push events to external URLs', + environments: ['node', 'browser', 'deno', 'cloudflare', 'bun'], + tools: ['Zapier', 'IFTTT', 'n8n', 'Custom APIs'] + } +} + +/** + * Detect current runtime environment + */ +export function detectEnvironment(): RuntimeEnvironment { + // Deno + if (typeof (globalThis as any).Deno !== 'undefined') { + return 'deno' + } + + // Bun + if (typeof (globalThis as any).Bun !== 'undefined') { + return 'bun' + } + + // Cloudflare Workers + if ( + typeof (globalThis as any).caches !== 'undefined' && + typeof (globalThis as any).HTMLRewriter !== 'undefined' + ) { + return 'cloudflare' + } + + // Node.js + if ( + typeof process !== 'undefined' && + process.versions && + process.versions.node + ) { + return 'node' + } + + // Browser + if (typeof window !== 'undefined') { + return 'browser' + } + + return 'node' +} + +/** + * Integration loader configuration + */ +export interface IntegrationLoaderConfig { + /** Which integrations to load: array of types, 'all', or 'none' */ + integrations?: IntegrationType[] | 'all' | 'none' + + /** Override configs per integration */ + config?: Partial> + + /** Custom integrations to add */ + custom?: IntegrationBase[] +} + +/** + * Lazy-loading integration manager + * + * @example + * ```typescript + * // Load all integrations (recommended) + * const loader = new IntegrationLoader({ integrations: 'all' }) + * const integrations = await loader.load() + * + * // Load specific integrations + * const loader = new IntegrationLoader({ + * integrations: ['odata', 'sheets'] + * }) + * ``` + */ +export class IntegrationLoader { + private config: IntegrationLoaderConfig + private environment: RuntimeEnvironment + private loaded: Map = new Map() + + constructor(config: IntegrationLoaderConfig = {}) { + this.config = { + integrations: config.integrations ?? 'none', + config: config.config ?? {}, + custom: config.custom ?? [] + } + this.environment = detectEnvironment() + } + + /** + * Get current runtime environment + */ + getEnvironment(): RuntimeEnvironment { + return this.environment + } + + /** + * Get all available integrations + */ + getAvailable(): IntegrationInfo[] { + return Object.values(INTEGRATION_CATALOG).filter((info) => + info.environments.includes(this.environment) + ) + } + + /** + * Check if an integration is available + */ + isAvailable(type: IntegrationType): boolean { + const info = INTEGRATION_CATALOG[type] + return info?.environments.includes(this.environment) ?? false + } + + /** + * Load and instantiate integrations + */ + async load(): Promise { + const toLoad = this.resolveIntegrations() + const results: IntegrationBase[] = [] + + // Load integrations in parallel for speed + const loadPromises = toLoad.map(async (type) => { + try { + const integration = await this.loadOne(type) + if (integration) { + this.loaded.set(type, integration) + return integration + } + } catch (error: any) { + console.warn(`[Brainy] Failed to load ${type}: ${error.message}`) + } + return null + }) + + const loadedIntegrations = await Promise.all(loadPromises) + results.push(...loadedIntegrations.filter((i): i is IntegrationBase => i !== null)) + + // Add custom integrations + for (const custom of this.config.custom ?? []) { + results.push(custom) + } + + return results + } + + /** + * Get a loaded integration by type + */ + get(type: IntegrationType): T | undefined { + return this.loaded.get(type) as T | undefined + } + + /** + * Check if an integration is loaded + */ + has(type: IntegrationType): boolean { + return this.loaded.has(type) + } + + /** + * Get all loaded integrations + */ + all(): IntegrationBase[] { + return Array.from(this.loaded.values()) + } + + private resolveIntegrations(): IntegrationType[] { + const { integrations } = this.config + + if (integrations === 'none') { + return [] + } + + if (integrations === 'all') { + return this.getAvailable().map((info) => info.id) + } + + return (integrations || []).filter((type) => this.isAvailable(type)) + } + + private async loadOne(type: IntegrationType): Promise { + const cfg = this.config.config?.[type] + + switch (type) { + case 'odata': { + const { ODataIntegration } = await import('../odata/ODataIntegration.js') + return new ODataIntegration(cfg) + } + case 'sheets': { + const { GoogleSheetsIntegration } = await import('../sheets/GoogleSheetsIntegration.js') + return new GoogleSheetsIntegration(cfg) + } + case 'sse': { + const { SSEIntegration } = await import('../sse/SSEIntegration.js') + return new SSEIntegration(cfg) + } + case 'webhooks': { + const { WebhookIntegration } = await import('../webhooks/WebhookIntegration.js') + return new WebhookIntegration(cfg) + } + default: + return null + } + } +} + +/** + * Create an integration loader + */ +export function createIntegrationLoader(config?: IntegrationLoaderConfig): IntegrationLoader { + return new IntegrationLoader(config) +} diff --git a/src/integrations/core/TabularExporter.ts b/src/integrations/core/TabularExporter.ts new file mode 100644 index 00000000..e7e48625 --- /dev/null +++ b/src/integrations/core/TabularExporter.ts @@ -0,0 +1,574 @@ +/** + * Integration Hub - Tabular Exporter + * + * Converts Brainy entities to tabular formats (rows/columns) for use in + * spreadsheets, SQL databases, and BI tools. + */ + +import { Entity, Relation } from '../../types/brainy.types.js' +import { + TabularRow, + RelationTabularRow, + TabularExporterConfig +} from './types.js' + +/** + * Converts Brainy entities to tabular formats + * + * Used by SQL, OData, Google Sheets, and CSV integrations to maintain + * consistent entity-to-row mapping across all external tools. + * + * @example + * ```typescript + * const exporter = new TabularExporter({ + * flattenMetadata: true, + * includeVectors: false + * }) + * + * const rows = exporter.entitiesToRows(entities) + * const csv = exporter.toCSV(entities) + * const odata = exporter.toOData(entities) + * ``` + */ +export class TabularExporter { + private config: Required + + constructor(config: TabularExporterConfig = {}) { + this.config = { + flattenMetadata: config.flattenMetadata ?? true, + metadataPrefix: config.metadataPrefix ?? 'Metadata_', + includeVectors: config.includeVectors ?? false, + dateFormat: config.dateFormat ?? 'ISO8601', + jsonStringify: config.jsonStringify ?? ['data'], + maxFlattenDepth: config.maxFlattenDepth ?? 1, // Flatten one level, stringify deeper + excludeColumns: config.excludeColumns ?? [] + } + } + + /** + * Convert entities to tabular rows + */ + entitiesToRows(entities: Entity[]): TabularRow[] { + return entities.map((entity) => this.entityToRow(entity)) + } + + /** + * Convert a single entity to a tabular row + */ + entityToRow(entity: Entity): TabularRow { + const row: TabularRow = { + Id: entity.id, + Type: entity.type, + CreatedAt: this.formatDate(entity.createdAt), + UpdatedAt: entity.updatedAt + ? this.formatDate(entity.updatedAt) + : this.formatDate(entity.createdAt), + Confidence: entity.confidence ?? null, + Weight: entity.weight ?? null, + Service: entity.service ?? null, + Data: this.config.jsonStringify.includes('data') + ? JSON.stringify(entity.data ?? null) + : entity.data ?? null + } + + // Include vector if configured + if (this.config.includeVectors && entity.vector) { + row.Vector = JSON.stringify(Array.from(entity.vector)) + } + + // Flatten metadata + if (this.config.flattenMetadata && entity.metadata) { + const flatMetadata = this.flattenObject( + entity.metadata, + this.config.metadataPrefix, + this.config.maxFlattenDepth + ) + Object.assign(row, flatMetadata) + } else if (entity.metadata) { + row.Metadata = JSON.stringify(entity.metadata) + } + + // Remove excluded columns + for (const col of this.config.excludeColumns) { + delete row[col] + } + + return row + } + + /** + * Convert relations to tabular rows + */ + relationsToRows(relations: Relation[]): RelationTabularRow[] { + return relations.map((rel) => this.relationToRow(rel)) + } + + /** + * Convert a single relation to a tabular row + */ + relationToRow(relation: Relation): RelationTabularRow { + return { + Id: relation.id, + FromId: relation.from, + ToId: relation.to, + Type: relation.type, + Weight: relation.weight ?? null, + Confidence: relation.confidence ?? null, + CreatedAt: this.formatDate(relation.createdAt), + UpdatedAt: relation.updatedAt + ? this.formatDate(relation.updatedAt) + : this.formatDate(relation.createdAt), + Service: relation.service ?? null, + Metadata: relation.metadata ? JSON.stringify(relation.metadata) : null + } + } + + /** + * Convert entities to CSV string + */ + toCSV(entities: Entity[], options?: { delimiter?: string }): string { + const delimiter = options?.delimiter ?? ',' + const rows = this.entitiesToRows(entities) + + if (rows.length === 0) { + return '' + } + + // Get all columns from all rows + const columns = this.getAllColumns(rows) + + // Header row + const header = columns.map((col) => this.escapeCSV(col)).join(delimiter) + + // Data rows + const dataRows = rows.map((row) => + columns + .map((col) => { + const value = row[col] + return this.escapeCSV( + value === null || value === undefined ? '' : String(value) + ) + }) + .join(delimiter) + ) + + return [header, ...dataRows].join('\n') + } + + /** + * Convert relations to CSV string + */ + relationsToCSV( + relations: Relation[], + options?: { delimiter?: string } + ): string { + const delimiter = options?.delimiter ?? ',' + const rows = this.relationsToRows(relations) + + if (rows.length === 0) { + return '' + } + + const columns: (keyof RelationTabularRow)[] = [ + 'Id', + 'FromId', + 'ToId', + 'Type', + 'Weight', + 'Confidence', + 'CreatedAt', + 'UpdatedAt', + 'Service', + 'Metadata' + ] + + const header = columns.join(delimiter) + const dataRows = rows.map((row) => + columns + .map((col) => { + const value = row[col] + return this.escapeCSV( + value === null || value === undefined ? '' : String(value) + ) + }) + .join(delimiter) + ) + + return [header, ...dataRows].join('\n') + } + + /** + * Convert entities to OData format (JSON with annotations) + */ + toOData( + entities: Entity[], + options?: { + context?: string + count?: number + nextLink?: string + } + ): object { + const rows = this.entitiesToRows(entities) + + const result: any = { + '@odata.context': options?.context ?? '$metadata#Entities' + } + + if (options?.count !== undefined) { + result['@odata.count'] = options.count + } + + result.value = rows.map((row) => this.rowToODataEntity(row)) + + if (options?.nextLink) { + result['@odata.nextLink'] = options.nextLink + } + + return result + } + + /** + * Convert relations to OData format + */ + relationsToOData( + relations: Relation[], + options?: { + context?: string + count?: number + nextLink?: string + } + ): object { + const rows = this.relationsToRows(relations) + + const result: any = { + '@odata.context': options?.context ?? '$metadata#Relationships' + } + + if (options?.count !== undefined) { + result['@odata.count'] = options.count + } + + result.value = rows + + if (options?.nextLink) { + result['@odata.nextLink'] = options.nextLink + } + + return result + } + + /** + * Parse CSV string to entity-like objects + */ + parseCSV( + csv: string, + options?: { delimiter?: string } + ): Partial[] { + const delimiter = options?.delimiter ?? ',' + const lines = csv.split('\n').filter((line) => line.trim()) + + if (lines.length < 2) { + return [] + } + + const headers = this.parseCSVLine(lines[0], delimiter) + const entities: Partial[] = [] + + for (let i = 1; i < lines.length; i++) { + const values = this.parseCSVLine(lines[i], delimiter) + const row: Record = {} + + for (let j = 0; j < headers.length; j++) { + row[headers[j]] = values[j] ?? '' + } + + entities.push(this.rowToEntity(row)) + } + + return entities + } + + /** + * Get schema from entities (column names and types) + */ + getSchema(entities: Entity[]): Array<{ + name: string + type: 'string' | 'number' | 'boolean' | 'datetime' | 'json' + nullable: boolean + }> { + const rows = this.entitiesToRows(entities.slice(0, 100)) // Sample first 100 + const columns = this.getAllColumns(rows) + const schema: Array<{ + name: string + type: 'string' | 'number' | 'boolean' | 'datetime' | 'json' + nullable: boolean + }> = [] + + for (const col of columns) { + let type: 'string' | 'number' | 'boolean' | 'datetime' | 'json' = 'string' + let nullable = false + + for (const row of rows) { + const value = row[col] + + if (value === null || value === undefined) { + nullable = true + continue + } + + const inferredType = this.inferType(value) + if (type === 'string') { + type = inferredType + } else if (type !== inferredType) { + // Mixed types, fall back to string + type = 'string' + break + } + } + + schema.push({ name: col, type, nullable }) + } + + return schema + } + + // Private helper methods + + private formatDate(timestamp: number): string { + switch (this.config.dateFormat) { + case 'unix': + return Math.floor(timestamp / 1000).toString() + case 'unix_ms': + return timestamp.toString() + case 'ISO8601': + default: + return new Date(timestamp).toISOString() + } + } + + private flattenObject( + obj: Record, + prefix: string, + maxDepth: number, + currentDepth = 0 + ): Record { + const result: Record = {} + + for (const [key, value] of Object.entries(obj)) { + const newKey = `${prefix}${key}` + + if (value === null || value === undefined) { + result[newKey] = null + } else if (Array.isArray(value)) { + // Arrays are always JSON stringified + result[newKey] = JSON.stringify(value) + } else if (typeof value === 'object') { + // Objects: flatten if under depth limit, otherwise stringify + if (currentDepth < maxDepth - 1) { + Object.assign( + result, + this.flattenObject(value, `${newKey}_`, maxDepth, currentDepth + 1) + ) + } else { + // Max depth reached - stringify the object + result[newKey] = JSON.stringify(value) + } + } else { + // Primitives: use as-is + result[newKey] = value + } + } + + return result + } + + private getAllColumns(rows: TabularRow[]): string[] { + const columnSet = new Set() + + // Standard columns first + const standardColumns = [ + 'Id', + 'Type', + 'CreatedAt', + 'UpdatedAt', + 'Confidence', + 'Weight', + 'Service', + 'Data' + ] + + for (const col of standardColumns) { + columnSet.add(col) + } + + // Add all other columns from rows + for (const row of rows) { + for (const key of Object.keys(row)) { + columnSet.add(key) + } + } + + return Array.from(columnSet) + } + + private escapeCSV(value: string): string { + // Escape quotes and wrap in quotes if contains special characters + if ( + value.includes(',') || + value.includes('"') || + value.includes('\n') || + value.includes('\r') + ) { + return `"${value.replace(/"/g, '""')}"` + } + return value + } + + private parseCSVLine(line: string, delimiter: string): string[] { + const result: string[] = [] + let current = '' + let inQuotes = false + + for (let i = 0; i < line.length; i++) { + const char = line[i] + + if (inQuotes) { + if (char === '"') { + if (line[i + 1] === '"') { + current += '"' + i++ + } else { + inQuotes = false + } + } else { + current += char + } + } else { + if (char === '"') { + inQuotes = true + } else if (char === delimiter) { + result.push(current) + current = '' + } else { + current += char + } + } + } + + result.push(current) + return result + } + + private rowToEntity(row: Record): Partial { + const entity: Partial = {} + + // Map standard columns + if (row.Id) entity.id = row.Id + if (row.Type) entity.type = row.Type as any + if (row.Service) entity.service = row.Service + if (row.Confidence) entity.confidence = parseFloat(row.Confidence) + if (row.Weight) entity.weight = parseFloat(row.Weight) + + // Parse timestamps + if (row.CreatedAt) { + entity.createdAt = this.parseDate(row.CreatedAt) + } + if (row.UpdatedAt) { + entity.updatedAt = this.parseDate(row.UpdatedAt) + } + + // Parse data + if (row.Data) { + try { + entity.data = JSON.parse(row.Data) + } catch { + entity.data = row.Data + } + } + + // Collect metadata from prefixed columns + const metadata: Record = {} + for (const [key, value] of Object.entries(row)) { + if (key.startsWith(this.config.metadataPrefix)) { + const metaKey = key.slice(this.config.metadataPrefix.length) + try { + metadata[metaKey] = JSON.parse(value as string) + } catch { + metadata[metaKey] = value + } + } + } + if (Object.keys(metadata).length > 0) { + entity.metadata = metadata + } + + return entity + } + + private parseDate(value: string): number { + // Try parsing as number (unix timestamp) + const num = Number(value) + if (!isNaN(num)) { + // If it looks like seconds (< year 3000 in seconds) + if (num < 32503680000) { + return num * 1000 + } + return num + } + + // Try parsing as ISO date + const date = new Date(value) + if (!isNaN(date.getTime())) { + return date.getTime() + } + + return Date.now() + } + + private rowToODataEntity(row: TabularRow): object { + const result: any = {} + + for (const [key, value] of Object.entries(row)) { + if (value === null) { + result[key] = null + } else if (key === 'CreatedAt' || key === 'UpdatedAt') { + // OData datetime format + result[key] = value + } else { + result[key] = value + } + } + + return result + } + + private inferType( + value: any + ): 'string' | 'number' | 'boolean' | 'datetime' | 'json' { + if (typeof value === 'number') { + return 'number' + } + if (typeof value === 'boolean') { + return 'boolean' + } + if (typeof value === 'string') { + // Check if it's a date + if (/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/.test(value)) { + return 'datetime' + } + // Check if it's JSON + if ( + (value.startsWith('{') && value.endsWith('}')) || + (value.startsWith('[') && value.endsWith(']')) + ) { + try { + JSON.parse(value) + return 'json' + } catch { + // Not valid JSON + } + } + } + return 'string' + } +} diff --git a/src/integrations/core/index.ts b/src/integrations/core/index.ts new file mode 100644 index 00000000..f0802090 --- /dev/null +++ b/src/integrations/core/index.ts @@ -0,0 +1,64 @@ +/** + * Integration Hub - Core Infrastructure + * + * Shared foundation for all integrations: + * - EventBus: Real-time change notifications + * - TabularExporter: Entity to rows/columns conversion + * - IntegrationBase: Base class for integrations + * - IntegrationHub: Zero-config integration manager + */ + +// Event system +export { EventBus } from './EventBus.js' + +// Tabular export +export { TabularExporter } from './TabularExporter.js' + +// Base class +export { + IntegrationBase, + type HTTPIntegration, + type StreamingIntegration +} from './IntegrationBase.js' + +// Integration loader +export { + IntegrationLoader, + createIntegrationLoader, + detectEnvironment, + INTEGRATION_CATALOG, + type IntegrationType, + type RuntimeEnvironment, + type IntegrationInfo, + type IntegrationLoaderConfig +} from './IntegrationLoader.js' + +// Zero-config hub +export { + IntegrationHub, + createIntegrationHub, + type IntegrationHubConfig, + type IntegrationRequest, + type IntegrationResponse +} from './IntegrationHub.js' + +// Types +export type { + // Events + BrainyEvent, + EventFilter, + EventHandler, + EventSubscription, + // Tabular + TabularRow, + RelationTabularRow, + TabularExporterConfig, + // Config + IntegrationConfig, + IntegrationHealthStatus, + // OData + ODataQueryOptions, + // Webhooks + WebhookRegistration, + WebhookDeliveryResult +} from './types.js' diff --git a/src/integrations/core/types.ts b/src/integrations/core/types.ts new file mode 100644 index 00000000..ba53d682 --- /dev/null +++ b/src/integrations/core/types.ts @@ -0,0 +1,261 @@ +/** + * Integration Hub - Shared Types + * + * Types for OData, Google Sheets, SSE, and Webhooks integrations. + * Zero external dependencies. + */ + +import { Entity, Relation } from '../../types/brainy.types.js' +import { NounType, VerbType } from '../../types/graphTypes.js' + +// ============================================================================ +// Events - Real-time change notifications +// ============================================================================ + +/** + * Real-time event emitted when Brainy data changes + */ +export interface BrainyEvent { + /** Unique event identifier */ + id: string + + /** What changed: noun, verb, or VFS */ + entityType: 'noun' | 'verb' | 'vfs' + + /** What happened */ + operation: 'create' | 'update' | 'delete' + + /** The entity ID that was affected */ + entityId: string + + /** Unix timestamp in milliseconds */ + timestamp: number + + /** Monotonically increasing sequence ID for ordering/resumption */ + sequenceId: bigint + + /** NounType if entityType is 'noun' */ + nounType?: NounType + + /** VerbType if entityType is 'verb' */ + verbType?: VerbType + + /** Service (multi-tenancy) */ + service?: string + + /** Full entity data (if includeData is enabled) */ + data?: Entity | Relation +} + +/** + * Filter for subscribing to events + */ +export interface EventFilter { + /** Filter by entity types */ + entityTypes?: ('noun' | 'verb' | 'vfs')[] + + /** Filter by operations */ + operations?: ('create' | 'update' | 'delete')[] + + /** Filter by noun types */ + nounTypes?: NounType[] + + /** Filter by verb types */ + verbTypes?: VerbType[] + + /** Filter by service */ + service?: string + + /** Resume from this sequence ID */ + since?: bigint +} + +/** + * Event handler function + */ +export type EventHandler = (event: BrainyEvent) => void | Promise + +/** + * Event subscription handle + */ +export interface EventSubscription { + id: string + unsubscribe(): void +} + +// ============================================================================ +// Tabular Export - Entity to rows/columns conversion +// ============================================================================ + +/** + * Tabular row representation of an entity + */ +export interface TabularRow { + Id: string + Type: string + CreatedAt: string + UpdatedAt: string + Confidence: number | null + Weight: number | null + Service: string | null + Data: string | null + /** Flattened metadata columns (Metadata_*) */ + [key: string]: any +} + +/** + * Tabular row for relations + */ +export interface RelationTabularRow { + Id: string + FromId: string + ToId: string + Type: string + Weight: number | null + Confidence: number | null + CreatedAt: string + UpdatedAt: string + Service: string | null + Metadata: string | null +} + +/** + * Configuration for TabularExporter + */ +export interface TabularExporterConfig { + /** Flatten metadata into separate columns (default: true) */ + flattenMetadata?: boolean + + /** Prefix for metadata columns (default: 'Metadata_') */ + metadataPrefix?: string + + /** Include vector embeddings (default: false) */ + includeVectors?: boolean + + /** Date format (default: 'ISO8601') */ + dateFormat?: 'ISO8601' | 'unix' | 'unix_ms' + + /** Fields to JSON.stringify (default: ['data']) */ + jsonStringify?: string[] + + /** Max depth for flattening nested objects (default: 2) */ + maxFlattenDepth?: number + + /** Columns to exclude */ + excludeColumns?: string[] +} + +// ============================================================================ +// Integration Configuration +// ============================================================================ + +/** + * Base configuration for all integrations + */ +export interface IntegrationConfig { + /** Enable/disable the integration */ + enabled?: boolean + + /** Rate limiting */ + rateLimit?: { + max: number + windowMs: number + } + + /** Authentication */ + auth?: { + required: boolean + apiKeys?: string[] + } + + /** CORS settings */ + cors?: { + origin: string | string[] + methods?: string[] + credentials?: boolean + } +} + +/** + * Health status for an integration + */ +export interface IntegrationHealthStatus { + name: string + status: 'healthy' | 'degraded' | 'unhealthy' | 'stopped' + message?: string + uptimeMs?: number + requestCount?: number + errorCount?: number + lastError?: string + checkedAt: number +} + +// ============================================================================ +// OData - Excel Power Query, Power BI, Tableau +// ============================================================================ + +/** + * OData query options parsed from URL + */ +export interface ODataQueryOptions { + /** $filter expression */ + filter?: string + + /** $select columns */ + select?: string[] + + /** $orderby specification */ + orderBy?: Array<{ field: string; direction: 'asc' | 'desc' }> + + /** $top (limit) */ + top?: number + + /** $skip (offset) */ + skip?: number + + /** $expand relations */ + expand?: string[] + + /** $count - include total count */ + count?: boolean + + /** $search - full text search */ + search?: string +} + +// ============================================================================ +// Webhooks - Push notifications +// ============================================================================ + +/** + * Webhook registration + */ +export interface WebhookRegistration { + id: string + url: string + events: EventFilter + secret?: string + active: boolean + retryPolicy?: { + maxRetries: number + backoffMultiplier: number + initialDelayMs: number + maxDelayMs: number + } + createdAt: number + lastDeliveryAt?: number + failureCount?: number +} + +/** + * Webhook delivery result + */ +export interface WebhookDeliveryResult { + webhookId: string + eventId: string + success: boolean + statusCode?: number + error?: string + attempts: number + timestamp: number +} diff --git a/src/integrations/index.ts b/src/integrations/index.ts new file mode 100644 index 00000000..804d272c --- /dev/null +++ b/src/integrations/index.ts @@ -0,0 +1,138 @@ +/** + * Brainy Integration Hub + * + * Connect Brainy to external tools with zero configuration: + * - Google Sheets (two-way sync) + * - Excel / Power BI / Tableau (OData 4.0) + * - Real-time dashboards (SSE streaming) + * - External webhooks (push notifications) + * + * @example Enable integrations (v7.4.0 - recommended) + * ```typescript + * import { Brainy } from '@soulcraft/brainy' + * + * const brain = new Brainy({ integrations: true }) + * await brain.init() + * + * // Access the hub + * console.log(brain.hub.endpoints) + * // { odata: '/odata', sheets: '/sheets', sse: '/events', webhooks: '/webhooks' } + * + * // Handle requests (Express, Hono, etc.) + * app.all('/odata/*', async (req, res) => { + * const response = await brain.hub.handleRequest(req) + * res.status(response.status).json(response.body) + * }) + * ``` + */ + +// ============================================================================ +// Integration Hub (used internally by brain.hub) +// ============================================================================ + +export { + IntegrationHub, + createIntegrationHub, + type IntegrationHubConfig, + type IntegrationRequest, + type IntegrationResponse +} from './core/IntegrationHub.js' + +// ============================================================================ +// Core Infrastructure +// ============================================================================ + +export { + // Event system for real-time updates + EventBus, + // Entity → rows/columns conversion + TabularExporter, + // Base class for custom integrations + IntegrationBase, + // Lazy-loading manager + IntegrationLoader, + createIntegrationLoader, + // Environment detection + detectEnvironment, + // Integration catalog + INTEGRATION_CATALOG +} from './core/index.js' + +// Types +export type { + HTTPIntegration, + StreamingIntegration, + IntegrationType, + RuntimeEnvironment, + IntegrationInfo, + IntegrationLoaderConfig +} from './core/index.js' + +export type { + BrainyEvent, + EventFilter, + EventHandler, + EventSubscription, + TabularRow, + RelationTabularRow, + TabularExporterConfig, + IntegrationConfig, + IntegrationHealthStatus, + ODataQueryOptions, + WebhookRegistration, + WebhookDeliveryResult +} from './core/index.js' + +// ============================================================================ +// Individual Integrations +// ============================================================================ + +/** + * OData 4.0 Integration + * + * Connect Excel Power Query, Power BI, Tableau, and any OData client. + */ +export { ODataIntegration, type ODataConfig } from './odata/index.js' + +/** + * Google Sheets Integration + * + * Two-way sync between Brainy and Google Sheets via Apps Script. + */ +export { GoogleSheetsIntegration, type GoogleSheetsConfig } from './sheets/index.js' + +/** + * SSE Streaming Integration + * + * Real-time event streaming via Server-Sent Events. + */ +export { SSEIntegration, type SSEConfig } from './sse/index.js' + +/** + * Webhook Integration + * + * Push events to external URLs with retry and signing. + */ +export { WebhookIntegration, type WebhookConfig } from './webhooks/index.js' + +// ============================================================================ +// OData Utilities (for advanced use) +// ============================================================================ + +export { + parseODataQuery, + parseFilter, + parseOrderBy, + parseSelect, + odataToFindParams, + applyFilter, + applySelect, + applyOrderBy, + applyPagination +} from './odata/ODataQueryParser.js' + +export { + generateEdmx, + generateMetadataJson, + generateServiceDocument +} from './odata/EdmxGenerator.js' diff --git a/src/integrations/odata/EdmxGenerator.ts b/src/integrations/odata/EdmxGenerator.ts new file mode 100644 index 00000000..3755c4ff --- /dev/null +++ b/src/integrations/odata/EdmxGenerator.ts @@ -0,0 +1,238 @@ +/** + * EDMX Metadata Generator + * + * Generates OData $metadata XML document describing the Brainy data model. + * This allows tools like Excel Power Query and Power BI to discover the schema. + */ + +import { NounType, VerbType } from '../../types/graphTypes.js' + +/** + * OData property type mapping + */ +interface PropertyDef { + name: string + type: string + nullable: boolean +} + +/** + * Generate EDMX metadata XML for Brainy schema + * + * @param options Configuration options + * @returns XML string + */ +export function generateEdmx(options?: { + namespace?: string + containerName?: string + includeRelationships?: boolean +}): string { + const namespace = options?.namespace ?? 'Brainy' + const containerName = options?.containerName ?? 'BrainyContainer' + const includeRelationships = options?.includeRelationships ?? true + + const entityProperties = getEntityProperties() + const relationshipProperties = getRelationshipProperties() + + let xml = ` + + + + + + + + + +${entityProperties.map((p) => ` `).join('\n')} + + +${includeRelationships ? generateRelationshipType(relationshipProperties) : ''} + + + +${Object.values(NounType) + .map((v, i) => ` `) + .join('\n')} + + +${includeRelationships ? generateVerbTypeEnum() : ''} + + + + +${includeRelationships ? ` ` : ''} + + + + +` + + return xml +} + +/** + * Generate JSON-based OData metadata (for modern clients) + */ +export function generateMetadataJson(options?: { + namespace?: string + includeRelationships?: boolean +}): object { + const namespace = options?.namespace ?? 'Brainy' + const includeRelationships = options?.includeRelationships ?? true + + const entityProps = getEntityProperties() + const relProps = getRelationshipProperties() + + const schema: any = { + $Version: '4.0', + [`${namespace}`]: { + Entity: { + $Kind: 'EntityType', + $Key: ['Id'], + ...Object.fromEntries( + entityProps.map((p) => [ + p.name, + { + $Type: p.type.replace('Edm.', ''), + $Nullable: p.nullable + } + ]) + ) + }, + NounType: { + $Kind: 'EnumType', + ...Object.fromEntries( + Object.values(NounType).map((v, i) => [v, i]) + ) + } + }, + [`${namespace}.Container`]: { + $Kind: 'EntityContainer', + Entities: { + $Collection: true, + $Type: `${namespace}.Entity` + } + } + } + + if (includeRelationships) { + schema[namespace].Relationship = { + $Kind: 'EntityType', + $Key: ['Id'], + ...Object.fromEntries( + relProps.map((p) => [ + p.name, + { + $Type: p.type.replace('Edm.', ''), + $Nullable: p.nullable + } + ]) + ) + } + + schema[namespace].VerbType = { + $Kind: 'EnumType', + ...Object.fromEntries( + Object.values(VerbType).map((v, i) => [v, i]) + ) + } + + schema[`${namespace}.Container`].Relationships = { + $Collection: true, + $Type: `${namespace}.Relationship` + } + } + + return schema +} + +/** + * Get service document (root OData response) + */ +export function generateServiceDocument( + baseUrl: string, + options?: { includeRelationships?: boolean } +): object { + const includeRelationships = options?.includeRelationships ?? true + + const collections = [ + { + name: 'Entities', + kind: 'EntitySet', + url: 'Entities' + } + ] + + if (includeRelationships) { + collections.push({ + name: 'Relationships', + kind: 'EntitySet', + url: 'Relationships' + }) + } + + return { + '@odata.context': `${baseUrl}/$metadata`, + value: collections + } +} + +// Private helpers + +function getEntityProperties(): PropertyDef[] { + return [ + { name: 'Id', type: 'Edm.String', nullable: false }, + { name: 'Type', type: 'Edm.String', nullable: false }, + { name: 'CreatedAt', type: 'Edm.DateTimeOffset', nullable: false }, + { name: 'UpdatedAt', type: 'Edm.DateTimeOffset', nullable: true }, + { name: 'Confidence', type: 'Edm.Double', nullable: true }, + { name: 'Weight', type: 'Edm.Double', nullable: true }, + { name: 'Service', type: 'Edm.String', nullable: true }, + { name: 'Data', type: 'Edm.String', nullable: true }, + // Common metadata fields (flattened) + { name: 'Metadata_name', type: 'Edm.String', nullable: true }, + { name: 'Metadata_title', type: 'Edm.String', nullable: true }, + { name: 'Metadata_description', type: 'Edm.String', nullable: true }, + { name: 'Metadata_email', type: 'Edm.String', nullable: true }, + { name: 'Metadata_url', type: 'Edm.String', nullable: true }, + { name: 'Metadata_tags', type: 'Edm.String', nullable: true }, + { name: 'Metadata_category', type: 'Edm.String', nullable: true }, + { name: 'Metadata_status', type: 'Edm.String', nullable: true }, + { name: 'Metadata_priority', type: 'Edm.Int32', nullable: true } + ] +} + +function getRelationshipProperties(): PropertyDef[] { + return [ + { name: 'Id', type: 'Edm.String', nullable: false }, + { name: 'FromId', type: 'Edm.String', nullable: false }, + { name: 'ToId', type: 'Edm.String', nullable: false }, + { name: 'Type', type: 'Edm.String', nullable: false }, + { name: 'Weight', type: 'Edm.Double', nullable: true }, + { name: 'Confidence', type: 'Edm.Double', nullable: true }, + { name: 'CreatedAt', type: 'Edm.DateTimeOffset', nullable: false }, + { name: 'UpdatedAt', type: 'Edm.DateTimeOffset', nullable: true }, + { name: 'Service', type: 'Edm.String', nullable: true }, + { name: 'Metadata', type: 'Edm.String', nullable: true } + ] +} + +function generateRelationshipType(properties: PropertyDef[]): string { + return ` + + + + +${properties.map((p) => ` `).join('\n')} + ` +} + +function generateVerbTypeEnum(): string { + return ` + +${Object.values(VerbType) + .map((v, i) => ` `) + .join('\n')} + ` +} diff --git a/src/integrations/odata/ODataIntegration.ts b/src/integrations/odata/ODataIntegration.ts new file mode 100644 index 00000000..0e50d792 --- /dev/null +++ b/src/integrations/odata/ODataIntegration.ts @@ -0,0 +1,710 @@ +/** + * OData Integration + * + * Exposes Brainy data via OData 4.0 REST API for: + * - Excel Power Query + * - Power BI + * - Tableau + * - Any OData-compatible BI tool + * + * Zero external dependencies - works in all environments. + */ + +import { + IntegrationBase, + HTTPIntegration +} from '../core/IntegrationBase.js' +import { IntegrationConfig, ODataQueryOptions } from '../core/types.js' +import { AugmentationManifest } from '../../augmentations/manifest.js' +import { Entity, Relation, FindParams } from '../../types/brainy.types.js' +import { NounType } from '../../types/graphTypes.js' +import { + parseODataQuery, + applyFilter, + applySelect, + applyOrderBy, + applyPagination +} from './ODataQueryParser.js' +import { + generateEdmx, + generateMetadataJson, + generateServiceDocument +} from './EdmxGenerator.js' + +/** + * OData integration configuration + */ +export interface ODataConfig extends IntegrationConfig { + /** Base path for OData routes (default: '/odata') */ + basePath?: string + + /** Port to listen on (only used when running standalone server) */ + port?: number + + /** Include relationships endpoint (default: true) */ + includeRelationships?: boolean + + /** Maximum page size (default: 1000) */ + maxPageSize?: number + + /** Default page size (default: 100) */ + defaultPageSize?: number + + /** Namespace for metadata (default: 'Brainy') */ + namespace?: string +} + +/** + * OData request context + */ +interface ODataRequest { + method: string + path: string + query: string + body?: any + headers: Record +} + +/** + * OData response + */ +interface ODataResponse { + status: number + headers: Record + body: any +} + +/** + * OData Integration + * + * Provides OData 4.0 REST API for Brainy data access from spreadsheets and BI tools. + * + * Routes: + * - GET /odata - Service document + * - GET /odata/$metadata - EDMX metadata + * - GET /odata/Entities - List entities with OData queries + * - GET /odata/Entities('id') - Get single entity + * - GET /odata/Relationships - List relationships + * - GET /odata/Relationships('id') - Get single relationship + * - POST /odata/Entities - Create entity + * - PATCH /odata/Entities('id') - Update entity + * - DELETE /odata/Entities('id') - Delete entity + * + * Supports: $filter, $select, $orderby, $top, $skip, $count, $search + * + * @example + * ```typescript + * // Register with Brainy + * brain.augmentations.register(new ODataIntegration({ + * basePath: '/odata', + * maxPageSize: 1000 + * })) + * + * // Connect from Excel Power Query: + * // Data → Get Data → From OData Feed → http://localhost:3000/odata + * ``` + */ +export class ODataIntegration extends IntegrationBase implements HTTPIntegration { + readonly name = 'odata' + + // HTTPIntegration + port: number + basePath: string + + private odataConfig: ODataConfig & { + enabled: boolean + basePath: string + port: number + includeRelationships: boolean + maxPageSize: number + defaultPageSize: number + namespace: string + } + + constructor(config?: ODataConfig) { + super(config) + + this.odataConfig = { + enabled: config?.enabled ?? true, + basePath: config?.basePath ?? '/odata', + port: config?.port ?? 0, + includeRelationships: config?.includeRelationships ?? true, + maxPageSize: config?.maxPageSize ?? 1000, + defaultPageSize: config?.defaultPageSize ?? 100, + namespace: config?.namespace ?? 'Brainy', + rateLimit: config?.rateLimit, + auth: config?.auth, + cors: config?.cors + } + + this.port = this.odataConfig.port + this.basePath = this.odataConfig.basePath + } + + /** + * Start the integration (registers routes with API server) + */ + protected async onStart(): Promise { + this.log('OData integration started') + } + + /** + * Stop the integration + */ + protected async onStop(): Promise { + this.log('OData integration stopped') + } + + /** + * Handle an OData request + * + * This is the main entry point for processing OData requests. + * Can be called directly or integrated with an HTTP server. + * + * @param request OData request + * @returns OData response + */ + async handleRequest(request: ODataRequest): Promise { + this.recordRequest() + + try { + const { method, path } = request + const relativePath = path.startsWith(this.basePath) + ? path.slice(this.basePath.length) + : path + + // Route the request + if (method === 'GET') { + if (relativePath === '' || relativePath === '/') { + return this.getServiceDocument(request) + } + if (relativePath === '/$metadata') { + return this.getMetadata(request) + } + if (relativePath.startsWith('/Entities')) { + return this.handleEntities(request, relativePath) + } + if (relativePath.startsWith('/Relationships')) { + return this.handleRelationships(request, relativePath) + } + } + + if (method === 'POST') { + if (relativePath === '/Entities') { + return this.createEntity(request) + } + if (relativePath === '/Relationships') { + return this.createRelationship(request) + } + } + + if (method === 'PATCH') { + if (relativePath.startsWith('/Entities(')) { + return this.updateEntity(request, relativePath) + } + } + + if (method === 'DELETE') { + if (relativePath.startsWith('/Entities(')) { + return this.deleteEntity(request, relativePath) + } + if (relativePath.startsWith('/Relationships(')) { + return this.deleteRelationship(request, relativePath) + } + } + + return this.errorResponse(404, 'Not Found') + } catch (error: any) { + this.recordError(error) + return this.errorResponse(500, error.message) + } + } + + /** + * Get registered routes + */ + getRoutes(): Array<{ method: string; path: string; description: string }> { + const routes = [ + { method: 'GET', path: `${this.basePath}`, description: 'Service document' }, + { method: 'GET', path: `${this.basePath}/$metadata`, description: 'EDMX metadata' }, + { method: 'GET', path: `${this.basePath}/Entities`, description: 'List entities' }, + { method: 'GET', path: `${this.basePath}/Entities('id')`, description: 'Get entity' }, + { method: 'POST', path: `${this.basePath}/Entities`, description: 'Create entity' }, + { method: 'PATCH', path: `${this.basePath}/Entities('id')`, description: 'Update entity' }, + { method: 'DELETE', path: `${this.basePath}/Entities('id')`, description: 'Delete entity' } + ] + + if (this.odataConfig.includeRelationships) { + routes.push( + { method: 'GET', path: `${this.basePath}/Relationships`, description: 'List relationships' }, + { method: 'GET', path: `${this.basePath}/Relationships('id')`, description: 'Get relationship' }, + { method: 'POST', path: `${this.basePath}/Relationships`, description: 'Create relationship' }, + { method: 'DELETE', path: `${this.basePath}/Relationships('id')`, description: 'Delete relationship' } + ) + } + + return routes + } + + /** + * Get augmentation manifest + */ + getManifest(): AugmentationManifest { + return { + id: 'odata', + name: 'OData Integration', + version: '1.0.0', + description: 'OData 4.0 API for Excel, Power BI, and BI tools', + longDescription: + 'Exposes Brainy data via OData 4.0 REST API, enabling direct connections from Excel Power Query, Power BI, Tableau, and any OData-compatible tool. Supports $filter, $select, $orderby, $top, $skip, $count queries.', + category: 'integration', + status: 'stable', + configSchema: { + type: 'object', + properties: { + enabled: { type: 'boolean', default: true }, + basePath: { type: 'string', default: '/odata' }, + maxPageSize: { type: 'number', default: 1000 }, + defaultPageSize: { type: 'number', default: 100 }, + includeRelationships: { type: 'boolean', default: true }, + namespace: { type: 'string', default: 'Brainy' } + } + }, + configDefaults: { + enabled: true, + basePath: '/odata', + maxPageSize: 1000, + defaultPageSize: 100, + includeRelationships: true, + namespace: 'Brainy' + }, + features: [ + 'Excel Power Query support', + 'Power BI direct connect', + 'OData $filter queries', + 'OData $select, $orderby', + 'Pagination with $top/$skip', + '$count support', + '$search semantic search' + ], + keywords: ['odata', 'excel', 'power-bi', 'bi', 'rest', 'api'] + } + } + + // Route handlers + + private getServiceDocument(_request: ODataRequest): ODataResponse { + const baseUrl = `${this.basePath}` + return this.jsonResponse( + generateServiceDocument(baseUrl, { + includeRelationships: this.odataConfig.includeRelationships + }) + ) + } + + private getMetadata(request: ODataRequest): ODataResponse { + // Check Accept header for JSON vs XML + const accept = request.headers['accept'] || '' + + if (accept.includes('application/json')) { + return this.jsonResponse( + generateMetadataJson({ + namespace: this.odataConfig.namespace, + includeRelationships: this.odataConfig.includeRelationships + }) + ) + } + + // Default to XML EDMX + return { + status: 200, + headers: { + 'Content-Type': 'application/xml', + 'OData-Version': '4.0' + }, + body: generateEdmx({ + namespace: this.odataConfig.namespace, + includeRelationships: this.odataConfig.includeRelationships + }) + } + } + + private async handleEntities( + request: ODataRequest, + path: string + ): Promise { + // Single entity: /Entities('id') + const idMatch = path.match(/^\/Entities\('([^']+)'\)/) + if (idMatch) { + return this.handleGetEntity(idMatch[1]) + } + + // Collection: /Entities?$filter=... + return this.listEntities(request) + } + + private async listEntities(request: ODataRequest): Promise { + const options = parseODataQuery(request.query) + + // Apply pagination limits + const pageSize = Math.min( + options.top ?? this.odataConfig.defaultPageSize, + this.odataConfig.maxPageSize + ) + options.top = pageSize + + // Query from Brainy + const findParams: FindParams = { + limit: pageSize + 1, // +1 to detect if there are more + offset: options.skip + } + + // Apply orderby + if (options.orderBy && options.orderBy.length > 0) { + findParams.orderBy = options.orderBy[0].field + findParams.order = options.orderBy[0].direction + } + + // Apply $search as semantic query + if (options.search) { + findParams.query = options.search + } + + // Get entities + let entities = await this.queryEntities(findParams) + + // Apply $filter (in-memory for complex filters) + if (options.filter) { + const rows = this.exporter.entitiesToRows(entities) + const filteredRows = applyFilter(rows, options.filter) + // Map back to entities (simple approach) + const filteredIds = new Set(filteredRows.map((r) => r.Id)) + entities = entities.filter((e) => filteredIds.has(e.id)) + } + + // Check for more results + const hasMore = entities.length > pageSize + if (hasMore) { + entities = entities.slice(0, pageSize) + } + + // Convert to tabular format + let rows = this.exporter.entitiesToRows(entities) + + // Apply $select + if (options.select && options.select.length > 0) { + rows = applySelect(rows, options.select) as any + } + + // Apply $orderby (if not handled by storage) + if (options.orderBy && options.orderBy.length > 0) { + rows = applyOrderBy(rows, options.orderBy) + } + + // Build response + const result: any = { + '@odata.context': `${this.basePath}/$metadata#Entities` + } + + // $count + if (options.count) { + // For accurate count, we'd need to query without limit + // For now, use the page count + result['@odata.count'] = entities.length + } + + result.value = rows + + // Next link for pagination + if (hasMore) { + const nextSkip = (options.skip ?? 0) + pageSize + result['@odata.nextLink'] = `${this.basePath}/Entities?$top=${pageSize}&$skip=${nextSkip}` + } + + return this.jsonResponse(result) + } + + private async handleGetEntity(id: string): Promise { + const entity = await super.getEntity(id) + + if (!entity) { + return this.errorResponse(404, `Entity '${id}' not found`) + } + + const row = this.exporter.entityToRow(entity) + + return this.jsonResponse({ + '@odata.context': `${this.basePath}/$metadata#Entities/$entity`, + ...row + }) + } + + private async createEntity(request: ODataRequest): Promise { + if (!this.context) { + return this.errorResponse(500, 'Integration not initialized') + } + + const body = request.body + if (!body || !body.Type) { + return this.errorResponse(400, 'Missing required field: Type') + } + + const entity = await this.context.brain.add({ + type: body.Type as NounType, + data: body.Data ? JSON.parse(body.Data) : undefined, + metadata: this.extractMetadata(body), + confidence: body.Confidence, + weight: body.Weight, + service: body.Service + }) + + const row = this.exporter.entityToRow(entity) + + return { + status: 201, + headers: { + 'Content-Type': 'application/json', + 'OData-Version': '4.0', + 'Location': `${this.basePath}/Entities('${entity.id}')` + }, + body: JSON.stringify({ + '@odata.context': `${this.basePath}/$metadata#Entities/$entity`, + ...row + }) + } + } + + private async updateEntity( + request: ODataRequest, + path: string + ): Promise { + if (!this.context) { + return this.errorResponse(500, 'Integration not initialized') + } + + const idMatch = path.match(/^\/Entities\('([^']+)'\)/) + if (!idMatch) { + return this.errorResponse(400, 'Invalid entity ID') + } + + const id = idMatch[1] + const body = request.body + + const updates: any = { id } + + if (body.Type) updates.type = body.Type as NounType + if (body.Data) updates.data = JSON.parse(body.Data) + if (body.Confidence !== undefined) updates.confidence = body.Confidence + if (body.Weight !== undefined) updates.weight = body.Weight + + const metadata = this.extractMetadata(body) + if (Object.keys(metadata).length > 0) { + updates.metadata = metadata + updates.merge = true + } + + await this.context.brain.update(updates) + + return { + status: 204, + headers: { 'OData-Version': '4.0' }, + body: null + } + } + + private async deleteEntity( + _request: ODataRequest, + path: string + ): Promise { + if (!this.context) { + return this.errorResponse(500, 'Integration not initialized') + } + + const idMatch = path.match(/^\/Entities\('([^']+)'\)/) + if (!idMatch) { + return this.errorResponse(400, 'Invalid entity ID') + } + + await this.context.brain.delete(idMatch[1]) + + return { + status: 204, + headers: { 'OData-Version': '4.0' }, + body: null + } + } + + private async handleRelationships( + request: ODataRequest, + path: string + ): Promise { + // Single relationship: /Relationships('id') + const idMatch = path.match(/^\/Relationships\('([^']+)'\)/) + if (idMatch) { + return this.getRelationship(idMatch[1]) + } + + // Collection + return this.listRelationships(request) + } + + private async listRelationships( + request: ODataRequest + ): Promise { + const options = parseODataQuery(request.query) + + const pageSize = Math.min( + options.top ?? this.odataConfig.defaultPageSize, + this.odataConfig.maxPageSize + ) + + const relations = await this.queryRelations({ + limit: pageSize, + offset: options.skip + }) + + let rows = this.exporter.relationsToRows(relations) + + // Apply $filter + if (options.filter) { + rows = applyFilter(rows, options.filter) as any + } + + // Apply $select + if (options.select && options.select.length > 0) { + rows = applySelect(rows, options.select) as any + } + + // Apply $orderby + if (options.orderBy && options.orderBy.length > 0) { + rows = applyOrderBy(rows, options.orderBy) + } + + return this.jsonResponse({ + '@odata.context': `${this.basePath}/$metadata#Relationships`, + value: rows + }) + } + + private async getRelationship(id: string): Promise { + const relations = await this.queryRelations({ limit: 1000 }) + const relation = relations.find((r) => r.id === id) + + if (!relation) { + return this.errorResponse(404, `Relationship '${id}' not found`) + } + + const row = this.exporter.relationToRow(relation) + + return this.jsonResponse({ + '@odata.context': `${this.basePath}/$metadata#Relationships/$entity`, + ...row + }) + } + + private async createRelationship( + request: ODataRequest + ): Promise { + if (!this.context) { + return this.errorResponse(500, 'Integration not initialized') + } + + const body = request.body + if (!body || !body.FromId || !body.ToId || !body.Type) { + return this.errorResponse( + 400, + 'Missing required fields: FromId, ToId, Type' + ) + } + + const relation = await this.context.brain.relate({ + from: body.FromId, + to: body.ToId, + type: body.Type, + weight: body.Weight, + confidence: body.Confidence, + metadata: body.Metadata ? JSON.parse(body.Metadata) : undefined, + service: body.Service + }) + + const row = this.exporter.relationToRow(relation) + + return { + status: 201, + headers: { + 'Content-Type': 'application/json', + 'OData-Version': '4.0', + 'Location': `${this.basePath}/Relationships('${relation.id}')` + }, + body: JSON.stringify({ + '@odata.context': `${this.basePath}/$metadata#Relationships/$entity`, + ...row + }) + } + } + + private async deleteRelationship( + _request: ODataRequest, + path: string + ): Promise { + if (!this.context) { + return this.errorResponse(500, 'Integration not initialized') + } + + const idMatch = path.match(/^\/Relationships\('([^']+)'\)/) + if (!idMatch) { + return this.errorResponse(400, 'Invalid relationship ID') + } + + await this.context.brain.unrelate(idMatch[1]) + + return { + status: 204, + headers: { 'OData-Version': '4.0' }, + body: null + } + } + + // Helpers + + private jsonResponse(data: any): ODataResponse { + return { + status: 200, + headers: { + 'Content-Type': 'application/json', + 'OData-Version': '4.0' + }, + body: JSON.stringify(data) + } + } + + private errorResponse(status: number, message: string): ODataResponse { + return { + status, + headers: { + 'Content-Type': 'application/json', + 'OData-Version': '4.0' + }, + body: JSON.stringify({ + error: { + code: String(status), + message + } + }) + } + } + + private extractMetadata(body: Record): Record { + const metadata: Record = {} + const prefix = 'Metadata_' + + for (const [key, value] of Object.entries(body)) { + if (key.startsWith(prefix)) { + const metaKey = key.slice(prefix.length) + metadata[metaKey] = value + } + } + + return metadata + } +} diff --git a/src/integrations/odata/ODataQueryParser.ts b/src/integrations/odata/ODataQueryParser.ts new file mode 100644 index 00000000..c9808f54 --- /dev/null +++ b/src/integrations/odata/ODataQueryParser.ts @@ -0,0 +1,650 @@ +/** + * OData Query Parser + * + * Lightweight parser for OData query parameters without heavy dependencies. + * Supports $filter, $select, $orderby, $top, $skip, $count, $search, $expand. + */ + +import { ODataQueryOptions } from '../core/types.js' + +/** + * Parse OData filter expression to a structured filter object + * + * Supports: + * - eq, ne, gt, ge, lt, le comparisons + * - and, or logical operators + * - contains(), startswith(), endswith() string functions + * - Parentheses for grouping + * + * @param filter OData $filter string + * @returns Parsed filter object for query execution + */ +export function parseFilter(filter: string): any { + if (!filter || filter.trim() === '') { + return null + } + + // Tokenize + const tokens = tokenize(filter) + + // Parse expression + return parseExpression(tokens, 0).result +} + +/** + * Parse OData $orderby specification + */ +export function parseOrderBy( + orderby: string +): Array<{ field: string; direction: 'asc' | 'desc' }> { + if (!orderby || orderby.trim() === '') { + return [] + } + + return orderby.split(',').map((part) => { + const trimmed = part.trim() + const parts = trimmed.split(/\s+/) + return { + field: parts[0], + direction: (parts[1]?.toLowerCase() as 'asc' | 'desc') || 'asc' + } + }) +} + +/** + * Parse OData $select specification + */ +export function parseSelect(select: string): string[] { + if (!select || select.trim() === '') { + return [] + } + + return select.split(',').map((s) => s.trim()) +} + +/** + * Parse OData $expand specification + */ +export function parseExpand(expand: string): string[] { + if (!expand || expand.trim() === '') { + return [] + } + + return expand.split(',').map((s) => s.trim()) +} + +/** + * Parse full OData query string to options object + */ +export function parseODataQuery(queryString: string): ODataQueryOptions { + const params = new URLSearchParams(queryString) + const options: ODataQueryOptions = {} + + // $filter + const filter = params.get('$filter') + if (filter) { + options.filter = filter + } + + // $select + const select = params.get('$select') + if (select) { + options.select = parseSelect(select) + } + + // $orderby + const orderby = params.get('$orderby') + if (orderby) { + options.orderBy = parseOrderBy(orderby) + } + + // $top + const top = params.get('$top') + if (top) { + options.top = parseInt(top, 10) + } + + // $skip + const skip = params.get('$skip') + if (skip) { + options.skip = parseInt(skip, 10) + } + + // $expand + const expand = params.get('$expand') + if (expand) { + options.expand = parseExpand(expand) + } + + // $count + const count = params.get('$count') + if (count === 'true') { + options.count = true + } + + // $search + const search = params.get('$search') + if (search) { + options.search = search + } + + return options +} + +/** + * Convert OData options to Brainy FindParams + */ +export function odataToFindParams(options: ODataQueryOptions): any { + const findParams: any = {} + + // Top/Skip -> Limit/Offset + if (options.top !== undefined) { + findParams.limit = options.top + } + if (options.skip !== undefined) { + findParams.offset = options.skip + } + + // OrderBy + if (options.orderBy && options.orderBy.length > 0) { + findParams.orderBy = options.orderBy[0].field + findParams.order = options.orderBy[0].direction + } + + // Search -> Query + if (options.search) { + findParams.query = options.search + } + + // Filter -> Where (simplified) + if (options.filter) { + const parsed = parseFilter(options.filter) + if (parsed) { + findParams.where = filterToWhere(parsed) + } + } + + return findParams +} + +/** + * Apply OData filter to an array of entities + */ +export function applyFilter>( + entities: T[], + filter: string +): T[] { + if (!filter) return entities + + const parsed = parseFilter(filter) + if (!parsed) return entities + + return entities.filter((entity) => evaluateFilter(entity, parsed)) +} + +/** + * Apply OData select to transform entities + */ +export function applySelect>( + entities: T[], + select: string[] +): Partial[] { + if (!select || select.length === 0) return entities + + return entities.map((entity) => { + const result: Partial = {} + for (const field of select) { + if (field in entity) { + result[field as keyof T] = entity[field] + } + } + return result + }) +} + +/** + * Apply OData orderby to sort entities + */ +export function applyOrderBy>( + entities: T[], + orderBy: Array<{ field: string; direction: 'asc' | 'desc' }> +): T[] { + if (!orderBy || orderBy.length === 0) return entities + + return [...entities].sort((a, b) => { + for (const { field, direction } of orderBy) { + const aVal = getNestedValue(a, field) + const bVal = getNestedValue(b, field) + + let cmp = 0 + if (aVal === bVal) { + cmp = 0 + } else if (aVal === null || aVal === undefined) { + cmp = 1 + } else if (bVal === null || bVal === undefined) { + cmp = -1 + } else if (typeof aVal === 'string' && typeof bVal === 'string') { + cmp = aVal.localeCompare(bVal) + } else { + cmp = aVal < bVal ? -1 : 1 + } + + if (cmp !== 0) { + return direction === 'desc' ? -cmp : cmp + } + } + return 0 + }) +} + +/** + * Apply top/skip pagination + */ +export function applyPagination( + entities: T[], + top?: number, + skip?: number +): T[] { + let result = entities + + if (skip !== undefined && skip > 0) { + result = result.slice(skip) + } + + if (top !== undefined && top > 0) { + result = result.slice(0, top) + } + + return result +} + +// Private helpers + +type Token = { + type: + | 'identifier' + | 'string' + | 'number' + | 'boolean' + | 'null' + | 'operator' + | 'function' + | 'lparen' + | 'rparen' + | 'comma' + value: string +} + +function tokenize(input: string): Token[] { + const tokens: Token[] = [] + let i = 0 + + while (i < input.length) { + // Skip whitespace + if (/\s/.test(input[i])) { + i++ + continue + } + + // String literal + if (input[i] === "'") { + let str = '' + i++ + while (i < input.length && input[i] !== "'") { + if (input[i] === "'" && input[i + 1] === "'") { + str += "'" + i += 2 + } else { + str += input[i] + i++ + } + } + i++ // Skip closing quote + tokens.push({ type: 'string', value: str }) + continue + } + + // Number + if (/\d/.test(input[i]) || (input[i] === '-' && /\d/.test(input[i + 1]))) { + let num = '' + while (i < input.length && /[\d.\-]/.test(input[i])) { + num += input[i] + i++ + } + tokens.push({ type: 'number', value: num }) + continue + } + + // Parentheses + if (input[i] === '(') { + tokens.push({ type: 'lparen', value: '(' }) + i++ + continue + } + if (input[i] === ')') { + tokens.push({ type: 'rparen', value: ')' }) + i++ + continue + } + + // Comma + if (input[i] === ',') { + tokens.push({ type: 'comma', value: ',' }) + i++ + continue + } + + // Identifier or keyword + if (/[a-zA-Z_]/.test(input[i])) { + let ident = '' + while (i < input.length && /[a-zA-Z0-9_./]/.test(input[i])) { + ident += input[i] + i++ + } + + const lower = ident.toLowerCase() + + // Operators + if (['eq', 'ne', 'gt', 'ge', 'lt', 'le', 'and', 'or', 'not'].includes(lower)) { + tokens.push({ type: 'operator', value: lower }) + } + // Boolean + else if (lower === 'true' || lower === 'false') { + tokens.push({ type: 'boolean', value: lower }) + } + // Null + else if (lower === 'null') { + tokens.push({ type: 'null', value: 'null' }) + } + // Functions + else if ( + ['contains', 'startswith', 'endswith', 'tolower', 'toupper', 'length', 'trim', 'substring'].includes( + lower + ) + ) { + tokens.push({ type: 'function', value: lower }) + } + // Identifier (field name) + else { + tokens.push({ type: 'identifier', value: ident }) + } + continue + } + + // Unknown character, skip + i++ + } + + return tokens +} + +function parseExpression( + tokens: Token[], + pos: number +): { result: any; pos: number } { + return parseOr(tokens, pos) +} + +function parseOr( + tokens: Token[], + pos: number +): { result: any; pos: number } { + let { result: left, pos: nextPos } = parseAnd(tokens, pos) + + while (nextPos < tokens.length && tokens[nextPos]?.value === 'or') { + nextPos++ // Skip 'or' + const { result: right, pos: newPos } = parseAnd(tokens, nextPos) + left = { or: [left, right] } + nextPos = newPos + } + + return { result: left, pos: nextPos } +} + +function parseAnd( + tokens: Token[], + pos: number +): { result: any; pos: number } { + let { result: left, pos: nextPos } = parsePrimary(tokens, pos) + + while (nextPos < tokens.length && tokens[nextPos]?.value === 'and') { + nextPos++ // Skip 'and' + const { result: right, pos: newPos } = parsePrimary(tokens, nextPos) + left = { and: [left, right] } + nextPos = newPos + } + + return { result: left, pos: nextPos } +} + +function parsePrimary( + tokens: Token[], + pos: number +): { result: any; pos: number } { + if (pos >= tokens.length) { + return { result: null, pos } + } + + const token = tokens[pos] + + // Parenthesized expression + if (token.type === 'lparen') { + const { result, pos: endPos } = parseExpression(tokens, pos + 1) + // Skip rparen + return { result, pos: endPos + 1 } + } + + // Function call + if (token.type === 'function') { + return parseFunction(tokens, pos) + } + + // Not operator + if (token.value === 'not') { + const { result, pos: endPos } = parsePrimary(tokens, pos + 1) + return { result: { not: result }, pos: endPos } + } + + // Comparison: identifier operator value + if (token.type === 'identifier') { + const field = token.value + pos++ + + if (pos >= tokens.length || tokens[pos].type !== 'operator') { + return { result: { field, exists: true }, pos } + } + + const op = tokens[pos].value + pos++ + + if (pos >= tokens.length) { + return { result: { field, op, value: null }, pos } + } + + const valueToken = tokens[pos] + let value: any + + switch (valueToken.type) { + case 'string': + value = valueToken.value + break + case 'number': + value = parseFloat(valueToken.value) + break + case 'boolean': + value = valueToken.value === 'true' + break + case 'null': + value = null + break + default: + value = valueToken.value + } + + return { result: { field, op, value }, pos: pos + 1 } + } + + return { result: null, pos: pos + 1 } +} + +function parseFunction( + tokens: Token[], + pos: number +): { result: any; pos: number } { + const funcName = tokens[pos].value + pos++ // Skip function name + + if (tokens[pos]?.type !== 'lparen') { + return { result: null, pos } + } + pos++ // Skip '(' + + const args: any[] = [] + + while (pos < tokens.length && tokens[pos]?.type !== 'rparen') { + if (tokens[pos].type === 'comma') { + pos++ + continue + } + + if (tokens[pos].type === 'identifier') { + args.push({ field: tokens[pos].value }) + pos++ + } else if (tokens[pos].type === 'string') { + args.push({ value: tokens[pos].value }) + pos++ + } else if (tokens[pos].type === 'number') { + args.push({ value: parseFloat(tokens[pos].value) }) + pos++ + } else { + pos++ + } + } + + pos++ // Skip ')' + + return { result: { func: funcName, args }, pos } +} + +function evaluateFilter(entity: Record, filter: any): boolean { + if (!filter) return true + + // Logical operators + if (filter.and) { + return filter.and.every((f: any) => evaluateFilter(entity, f)) + } + if (filter.or) { + return filter.or.some((f: any) => evaluateFilter(entity, f)) + } + if (filter.not) { + return !evaluateFilter(entity, filter.not) + } + + // Function + if (filter.func) { + return evaluateFunction(entity, filter.func, filter.args) + } + + // Comparison + if (filter.field && filter.op) { + const fieldValue = getNestedValue(entity, filter.field) + return compareValues(fieldValue, filter.op, filter.value) + } + + return true +} + +function evaluateFunction( + entity: Record, + func: string, + args: any[] +): boolean { + if (args.length < 2) return false + + const fieldValue = args[0].field + ? String(getNestedValue(entity, args[0].field) ?? '') + : '' + const searchValue = args[1].value ?? '' + + switch (func) { + case 'contains': + return fieldValue.toLowerCase().includes(searchValue.toLowerCase()) + case 'startswith': + return fieldValue.toLowerCase().startsWith(searchValue.toLowerCase()) + case 'endswith': + return fieldValue.toLowerCase().endsWith(searchValue.toLowerCase()) + default: + return false + } +} + +function compareValues(fieldValue: any, op: string, filterValue: any): boolean { + // Handle null comparisons + if (filterValue === null) { + switch (op) { + case 'eq': + return fieldValue === null || fieldValue === undefined + case 'ne': + return fieldValue !== null && fieldValue !== undefined + default: + return false + } + } + + // Null field value + if (fieldValue === null || fieldValue === undefined) { + return op === 'ne' + } + + // Type coercion for comparisons + let fv = fieldValue + let cv = filterValue + + if (typeof filterValue === 'number' && typeof fieldValue === 'string') { + fv = parseFloat(fieldValue) + } + if (typeof filterValue === 'string' && typeof fieldValue === 'number') { + cv = parseFloat(filterValue) + } + + switch (op) { + case 'eq': + return fv === cv || (typeof fv === 'string' && fv.toLowerCase() === String(cv).toLowerCase()) + case 'ne': + return fv !== cv + case 'gt': + return fv > cv + case 'ge': + return fv >= cv + case 'lt': + return fv < cv + case 'le': + return fv <= cv + default: + return false + } +} + +function getNestedValue(obj: Record, path: string): any { + const parts = path.split(/[./]/) + let value: any = obj + + for (const part of parts) { + if (value === null || value === undefined) return undefined + value = value[part] + } + + return value +} + +function filterToWhere(filter: any): any { + if (!filter) return {} + + // Simple comparison + if (filter.field && filter.op && filter.op === 'eq') { + return { [filter.field]: filter.value } + } + + // For more complex filters, return as-is for storage adapter to handle + return filter +} diff --git a/src/integrations/odata/index.ts b/src/integrations/odata/index.ts new file mode 100644 index 00000000..1dbd45e6 --- /dev/null +++ b/src/integrations/odata/index.ts @@ -0,0 +1,24 @@ +/** + * OData Integration Module + * + * Provides OData 4.0 REST API for Excel, Power BI, and BI tools. + */ + +export { ODataIntegration, type ODataConfig } from './ODataIntegration.js' +export { + parseODataQuery, + parseFilter, + parseOrderBy, + parseSelect, + parseExpand, + odataToFindParams, + applyFilter, + applySelect, + applyOrderBy, + applyPagination +} from './ODataQueryParser.js' +export { + generateEdmx, + generateMetadataJson, + generateServiceDocument +} from './EdmxGenerator.js' diff --git a/src/integrations/sheets/GoogleSheetsIntegration.ts b/src/integrations/sheets/GoogleSheetsIntegration.ts new file mode 100644 index 00000000..08dbd941 --- /dev/null +++ b/src/integrations/sheets/GoogleSheetsIntegration.ts @@ -0,0 +1,771 @@ +/** + * Google Sheets Integration + * + * Provides REST API endpoints optimized for Google Apps Script to enable + * two-way sync between Brainy and Google Sheets. + * + * Features: + * - Custom function support (=BRAINY_QUERY(), =BRAINY_ADD()) + * - Real-time updates via SSE subscription + * - Batch operations for performance + * - Simple authentication (API key or Bearer token) + * + * Zero external dependencies - works in all environments. + */ + +import { IntegrationBase, HTTPIntegration } from '../core/IntegrationBase.js' +import { IntegrationConfig } from '../core/types.js' +import { AugmentationManifest } from '../../augmentations/manifest.js' +import { Entity, FindParams } from '../../types/brainy.types.js' +import { NounType, VerbType } from '../../types/graphTypes.js' + +/** + * Google Sheets integration configuration + */ +export interface GoogleSheetsConfig extends IntegrationConfig { + /** Base path for API routes (default: '/sheets') */ + basePath?: string + + /** Port to listen on (only for standalone) */ + port?: number + + /** Maximum results per query (default: 1000) */ + maxResults?: number + + /** Default query limit (default: 100) */ + defaultLimit?: number + + /** Allow write operations (default: true) */ + allowWrite?: boolean + + /** CORS origins to allow (default: Google Sheets) */ + allowedOrigins?: string[] +} + +/** + * Sheets API request + */ +interface SheetsRequest { + method: string + path: string + query: Record + body?: any + headers: Record +} + +/** + * Sheets API response + */ +interface SheetsResponse { + status: number + headers: Record + body: any +} + +/** + * Google Sheets Integration + * + * Provides a simple REST API optimized for Google Apps Script custom functions. + * + * Endpoints: + * - GET /sheets/query - Query entities (for =BRAINY_QUERY()) + * - GET /sheets/entity/:id - Get single entity (for =BRAINY_GET()) + * - GET /sheets/similar - Semantic search (for =BRAINY_SIMILAR()) + * - GET /sheets/relations - Get relationships (for =BRAINY_RELATIONS()) + * - POST /sheets/add - Add entity (for sidebar) + * - POST /sheets/batch - Batch operations (for range sync) + * - GET /sheets/schema - Get type schema (for sidebar dropdown) + * - GET /sheets/stream - SSE for real-time updates + * + * Response Format (optimized for Sheets): + * ```json + * { + * "headers": ["Id", "Type", "Name", "Email"], + * "rows": [ + * ["uuid-1", "person", "John Doe", "john@example.com"], + * ["uuid-2", "person", "Jane Doe", "jane@example.com"] + * ], + * "count": 2, + * "hasMore": false + * } + * ``` + * + * @example + * ```typescript + * brain.augmentations.register(new GoogleSheetsIntegration({ + * basePath: '/sheets', + * maxResults: 1000 + * })) + * ``` + */ +export class GoogleSheetsIntegration + extends IntegrationBase + implements HTTPIntegration +{ + readonly name = 'sheets' + + port: number + basePath: string + + private sheetsConfig: GoogleSheetsConfig & { + enabled: boolean + basePath: string + port: number + maxResults: number + defaultLimit: number + allowWrite: boolean + allowedOrigins: string[] + } + private sseClients: Map void> = + new Map() + + constructor(config?: GoogleSheetsConfig) { + super(config) + + this.sheetsConfig = { + enabled: config?.enabled ?? true, + basePath: config?.basePath ?? '/sheets', + port: config?.port ?? 0, + maxResults: config?.maxResults ?? 1000, + defaultLimit: config?.defaultLimit ?? 100, + allowWrite: config?.allowWrite ?? true, + allowedOrigins: config?.allowedOrigins ?? [ + 'https://docs.google.com', + 'https://script.google.com' + ], + rateLimit: config?.rateLimit, + auth: config?.auth, + cors: config?.cors ?? { + origin: [ + 'https://docs.google.com', + 'https://script.google.com', + '*' + ], + methods: ['GET', 'POST', 'OPTIONS'], + credentials: true + } + } + + this.port = this.sheetsConfig.port + this.basePath = this.sheetsConfig.basePath + } + + protected async onStart(): Promise { + // Subscribe to changes for real-time sync + this.subscribeToChanges( + { entityTypes: ['noun', 'verb'] }, + (event) => { + // Broadcast to all SSE clients + this.broadcastSSE('change', { + type: event.entityType, + operation: event.operation, + entityId: event.entityId, + timestamp: event.timestamp + }) + } + ) + + this.log('Google Sheets integration started') + } + + protected async onStop(): Promise { + // Close all SSE connections + this.sseClients.clear() + this.log('Google Sheets integration stopped') + } + + /** + * Handle a Sheets API request + */ + async handleRequest(request: SheetsRequest): Promise { + this.recordRequest() + + try { + const { method, path } = request + const relativePath = path.startsWith(this.basePath) + ? path.slice(this.basePath.length) + : path + + // CORS preflight + if (method === 'OPTIONS') { + return this.corsResponse() + } + + // Route the request + if (method === 'GET') { + if (relativePath === '/query' || relativePath === '') { + return this.handleQuery(request) + } + if (relativePath.startsWith('/entity/')) { + const id = relativePath.slice('/entity/'.length) + return this.handleGetEntity(id) + } + if (relativePath === '/similar') { + return this.handleSimilar(request) + } + if (relativePath === '/relations') { + return this.handleRelations(request) + } + if (relativePath === '/schema') { + return this.handleSchema() + } + if (relativePath === '/stream') { + return this.handleStream(request) + } + if (relativePath === '/health') { + return this.handleHealth() + } + } + + if (method === 'POST' && this.sheetsConfig.allowWrite) { + if (relativePath === '/add') { + return this.handleAdd(request) + } + if (relativePath === '/update') { + return this.handleUpdate(request) + } + if (relativePath === '/delete') { + return this.handleDelete(request) + } + if (relativePath === '/batch') { + return this.handleBatch(request) + } + if (relativePath === '/relate') { + return this.handleRelate(request) + } + } + + return this.errorResponse(404, 'Not Found') + } catch (error: any) { + this.recordError(error) + return this.errorResponse(500, error.message) + } + } + + /** + * Get registered routes + */ + getRoutes(): Array<{ method: string; path: string; description: string }> { + const routes = [ + { method: 'GET', path: `${this.basePath}/query`, description: 'Query entities' }, + { method: 'GET', path: `${this.basePath}/entity/:id`, description: 'Get entity by ID' }, + { method: 'GET', path: `${this.basePath}/similar`, description: 'Semantic search' }, + { method: 'GET', path: `${this.basePath}/relations`, description: 'Get relationships' }, + { method: 'GET', path: `${this.basePath}/schema`, description: 'Get schema' }, + { method: 'GET', path: `${this.basePath}/stream`, description: 'SSE stream' }, + { method: 'GET', path: `${this.basePath}/health`, description: 'Health check' } + ] + + if (this.sheetsConfig.allowWrite) { + routes.push( + { method: 'POST', path: `${this.basePath}/add`, description: 'Add entity' }, + { method: 'POST', path: `${this.basePath}/update`, description: 'Update entity' }, + { method: 'POST', path: `${this.basePath}/delete`, description: 'Delete entity' }, + { method: 'POST', path: `${this.basePath}/batch`, description: 'Batch operations' }, + { method: 'POST', path: `${this.basePath}/relate`, description: 'Create relationship' } + ) + } + + return routes + } + + /** + * Register an SSE client + */ + registerSSEClient( + clientId: string, + sendFn: (event: string, data: any) => void + ): () => void { + this.sseClients.set(clientId, sendFn) + return () => this.sseClients.delete(clientId) + } + + /** + * Get manifest + */ + getManifest(): AugmentationManifest { + return { + id: 'sheets', + name: 'Google Sheets Integration', + version: '1.0.0', + description: 'Two-way sync between Brainy and Google Sheets', + longDescription: + 'Enables real-time bidirectional synchronization with Google Sheets. Use custom functions like =BRAINY_QUERY() directly in cells, or the sidebar for browsing and editing.', + category: 'integration', + status: 'stable', + configSchema: { + type: 'object', + properties: { + enabled: { type: 'boolean', default: true }, + basePath: { type: 'string', default: '/sheets' }, + maxResults: { type: 'number', default: 1000 }, + defaultLimit: { type: 'number', default: 100 }, + allowWrite: { type: 'boolean', default: true } + } + }, + configDefaults: { + enabled: true, + basePath: '/sheets', + maxResults: 1000, + defaultLimit: 100, + allowWrite: true + }, + features: [ + 'Custom functions (=BRAINY_QUERY)', + 'Real-time sync via SSE', + 'Batch operations', + 'Semantic search support', + 'Type schema discovery' + ], + keywords: ['google-sheets', 'spreadsheet', 'sync', 'real-time'] + } + } + + // Route handlers + + private async handleQuery(request: SheetsRequest): Promise { + const { query } = request + + // Build find params + const findParams: FindParams = { + limit: Math.min( + parseInt(query.limit) || this.sheetsConfig.defaultLimit, + this.sheetsConfig.maxResults + ) + } + + // Query string (semantic search) + if (query.q) { + findParams.query = query.q + } + + // Type filter + if (query.type) { + const types = query.type.split(',') as NounType[] + findParams.type = types.length === 1 ? types[0] : types + } + + // Offset pagination + if (query.offset) { + findParams.offset = parseInt(query.offset) + } + + // Sort + if (query.orderBy) { + findParams.orderBy = query.orderBy + findParams.order = (query.order as 'asc' | 'desc') || 'desc' + } + + // Execute query + const entities = await this.queryEntities(findParams) + + // Convert to sheets format + return this.entitiesToSheetsResponse(entities) + } + + private async handleGetEntity(id: string): Promise { + const entity = await this.getEntity(id) + + if (!entity) { + return this.errorResponse(404, 'Entity not found') + } + + return this.entitiesToSheetsResponse([entity]) + } + + private async handleSimilar(request: SheetsRequest): Promise { + if (!this.context) { + return this.errorResponse(500, 'Not initialized') + } + + const { query } = request + + if (!query.q && !query.to) { + return this.errorResponse(400, 'Missing q or to parameter') + } + + const limit = Math.min( + parseInt(query.limit) || 10, + this.sheetsConfig.maxResults + ) + + // Semantic similarity search + let results: any[] + + if (query.to) { + // Similar to entity + results = await this.context.brain.similar({ + to: query.to, + limit, + threshold: query.threshold ? parseFloat(query.threshold) : undefined + }) + } else { + // Similar to query text + results = await this.context.brain.find({ + query: query.q, + limit + }) + } + + const entities = results.map((r: any) => r.entity || r) + return this.entitiesToSheetsResponse(entities) + } + + private async handleRelations( + request: SheetsRequest + ): Promise { + const { query } = request + + const params: any = { + limit: Math.min( + parseInt(query.limit) || 100, + this.sheetsConfig.maxResults + ) + } + + if (query.from) params.from = query.from + if (query.to) params.to = query.to + if (query.type) params.type = query.type as VerbType + + const relations = await this.queryRelations(params) + + // Convert to sheets format + const headers = [ + 'Id', + 'FromId', + 'ToId', + 'Type', + 'Weight', + 'Confidence', + 'CreatedAt' + ] + const rows = relations.map((r) => [ + r.id, + r.from, + r.to, + r.type, + r.weight ?? 1, + r.confidence ?? 1, + new Date(r.createdAt).toISOString() + ]) + + return this.jsonResponse({ + headers, + rows, + count: rows.length, + hasMore: false + }) + } + + private handleSchema(): SheetsResponse { + return this.jsonResponse({ + nounTypes: Object.values(NounType), + verbTypes: Object.values(VerbType), + entityFields: ['id', 'type', 'data', 'metadata', 'confidence', 'weight', 'service', 'createdAt', 'updatedAt'], + commonMetadataFields: ['name', 'title', 'description', 'email', 'url', 'tags', 'category', 'status', 'priority'] + }) + } + + private handleStream(_request: SheetsRequest): SheetsResponse { + // This returns headers for SSE - actual streaming is handled by the HTTP server + return { + status: 200, + headers: { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + 'Access-Control-Allow-Origin': '*' + }, + body: 'sse' // Signal to HTTP server to handle as SSE + } + } + + private handleHealth(): SheetsResponse { + return this.jsonResponse({ + status: 'ok', + integration: 'sheets', + uptime: this.startedAt ? Date.now() - this.startedAt : 0, + requests: this.requestCount + }) + } + + private async handleAdd(request: SheetsRequest): Promise { + if (!this.context) { + return this.errorResponse(500, 'Not initialized') + } + + const { body } = request + + if (!body || !body.type) { + return this.errorResponse(400, 'Missing required field: type') + } + + const entity = await this.context.brain.add({ + type: body.type as NounType, + data: body.data, + metadata: body.metadata, + confidence: body.confidence, + weight: body.weight, + service: body.service + }) + + return this.entitiesToSheetsResponse([entity]) + } + + private async handleUpdate(request: SheetsRequest): Promise { + if (!this.context) { + return this.errorResponse(500, 'Not initialized') + } + + const { body } = request + + if (!body || !body.id) { + return this.errorResponse(400, 'Missing required field: id') + } + + await this.context.brain.update({ + id: body.id, + data: body.data, + metadata: body.metadata, + type: body.type as NounType, + confidence: body.confidence, + weight: body.weight, + merge: body.merge ?? true + }) + + const updated = await this.getEntity(body.id) + return this.entitiesToSheetsResponse(updated ? [updated] : []) + } + + private async handleDelete(request: SheetsRequest): Promise { + if (!this.context) { + return this.errorResponse(500, 'Not initialized') + } + + const { body } = request + + if (!body || !body.id) { + return this.errorResponse(400, 'Missing required field: id') + } + + await this.context.brain.delete(body.id) + + return this.jsonResponse({ success: true, deleted: body.id }) + } + + private async handleBatch(request: SheetsRequest): Promise { + if (!this.context) { + return this.errorResponse(500, 'Not initialized') + } + + const { body } = request + + if (!body || !Array.isArray(body.operations)) { + return this.errorResponse(400, 'Missing operations array') + } + + const results: any[] = [] + + for (const op of body.operations) { + try { + switch (op.action) { + case 'add': + const added = await this.context.brain.add({ + type: op.type as NounType, + data: op.data, + metadata: op.metadata + }) + results.push({ success: true, id: added.id, action: 'add' }) + break + + case 'update': + await this.context.brain.update({ + id: op.id, + data: op.data, + metadata: op.metadata, + merge: true + }) + results.push({ success: true, id: op.id, action: 'update' }) + break + + case 'delete': + await this.context.brain.delete(op.id) + results.push({ success: true, id: op.id, action: 'delete' }) + break + + default: + results.push({ + success: false, + error: `Unknown action: ${op.action}` + }) + } + } catch (error: any) { + results.push({ + success: false, + id: op.id, + action: op.action, + error: error.message + }) + } + } + + return this.jsonResponse({ + results, + total: body.operations.length, + successful: results.filter((r) => r.success).length, + failed: results.filter((r) => !r.success).length + }) + } + + private async handleRelate(request: SheetsRequest): Promise { + if (!this.context) { + return this.errorResponse(500, 'Not initialized') + } + + const { body } = request + + if (!body || !body.from || !body.to || !body.type) { + return this.errorResponse(400, 'Missing required fields: from, to, type') + } + + const relation = await this.context.brain.relate({ + from: body.from, + to: body.to, + type: body.type as VerbType, + weight: body.weight, + metadata: body.metadata + }) + + return this.jsonResponse({ + success: true, + relation: { + id: relation.id, + from: relation.from, + to: relation.to, + type: relation.type + } + }) + } + + // Helpers + + private entitiesToSheetsResponse(entities: Entity[]): SheetsResponse { + if (entities.length === 0) { + return this.jsonResponse({ + headers: ['Id', 'Type', 'CreatedAt'], + rows: [], + count: 0, + hasMore: false + }) + } + + // Collect all unique metadata keys + const metadataKeys = new Set() + for (const entity of entities) { + if (entity.metadata) { + for (const key of Object.keys(entity.metadata)) { + metadataKeys.add(key) + } + } + } + + // Build headers + const baseHeaders = ['Id', 'Type', 'Confidence', 'Weight', 'CreatedAt'] + const metaHeaders = Array.from(metadataKeys).sort() + const headers = [...baseHeaders, ...metaHeaders, 'Data'] + + // Build rows + const rows = entities.map((entity) => { + const baseValues = [ + entity.id, + entity.type, + entity.confidence ?? '', + entity.weight ?? '', + new Date(entity.createdAt).toISOString() + ] + + const metaValues = metaHeaders.map((key) => { + const val = entity.metadata?.[key] + if (val === undefined || val === null) return '' + if (typeof val === 'object') return JSON.stringify(val) + return val + }) + + const dataValue = entity.data ? JSON.stringify(entity.data) : '' + + return [...baseValues, ...metaValues, dataValue] + }) + + return this.jsonResponse({ + headers, + rows, + count: rows.length, + hasMore: false + }) + } + + private broadcastSSE(event: string, data: any): void { + for (const [_, sendFn] of this.sseClients) { + try { + sendFn(event, data) + } catch { + // Client disconnected + } + } + } + + private jsonResponse(data: any): SheetsResponse { + return { + status: 200, + headers: { + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'GET, POST, OPTIONS', + 'Access-Control-Allow-Headers': 'Content-Type, Authorization' + }, + body: JSON.stringify(data) + } + } + + private errorResponse(status: number, message: string): SheetsResponse { + return { + status, + headers: { + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*' + }, + body: JSON.stringify({ error: message }) + } + } + + private corsResponse(): SheetsResponse { + return { + status: 204, + headers: { + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'GET, POST, OPTIONS', + 'Access-Control-Allow-Headers': 'Content-Type, Authorization', + 'Access-Control-Max-Age': '86400' + }, + body: null + } + } +} + +/** + * Package export for @soulcraft/brainy-sheets + */ +export const integration = { + name: 'sheets', + version: '1.0.0', + description: 'Google Sheets two-way sync with real-time updates', + environments: ['node', 'browser', 'deno', 'cloudflare', 'bun'], + create: (brain: any, config?: GoogleSheetsConfig) => + new GoogleSheetsIntegration(config), + defaultConfig: { + basePath: '/sheets', + maxResults: 1000, + defaultLimit: 100, + allowWrite: true + } +} diff --git a/src/integrations/sheets/index.ts b/src/integrations/sheets/index.ts new file mode 100644 index 00000000..52ba1b81 --- /dev/null +++ b/src/integrations/sheets/index.ts @@ -0,0 +1,12 @@ +/** + * Google Sheets Integration Module + * + * Provides two-way sync between Brainy and Google Sheets. + * Works in all environments (Node.js, browser, Deno, Cloudflare, Bun). + */ + +export { + GoogleSheetsIntegration, + integration, + type GoogleSheetsConfig +} from './GoogleSheetsIntegration.js' diff --git a/src/integrations/sse/SSEIntegration.ts b/src/integrations/sse/SSEIntegration.ts new file mode 100644 index 00000000..8e09ec02 --- /dev/null +++ b/src/integrations/sse/SSEIntegration.ts @@ -0,0 +1,449 @@ +/** + * SSE (Server-Sent Events) Integration + * + * Provides real-time streaming of Brainy events to clients. + * Universal - works in all environments. + */ + +import { + IntegrationBase, + HTTPIntegration, + StreamingIntegration +} from '../core/IntegrationBase.js' +import { + IntegrationConfig, + EventFilter, + BrainyEvent +} from '../core/types.js' +import { AugmentationManifest } from '../../augmentations/manifest.js' + +/** + * SSE integration configuration + */ +export interface SSEConfig extends IntegrationConfig { + /** Base path for SSE endpoint (default: '/events') */ + basePath?: string + + /** Heartbeat interval in ms (default: 30000) */ + heartbeatInterval?: number + + /** Maximum clients (default: 1000) */ + maxClients?: number + + /** Include full entity data in events (default: false) */ + includeData?: boolean +} + +/** + * SSE client connection + */ +interface SSEClient { + id: string + filter: EventFilter + send: (event: string, data: any, id?: string) => void + close: () => void + lastEventId?: string + connectedAt: number +} + +/** + * SSE Integration + * + * Enables real-time streaming of Brainy changes via Server-Sent Events. + * Clients can subscribe to specific event types and receive updates instantly. + * + * Endpoint: + * - GET /events - SSE stream + * + * Query Parameters: + * - types: Entity types to subscribe to (noun,verb,vfs) + * - operations: Operations to subscribe to (create,update,delete) + * - nounTypes: Specific noun types (person,document,...) + * - since: Resume from sequence ID (Last-Event-ID) + * + * @example + * ```typescript + * brain.augmentations.register(new SSEIntegration({ + * basePath: '/events', + * heartbeatInterval: 30000 + * })) + * + * // Client-side: + * const source = new EventSource('/events?types=noun&operations=create,update') + * source.onmessage = (event) => { + * const data = JSON.parse(event.data) + * console.log('Change:', data) + * } + * ``` + */ +export class SSEIntegration + extends IntegrationBase + implements HTTPIntegration, StreamingIntegration +{ + readonly name = 'sse' + + port: number + basePath: string + + private sseConfig: SSEConfig & { + enabled: boolean + basePath: string + heartbeatInterval: number + maxClients: number + includeData: boolean + } + private clients: Map = new Map() + private heartbeatTimer?: ReturnType + private clientIdCounter = 0 + + constructor(config?: SSEConfig) { + super(config) + + this.sseConfig = { + enabled: config?.enabled ?? true, + basePath: config?.basePath ?? '/events', + heartbeatInterval: config?.heartbeatInterval ?? 30000, + maxClients: config?.maxClients ?? 1000, + includeData: config?.includeData ?? false, + rateLimit: config?.rateLimit, + auth: config?.auth, + cors: config?.cors + } + + this.port = 0 + this.basePath = this.sseConfig.basePath + } + + protected async onStart(): Promise { + // Subscribe to all Brainy events + this.subscribeToChanges({}, (event) => { + this.broadcastEvent(event) + }) + + // Start heartbeat + this.heartbeatTimer = setInterval(() => { + this.sendHeartbeat() + }, this.sseConfig.heartbeatInterval) + + this.log('SSE integration started') + } + + protected async onStop(): Promise { + // Clear heartbeat + if (this.heartbeatTimer) { + clearInterval(this.heartbeatTimer) + } + + // Close all clients + for (const [_, client] of this.clients) { + client.close() + } + this.clients.clear() + + this.log('SSE integration stopped') + } + + /** + * Handle incoming HTTP request for SSE + */ + handleRequest(request: { + method: string + path: string + query: Record + headers: Record + }): { + status: number + headers: Record + body: string | null + isSSE: boolean + } { + const { method, path, query, headers } = request + + if (method !== 'GET') { + return { + status: 405, + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ error: 'Method not allowed' }), + isSSE: false + } + } + + const relativePath = path.startsWith(this.basePath) + ? path.slice(this.basePath.length) + : path + + if (relativePath !== '' && relativePath !== '/') { + return { + status: 404, + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ error: 'Not found' }), + isSSE: false + } + } + + // Parse filter from query params + const filter = this.parseFilter(query, headers) + + // Return SSE headers - actual stream handling is done by the server + return { + status: 200, + headers: { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + 'Access-Control-Allow-Origin': '*', + 'X-Accel-Buffering': 'no' + }, + body: null, + isSSE: true + } + } + + /** + * Register an SSE client (called by HTTP server) + */ + registerClient( + sendFn: (event: string, data: any, id?: string) => void, + closeFn: () => void, + query: Record, + headers: Record + ): string { + if (this.clients.size >= this.sseConfig.maxClients) { + throw new Error('Maximum clients reached') + } + + const clientId = `sse-${++this.clientIdCounter}-${Date.now()}` + const filter = this.parseFilter(query, headers) + + const client: SSEClient = { + id: clientId, + filter, + send: sendFn, + close: closeFn, + lastEventId: headers['last-event-id'], + connectedAt: Date.now() + } + + this.clients.set(clientId, client) + this.recordRequest() + + // Send initial connection event + sendFn('connected', { clientId, filter }, clientId) + + // Replay missed events if resuming + if (filter.since !== undefined) { + const missed = this.eventBus.getEventsSince(filter.since) + for (const event of missed) { + if (this.matchesFilter(event, filter)) { + sendFn('change', this.formatEvent(event), String(event.sequenceId)) + } + } + } + + this.log(`SSE client connected: ${clientId}`) + + return clientId + } + + /** + * Unregister an SSE client + */ + unregisterClient(clientId: string): void { + this.clients.delete(clientId) + this.log(`SSE client disconnected: ${clientId}`) + } + + /** + * Get connected client count + */ + getClientCount(): number { + return this.clients.size + } + + /** + * Stream implementation for StreamingIntegration interface + */ + stream( + filter: EventFilter, + callback: (event: any) => void + ): { close: () => void } { + const subscription = this.eventBus.subscribe(filter, (event) => { + callback(this.formatEvent(event)) + }) + + return { + close: () => subscription.unsubscribe() + } + } + + /** + * Get routes + */ + getRoutes(): Array<{ method: string; path: string; description: string }> { + return [ + { + method: 'GET', + path: this.basePath, + description: 'Server-Sent Events stream' + } + ] + } + + /** + * Get manifest + */ + getManifest(): AugmentationManifest { + return { + id: 'sse', + name: 'SSE Streaming', + version: '1.0.0', + description: 'Real-time event streaming via Server-Sent Events', + longDescription: + 'Enables real-time streaming of Brainy changes to web clients, Google Sheets, and dashboards via standard Server-Sent Events.', + category: 'integration', + status: 'stable', + configSchema: { + type: 'object', + properties: { + enabled: { type: 'boolean', default: true }, + basePath: { type: 'string', default: '/events' }, + heartbeatInterval: { type: 'number', default: 30000 }, + maxClients: { type: 'number', default: 1000 }, + includeData: { type: 'boolean', default: false } + } + }, + configDefaults: { + enabled: true, + basePath: '/events', + heartbeatInterval: 30000, + maxClients: 1000, + includeData: false + }, + features: [ + 'Real-time streaming', + 'Event filtering', + 'Automatic reconnection support', + 'Event replay on reconnect', + 'Heartbeat keep-alive' + ], + keywords: ['sse', 'streaming', 'real-time', 'events'] + } + } + + // Private methods + + private parseFilter( + query: Record, + headers: Record + ): EventFilter { + const filter: EventFilter = {} + + if (query.types) { + filter.entityTypes = query.types.split(',') as any[] + } + + if (query.operations) { + filter.operations = query.operations.split(',') as any[] + } + + if (query.nounTypes) { + filter.nounTypes = query.nounTypes.split(',') as any[] + } + + if (query.verbTypes) { + filter.verbTypes = query.verbTypes.split(',') as any[] + } + + if (query.service) { + filter.service = query.service + } + + // Resume from Last-Event-ID header or query param + const since = headers['last-event-id'] || query.since + if (since) { + filter.since = BigInt(since) + } + + return filter + } + + private matchesFilter(event: BrainyEvent, filter: EventFilter): boolean { + if (filter.entityTypes?.length && !filter.entityTypes.includes(event.entityType)) { + return false + } + if (filter.operations?.length && !filter.operations.includes(event.operation)) { + return false + } + if (filter.nounTypes?.length && event.nounType && !filter.nounTypes.includes(event.nounType)) { + return false + } + if (filter.verbTypes?.length && event.verbType && !filter.verbTypes.includes(event.verbType)) { + return false + } + if (filter.service && event.service !== filter.service) { + return false + } + return true + } + + private formatEvent(event: BrainyEvent): any { + const formatted: any = { + id: event.id, + type: event.entityType, + operation: event.operation, + entityId: event.entityId, + timestamp: event.timestamp + } + + if (event.nounType) formatted.nounType = event.nounType + if (event.verbType) formatted.verbType = event.verbType + if (event.service) formatted.service = event.service + + if (this.sseConfig.includeData && event.data) { + formatted.data = event.data + } + + return formatted + } + + private broadcastEvent(event: BrainyEvent): void { + const eventId = String(event.sequenceId) + + for (const [_, client] of this.clients) { + if (this.matchesFilter(event, client.filter)) { + try { + client.send('change', this.formatEvent(event), eventId) + } catch { + // Client disconnected, will be cleaned up + } + } + } + } + + private sendHeartbeat(): void { + const timestamp = Date.now() + for (const [_, client] of this.clients) { + try { + client.send('heartbeat', { timestamp }) + } catch { + // Client disconnected + } + } + } +} + +/** + * Package export for @soulcraft/brainy-sse + */ +export const integration = { + name: 'sse', + version: '1.0.0', + description: 'Real-time event streaming via Server-Sent Events', + environments: ['node', 'browser', 'deno', 'cloudflare', 'bun'], + create: (brain: any, config?: SSEConfig) => new SSEIntegration(config), + defaultConfig: { + basePath: '/events', + heartbeatInterval: 30000, + maxClients: 1000 + } +} diff --git a/src/integrations/sse/index.ts b/src/integrations/sse/index.ts new file mode 100644 index 00000000..ee4547b1 --- /dev/null +++ b/src/integrations/sse/index.ts @@ -0,0 +1,12 @@ +/** + * SSE (Server-Sent Events) Integration Module + * + * Provides real-time streaming of Brainy events. + * Works in all environments. + */ + +export { + SSEIntegration, + integration, + type SSEConfig +} from './SSEIntegration.js' diff --git a/src/integrations/webhooks/WebhookIntegration.ts b/src/integrations/webhooks/WebhookIntegration.ts new file mode 100644 index 00000000..c9caf238 --- /dev/null +++ b/src/integrations/webhooks/WebhookIntegration.ts @@ -0,0 +1,548 @@ +/** + * Webhook Integration + * + * Delivers Brainy events to external URLs via HTTP POST. + * Supports HMAC signing, retry policies, and batching. + * + * Zero external dependencies - works in all environments. + */ + +import { IntegrationBase } from '../core/IntegrationBase.js' +import { + IntegrationConfig, + EventFilter, + WebhookRegistration, + WebhookDeliveryResult, + BrainyEvent +} from '../core/types.js' +import { AugmentationManifest } from '../../augmentations/manifest.js' + +/** + * Webhook integration configuration + */ +export interface WebhookConfig extends IntegrationConfig { + /** Default retry policy */ + defaultRetryPolicy?: { + maxRetries: number + backoffMultiplier: number + initialDelayMs: number + maxDelayMs: number + } + + /** Batch delivery settings */ + batch?: { + enabled: boolean + maxSize: number + maxDelayMs: number + } + + /** Request timeout in ms (default: 30000) */ + timeout?: number + + /** Include full entity data (default: false) */ + includeData?: boolean +} + +/** + * Pending delivery + */ +interface PendingDelivery { + webhook: WebhookRegistration + events: BrainyEvent[] + attempts: number + nextAttempt: number +} + +/** + * Webhook Integration + * + * Enables push notifications to external systems when Brainy data changes. + * Supports HMAC-SHA256 signing for security, automatic retry with exponential + * backoff, and optional event batching. + * + * Methods: + * - register(webhook) - Register a new webhook + * - unregister(id) - Remove a webhook + * - list() - List all webhooks + * - getDeliveryHistory(webhookId) - Get delivery history + * + * @example + * ```typescript + * const webhooks = new WebhookIntegration() + * brain.augmentations.register(webhooks) + * + * // Register a webhook + * await webhooks.register({ + * url: 'https://example.com/webhook', + * events: { entityTypes: ['noun'], operations: ['create', 'update'] }, + * secret: 'my-signing-secret' + * }) + * ``` + */ +export class WebhookIntegration extends IntegrationBase { + readonly name = 'webhooks' + + private webhookConfig: WebhookConfig & { + enabled: boolean + defaultRetryPolicy: { + maxRetries: number + backoffMultiplier: number + initialDelayMs: number + maxDelayMs: number + } + batch: { + enabled: boolean + maxSize: number + maxDelayMs: number + } + timeout: number + includeData: boolean + } + private webhooks: Map = new Map() + private pendingDeliveries: PendingDelivery[] = [] + private deliveryHistory: WebhookDeliveryResult[] = [] + private deliveryTimer?: ReturnType + private batchBuffer: Map = new Map() + private batchTimer?: ReturnType + private webhookIdCounter = 0 + + constructor(config?: WebhookConfig) { + super(config) + + this.webhookConfig = { + enabled: config?.enabled ?? true, + defaultRetryPolicy: config?.defaultRetryPolicy ?? { + maxRetries: 5, + backoffMultiplier: 2, + initialDelayMs: 1000, + maxDelayMs: 60000 + }, + batch: config?.batch ?? { + enabled: false, + maxSize: 100, + maxDelayMs: 5000 + }, + timeout: config?.timeout ?? 30000, + includeData: config?.includeData ?? false, + rateLimit: config?.rateLimit, + auth: config?.auth, + cors: config?.cors + } + } + + protected async onStart(): Promise { + // Subscribe to all events + this.subscribeToChanges({}, (event) => { + this.handleEvent(event) + }) + + // Start delivery processor + this.deliveryTimer = setInterval(() => { + this.processDeliveries() + }, 1000) + + this.log('Webhook integration started') + } + + protected async onStop(): Promise { + if (this.deliveryTimer) { + clearInterval(this.deliveryTimer) + } + if (this.batchTimer) { + clearTimeout(this.batchTimer) + } + + // Attempt to deliver remaining events + await this.flushBatches() + await this.processDeliveries() + + this.log('Webhook integration stopped') + } + + /** + * Register a new webhook + */ + async register( + webhook: Omit + ): Promise { + const id = `wh-${++this.webhookIdCounter}-${Date.now()}` + + const registration: WebhookRegistration = { + id, + url: webhook.url, + events: webhook.events, + secret: webhook.secret, + active: true, + retryPolicy: webhook.retryPolicy ?? this.webhookConfig.defaultRetryPolicy, + createdAt: Date.now() + } + + this.webhooks.set(id, registration) + this.log(`Registered webhook: ${id} -> ${webhook.url}`) + + return registration + } + + /** + * Unregister a webhook + */ + async unregister(id: string): Promise { + const existed = this.webhooks.delete(id) + if (existed) { + this.log(`Unregistered webhook: ${id}`) + } + return existed + } + + /** + * List all webhooks + */ + list(): WebhookRegistration[] { + return Array.from(this.webhooks.values()) + } + + /** + * Get a specific webhook + */ + get(id: string): WebhookRegistration | undefined { + return this.webhooks.get(id) + } + + /** + * Update a webhook + */ + async update( + id: string, + updates: Partial> + ): Promise { + const webhook = this.webhooks.get(id) + if (!webhook) return null + + const updated = { ...webhook, ...updates } + this.webhooks.set(id, updated) + return updated + } + + /** + * Get delivery history for a webhook + */ + getDeliveryHistory( + webhookId?: string, + limit = 100 + ): WebhookDeliveryResult[] { + let history = this.deliveryHistory + + if (webhookId) { + history = history.filter((d) => d.webhookId === webhookId) + } + + return history.slice(-limit) + } + + /** + * Manually trigger a test delivery + */ + async testDelivery(webhookId: string): Promise { + const webhook = this.webhooks.get(webhookId) + if (!webhook) { + throw new Error(`Webhook not found: ${webhookId}`) + } + + const testEvent: BrainyEvent = { + id: 'test-event', + entityType: 'noun', + operation: 'create', + entityId: 'test-entity', + timestamp: Date.now(), + sequenceId: 0n + } + + return this.deliverEvent(webhook, [testEvent]) + } + + /** + * Get manifest + */ + getManifest(): AugmentationManifest { + return { + id: 'webhooks', + name: 'Webhooks', + version: '1.0.0', + description: 'Push events to external URLs', + longDescription: + 'Delivers Brainy events to external webhooks via HTTP POST. Supports HMAC-SHA256 signing, exponential backoff retry, and event batching.', + category: 'integration', + status: 'stable', + configSchema: { + type: 'object', + properties: { + enabled: { type: 'boolean', default: true }, + timeout: { type: 'number', default: 30000 }, + includeData: { type: 'boolean', default: false } + } + }, + configDefaults: { + enabled: true, + timeout: 30000, + includeData: false + }, + features: [ + 'HMAC-SHA256 signing', + 'Exponential backoff retry', + 'Event batching', + 'Delivery history tracking' + ], + keywords: ['webhooks', 'push', 'notifications', 'events'] + } + } + + // Private methods + + private handleEvent(event: BrainyEvent): void { + for (const [_, webhook] of this.webhooks) { + if (!webhook.active) continue + if (!this.matchesFilter(event, webhook.events)) continue + + if (this.webhookConfig.batch.enabled) { + this.addToBatch(webhook.id, event) + } else { + this.queueDelivery(webhook, [event]) + } + } + } + + private matchesFilter(event: BrainyEvent, filter: EventFilter): boolean { + if ( + filter.entityTypes?.length && + !filter.entityTypes.includes(event.entityType) + ) { + return false + } + if ( + filter.operations?.length && + !filter.operations.includes(event.operation) + ) { + return false + } + if ( + filter.nounTypes?.length && + event.nounType && + !filter.nounTypes.includes(event.nounType) + ) { + return false + } + return true + } + + private addToBatch(webhookId: string, event: BrainyEvent): void { + let batch = this.batchBuffer.get(webhookId) + if (!batch) { + batch = [] + this.batchBuffer.set(webhookId, batch) + } + + batch.push(event) + + // Flush if batch is full + if (batch.length >= this.webhookConfig.batch.maxSize) { + this.flushBatch(webhookId) + } + + // Set timer for delayed flush + if (!this.batchTimer) { + this.batchTimer = setTimeout(() => { + this.flushBatches() + this.batchTimer = undefined + }, this.webhookConfig.batch.maxDelayMs) + } + } + + private flushBatch(webhookId: string): void { + const batch = this.batchBuffer.get(webhookId) + if (!batch || batch.length === 0) return + + const webhook = this.webhooks.get(webhookId) + if (webhook) { + this.queueDelivery(webhook, [...batch]) + } + + this.batchBuffer.delete(webhookId) + } + + private async flushBatches(): Promise { + for (const webhookId of this.batchBuffer.keys()) { + this.flushBatch(webhookId) + } + } + + private queueDelivery( + webhook: WebhookRegistration, + events: BrainyEvent[] + ): void { + this.pendingDeliveries.push({ + webhook, + events, + attempts: 0, + nextAttempt: Date.now() + }) + } + + private async processDeliveries(): Promise { + const now = Date.now() + const ready = this.pendingDeliveries.filter((d) => d.nextAttempt <= now) + + for (const delivery of ready) { + const result = await this.deliverEvent(delivery.webhook, delivery.events) + + // Remove from pending + const index = this.pendingDeliveries.indexOf(delivery) + if (index >= 0) { + this.pendingDeliveries.splice(index, 1) + } + + // Track history + this.deliveryHistory.push(result) + if (this.deliveryHistory.length > 1000) { + this.deliveryHistory.shift() + } + + // Retry if failed + if (!result.success && delivery.attempts < (delivery.webhook.retryPolicy?.maxRetries ?? 5)) { + const policy = delivery.webhook.retryPolicy ?? this.webhookConfig.defaultRetryPolicy + const delay = Math.min( + policy.initialDelayMs * Math.pow(policy.backoffMultiplier, delivery.attempts), + policy.maxDelayMs + ) + + this.pendingDeliveries.push({ + ...delivery, + attempts: delivery.attempts + 1, + nextAttempt: now + delay + }) + } else if (!result.success) { + // Max retries reached - update webhook failure count + const webhook = this.webhooks.get(delivery.webhook.id) + if (webhook) { + webhook.failureCount = (webhook.failureCount ?? 0) + 1 + } + } + } + } + + private async deliverEvent( + webhook: WebhookRegistration, + events: BrainyEvent[] + ): Promise { + const payload = { + events: events.map((e) => ({ + id: e.id, + type: e.entityType, + operation: e.operation, + entityId: e.entityId, + timestamp: e.timestamp, + nounType: e.nounType, + verbType: e.verbType, + service: e.service, + data: this.webhookConfig.includeData ? e.data : undefined + })), + deliveredAt: Date.now() + } + + const body = JSON.stringify(payload) + const headers: Record = { + 'Content-Type': 'application/json', + 'User-Agent': 'Brainy-Webhook/1.0', + 'X-Brainy-Delivery': events[0]?.id ?? 'batch' + } + + // Sign if secret is configured + if (webhook.secret) { + const signature = await this.sign(body, webhook.secret) + headers['X-Brainy-Signature'] = `sha256=${signature}` + } + + try { + const controller = new AbortController() + const timeoutId = setTimeout( + () => controller.abort(), + this.webhookConfig.timeout + ) + + const response = await fetch(webhook.url, { + method: 'POST', + headers, + body, + signal: controller.signal + }) + + clearTimeout(timeoutId) + + const success = response.ok + + if (success) { + // Update last delivery time + const wh = this.webhooks.get(webhook.id) + if (wh) { + wh.lastDeliveryAt = Date.now() + wh.failureCount = 0 + } + } + + return { + webhookId: webhook.id, + eventId: events[0]?.id ?? 'batch', + success, + statusCode: response.status, + attempts: 1, + timestamp: Date.now() + } + } catch (error: any) { + return { + webhookId: webhook.id, + eventId: events[0]?.id ?? 'batch', + success: false, + error: error.message, + attempts: 1, + timestamp: Date.now() + } + } + } + + private async sign(payload: string, secret: string): Promise { + // Use Web Crypto API (works in all environments) + const encoder = new TextEncoder() + const keyData = encoder.encode(secret) + const messageData = encoder.encode(payload) + + const key = await crypto.subtle.importKey( + 'raw', + keyData, + { name: 'HMAC', hash: 'SHA-256' }, + false, + ['sign'] + ) + + const signature = await crypto.subtle.sign('HMAC', key, messageData) + const signatureArray = new Uint8Array(signature) + + return Array.from(signatureArray) + .map((b) => b.toString(16).padStart(2, '0')) + .join('') + } +} + +/** + * Package export for @soulcraft/brainy-webhooks + */ +export const integration = { + name: 'webhooks', + version: '1.0.0', + description: 'Push events to external URLs via webhooks', + environments: ['node', 'browser', 'deno', 'cloudflare', 'bun'], + create: (brain: any, config?: WebhookConfig) => + new WebhookIntegration(config), + defaultConfig: { + timeout: 30000, + includeData: false + } +} diff --git a/src/integrations/webhooks/index.ts b/src/integrations/webhooks/index.ts new file mode 100644 index 00000000..c6307eb1 --- /dev/null +++ b/src/integrations/webhooks/index.ts @@ -0,0 +1,12 @@ +/** + * Webhook Integration Module + * + * Push events to external URLs via HTTP POST. + * Works in all environments. + */ + +export { + WebhookIntegration, + integration, + type WebhookConfig +} from './WebhookIntegration.js' diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index 3d9a3569..e4a65bc1 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -622,6 +622,42 @@ export type AggregateMetric = // ============= Configuration ============= +/** + * Integration Hub configuration (v7.4.0) + * + * Enables external tool integrations: Excel, Power BI, Google Sheets, etc. + * Works in all environments with zero external dependencies. + * + * @example + * ```typescript + * // Enable all integrations with defaults + * new Brainy({ integrations: true }) + * + * // Custom configuration + * new Brainy({ + * integrations: { + * basePath: '/api/v1', + * enable: ['odata', 'sheets'] + * } + * }) + * ``` + */ +export interface IntegrationsConfig { + /** Base path for all integration endpoints (default: '') */ + basePath?: string + + /** Which integrations to enable (default: all) */ + enable?: ('odata' | 'sheets' | 'sse' | 'webhooks')[] | 'all' + + /** Per-integration config overrides */ + config?: { + odata?: { basePath?: string } + sheets?: { basePath?: string } + sse?: { basePath?: string; heartbeatInterval?: number } + webhooks?: { maxRetries?: number } + } +} + /** * Brainy configuration */ @@ -696,6 +732,13 @@ export interface BrainyConfig { // Logging configuration verbose?: boolean // Enable verbose logging silent?: boolean // Suppress all logging output + + // Integration Hub (v7.4.0) + // Enable external tool integrations: Excel, Power BI, Google Sheets, etc. + // - true: Enable all integrations with default paths + // - false/undefined: Disable integrations (default) + // - IntegrationsConfig: Custom configuration + integrations?: boolean | IntegrationsConfig } // ============= Neural API Types ============= diff --git a/tests/integrations/core.test.ts b/tests/integrations/core.test.ts new file mode 100644 index 00000000..1334e627 --- /dev/null +++ b/tests/integrations/core.test.ts @@ -0,0 +1,427 @@ +/** + * Integration Hub - Core Tests + * + * Tests for EventBus, TabularExporter, and core functionality. + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest' +import { EventBus } from '../../src/integrations/core/EventBus.js' +import { TabularExporter } from '../../src/integrations/core/TabularExporter.js' +import { NounType, VerbType } from '../../src/types/graphTypes.js' +import type { Entity, Relation } from '../../src/types/brainy.types.js' + +describe('EventBus', () => { + let eventBus: EventBus + + beforeEach(() => { + eventBus = new EventBus() + }) + + describe('subscribe and emit', () => { + it('should emit events to subscribers', async () => { + const handler = vi.fn() + + eventBus.subscribe({}, handler) + + eventBus.emit({ + entityType: 'noun', + operation: 'create', + entityId: 'test-123' + }) + + expect(handler).toHaveBeenCalledTimes(1) + expect(handler).toHaveBeenCalledWith( + expect.objectContaining({ + entityType: 'noun', + operation: 'create', + entityId: 'test-123' + }) + ) + }) + + it('should filter by entity type', () => { + const handler = vi.fn() + + eventBus.subscribe({ entityTypes: ['verb'] }, handler) + + eventBus.emit({ + entityType: 'noun', + operation: 'create', + entityId: 'test-123' + }) + + expect(handler).not.toHaveBeenCalled() + + eventBus.emit({ + entityType: 'verb', + operation: 'create', + entityId: 'test-456' + }) + + expect(handler).toHaveBeenCalledTimes(1) + }) + + it('should filter by operation', () => { + const handler = vi.fn() + + eventBus.subscribe({ operations: ['update', 'delete'] }, handler) + + eventBus.emit({ + entityType: 'noun', + operation: 'create', + entityId: 'test-123' + }) + + expect(handler).not.toHaveBeenCalled() + + eventBus.emit({ + entityType: 'noun', + operation: 'update', + entityId: 'test-456' + }) + + expect(handler).toHaveBeenCalledTimes(1) + }) + + it('should filter by noun type', () => { + const handler = vi.fn() + + eventBus.subscribe({ nounTypes: [NounType.Person] }, handler) + + eventBus.emit({ + entityType: 'noun', + operation: 'create', + entityId: 'test-123', + nounType: NounType.Document + }) + + expect(handler).not.toHaveBeenCalled() + + eventBus.emit({ + entityType: 'noun', + operation: 'create', + entityId: 'test-456', + nounType: NounType.Person + }) + + expect(handler).toHaveBeenCalledTimes(1) + }) + }) + + describe('unsubscribe', () => { + it('should stop receiving events after unsubscribe', () => { + const handler = vi.fn() + + const subscription = eventBus.subscribe({}, handler) + + eventBus.emit({ + entityType: 'noun', + operation: 'create', + entityId: 'test-123' + }) + + expect(handler).toHaveBeenCalledTimes(1) + + subscription.unsubscribe() + + eventBus.emit({ + entityType: 'noun', + operation: 'create', + entityId: 'test-456' + }) + + expect(handler).toHaveBeenCalledTimes(1) // Still 1 + }) + }) + + describe('sequence IDs', () => { + it('should increment sequence IDs', () => { + const events: any[] = [] + eventBus.subscribe({}, (e) => events.push(e)) + + eventBus.emit({ entityType: 'noun', operation: 'create', entityId: '1' }) + eventBus.emit({ entityType: 'noun', operation: 'create', entityId: '2' }) + eventBus.emit({ entityType: 'noun', operation: 'create', entityId: '3' }) + + expect(events[0].sequenceId).toBe(1n) + expect(events[1].sequenceId).toBe(2n) + expect(events[2].sequenceId).toBe(3n) + }) + + it('should return current sequence ID', () => { + expect(eventBus.getCurrentSequenceId()).toBe(0n) + + eventBus.emit({ entityType: 'noun', operation: 'create', entityId: '1' }) + expect(eventBus.getCurrentSequenceId()).toBe(1n) + + eventBus.emit({ entityType: 'noun', operation: 'create', entityId: '2' }) + expect(eventBus.getCurrentSequenceId()).toBe(2n) + }) + }) + + describe('event buffer', () => { + it('should buffer events for replay', () => { + eventBus.emit({ entityType: 'noun', operation: 'create', entityId: '1' }) + eventBus.emit({ entityType: 'noun', operation: 'create', entityId: '2' }) + eventBus.emit({ entityType: 'noun', operation: 'create', entityId: '3' }) + + const events = eventBus.getEventsSince(1n) + + expect(events).toHaveLength(2) + expect(events[0].entityId).toBe('2') + expect(events[1].entityId).toBe('3') + }) + }) + + describe('helper methods', () => { + it('should emit noun events', () => { + const handler = vi.fn() + eventBus.subscribe({}, handler) + + eventBus.emitNoun('create', 'entity-1', NounType.Person) + + expect(handler).toHaveBeenCalledWith( + expect.objectContaining({ + entityType: 'noun', + operation: 'create', + entityId: 'entity-1', + nounType: NounType.Person + }) + ) + }) + + it('should emit verb events', () => { + const handler = vi.fn() + eventBus.subscribe({}, handler) + + eventBus.emitVerb('create', 'rel-1', VerbType.RelatedTo) + + expect(handler).toHaveBeenCalledWith( + expect.objectContaining({ + entityType: 'verb', + operation: 'create', + entityId: 'rel-1', + verbType: VerbType.RelatedTo + }) + ) + }) + + it('should emit VFS events', () => { + const handler = vi.fn() + eventBus.subscribe({}, handler) + + eventBus.emitVFS('create', 'file-1') + + expect(handler).toHaveBeenCalledWith( + expect.objectContaining({ + entityType: 'vfs', + operation: 'create', + entityId: 'file-1' + }) + ) + }) + }) +}) + +describe('TabularExporter', () => { + let exporter: TabularExporter + + const mockEntity: Entity = { + id: 'entity-123', + type: NounType.Person, + createdAt: 1704067200000, // 2024-01-01 + updatedAt: 1704153600000, // 2024-01-02 + vector: new Float32Array([0.1, 0.2, 0.3]), + confidence: 0.95, + weight: 0.8, + service: 'test-service', + data: { content: 'Hello world' }, + metadata: { + name: 'John Doe', + email: 'john@example.com', + nested: { key: 'value' } + } + } + + const mockRelation: Relation = { + id: 'rel-123', + from: 'entity-1', + to: 'entity-2', + type: VerbType.RelatedTo, + weight: 0.9, + confidence: 0.85, + createdAt: 1704067200000, + service: 'test-service', + metadata: { reason: 'test' } + } + + beforeEach(() => { + exporter = new TabularExporter() + }) + + describe('entitiesToRows', () => { + it('should convert entity to row', () => { + const rows = exporter.entitiesToRows([mockEntity]) + + expect(rows).toHaveLength(1) + expect(rows[0].Id).toBe('entity-123') + expect(rows[0].Type).toBe('person') + expect(rows[0].Confidence).toBe(0.95) + expect(rows[0].Weight).toBe(0.8) + expect(rows[0].Service).toBe('test-service') + }) + + it('should flatten metadata by default', () => { + const rows = exporter.entitiesToRows([mockEntity]) + + expect(rows[0].Metadata_name).toBe('John Doe') + expect(rows[0].Metadata_email).toBe('john@example.com') + }) + + it('should JSON stringify nested metadata', () => { + const rows = exporter.entitiesToRows([mockEntity]) + + expect(rows[0].Metadata_nested).toBe('{"key":"value"}') + }) + + it('should JSON stringify data field', () => { + const rows = exporter.entitiesToRows([mockEntity]) + + expect(rows[0].Data).toBe('{"content":"Hello world"}') + }) + + it('should format dates as ISO 8601', () => { + const rows = exporter.entitiesToRows([mockEntity]) + + expect(rows[0].CreatedAt).toMatch(/^\d{4}-\d{2}-\d{2}T/) + }) + }) + + describe('configuration', () => { + it('should not flatten metadata when disabled', () => { + exporter = new TabularExporter({ flattenMetadata: false }) + const rows = exporter.entitiesToRows([mockEntity]) + + expect(rows[0].Metadata_name).toBeUndefined() + expect(rows[0].Metadata).toBeDefined() + }) + + it('should include vectors when enabled', () => { + exporter = new TabularExporter({ includeVectors: true }) + const rows = exporter.entitiesToRows([mockEntity]) + + expect(rows[0].Vector).toBeDefined() + }) + + it('should use custom metadata prefix', () => { + exporter = new TabularExporter({ metadataPrefix: 'M_' }) + const rows = exporter.entitiesToRows([mockEntity]) + + expect(rows[0].M_name).toBe('John Doe') + expect(rows[0].Metadata_name).toBeUndefined() + }) + + it('should format dates as unix timestamps', () => { + exporter = new TabularExporter({ dateFormat: 'unix' }) + const rows = exporter.entitiesToRows([mockEntity]) + + expect(rows[0].CreatedAt).toBe('1704067200') + }) + }) + + describe('relationsToRows', () => { + it('should convert relation to row', () => { + const rows = exporter.relationsToRows([mockRelation]) + + expect(rows).toHaveLength(1) + expect(rows[0].Id).toBe('rel-123') + expect(rows[0].FromId).toBe('entity-1') + expect(rows[0].ToId).toBe('entity-2') + expect(rows[0].Type).toBe('relatedTo') + expect(rows[0].Weight).toBe(0.9) + expect(rows[0].Confidence).toBe(0.85) + }) + }) + + describe('toCSV', () => { + it('should generate valid CSV', () => { + const csv = exporter.toCSV([mockEntity]) + const lines = csv.split('\n') + + expect(lines.length).toBeGreaterThan(1) + expect(lines[0]).toContain('Id') + expect(lines[0]).toContain('Type') + expect(lines[1]).toContain('entity-123') + }) + + it('should escape special characters', () => { + const entityWithComma: Entity = { + ...mockEntity, + metadata: { name: 'Doe, John' } + } + + const csv = exporter.toCSV([entityWithComma]) + + expect(csv).toContain('"Doe, John"') + }) + + it('should escape quotes', () => { + const entityWithQuotes: Entity = { + ...mockEntity, + metadata: { name: 'John "Jack" Doe' } + } + + const csv = exporter.toCSV([entityWithQuotes]) + + expect(csv).toContain('""Jack""') + }) + }) + + describe('parseCSV', () => { + it('should parse CSV back to entities', () => { + const csv = exporter.toCSV([mockEntity]) + const entities = exporter.parseCSV(csv) + + expect(entities).toHaveLength(1) + expect(entities[0].id).toBe('entity-123') + expect(entities[0].type).toBe('person') + }) + }) + + describe('toOData', () => { + it('should generate OData response', () => { + const odata = exporter.toOData([mockEntity]) + + expect(odata).toHaveProperty('@odata.context') + expect(odata).toHaveProperty('value') + expect((odata as any).value).toHaveLength(1) + }) + + it('should include count when provided', () => { + const odata = exporter.toOData([mockEntity], { count: 100 }) + + expect((odata as any)['@odata.count']).toBe(100) + }) + + it('should include nextLink when provided', () => { + const odata = exporter.toOData([mockEntity], { + nextLink: '/odata/Entities?$skip=100' + }) + + expect((odata as any)['@odata.nextLink']).toBe('/odata/Entities?$skip=100') + }) + }) + + describe('getSchema', () => { + it('should infer schema from entities', () => { + const schema = exporter.getSchema([mockEntity]) + + expect(schema).toContainEqual( + expect.objectContaining({ name: 'Id', type: 'string' }) + ) + expect(schema).toContainEqual( + expect.objectContaining({ name: 'Confidence', type: 'number' }) + ) + }) + }) +}) diff --git a/tests/integrations/odata.test.ts b/tests/integrations/odata.test.ts new file mode 100644 index 00000000..d23da805 --- /dev/null +++ b/tests/integrations/odata.test.ts @@ -0,0 +1,343 @@ +/** + * OData Integration Tests + * + * Tests for OData query parsing, EDMX generation, and filtering. + */ + +import { describe, it, expect } from 'vitest' +import { + parseODataQuery, + parseFilter, + parseOrderBy, + parseSelect, + applyFilter, + applySelect, + applyOrderBy, + applyPagination +} from '../../src/integrations/odata/ODataQueryParser.js' +import { + generateEdmx, + generateMetadataJson, + generateServiceDocument +} from '../../src/integrations/odata/EdmxGenerator.js' + +describe('OData Query Parser', () => { + describe('parseODataQuery', () => { + it('should parse $top', () => { + const options = parseODataQuery('$top=10') + expect(options.top).toBe(10) + }) + + it('should parse $skip', () => { + const options = parseODataQuery('$skip=20') + expect(options.skip).toBe(20) + }) + + it('should parse $select', () => { + const options = parseODataQuery('$select=Id,Type,Name') + expect(options.select).toEqual(['Id', 'Type', 'Name']) + }) + + it('should parse $orderby', () => { + const options = parseODataQuery('$orderby=CreatedAt desc') + expect(options.orderBy).toEqual([ + { field: 'CreatedAt', direction: 'desc' } + ]) + }) + + it('should parse $count', () => { + const options = parseODataQuery('$count=true') + expect(options.count).toBe(true) + }) + + it('should parse $filter', () => { + const options = parseODataQuery("$filter=Type eq 'person'") + expect(options.filter).toBe("Type eq 'person'") + }) + + it('should parse $search', () => { + const options = parseODataQuery('$search=machine learning') + expect(options.search).toBe('machine learning') + }) + + it('should parse multiple options', () => { + const options = parseODataQuery( + '$top=10&$skip=20&$orderby=Name asc&$count=true' + ) + expect(options.top).toBe(10) + expect(options.skip).toBe(20) + expect(options.orderBy).toEqual([{ field: 'Name', direction: 'asc' }]) + expect(options.count).toBe(true) + }) + }) + + describe('parseFilter', () => { + it('should parse simple eq comparison', () => { + const filter = parseFilter("Type eq 'person'") + expect(filter).toEqual({ + field: 'Type', + op: 'eq', + value: 'person' + }) + }) + + it('should parse ne comparison', () => { + const filter = parseFilter("Status ne 'deleted'") + expect(filter).toEqual({ + field: 'Status', + op: 'ne', + value: 'deleted' + }) + }) + + it('should parse numeric comparisons', () => { + const gtFilter = parseFilter('Weight gt 0.5') + expect(gtFilter.op).toBe('gt') + expect(gtFilter.value).toBe(0.5) + + const ltFilter = parseFilter('Count lt 100') + expect(ltFilter.op).toBe('lt') + expect(ltFilter.value).toBe(100) + }) + + it('should parse boolean values', () => { + const filter = parseFilter('Active eq true') + expect(filter.value).toBe(true) + }) + + it('should parse null values', () => { + const filter = parseFilter('DeletedAt eq null') + expect(filter.value).toBe(null) + }) + + it('should parse and expressions', () => { + const filter = parseFilter("Type eq 'person' and Status eq 'active'") + expect(filter).toEqual({ + and: [ + { field: 'Type', op: 'eq', value: 'person' }, + { field: 'Status', op: 'eq', value: 'active' } + ] + }) + }) + + it('should parse or expressions', () => { + const filter = parseFilter("Type eq 'person' or Type eq 'organization'") + expect(filter).toEqual({ + or: [ + { field: 'Type', op: 'eq', value: 'person' }, + { field: 'Type', op: 'eq', value: 'organization' } + ] + }) + }) + + it('should parse contains function', () => { + const filter = parseFilter("contains(Name, 'John')") + expect(filter).toEqual({ + func: 'contains', + args: [{ field: 'Name' }, { value: 'John' }] + }) + }) + + it('should parse startswith function', () => { + const filter = parseFilter("startswith(Email, 'admin')") + expect(filter).toEqual({ + func: 'startswith', + args: [{ field: 'Email' }, { value: 'admin' }] + }) + }) + }) + + describe('parseOrderBy', () => { + it('should parse single field', () => { + const orderBy = parseOrderBy('Name') + expect(orderBy).toEqual([{ field: 'Name', direction: 'asc' }]) + }) + + it('should parse with direction', () => { + const orderBy = parseOrderBy('CreatedAt desc') + expect(orderBy).toEqual([{ field: 'CreatedAt', direction: 'desc' }]) + }) + + it('should parse multiple fields', () => { + const orderBy = parseOrderBy('Type asc, CreatedAt desc') + expect(orderBy).toEqual([ + { field: 'Type', direction: 'asc' }, + { field: 'CreatedAt', direction: 'desc' } + ]) + }) + }) + + describe('parseSelect', () => { + it('should parse comma-separated fields', () => { + const select = parseSelect('Id, Type, Name') + expect(select).toEqual(['Id', 'Type', 'Name']) + }) + }) + + describe('applyFilter', () => { + const testData = [ + { Id: '1', Type: 'person', Name: 'John Doe', Weight: 0.8 }, + { Id: '2', Type: 'person', Name: 'Jane Smith', Weight: 0.6 }, + { Id: '3', Type: 'organization', Name: 'Acme Corp', Weight: 0.9 }, + { Id: '4', Type: 'document', Name: 'Report', Weight: 0.5 } + ] + + it('should filter by eq', () => { + const result = applyFilter(testData, "Type eq 'person'") + expect(result).toHaveLength(2) + expect(result.every((r) => r.Type === 'person')).toBe(true) + }) + + it('should filter by gt', () => { + const result = applyFilter(testData, 'Weight gt 0.7') + expect(result).toHaveLength(2) + expect(result.every((r) => r.Weight > 0.7)).toBe(true) + }) + + it('should filter with contains', () => { + const result = applyFilter(testData, "contains(Name, 'Doe')") + expect(result).toHaveLength(1) + expect(result[0].Name).toBe('John Doe') + }) + + it('should filter with and', () => { + const result = applyFilter( + testData, + "Type eq 'person' and Weight gt 0.7" + ) + expect(result).toHaveLength(1) + expect(result[0].Name).toBe('John Doe') + }) + }) + + describe('applySelect', () => { + const testData = [ + { Id: '1', Type: 'person', Name: 'John', Extra: 'data' } + ] + + it('should select specified fields only', () => { + const result = applySelect(testData, ['Id', 'Name']) + expect(result[0]).toHaveProperty('Id') + expect(result[0]).toHaveProperty('Name') + expect(result[0]).not.toHaveProperty('Type') + expect(result[0]).not.toHaveProperty('Extra') + }) + + it('should return all fields if select is empty', () => { + const result = applySelect(testData, []) + expect(result[0]).toHaveProperty('Extra') + }) + }) + + describe('applyOrderBy', () => { + const testData = [ + { Name: 'Charlie', Weight: 0.5 }, + { Name: 'Alice', Weight: 0.9 }, + { Name: 'Bob', Weight: 0.7 } + ] + + it('should sort ascending by default', () => { + const result = applyOrderBy(testData, [ + { field: 'Name', direction: 'asc' } + ]) + expect(result[0].Name).toBe('Alice') + expect(result[1].Name).toBe('Bob') + expect(result[2].Name).toBe('Charlie') + }) + + it('should sort descending', () => { + const result = applyOrderBy(testData, [ + { field: 'Weight', direction: 'desc' } + ]) + expect(result[0].Weight).toBe(0.9) + expect(result[2].Weight).toBe(0.5) + }) + }) + + describe('applyPagination', () => { + const testData = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + + it('should apply top (limit)', () => { + const result = applyPagination(testData, 3) + expect(result).toEqual([1, 2, 3]) + }) + + it('should apply skip (offset)', () => { + const result = applyPagination(testData, undefined, 5) + expect(result).toEqual([6, 7, 8, 9, 10]) + }) + + it('should apply both top and skip', () => { + const result = applyPagination(testData, 3, 2) + expect(result).toEqual([3, 4, 5]) + }) + }) +}) + +describe('EDMX Generator', () => { + describe('generateEdmx', () => { + it('should generate valid XML', () => { + const xml = generateEdmx() + expect(xml).toContain(' { + const xml = generateEdmx() + expect(xml).toContain('Property Name="Id"') + expect(xml).toContain('Property Name="Type"') + expect(xml).toContain('Property Name="CreatedAt"') + }) + + it('should include NounType enum', () => { + const xml = generateEdmx() + expect(xml).toContain('EnumType Name="NounType"') + expect(xml).toContain('Member Name="person"') + }) + + it('should include relationships when enabled', () => { + const xml = generateEdmx({ includeRelationships: true }) + expect(xml).toContain('EntityType Name="Relationship"') + expect(xml).toContain('EnumType Name="VerbType"') + }) + + it('should exclude relationships when disabled', () => { + const xml = generateEdmx({ includeRelationships: false }) + expect(xml).not.toContain('EntityType Name="Relationship"') + }) + + it('should use custom namespace', () => { + const xml = generateEdmx({ namespace: 'MyApp' }) + expect(xml).toContain('Namespace="MyApp"') + }) + }) + + describe('generateMetadataJson', () => { + it('should generate JSON metadata', () => { + const metadata = generateMetadataJson() + expect(metadata).toHaveProperty('$Version', '4.0') + expect(metadata).toHaveProperty('Brainy') + }) + }) + + describe('generateServiceDocument', () => { + it('should generate service document', () => { + const doc = generateServiceDocument('http://localhost/odata') + expect(doc).toHaveProperty('@odata.context') + expect(doc).toHaveProperty('value') + expect((doc as any).value).toContainEqual( + expect.objectContaining({ name: 'Entities' }) + ) + }) + + it('should include relationships when enabled', () => { + const doc = generateServiceDocument('http://localhost/odata', { + includeRelationships: true + }) + expect((doc as any).value).toContainEqual( + expect.objectContaining({ name: 'Relationships' }) + ) + }) + }) +})