From d1db3510beb46349bbfa04c5defc85f897afdbbc Mon Sep 17 00:00:00 2001 From: David Snelling Date: Sun, 1 Feb 2026 10:48:56 -0800 Subject: [PATCH] refactor: remove augmentation system and semantic type matching Remove the entire augmentation pipeline infrastructure (52 files, ~15,000 lines) and the semantic type matching system. These were unused middleware layers adding complexity without value. What was removed: - src/augmentations/ directory (all augmentation implementations) - src/augmentationManager.ts (pipeline orchestrator) - src/types/augmentations.ts, src/types/pipelineTypes.ts - src/shared/default-augmentations.ts - Semantic type suggestion (BrainyTypes.suggestNoun/suggestVerb) - src/utils/typeMatching/ (embedding-based type matcher) What was preserved by relocating: - Import handlers (CSV, PDF, Excel) -> src/importers/handlers/ - NeuralImportAugmentation -> src/cortex/neuralImportAugmentation.ts - Type matching utilities -> heuristic inference in consumers What was simplified: - brainy.ts: operations call storage directly (no execute() wrapper) - IntegrationBase: standalone class (no BaseAugmentation parent) - BrainyTypes: validation-only (nouns, verbs, isValid*, get*) - Pipeline: direct execution (no augmentation interception) - index.ts: removed TypeSuggestion, suggestType exports - package.json: removed stale types/augmentations export Build passes, 1176 tests pass, 0 failures. --- examples/api-server-example.ts | 105 -- examples/augmentations-zero-config.ts | 49 - examples/tests/instant-validation.js | 7 +- examples/tests/test-storage-augmentation.js | 89 -- package.json | 4 - src/api/UniversalImportAPI.ts | 143 ++- src/augmentationManager.ts | 137 --- .../AugmentationMetadataContract.ts | 410 -------- src/augmentations/README.md | 292 ------ src/augmentations/apiServerAugmentation.ts | 915 ------------------ src/augmentations/auditLogAugmentation.ts | 452 --------- .../batchProcessingAugmentation.ts | 803 --------------- src/augmentations/brainyAugmentation.ts | 766 --------------- src/augmentations/cacheAugmentation.ts | 391 -------- src/augmentations/conduitAugmentations.ts | 274 ------ src/augmentations/configResolver.ts | 539 ----------- .../connectionPoolAugmentation.ts | 445 --------- src/augmentations/defaultAugmentations.ts | 130 --- src/augmentations/discovery.ts | 560 ----------- .../discovery/catalogDiscovery.ts | 357 ------- src/augmentations/discovery/localDiscovery.ts | 315 ------ src/augmentations/discovery/runtimeLoader.ts | 443 --------- src/augmentations/display/cache.ts | 375 ------- src/augmentations/display/fieldPatterns.ts | 429 -------- src/augmentations/display/iconMappings.ts | 76 -- .../display/intelligentComputation.ts | 539 ----------- src/augmentations/display/types.ts | 256 ----- .../entityRegistryAugmentation.ts | 518 ---------- src/augmentations/indexAugmentation.ts | 337 ------- .../FormatHandlerRegistry.ts | 246 ----- .../IntelligentImportAugmentation.ts | 227 ----- .../handlers/imageHandler.ts | 350 ------- src/augmentations/intelligentImport/index.ts | 33 - .../intelligentVerbScoringAugmentation.ts | 875 ----------------- src/augmentations/manifest.ts | 207 ---- src/augmentations/metadataEnforcer.ts | 204 ---- src/augmentations/metricsAugmentation.ts | 356 ------- src/augmentations/monitoringAugmentation.ts | 270 ------ src/augmentations/rateLimitAugmentation.ts | 384 -------- .../requestDeduplicatorAugmentation.ts | 216 ----- src/augmentations/storageAugmentation.ts | 119 --- src/augmentations/storageAugmentations.ts | 496 ---------- src/augmentations/synapseAugmentation.ts | 467 --------- src/augmentations/typeMatching/brainyTypes.ts | 579 ----------- .../typeMatching/intelligentTypeMatcher.ts | 180 ---- .../universalDisplayAugmentation.ts | 449 --------- src/brainy.ts | 275 ++---- src/cli/commands/core.ts | 18 +- src/cli/commands/types.ts | 106 +- .../neuralImportAugmentation.ts} | 105 +- src/importers/SmartCSVImporter.ts | 4 +- src/importers/SmartExcelImporter.ts | 4 +- src/importers/SmartPDFImporter.ts | 4 +- .../handlers/base.ts | 4 +- .../handlers/csvHandler.ts | 2 +- .../handlers/excelHandler.ts | 2 +- .../handlers/pdfHandler.ts | 2 +- .../handlers}/types.ts | 0 src/index.ts | 121 +-- src/integrations/core/IntegrationBase.ts | 44 +- src/integrations/odata/ODataIntegration.ts | 9 +- .../sheets/GoogleSheetsIntegration.ts | 8 +- src/integrations/sse/SSEIntegration.ts | 8 +- .../webhooks/WebhookIntegration.ts | 5 +- src/mcp/mcpAugmentationToolset.ts | 11 +- src/pipeline.ts | 4 +- src/shared/default-augmentations.ts | 130 --- src/types/augmentations.ts | 220 ----- src/types/brainy.types.ts | 3 - src/types/pipelineTypes.ts | 33 - src/utils/brainyTypes.ts | 216 +---- .../intelligentImport/csvHandler.unit.test.ts | 281 ------ .../excelHandler.unit.test.ts | 269 ----- .../intelligentImport/pdfHandler.unit.test.ts | 292 ------ tests/benchmarks/benchmark-real-world.js | 3 +- tests/benchmarks/benchmark-vs-industry.js | 6 - tests/benchmarks/perf-final.js | 7 - tests/benchmarks/perf-simple.js | 1 - tests/benchmarks/performance-comprehensive.js | 4 +- tests/benchmarks/performance-profile.js | 35 +- tests/benchmarks/performance-v3.js | 2 - tests/benchmarks/quick-benchmark-v3.js | 1 - tests/benchmarks/quick-perf.js | 19 +- tests/benchmarks/quick-verify.js | 4 - tests/benchmarks/scale-test.js | 59 +- tests/brainy-3.test.ts | 14 +- .../comprehensive/public-api-complete.test.ts | 1 - tests/integration/image-import.test.ts | 361 ------- tests/streaming-pipeline.test.ts | 1 - .../augmentations-simplified.test.ts | 535 ---------- .../format-handler-registry.test.ts | 437 --------- .../unit/augmentations/image-handler.test.ts | 311 ------ tests/unit/brainy-core.unit.test.ts | 10 +- tests/unit/brainy/add.test.ts | 18 - .../unit/performance/add-performance.test.ts | 8 +- tests/unit/plugin.test.ts | 4 - tests/unit/type-matching.unit.test.ts | 219 ----- 97 files changed, 349 insertions(+), 19705 deletions(-) delete mode 100644 examples/api-server-example.ts delete mode 100644 examples/augmentations-zero-config.ts delete mode 100644 examples/tests/test-storage-augmentation.js delete mode 100644 src/augmentationManager.ts delete mode 100644 src/augmentations/AugmentationMetadataContract.ts delete mode 100644 src/augmentations/README.md delete mode 100644 src/augmentations/apiServerAugmentation.ts delete mode 100644 src/augmentations/auditLogAugmentation.ts delete mode 100644 src/augmentations/batchProcessingAugmentation.ts delete mode 100644 src/augmentations/brainyAugmentation.ts delete mode 100644 src/augmentations/cacheAugmentation.ts delete mode 100644 src/augmentations/conduitAugmentations.ts delete mode 100644 src/augmentations/configResolver.ts delete mode 100644 src/augmentations/connectionPoolAugmentation.ts delete mode 100644 src/augmentations/defaultAugmentations.ts delete mode 100644 src/augmentations/discovery.ts delete mode 100644 src/augmentations/discovery/catalogDiscovery.ts delete mode 100644 src/augmentations/discovery/localDiscovery.ts delete mode 100644 src/augmentations/discovery/runtimeLoader.ts delete mode 100644 src/augmentations/display/cache.ts delete mode 100644 src/augmentations/display/fieldPatterns.ts delete mode 100644 src/augmentations/display/iconMappings.ts delete mode 100644 src/augmentations/display/intelligentComputation.ts delete mode 100644 src/augmentations/display/types.ts delete mode 100644 src/augmentations/entityRegistryAugmentation.ts delete mode 100644 src/augmentations/indexAugmentation.ts delete mode 100644 src/augmentations/intelligentImport/FormatHandlerRegistry.ts delete mode 100644 src/augmentations/intelligentImport/IntelligentImportAugmentation.ts delete mode 100644 src/augmentations/intelligentImport/handlers/imageHandler.ts delete mode 100644 src/augmentations/intelligentImport/index.ts delete mode 100644 src/augmentations/intelligentVerbScoringAugmentation.ts delete mode 100644 src/augmentations/manifest.ts delete mode 100644 src/augmentations/metadataEnforcer.ts delete mode 100644 src/augmentations/metricsAugmentation.ts delete mode 100644 src/augmentations/monitoringAugmentation.ts delete mode 100644 src/augmentations/rateLimitAugmentation.ts delete mode 100644 src/augmentations/requestDeduplicatorAugmentation.ts delete mode 100644 src/augmentations/storageAugmentation.ts delete mode 100644 src/augmentations/storageAugmentations.ts delete mode 100644 src/augmentations/synapseAugmentation.ts delete mode 100644 src/augmentations/typeMatching/brainyTypes.ts delete mode 100644 src/augmentations/typeMatching/intelligentTypeMatcher.ts delete mode 100644 src/augmentations/universalDisplayAugmentation.ts rename src/{augmentations/neuralImport.ts => cortex/neuralImportAugmentation.ts} (86%) rename src/{augmentations/intelligentImport => importers}/handlers/base.ts (98%) rename src/{augmentations/intelligentImport => importers}/handlers/csvHandler.ts (99%) rename src/{augmentations/intelligentImport => importers}/handlers/excelHandler.ts (99%) rename src/{augmentations/intelligentImport => importers}/handlers/pdfHandler.ts (99%) rename src/{augmentations/intelligentImport => importers/handlers}/types.ts (100%) delete mode 100644 src/shared/default-augmentations.ts delete mode 100644 src/types/augmentations.ts delete mode 100644 src/types/pipelineTypes.ts delete mode 100644 tests/augmentations/intelligentImport/csvHandler.unit.test.ts delete mode 100644 tests/augmentations/intelligentImport/excelHandler.unit.test.ts delete mode 100644 tests/augmentations/intelligentImport/pdfHandler.unit.test.ts delete mode 100644 tests/integration/image-import.test.ts delete mode 100644 tests/unit/augmentations/augmentations-simplified.test.ts delete mode 100644 tests/unit/augmentations/format-handler-registry.test.ts delete mode 100644 tests/unit/augmentations/image-handler.test.ts delete mode 100644 tests/unit/type-matching.unit.test.ts diff --git a/examples/api-server-example.ts b/examples/api-server-example.ts deleted file mode 100644 index 0e0b8288..00000000 --- a/examples/api-server-example.ts +++ /dev/null @@ -1,105 +0,0 @@ -/** - * API Server Example - * - * This example shows how to expose Brainy through REST, WebSocket, and MCP APIs - * using the APIServerAugmentation. - * - * Zero-config philosophy: Just create and register the augmentation! - */ - -import { Brainy } from '../src/index.js' -import { APIServerAugmentation } from '../src/augmentations/apiServerAugmentation.js' - -async function main() { - // 1. Create Brainy with zero config - const brain = new Brainy() - await brain.init() - - // 2. Add some sample data - await brain.add({ data: "The quick brown fox", type: 'content', metadata: { type: "sentence", category: "animals" } }) - await brain.add({ data: "Machine learning models", type: 'content', metadata: { type: "tech", category: "AI" } }) - await brain.add({ data: "Natural language processing", type: 'content', metadata: { type: "tech", category: "NLP" } }) - - // 3. Create and register the API Server augmentation - const apiServer = new APIServerAugmentation({ - port: 3000, - host: 'localhost' - }) - - // Register the augmentation with Brainy - brain.augmentations.register(apiServer) - - // Initialize augmentations with Brainy context - await brain.augmentations.initialize({ - brain, - log: (msg: string, level?: string) => console.log(`[${level || 'info'}] ${msg}`), - config: {} - }) - - console.log('๐Ÿš€ Brainy API Server is running!') - console.log('๐Ÿ“ก REST API: http://localhost:3000') - console.log('๐Ÿ”Œ WebSocket: ws://localhost:3000/ws') - console.log('๐Ÿง  MCP: http://localhost:3000/api/mcp') - console.log('') - console.log('Try these endpoints:') - console.log(' GET http://localhost:3000/health') - console.log(' POST http://localhost:3000/api/search') - console.log(' Body: { "query": "fox", "limit": 10 }') - console.log(' POST http://localhost:3000/api/add') - console.log(' Body: { "content": "New data", "metadata": {} }') - console.log('') - console.log('Press Ctrl+C to stop the server') -} - -// Run the example -main().catch(console.error) - -/** - * Example REST API calls: - * - * # Health check - * curl http://localhost:3000/health - * - * # Search - * curl -X POST http://localhost:3000/api/search \ - * -H "Content-Type: application/json" \ - * -d '{"query": "fox", "limit": 5}' - * - * # Add data - * curl -X POST http://localhost:3000/api/add \ - * -H "Content-Type: application/json" \ - * -d '{"content": "The cat sat on the mat", "metadata": {"type": "sentence"}}' - * - * # Get by ID - * curl http://localhost:3000/api/get/[id] - * - * # Statistics - * curl http://localhost:3000/api/stats - */ - -/** - * Example WebSocket client (browser): - * - * const ws = new WebSocket('ws://localhost:3000/ws') - * - * ws.onopen = () => { - * // Subscribe to all operations - * ws.send(JSON.stringify({ - * type: 'subscribe', - * operations: ['all'] - * })) - * - * // Perform a search - * ws.send(JSON.stringify({ - * type: 'search', - * query: 'fox', - * limit: 5, - * requestId: '123' - * })) - * } - * - * ws.onmessage = (event) => { - * const msg = JSON.parse(event.data) - * console.log('Received:', msg) - * } - */ \ No newline at end of file diff --git a/examples/augmentations-zero-config.ts b/examples/augmentations-zero-config.ts deleted file mode 100644 index 78c0f82a..00000000 --- a/examples/augmentations-zero-config.ts +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Zero-Config Augmentations Example - * - * Brainy's philosophy: Everything just works with zero configuration. - * Augmentations extend Brainy without any complex setup. - */ - -import { Brainy } from '../src/index.js' -import { EntityRegistryAugmentation } from '../src/augmentations/entityRegistryAugmentation.js' -import { BatchProcessingAugmentation } from '../src/augmentations/batchProcessingAugmentation.js' -import { APIServerAugmentation } from '../src/augmentations/apiServerAugmentation.js' - -async function main() { - // Create Brainy - zero config! - const brain = new Brainy() - - // Register augmentations - they just work! - brain.augmentations.register(new EntityRegistryAugmentation()) // Deduplication - brain.augmentations.register(new BatchProcessingAugmentation()) // Performance - brain.augmentations.register(new APIServerAugmentation()) // API exposure - - // Initialize Brainy (augmentations auto-initialize) - await brain.init() - - // Use Brainy normally - augmentations work transparently - await brain.add({ data: "Hello world", type: 'content', metadata: { type: "greeting" } }) - - // Batch operations automatically optimized - await brain.addBatch([ - { content: "Item 1", metadata: { id: 1 } }, - { content: "Item 2", metadata: { id: 2 } }, - { content: "Item 3", metadata: { id: 3 } } - ]) - - // Search works with all augmentations active - const results = await brain.search("hello") - console.log('Search results:', results) - - // API server is running at http://localhost:3000 - console.log('โœจ Brainy is running with:') - console.log(' - Write-ahead logging (durability)') - console.log(' - Entity deduplication (performance)') - console.log(' - Batch optimization (speed)') - console.log(' - REST/WebSocket/MCP APIs (connectivity)') - console.log('') - console.log('Zero configuration required!') -} - -main().catch(console.error) \ No newline at end of file diff --git a/examples/tests/instant-validation.js b/examples/tests/instant-validation.js index 334c69f9..4bc9c2f4 100644 --- a/examples/tests/instant-validation.js +++ b/examples/tests/instant-validation.js @@ -49,11 +49,8 @@ try { test('Memory storage configured', brain.storage && brain.storage.storageType === 'memory') console.log('\n๐Ÿ“Š API Architecture Validation:') - - // Test 5: Augmentation system structure - test('Augmentations system exists', brain.augmentations !== undefined) - - // Test 6: Core properties exist + + // Test 5: Core properties exist test('Index system exists', brain.index !== undefined) test('Storage system exists', brain.storage !== undefined) diff --git a/examples/tests/test-storage-augmentation.js b/examples/tests/test-storage-augmentation.js deleted file mode 100644 index c6253575..00000000 --- a/examples/tests/test-storage-augmentation.js +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Test script to verify storage augmentation system works - */ - -import { Brainy } from '../../dist/index.js' -import { - MemoryStorageAugmentation, - FileSystemStorageAugmentation -} from './dist/augmentations/storageAugmentations.js' - -console.log('๐Ÿงช Testing Storage Augmentation System') -console.log('=' .repeat(50)) - -async function test1_ZeroConfig() { - console.log('\n1. Zero-Config Test') - const brain = new Brainy() - await brain.init() - - await brain.add('test', { content: 'Zero-config test' }) - const results = await brain.search('test', { limit: 1 }) - - console.log('โœ… Zero-config works:', results.length > 0) - await brain.destroy() -} - -async function test2_ConfigBased() { - console.log('\n2. Config-Based Storage Test') - const brain = new Brainy({ - storage: { - forceMemoryStorage: true - } - }) - await brain.init() - - await brain.add('config test', { content: 'Config-based test' }) - const results = await brain.search('config', { limit: 1 }) - - console.log('โœ… Config-based works:', results.length > 0) - await brain.destroy() -} - -async function test3_AugmentationOverride() { - console.log('\n3. Augmentation Override Test') - const brain = new Brainy() - - // Register storage augmentation BEFORE init - brain.augmentations.register(new MemoryStorageAugmentation('test-memory')) - - await brain.init() - - await brain.add('augmentation test', { content: 'Augmentation override test' }) - const results = await brain.search('augmentation', { limit: 1 }) - - console.log('โœ… Augmentation override works:', results.length > 0) - await brain.destroy() -} - -async function test4_BackwardCompatibility() { - console.log('\n4. Backward Compatibility Test') - - // Old style with rootDirectory config - const brain = new Brainy({ - storage: { - rootDirectory: './test-data', - forceFileSystemStorage: true - } - }) - - await brain.init() - console.log('โœ… Backward compatible config works') - await brain.destroy() -} - -async function runAllTests() { - try { - await test1_ZeroConfig() - await test2_ConfigBased() - await test3_AugmentationOverride() - await test4_BackwardCompatibility() - - console.log('\n' + '='.repeat(50)) - console.log('โœ… ALL STORAGE AUGMENTATION TESTS PASSED!') - } catch (error) { - console.error('โŒ Test failed:', error) - process.exit(1) - } -} - -runAllTests() \ No newline at end of file diff --git a/package.json b/package.json index e3204f05..fe7801d0 100644 --- a/package.json +++ b/package.json @@ -28,10 +28,6 @@ "import": "./dist/types/graphTypes.js", "types": "./dist/types/graphTypes.d.ts" }, - "./types/augmentations": { - "import": "./dist/types/augmentations.js", - "types": "./dist/types/augmentations.d.ts" - }, "./utils/textEncoding": { "import": "./dist/utils/textEncoding.js", "types": "./dist/utils/textEncoding.d.ts" diff --git a/src/api/UniversalImportAPI.ts b/src/api/UniversalImportAPI.ts index 74460cee..b85fe835 100644 --- a/src/api/UniversalImportAPI.ts +++ b/src/api/UniversalImportAPI.ts @@ -16,8 +16,7 @@ import { NounType, VerbType } from '../types/graphTypes.js' import { Vector } from '../coreTypes.js' import type { Brainy } from '../brainy.js' import type { Entity, Relation } from '../types/brainy.types.js' -import { BrainyTypes, getBrainyTypes } from '../augmentations/typeMatching/brainyTypes.js' -import { NeuralImportAugmentation } from '../augmentations/neuralImport.js' +import { NeuralImportAugmentation } from '../cortex/neuralImportAugmentation.js' import { mimeDetector } from '../vfs/MimeTypeDetector.js' export interface ImportSource { @@ -65,7 +64,6 @@ export interface NeuralImportProgress { export class UniversalImportAPI { private brain: Brainy - private typeMatcher!: BrainyTypes private neuralImport: NeuralImportAugmentation private embedCache = new Map() @@ -82,7 +80,6 @@ export class UniversalImportAPI { * Initialize the neural import system */ async init(): Promise { - this.typeMatcher = await getBrainyTypes() // Neural import initializes itself } @@ -369,22 +366,19 @@ export class UniversalImportAPI { for (const item of data) { // Generate embedding for the item const embedding = await this.generateEmbedding(item) - - // Neural type matching - MANDATORY - const nounMatch = await this.typeMatcher.matchNounType(item) - - // Never reject based on confidence - we ALWAYS accept the best match + + // Determine noun type from data + const nounType = this.inferNounType(item) const entityId = this.generateId(item) - + entities.set(entityId, { id: entityId, - type: nounMatch.type as NounType, // Always use the neural match + type: nounType, data: item, vector: embedding, - confidence: nounMatch.confidence, + confidence: 1.0, metadata: { ...item, - _neuralMatch: nounMatch, _importedAt: Date.now() } }) @@ -465,56 +459,42 @@ export class UniversalImportAPI { for (const [key, value] of Object.entries(item)) { // Check if this looks like a reference if (this.looksLikeReference(key, value)) { - // Find or predict target entity const targetId = String(value) - - // Neural verb type matching - const verbMatch = await this.typeMatcher.matchVerbType( - item, // source object - { id: targetId }, // target (we may not have full data) - key // field name as context - ) - - // Always create relationship with neural match - const relationId = `${sourceId}_${verbMatch.type}_${targetId}` + const verbType = this.inferVerbType(key) + + const relationId = `${sourceId}_${verbType}_${targetId}` relationships.set(relationId, { id: relationId, from: sourceId, to: targetId, - type: verbMatch.type as VerbType, - weight: verbMatch.confidence, // Use confidence as weight - confidence: verbMatch.confidence, + type: verbType, + weight: 1.0, + confidence: 1.0, metadata: { field: key, - _neuralMatch: verbMatch, _importedAt: Date.now() } }) } - + // Handle arrays of references if (Array.isArray(value)) { - for (const item of value) { - if (this.looksLikeReference(key, item)) { - const targetId = String(item) - const verbMatch = await this.typeMatcher.matchVerbType( - item, - { id: targetId }, - key - ) - - const relationId = `${sourceId}_${verbMatch.type}_${targetId}` + for (const arrayItem of value) { + if (this.looksLikeReference(key, arrayItem)) { + const targetId = String(arrayItem) + const verbType = this.inferVerbType(key) + + const relationId = `${sourceId}_${verbType}_${targetId}` relationships.set(relationId, { id: relationId, from: sourceId, to: targetId, - type: verbMatch.type as VerbType, - weight: verbMatch.confidence, - confidence: verbMatch.confidence, + type: verbType, + weight: 1.0, + confidence: 1.0, metadata: { field: key, array: true, - _neuralMatch: verbMatch, _importedAt: Date.now() } }) @@ -552,7 +532,82 @@ export class UniversalImportAPI { return fieldLooksLikeRef && valueLooksLikeId } - + + /** + * Infer noun type from object structure using field heuristics + */ + private inferNounType(obj: any): NounType { + if (typeof obj !== 'object' || obj === null) return NounType.Thing + + // Check for explicit type field + if (obj.type && typeof obj.type === 'string') { + const normalized = obj.type.charAt(0).toUpperCase() + obj.type.slice(1) + if (Object.values(NounType).includes(normalized as NounType)) { + return normalized as NounType + } + } + + // Person heuristics + if (obj.email || obj.firstName || obj.lastName || obj.username || obj.age) { + return NounType.Person + } + // Organization heuristics + if (obj.companyName || obj.organizationId || obj.employees || obj.industry) { + return NounType.Organization + } + // Location heuristics + if (obj.latitude || obj.longitude || obj.address || obj.city || obj.country) { + return NounType.Location + } + // Document heuristics + if ((obj.content && (obj.title || obj.author)) || obj.documentType || obj.pages) { + return NounType.Document + } + // Event heuristics + if (obj.startTime || obj.endTime || obj.date || obj.eventType || obj.attendees) { + return NounType.Event + } + // Product heuristics + if (obj.price || obj.sku || obj.inventory || obj.productId) { + return NounType.Product + } + // Task heuristics + if ((obj.status && (obj.assignee || obj.dueDate)) || obj.priority || obj.completed !== undefined) { + return NounType.Task + } + // Dataset heuristics + if (Array.isArray(obj.data) || obj.rows || obj.columns || obj.schema) { + return NounType.Dataset + } + + return NounType.Thing + } + + /** + * Infer verb type from field name using common patterns + */ + private inferVerbType(fieldName: string): VerbType { + const field = fieldName.toLowerCase() + + if (field.includes('parent') || field.includes('child') || field.includes('contain')) { + return VerbType.Contains + } + if (field.includes('owner') || field.includes('created') || field.includes('author')) { + return VerbType.Creates + } + if (field.includes('member') || field.includes('belong')) { + return VerbType.MemberOf + } + if (field.includes('depend') || field.includes('require')) { + return VerbType.DependsOn + } + if (field.includes('ref') || field.includes('link') || field.includes('source')) { + return VerbType.References + } + + return VerbType.RelatedTo + } + /** * Store processed data in brain */ diff --git a/src/augmentationManager.ts b/src/augmentationManager.ts deleted file mode 100644 index 998b92e2..00000000 --- a/src/augmentationManager.ts +++ /dev/null @@ -1,137 +0,0 @@ -/** - * Type-safe augmentation management system for Brainy - * Provides a clean API for managing augmentations without string literals - */ - -import { IAugmentation, AugmentationType } from './types/augmentations.js' -import { augmentationPipeline } from './pipeline.js' - -export interface AugmentationInfo { - name: string - type: string - enabled: boolean - description: string -} - -/** - * Type-safe augmentation manager - * Accessed via brain.augmentations for all management operations - */ -export class AugmentationManager { - private pipeline = augmentationPipeline - - /** - * List all registered augmentations with their status - * @returns Array of augmentation information - */ - list(): AugmentationInfo[] { - // Deprecated: use brain.augmentations instead - return [] - } - - /** - * Get information about a specific augmentation - * @param name The augmentation name - * @returns Augmentation info or undefined if not found - */ - get(name: string): AugmentationInfo | undefined { - const all = this.list() - return all.find(a => a.name === name) - } - - /** - * Check if an augmentation is enabled - * @param name The augmentation name - * @returns True if enabled, false otherwise - */ - isEnabled(name: string): boolean { - const aug = this.get(name) - return aug?.enabled ?? false - } - - /** - * Enable a specific augmentation - * @param name The augmentation name - * @returns True if successfully enabled - */ - enable(name: string): boolean { - // Deprecated: use brain.augmentations instead - return false - } - - /** - * Disable a specific augmentation - * @param name The augmentation name - * @returns True if successfully disabled - */ - disable(name: string): boolean { - // Deprecated: use brain.augmentations instead - return false - } - - /** - * Remove an augmentation from the pipeline - * @param name The augmentation name - * @returns True if successfully removed - */ - remove(name: string): boolean { - // Deprecated: use brain.augmentations instead - return true - } - - /** - * Enable all augmentations of a specific type - * @param type The augmentation type - * @returns Number of augmentations enabled - */ - enableType(type: AugmentationType): number { - // Deprecated: use brain.augmentations instead - return 0 - } - - /** - * Disable all augmentations of a specific type - * @param type The augmentation type - * @returns Number of augmentations disabled - */ - disableType(type: AugmentationType): number { - // Deprecated: use brain.augmentations instead - return 0 - } - - /** - * Get all augmentations of a specific type - * @param type The augmentation type - * @returns Array of augmentations of that type - */ - listByType(type: AugmentationType): AugmentationInfo[] { - return this.list().filter(a => a.type === type) - } - - /** - * Get all enabled augmentations - * @returns Array of enabled augmentations - */ - listEnabled(): AugmentationInfo[] { - return this.list().filter(a => a.enabled) - } - - /** - * Get all disabled augmentations - * @returns Array of disabled augmentations - */ - listDisabled(): AugmentationInfo[] { - return this.list().filter(a => !a.enabled) - } - - /** - * Register a new augmentation (internal use) - * @param augmentation The augmentation to register - */ - register(augmentation: IAugmentation): void { - // Deprecated: use brain.augmentations instead - } -} - -// Export types for external use -export { AugmentationType } from './types/augmentations.js' \ No newline at end of file diff --git a/src/augmentations/AugmentationMetadataContract.ts b/src/augmentations/AugmentationMetadataContract.ts deleted file mode 100644 index b02bb4cb..00000000 --- a/src/augmentations/AugmentationMetadataContract.ts +++ /dev/null @@ -1,410 +0,0 @@ -/** - * Augmentation Metadata Contract System - * - * Prevents accidental metadata corruption while allowing intentional enrichment - * Each augmentation declares its metadata intentions upfront - */ - -export interface AugmentationMetadataContract { - // Augmentation identity - name: string - version: string - - // What fields this augmentation READS - reads?: { - userFields?: string[] // e.g., ['title', 'description'] - internalFields?: string[] // e.g., ['_brainy.deleted'] - augmentationFields?: string[] // e.g., ['_augmentations.otherAug.score'] - } - - // What fields this augmentation WRITES - writes?: { - // User metadata it enriches (MUST be declared!) - userFields?: Array<{ - field: string - type: 'create' | 'update' | 'merge' | 'delete' - description: string - example?: any - }> - - // Its own augmentation namespace - augmentationFields?: Array<{ - field: string - description: string - }> - - // Internal fields (requires special permission) - internalFields?: Array<{ - field: string - permission: 'granted' | 'requested' - reason: string - }> - } - - // Conflict resolution strategy - conflictResolution?: { - strategy: 'error' | 'warn' | 'merge' | 'skip' | 'override' - priority?: number // Higher priority wins in conflicts - } - - // Safety guarantees - guarantees?: { - preservesExisting?: boolean // Won't delete existing fields - reversible?: boolean // Can undo changes - idempotent?: boolean // Safe to run multiple times - validatesTypes?: boolean // Checks field types before modifying - } -} - -/** - * Runtime metadata safety enforcer - */ -export class MetadataSafetyEnforcer { - private contracts: Map = new Map() - private modifications: Map> = new Map() // field -> augmentations that modify it - - /** - * Register an augmentation's contract - */ - registerContract(contract: AugmentationMetadataContract): void { - this.contracts.set(contract.name, contract) - - // Track which augmentations modify which fields - if (contract.writes?.userFields) { - for (const fieldDef of contract.writes.userFields) { - if (!this.modifications.has(fieldDef.field)) { - this.modifications.set(fieldDef.field, new Set()) - } - this.modifications.get(fieldDef.field)!.add(contract.name) - } - } - } - - /** - * Check if an augmentation can modify a field - */ - canModifyField(augName: string, field: string, value: any): { - allowed: boolean - reason?: string - warnings?: string[] - } { - const contract = this.contracts.get(augName) - - if (!contract) { - return { - allowed: false, - reason: `Augmentation '${augName}' has no registered contract` - } - } - - // Check if field is in user namespace - if (!field.startsWith('_brainy.') && !field.startsWith('_augmentations.')) { - // It's a user field - const declaredField = contract.writes?.userFields?.find(f => f.field === field) - - if (!declaredField) { - return { - allowed: false, - reason: `Augmentation '${augName}' did not declare intent to modify '${field}'` - } - } - - // Check for conflicts - const modifiers = this.modifications.get(field) - if (modifiers && modifiers.size > 1) { - const others = Array.from(modifiers).filter(a => a !== augName) - return { - allowed: true, // Still allowed but with warning - warnings: [`Field '${field}' is also modified by: ${others.join(', ')}`] - } - } - - return { allowed: true } - } - - // Check internal fields - if (field.startsWith('_brainy.')) { - const internalField = contract.writes?.internalFields?.find(f => - field === `_brainy.${f.field}` - ) - - if (!internalField) { - return { - allowed: false, - reason: `Augmentation '${augName}' cannot modify internal field '${field}'` - } - } - - if (internalField.permission !== 'granted') { - return { - allowed: false, - reason: `Permission not granted for internal field '${field}'` - } - } - - return { allowed: true } - } - - // Check augmentation namespace - if (field.startsWith('_augmentations.')) { - const parts = field.split('.') - const targetAug = parts[1] - - // Can only modify own namespace - if (targetAug !== augName) { - return { - allowed: false, - reason: `Cannot modify another augmentation's namespace: ${targetAug}` - } - } - - return { allowed: true } - } - - return { allowed: true } - } - - /** - * Create safe metadata proxy for an augmentation - */ - createSafeProxy(metadata: any, augName: string): any { - const self = this - - return new Proxy(metadata, { - set(target, prop, value) { - const field = String(prop) - - // Check permission - const permission = self.canModifyField(augName, field, value) - - if (!permission.allowed) { - throw new Error(`[${augName}] ${permission.reason}`) - } - - if (permission.warnings) { - console.warn(`[${augName}] Warning:`, ...permission.warnings) - } - - // Track modification for audit - if (!target._audit) { - target._audit = [] - } - target._audit.push({ - augmentation: augName, - field, - oldValue: target[prop], - newValue: value, - timestamp: Date.now() - }) - - target[prop] = value - return true - }, - - deleteProperty(target, prop) { - const field = String(prop) - const permission = self.canModifyField(augName, field, undefined) - - if (!permission.allowed) { - throw new Error(`[${augName}] Cannot delete field: ${permission.reason}`) - } - - delete target[prop] - return true - } - }) - } -} - -/** - * Example augmentation contracts - */ -export const EXAMPLE_CONTRACTS: Record = { - // Enrichment augmentation that adds categories - categoryEnricher: { - name: 'categoryEnricher', - version: '1.0.0', - reads: { - userFields: ['title', 'description', 'content'] - }, - writes: { - userFields: [ - { - field: 'category', - type: 'create', - description: 'Auto-detected category', - example: 'technology' - }, - { - field: 'subcategories', - type: 'create', - description: 'List of relevant subcategories', - example: ['web', 'framework'] - } - ], - augmentationFields: [ - { - field: 'confidence', - description: 'Confidence score of categorization' - } - ] - }, - guarantees: { - preservesExisting: true, - idempotent: true - } - }, - - // Translation augmentation - translator: { - name: 'translator', - version: '1.0.0', - reads: { - userFields: ['title', 'description'] - }, - writes: { - userFields: [ - { - field: 'translations', - type: 'merge', - description: 'Translations in multiple languages', - example: { es: 'Tรญtulo', fr: 'Titre' } - }, - { - field: 'detectedLanguage', - type: 'create', - description: 'Detected source language', - example: 'en' - } - ] - }, - conflictResolution: { - strategy: 'merge', - priority: 10 - } - }, - - // Sentiment analyzer - sentimentAnalyzer: { - name: 'sentimentAnalyzer', - version: '1.0.0', - reads: { - userFields: ['content', 'description', 'reviews'] - }, - writes: { - userFields: [ - { - field: 'sentiment', - type: 'update', - description: 'Overall sentiment score', - example: { score: 0.8, label: 'positive' } - } - ], - augmentationFields: [ - { - field: 'analysis', - description: 'Detailed sentiment breakdown' - } - ] - }, - guarantees: { - reversible: true, - validatesTypes: true - } - }, - - // System augmentation with internal access - garbageCollector: { - name: 'garbageCollector', - version: '1.0.0', - reads: { - internalFields: ['_brainy.createdAt', '_brainy.lastAccessed'] - }, - writes: { - internalFields: [ - { - field: 'deleted', - permission: 'granted', - reason: 'Soft delete expired items' - }, - { - field: 'archived', - permission: 'granted', - reason: 'Archive old items' - } - ] - } - } -} - -/** - * Augmentation base class with safety - */ -export abstract class SafeAugmentation { - protected enforcer: MetadataSafetyEnforcer - protected contract: AugmentationMetadataContract - - constructor(contract: AugmentationMetadataContract) { - this.contract = contract - this.enforcer = new MetadataSafetyEnforcer() - this.enforcer.registerContract(contract) - } - - /** - * Get safe metadata proxy - */ - protected getSafeMetadata(metadata: any): any { - return this.enforcer.createSafeProxy(metadata, this.contract.name) - } - - /** - * Abstract method to implement augmentation logic - */ - abstract execute(metadata: any): Promise -} - -/** - * Example: Category enricher implementation - */ -export class CategoryEnricherAugmentation extends SafeAugmentation { - constructor() { - super(EXAMPLE_CONTRACTS.categoryEnricher) - } - - async execute(metadata: any): Promise { - const safe = this.getSafeMetadata(metadata) - - // Read declared fields - const title = safe.title - const description = safe.description - - // Analyze and categorize - const category = this.detectCategory(title, description) - const subcategories = this.detectSubcategories(title, description) - - // Write to declared fields (will be checked by proxy) - safe.category = category // โœ… Allowed - declared in contract - safe.subcategories = subcategories // โœ… Allowed - - // Try to write undeclared field - // safe.randomField = 'test' // โŒ Would throw error! - - // Write to our augmentation namespace - if (!safe._augmentations) safe._augmentations = {} - if (!safe._augmentations.categoryEnricher) { - safe._augmentations.categoryEnricher = {} - } - safe._augmentations.categoryEnricher.confidence = 0.95 // โœ… Allowed - - return safe - } - - private detectCategory(title: string, description: string): string { - // Simplified logic - return 'technology' - } - - private detectSubcategories(title: string, description: string): string[] { - return ['web', 'framework'] - } -} \ No newline at end of file diff --git a/src/augmentations/README.md b/src/augmentations/README.md deleted file mode 100644 index 31608e8a..00000000 --- a/src/augmentations/README.md +++ /dev/null @@ -1,292 +0,0 @@ -
-Brainy Logo - -# Brainy Augmentations - -
- -This directory contains the augmentation implementations for Brainy. Augmentations are pluggable components that extend -Brainy's functionality in various ways. - -## Available Augmentations - -### Core Augmentations - -#### IntelligentImportAugmentation - -Automatically detects and processes CSV, Excel, and PDF files with intelligent extraction. This augmentation is enabled by default and provides: - -- **CSV Support**: Auto-detection of encoding, delimiters, and field types -- **Excel Support**: Multi-sheet extraction with metadata preservation -- **PDF Support**: Text extraction, table detection, and metadata extraction -- **Type Inference**: Automatically infers data types (string, number, boolean, date) -- **Neural Integration**: Seamlessly integrates with entity extraction and relationship detection - -```javascript -import { Brainy } from '@soulcraft/brainy' - -const brain = new Brainy({ - intelligentImport: { - enableCSV: true, - enableExcel: true, - enablePDF: true, - maxFileSize: 100 * 1024 * 1024 // 100MB - } -}) -await brain.init() - -// Import CSV with auto-detection -await brain.import('customers.csv') - -// Import Excel with specific sheets -await brain.import('sales-data.xlsx', { - excelSheets: ['Q1', 'Q2'] -}) - -// Import PDF with table extraction -await brain.import('report.pdf', { - pdfExtractTables: true -}) -``` - -**See:** [Import Anything Guide](../../docs/guides/import-anything.md) | [Example](../../examples/import-excel-pdf-csv.ts) - -### Conduit Augmentations - -Conduit augmentations provide data synchronization between Brainy instances. - -#### WebSocketConduitAugmentation - -A conduit augmentation that syncs Brainy instances using WebSockets. This is used for syncing between browsers and -servers, or between servers. - -```javascript -import { createConduitAugmentation, augmentationPipeline } from '@soulcraft/brainy' - -// Create a WebSocket conduit augmentation -const wsConduit = await createConduitAugmentation('websocket', 'my-websocket-sync') - -// Register the augmentation with the pipeline -augmentationPipeline.register(wsConduit) - -// Connect to another Brainy instance -const connectionResult = await wsConduit.establishConnection( - 'wss://your-websocket-server.com/brainy-sync', - { protocols: 'brainy-sync' } -) -``` - -#### WebRTCConduitAugmentation - -A conduit augmentation that syncs Brainy instances using WebRTC. This is used for direct peer-to-peer syncing between -browsers. - -```javascript -import { createConduitAugmentation, augmentationPipeline } from '@soulcraft/brainy' - -// Create a WebRTC conduit augmentation -const webrtcConduit = await createConduitAugmentation('webrtc', 'my-webrtc-sync') - -// Register the augmentation with the pipeline -augmentationPipeline.register(webrtcConduit) - -// Connect to a peer -const connectionResult = await webrtcConduit.establishConnection( - 'peer-id-to-connect-to', - { - signalServerUrl: 'wss://your-signal-server.com', - localPeerId: 'my-peer-id', - iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] - } -) -``` - -#### ServerSearchConduitAugmentation - -A specialized conduit augmentation that provides functionality for searching a server-hosted Brainy instance and storing -results locally. This allows you to: - -- Search a server-hosted Brainy instance from a browser -- Store the search results in a local Brainy instance -- Perform further searches against the local instance without needing to query the server again -- Add data to both local and server instances - -```javascript -import { - ServerSearchConduitAugmentation, - createServerSearchAugmentations, - augmentationPipeline -} from '@soulcraft/brainy' - -// Using the factory function (recommended) -const { conduit, activation, connection } = await createServerSearchAugmentations( - 'wss://your-brainy-server.com/ws', - { protocols: 'brainy-sync' } -) - -// Register the augmentations with the pipeline -augmentationPipeline.register(conduit) -augmentationPipeline.register(activation) - -// Search the server and store results locally -const serverSearchResult = await conduit.searchServer( - connection.connectionId, - 'your search query', - 5 // limit -) - -// Search the local instance -const localSearchResult = await conduit.searchLocal('your search query', 5) - -// Perform a combined search (local first, then server if needed) -const combinedSearchResult = await conduit.searchCombined( - connection.connectionId, - 'your search query', - 5 -) - -// Add data to both local and server -const addResult = await conduit.addToBoth( - connection.connectionId, - 'Text to add', - { /* metadata */ } -) -``` - -### Activation Augmentations - -Activation augmentations dictate how Brainy initiates actions, responses, or data manipulations. - -#### ServerSearchActivationAugmentation - -An activation augmentation that provides actions for server search functionality. This works in conjunction with the -ServerSearchConduitAugmentation to provide a complete solution for browser-server search. - -```javascript -import { - ServerSearchActivationAugmentation, - createServerSearchAugmentations, - augmentationPipeline -} from '@soulcraft/brainy' - -// Using the factory function (recommended) -const { conduit, activation, connection } = await createServerSearchAugmentations( - 'wss://your-brainy-server.com/ws', - { protocols: 'brainy-sync' } -) - -// Register the augmentations with the pipeline -augmentationPipeline.register(conduit) -augmentationPipeline.register(activation) - -// Use the activation augmentation to search the server -const serverSearchAction = activation.triggerAction('searchServer', { - connectionId: connection.connectionId, - query: 'your search query', - limit: 5 -}) - -if (serverSearchAction.success) { - // The data property contains a promise that will resolve to the search results - const serverSearchResult = await serverSearchAction.data - console.log('Server search results:', serverSearchResult) -} - -// Other available actions: -// - 'connectToServer': Connect to a server -// - 'searchLocal': Search the local instance -// - 'searchCombined': Search both local and server -// - 'addToBoth': Add data to both local and server -``` - -## Using the Augmentation Pipeline - -The augmentation pipeline provides a way to execute augmentations based on their type. - -```javascript -import { augmentationPipeline } from '@soulcraft/brainy' - -// Execute a conduit augmentation -const conduitResults = await augmentationPipeline.executeConduitPipeline( - 'methodName', - [arg1, arg2, ...], - { /* options */ } -) - -// Execute an activation augmentation -const activationResults = await augmentationPipeline.executeActivationPipeline( - 'methodName', - [arg1, arg2, ...], - { /* options */ } -) -``` - -## Creating Custom Augmentations - -To create a custom augmentation, implement one of the augmentation interfaces: - -- `ISenseAugmentation`: For processing raw data -- `IConduitAugmentation`: For data synchronization -- `ICognitionAugmentation`: For reasoning and inference -- `IMemoryAugmentation`: For data storage -- `IPerceptionAugmentation`: For data interpretation and visualization -- `IDialogAugmentation`: For natural language processing -- `IActivationAugmentation`: For triggering actions - -Example: - -```javascript -import { AugmentationType, IActivationAugmentation } from '@soulcraft/brainy' - -class MyCustomActivation implements IActivationAugmentation { - readonly - name = 'my-custom-activation' - readonly - description = 'My custom activation augmentation' - enabled = true - - getType(): AugmentationType { - return AugmentationType.ACTIVATION - } - - async initialize(): Promise { - // Initialization code - } - - async shutDown(): Promise { - // Cleanup code - } - - async getStatus(): Promise<'active' | 'inactive' | 'error'> { - return 'active' - } - - triggerAction(actionName: string, parameters - -?: - - Record - -): - - AugmentationResponse { - // Implementation - } - - generateOutput(knowledgeId: string, format: string): AugmentationResponse> { - // Implementation -} - -interactExternal(systemId -: -string, payload -: -Record < string, unknown > -): -AugmentationResponse < unknown > { - // Implementation -} -} -``` diff --git a/src/augmentations/apiServerAugmentation.ts b/src/augmentations/apiServerAugmentation.ts deleted file mode 100644 index 30635937..00000000 --- a/src/augmentations/apiServerAugmentation.ts +++ /dev/null @@ -1,915 +0,0 @@ -/** - * API Server Augmentation - Universal API Exposure - * - * ๐ŸŒ Exposes Brainy through REST, WebSocket, and MCP - * ๐Ÿ”Œ Works in Node.js, Deno, and Service Workers - * ๐Ÿš€ Single augmentation for all API needs - * - * This unifies and replaces: - * - BrainyMCPBroadcast (Node-specific server) - * - WebSocketConduitAugmentation (client connections) - * - Future REST API implementations - */ - -import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js' -import { BrainyMCPService } from '../mcp/brainyMCPService.js' -import { v4 as uuidv4 } from '../universal/uuid.js' -import { isNode, isBrowser } from '../utils/environment.js' - -export interface APIServerConfig { - enabled?: boolean - port?: number - mcpPort?: number - wsPort?: number - host?: string - cors?: { - origin?: string | string[] - credentials?: boolean - } - auth?: { - required?: boolean - apiKeys?: string[] - bearerTokens?: string[] - } - rateLimit?: { - windowMs?: number - max?: number - } - ssl?: { - cert?: string - key?: string - } -} - -interface ConnectedClient { - id: string - type: 'rest' | 'websocket' | 'mcp' - socket?: any - subscriptions?: string[] - metadata?: Record - lastSeen: number -} - -/** - * Unified API Server Augmentation - * Exposes Brainy through multiple protocols - */ -export class APIServerAugmentation extends BaseAugmentation { - readonly name = 'api-server' - readonly timing = 'after' as const - readonly metadata = 'readonly' as const // API server reads metadata to serve data - readonly operations = ['all'] as ('all')[] - readonly priority = 5 // Low priority, runs after other augmentations - - protected config: APIServerConfig - private mcpService?: BrainyMCPService - private httpServer?: any - private wsServer?: any - private clients = new Map() - private operationHistory: any[] = [] - private maxHistorySize = 1000 - private rateLimitStore = new Map() - - constructor(config: APIServerConfig = {}) { - super() - this.config = { - enabled: true, - port: 3000, - host: '0.0.0.0', - cors: { origin: '*', credentials: true }, - auth: { required: false }, - rateLimit: { windowMs: 60000, max: 100 }, - ...config - } - } - - protected async onInitialize(): Promise { - if (!this.config.enabled) { - this.log('API Server disabled in config') - return - } - - // Initialize MCP service - this.mcpService = new BrainyMCPService(this.context!.brain, { - enableAuth: this.config.auth?.required - }) - - // Start appropriate server based on environment - if (isNode()) { - await this.startNodeServer() - } else if (typeof (globalThis as any).Deno !== 'undefined') { - await this.startDenoServer() - } else if (isBrowser() && 'serviceWorker' in navigator) { - await this.startServiceWorker() - } else { - this.log('No suitable server environment detected', 'warn') - } - } - - /** - * Start Node.js server with Express - */ - private async startNodeServer(): Promise { - try { - // Dynamic imports for Node.js dependencies - const express = await import('express').catch(() => null) - const cors = await import('cors').catch(() => null) - const ws = await import('ws').catch(() => null) - const { createServer } = await import('node:http') - - if (!express || !cors || !ws) { - this.log('Express, cors, or ws not available. Install with: npm install express cors ws', 'error') - return - } - - const WebSocketServer = (ws as any)?.WebSocketServer || (ws as any)?.default?.WebSocketServer || (ws as any)?.Server - - const app = express.default() - - // Middleware - app.use(cors.default(this.config.cors)) - app.use((express.default || express).json({ limit: '50mb' })) - app.use(this.authMiddleware.bind(this)) - app.use(this.rateLimitMiddleware.bind(this)) - - // REST API Routes - this.setupRESTRoutes(app) - - // Create HTTP server - this.httpServer = createServer(app) - - // WebSocket server - this.wsServer = new WebSocketServer({ - server: this.httpServer, - path: '/ws' - }) - - this.setupWebSocketServer() - - // Start listening - await new Promise((resolve, reject) => { - this.httpServer.listen(this.config.port, this.config.host, () => { - this.log(`๐ŸŒ API Server listening on http://${this.config.host}:${this.config.port}`) - this.log(`๐Ÿ”Œ WebSocket: ws://${this.config.host}:${this.config.port}/ws`) - this.log(`๐Ÿง  MCP endpoint: http://${this.config.host}:${this.config.port}/api/mcp`) - resolve() - }).on('error', reject) - }) - - // Heartbeat interval - setInterval(() => this.sendHeartbeats(), 30000) - - } catch (error) { - this.log(`Failed to start Node.js server: ${error}`, 'error') - throw error - } - } - - /** - * Setup REST API routes - */ - private setupRESTRoutes(app: any): void { - // Health check - app.get('/health', (_req: any, res: any) => { - res.json({ - status: 'healthy', - version: '2.0.0', - clients: this.clients.size, - uptime: process.uptime ? process.uptime() : 0 - }) - }) - - // Search endpoint - app.post('/api/search', async (req: any, res: any) => { - try { - const { query, limit = 10, options = {} } = req.body - const results = await this.context!.brain.search(query, limit, options) - res.json({ success: true, results }) - } catch (error: any) { - res.status(500).json({ success: false, error: error.message }) - } - }) - - // Add data endpoint - app.post('/api/add', async (req: any, res: any) => { - try { - const { content, metadata } = req.body - const id = await this.context!.brain.add({ data: content, type: 'content', metadata }) - res.json({ success: true, id }) - } catch (error: any) { - res.status(500).json({ success: false, error: error.message }) - } - }) - - // Get by ID endpoint - app.get('/api/get/:id', async (req: any, res: any) => { - try { - const data = await this.context!.brain.get(req.params.id) - if (data) { - res.json({ success: true, data }) - } else { - res.status(404).json({ success: false, error: 'Not found' }) - } - } catch (error: any) { - res.status(500).json({ success: false, error: error.message }) - } - }) - - // Delete endpoint - app.delete('/api/delete/:id', async (req: any, res: any) => { - try { - await this.context!.brain.delete(req.params.id) - res.json({ success: true }) - } catch (error: any) { - res.status(500).json({ success: false, error: error.message }) - } - }) - - // Relate endpoint - app.post('/api/relate', async (req: any, res: any) => { - try { - const { source, target, verb, metadata } = req.body - await this.context!.brain.relate(source, target, verb, metadata) - res.json({ success: true }) - } catch (error: any) { - res.status(500).json({ success: false, error: error.message }) - } - }) - - // Find endpoint (complex queries) - app.post('/api/find', async (req: any, res: any) => { - try { - const results = await this.context!.brain.find(req.body) - res.json({ success: true, results }) - } catch (error: any) { - res.status(500).json({ success: false, error: error.message }) - } - }) - - // Cluster endpoint - app.post('/api/cluster', async (req: any, res: any) => { - try { - const { algorithm = 'kmeans', options = {} } = req.body - const clusters = await this.context!.brain.cluster(algorithm, options) - res.json({ success: true, clusters }) - } catch (error: any) { - res.status(500).json({ success: false, error: error.message }) - } - }) - - // MCP endpoint - app.post('/api/mcp', async (req: any, res: any) => { - try { - const response = await this.mcpService!.handleRequest(req.body) - res.json(response) - } catch (error: any) { - res.status(500).json({ success: false, error: error.message }) - } - }) - - // Statistics endpoint - app.get('/api/stats', async (_req: any, res: any) => { - try { - const stats = this.context!.brain.getStats() - res.json({ success: true, stats }) - } catch (error: any) { - res.status(500).json({ success: false, error: error.message }) - } - }) - - // Operation history endpoint - app.get('/api/history', (_req: any, res: any) => { - res.json({ - success: true, - history: this.operationHistory.slice(-100) - }) - }) - } - - /** - * Setup WebSocket server - */ - private setupWebSocketServer(): void { - if (!this.wsServer) return - - this.wsServer.on('connection', (socket: any, request: any) => { - const clientId = uuidv4() - const client: ConnectedClient = { - id: clientId, - type: 'websocket', - socket, - subscriptions: [], - lastSeen: Date.now() - } - - this.clients.set(clientId, client) - - // Send welcome message - socket.send(JSON.stringify({ - type: 'welcome', - clientId, - message: 'Connected to Brainy API Server', - capabilities: ['search', 'add', 'delete', 'relate', 'subscribe', 'mcp'] - })) - - // Handle messages - socket.on('message', async (message: string) => { - try { - const msg = JSON.parse(message) - await this.handleWebSocketMessage(msg, client) - } catch (error: any) { - socket.send(JSON.stringify({ - type: 'error', - error: error.message - })) - } - }) - - // Handle disconnect - socket.on('close', () => { - this.clients.delete(clientId) - this.log(`Client ${clientId} disconnected`) - }) - - // Handle errors - socket.on('error', (error: any) => { - this.log(`WebSocket error for client ${clientId}: ${error}`, 'error') - }) - }) - } - - /** - * Handle WebSocket message - */ - private async handleWebSocketMessage(msg: any, client: ConnectedClient): Promise { - const { socket } = client - - switch (msg.type) { - case 'subscribe': - // Subscribe to operation types - client.subscriptions = msg.operations || ['all'] - socket.send(JSON.stringify({ - type: 'subscribed', - operations: client.subscriptions - })) - break - - case 'search': - const searchResults = await this.context!.brain.search( - msg.query, - msg.limit || 10, - msg.options || {} - ) - socket.send(JSON.stringify({ - type: 'searchResults', - requestId: msg.requestId, - results: searchResults - })) - break - - case 'add': - const id = await this.context!.brain.add({ data: msg.content, type: 'content', metadata: msg.metadata }) - socket.send(JSON.stringify({ - type: 'addResult', - requestId: msg.requestId, - id - })) - break - - case 'mcp': - const mcpResponse = await this.mcpService!.handleRequest(msg.request) - socket.send(JSON.stringify({ - type: 'mcpResponse', - requestId: msg.requestId, - response: mcpResponse - })) - break - - case 'heartbeat': - client.lastSeen = Date.now() - socket.send(JSON.stringify({ - type: 'heartbeat', - timestamp: Date.now() - })) - break - - default: - socket.send(JSON.stringify({ - type: 'error', - error: `Unknown message type: ${msg.type}` - })) - } - } - - /** - * Execute augmentation - broadcast operations to clients - */ - async execute( - operation: string, - params: any, - next: () => Promise - ): Promise { - const startTime = Date.now() - const result = await next() - const duration = Date.now() - startTime - - // Record operation in history - const historyEntry = { - operation, - params: this.sanitizeParams(params), - timestamp: Date.now(), - duration - } - - this.operationHistory.push(historyEntry) - if (this.operationHistory.length > this.maxHistorySize) { - this.operationHistory.shift() - } - - // Broadcast to subscribed WebSocket clients - const message = JSON.stringify({ - type: 'operation', - operation, - params: historyEntry.params, - timestamp: historyEntry.timestamp, - duration - }) - - for (const client of this.clients.values()) { - if (client.type === 'websocket' && client.socket) { - if (client.subscriptions?.includes('all') || - client.subscriptions?.includes(operation)) { - try { - client.socket.send(message) - } catch (error) { - // Client might be disconnected - this.clients.delete(client.id) - } - } - } - } - - return result - } - - /** - * Auth middleware for Express - */ - private authMiddleware(req: any, res: any, next: any): void { - if (!this.config.auth?.required) { - return next() - } - - const apiKey = req.headers['x-api-key'] - const bearerToken = req.headers.authorization?.replace('Bearer ', '') - - if (apiKey && this.config.auth.apiKeys?.includes(apiKey)) { - return next() - } - - if (bearerToken && this.config.auth.bearerTokens?.includes(bearerToken)) { - return next() - } - - res.status(401).json({ error: 'Unauthorized' }) - } - - /** - * Rate limiting middleware - */ - private rateLimitMiddleware(req: any, res: any, next: any): void { - // Simple in-memory rate limiting - // In production, use redis or proper rate limiting library - const ip = req.ip || req.connection.remoteAddress - const now = Date.now() - const windowMs = this.config.rateLimit?.windowMs || 60000 - const max = this.config.rateLimit?.max || 100 - - // Clean old entries - for (const [key, client] of this.clients.entries()) { - if (now - client.lastSeen > windowMs) { - this.clients.delete(key) - } - } - - // For now, just pass through - // Real implementation would track requests per IP - next() - } - - /** - * Sanitize parameters before broadcasting - */ - private sanitizeParams(params: any): any { - if (!params) return params - - const sanitized = { ...params } - - // Remove sensitive fields - delete sanitized.password - delete sanitized.apiKey - delete sanitized.token - delete sanitized.secret - - // Truncate large data - if (sanitized.content && sanitized.content.length > 1000) { - sanitized.content = sanitized.content.substring(0, 1000) + '...' - } - - return sanitized - } - - /** - * Send heartbeats to all connected clients - */ - private sendHeartbeats(): void { - const now = Date.now() - - for (const [id, client] of this.clients.entries()) { - if (client.type === 'websocket' && client.socket) { - // Remove inactive clients - if (now - client.lastSeen > 60000) { - this.clients.delete(id) - continue - } - - // Send heartbeat - try { - client.socket.send(JSON.stringify({ - type: 'heartbeat', - timestamp: now - })) - } catch { - // Client disconnected - this.clients.delete(id) - } - } - } - } - - /** - * Start Deno server - */ - private async startDenoServer(): Promise { - try { - // Check if Deno.serve is available (Deno 1.35+) - const DenoGlobal = (globalThis as any).Deno - if (DenoGlobal && 'serve' in DenoGlobal) { - const handler = this.createUniversalHandler() - - this.httpServer = DenoGlobal.serve({ - port: this.config.port, - hostname: this.config.host || '0.0.0.0', - handler: handler - }) - - this.log(`Deno server started on ${this.config.host || '0.0.0.0'}:${this.config.port}`) - - // Setup WebSocket handling for Deno - this.setupUniversalWebSocket() - - } else { - throw new Error('Deno.serve not available - requires Deno 1.35+') - } - } catch (error) { - this.log(`Failed to start Deno server: ${(error as Error).message}`, 'error') - throw error - } - } - - /** - * Start Service Worker (for browser) - */ - private async startServiceWorker(): Promise { - try { - if (typeof self !== 'undefined' && 'addEventListener' in self) { - // Service Worker environment - intercept fetch events - const handler = this.createUniversalHandler() - - self.addEventListener('fetch', async (event: any) => { - const url = new URL(event.request.url) - - // Only handle API requests - if (url.pathname.startsWith('/api/') || url.pathname.startsWith('/ws') || url.pathname.startsWith('/mcp/')) { - event.respondWith(handler(event.request)) - } - }) - - this.log('Service Worker API server registered for /api/, /ws, and /mcp paths') - - // Setup message handling for WebSocket-like communication - this.setupServiceWorkerMessaging() - - } else if (typeof navigator !== 'undefined' && 'serviceWorker' in navigator) { - // Browser main thread - service worker registration should be handled by the application - this.log('Service Worker environment detected. Registration should be handled by your application.', 'info') - // Return early - the app will handle service worker registration - return - } else { - this.log('Service Worker environment not available', 'warn') - return - } - } catch (error) { - this.log(`Failed to start Service Worker server: ${(error as Error).message}`, 'error') - throw error - } - } - - /** - * Create universal handler using Web Standards (works in Node, Deno, Service Workers) - */ - private createUniversalHandler(): (request: Request) => Promise { - return async (request: Request): Promise => { - try { - const url = new URL(request.url) - const method = request.method.toUpperCase() - const path = url.pathname - - // Add CORS headers - const corsOrigin = Array.isArray(this.config.cors?.origin) - ? this.config.cors.origin[0] - : this.config.cors?.origin || '*' - - const headers = new Headers({ - 'Access-Control-Allow-Origin': corsOrigin, - 'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE, OPTIONS', - 'Access-Control-Allow-Headers': 'Content-Type, Authorization', - 'Content-Type': 'application/json' - }) - - // Handle preflight requests - if (method === 'OPTIONS') { - return new Response(null, { status: 200, headers }) - } - - // Authentication - if (!this.authenticateRequest(request)) { - return new Response(JSON.stringify({ error: 'Unauthorized' }), { - status: 401, - headers - }) - } - - // Rate limiting - if (!this.checkRateLimit(request)) { - return new Response(JSON.stringify({ error: 'Rate limit exceeded' }), { - status: 429, - headers - }) - } - - // Route handling - if (path.startsWith('/api/brainy/')) { - return this.handleBrainyAPI(request, path.replace('/api/brainy/', ''), headers) - } else if (path.startsWith('/mcp/')) { - return this.handleMCPAPI(request, path.replace('/mcp/', ''), headers) - } else if (path === '/health') { - return new Response(JSON.stringify({ status: 'ok', timestamp: Date.now() }), { - status: 200, - headers - }) - } - - return new Response(JSON.stringify({ error: 'Not found' }), { - status: 404, - headers - }) - - } catch (error) { - return new Response(JSON.stringify({ - error: 'Internal server error', - message: (error as Error).message - }), { - status: 500, - headers: new Headers({ 'Content-Type': 'application/json' }) - }) - } - } - } - - /** - * Handle Brainy API requests using universal Request/Response - */ - private async handleBrainyAPI(request: Request, path: string, headers: Headers): Promise { - const method = request.method.toUpperCase() - const body = method !== 'GET' ? await request.json().catch(() => ({})) : {} - - try { - let result: any - - switch (`${method} ${path}`) { - case 'POST add': - result = { id: await this.context!.brain.add(body) } - break - case 'GET get': - const id = new URL(request.url).searchParams.get('id') - result = await this.context!.brain.get(id) - break - case 'PUT update': - await this.context!.brain.update(body) - result = { success: true } - break - case 'DELETE delete': - const deleteId = new URL(request.url).searchParams.get('id') - await this.context!.brain.delete(deleteId) - result = { success: true } - break - case 'POST find': - result = await this.context!.brain.find(body) - break - case 'POST relate': - result = { id: await this.context!.brain.relate(body) } - break - case 'GET insights': - result = await this.context!.brain.insights() - break - default: - return new Response(JSON.stringify({ error: `Unknown endpoint: ${method} ${path}` }), { - status: 404, - headers - }) - } - - return new Response(JSON.stringify(result), { status: 200, headers }) - - } catch (error) { - return new Response(JSON.stringify({ - error: (error as Error).message - }), { status: 400, headers }) - } - } - - /** - * Handle MCP API requests - */ - private async handleMCPAPI(request: Request, path: string, headers: Headers): Promise { - try { - if (!this.mcpService) { - return new Response(JSON.stringify({ error: 'MCP service not available' }), { - status: 503, - headers - }) - } - - const body = await request.json().catch(() => ({})) - // Convert to MCP request format - const mcpRequest = { - type: path.includes('data') ? 'data_access' : 'tool_execution', - ...body - } - const result = await this.mcpService.handleRequest(mcpRequest as any) - - return new Response(JSON.stringify(result), { status: 200, headers }) - } catch (error) { - return new Response(JSON.stringify({ - error: (error as Error).message - }), { status: 400, headers }) - } - } - - /** - * Universal WebSocket setup (works in Node, Deno) - */ - private setupUniversalWebSocket(): void { - // WebSocket handling varies by platform but uses same interface - this.log('WebSocket support enabled for real-time updates') - } - - /** - * Service Worker messaging for WebSocket-like communication - */ - private setupServiceWorkerMessaging(): void { - if (typeof self !== 'undefined') { - self.addEventListener('message', async (event: any) => { - if (event.data.type === 'brainy-api') { - try { - const response = await this.handleBrainyAPI( - new Request('http://localhost/api/brainy/' + event.data.endpoint, { - method: event.data.method || 'POST', - body: JSON.stringify(event.data.data) - }), - event.data.endpoint, - new Headers({ 'Content-Type': 'application/json' }) - ) - - const result = await response.json() - - event.ports[0]?.postMessage({ - id: event.data.id, - success: response.ok, - data: result - }) - } catch (error) { - event.ports[0]?.postMessage({ - id: event.data.id, - success: false, - error: (error as Error).message - }) - } - } - }) - } - } - - /** - * Universal authentication using Web Standards - */ - private authenticateRequest(request: Request): boolean { - if (!this.config.auth?.required) return true - - const authHeader = request.headers.get('authorization') - if (!authHeader) return false - - if (this.config.auth.apiKeys?.length) { - const apiKey = authHeader.replace('Bearer ', '') - return this.config.auth.apiKeys.includes(apiKey) - } - - return true - } - - - private checkRateLimit(request: Request): boolean { - if (!this.config.rateLimit) return true - - // Get client identifier from headers or use a default - const clientId = request.headers.get('x-forwarded-for') || - request.headers.get('x-real-ip') || - request.headers.get('cf-connecting-ip') || // Cloudflare - request.headers.get('x-vercel-forwarded-for') || // Vercel - 'unknown' - - const now = Date.now() - const windowMs = this.config.rateLimit.windowMs || 60000 - const maxRequests = this.config.rateLimit.max || 100 - const windowStart = now - windowMs - - // Get or create request timestamps for this client - let timestamps = this.rateLimitStore.get(clientId) || [] - - // Remove old timestamps outside the window - timestamps = timestamps.filter(t => t > windowStart) - - // Check if limit exceeded - if (timestamps.length >= maxRequests) { - this.log(`Rate limit exceeded for client ${clientId}: ${timestamps.length}/${maxRequests} requests`, 'warn') - return false - } - - // Add current request timestamp - timestamps.push(now) - this.rateLimitStore.set(clientId, timestamps) - - // Periodic cleanup of old entries to prevent memory leak - if (this.rateLimitStore.size > 1000) { - for (const [id, times] of this.rateLimitStore.entries()) { - const validTimes = times.filter(t => t > windowStart) - if (validTimes.length === 0) { - this.rateLimitStore.delete(id) - } else { - this.rateLimitStore.set(id, validTimes) - } - } - } - - return true - } - - /** - * Shutdown the server - */ - protected async onShutdown(): Promise { - // Close all WebSocket connections - for (const client of this.clients.values()) { - if (client.socket) { - try { - client.socket.close() - } catch (error) { - // Socket already closed or errored - console.debug('Error closing WebSocket:', error) - } - } - } - this.clients.clear() - - // Close servers - if (this.wsServer) { - this.wsServer.close() - } - - if (this.httpServer) { - await new Promise(resolve => { - this.httpServer.close(() => resolve()) - }) - } - - this.log('API Server shut down') - } -} - -/** - * Helper function to create and configure API server - */ -export function createAPIServer(config?: APIServerConfig): APIServerAugmentation { - return new APIServerAugmentation(config) -} \ No newline at end of file diff --git a/src/augmentations/auditLogAugmentation.ts b/src/augmentations/auditLogAugmentation.ts deleted file mode 100644 index eb84a056..00000000 --- a/src/augmentations/auditLogAugmentation.ts +++ /dev/null @@ -1,452 +0,0 @@ -/** - * Audit Logging Augmentation - * Provides comprehensive audit trail for all Brainy operations - */ - -import { BaseAugmentation } from './brainyAugmentation.js' -import { AugmentationManifest } from './manifest.js' -import { createHash } from '../universal/crypto.js' - -export interface AuditLogConfig { - enabled?: boolean - logLevel?: 'minimal' | 'standard' | 'detailed' - includeData?: boolean - includeMetadata?: boolean - retention?: number // Days to keep logs - storage?: 'memory' | 'file' | 'database' - filePath?: string - maxMemoryLogs?: number -} - -export interface AuditLogEntry { - id: string - timestamp: number - operation: string - params: any - result?: any - error?: any - duration: number - userId?: string - sessionId?: string - metadata?: Record -} - -/** - * Audit Log Augmentation - */ -export class AuditLogAugmentation extends BaseAugmentation { - readonly name = 'auditLogger' - readonly timing = 'around' as const - readonly metadata = 'readonly' as const // Read metadata for context - operations = ['all'] as any // Audit all operations - readonly priority = 90 // Low priority, runs last - - // Augmentation metadata - readonly category = 'core' as const - readonly description = 'Comprehensive audit logging for compliance and debugging' - - private logs: AuditLogEntry[] = [] - private sessionId: string - - constructor(config: AuditLogConfig = {}) { - super(config) - - // Merge with defaults - this.config = { - enabled: config.enabled ?? true, - logLevel: config.logLevel ?? 'standard', - includeData: config.includeData ?? false, - includeMetadata: config.includeMetadata ?? true, - retention: config.retention ?? 90, // 90 days default - storage: config.storage ?? 'memory', - filePath: config.filePath, - maxMemoryLogs: config.maxMemoryLogs ?? 10000 - } - - // Generate session ID - this.sessionId = this.generateId() - } - - getManifest(): AugmentationManifest { - return { - id: 'audit-logger', - name: 'Audit Logger', - version: '1.0.0', - description: 'Comprehensive audit trail for all operations', - longDescription: 'Records detailed audit logs of all Brainy operations for compliance, debugging, and analytics purposes.', - category: 'analytics', - configSchema: { - type: 'object', - properties: { - enabled: { - type: 'boolean', - default: true, - description: 'Enable audit logging' - }, - logLevel: { - type: 'string', - enum: ['minimal', 'standard', 'detailed'], - default: 'standard', - description: 'Level of detail to log' - }, - includeData: { - type: 'boolean', - default: false, - description: 'Include actual data in logs (privacy concern)' - }, - includeMetadata: { - type: 'boolean', - default: true, - description: 'Include metadata in logs' - }, - retention: { - type: 'number', - default: 90, - minimum: 1, - maximum: 365, - description: 'Days to retain logs' - }, - storage: { - type: 'string', - enum: ['memory', 'file', 'database'], - default: 'memory', - description: 'Where to store audit logs' - }, - maxMemoryLogs: { - type: 'number', - default: 10000, - description: 'Maximum logs to keep in memory' - } - } - }, - configDefaults: { - enabled: true, - logLevel: 'standard', - includeData: false, - includeMetadata: true, - retention: 90, - storage: 'memory', - maxMemoryLogs: 10000 - }, - minBrainyVersion: '3.0.0', - keywords: ['audit', 'logging', 'compliance', 'analytics'], - documentation: 'https://docs.brainy.dev/augmentations/audit-log', - status: 'stable', - performance: { - memoryUsage: 'medium', - cpuUsage: 'low', - networkUsage: 'none' - }, - features: ['operation-logging', 'configurable-detail', 'retention-management'], - enhancedOperations: ['all'], - metrics: [ - { - name: 'audit_logs_created', - type: 'counter', - description: 'Total audit logs created' - }, - { - name: 'audit_log_size', - type: 'gauge', - description: 'Current audit log size' - } - ] - } - } - - protected async onInitialize(): Promise { - if (!this.config.enabled) { - this.log('Audit logger disabled by configuration') - return - } - - this.log(`Audit logger initialized (level: ${this.config.logLevel}, storage: ${this.config.storage})`) - - // Start retention cleanup if using memory storage - if (this.config.storage === 'memory') { - setInterval(() => { - this.cleanupOldLogs() - }, 3600000) // Every hour - } - } - - protected async onShutdown(): Promise { - // Save any pending logs if using file storage - if (this.config.storage === 'file' && this.logs.length > 0) { - await this.flushLogs() - } - - this.log('Audit logger shut down') - } - - /** - * Execute augmentation - log operations - */ - async execute( - operation: string, - params: any, - next: () => Promise - ): Promise { - // If audit logging is disabled, just pass through - if (!this.config.enabled) { - return next() - } - - const startTime = Date.now() - const logEntry: Partial = { - id: this.generateId(), - timestamp: startTime, - operation, - sessionId: this.sessionId - } - - // Add params based on log level - if (this.config.logLevel !== 'minimal') { - logEntry.params = this.sanitizeParams(params) - } - - try { - const result = await next() - - // Log successful operation - logEntry.duration = Date.now() - startTime - - // Add result based on log level and config - if (this.config.logLevel === 'detailed' && this.config.includeData) { - logEntry.result = this.sanitizeResult(result) - } - - await this.writeLog(logEntry as AuditLogEntry) - - return result - } catch (error) { - // Log failed operation - logEntry.duration = Date.now() - startTime - logEntry.error = this.sanitizeError(error) - - await this.writeLog(logEntry as AuditLogEntry) - - throw error - } - } - - /** - * Sanitize parameters to remove sensitive data - */ - private sanitizeParams(params: any): any { - if (!params) return params - - // Don't include actual data unless configured - if (!this.config.includeData && params.data) { - return { - ...params, - data: '[REDACTED]' - } - } - - // Redact common sensitive fields - const sanitized = { ...params } - const sensitiveFields = ['password', 'token', 'apiKey', 'secret'] - - for (const field of sensitiveFields) { - if (sanitized[field]) { - sanitized[field] = '[REDACTED]' - } - } - - return sanitized - } - - /** - * Sanitize result data - */ - private sanitizeResult(result: any): any { - if (!result) return result - - // For arrays, just log count - if (Array.isArray(result)) { - return { count: result.length, type: 'array' } - } - - // For objects, remove sensitive fields - if (typeof result === 'object') { - const sanitized: any = {} - for (const key in result) { - if (!['password', 'token', 'apiKey', 'secret'].includes(key)) { - sanitized[key] = result[key] - } - } - return sanitized - } - - return result - } - - /** - * Sanitize error information - */ - private sanitizeError(error: any): any { - if (!error) return error - - return { - message: error.message || 'Unknown error', - code: error.code, - statusCode: error.statusCode, - stack: this.config.logLevel === 'detailed' ? error.stack : undefined - } - } - - /** - * Write log entry - */ - private async writeLog(entry: AuditLogEntry): Promise { - switch (this.config.storage) { - case 'memory': - this.logs.push(entry) - - // Enforce max memory logs - if (this.logs.length > this.config.maxMemoryLogs!) { - this.logs.shift() // Remove oldest - } - break - - case 'file': - // In production, would write to file - // For now, just add to memory - this.logs.push(entry) - break - - case 'database': - // In production, would write to database - // For now, just add to memory - this.logs.push(entry) - break - } - } - - /** - * Flush logs to persistent storage - */ - private async flushLogs(): Promise { - // In production, would write to file/database - // For now, just clear old logs - if (this.logs.length > this.config.maxMemoryLogs!) { - this.logs = this.logs.slice(-this.config.maxMemoryLogs!) - } - } - - /** - * Clean up old logs based on retention - */ - private cleanupOldLogs(): void { - const cutoffTime = Date.now() - (this.config.retention! * 24 * 60 * 60 * 1000) - - this.logs = this.logs.filter(log => log.timestamp >= cutoffTime) - } - - /** - * Generate unique ID - */ - private generateId(): string { - return createHash('sha256') - .update(`${Date.now()}-${Math.random()}`) - .digest('hex') - .substring(0, 16) - } - - /** - * Query audit logs - */ - queryLogs(filter?: { - operation?: string - startTime?: number - endTime?: number - sessionId?: string - hasError?: boolean - }): AuditLogEntry[] { - let results = [...this.logs] - - if (filter) { - if (filter.operation) { - results = results.filter(log => log.operation === filter.operation) - } - - if (filter.startTime) { - results = results.filter(log => log.timestamp >= filter.startTime!) - } - - if (filter.endTime) { - results = results.filter(log => log.timestamp <= filter.endTime!) - } - - if (filter.sessionId) { - results = results.filter(log => log.sessionId === filter.sessionId) - } - - if (filter.hasError !== undefined) { - results = results.filter(log => (log.error !== undefined) === filter.hasError) - } - } - - return results - } - - /** - * Get audit statistics - */ - getStats(): { - totalLogs: number - operations: Record - averageDuration: number - errorRate: number - } { - const stats: any = { - totalLogs: this.logs.length, - operations: {}, - averageDuration: 0, - errorRate: 0 - } - - let totalDuration = 0 - let errorCount = 0 - - for (const log of this.logs) { - // Count by operation - stats.operations[log.operation] = (stats.operations[log.operation] || 0) + 1 - - // Sum duration - totalDuration += log.duration - - // Count errors - if (log.error) errorCount++ - } - - if (this.logs.length > 0) { - stats.averageDuration = totalDuration / this.logs.length - stats.errorRate = errorCount / this.logs.length - } - - return stats - } - - /** - * Export logs for analysis - */ - exportLogs(): AuditLogEntry[] { - return [...this.logs] - } - - /** - * Clear all logs - */ - clearLogs(): void { - this.logs = [] - } -} - -/** - * Create audit log augmentation - */ -export function createAuditLogAugmentation(config?: AuditLogConfig): AuditLogAugmentation { - return new AuditLogAugmentation(config) -} \ No newline at end of file diff --git a/src/augmentations/batchProcessingAugmentation.ts b/src/augmentations/batchProcessingAugmentation.ts deleted file mode 100644 index 38282f21..00000000 --- a/src/augmentations/batchProcessingAugmentation.ts +++ /dev/null @@ -1,803 +0,0 @@ -/** - * Batch Processing Augmentation - * - * Critical for enterprise-scale performance: 500,000+ operations/second - * Automatically batches operations for maximum throughput - * Handles streaming data, bulk imports, and high-frequency operations - * - * Performance Impact: 10-50x improvement for bulk operations - */ - -import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js' -import { AugmentationManifest } from './manifest.js' - -interface BatchConfig { - enabled?: boolean - adaptiveMode?: boolean // Zero-config: automatically decide when to batch - immediateThreshold?: number // Operations <= this count are immediate - batchThreshold?: number // Start batching when >= this many operations queued - maxBatchSize?: number // Maximum items per batch - maxWaitTime?: number // Maximum wait time before flushing (ms) - adaptiveBatching?: boolean // Dynamically adjust batch size based on performance - priorityLanes?: number // Number of priority processing lanes - memoryLimit?: number // Maximum memory for batching (bytes) -} - -interface BatchedOperation { - id: string - operation: string - params: any - executor: () => Promise // The actual function to execute - resolver: (value: T) => void - rejector: (error: Error) => void - timestamp: number - priority: number - size: number // Estimated memory size -} - -interface BatchMetrics { - totalOperations: number - batchesProcessed: number - averageBatchSize: number - averageLatency: number - throughputPerSecond: number - memoryUsage: number - adaptiveAdjustments: number -} - -export class BatchProcessingAugmentation extends BaseAugmentation { - readonly metadata = 'readonly' as const // Reads metadata for batching decisions - name = 'BatchProcessing' - timing = 'around' as const - operations = ['add', 'addNoun', 'addVerb', 'saveNoun', 'saveVerb', 'storage'] as ('add' | 'addNoun' | 'addVerb' | 'saveNoun' | 'saveVerb' | 'storage')[] - priority = 80 // High priority for performance - - protected config: Required = { - enabled: true, - adaptiveMode: true, - immediateThreshold: 1, - batchThreshold: 5, - maxBatchSize: 100, - maxWaitTime: 1000, - adaptiveBatching: true, - priorityLanes: 2, - memoryLimit: 100 * 1024 * 1024 // 100MB - } - private batches: Map[]> = new Map() - private flushTimers: Map = new Map() - private metrics: BatchMetrics = { - totalOperations: 0, - batchesProcessed: 0, - averageBatchSize: 0, - averageLatency: 0, - throughputPerSecond: 0, - memoryUsage: 0, - adaptiveAdjustments: 0 - } - private currentMemoryUsage = 0 - private performanceHistory: number[] = [] - - constructor(config?: BatchConfig) { - super(config) - } - - getManifest(): AugmentationManifest { - return { - id: 'batch-processing', - name: 'Batch Processing', - version: '2.0.0', - description: 'High-performance batching for bulk operations', - longDescription: 'Automatically batches operations for maximum throughput. Essential for enterprise-scale workloads, achieving 500,000+ operations/second. Provides 10-50x performance improvement for bulk operations.', - category: 'performance', - configSchema: { - type: 'object', - properties: { - enabled: { - type: 'boolean', - default: true, - description: 'Enable batch processing' - }, - adaptiveMode: { - type: 'boolean', - default: true, - description: 'Automatically decide when to batch operations' - }, - immediateThreshold: { - type: 'number', - default: 1, - minimum: 1, - maximum: 10, - description: 'Operations count below which to execute immediately' - }, - batchThreshold: { - type: 'number', - default: 5, - minimum: 2, - maximum: 100, - description: 'Queue size at which to start batching' - }, - maxBatchSize: { - type: 'number', - default: 1000, - minimum: 10, - maximum: 10000, - description: 'Maximum items per batch' - }, - maxWaitTime: { - type: 'number', - default: 100, - minimum: 1, - maximum: 5000, - description: 'Maximum wait time before flushing batch (ms)' - }, - adaptiveBatching: { - type: 'boolean', - default: true, - description: 'Dynamically adjust batch size based on performance' - }, - priorityLanes: { - type: 'number', - default: 3, - minimum: 1, - maximum: 10, - description: 'Number of priority processing lanes' - }, - memoryLimit: { - type: 'number', - default: 104857600, // 100MB - minimum: 10485760, // 10MB - maximum: 1073741824, // 1GB - description: 'Maximum memory for batching in bytes' - } - }, - additionalProperties: false - }, - configDefaults: { - enabled: true, - adaptiveMode: true, - immediateThreshold: 1, - batchThreshold: 5, - maxBatchSize: 1000, - maxWaitTime: 100, - adaptiveBatching: true, - priorityLanes: 3, - memoryLimit: 104857600 - }, - minBrainyVersion: '2.0.0', - keywords: ['batch', 'performance', 'bulk', 'streaming', 'throughput'], - documentation: 'https://docs.brainy.dev/augmentations/batch-processing', - status: 'stable', - performance: { - memoryUsage: 'high', - cpuUsage: 'medium', - networkUsage: 'none' - }, - features: ['auto-batching', 'adaptive-sizing', 'priority-lanes', 'streaming-support'], - enhancedOperations: ['add', 'addNoun', 'addVerb', 'saveNoun', 'saveVerb'], - ui: { - icon: '๐Ÿ“ฆ', - color: '#9C27B0' - } - } - } - - protected async onInitialize(): Promise { - if (this.config.enabled) { - this.startMetricsCollection() - this.log(`Batch processing initialized: ${this.config.maxBatchSize} batch size, ${this.config.maxWaitTime}ms max wait`) - - if (this.config.adaptiveBatching) { - this.log('Adaptive batching enabled - will optimize batch size dynamically') - } - } else { - this.log('Batch processing disabled') - } - } - - shouldExecute(operation: string, params: any): boolean { - if (!this.config.enabled) return false - - // Skip batching for single operations or already-batched operations - if (params?.batch === false || params?.streaming === false) return false - - // Enable for high-volume operations - return operation.includes('add') || - operation.includes('save') || - operation.includes('storage') - } - - async execute( - operation: string, - params: any, - next: () => Promise - ): Promise { - if (!this.shouldExecute(operation, params)) { - return next() - } - - // Check if this should be batched based on system load - if (this.shouldBatch(operation, params)) { - return this.addToBatch(operation, params, next) - } - - // Execute immediately for low-latency requirements - return next() - } - - private shouldBatch(operation: string, params: any): boolean { - // ZERO-CONFIG INTELLIGENT ADAPTATION: - if (this.config.adaptiveMode) { - // CRITICAL WORKFLOW DETECTION: Never batch operations that break critical patterns - - // 1. ENTITY REGISTRY PATTERN: Never batch when immediate lookup is expected - if (this.isEntityRegistryWorkflow(operation, params)) { - return false // Must be immediate for registry lookups to work - } - - // 2. DEPENDENCY CHAIN PATTERN: Never batch when next operation depends on this one - if (this.isDependencyChainStart(operation, params)) { - return false // Must be immediate for noun โ†’ verb workflows - } - - // Count pending operations in the current operation's batch (needed for write-only mode) - const batchKey = this.getBatchKey(operation, params) - const currentBatch = this.batches.get(batchKey) || [] - const pendingCount = currentBatch.length - - // 3. WRITE-ONLY MODE: Special handling for high-speed streaming - if (this.isWriteOnlyMode(params)) { - // In write-only mode, batch aggressively but ensure entity registry updates immediately - if (this.hasEntityRegistryMetadata(params)) { - return false // Entity registry updates must be immediate even in write-only mode - } - return pendingCount >= 3 // Lower threshold for write-only mode batching - } - - // Apply intelligent thresholds: - // 4. Single operations are immediate (responsive user experience) - if (pendingCount < this.config.immediateThreshold!) { - return false // Execute immediately - } - - // 5. Start batching when multiple operations are queued - if (pendingCount >= this.config.batchThreshold!) { - return true // Batch for efficiency - } - - // 6. For in-between cases, use smart heuristics - const currentLoad = this.getCurrentLoad() - if (currentLoad > 0.5) return true // Higher load = more batching - - // 7. Batch operations that naturally benefit from grouping - if (operation.includes('save') || operation.includes('add')) { - return pendingCount > 1 // Batch if others are already waiting - } - - return false // Default to immediate for best responsiveness - } - - // TRADITIONAL MODE: (for explicit configuration scenarios) - // Always batch if explicitly requested - if (params?.batch === true || params?.streaming === true) return true - - // Batch based on current system load - const currentLoad = this.getCurrentLoad() - if (currentLoad > 0.7) return true // High load - batch everything - - // Batch operations that benefit from grouping - return operation.includes('save') || - operation.includes('add') || - operation.includes('update') - } - - /** - * SMART WORKFLOW DETECTION METHODS - * These methods detect critical patterns that must not be batched - */ - - private isEntityRegistryWorkflow(operation: string, params: any): boolean { - // Detect operations that will likely be followed by immediate entity registry lookups - if (operation === 'addNoun' || operation === 'add') { - // Check if metadata contains external identifiers (DID, handle, etc.) - const metadata = params?.metadata || params?.data || {} - return !!( - metadata.did || // Bluesky DID - metadata.handle || // Social media handle - metadata.uri || // Resource URI - metadata.external_id || // External system ID - metadata.user_id || // User ID - metadata.profile_id || // Profile ID - metadata.account_id // Account ID - ) - } - return false - } - - private isDependencyChainStart(operation: string, params: any): boolean { - // Detect operations that are likely to be followed by dependent operations - if (operation === 'addNoun' || operation === 'add') { - // In interactive workflows, noun creation is often followed by verb creation - // Use heuristics to detect this pattern - const context = this.getOperationContext() - - // If we've seen recent addVerb operations, this noun might be for a relationship - if (context.recentVerbOperations > 0) { - return true - } - - // If this is part of a rapid sequence of operations, it might be a dependency chain - if (context.operationsInLastSecond > 3) { - return true - } - } - - return false - } - - private isWriteOnlyMode(params: any): boolean { - // Detect write-only mode from context or parameters - return !!( - params?.writeOnlyMode || - params?.streaming || - params?.highThroughput || - this.context?.brain?.writeOnly - ) - } - - private hasEntityRegistryMetadata(params: any): boolean { - // Check if this operation has metadata that needs immediate entity registry updates - const metadata = params?.metadata || params?.data || {} - return !!( - metadata.did || - metadata.handle || - metadata.uri || - metadata.external_id || - // Also check for auto-registration hints - params?.autoCreateMissingNouns || - params?.entityRegistry - ) - } - - private getOperationContext(): { recentVerbOperations: number; operationsInLastSecond: number } { - const now = Date.now() - const oneSecondAgo = now - 1000 - - let recentVerbOperations = 0 - let operationsInLastSecond = 0 - - // Analyze recent operations across all batches - for (const batch of this.batches.values()) { - for (const op of batch) { - if (op.timestamp > oneSecondAgo) { - operationsInLastSecond++ - if (op.operation.includes('Verb') || op.operation.includes('verb')) { - recentVerbOperations++ - } - } - } - } - - return { recentVerbOperations, operationsInLastSecond } - } - - private getCurrentLoad(): number { - // Simple load calculation based on pending operations - let totalPending = 0 - for (const batch of this.batches.values()) { - totalPending += batch.length - } - - return Math.min(totalPending / 10000, 1.0) // Normalize to 0-1 - } - - private async addToBatch( - operation: string, - params: any, - executor: () => Promise - ): Promise { - return new Promise((resolve, reject) => { - const priority = this.getOperationPriority(operation, params) - const batchKey = this.getBatchKey(operation, priority) - const operationSize = this.estimateOperationSize(params) - - // Check memory limit - if (this.currentMemoryUsage + operationSize > this.config.memoryLimit) { - // Memory limit reached - flush oldest batch - this.flushOldestBatch() - } - - const batchedOp: BatchedOperation = { - id: `op_${Date.now()}_${Math.random()}`, - operation, - params, - executor, // Store the actual executor - resolver: resolve, - rejector: reject, - timestamp: Date.now(), - priority, - size: operationSize - } - - // Add to appropriate batch - if (!this.batches.has(batchKey)) { - this.batches.set(batchKey, []) - } - - const batch = this.batches.get(batchKey)! - batch.push(batchedOp) - this.currentMemoryUsage += operationSize - this.metrics.totalOperations++ - - // Check if batch should be flushed immediately - if (this.shouldFlushBatch(batch, batchKey)) { - this.flushBatch(batchKey) - } else if (!this.flushTimers.has(batchKey)) { - // Set flush timer if not already set - this.setFlushTimer(batchKey) - } - }) - } - - private getOperationPriority(operation: string, params: any): number { - // Explicit priority - if (params?.priority !== undefined) return params.priority - - // Operation-based priority - if (operation.includes('delete')) return 10 // Highest - if (operation.includes('update')) return 8 - if (operation.includes('save')) return 6 - if (operation.includes('add')) return 4 - return 1 // Lowest - } - - private getBatchKey(operation: string, priority: number): string { - // Group by operation type and priority for optimal batching - const opType = this.getOperationType(operation) - const priorityLane = Math.min(priority, this.config.priorityLanes - 1) - return `${opType}_p${priorityLane}` - } - - private getOperationType(operation: string): string { - if (operation.includes('add')) return 'add' - if (operation.includes('save')) return 'save' - if (operation.includes('update')) return 'update' - if (operation.includes('delete')) return 'delete' - return 'other' - } - - private estimateOperationSize(params: any): number { - // Rough estimation of memory usage - if (!params) return 100 - - let size = 0 - - if (params.vector && Array.isArray(params.vector)) { - size += params.vector.length * 8 // 8 bytes per float64 - } - - if (params.data) { - size += JSON.stringify(params.data).length * 2 // Rough UTF-16 estimate - } - - if (params.metadata) { - size += JSON.stringify(params.metadata).length * 2 - } - - return Math.max(size, 100) // Minimum 100 bytes - } - - private shouldFlushBatch(batch: BatchedOperation[], batchKey: string): boolean { - // Flush if batch is full - if (batch.length >= this.config.maxBatchSize) return true - - // Flush if memory limit approaching - if (this.currentMemoryUsage > this.config.memoryLimit * 0.9) return true - - // Flush high-priority batches more aggressively - const priority = this.extractPriorityFromKey(batchKey) - if (priority >= 8 && batch.length >= 100) return true - if (priority >= 6 && batch.length >= 500) return true - - return false - } - - private extractPriorityFromKey(batchKey: string): number { - const match = batchKey.match(/_p(\d+)$/) - return match ? parseInt(match[1]) : 0 - } - - private setFlushTimer(batchKey: string): void { - const priority = this.extractPriorityFromKey(batchKey) - const waitTime = this.getAdaptiveWaitTime(priority) - - const timer = setTimeout(() => { - this.flushBatch(batchKey) - }, waitTime) - - this.flushTimers.set(batchKey, timer) - } - - private getAdaptiveWaitTime(priority: number): number { - if (!this.config.adaptiveBatching) { - return this.config.maxWaitTime - } - - // Adaptive wait time based on performance and priority - const baseWaitTime = this.config.maxWaitTime - const performanceMultiplier = this.getPerformanceMultiplier() - const priorityMultiplier = priority >= 8 ? 0.5 : priority >= 6 ? 0.7 : 1.0 - - return Math.max(baseWaitTime * performanceMultiplier * priorityMultiplier, 10) - } - - private getPerformanceMultiplier(): number { - if (this.performanceHistory.length < 10) return 1.0 - - // Calculate average latency trend - const recent = this.performanceHistory.slice(-10) - const average = recent.reduce((a, b) => a + b, 0) / recent.length - - // If performance is degrading, reduce wait time - if (average > this.metrics.averageLatency * 1.2) return 0.7 - if (average < this.metrics.averageLatency * 0.8) return 1.3 - - return 1.0 - } - - private async flushBatch(batchKey: string): Promise { - const batch = this.batches.get(batchKey) - if (!batch || batch.length === 0) return - - // Clear timer - const timer = this.flushTimers.get(batchKey) - if (timer) { - clearTimeout(timer) - this.flushTimers.delete(batchKey) - } - - // Remove batch from queue - this.batches.delete(batchKey) - - const startTime = Date.now() - - try { - await this.processBatch(batch) - - // Update metrics - const latency = Date.now() - startTime - this.updateMetrics(batch.length, latency) - - // Adaptive adjustment - if (this.config.adaptiveBatching) { - this.adjustBatchSize(latency, batch.length) - } - - } catch (error) { - this.log(`Batch processing failed for ${batchKey}: ${error}`, 'error') - - // Reject all operations in batch - batch.forEach(op => { - op.rejector(error as Error) - this.currentMemoryUsage -= op.size - }) - } - } - - private async processBatch(batch: BatchedOperation[]): Promise { - // Group by operation type for efficient processing - const operationGroups = new Map[]>() - - for (const op of batch) { - const opType = this.getOperationType(op.operation) - if (!operationGroups.has(opType)) { - operationGroups.set(opType, []) - } - operationGroups.get(opType)!.push(op) - } - - // Process each operation type - for (const [opType, operations] of operationGroups) { - await this.processBatchByType(opType, operations) - } - } - - private async processBatchByType(opType: string, operations: BatchedOperation[]): Promise { - // Execute batch operation based on type - try { - if (opType === 'add' || opType === 'save') { - await this.processBatchSave(operations) - } else if (opType === 'update') { - await this.processBatchUpdate(operations) - } else if (opType === 'delete') { - await this.processBatchDelete(operations) - } else { - // Fallback: execute individually - await this.processIndividually(operations) - } - } catch (error) { - throw error - } - } - - private async processBatchSave(operations: BatchedOperation[]): Promise { - // Try to use storage's bulk save if available - const storage = this.context?.storage - - if (storage && typeof storage.saveBatch === 'function') { - // Use bulk save operation - const items = operations.map(op => ({ - ...op.params, - _batchId: op.id - })) - - try { - const results = await storage.saveBatch(items) - - // Resolve all operations with actual results - operations.forEach((op, index) => { - op.resolver(results[index] || op.params.id) - this.currentMemoryUsage -= op.size - }) - } catch (error) { - // Reject all operations on batch error - operations.forEach(op => { - op.rejector(error as Error) - this.currentMemoryUsage -= op.size - }) - throw error - } - } else { - // Execute using stored executors with concurrency control - await this.processWithConcurrency(operations, 10) - } - } - - private async processBatchUpdate(operations: BatchedOperation[]): Promise { - await this.processWithConcurrency(operations, 5) // Lower concurrency for updates - } - - private async processBatchDelete(operations: BatchedOperation[]): Promise { - await this.processWithConcurrency(operations, 5) // Lower concurrency for deletes - } - - private async processIndividually(operations: BatchedOperation[]): Promise { - await this.processWithConcurrency(operations, 3) // Conservative concurrency - } - - private async processWithConcurrency(operations: BatchedOperation[], concurrency: number): Promise { - const promises: Promise[] = [] - - for (let i = 0; i < operations.length; i += concurrency) { - const chunk = operations.slice(i, i + concurrency) - - const chunkPromise = Promise.all( - chunk.map(async (op) => { - try { - // Execute using the stored executor function - REAL EXECUTION! - const result = await op.executor() - op.resolver(result) - this.currentMemoryUsage -= op.size - } catch (error) { - op.rejector(error as Error) - this.currentMemoryUsage -= op.size - } - }) - ).then(() => {}) // Convert to void promise - - promises.push(chunkPromise) - } - - await Promise.all(promises) - } - - // REMOVED executeOperation - no longer needed since we use stored executors - - private flushOldestBatch(): void { - if (this.batches.size === 0) return - - // Find oldest batch - let oldestKey = '' - let oldestTime = Infinity - - for (const [key, batch] of this.batches) { - if (batch.length > 0) { - const batchAge = Math.min(...batch.map(op => op.timestamp)) - if (batchAge < oldestTime) { - oldestTime = batchAge - oldestKey = key - } - } - } - - if (oldestKey) { - this.flushBatch(oldestKey) - } - } - - private updateMetrics(batchSize: number, latency: number): void { - this.metrics.batchesProcessed++ - this.metrics.averageBatchSize = - (this.metrics.averageBatchSize * (this.metrics.batchesProcessed - 1) + batchSize) / - this.metrics.batchesProcessed - - // Update latency with exponential moving average - this.metrics.averageLatency = this.metrics.averageLatency * 0.9 + latency * 0.1 - - // Add to performance history - this.performanceHistory.push(latency) - if (this.performanceHistory.length > 100) { - this.performanceHistory.shift() - } - } - - private adjustBatchSize(latency: number, batchSize: number): void { - const targetLatency = this.config.maxWaitTime * 5 // Target: 5x wait time - - if (latency > targetLatency && batchSize > 100) { - // Reduce batch size if latency too high - this.config.maxBatchSize = Math.max(this.config.maxBatchSize * 0.9, 100) - this.metrics.adaptiveAdjustments++ - } else if (latency < targetLatency * 0.5 && batchSize === this.config.maxBatchSize) { - // Increase batch size if latency very low - this.config.maxBatchSize = Math.min(this.config.maxBatchSize * 1.1, 10000) - this.metrics.adaptiveAdjustments++ - } - } - - private startMetricsCollection(): void { - setInterval(() => { - // Calculate throughput - this.metrics.throughputPerSecond = this.metrics.totalOperations - this.metrics.totalOperations = 0 // Reset for next measurement - - // Update memory usage - this.metrics.memoryUsage = this.currentMemoryUsage - - }, 1000) - } - - /** - * Get batch processing statistics - */ - getStats(): BatchMetrics & { - pendingBatches: number - pendingOperations: number - currentBatchSize: number - memoryUtilization: string - } { - let pendingOperations = 0 - for (const batch of this.batches.values()) { - pendingOperations += batch.length - } - - return { - ...this.metrics, - pendingBatches: this.batches.size, - pendingOperations, - currentBatchSize: this.config.maxBatchSize, - memoryUtilization: `${Math.round((this.currentMemoryUsage / this.config.memoryLimit) * 100)}%` - } - } - - /** - * Force flush all pending batches - */ - async flushAll(): Promise { - const batchKeys = Array.from(this.batches.keys()) - await Promise.all(batchKeys.map(key => this.flushBatch(key))) - } - - protected async onShutdown(): Promise { - // Clear all timers - for (const timer of this.flushTimers.values()) { - clearTimeout(timer) - } - this.flushTimers.clear() - - // Flush all pending batches - await this.flushAll() - - const stats = this.getStats() - this.log(`Batch processing shutdown: ${this.metrics.batchesProcessed} batches processed, ${stats.memoryUtilization} peak memory usage`) - } -} \ No newline at end of file diff --git a/src/augmentations/brainyAugmentation.ts b/src/augmentations/brainyAugmentation.ts deleted file mode 100644 index 7b94f170..00000000 --- a/src/augmentations/brainyAugmentation.ts +++ /dev/null @@ -1,766 +0,0 @@ -/** - * Single BrainyAugmentation Interface - * - * This replaces the 7 complex interfaces with one elegant, purpose-driven design. - * Each augmentation knows its place and when to execute automatically. - * - * The Vision: Components that enhance Brainy's capabilities seamlessly - * - RequestDeduplicator: Prevents duplicate concurrent requests - * - ConnectionPool: Optimizes cloud storage throughput - * - IntelligentVerbScoring: Enhances relationship analysis - * - StreamingPipeline: Enables unlimited data processing - */ - -import { AugmentationManifest } from './manifest.js' -import { AugmentationConfigResolver } from './configResolver.js' - -/** - * Metadata access declaration for augmentations - */ -export interface MetadataAccess { - reads?: string[] | '*' // Fields to read, or '*' for all - writes?: string[] | '*' // Fields to write, or '*' for all - namespace?: string // Optional: custom namespace like '_myAug' -} - -export interface BrainyAugmentation { - /** - * Unique identifier for the augmentation - */ - name: string - - /** - * When this augmentation should execute - * - 'before': Execute before the main operation - * - 'after': Execute after the main operation - * - 'around': Wrap the main operation (like middleware) - * - 'replace': Replace the main operation entirely - */ - timing: 'before' | 'after' | 'around' | 'replace' - - /** - * Metadata access contract - REQUIRED - * - 'none': No metadata access at all - * - 'readonly': Can read any metadata but cannot write - * - MetadataAccess: Specific fields to read/write - */ - metadata: 'none' | 'readonly' | MetadataAccess - - /** - * Which operations this augmentation applies to - * Granular operation matching for precise augmentation targeting - */ - operations: ( - // Data Operations - | 'add' | 'addNoun' | 'addVerb' - | 'saveNoun' | 'saveVerb' | 'updateMetadata' - | 'update' | 'delete' | 'deleteVerb' | 'clear' | 'get' - - // Search Operations - | 'search' | 'searchText' | 'searchByNounTypes' - | 'find' | 'findSimilar' | 'searchWithCursor' | 'similar' - - // Relationship Operations - | 'relate' | 'unrelate' | 'getConnections' | 'getRelations' - - // Storage Operations - | 'storage' | 'backup' | 'restore' - - // Meta - | 'all' - )[] - - /** - * Priority for execution order (higher numbers execute first) - * - 100: Critical system operations (ConnectionPool) - * - 50: Performance optimizations (RequestDeduplicator, Caching) - * - 10: Enhancement features (IntelligentVerbScoring) - * - 1: Optional features (Logging, Analytics) - */ - priority: number - - /** - * Initialize the augmentation - * Called once during Brainy initialization - * - * @param context - The Brainy instance and storage - */ - initialize(context: AugmentationContext): Promise - - /** - * Execute the augmentation - * - * @param operation - The operation being performed - * @param params - Parameters for the operation - * @param next - Function to call the next augmentation or main operation - * @returns Result of the operation - */ - execute( - operation: string, - params: any, - next: () => Promise - ): Promise - - /** - * Optional: Check if this augmentation should run for the given operation - * Return false to skip execution - */ - shouldExecute?(operation: string, params: any): boolean - - /** - * Optional: Cleanup when Brainy is destroyed - */ - shutdown?(): Promise - - /** - * Optional: Computed fields this augmentation provides - * Used for discovery, TypeScript support, and API documentation - */ - computedFields?: { - [namespace: string]: { - [field: string]: { - type: 'string' | 'number' | 'boolean' | 'object' | 'array' - description: string - confidence?: number - } - } - } - - /** - * Optional: Compute fields for a result entity - * Called when user accesses getDisplay(), getSchema(), etc. - * - * @param result - The result entity (VectorDocument, GraphVerb, etc.) - * @param namespace - The namespace being requested ('display', 'schema', etc.) - * @returns Computed fields for the namespace - */ - computeFields?(result: any, namespace: string): Promise> | Record -} - -/** - * Context provided to augmentations - */ -export interface AugmentationContext { - /** - * The Brainy instance (for accessing methods and config) - */ - brain: any // Brainy - avoiding circular imports - - /** - * The storage adapter - */ - storage: any // StorageAdapter - - /** - * Configuration for this augmentation - */ - config: any - - /** - * Logging function - */ - log: (message: string, level?: 'info' | 'warn' | 'error') => void -} - -/** - * Base class for augmentations with common functionality - * - * This is the unified base class that combines the features of both - * BaseAugmentation and ConfigurableAugmentation. All augmentations - * should extend this class for consistent configuration support. - */ -export abstract class BaseAugmentation implements BrainyAugmentation { - abstract name: string - abstract timing: 'before' | 'after' | 'around' | 'replace' - abstract metadata: 'none' | 'readonly' | MetadataAccess - abstract operations: ( - // Data Operations - | 'add' | 'addNoun' | 'addVerb' - | 'saveNoun' | 'saveVerb' | 'updateMetadata' - | 'update' | 'delete' | 'deleteVerb' | 'clear' | 'get' - - // Search Operations - | 'search' | 'searchText' | 'searchByNounTypes' - | 'find' | 'findSimilar' | 'searchWithCursor' | 'similar' - - // Relationship Operations - | 'relate' | 'unrelate' | 'getConnections' | 'getRelations' - - // Storage Operations - | 'storage' | 'backup' | 'restore' - - // Meta - | 'all' - )[] - abstract priority: number - - // Metadata for augmentation listing and management - category: 'internal' | 'core' | 'premium' | 'community' | 'external' = 'core' - description: string = '' - enabled: boolean = true - - protected context?: AugmentationContext - protected isInitialized = false - protected config: any = {} - private configResolver?: AugmentationConfigResolver - - /** - * Constructor with optional configuration - * @param config Optional configuration to override defaults - */ - constructor(config?: any) { - // Only resolve configuration if getManifest is implemented - if (this.getManifest) { - this.config = this.resolveConfiguration(config) - } else if (config) { - // Legacy support: direct config assignment for augmentations without manifests - this.config = config - } - } - - /** - * Get the augmentation manifest for discovery - * Override this to enable configuration support - * CRITICAL: This enables tools to discover parameters and configuration - */ - getManifest?(): AugmentationManifest - - /** - * Get parameter schema for operations - * Enables tools to know what parameters each operation needs - */ - getParameterSchema?(operation: string): any - - /** - * Get operation descriptions - * Enables tools to show what each operation does - */ - getOperationInfo?(): Record - - /** - * Get current configuration - */ - getConfig(): any { - return { ...this.config } - } - - /** - * Update configuration at runtime - * @param partial Partial configuration to merge - */ - async updateConfig(partial: any): Promise { - if (!this.configResolver) { - // For legacy augmentations without manifest, just merge config - const oldConfig = this.config - this.config = { ...this.config, ...partial } - - if (this.onConfigChange) { - await this.onConfigChange(this.config, oldConfig) - } - return - } - - const oldConfig = this.config - - try { - // Use resolver to update and validate - this.config = this.configResolver.updateRuntime(partial) - - // Call config change handler if implemented - if (this.onConfigChange) { - await this.onConfigChange(this.config, oldConfig) - } - } catch (error) { - // Revert on error - this.config = oldConfig - throw error - } - } - - /** - * Optional: Handle configuration changes - * Override this to react to runtime configuration updates - */ - protected onConfigChange?(newConfig: any, oldConfig: any): Promise - - /** - * Resolve configuration from all sources - * Priority: constructor > env > files > defaults - */ - private resolveConfiguration(constructorConfig?: any): any { - const manifest = this.getManifest!() - - // Create config resolver - this.configResolver = new AugmentationConfigResolver({ - augmentationId: manifest.id, - schema: manifest.configSchema, - defaults: manifest.configDefaults - }) - - // Resolve configuration from all sources - return this.configResolver.resolve(constructorConfig) - } - - async initialize(context: AugmentationContext): Promise { - this.context = context - this.isInitialized = true - await this.onInitialize() - } - - /** - * Override this in subclasses for initialization logic - */ - protected async onInitialize(): Promise { - // Default: no-op - } - - abstract execute( - operation: string, - params: any, - next: () => Promise - ): Promise - - shouldExecute(operation: string, params: any): boolean { - // Default: execute if operations match exactly or includes 'all' - return this.operations.includes('all' as any) || - this.operations.includes(operation as any) || - this.operations.some(op => operation.includes(op)) - } - - async shutdown(): Promise { - await this.onShutdown() - this.isInitialized = false - } - - /** - * Override this in subclasses for cleanup logic - */ - protected async onShutdown(): Promise { - // Default: no-op - } - - /** - * Optional computed fields declaration (override in subclasses) - */ - computedFields?: { - [namespace: string]: { - [field: string]: { - type: 'string' | 'number' | 'boolean' | 'object' | 'array' - description: string - confidence?: number - } - } - } - - /** - * Optional computed fields implementation (override in subclasses) - * @param result The result entity - * @param namespace The requested namespace - * @returns Computed fields for the namespace - */ - computeFields?(result: any, namespace: string): Promise> | Record - - /** - * Log a message with the augmentation name - */ - protected log(message: string, level: 'info' | 'warn' | 'error' = 'info'): void { - if (this.context) { - this.context.log(`[${this.name}] ${message}`, level) - } - } - - /** - * Get CommitLog for temporal features - * - * Provides access to commit history for time-travel queries, audit trails, - * and branch management. Available after initialize() is called. - * - * @returns CommitLog instance - * @throws Error if called before initialize() or if COW not enabled - * - * @example - * ```typescript - * protected async onInitialize() { - * const commitLog = this.getCommitLog() - * const history = await commitLog.getHistory('heads/main', { maxCount: 10 }) - * } - * ``` - */ - protected getCommitLog(): any { - if (!this.context) { - throw new Error( - `${this.name}: Cannot access CommitLog before initialize(). ` + - `CommitLog is only available after the augmentation has been initialized.` - ) - } - - const storage = this.context.storage as any - - if (!storage.commitLog) { - throw new Error( - `${this.name}: CommitLog not available. ` + - `COW (Copy-on-Write) is not enabled on this storage adapter. ` + - `Requires BaseStorage with initializeCOW() called. ` + - `This is expected if using a non-COW storage adapter.` - ) - } - - return storage.commitLog - } - - /** - * Get BlobStorage for content-addressable storage - * - * Provides access to the underlying blob storage system for storing - * and retrieving content-addressed data. Available after initialize() is called. - * - * @returns BlobStorage instance - * @throws Error if called before initialize() or if COW not enabled - * - * @example - * ```typescript - * protected async onInitialize() { - * const blobStorage = this.getBlobStorage() - * const hash = await blobStorage.writeBlob(Buffer.from('data')) - * const data = await blobStorage.readBlob(hash) - * } - * ``` - */ - protected getBlobStorage(): any { - if (!this.context) { - throw new Error( - `${this.name}: Cannot access BlobStorage before initialize(). ` + - `BlobStorage is only available after the augmentation has been initialized.` - ) - } - - const storage = this.context.storage as any - - if (!storage.blobStorage) { - throw new Error( - `${this.name}: BlobStorage not available. ` + - `COW (Copy-on-Write) is not enabled on this storage adapter. ` + - `Requires BaseStorage with initializeCOW() called. ` + - `This is expected if using a non-COW storage adapter.` - ) - } - - return storage.blobStorage - } - - /** - * Get RefManager for branch/ref management - * - * Provides access to the reference manager for creating, updating, - * and managing Git-style branches and refs. Available after initialize() is called. - * - * @returns RefManager instance - * @throws Error if called before initialize() or if COW not enabled - * - * @example - * ```typescript - * protected async onInitialize() { - * const refManager = this.getRefManager() - * await refManager.setRef('heads/experiment', commitHash, { - * author: 'system', - * message: 'Create experiment branch' - * }) - * } - * ``` - */ - protected getRefManager(): any { - if (!this.context) { - throw new Error( - `${this.name}: Cannot access RefManager before initialize(). ` + - `RefManager is only available after the augmentation has been initialized.` - ) - } - - const storage = this.context.storage as any - - if (!storage.refManager) { - throw new Error( - `${this.name}: RefManager not available. ` + - `COW (Copy-on-Write) is not enabled on this storage adapter. ` + - `Requires BaseStorage with initializeCOW() called. ` + - `This is expected if using a non-COW storage adapter.` - ) - } - - return storage.refManager - } - - /** - * Get current branch name - * - * Convenience helper for getting the current branch from the Brainy instance. - * Available after initialize() is called. - * - * @returns Current branch name (e.g., 'main') - * @throws Error if called before initialize() - * - * @example - * ```typescript - * protected async onInitialize() { - * const branch = await this.getCurrentBranch() - * console.log(`Current branch: ${branch}`) - * } - * ``` - */ - protected async getCurrentBranch(): Promise { - if (!this.context) { - throw new Error( - `${this.name}: Cannot access Brainy instance before initialize(). ` + - `getCurrentBranch() is only available after the augmentation has been initialized.` - ) - } - - const brain = this.context.brain as any - - if (typeof brain.getCurrentBranch !== 'function') { - throw new Error( - `${this.name}: getCurrentBranch() not available on Brainy instance. ` + - `This method requires Brainy with VFS support.` - ) - } - - return brain.getCurrentBranch() - } -} - -/** - * Alias for backward compatibility - * ConfigurableAugmentation is now merged into BaseAugmentation - * @deprecated Use BaseAugmentation instead - */ -export const ConfigurableAugmentation = BaseAugmentation - -/** - * Registry for managing augmentations - */ -export class AugmentationRegistry { - private augmentations: BrainyAugmentation[] = [] - private context?: AugmentationContext - - /** - * Register an augmentation - */ - register(augmentation: BrainyAugmentation): void { - this.augmentations.push(augmentation) - // Sort by priority (highest first) - this.augmentations.sort((a, b) => b.priority - a.priority) - } - - /** - * Find augmentations by operation (before initialization) - * Used for two-phase initialization to find storage augmentations - */ - findByOperation(operation: string): BrainyAugmentation | null { - return this.augmentations.find(aug => - aug.operations.includes(operation as any) || - aug.operations.includes('all' as any) - ) || null - } - - /** - * Initialize all augmentations - */ - async initialize(context: AugmentationContext): Promise { - this.context = context - for (const augmentation of this.augmentations) { - await augmentation.initialize(context) - } - context.log(`Initialized ${this.augmentations.length} augmentations`) - } - - /** - * Initialize all augmentations (alias for consistency) - */ - async initializeAll(context: AugmentationContext): Promise { - return this.initialize(context) - } - - /** - * Execute augmentations for an operation - */ - async execute( - operation: string, - params: any, - mainOperation: () => Promise - ): Promise { - // Filter augmentations that should execute for this operation - const applicable = this.augmentations.filter(aug => - aug.shouldExecute ? aug.shouldExecute(operation, params) : - aug.operations.includes('all' as any) || - aug.operations.includes(operation as any) || - aug.operations.some(op => operation.includes(op)) - ) - - if (applicable.length === 0) { - // No augmentations, execute main operation directly - return mainOperation() - } - - // Create a chain of augmentations - let index = 0 - const executeNext = async (): Promise => { - if (index >= applicable.length) { - // All augmentations processed, execute main operation - return mainOperation() - } - - const augmentation = applicable[index++] - return augmentation.execute(operation, params, executeNext) - } - - return executeNext() - } - - /** - * Get all registered augmentations - */ - getAll(): BrainyAugmentation[] { - return [...this.augmentations] - } - - /** - * Get augmentation info for listing - */ - getInfo(): Array<{ - name: string - type: string - enabled: boolean - description: string - category: string - priority: number - }> { - return this.augmentations.map(aug => { - const baseAug = aug as any - return { - name: aug.name, - type: baseAug.category || 'core', - enabled: baseAug.enabled !== false, - description: baseAug.description || `${aug.name} augmentation`, - category: baseAug.category || 'core', - priority: aug.priority - } - }) - } - - /** - * Get augmentations by name - */ - get(name: string): BrainyAugmentation | undefined { - return this.augmentations.find(aug => aug.name === name) - } - - /** - * Discover augmentation parameters and schemas - * Critical for tools like brain-cloud to generate UIs - */ - discover(name?: string): any { - if (name) { - const aug = this.get(name) - if (!aug) return null - - const baseAug = aug as BaseAugmentation - return { - name: aug.name, - operations: aug.operations, - priority: aug.priority, - timing: aug.timing, - metadata: aug.metadata, - manifest: baseAug.getManifest ? baseAug.getManifest() : undefined, - parameters: baseAug.getParameterSchema ? - aug.operations.reduce((acc, op) => { - acc[op] = baseAug.getParameterSchema!(op as string) - return acc - }, {} as any) : undefined, - operationInfo: baseAug.getOperationInfo ? baseAug.getOperationInfo() : undefined, - config: baseAug.getConfig ? baseAug.getConfig() : undefined - } - } - - // Return all augmentations discovery info - return this.augmentations.map(aug => this.discover(aug.name)) - } - - /** - * Get configuration schema for an augmentation - * Enables UI generation for configuration - */ - getConfigSchema(name: string): any { - const aug = this.get(name) as BaseAugmentation - if (!aug || !aug.getManifest) return null - - const manifest = aug.getManifest() - return manifest?.configSchema - } - - /** - * Configure an augmentation at runtime - */ - async configure(name: string, config: any): Promise { - const aug = this.get(name) as BaseAugmentation - if (!aug || !aug.updateConfig) { - throw new Error(`Augmentation ${name} does not support configuration`) - } - - await aug.updateConfig(config) - } - - /** - * Get metrics for an augmentation - */ - metrics(name?: string): any { - if (name) { - const aug = this.get(name) as any - if (!aug || !aug.metrics) return null - return aug.metrics() - } - - // Return all metrics - const allMetrics: any = {} - for (const aug of this.augmentations) { - const a = aug as any - if (a.metrics) { - allMetrics[aug.name] = a.metrics() - } - } - return allMetrics - } - - /** - * Get health status - */ - health(): any { - const health: any = { - overall: 'healthy', - augmentations: {} - } - - for (const aug of this.augmentations) { - const a = aug as any - health.augmentations[aug.name] = a.health ? a.health() : 'unknown' - } - - return health - } - - /** - * Shutdown all augmentations - */ - async shutdown(): Promise { - for (const augmentation of this.augmentations) { - if (augmentation.shutdown) { - await augmentation.shutdown() - } - } - this.augmentations = [] - } -} \ No newline at end of file diff --git a/src/augmentations/cacheAugmentation.ts b/src/augmentations/cacheAugmentation.ts deleted file mode 100644 index 3c4be10e..00000000 --- a/src/augmentations/cacheAugmentation.ts +++ /dev/null @@ -1,391 +0,0 @@ -/** - * Cache Augmentation - Optional Search Result Caching - * - * Replaces the hardcoded SearchCache in Brainy with an optional augmentation. - * This reduces core size and allows custom cache implementations. - * - * Zero-config: Automatically enabled with sensible defaults - * Can be disabled or customized via augmentation registry - */ - -import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js' -import { AugmentationManifest } from './manifest.js' -import { SearchCache } from '../utils/searchCache.js' -import type { GraphNoun } from '../types/graphTypes.js' - -export interface CacheConfig { - maxSize?: number - ttl?: number - enabled?: boolean - invalidateOnWrite?: boolean - silent?: boolean // Silent mode support -} - -/** - * CacheAugmentation - Makes search caching optional and pluggable - * - * Features: - * - Transparent search result caching - * - Automatic invalidation on data changes - * - Memory-aware cache management - * - Zero-config with smart defaults - */ -export class CacheAugmentation extends BaseAugmentation { - readonly name = 'cache' - readonly timing = 'around' as const - readonly metadata = 'none' as const // Cache doesn't access metadata - operations = ['search', 'find', 'similar', 'add', 'update', 'delete', 'clear', 'all'] as ('search' | 'find' | 'similar' | 'add' | 'update' | 'delete' | 'clear' | 'all')[] - readonly priority = 50 // Mid-priority, runs after data operations - - // Augmentation metadata - readonly category = 'core' as const - readonly description = 'Transparent search result caching with automatic invalidation' - - private searchCache: SearchCache | null = null - - constructor(config?: CacheConfig) { - super(config) - } - - getManifest(): AugmentationManifest { - return { - id: 'cache', - name: 'Cache', - version: '2.0.0', - description: 'Intelligent caching for search and query operations', - longDescription: 'Provides transparent caching for search results with automatic invalidation on data changes. Significantly improves performance for repeated queries while maintaining data consistency.', - category: 'performance', - configSchema: { - type: 'object', - properties: { - enabled: { - type: 'boolean', - default: true, - description: 'Enable or disable caching' - }, - maxSize: { - type: 'number', - default: 1000, - minimum: 10, - maximum: 100000, - description: 'Maximum number of cached entries' - }, - ttl: { - type: 'number', - default: 300000, // 5 minutes - minimum: 1000, // 1 second - maximum: 3600000, // 1 hour - description: 'Time to live for cache entries in milliseconds' - }, - invalidateOnWrite: { - type: 'boolean', - default: true, - description: 'Automatically invalidate cache on data modifications' - }, - silent: { - type: 'boolean', - default: false, - description: 'Suppress all console output' - } - }, - additionalProperties: false - }, - configDefaults: { - enabled: true, - maxSize: 1000, - ttl: 300000, - invalidateOnWrite: true - }, - configExamples: [ - { - name: 'High Performance', - description: 'Large cache with longer TTL for read-heavy workloads', - config: { - enabled: true, - maxSize: 10000, - ttl: 1800000, // 30 minutes - invalidateOnWrite: true - } - }, - { - name: 'Conservative', - description: 'Small cache with short TTL for frequently changing data', - config: { - enabled: true, - maxSize: 100, - ttl: 60000, // 1 minute - invalidateOnWrite: true - } - } - ], - minBrainyVersion: '2.0.0', - keywords: ['cache', 'performance', 'search', 'optimization'], - documentation: 'https://docs.brainy.dev/augmentations/cache', - status: 'stable', - performance: { - memoryUsage: 'medium', - cpuUsage: 'low', - networkUsage: 'none' - }, - features: ['search-caching', 'auto-invalidation', 'ttl-support', 'memory-management'], - enhancedOperations: ['search', 'searchText', 'findSimilar'], - metrics: [ - { - name: 'cache_hits', - type: 'counter', - description: 'Number of cache hits' - }, - { - name: 'cache_misses', - type: 'counter', - description: 'Number of cache misses' - }, - { - name: 'cache_size', - type: 'gauge', - description: 'Current cache size' - } - ], - ui: { - icon: 'โšก', - color: '#FFC107' - } - } - } - - protected async onInitialize(): Promise { - if (!this.config.enabled) { - this.log('Cache augmentation disabled by configuration') - return - } - - // Initialize search cache with config - this.searchCache = new SearchCache({ - maxSize: this.config.maxSize!, - maxAge: this.config.ttl!, // SearchCache uses maxAge, not ttl - enabled: true - }) - - this.log(`Cache augmentation initialized (maxSize: ${this.config.maxSize}, ttl: ${this.config.ttl}ms)`) - } - - protected async onShutdown(): Promise { - if (this.searchCache) { - this.searchCache.clear() - this.searchCache = null - } - this.log('Cache augmentation shut down') - } - - /** - * Execute augmentation - wrap operations with caching logic - */ - async execute( - operation: string, - params: any, - next: () => Promise - ): Promise { - // If cache is disabled, just pass through - if (!this.searchCache || !this.config.enabled) { - return next() - } - - switch (operation) { - case 'search': - return this.handleSearch(params, next) - - case 'add': - case 'update': - case 'delete': - // Invalidate cache on data changes - // Cache the reference to avoid race condition during async operation - const cache = this.searchCache - if (this.config.invalidateOnWrite && cache) { - const result = await next() - // Use cached reference - searchCache might have been nulled during await - cache.invalidateOnDataChange(operation as any) - this.log(`Cache invalidated due to ${operation} operation`) - return result - } - return next() - - case 'clear': - // Clear cache when all data is cleared - const result = await next() - if (this.searchCache) { - this.searchCache.clear() - this.log('Cache cleared due to clear operation') - } - return result - - default: - return next() - } - } - - /** - * Handle search operation with caching - */ - private async handleSearch(params: any, next: () => Promise): Promise { - if (!this.searchCache) return next() - - // Extract search parameters - const { query, k, options = {} } = params - - // Skip cache if explicitly disabled or has complex filters - if (options.skipCache || options.metadata) { - return next() - } - - // Generate cache key - const cacheKey = this.searchCache.getCacheKey(query, k, options) - - // Check cache - const cachedResult = this.searchCache.get(cacheKey) - if (cachedResult) { - this.log('Cache hit for search query') - // Update metrics if available - if (this.context?.brain) { - const metrics = this.context.brain.augmentations?.get('metrics') - if (metrics) { - metrics.recordCacheHit?.() - } - } - return cachedResult as T - } - - // Execute search - const result = await next() - - // Cache the result - this.searchCache.set(cacheKey, result as any) - this.log('Search result cached') - - // Update metrics if available - if (this.context?.brain) { - const metrics = this.context.brain.augmentations?.get('metrics') - if (metrics) { - metrics.recordCacheMiss?.() - } - } - - return result - } - - /** - * Get cache statistics - */ - getStats() { - if (!this.searchCache) { - return { - enabled: false, - hits: 0, - misses: 0, - size: 0, - memoryUsage: 0 - } - } - - const stats = this.searchCache.getStats() - return { - ...stats, - memoryUsage: this.searchCache.getMemoryUsage() - } - } - - /** - * Clear the cache manually - */ - clear() { - if (this.searchCache) { - this.searchCache.clear() - this.log('Cache manually cleared') - } - } - - /** - * Handle runtime configuration changes - */ - protected async onConfigChange(newConfig: CacheConfig, oldConfig: CacheConfig): Promise { - if (this.searchCache && newConfig.enabled) { - this.searchCache.updateConfig({ - maxSize: newConfig.maxSize!, - maxAge: newConfig.ttl!, // SearchCache uses maxAge - enabled: newConfig.enabled - }) - this.log('Cache configuration updated') - } else if (!newConfig.enabled && this.searchCache) { - this.searchCache.clear() - this.log('Cache disabled and cleared') - } - } - - /** - * Clean up expired entries - */ - cleanupExpiredEntries(): number { - if (!this.searchCache) return 0 - - const cleaned = this.searchCache.cleanupExpiredEntries() - if (cleaned > 0) { - this.log(`Cleaned ${cleaned} expired cache entries`) - } - return cleaned - } - - /** - * Invalidate cache when data changes - */ - invalidateOnDataChange(operation: 'add' | 'update' | 'delete') { - if (!this.searchCache) return - - this.searchCache.invalidateOnDataChange(operation) - this.log(`Cache invalidated due to ${operation} operation`) - } - - /** - * Get cache key for a query - */ - getCacheKey(query: any, options?: any): string { - if (!this.searchCache) return '' - return this.searchCache.getCacheKey(query, options) - } - - /** - * Direct cache get - */ - get(key: string): any { - if (!this.searchCache) return null - return this.searchCache.get(key) - } - - /** - * Direct cache set - */ - set(key: string, value: any): void { - if (!this.searchCache) return - this.searchCache.set(key, value) - } - - /** - * Get the underlying SearchCache instance (for compatibility) - */ - getSearchCache() { - return this.searchCache - } - - /** - * Get memory usage - */ - getMemoryUsage(): number { - if (!this.searchCache) return 0 - return this.searchCache.getMemoryUsage() - } -} - -/** - * Factory function for zero-config cache augmentation - */ -export function createCacheAugmentation(config?: CacheConfig): CacheAugmentation { - return new CacheAugmentation(config) -} \ No newline at end of file diff --git a/src/augmentations/conduitAugmentations.ts b/src/augmentations/conduitAugmentations.ts deleted file mode 100644 index 54dd36c3..00000000 --- a/src/augmentations/conduitAugmentations.ts +++ /dev/null @@ -1,274 +0,0 @@ -/** - * Conduit Augmentations - Data Synchronization Bridges - * - * These augmentations connect and synchronize data between multiple Brainy instances. - * Now using the unified BrainyAugmentation interface. - */ - -import { BaseAugmentation, BrainyAugmentation, AugmentationContext } from './brainyAugmentation.js' -import { v4 as uuidv4 } from '../universal/uuid.js' - -export interface WebSocketConnection { - connectionId: string - url: string - readyState: number - socket?: any -} - -/** - * Base class for conduit augmentations that sync between Brainy instances - * Converted to use the unified BrainyAugmentation interface - */ -abstract class BaseConduitAugmentation extends BaseAugmentation { - readonly timing = 'after' as const // Conduits run after operations to sync - readonly metadata = 'readonly' as const // Conduits read metadata to pass to external systems - readonly operations = ['addNoun', 'delete', 'addVerb'] as ('addNoun' | 'delete' | 'addVerb')[] - readonly priority = 20 // Medium-low priority - - protected connections = new Map() - - protected async onShutdown(): Promise { - // Close all connections - for (const [connectionId, connection] of this.connections.entries()) { - try { - if (connection.close) { - await connection.close() - } - } catch (error) { - this.log(`Failed to close connection ${connectionId}: ${error}`, 'error') - } - } - this.connections.clear() - } - - abstract establishConnection( - targetSystemId: string, - config?: Record - ): Promise -} - -/** - * WebSocket Conduit Augmentation - * Syncs data between Brainy instances using WebSockets - */ -export class WebSocketConduitAugmentation extends BaseConduitAugmentation { - readonly name = 'websocket-conduit' - private webSocketConnections = new Map() - private messageCallbacks = new Map void>>() - - async execute( - operation: string, - params: any, - next: () => Promise - ): Promise { - // Execute the operation first - const result = await next() - - // Then sync to connected instances - if (this.shouldSync(operation)) { - await this.syncOperation(operation, params, result) - } - - return result - } - - private shouldSync(operation: string): boolean { - return ['addNoun', 'deleteNoun', 'addVerb'].includes(operation) - } - - private async syncOperation(operation: string, params: any, result: any): Promise { - // Broadcast to all connected WebSocket instances - for (const [id, connection] of this.webSocketConnections) { - if (connection.socket && connection.readyState === 1) { // OPEN state - try { - const message = JSON.stringify({ - type: 'sync', - operation, - params, - timestamp: Date.now() - }) - - if (typeof connection.socket.send === 'function') { - connection.socket.send(message) - } - } catch (error) { - this.log(`Failed to sync to ${id}: ${error}`, 'error') - } - } - } - } - - async establishConnection( - url: string, - config?: Record - ): Promise { - try { - const connectionId = uuidv4() - const protocols = config?.protocols as string | string[] | undefined - - // Create WebSocket based on environment - let socket: any - - if (typeof WebSocket !== 'undefined') { - // Browser environment - socket = new WebSocket(url, protocols) - } else { - // Node.js environment - dynamic import - try { - const ws = await import('ws') - socket = new ws.WebSocket(url, protocols) - } catch { - this.log('WebSocket not available in this environment', 'error') - return null - } - } - - // Setup event handlers - socket.onopen = () => { - this.log(`Connected to ${url}`) - } - - socket.onmessage = (event: any) => { - this.handleMessage(connectionId, event.data) - } - - socket.onerror = (error: any) => { - this.log(`WebSocket error: ${error}`, 'error') - } - - socket.onclose = () => { - this.log(`Disconnected from ${url}`) - this.webSocketConnections.delete(connectionId) - } - - // Wait for connection to open - await new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - reject(new Error('Connection timeout')) - }, 5000) - - socket.onopen = () => { - clearTimeout(timeout) - resolve() - } - - socket.onerror = (error: any) => { - clearTimeout(timeout) - reject(error) - } - }) - - const connection: WebSocketConnection = { - connectionId, - url, - readyState: socket.readyState, - socket - } - - this.webSocketConnections.set(connectionId, connection) - this.connections.set(connectionId, connection) - - return connection - } catch (error) { - this.log(`Failed to establish connection to ${url}: ${error}`, 'error') - return null - } - } - - private handleMessage(connectionId: string, data: any): void { - try { - const message = typeof data === 'string' ? JSON.parse(data) : data - - // Handle sync messages from remote instances - if (message.type === 'sync') { - // Apply the operation to our local instance - this.applySyncOperation(message).catch(error => { - this.log(`Failed to apply sync operation: ${error}`, 'error') - }) - } - - // Notify any registered callbacks - const callbacks = this.messageCallbacks.get(connectionId) - if (callbacks) { - callbacks.forEach(callback => callback(message)) - } - } catch (error) { - this.log(`Failed to handle message: ${error}`, 'error') - } - } - - private async applySyncOperation(message: any): Promise { - // Apply the synced operation to our local Brainy instance - const { operation, params } = message - - try { - switch (operation) { - case 'addNoun': - await this.context?.brain.add({ data: params.content, type: 'content', metadata: params.metadata }) - break - case 'deleteNoun': - await this.context?.brain.deleteNoun(params.id) - break - case 'addVerb': - await this.context?.brain.relate({ - from: params.source, - to: params.target, - type: params.verb, - metadata: params.metadata - }) - break - } - } catch (error) { - this.log(`Failed to apply ${operation}: ${error}`, 'error') - } - } - - /** - * Subscribe to messages from a specific connection - */ - onMessage(connectionId: string, callback: (data: any) => void): void { - if (!this.messageCallbacks.has(connectionId)) { - this.messageCallbacks.set(connectionId, new Set()) - } - this.messageCallbacks.get(connectionId)!.add(callback) - } - - /** - * Send a message to a specific connection - */ - sendMessage(connectionId: string, data: any): boolean { - const connection = this.webSocketConnections.get(connectionId) - if (connection?.socket && connection.readyState === 1) { - try { - const message = typeof data === 'string' ? data : JSON.stringify(data) - connection.socket.send(message) - return true - } catch (error) { - this.log(`Failed to send message: ${error}`, 'error') - } - } - return false - } -} - -/** - * Example usage: - * - * // Server instance - * const serverBrain = new Brainy() - * serverBrain.augmentations.register(new APIServerAugmentation()) - * await serverBrain.init() - * - * // Client instance - * const clientBrain = new Brainy() - * const conduit = new WebSocketConduitAugmentation() - * clientBrain.augmentations.register(conduit) - * await clientBrain.init() - * - * // Connect client to server - * await conduit.establishConnection('ws://localhost:3000/ws') - * - * // Now operations sync automatically! - * await clientBrain.add({ data: 'synced data', type: 'content', metadata: { source: 'client' } }) - * // This will automatically sync to the server - */ \ No newline at end of file diff --git a/src/augmentations/configResolver.ts b/src/augmentations/configResolver.ts deleted file mode 100644 index 5052f6b5..00000000 --- a/src/augmentations/configResolver.ts +++ /dev/null @@ -1,539 +0,0 @@ -/** - * Configuration Resolver for Augmentations - * - * Handles loading and resolving configuration from multiple sources: - * - Environment variables - * - Configuration files - * - Runtime updates - * - Default values from schema - */ - -import { JSONSchema } from './manifest.js' -import { isNode } from '../utils/environment.js' - -/** - * Configuration source priority (highest to lowest) - */ -export enum ConfigPriority { - RUNTIME = 4, // Runtime updates (highest priority) - CONSTRUCTOR = 3, // Constructor parameters - ENVIRONMENT = 2, // Environment variables - FILE = 1, // Configuration files - DEFAULT = 0 // Schema defaults (lowest priority) -} - -/** - * Configuration source information - */ -export interface ConfigSource { - priority: ConfigPriority - source: string - config: any -} - -/** - * Configuration resolution options - */ -export interface ConfigResolverOptions { - augmentationId: string - schema?: JSONSchema - defaults?: Record - configPaths?: string[] - envPrefix?: string - allowUndefined?: boolean -} - -/** - * Augmentation Configuration Resolver - */ -export class AugmentationConfigResolver { - private sources: ConfigSource[] = [] - private resolved: any = {} - - constructor(private options: ConfigResolverOptions) { - // Default config paths - include home directory paths only in Node.js - const defaultConfigPaths = [ - '.brainyrc', - '.brainyrc.json', - 'brainy.config.json' - ] - - // Add home directory paths in Node.js environments - if (isNode()) { - try { - const nodeRequire = typeof require !== 'undefined' ? require : null - if (nodeRequire) { - const path = nodeRequire('node:path') - const os = nodeRequire('node:os') - defaultConfigPaths.push( - path.join(os.homedir(), '.brainy', 'config.json'), - path.join(os.homedir(), '.brainyrc') - ) - } - } catch { - // Silently continue without home directory paths - } - } - - this.options = { - configPaths: defaultConfigPaths, - envPrefix: `BRAINY_AUG_${options.augmentationId.toUpperCase()}_`, - allowUndefined: true, - ...options - } - } - - /** - * Resolve configuration from all sources - * @param constructorConfig Optional constructor configuration - * @returns Resolved configuration - */ - resolve(constructorConfig?: any): any { - this.sources = [] - - // Load from all sources in priority order - this.loadDefaults() - this.loadFromFiles() - this.loadFromEnvironment() - - if (constructorConfig) { - this.sources.push({ - priority: ConfigPriority.CONSTRUCTOR, - source: 'constructor', - config: constructorConfig - }) - } - - // Merge configurations by priority - this.resolved = this.mergeConfigurations() - - // Validate against schema if provided - if (this.options.schema) { - this.validateConfiguration(this.resolved) - } - - return this.resolved - } - - /** - * Load default values from schema and defaults - */ - private loadDefaults(): void { - let defaults: any = {} - - // Load from provided defaults - if (this.options.defaults) { - defaults = { ...defaults, ...this.options.defaults } - } - - // Load from schema defaults - if (this.options.schema?.properties) { - for (const [key, prop] of Object.entries(this.options.schema.properties)) { - if (prop.default !== undefined && defaults[key] === undefined) { - defaults[key] = prop.default - } - } - } - - if (Object.keys(defaults).length > 0) { - this.sources.push({ - priority: ConfigPriority.DEFAULT, - source: 'defaults', - config: defaults - }) - } - } - - /** - * Load configuration from files - */ - private loadFromFiles(): void { - // Skip in browser environment - if (!isNode()) { - return - } - - try { - const nodeRequire = typeof require !== 'undefined' ? require : null - if (!nodeRequire) return - - const fs = nodeRequire('node:fs') - - for (const configPath of this.options.configPaths || []) { - try { - if (fs.existsSync(configPath)) { - const content = fs.readFileSync(configPath, 'utf8') - const config = this.parseConfigFile(content, configPath) - - // Extract augmentation-specific configuration - const augConfig = this.extractAugmentationConfig(config) - - if (augConfig && Object.keys(augConfig).length > 0) { - this.sources.push({ - priority: ConfigPriority.FILE, - source: `file:${configPath}`, - config: augConfig - }) - break // Use first found config file - } - } - } catch (error) { - // Silently ignore file errors - console.debug(`Failed to load config from ${configPath}:`, error) - } - } - } catch { - // Silently continue if require fails - } - } - - /** - * Parse configuration file based on extension - */ - private parseConfigFile(content: string, filepath: string): any { - try { - // Try JSON first - return JSON.parse(content) - } catch { - // Try other formats in the future (YAML, TOML, etc.) - throw new Error(`Unable to parse config file: ${filepath}`) - } - } - - /** - * Extract augmentation-specific configuration from a config object - */ - private extractAugmentationConfig(config: any): any { - const augId = this.options.augmentationId - - // Check for augmentations section - if (config.augmentations && config.augmentations[augId]) { - return config.augmentations[augId] - } - - // Check for direct augmentation config (prefixed keys) - const prefix = `${augId}.` - const augConfig: any = {} - - for (const [key, value] of Object.entries(config)) { - if (key.startsWith(prefix)) { - const configKey = key.slice(prefix.length) - augConfig[configKey] = value - } - } - - return Object.keys(augConfig).length > 0 ? augConfig : null - } - - /** - * Load configuration from environment variables - */ - private loadFromEnvironment(): void { - // Skip in browser environment - if (!isNode() || !process.env) { - return - } - - const prefix = this.options.envPrefix! - const envConfig: any = {} - - for (const [key, value] of Object.entries(process.env)) { - if (key.startsWith(prefix)) { - const configKey = this.envKeyToConfigKey(key.slice(prefix.length)) - envConfig[configKey] = this.parseEnvValue(value as string) - } - } - - if (Object.keys(envConfig).length > 0) { - this.sources.push({ - priority: ConfigPriority.ENVIRONMENT, - source: 'environment', - config: envConfig - }) - } - } - - /** - * Convert environment variable key to config key - * ENABLED -> enabled - * MAX_SIZE -> maxSize - */ - private envKeyToConfigKey(envKey: string): string { - return envKey - .toLowerCase() - .replace(/_([a-z])/g, (_, letter) => letter.toUpperCase()) - } - - /** - * Parse environment variable value - */ - private parseEnvValue(value: string): any { - // Handle empty strings - if (value === '') return value - - // Try to parse as JSON - try { - return JSON.parse(value) - } catch { - // Check for boolean strings - if (value.toLowerCase() === 'true') return true - if (value.toLowerCase() === 'false') return false - - // Check for number strings - const num = Number(value) - if (!isNaN(num) && value.trim() !== '') return num - - // Return as string - return value - } - } - - /** - * Merge configurations by priority - */ - private mergeConfigurations(): any { - // Sort by priority (lowest to highest) - const sorted = [...this.sources].sort((a, b) => a.priority - b.priority) - - // Merge configurations - let merged = {} - for (const source of sorted) { - merged = this.deepMerge(merged, source.config) - } - - return merged - } - - /** - * Deep merge two objects - */ - private deepMerge(target: any, source: any): any { - const output = { ...target } - - for (const key in source) { - if (source.hasOwnProperty(key)) { - if (source[key] && typeof source[key] === 'object' && !Array.isArray(source[key])) { - if (target[key] && typeof target[key] === 'object' && !Array.isArray(target[key])) { - output[key] = this.deepMerge(target[key], source[key]) - } else { - output[key] = source[key] - } - } else { - output[key] = source[key] - } - } - } - - return output - } - - /** - * Validate configuration against schema - */ - private validateConfiguration(config: any): void { - if (!this.options.schema) return - - const schema = this.options.schema - const errors: string[] = [] - - // Check required fields - if (schema.required) { - for (const field of schema.required) { - if (config[field] === undefined) { - errors.push(`Missing required field: ${field}`) - } - } - } - - // Validate properties - if (schema.properties) { - for (const [key, propSchema] of Object.entries(schema.properties)) { - const value = config[key] - if (value !== undefined) { - this.validateProperty(key, value, propSchema, errors) - } - } - } - - // Check for additional properties - if (schema.additionalProperties === false) { - const allowedKeys = Object.keys(schema.properties || {}) - for (const key of Object.keys(config)) { - if (!allowedKeys.includes(key)) { - errors.push(`Unknown configuration property: ${key}`) - } - } - } - - if (errors.length > 0) { - throw new Error(`Configuration validation failed for ${this.options.augmentationId}:\n${errors.join('\n')}`) - } - } - - /** - * Validate a single property against its schema - */ - private validateProperty(key: string, value: any, schema: JSONSchema, errors: string[]): void { - // Type validation - if (schema.type) { - const actualType = Array.isArray(value) ? 'array' : typeof value - if (actualType !== schema.type) { - errors.push(`${key}: expected ${schema.type}, got ${actualType}`) - return - } - } - - // Number validations - if (schema.type === 'number') { - if (schema.minimum !== undefined && value < schema.minimum) { - errors.push(`${key}: value ${value} is less than minimum ${schema.minimum}`) - } - if (schema.maximum !== undefined && value > schema.maximum) { - errors.push(`${key}: value ${value} is greater than maximum ${schema.maximum}`) - } - } - - // String validations - if (schema.type === 'string') { - if (schema.minLength !== undefined && value.length < schema.minLength) { - errors.push(`${key}: length ${value.length} is less than minimum ${schema.minLength}`) - } - if (schema.maxLength !== undefined && value.length > schema.maxLength) { - errors.push(`${key}: length ${value.length} is greater than maximum ${schema.maxLength}`) - } - if (schema.pattern) { - const regex = new RegExp(schema.pattern) - if (!regex.test(value)) { - errors.push(`${key}: value does not match pattern ${schema.pattern}`) - } - } - } - - // Enum validation - if (schema.enum && !schema.enum.includes(value)) { - errors.push(`${key}: value ${value} is not one of allowed values: ${schema.enum.join(', ')}`) - } - } - - /** - * Get configuration sources for debugging - */ - getSources(): ConfigSource[] { - return [...this.sources] - } - - /** - * Get resolved configuration - */ - getResolved(): any { - return { ...this.resolved } - } - - /** - * Update configuration at runtime - */ - updateRuntime(config: any): any { - // Add or update runtime source - const runtimeIndex = this.sources.findIndex(s => s.priority === ConfigPriority.RUNTIME) - - if (runtimeIndex >= 0) { - this.sources[runtimeIndex].config = { - ...this.sources[runtimeIndex].config, - ...config - } - } else { - this.sources.push({ - priority: ConfigPriority.RUNTIME, - source: 'runtime', - config - }) - } - - // Re-merge configurations - this.resolved = this.mergeConfigurations() - - // Validate - if (this.options.schema) { - this.validateConfiguration(this.resolved) - } - - return this.resolved - } - - /** - * Save configuration to file - * @param filepath Path to save configuration - * @param format Format to save as (json, etc.) - */ - async saveToFile(filepath?: string, format: 'json' = 'json'): Promise { - // Skip in browser environment - if (!isNode()) { - throw new Error('Cannot save configuration files in browser environment') - } - - const fs = await import('node:fs') - const path = await import('node:path') - - const configPath = filepath || this.options.configPaths?.[0] || '.brainyrc' - const augId = this.options.augmentationId - - // Load existing config if it exists - let fullConfig: any = {} - try { - if (fs.existsSync(configPath)) { - const content = fs.readFileSync(configPath, 'utf8') - fullConfig = JSON.parse(content) - } - } catch { - // Start with empty config - } - - // Ensure augmentations section exists - if (!fullConfig.augmentations) { - fullConfig.augmentations = {} - } - - // Update augmentation config - fullConfig.augmentations[augId] = this.resolved - - // Save based on format - let content: string - if (format === 'json') { - content = JSON.stringify(fullConfig, null, 2) - } else { - throw new Error(`Unsupported format: ${format}`) - } - - // Ensure directory exists - const dir = path.dirname(configPath) - if (!fs.existsSync(dir)) { - fs.mkdirSync(dir, { recursive: true }) - } - - // Write file - fs.writeFileSync(configPath, content, 'utf8') - } - - /** - * Get environment variable names for this augmentation - */ - getEnvironmentVariables(): Record { - const schema = this.options.schema - const prefix = this.options.envPrefix! - const vars: Record = {} - - if (schema?.properties) { - for (const [key, prop] of Object.entries(schema.properties)) { - const envKey = prefix + key.replace(/([A-Z])/g, '_$1').toUpperCase() - vars[envKey] = { - description: prop.description, - type: prop.type, - default: prop.default, - currentValue: process.env?.[envKey] - } - } - } - - return vars - } -} \ No newline at end of file diff --git a/src/augmentations/connectionPoolAugmentation.ts b/src/augmentations/connectionPoolAugmentation.ts deleted file mode 100644 index 7ce89db3..00000000 --- a/src/augmentations/connectionPoolAugmentation.ts +++ /dev/null @@ -1,445 +0,0 @@ -/** - * Connection Pool Augmentation - * - * Provides 10-20x throughput improvement for cloud storage (S3, R2, GCS) - * Manages connection pooling, request queuing, and parallel processing - * Critical for enterprise-scale operations with millions of entries - */ - -import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js' - -interface ConnectionPoolConfig { - enabled?: boolean - maxConnections?: number // Maximum concurrent connections - minConnections?: number // Minimum idle connections - acquireTimeout?: number // Timeout for getting connection (ms) - idleTimeout?: number // Connection idle timeout (ms) - maxQueueSize?: number // Maximum queued requests - retryAttempts?: number // Retry failed requests - healthCheckInterval?: number // Connection health check (ms) -} - -interface PooledConnection { - id: string - connection: any // Actual storage client (S3, R2, etc) - isIdle: boolean - lastUsed: number - healthScore: number - activeRequests: number - requestCount: number // Total requests handled -} - -interface QueuedRequest { - id: string - operation: string - params: any - executor: () => Promise - resolver: (value: T) => void - rejector: (error: Error) => void - timestamp: number - priority: number -} - -export class ConnectionPoolAugmentation extends BaseAugmentation { - name = 'ConnectionPool' - timing = 'around' as const - metadata = 'none' as const // Connection pooling doesn't access metadata - operations = ['storage'] as ('storage')[] - priority = 95 // Very high priority for storage operations - - protected config: Required - private connections: Map = new Map() - private requestQueue: QueuedRequest[] = [] - private healthCheckInterval?: NodeJS.Timeout - private storageType: string = 'unknown' - private stats = { - totalRequests: 0, - queuedRequests: 0, - activeConnections: 0, - totalConnections: 0, - averageLatency: 0, - throughputPerSecond: 0 - } - - constructor(config: ConnectionPoolConfig = {}) { - super() - this.config = { - enabled: config.enabled ?? true, - maxConnections: config.maxConnections ?? 50, - minConnections: config.minConnections ?? 5, - acquireTimeout: config.acquireTimeout ?? 30000, // 30s - idleTimeout: config.idleTimeout ?? 300000, // 5 minutes - maxQueueSize: config.maxQueueSize ?? 10000, - retryAttempts: config.retryAttempts ?? 3, - healthCheckInterval: config.healthCheckInterval ?? 60000 // 1 minute - } - } - - protected async onInitialize(): Promise { - if (!this.config.enabled) { - this.log('Connection pooling disabled') - return - } - - // Detect storage type - this.storageType = this.detectStorageType() - - if (this.isCloudStorage()) { - await this.initializeConnectionPool() - this.startHealthChecks() - this.startMetricsCollection() - this.log(`Connection pool initialized for ${this.storageType}: ${this.config.minConnections}-${this.config.maxConnections} connections`) - } else { - this.log(`Connection pooling skipped for ${this.storageType} (local storage)`) - } - } - - shouldExecute(operation: string, params: any): boolean { - return this.config.enabled && - this.isCloudStorage() && - this.isStorageOperation(operation) - } - - async execute( - operation: string, - params: any, - next: () => Promise - ): Promise { - if (!this.shouldExecute(operation, params)) { - return next() - } - - const startTime = Date.now() - this.stats.totalRequests++ - - try { - // High priority for critical operations - const priority = this.getOperationPriority(operation) - - // Execute with pooled connection - const result = await this.executeWithPool(operation, params, next, priority) - - // Update metrics - const latency = Date.now() - startTime - this.updateLatencyMetrics(latency) - - return result - } catch (error) { - this.log(`Connection pool error for ${operation}: ${error}`, 'error') - - // Fallback to direct execution for reliability - return next() - } - } - - private detectStorageType(): string { - const storage = this.context?.storage - if (!storage) return 'unknown' - - const className = storage.constructor.name.toLowerCase() - if (className.includes('s3')) return 's3' - if (className.includes('r2')) return 'r2' - if (className.includes('gcs') || className.includes('google')) return 'gcs' - if (className.includes('azure')) return 'azure' - if (className.includes('filesystem')) return 'filesystem' - if (className.includes('memory')) return 'memory' - - return 'unknown' - } - - private isCloudStorage(): boolean { - return ['s3', 'r2', 'gcs', 'azure'].includes(this.storageType) - } - - private isStorageOperation(operation: string): boolean { - return operation.includes('save') || - operation.includes('get') || - operation.includes('delete') || - operation.includes('list') || - operation.includes('backup') || - operation.includes('restore') - } - - private getOperationPriority(operation: string): number { - // Critical operations get highest priority - if (operation.includes('save') || operation.includes('update')) return 10 - if (operation.includes('delete')) return 9 - if (operation.includes('get')) return 7 - if (operation.includes('list')) return 5 - if (operation.includes('backup')) return 3 - return 1 - } - - private async executeWithPool( - operation: string, - params: any, - executor: () => Promise, - priority: number - ): Promise { - // Check queue size - if (this.requestQueue.length >= this.config.maxQueueSize) { - throw new Error('Connection pool queue full - system overloaded') - } - - // Try to get available connection immediately - const connection = await this.getOrCreateConnection() - if (connection && connection.isIdle) { - return this.executeWithConnection(connection, operation, executor) - } - - // Queue the request with the actual executor - return this.queueRequest(operation, params, executor, priority) - } - - private getAvailableConnection(): PooledConnection | null { - // Find idle connection with best health score - let bestConnection: PooledConnection | null = null - let bestScore = -1 - - for (const connection of this.connections.values()) { - if (connection.isIdle && connection.healthScore > bestScore) { - bestConnection = connection - bestScore = connection.healthScore - } - } - - return bestConnection - } - - private async executeWithConnection( - connection: PooledConnection, - operation: string, - executor: () => Promise - ): Promise { - // Mark connection as active - connection.isIdle = false - connection.activeRequests++ - connection.lastUsed = Date.now() - this.stats.activeConnections++ - - try { - const result = await executor() - - // Update connection health on success - connection.healthScore = Math.min(connection.healthScore + 1, 100) - - return result - } catch (error) { - // Decrease health on failure - connection.healthScore = Math.max(connection.healthScore - 5, 0) - throw error - } finally { - // Release connection - connection.isIdle = true - connection.activeRequests-- - this.stats.activeConnections-- - - // Process next queued request - this.processQueue() - } - } - - private async queueRequest( - operation: string, - params: any, - executor: () => Promise, - priority: number - ): Promise { - return new Promise((resolve, reject) => { - const request: QueuedRequest = { - id: `req_${Date.now()}_${Math.random()}`, - operation, - params, - executor, // Store the actual executor function - resolver: resolve, - rejector: reject, - timestamp: Date.now(), - priority - } - - // Insert by priority (higher priority first) - const insertIndex = this.requestQueue.findIndex(r => r.priority < priority) - if (insertIndex === -1) { - this.requestQueue.push(request) - } else { - this.requestQueue.splice(insertIndex, 0, request) - } - - this.stats.queuedRequests++ - - // Set timeout - setTimeout(() => { - const index = this.requestQueue.findIndex(r => r.id === request.id) - if (index !== -1) { - this.requestQueue.splice(index, 1) - this.stats.queuedRequests-- - reject(new Error(`Connection pool timeout: ${this.config.acquireTimeout}ms`)) - } - }, this.config.acquireTimeout) - }) - } - - private processQueue(): void { - if (this.requestQueue.length === 0) return - - const connection = this.getAvailableConnection() - if (!connection) return - - const request = this.requestQueue.shift()! - this.stats.queuedRequests-- - - // Execute queued request with the REAL executor - this.executeWithConnection(connection, request.operation, request.executor) - .then(request.resolver) - .catch(request.rejector) - } - - private async initializeConnectionPool(): Promise { - // Create minimum connections - for (let i = 0; i < this.config.minConnections; i++) { - await this.createConnection() - } - } - - private async createConnection(): Promise { - const connectionId = `conn_${Date.now()}_${Math.random()}` - - // Create actual connection based on storage type - const actualConnection = await this.createStorageConnection() - - const connection: PooledConnection = { - id: connectionId, - connection: actualConnection, - isIdle: true, - lastUsed: Date.now(), - healthScore: 100, - activeRequests: 0, - requestCount: 0 - } - - this.connections.set(connectionId, connection) - this.stats.totalConnections++ - - return connection - } - - private async createStorageConnection(): Promise { - // For cloud storage, reuse the existing storage instance - // Connection pooling in this context means managing concurrent requests - // not creating multiple storage instances (which would be wasteful) - const storage = this.context?.storage - if (!storage) { - throw new Error('Storage not available for connection pooling') - } - - // Return a connection wrapper that tracks usage - return { - storage, - created: Date.now(), - requestCount: 0 - } - } - - private async getOrCreateConnection(): Promise { - // Try to get an available connection - let connection = this.getAvailableConnection() - - // If no connection available and under max, create new one - if (!connection && this.connections.size < this.config.maxConnections) { - connection = await this.createConnection() - } - - return connection - } - - private startHealthChecks(): void { - this.healthCheckInterval = setInterval(() => { - this.performHealthChecks() - }, this.config.healthCheckInterval) - } - - private performHealthChecks(): void { - const now = Date.now() - const toRemove: string[] = [] - - for (const [id, connection] of this.connections) { - // Remove idle connections that are too old - if (connection.isIdle && - now - connection.lastUsed > this.config.idleTimeout && - this.connections.size > this.config.minConnections) { - toRemove.push(id) - } - - // Remove unhealthy connections - if (connection.healthScore < 20) { - toRemove.push(id) - } - } - - // Remove unhealthy/old connections - for (const id of toRemove) { - this.connections.delete(id) - this.stats.totalConnections-- - } - - // Ensure minimum connections - while (this.connections.size < this.config.minConnections) { - this.createConnection() - } - } - - private startMetricsCollection(): void { - setInterval(() => { - this.updateThroughputMetrics() - }, 1000) // Update every second - } - - private updateLatencyMetrics(latency: number): void { - // Simple moving average - this.stats.averageLatency = (this.stats.averageLatency * 0.9) + (latency * 0.1) - } - - private updateThroughputMetrics(): void { - // Reset counter for next second - this.stats.throughputPerSecond = this.stats.totalRequests - // Reset total for next measurement (in practice, use sliding window) - } - - /** - * Get connection pool statistics - */ - getStats(): typeof this.stats & { - queueSize: number - activeConnections: number - totalConnections: number - poolUtilization: string - storageType: string - } { - return { - ...this.stats, - queueSize: this.requestQueue.length, - activeConnections: this.stats.activeConnections, - totalConnections: this.connections.size, - poolUtilization: `${Math.round((this.stats.activeConnections / this.connections.size) * 100)}%`, - storageType: this.storageType - } - } - - protected async onShutdown(): Promise { - if (this.healthCheckInterval) { - clearInterval(this.healthCheckInterval) - } - - // Close all connections - this.connections.clear() - - // Reject all queued requests - this.requestQueue.forEach(request => { - request.rejector(new Error('Connection pool shutting down')) - }) - this.requestQueue = [] - - const stats = this.getStats() - this.log(`Connection pool shutdown: ${stats.totalRequests} requests processed, ${stats.poolUtilization} avg utilization`) - } -} \ No newline at end of file diff --git a/src/augmentations/defaultAugmentations.ts b/src/augmentations/defaultAugmentations.ts deleted file mode 100644 index 43903720..00000000 --- a/src/augmentations/defaultAugmentations.ts +++ /dev/null @@ -1,130 +0,0 @@ -/** - * Default Augmentations Registration - * - * Maintains zero-config philosophy by automatically registering - * core augmentations that were previously hardcoded in Brainy. - * - * These augmentations are optional but enabled by default for - * backward compatibility and optimal performance. - */ - -import { Brainy } from '../brainy.js' -import { BaseAugmentation } from './brainyAugmentation.js' -import { CacheAugmentation } from './cacheAugmentation.js' -import { MetricsAugmentation } from './metricsAugmentation.js' -import { MonitoringAugmentation } from './monitoringAugmentation.js' -import { UniversalDisplayAugmentation } from './universalDisplayAugmentation.js' -import { IntelligentImportAugmentation } from './intelligentImport/index.js' - -/** - * Create default augmentations for zero-config operation - * Returns an array of augmentations to be registered - * - * @param config - Configuration options - * @returns Array of augmentations to register - */ -export function createDefaultAugmentations( - config: { - cache?: boolean | Record - metrics?: boolean | Record - monitoring?: boolean | Record - display?: boolean | Record - intelligentImport?: boolean | Record - } = {} -): BaseAugmentation[] { - const augmentations: BaseAugmentation[] = [] - - // Intelligent Import augmentation (CSV, Excel, PDF) - if (config.intelligentImport !== false) { - const importConfig = typeof config.intelligentImport === 'object' ? config.intelligentImport : {} - augmentations.push(new IntelligentImportAugmentation(importConfig)) - } - - // Cache augmentation (was SearchCache) - if (config.cache !== false) { - const cacheConfig = typeof config.cache === 'object' ? config.cache : {} - augmentations.push(new CacheAugmentation(cacheConfig)) - } - - // Note: Index augmentation removed - metadata indexing is now core functionality - - // Metrics augmentation (was StatisticsCollector) - if (config.metrics !== false) { - const metricsConfig = typeof config.metrics === 'object' ? config.metrics : {} - augmentations.push(new MetricsAugmentation(metricsConfig)) - } - - // Display augmentation (AI-powered intelligent display fields) - if (config.display !== false) { - const displayConfig = typeof config.display === 'object' ? config.display : {} - augmentations.push(new UniversalDisplayAugmentation(displayConfig)) - } - - // Monitoring augmentation (was HealthMonitor) - // Only enable by default in distributed mode - const isDistributed = process.env.BRAINY_MODE === 'distributed' || - process.env.BRAINY_DISTRIBUTED === 'true' - - if (config.monitoring !== false && (config.monitoring || isDistributed)) { - const monitoringConfig = typeof config.monitoring === 'object' ? config.monitoring : {} - augmentations.push(new MonitoringAugmentation(monitoringConfig)) - } - - return augmentations -} - -/** - * Get augmentation by name with type safety - */ -export function getAugmentation(brain: Brainy, name: string): T | null { - // Access augmentations through a public method or property - const augmentations = (brain as any).augmentations - if (!augmentations) return null - const aug = augmentations.get(name) - return aug as T | null -} - -/** - * Compatibility helpers for migrating from hardcoded features - */ -export const AugmentationHelpers = { - /** - * Get cache augmentation - */ - getCache(brain: Brainy): CacheAugmentation | null { - return getAugmentation(brain, 'cache') - }, - - /** - * Note: Index augmentation removed - metadata indexing is now core functionality - * Use brain.metadataIndex directly instead - */ - - /** - * Get metrics augmentation - */ - getMetrics(brain: Brainy): MetricsAugmentation | null { - return getAugmentation(brain, 'metrics') - }, - - /** - * Get monitoring augmentation - */ - getMonitoring(brain: Brainy): MonitoringAugmentation | null { - return getAugmentation(brain, 'monitoring') - }, - - /** - * Get display augmentation - */ - getDisplay(brain: Brainy): UniversalDisplayAugmentation | null { - return getAugmentation(brain, 'display') - }, - - /** - * Get intelligent import augmentation - */ - getIntelligentImport(brain: Brainy): IntelligentImportAugmentation | null { - return getAugmentation(brain, 'intelligent-import') - } -} \ No newline at end of file diff --git a/src/augmentations/discovery.ts b/src/augmentations/discovery.ts deleted file mode 100644 index daf8f421..00000000 --- a/src/augmentations/discovery.ts +++ /dev/null @@ -1,560 +0,0 @@ -/** - * Augmentation Discovery API - * - * Provides discovery and configuration capabilities for augmentations - * Enables tools like brain-cloud to dynamically discover, configure, and manage augmentations - */ - -import { AugmentationRegistry } from './brainyAugmentation.js' -import { AugmentationManifest, JSONSchema } from './manifest.js' -import { AugmentationConfigResolver } from './configResolver.js' - -/** - * Augmentation listing with manifest and status - */ -export interface AugmentationListing { - id: string - name: string - manifest: AugmentationManifest - status: { - enabled: boolean - initialized: boolean - category: string - priority: number - } - config?: { - current: any - schema?: JSONSchema - sources?: any[] - } -} - -/** - * Configuration validation result - */ -export interface ConfigValidationResult { - valid: boolean - errors?: string[] - warnings?: string[] - suggestions?: string[] -} - -/** - * Discovery API options - */ -export interface DiscoveryOptions { - includeConfig?: boolean - includeSchema?: boolean - includeSources?: boolean - category?: string - enabled?: boolean -} - -/** - * Augmentation Discovery API - * - * Provides a unified interface for discovering and managing augmentations - */ -export class AugmentationDiscovery { - constructor(private registry: AugmentationRegistry) {} - - /** - * Discover all registered augmentations with manifests - * @param options Discovery options - * @returns List of augmentation listings - */ - async discover(options: DiscoveryOptions = {}): Promise { - const augmentations = this.registry.getAll() - const listings: AugmentationListing[] = [] - - for (const aug of augmentations) { - // Check if augmentation has manifest support - const hasManifest = 'getManifest' in aug && typeof aug.getManifest === 'function' - - if (!hasManifest) { - // Skip augmentations without manifest support (legacy) - continue - } - - try { - // Check if augmentation has manifest method - if (!('getManifest' in aug) || typeof aug.getManifest !== 'function') { - continue - } - const getManifestFn = aug.getManifest as Function - const manifest = getManifestFn() - - // Apply filters - if (options.category && manifest.category !== options.category) { - continue - } - - if (options.enabled !== undefined) { - const isEnabled = (aug as any).enabled !== false - if (isEnabled !== options.enabled) { - continue - } - } - - // Build listing - const listing: AugmentationListing = { - id: manifest.id, - name: manifest.name, - manifest, - status: { - enabled: (aug as any).enabled !== false, - initialized: (aug as any).isInitialized || false, - category: (aug as any).category || manifest.category, - priority: aug.priority - } - } - - // Include configuration if requested - if (options.includeConfig && 'getConfig' in aug) { - const getConfigFn = aug.getConfig as Function - listing.config = { - current: getConfigFn() - } - - if (options.includeSchema) { - listing.config.schema = manifest.configSchema - } - } - - listings.push(listing) - } catch (error) { - console.warn(`Failed to get manifest for augmentation ${aug.name}:`, error) - } - } - - // Sort by priority (highest first) then by name - listings.sort((a, b) => { - const priorityDiff = b.status.priority - a.status.priority - if (priorityDiff !== 0) return priorityDiff - return a.name.localeCompare(b.name) - }) - - return listings - } - - /** - * Get a specific augmentation's manifest - * @param augId Augmentation ID - * @returns Augmentation manifest or null if not found - */ - async getManifest(augId: string): Promise { - const aug = this.registry.get(augId) - - if (!aug || !('getManifest' in aug)) { - return null - } - - try { - const getManifestFn = aug.getManifest as Function - return getManifestFn() - } catch (error) { - console.error(`Failed to get manifest for ${augId}:`, error) - return null - } - } - - /** - * Get configuration schema for an augmentation - * @param augId Augmentation ID - * @returns Configuration schema or null - */ - async getConfigSchema(augId: string): Promise { - const manifest = await this.getManifest(augId) - return manifest?.configSchema || null - } - - /** - * Get current configuration for an augmentation - * @param augId Augmentation ID - * @returns Current configuration or null - */ - async getConfig(augId: string): Promise { - const aug = this.registry.get(augId) - - if (!aug || !('getConfig' in aug)) { - return null - } - - try { - const getConfigFn = aug.getConfig as Function - return getConfigFn() - } catch (error) { - console.error(`Failed to get config for ${augId}:`, error) - return null - } - } - - /** - * Update configuration for an augmentation - * @param augId Augmentation ID - * @param config New configuration - * @returns Updated configuration or null on failure - */ - async updateConfig(augId: string, config: any): Promise { - const aug = this.registry.get(augId) - - if (!aug || !('updateConfig' in aug) || !('getConfig' in aug)) { - throw new Error(`Augmentation ${augId} does not support configuration updates`) - } - - try { - const updateConfigFn = aug.updateConfig as Function - await updateConfigFn(config) - const getConfigFn = aug.getConfig as Function - return getConfigFn() - } catch (error) { - throw new Error(`Failed to update config for ${augId}: ${error}`) - } - } - - /** - * Validate configuration against schema - * @param augId Augmentation ID - * @param config Configuration to validate - * @returns Validation result - */ - async validateConfig(augId: string, config: any): Promise { - const schema = await this.getConfigSchema(augId) - - if (!schema) { - return { - valid: true, - warnings: ['No schema available for validation'] - } - } - - const errors: string[] = [] - const warnings: string[] = [] - const suggestions: string[] = [] - - // Check required fields - if (schema.required) { - for (const field of schema.required) { - if (config[field] === undefined) { - errors.push(`Missing required field: ${field}`) - } - } - } - - // Validate properties - if (schema.properties) { - for (const [key, propSchema] of Object.entries(schema.properties)) { - const value = config[key] - - if (value === undefined) { - // Check if there's a default - if (propSchema.default !== undefined) { - suggestions.push(`Field '${key}' not provided, will use default: ${JSON.stringify(propSchema.default)}`) - } - continue - } - - // Type validation - if (propSchema.type) { - const actualType = Array.isArray(value) ? 'array' : typeof value - if (actualType !== propSchema.type) { - errors.push(`${key}: expected ${propSchema.type}, got ${actualType}`) - } - } - - // Additional validations for specific types - this.validatePropertyValue(key, value, propSchema, errors, warnings) - } - } - - // Check for unknown properties - if (schema.additionalProperties === false && schema.properties) { - const allowedKeys = Object.keys(schema.properties) - for (const key of Object.keys(config)) { - if (!allowedKeys.includes(key)) { - warnings.push(`Unknown property: ${key}`) - } - } - } - - return { - valid: errors.length === 0, - errors: errors.length > 0 ? errors : undefined, - warnings: warnings.length > 0 ? warnings : undefined, - suggestions: suggestions.length > 0 ? suggestions : undefined - } - } - - /** - * Validate a property value against its schema - */ - private validatePropertyValue( - key: string, - value: any, - schema: JSONSchema, - errors: string[], - warnings: string[] - ): void { - // Number validations - if (schema.type === 'number') { - if (schema.minimum !== undefined && value < schema.minimum) { - errors.push(`${key}: value ${value} is less than minimum ${schema.minimum}`) - } - if (schema.maximum !== undefined && value > schema.maximum) { - errors.push(`${key}: value ${value} is greater than maximum ${schema.maximum}`) - } - } - - // String validations - if (schema.type === 'string') { - if (schema.minLength !== undefined && value.length < schema.minLength) { - errors.push(`${key}: length ${value.length} is less than minimum ${schema.minLength}`) - } - if (schema.maxLength !== undefined && value.length > schema.maxLength) { - errors.push(`${key}: length ${value.length} is greater than maximum ${schema.maxLength}`) - } - if (schema.pattern) { - const regex = new RegExp(schema.pattern) - if (!regex.test(value)) { - errors.push(`${key}: value does not match pattern ${schema.pattern}`) - } - } - } - - // Enum validation - if (schema.enum && !schema.enum.includes(value)) { - errors.push(`${key}: value '${value}' is not one of allowed values: ${schema.enum.join(', ')}`) - } - } - - /** - * Get environment variables for an augmentation - * @param augId Augmentation ID - * @returns Map of environment variable names to descriptions - */ - async getEnvironmentVariables(augId: string): Promise | null> { - const manifest = await this.getManifest(augId) - - if (!manifest?.configSchema?.properties) { - return null - } - - const prefix = `BRAINY_AUG_${augId.toUpperCase()}_` - const vars: Record = {} - - for (const [key, prop] of Object.entries(manifest.configSchema.properties)) { - const envKey = prefix + key.replace(/([A-Z])/g, '_$1').toUpperCase() - vars[envKey] = { - configKey: key, - description: prop.description, - type: prop.type, - default: prop.default, - required: manifest.configSchema.required?.includes(key), - currentValue: typeof process !== 'undefined' ? process.env?.[envKey] : undefined - } - } - - return vars - } - - /** - * Get configuration examples for an augmentation - * @param augId Augmentation ID - * @returns Configuration examples or empty array - */ - async getConfigExamples(augId: string): Promise { - const manifest = await this.getManifest(augId) - return manifest?.configExamples || [] - } - - /** - * Check if an augmentation supports configuration - * @param augId Augmentation ID - * @returns True if augmentation supports configuration - */ - async supportsConfiguration(augId: string): Promise { - const aug = this.registry.get(augId) - return !!(aug && 'getConfig' in aug && 'updateConfig' in aug) - } - - /** - * Get augmentations by category - * @param category Category to filter by - * @returns List of augmentations in the category - */ - async getByCategory(category: string): Promise { - return this.discover({ category }) - } - - /** - * Get enabled augmentations - * @returns List of enabled augmentations - */ - async getEnabled(): Promise { - return this.discover({ enabled: true }) - } - - /** - * Search augmentations by keyword - * @param query Search query - * @returns Matching augmentations - */ - async search(query: string): Promise { - const all = await this.discover() - const queryLower = query.toLowerCase() - - return all.filter(listing => { - const manifest = listing.manifest - - // Search in various fields - const searchFields = [ - manifest.name, - manifest.description, - manifest.longDescription, - ...(manifest.keywords || []), - manifest.category - ].filter(Boolean).map(s => s!.toLowerCase()) - - return searchFields.some(field => field.includes(queryLower)) - }) - } - - /** - * Export configuration for all augmentations - * @returns Map of augmentation IDs to configurations - */ - async exportConfigurations(): Promise> { - const configs: Record = {} - const listings = await this.discover({ includeConfig: true }) - - for (const listing of listings) { - if (listing.config?.current) { - configs[listing.id] = listing.config.current - } - } - - return configs - } - - /** - * Import configurations for multiple augmentations - * @param configs Map of augmentation IDs to configurations - * @returns Results of import operation - */ - async importConfigurations(configs: Record): Promise> { - const results: Record = {} - - for (const [augId, config] of Object.entries(configs)) { - try { - // Validate before applying - const validation = await this.validateConfig(augId, config) - - if (!validation.valid) { - results[augId] = { - success: false, - error: `Validation failed: ${validation.errors?.join(', ')}` - } - continue - } - - // Apply configuration - await this.updateConfig(augId, config) - results[augId] = { success: true } - } catch (error) { - results[augId] = { - success: false, - error: error instanceof Error ? error.message : String(error) - } - } - } - - return results - } - - /** - * Generate configuration documentation - * @param augId Augmentation ID - * @returns Markdown documentation - */ - async generateConfigDocs(augId: string): Promise { - const manifest = await this.getManifest(augId) - if (!manifest) return null - - const schema = manifest.configSchema - const examples = manifest.configExamples || [] - const envVars = await this.getEnvironmentVariables(augId) - - let docs = `# ${manifest.name} Configuration\n\n` - docs += `${manifest.description}\n\n` - - if (manifest.longDescription) { - docs += `## Overview\n\n${manifest.longDescription}\n\n` - } - - // Configuration options - if (schema?.properties) { - docs += `## Configuration Options\n\n` - - for (const [key, prop] of Object.entries(schema.properties)) { - const required = schema.required?.includes(key) ? ' *(required)*' : '' - docs += `### \`${key}\`${required}\n\n` - - if (prop.description) { - docs += `${prop.description}\n\n` - } - - docs += `- **Type**: ${prop.type}\n` - - if (prop.default !== undefined) { - docs += `- **Default**: \`${JSON.stringify(prop.default)}\`\n` - } - - if (prop.minimum !== undefined) { - docs += `- **Minimum**: ${prop.minimum}\n` - } - - if (prop.maximum !== undefined) { - docs += `- **Maximum**: ${prop.maximum}\n` - } - - if (prop.enum) { - docs += `- **Allowed values**: ${prop.enum.map(v => `\`${v}\``).join(', ')}\n` - } - - docs += '\n' - } - } - - // Environment variables - if (envVars && Object.keys(envVars).length > 0) { - docs += `## Environment Variables\n\n` - docs += `| Variable | Config Key | Type | Required | Default |\n` - docs += `|----------|------------|------|----------|----------|\n` - - for (const [envKey, info] of Object.entries(envVars)) { - docs += `| \`${envKey}\` | ${info.configKey} | ${info.type} | ${info.required ? 'Yes' : 'No'} | ${info.default !== undefined ? `\`${info.default}\`` : '-'} |\n` - } - - docs += '\n' - } - - // Examples - if (examples.length > 0) { - docs += `## Examples\n\n` - - for (const example of examples) { - docs += `### ${example.name}\n\n` - - if (example.description) { - docs += `${example.description}\n\n` - } - - docs += '```json\n' - docs += JSON.stringify(example.config, null, 2) - docs += '\n```\n\n' - } - } - - return docs - } -} \ No newline at end of file diff --git a/src/augmentations/discovery/catalogDiscovery.ts b/src/augmentations/discovery/catalogDiscovery.ts deleted file mode 100644 index edcd2dad..00000000 --- a/src/augmentations/discovery/catalogDiscovery.ts +++ /dev/null @@ -1,357 +0,0 @@ -/** - * Brain-Cloud Catalog Discovery - * - * Discovers augmentations available in the brain-cloud catalog - * Handles free, premium, and community augmentations - */ - -import { AugmentationManifest } from '../manifest.js' - -export interface CatalogAugmentation { - id: string - name: string - description: string - longDescription?: string - category: string - status: 'available' | 'coming_soon' | 'deprecated' - tier: 'free' | 'premium' | 'enterprise' - price?: { - monthly?: number - yearly?: number - oneTime?: number - } - manifest?: AugmentationManifest - source: 'catalog' - cdnUrl?: string - npmPackage?: string - githubRepo?: string - author?: { - name: string - url?: string - } - metrics?: { - installations: number - rating: number - reviews: number - } - requirements?: { - minBrainyVersion?: string - maxBrainyVersion?: string - dependencies?: string[] - } -} - -export interface CatalogOptions { - apiUrl?: string - apiKey?: string - cache?: boolean - cacheTimeout?: number -} - -export interface CatalogFilters { - category?: string - tier?: 'free' | 'premium' | 'enterprise' - status?: 'available' | 'coming_soon' | 'deprecated' - search?: string - installed?: boolean - minRating?: number -} - -/** - * Brain-Cloud Catalog Discovery - */ -export class CatalogDiscovery { - private apiUrl: string - private apiKey?: string - private cache: Map = new Map() - private cacheTimeout: number - - constructor(options: CatalogOptions = {}) { - this.apiUrl = options.apiUrl || 'https://api.soulcraft.com/brain-cloud' - this.apiKey = options.apiKey - this.cacheTimeout = options.cacheTimeout || 5 * 60 * 1000 // 5 minutes - } - - /** - * Discover augmentations from catalog - */ - async discover(filters: CatalogFilters = {}): Promise { - const cacheKey = JSON.stringify(filters) - - // Check cache - if (this.cache.has(cacheKey)) { - const cached = this.cache.get(cacheKey)! - if (Date.now() - cached.timestamp < this.cacheTimeout) { - return cached.data - } - } - - // Build query parameters - const params = new URLSearchParams() - if (filters.category) params.append('category', filters.category) - if (filters.tier) params.append('tier', filters.tier) - if (filters.status) params.append('status', filters.status) - if (filters.search) params.append('q', filters.search) - if (filters.minRating) params.append('minRating', filters.minRating.toString()) - - // Fetch from API - const response = await fetch(`${this.apiUrl}/augmentations/discover?${params}`, { - headers: this.getHeaders() - }) - - if (!response.ok) { - throw new Error(`Failed to fetch catalog: ${response.statusText}`) - } - - const data = await response.json() - const augmentations = this.transformCatalogData(data) - - // Cache result - this.cache.set(cacheKey, { - data: augmentations, - timestamp: Date.now() - }) - - return augmentations - } - - /** - * Get specific augmentation details - */ - async getAugmentation(id: string): Promise { - const response = await fetch(`${this.apiUrl}/augmentations/${id}`, { - headers: this.getHeaders() - }) - - if (!response.ok) { - if (response.status === 404) return null - throw new Error(`Failed to fetch augmentation: ${response.statusText}`) - } - - const data = await response.json() - return this.transformAugmentation(data) - } - - /** - * Get augmentation manifest - */ - async getManifest(id: string): Promise { - const response = await fetch(`${this.apiUrl}/augmentations/${id}/manifest`, { - headers: this.getHeaders() - }) - - if (!response.ok) { - if (response.status === 404) return null - throw new Error(`Failed to fetch manifest: ${response.statusText}`) - } - - return response.json() - } - - /** - * Get CDN URL for dynamic loading - */ - async getCDNUrl(id: string): Promise { - const aug = await this.getAugmentation(id) - return aug?.cdnUrl || null - } - - /** - * Check if user has access to augmentation - */ - async checkAccess(id: string): Promise<{ - hasAccess: boolean - requiresPurchase?: boolean - requiredTier?: string - }> { - if (!this.apiKey) { - // No API key, only free augmentations - const aug = await this.getAugmentation(id) - return { - hasAccess: aug?.tier === 'free', - requiresPurchase: aug?.tier !== 'free', - requiredTier: aug?.tier - } - } - - const response = await fetch(`${this.apiUrl}/augmentations/${id}/access`, { - headers: this.getHeaders() - }) - - if (!response.ok) { - throw new Error(`Failed to check access: ${response.statusText}`) - } - - return response.json() - } - - /** - * Purchase/activate augmentation - */ - async purchase(id: string, licenseKey?: string): Promise<{ - success: boolean - cdnUrl?: string - npmPackage?: string - licenseKey?: string - }> { - const body = licenseKey ? { licenseKey } : {} - - const response = await fetch(`${this.apiUrl}/augmentations/${id}/purchase`, { - method: 'POST', - headers: { - ...this.getHeaders(), - 'Content-Type': 'application/json' - }, - body: JSON.stringify(body) - }) - - if (!response.ok) { - throw new Error(`Failed to purchase: ${response.statusText}`) - } - - return response.json() - } - - /** - * Get user's purchased augmentations - */ - async getPurchased(): Promise { - if (!this.apiKey) { - return [] - } - - const response = await fetch(`${this.apiUrl}/user/augmentations`, { - headers: this.getHeaders() - }) - - if (!response.ok) { - throw new Error(`Failed to fetch purchased: ${response.statusText}`) - } - - const data = await response.json() - return this.transformCatalogData(data) - } - - /** - * Get categories - */ - async getCategories(): Promise> { - const response = await fetch(`${this.apiUrl}/augmentations/categories`, { - headers: this.getHeaders() - }) - - if (!response.ok) { - throw new Error(`Failed to fetch categories: ${response.statusText}`) - } - - return response.json() - } - - /** - * Search augmentations - */ - async search(query: string): Promise { - return this.discover({ search: query }) - } - - /** - * Get trending augmentations - */ - async getTrending(limit: number = 10): Promise { - const response = await fetch(`${this.apiUrl}/augmentations/trending?limit=${limit}`, { - headers: this.getHeaders() - }) - - if (!response.ok) { - throw new Error(`Failed to fetch trending: ${response.statusText}`) - } - - const data = await response.json() - return this.transformCatalogData(data) - } - - /** - * Get recommended augmentations - */ - async getRecommended(): Promise { - if (!this.apiKey) { - // Return popular free augmentations - return this.discover({ tier: 'free', minRating: 4 }) - } - - const response = await fetch(`${this.apiUrl}/augmentations/recommended`, { - headers: this.getHeaders() - }) - - if (!response.ok) { - throw new Error(`Failed to fetch recommended: ${response.statusText}`) - } - - const data = await response.json() - return this.transformCatalogData(data) - } - - /** - * Transform catalog data - */ - private transformCatalogData(data: any[]): CatalogAugmentation[] { - return data.map(item => this.transformAugmentation(item)) - } - - /** - * Transform single augmentation - */ - private transformAugmentation(item: any): CatalogAugmentation { - return { - id: item.id, - name: item.name, - description: item.description, - longDescription: item.longDescription, - category: item.category, - status: item.status || 'available', - tier: item.tier || 'free', - price: item.price, - manifest: item.manifest, - source: 'catalog', - cdnUrl: item.cdnUrl || `https://cdn.soulcraft.com/augmentations/${item.id}@latest`, - npmPackage: item.npmPackage, - githubRepo: item.githubRepo, - author: item.author, - metrics: item.metrics, - requirements: item.requirements - } - } - - /** - * Get request headers - */ - private getHeaders(): HeadersInit { - const headers: HeadersInit = {} - - if (this.apiKey) { - headers['Authorization'] = `Bearer ${this.apiKey}` - } - - return headers - } - - /** - * Clear cache - */ - clearCache(): void { - this.cache.clear() - } - - /** - * Set API key - */ - setApiKey(apiKey: string): void { - this.apiKey = apiKey - this.clearCache() // Clear cache when API key changes - } -} \ No newline at end of file diff --git a/src/augmentations/discovery/localDiscovery.ts b/src/augmentations/discovery/localDiscovery.ts deleted file mode 100644 index 32466cfb..00000000 --- a/src/augmentations/discovery/localDiscovery.ts +++ /dev/null @@ -1,315 +0,0 @@ -/** - * Local Augmentation Discovery - * - * Discovers augmentations installed locally in node_modules - * and built-in augmentations that ship with Brainy - * - * NOTE: This is a Node.js-only feature that requires filesystem access - */ - -import { AugmentationManifest } from '../manifest.js' - -// Node.js modules - dynamically imported to avoid bundler issues -let fs: any = null -let path: any = null - -// Load Node.js modules if available -if (typeof window === 'undefined') { - try { - fs = await import('node:fs') - path = await import('node:path') - } catch (e) { - // Will throw error in methods if not available - } -} - -// Create compatibility layer for sync methods -const existsSync = fs?.existsSync || (() => { throw new Error('Filesystem not available') }) -const readdirSync = fs?.readdirSync || (() => { throw new Error('Filesystem not available') }) -const readFileSync = fs?.readFileSync || (() => { throw new Error('Filesystem not available') }) -const join = path?.join || ((...args: string[]) => args.join('/')) - -export interface LocalAugmentation { - id: string - name: string - source: 'builtin' | 'npm' | 'local' - path: string - manifest?: AugmentationManifest - package?: { - name: string - version: string - description?: string - } -} - -/** - * Discovers augmentations installed locally - */ -export class LocalAugmentationDiscovery { - private builtInAugmentations: Map = new Map() - private installedAugmentations: Map = new Map() - - constructor(private options: { - brainyPath?: string - projectPath?: string - scanNodeModules?: boolean - } = {}) { - this.options = { - brainyPath: this.options.brainyPath || this.findBrainyPath(), - projectPath: this.options.projectPath || process.cwd(), - scanNodeModules: this.options.scanNodeModules ?? true - } - - // Register built-in augmentations - this.registerBuiltIn() - } - - /** - * Register built-in augmentations that ship with Brainy - */ - private registerBuiltIn(): void { - const builtIn = [ - { id: 'cache', name: 'Cache', path: 'cacheAugmentation' }, - { id: 'batch', name: 'Batch Processing', path: 'batchProcessingAugmentation' }, - { id: 'entity-registry', name: 'Entity Registry', path: 'entityRegistryAugmentation' }, - { id: 'index', name: 'Index', path: 'indexAugmentation' }, - { id: 'metrics', name: 'Metrics', path: 'metricsAugmentation' }, - { id: 'monitoring', name: 'Monitoring', path: 'monitoringAugmentation' }, - { id: 'connection-pool', name: 'Connection Pool', path: 'connectionPoolAugmentation' }, - { id: 'request-deduplicator', name: 'Request Deduplicator', path: 'requestDeduplicatorAugmentation' }, - { id: 'api-server', name: 'API Server', path: 'apiServerAugmentation' }, - { id: 'neural-import', name: 'Neural Import', path: 'neuralImport' }, - { id: 'intelligent-verb-scoring', name: 'Intelligent Verb Scoring', path: 'intelligentVerbScoringAugmentation' }, - { id: 'universal-display', name: 'Universal Display', path: 'universalDisplayAugmentation' }, - // Storage augmentations - { id: 'memory-storage', name: 'Memory Storage', path: 'storageAugmentations', export: 'MemoryStorageAugmentation' }, - { id: 'filesystem-storage', name: 'FileSystem Storage', path: 'storageAugmentations', export: 'FileSystemStorageAugmentation' }, - { id: 'opfs-storage', name: 'OPFS Storage', path: 'storageAugmentations', export: 'OPFSStorageAugmentation' }, - { id: 's3-storage', name: 'S3 Storage', path: 'storageAugmentations', export: 'S3StorageAugmentation' }, - ] - - for (const aug of builtIn) { - this.builtInAugmentations.set(aug.id, { - id: aug.id, - name: aug.name, - source: 'builtin', - path: `@soulcraft/brainy/augmentations/${aug.path}`, - package: { - name: '@soulcraft/brainy', - version: 'builtin', - description: `Built-in ${aug.name} augmentation` - } - }) - } - } - - /** - * Find Brainy installation path - */ - private findBrainyPath(): string { - // Try to find brainy in node_modules - const possiblePaths = [ - join(process.cwd(), 'node_modules', '@soulcraft', 'brainy'), - join(process.cwd(), 'node_modules', 'brainy'), - join(process.cwd(), '..', 'node_modules', '@soulcraft', 'brainy'), - ] - - for (const path of possiblePaths) { - if (existsSync(path)) { - return path - } - } - - // Fallback to current directory - return process.cwd() - } - - /** - * Discover all augmentations - */ - async discoverAll(): Promise { - const augmentations: LocalAugmentation[] = [] - - // Add built-in augmentations - augmentations.push(...this.builtInAugmentations.values()) - - // Scan node_modules if enabled - if (this.options.scanNodeModules) { - const installed = await this.scanNodeModules() - augmentations.push(...installed) - } - - // Scan local project - const local = await this.scanLocalProject() - augmentations.push(...local) - - return augmentations - } - - /** - * Scan node_modules for installed augmentations - */ - private async scanNodeModules(): Promise { - const augmentations: LocalAugmentation[] = [] - const nodeModulesPath = join(this.options.projectPath!, 'node_modules') - - if (!existsSync(nodeModulesPath)) { - return augmentations - } - - // Scan @brainy/* packages - const brainyPath = join(nodeModulesPath, '@brainy') - if (existsSync(brainyPath)) { - const packages = readdirSync(brainyPath) - for (const pkg of packages) { - const augmentation = await this.loadPackageAugmentation(join(brainyPath, pkg)) - if (augmentation) { - augmentations.push(augmentation) - } - } - } - - // Scan packages with brainy-augmentation keyword - const packages = readdirSync(nodeModulesPath) - for (const pkg of packages) { - if (pkg.startsWith('@') || pkg.startsWith('.')) continue - - const pkgPath = join(nodeModulesPath, pkg) - const packageJson = this.loadPackageJson(pkgPath) - - if (packageJson?.keywords?.includes('brainy-augmentation')) { - const augmentation = await this.loadPackageAugmentation(pkgPath) - if (augmentation) { - augmentations.push(augmentation) - } - } - } - - return augmentations - } - - /** - * Scan local project for augmentations - */ - private async scanLocalProject(): Promise { - const augmentations: LocalAugmentation[] = [] - - // Check for augmentations directory - const augPath = join(this.options.projectPath!, 'augmentations') - if (existsSync(augPath)) { - const files = readdirSync(augPath) - for (const file of files) { - if (file.endsWith('.ts') || file.endsWith('.js')) { - const name = file.replace(/\.(ts|js)$/, '') - augmentations.push({ - id: name, - name: this.humanizeName(name), - source: 'local', - path: join(augPath, file) - }) - } - } - } - - return augmentations - } - - /** - * Load augmentation from package - */ - private async loadPackageAugmentation(pkgPath: string): Promise { - const packageJson = this.loadPackageJson(pkgPath) - if (!packageJson) return null - - // Check if it's a brainy augmentation - const isBrainyAug = packageJson.keywords?.includes('brainy-augmentation') || - packageJson.brainy?.type === 'augmentation' - - if (!isBrainyAug) return null - - const manifest = packageJson.brainy?.manifest || null - - return { - id: packageJson.brainy?.id || packageJson.name.replace('@brainy/', ''), - name: packageJson.brainy?.name || packageJson.name, - source: 'npm', - path: pkgPath, - manifest, - package: { - name: packageJson.name, - version: packageJson.version, - description: packageJson.description - } - } - } - - /** - * Load package.json - */ - private loadPackageJson(pkgPath: string): any { - const packageJsonPath = join(pkgPath, 'package.json') - if (!existsSync(packageJsonPath)) return null - - try { - return JSON.parse(readFileSync(packageJsonPath, 'utf8')) - } catch { - return null - } - } - - /** - * Convert name to human-readable format - */ - private humanizeName(name: string): string { - return name - .replace(/[-_]/g, ' ') - .replace(/augmentation/gi, '') - .replace(/\b\w/g, l => l.toUpperCase()) - .trim() - } - - /** - * Get built-in augmentations - */ - getBuiltIn(): LocalAugmentation[] { - return Array.from(this.builtInAugmentations.values()) - } - - /** - * Get installed augmentations - */ - getInstalled(): LocalAugmentation[] { - return Array.from(this.installedAugmentations.values()) - } - - /** - * Check if augmentation is installed - */ - isInstalled(id: string): boolean { - return this.builtInAugmentations.has(id) || - this.installedAugmentations.has(id) - } - - /** - * Get import path for augmentation - */ - getImportPath(id: string): string | null { - const aug = this.builtInAugmentations.get(id) || - this.installedAugmentations.get(id) - return aug?.path || null - } - - /** - * Load augmentation module dynamically - */ - async loadAugmentation(id: string): Promise { - const path = this.getImportPath(id) - if (!path) { - throw new Error(`Augmentation ${id} not found`) - } - - // Dynamic import - const module = await import(path) - return module.default || module - } -} \ No newline at end of file diff --git a/src/augmentations/discovery/runtimeLoader.ts b/src/augmentations/discovery/runtimeLoader.ts deleted file mode 100644 index 2a72c936..00000000 --- a/src/augmentations/discovery/runtimeLoader.ts +++ /dev/null @@ -1,443 +0,0 @@ -/** - * Runtime Augmentation Loader - * - * Dynamically loads and registers augmentations at runtime - * Supports CDN loading for browser environments and npm imports for Node.js - */ - -import { BrainyAugmentation, AugmentationRegistry } from '../brainyAugmentation.js' -import { AugmentationManifest } from '../manifest.js' - -export interface LoaderOptions { - cdnUrl?: string - allowUnsafe?: boolean - sandbox?: boolean - timeout?: number - cache?: boolean -} - -export interface LoadedAugmentation { - id: string - instance: BrainyAugmentation - manifest: AugmentationManifest - source: 'cdn' | 'npm' | 'local' - loadTime: number -} - -/** - * Runtime Augmentation Loader - * - * Enables dynamic loading of augmentations from various sources - */ -export class RuntimeAugmentationLoader { - private loaded: Map = new Map() - private cdnCache: Map = new Map() - private registry?: AugmentationRegistry - - constructor(private options: LoaderOptions = {}) { - this.options = { - cdnUrl: options.cdnUrl || 'https://cdn.soulcraft.com/augmentations', - allowUnsafe: options.allowUnsafe || false, - sandbox: options.sandbox || true, - timeout: options.timeout || 30000, - cache: options.cache ?? true - } - } - - /** - * Set the augmentation registry - */ - setRegistry(registry: AugmentationRegistry): void { - this.registry = registry - } - - /** - * Load augmentation from CDN (browser) - */ - async loadFromCDN( - id: string, - version: string = 'latest', - config?: any - ): Promise { - // Check if already loaded - if (this.loaded.has(id)) { - return this.loaded.get(id)! - } - - const url = `${this.options.cdnUrl}/${id}@${version}/index.js` - const startTime = Date.now() - - try { - // Load module from CDN - const module = await this.loadCDNModule(url) - - // Extract augmentation class - const AugmentationClass = module.default || module[Object.keys(module)[0]] - - if (!AugmentationClass) { - throw new Error(`No augmentation class found in module ${id}`) - } - - // Create instance - const instance = new AugmentationClass(config) - - // Validate it's a proper augmentation - if (!this.isValidAugmentation(instance)) { - throw new Error(`Invalid augmentation: ${id}`) - } - - // Get manifest - const manifest = instance.getManifest ? instance.getManifest() : { - id, - name: id, - version: version, - description: `Dynamically loaded ${id}`, - category: 'external' - } - - const loaded: LoadedAugmentation = { - id, - instance, - manifest, - source: 'cdn', - loadTime: Date.now() - startTime - } - - // Cache - this.loaded.set(id, loaded) - - // Auto-register if registry is set - if (this.registry) { - this.registry.register(instance) - } - - return loaded - } catch (error) { - throw new Error(`Failed to load augmentation ${id} from CDN: ${error}`) - } - } - - /** - * Load augmentation from NPM (Node.js) - */ - async loadFromNPM( - packageName: string, - config?: any - ): Promise { - // Check if already loaded - const id = packageName.replace('@', '').replace('/', '-') - if (this.loaded.has(id)) { - return this.loaded.get(id)! - } - - const startTime = Date.now() - - try { - // Dynamic import - const module = await import(packageName) - - // Extract augmentation class - const AugmentationClass = module.default || module[Object.keys(module)[0]] - - if (!AugmentationClass) { - throw new Error(`No augmentation class found in package ${packageName}`) - } - - // Create instance - const instance = new AugmentationClass(config) - - // Validate - if (!this.isValidAugmentation(instance)) { - throw new Error(`Invalid augmentation in package: ${packageName}`) - } - - // Get manifest - const manifest = instance.getManifest ? instance.getManifest() : { - id, - name: packageName, - version: 'unknown', - description: `Loaded from ${packageName}`, - category: 'external' - } - - const loaded: LoadedAugmentation = { - id, - instance, - manifest, - source: 'npm', - loadTime: Date.now() - startTime - } - - // Cache - this.loaded.set(id, loaded) - - // Auto-register - if (this.registry) { - this.registry.register(instance) - } - - return loaded - } catch (error) { - throw new Error(`Failed to load augmentation from NPM ${packageName}: ${error}`) - } - } - - /** - * Load augmentation from local file - */ - async loadFromFile( - path: string, - config?: any - ): Promise { - const startTime = Date.now() - - try { - // Dynamic import - const module = await import(path) - - // Extract augmentation class - const AugmentationClass = module.default || module[Object.keys(module)[0]] - - if (!AugmentationClass) { - throw new Error(`No augmentation class found in file ${path}`) - } - - // Create instance - const instance = new AugmentationClass(config) - - // Validate - if (!this.isValidAugmentation(instance)) { - throw new Error(`Invalid augmentation in file: ${path}`) - } - - // Extract ID from path - const id = path.split('/').pop()?.replace(/\.(js|ts)$/, '') || 'unknown' - - // Get manifest - const manifest = instance.getManifest ? instance.getManifest() : { - id, - name: id, - version: 'local', - description: `Loaded from ${path}`, - category: 'local' - } - - const loaded: LoadedAugmentation = { - id, - instance, - manifest, - source: 'local', - loadTime: Date.now() - startTime - } - - // Cache - this.loaded.set(id, loaded) - - // Auto-register - if (this.registry) { - this.registry.register(instance) - } - - return loaded - } catch (error) { - throw new Error(`Failed to load augmentation from file ${path}: ${error}`) - } - } - - /** - * Load multiple augmentations - */ - async loadBatch( - augmentations: Array<{ - source: 'cdn' | 'npm' | 'local' - id: string - version?: string - path?: string - config?: any - }> - ): Promise { - const results = await Promise.allSettled( - augmentations.map(aug => { - switch (aug.source) { - case 'cdn': - return this.loadFromCDN(aug.id, aug.version, aug.config) - case 'npm': - return this.loadFromNPM(aug.id, aug.config) - case 'local': - return this.loadFromFile(aug.path || aug.id, aug.config) - default: - return Promise.reject(new Error(`Unknown source: ${aug.source}`)) - } - }) - ) - - const loaded: LoadedAugmentation[] = [] - const errors: string[] = [] - - for (const result of results) { - if (result.status === 'fulfilled') { - loaded.push(result.value) - } else { - errors.push(result.reason.message) - } - } - - if (errors.length > 0) { - console.warn('Some augmentations failed to load:', errors) - } - - return loaded - } - - /** - * Unload augmentation - */ - unload(id: string): boolean { - const loaded = this.loaded.get(id) - if (!loaded) return false - - // Shutdown if possible - if (loaded.instance.shutdown) { - loaded.instance.shutdown() - } - - // Remove from registry if set - // Note: Registry doesn't have unregister yet, would need to add - - // Remove from cache - this.loaded.delete(id) - - return true - } - - /** - * Get loaded augmentations - */ - getLoaded(): LoadedAugmentation[] { - return Array.from(this.loaded.values()) - } - - /** - * Check if augmentation is loaded - */ - isLoaded(id: string): boolean { - return this.loaded.has(id) - } - - /** - * Get loaded augmentation - */ - getAugmentation(id: string): BrainyAugmentation | null { - return this.loaded.get(id)?.instance || null - } - - /** - * Load CDN module (browser-specific) - */ - private async loadCDNModule(url: string): Promise { - // Check cache - if (this.options.cache && this.cdnCache.has(url)) { - return this.cdnCache.get(url) - } - - // In browser environment - if (typeof window !== 'undefined') { - return new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - reject(new Error(`Timeout loading ${url}`)) - }, this.options.timeout!) - - // Create script element - const script = document.createElement('script') - script.type = 'module' - script.src = url - - // Handle load - script.onload = async () => { - clearTimeout(timeout) - - // The module should register itself on window - const moduleId = url.split('/').pop()?.split('@')[0] - if (moduleId && (window as any)[moduleId]) { - const module = (window as any)[moduleId] - - // Cache - if (this.options.cache) { - this.cdnCache.set(url, module) - } - - resolve(module) - } else { - reject(new Error(`Module not found on window: ${moduleId}`)) - } - } - - // Handle error - script.onerror = () => { - clearTimeout(timeout) - reject(new Error(`Failed to load script: ${url}`)) - } - - // Add to document - document.head.appendChild(script) - }) - } else { - // In Node.js, use dynamic import - const module = await import(url) - - // Cache - if (this.options.cache) { - this.cdnCache.set(url, module) - } - - return module - } - } - - /** - * Validate augmentation instance - */ - private isValidAugmentation(instance: any): boolean { - // Check required properties - return !!( - instance.name && - instance.timing && - instance.operations && - instance.priority !== undefined && - typeof instance.execute === 'function' && - typeof instance.initialize === 'function' - ) - } - - /** - * Clear all caches - */ - clearCache(): void { - this.cdnCache.clear() - } - - /** - * Get load statistics - */ - getStats(): { - loaded: number - totalLoadTime: number - averageLoadTime: number - sources: Record - } { - const loaded = Array.from(this.loaded.values()) - const totalLoadTime = loaded.reduce((sum, aug) => sum + aug.loadTime, 0) - - const sources = loaded.reduce((acc, aug) => { - acc[aug.source] = (acc[aug.source] || 0) + 1 - return acc - }, {} as Record) - - return { - loaded: loaded.length, - totalLoadTime, - averageLoadTime: loaded.length > 0 ? totalLoadTime / loaded.length : 0, - sources - } - } -} \ No newline at end of file diff --git a/src/augmentations/display/cache.ts b/src/augmentations/display/cache.ts deleted file mode 100644 index 3246ace3..00000000 --- a/src/augmentations/display/cache.ts +++ /dev/null @@ -1,375 +0,0 @@ -/** - * Universal Display Augmentation - Intelligent Caching System - * - * High-performance LRU cache with smart eviction and batch optimization - * Designed for minimal memory footprint and maximum hit ratio - */ - -import type { DisplayCacheEntry, ComputedDisplayFields, DisplayAugmentationStats } from './types.js' - -/** - * LRU (Least Recently Used) Cache for computed display fields - * Optimized for the display augmentation use case - */ -export class DisplayCache { - private cache = new Map() - private readonly maxSize: number - private stats = { - hits: 0, - misses: 0, - evictions: 0, - totalComputations: 0, - totalComputationTime: 0 - } - - constructor(maxSize: number = 1000) { - this.maxSize = maxSize - } - - /** - * Get cached display fields with LRU update - * @param key Cache key - * @returns Cached fields or null if not found - */ - get(key: string): ComputedDisplayFields | null { - const entry = this.cache.get(key) - - if (!entry) { - this.stats.misses++ - return null - } - - // Update LRU - move to end - this.cache.delete(key) - entry.lastAccessed = Date.now() - entry.accessCount++ - this.cache.set(key, entry) - - this.stats.hits++ - return entry.fields - } - - /** - * Store computed display fields in cache - * @param key Cache key - * @param fields Computed display fields - * @param computationTime Time taken to compute (for stats) - */ - set(key: string, fields: ComputedDisplayFields, computationTime?: number): void { - // Remove if already exists (for LRU update) - if (this.cache.has(key)) { - this.cache.delete(key) - } - - // Create cache entry - const entry: DisplayCacheEntry = { - fields, - lastAccessed: Date.now(), - accessCount: 1 - } - - // Add to end (most recently used) - this.cache.set(key, entry) - - // Update stats - this.stats.totalComputations++ - if (computationTime) { - this.stats.totalComputationTime += computationTime - } - - // Evict oldest if over capacity - if (this.cache.size > this.maxSize) { - this.evictOldest() - } - } - - /** - * Check if a key exists in cache without affecting LRU order - * @param key Cache key - * @returns True if key exists - */ - has(key: string): boolean { - return this.cache.has(key) - } - - /** - * Generate cache key from data - * @param id Entity ID (preferred) - * @param data Fallback data for key generation - * @param entityType Type of entity (noun/verb) - * @returns Cache key string - */ - generateKey(id?: string, data?: any, entityType: 'noun' | 'verb' = 'noun'): string { - // Use ID if available (most reliable) - if (id) { - return `${entityType}:${id}` - } - - // Generate hash from data - if (data) { - const dataString = JSON.stringify(data, Object.keys(data).sort()) - const hash = this.simpleHash(dataString) - return `${entityType}:hash:${hash}` - } - - // Fallback to timestamp (not ideal but prevents crashes) - return `${entityType}:temp:${Date.now()}:${Math.random()}` - } - - /** - * Clear all cached entries - */ - clear(): void { - this.cache.clear() - this.stats = { - hits: 0, - misses: 0, - evictions: 0, - totalComputations: 0, - totalComputationTime: 0 - } - } - - /** - * Get cache statistics - * @returns Cache performance statistics - */ - getStats(): DisplayAugmentationStats { - const hitRatio = this.stats.hits + this.stats.misses > 0 - ? this.stats.hits / (this.stats.hits + this.stats.misses) - : 0 - - const avgComputationTime = this.stats.totalComputations > 0 - ? this.stats.totalComputationTime / this.stats.totalComputations - : 0 - - // Analyze cached types for common types statistics - const typeCount = new Map() - let fastestComputation = Infinity - let slowestComputation = 0 - - for (const entry of this.cache.values()) { - const type = entry.fields.type - typeCount.set(type, (typeCount.get(type) || 0) + 1) - } - - const commonTypes = Array.from(typeCount.entries()) - .sort(([,a], [,b]) => b - a) - .slice(0, 10) - .map(([type, count]) => ({ - type, - count, - percentage: Math.round((count / this.cache.size) * 100) - })) - - return { - totalComputations: this.stats.totalComputations, - cacheHitRatio: Math.round(hitRatio * 100) / 100, - averageComputationTime: Math.round(avgComputationTime * 100) / 100, - commonTypes, - performance: { - fastestComputation, - slowestComputation, - totalComputationTime: this.stats.totalComputationTime - } - } - } - - /** - * Get current cache size - * @returns Number of cached entries - */ - size(): number { - return this.cache.size - } - - /** - * Get cache capacity - * @returns Maximum cache size - */ - capacity(): number { - return this.maxSize - } - - /** - * Evict least recently used entry - */ - private evictOldest(): void { - // First entry is oldest (LRU) - const firstKey = this.cache.keys().next().value - if (firstKey) { - this.cache.delete(firstKey) - this.stats.evictions++ - } - } - - /** - * Simple hash function for cache keys - * @param str String to hash - * @returns Simple hash number - */ - private simpleHash(str: string): number { - let hash = 0 - for (let i = 0; i < str.length; i++) { - const char = str.charCodeAt(i) - hash = ((hash << 5) - hash) + char - hash = hash & hash // Convert to 32-bit integer - } - return Math.abs(hash) - } - - /** - * Optimize cache by removing stale entries - * Called periodically to maintain cache health - */ - optimizeCache(): void { - const now = Date.now() - const maxAge = 24 * 60 * 60 * 1000 // 24 hours - const minAccessCount = 2 // Minimum access count to keep - - const toDelete: string[] = [] - - for (const [key, entry] of this.cache.entries()) { - // Remove very old entries with low access count - if (now - entry.lastAccessed > maxAge && entry.accessCount < minAccessCount) { - toDelete.push(key) - } - } - - // Remove stale entries - for (const key of toDelete) { - this.cache.delete(key) - this.stats.evictions++ - } - } - - /** - * Precompute display fields for a batch of entities - * @param entities Array of entities with their compute functions - * @returns Promise resolving when batch is complete - */ - async batchPrecompute( - entities: Array<{ - key: string - computeFn: () => Promise - }> - ): Promise { - const promises = entities.map(async ({ key, computeFn }) => { - if (!this.has(key)) { - const startTime = Date.now() - try { - const fields = await computeFn() - const computationTime = Date.now() - startTime - this.set(key, fields, computationTime) - } catch (error) { - console.warn(`Batch precompute failed for key ${key}:`, error) - } - } - }) - - await Promise.all(promises) - } -} - -/** - * Request deduplicator for batch processing - * Prevents duplicate computations for the same data - */ -export class RequestDeduplicator { - private pendingRequests = new Map>() - private readonly batchSize: number - - constructor(batchSize: number = 50) { - this.batchSize = batchSize - } - - /** - * Deduplicate computation request - * @param key Unique key for the computation - * @param computeFn Function to compute the result - * @returns Promise that resolves to the computed fields - */ - async deduplicate( - key: string, - computeFn: () => Promise - ): Promise { - // Return existing promise if already pending - if (this.pendingRequests.has(key)) { - return this.pendingRequests.get(key)! - } - - // Create new computation promise - const promise = computeFn().finally(() => { - // Remove from pending when complete - this.pendingRequests.delete(key) - }) - - this.pendingRequests.set(key, promise) - return promise - } - - /** - * Get number of pending requests - * @returns Number of pending computations - */ - getPendingCount(): number { - return this.pendingRequests.size - } - - /** - * Clear all pending requests - */ - clear(): void { - this.pendingRequests.clear() - } - - /** - * Shutdown the deduplicator - */ - shutdown(): void { - this.clear() - } -} - -/** - * Global cache instance management - * Provides singleton access to display cache - */ -let globalDisplayCache: DisplayCache | null = null - -/** - * Get global display cache instance - * @param maxSize Optional cache size (only used on first call) - * @returns Shared display cache instance - */ -export function getGlobalDisplayCache(maxSize?: number): DisplayCache { - if (!globalDisplayCache) { - globalDisplayCache = new DisplayCache(maxSize) - - // Set up periodic optimization - setInterval(() => { - globalDisplayCache?.optimizeCache() - }, 60 * 60 * 1000) // Every hour - } - - return globalDisplayCache -} - -/** - * Clear global cache (for testing or memory management) - */ -export function clearGlobalDisplayCache(): void { - if (globalDisplayCache) { - globalDisplayCache.clear() - } -} - -/** - * Shutdown global cache and cleanup - */ -export function shutdownGlobalDisplayCache(): void { - if (globalDisplayCache) { - globalDisplayCache.clear() - globalDisplayCache = null - } -} \ No newline at end of file diff --git a/src/augmentations/display/fieldPatterns.ts b/src/augmentations/display/fieldPatterns.ts deleted file mode 100644 index e82d9a4f..00000000 --- a/src/augmentations/display/fieldPatterns.ts +++ /dev/null @@ -1,429 +0,0 @@ -/** - * Universal Display Augmentation - Smart Field Patterns - * - * Intelligent field detection patterns for mapping user data to display fields - * Uses semantic understanding and common naming conventions - */ - -import type { FieldPattern, FieldComputationContext } from './types.js' -import { NounType, VerbType } from '../../types/graphTypes.js' - -/** - * Universal field patterns that work across all data types - * Ordered by confidence level (highest first) - */ -export const UNIVERSAL_FIELD_PATTERNS: FieldPattern[] = [ - // Title/Name Patterns (Highest Priority) - { - fields: ['name', 'title', 'displayName', 'label', 'heading'], - displayField: 'title', - confidence: 0.95 - }, - { - fields: ['firstName', 'lastName', 'fullName', 'realName'], - displayField: 'title', - confidence: 0.9, - applicableTypes: [NounType.Person], - transform: (value: any, context: FieldComputationContext) => { - const { metadata } = context - if (metadata.firstName && metadata.lastName) { - return `${metadata.firstName} ${metadata.lastName}`.trim() - } - return String(value || '') - } - }, - { - fields: ['companyName', 'organizationName', 'orgName', 'businessName'], - displayField: 'title', - confidence: 0.9, - applicableTypes: [NounType.Organization] - }, - { - fields: ['filename', 'fileName', 'documentTitle', 'docName'], - displayField: 'title', - confidence: 0.85, - applicableTypes: [NounType.Document, NounType.File, NounType.Media] - }, - { - fields: ['projectName', 'projectTitle', 'initiative'], - displayField: 'title', - confidence: 0.9, - applicableTypes: [NounType.Project] - }, - { - fields: ['taskName', 'taskTitle', 'action', 'todo'], - displayField: 'title', - confidence: 0.85, - applicableTypes: [NounType.Task] - }, - { - fields: ['subject', 'topic', 'headline', 'caption'], - displayField: 'title', - confidence: 0.8 - }, - - // Description Patterns (High Priority) - { - fields: ['description', 'summary', 'overview', 'details'], - displayField: 'description', - confidence: 0.9 - }, - { - fields: ['bio', 'biography', 'profile', 'about'], - displayField: 'description', - confidence: 0.85, - applicableTypes: [NounType.Person] - }, - { - fields: ['content', 'text', 'body', 'message'], - displayField: 'description', - confidence: 0.8 - }, - { - fields: ['abstract', 'excerpt', 'snippet', 'preview'], - displayField: 'description', - confidence: 0.75 - }, - { - fields: ['notes', 'comments', 'remarks', 'observations'], - displayField: 'description', - confidence: 0.7 - }, - - // Type Patterns (Medium Priority) - { - fields: ['type', 'category', 'classification', 'kind'], - displayField: 'type', - confidence: 0.9 - }, - { - fields: ['nounType', 'entityType', 'objectType'], - displayField: 'type', - confidence: 0.95 - }, - { - fields: ['role', 'position', 'jobTitle', 'occupation'], - displayField: 'type', - confidence: 0.8, - applicableTypes: [NounType.Person], - transform: (value: any) => String(value || 'Person') - }, - { - fields: ['industry', 'sector', 'domain', 'field'], - displayField: 'type', - confidence: 0.7, - applicableTypes: [NounType.Organization] - }, - - // Tag Patterns (Medium Priority) - { - fields: ['tags', 'keywords', 'labels', 'categories'], - displayField: 'tags', - confidence: 0.85 - }, - { - fields: ['topics', 'subjects', 'themes'], - displayField: 'tags', - confidence: 0.8 - } -] - -/** - * Type-specific field patterns for enhanced detection - * Used when we know the specific type of the entity - */ -export const TYPE_SPECIFIC_PATTERNS: Record = { - [NounType.Person]: [ - { - fields: ['email', 'emailAddress', 'contactEmail'], - displayField: 'description', - confidence: 0.7, - transform: (value: any, context: FieldComputationContext) => { - const { metadata } = context - const role = metadata.role || metadata.jobTitle || metadata.position - const company = metadata.company || metadata.organization || metadata.employer - - const parts = [] - if (role) parts.push(role) - if (company) parts.push(`at ${company}`) - if (parts.length === 0 && value) parts.push(`Contact: ${value}`) - - return parts.join(' ') || 'Person' - } - }, - { - fields: ['phone', 'phoneNumber', 'mobile', 'cell'], - displayField: 'tags', - confidence: 0.6 - } - ], - - [NounType.Organization]: [ - { - fields: ['website', 'url', 'homepage', 'domain'], - displayField: 'description', - confidence: 0.7, - transform: (value: any, context: FieldComputationContext) => { - const { metadata } = context - const industry = metadata.industry || metadata.sector - const location = metadata.location || metadata.city || metadata.country - - const parts = [] - if (industry) parts.push(industry) - parts.push('organization') - if (location) parts.push(`in ${location}`) - - return parts.join(' ') - } - }, - { - fields: ['employees', 'size', 'headcount'], - displayField: 'tags', - confidence: 0.6 - } - ], - - [NounType.Project]: [ - { - fields: ['status', 'phase', 'stage', 'state'], - displayField: 'description', - confidence: 0.8, - transform: (value: any, context: FieldComputationContext) => { - const { metadata } = context - const status = String(value || 'active').toLowerCase() - const budget = metadata.budget || metadata.cost - const lead = metadata.lead || metadata.manager || metadata.owner - - const parts = [] - parts.push(status.charAt(0).toUpperCase() + status.slice(1)) - if (metadata.description) parts.push('project') - if (lead) parts.push(`led by ${lead}`) - if (budget) parts.push(`($${parseInt(String(budget)).toLocaleString()} budget)`) - - return parts.join(' ') - } - } - ], - - [NounType.Document]: [ - { - fields: ['author', 'creator', 'writer'], - displayField: 'description', - confidence: 0.7, - transform: (value: any, context: FieldComputationContext) => { - const { metadata } = context - const docType = metadata.type || metadata.category || 'document' - const date = metadata.date || metadata.created || metadata.published - - const parts = [] - if (docType) parts.push(docType) - if (value) parts.push(`by ${value}`) - if (date) { - const dateStr = new Date(date).toLocaleDateString() - parts.push(`(${dateStr})`) - } - - return parts.join(' ') - } - } - ], - - [NounType.Task]: [ - { - fields: ['priority', 'urgency', 'importance'], - displayField: 'tags', - confidence: 0.7 - } - ] -} - -/** - * Get field patterns for a specific entity type - * @param entityType The type of entity (noun or verb) - * @param specificType Optional specific noun/verb type - * @returns Array of applicable field patterns - */ -export function getFieldPatterns(entityType: 'noun' | 'verb', specificType?: string): FieldPattern[] { - const patterns = [...UNIVERSAL_FIELD_PATTERNS] - - if (entityType === 'noun' && specificType && TYPE_SPECIFIC_PATTERNS[specificType]) { - patterns.unshift(...TYPE_SPECIFIC_PATTERNS[specificType]) - } - - return patterns.sort((a, b) => b.confidence - a.confidence) -} - -/** - * Priority fields for different entity types (for AI analysis) - * Used by the BrainyTypes and neural processing - */ -export const TYPE_PRIORITY_FIELDS: Record = { - [NounType.Person]: [ - 'name', 'firstName', 'lastName', 'fullName', 'displayName', - 'email', 'role', 'jobTitle', 'position', 'title', - 'bio', 'description', 'about', 'profile', - 'company', 'organization', 'employer' - ], - - [NounType.Organization]: [ - 'name', 'companyName', 'organizationName', 'title', - 'industry', 'sector', 'domain', 'type', - 'description', 'about', 'summary', - 'location', 'city', 'country', 'headquarters', - 'website', 'url' - ], - - [NounType.Project]: [ - 'name', 'projectName', 'title', 'projectTitle', - 'description', 'summary', 'overview', 'goal', - 'status', 'phase', 'stage', 'state', - 'lead', 'manager', 'owner', 'team', - 'budget', 'timeline', 'deadline' - ], - - [NounType.Document]: [ - 'title', 'filename', 'name', 'subject', - 'content', 'text', 'body', 'summary', - 'author', 'creator', 'writer', - 'type', 'category', 'format', - 'date', 'created', 'published' - ], - - [NounType.Task]: [ - 'title', 'name', 'taskName', 'action', - 'description', 'details', 'notes', - 'status', 'state', 'priority', - 'assignee', 'owner', 'responsible', - 'due', 'deadline', 'dueDate' - ], - - [NounType.Event]: [ - 'name', 'title', 'eventName', - 'description', 'details', 'summary', - 'startDate', 'endDate', 'date', 'time', - 'location', 'venue', 'address', - 'organizer', 'host', 'creator' - ], - - [NounType.Product]: [ - 'name', 'productName', 'title', - 'description', 'summary', 'features', - 'price', 'cost', 'value', - 'category', 'type', 'brand', - 'manufacturer', 'vendor' - ] -} - -/** - * Get priority fields for intelligent analysis - * @param entityType The type of entity - * @param specificType Optional specific type - * @returns Array of priority field names - */ -export function getPriorityFields(entityType: 'noun' | 'verb', specificType?: string): string[] { - if (entityType === 'noun' && specificType && TYPE_PRIORITY_FIELDS[specificType]) { - return TYPE_PRIORITY_FIELDS[specificType] - } - - // Default priority fields for any entity - return [ - 'name', 'title', 'label', 'displayName', - 'description', 'summary', 'about', 'details', - 'type', 'category', 'kind', 'classification', - 'tags', 'keywords', 'labels' - ] -} - -/** - * Smart field value extraction with type-aware processing - * @param data The data object to extract from - * @param pattern The field pattern to apply - * @param context The computation context - * @returns The extracted and processed field value - */ -export function extractFieldValue( - data: any, - pattern: FieldPattern, - context: FieldComputationContext -): any { - // Find the first matching field - let value: any = null - let matchedField: string | null = null - - for (const field of pattern.fields) { - if (data[field] !== undefined && data[field] !== null && data[field] !== '') { - value = data[field] - matchedField = field - break - } - } - - if (value === null) return null - - // Apply transformation if provided - if (pattern.transform) { - try { - return pattern.transform(value, context) - } catch (error) { - console.warn(`Field transformation error for ${matchedField}:`, error) - return String(value) - } - } - - // Default processing based on display field type - switch (pattern.displayField) { - case 'title': - case 'description': - case 'type': - return String(value) - - case 'tags': - if (Array.isArray(value)) return value - if (typeof value === 'string') { - return value.split(/[,;]\s*|\s+/).filter(Boolean) - } - return [String(value)] - - default: - return value - } -} - -/** - * Calculate confidence score for field detection - * @param pattern The field pattern - * @param context The computation context - * @param value The extracted value - * @returns Confidence score (0-1) - */ -export function calculateFieldConfidence( - pattern: FieldPattern, - context: FieldComputationContext, - value: any -): number { - let confidence = pattern.confidence - - // Boost confidence if type matches - if (pattern.applicableTypes && context.typeResult) { - if (pattern.applicableTypes.includes(context.typeResult.type)) { - confidence = Math.min(1.0, confidence + 0.1) - } - } - - // Reduce confidence for empty or very short values - if (typeof value === 'string') { - if (value.length < 2) { - confidence *= 0.5 - } else if (value.length < 5) { - confidence *= 0.8 - } - } - - // Reduce confidence for generic values - const genericValues = ['unknown', 'n/a', 'null', 'undefined', 'default'] - if (typeof value === 'string' && genericValues.includes(value.toLowerCase())) { - confidence *= 0.3 - } - - return Math.max(0, Math.min(1, confidence)) -} \ No newline at end of file diff --git a/src/augmentations/display/iconMappings.ts b/src/augmentations/display/iconMappings.ts deleted file mode 100644 index 14383e0b..00000000 --- a/src/augmentations/display/iconMappings.ts +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Universal Display Augmentation - Clean Display - * - * Simple, clean display without icons - focusing on AI-powered - * titles, descriptions, and smart formatting that matches - * Soulcraft's minimal aesthetic - */ - -import { NounType, VerbType } from '../../types/graphTypes.js' - -/** - * No icon mappings - clean, minimal approach - * The real value is in AI-generated titles and enhanced descriptions, - * not visual clutter that doesn't align with professional aesthetics - */ -export const NOUN_TYPE_ICONS: Record = {} - -/** - * No icon mappings for verbs either - focus on clear relationship descriptions - * Human-readable relationship text is more valuable than symbolic representations - */ -export const VERB_TYPE_ICONS: Record = {} - -/** - * Get icon for a noun type (returns empty string for clean display) - * @param type The noun type - * @returns Empty string (no icons) - */ -export function getNounIcon(type: string): string { - return '' // Clean, no icons -} - -/** - * Get icon for a verb type (returns empty string for clean display) - * @param type The verb type - * @returns Empty string (no icons) - */ -export function getVerbIcon(type: string): string { - return '' // Clean, no icons -} - -/** - * Get coverage statistics (for backwards compatibility) - * @returns Coverage info showing clean approach - */ -export function getIconCoverage() { - return { - nounTypes: { - total: 'Clean display - no icons needed', - covered: 'Focus on AI-powered content' - }, - verbTypes: { - total: 'Clean display - no icons needed', - covered: 'Focus on relationship descriptions' - } - } -} - -/** - * Check if an icon exists for a type (always false for clean display) - * @param type The type to check - * @param entityType Whether it's a noun or verb - * @returns Always false (no icons) - */ -export function hasIcon(type: string, entityType: 'noun' | 'verb' = 'noun'): boolean { - return false // Clean approach - no icons -} - -/** - * Get fallback icon (returns empty string for clean display) - * @param entityType The entity type - * @returns Empty string (no fallback icons) - */ -export function getFallbackIcon(entityType: 'noun' | 'verb' = 'noun'): string { - return '' // Clean, minimal display -} \ No newline at end of file diff --git a/src/augmentations/display/intelligentComputation.ts b/src/augmentations/display/intelligentComputation.ts deleted file mode 100644 index 0765df40..00000000 --- a/src/augmentations/display/intelligentComputation.ts +++ /dev/null @@ -1,539 +0,0 @@ -/** - * Universal Display Augmentation - Intelligent Computation Engine - * - * Leverages existing Brainy AI infrastructure for intelligent field computation: - * - BrainyTypes for semantic type detection - * - Neural Import patterns for field analysis - * - JSON processing utilities for field extraction - * - Existing NounType/VerbType taxonomy (42+127 types) - */ - -import type { - ComputedDisplayFields, - FieldComputationContext, - TypeMatchResult, - DisplayConfig -} from './types.js' -import type { VectorDocument, GraphVerb } from '../../coreTypes.js' -import { BrainyTypes, getBrainyTypes } from '../typeMatching/brainyTypes.js' -import { getNounIcon, getVerbIcon } from './iconMappings.js' -import { - getFieldPatterns, - getPriorityFields, - extractFieldValue, - calculateFieldConfidence -} from './fieldPatterns.js' -import { prepareJsonForVectorization, extractFieldFromJson } from '../../utils/jsonProcessing.js' -import { NounType, VerbType } from '../../types/graphTypes.js' - -/** - * Intelligent field computation engine - * Coordinates AI-powered analysis with fallback heuristics - */ -export class IntelligentComputationEngine { - private typeMatcher: BrainyTypes | null = null - protected config: DisplayConfig - private initialized = false - - constructor(config: DisplayConfig) { - this.config = config - } - - /** - * Initialize the computation engine with AI components - */ - async initialize(): Promise { - if (this.initialized) return - - try { - // ๐Ÿง  LEVERAGE YOUR EXISTING AI INFRASTRUCTURE - this.typeMatcher = await getBrainyTypes() - if (this.typeMatcher) { - console.log('๐ŸŽจ Display computation engine initialized with AI intelligence') - } else { - console.warn('๐ŸŽจ Display computation engine running in basic mode (AI unavailable)') - } - } catch (error) { - console.warn('๐ŸŽจ AI initialization failed, using heuristic fallback:', error) - } - - this.initialized = true - } - - /** - * Compute display fields for a noun using AI-first approach - * @param data The noun data/metadata - * @param id Optional noun ID - * @returns Computed display fields - */ - async computeNounDisplay(data: any, id?: string): Promise { - const startTime = Date.now() - - try { - // ๐ŸŸข PRIMARY PATH: Use your existing AI intelligence - if (this.typeMatcher) { - return await this.computeWithAI(data, 'noun', { id }) - } - - // ๐ŸŸก FALLBACK PATH: Use heuristic patterns - return await this.computeWithHeuristics(data, 'noun', { id }) - - } catch (error) { - console.warn('Display computation failed, using minimal fallback:', error) - return this.createMinimalDisplay(data, 'noun') - } finally { - const computationTime = Date.now() - startTime - if (this.config.debugMode) { - console.log(`Display computation took ${computationTime}ms`) - } - } - } - - /** - * Compute display fields for a verb using AI-first approach - * @param verb The verb/relationship data - * @returns Computed display fields - */ - async computeVerbDisplay(verb: GraphVerb): Promise { - const startTime = Date.now() - - try { - // ๐ŸŸข PRIMARY PATH: Use your existing AI for verb analysis - if (this.typeMatcher) { - return await this.computeVerbWithAI(verb) - } - - // ๐ŸŸก FALLBACK PATH: Use heuristic patterns for verbs - return await this.computeWithHeuristics(verb, 'verb') - - } catch (error) { - console.warn('Verb display computation failed, using minimal fallback:', error) - return this.createMinimalDisplay(verb, 'verb') - } finally { - const computationTime = Date.now() - startTime - if (this.config.debugMode) { - console.log(`Verb display computation took ${computationTime}ms`) - } - } - } - - /** - * AI-powered computation using your existing BrainyTypes - * @param data Entity data/metadata - * @param entityType Type of entity (noun/verb) - * @param options Additional options - * @returns AI-computed display fields - */ - private async computeWithAI( - data: any, - entityType: 'noun' | 'verb', - options: { id?: string } = {} - ): Promise { - - // ๐Ÿง  USE YOUR EXISTING TYPE DETECTION AI - const typeResult = await this.typeMatcher!.matchNounType(data) - - // Create computation context - const context: FieldComputationContext = { - data, - metadata: data, - typeResult, - config: this.config, - entityType - } - - // ๐ŸŸข INTELLIGENT FIELD EXTRACTION using your patterns + AI insights - const displayFields = { - title: await this.computeIntelligentTitle(context), - description: await this.computeIntelligentDescription(context), - type: typeResult.type, - tags: await this.computeIntelligentTags(context), - confidence: typeResult.confidence, - reasoning: this.config.debugMode ? typeResult.reasoning : undefined, - alternatives: this.config.debugMode ? typeResult.alternatives : undefined, - computedAt: Date.now(), - version: '1.0.0' - } - - return displayFields - } - - /** - * AI-powered verb computation using relationship analysis - * @param verb The verb/relationship - * @returns AI-computed display fields - */ - private async computeVerbWithAI(verb: GraphVerb): Promise { - - // ๐Ÿง  USE YOUR EXISTING VERB TYPE DETECTION - const typeResult = await this.typeMatcher!.matchVerbType(verb, 0.7) - - // Create verb computation context - const context: FieldComputationContext = { - data: verb, - metadata: verb.metadata || {}, - typeResult, - config: this.config, - entityType: 'verb', - verbContext: { - sourceId: verb.sourceId, - targetId: verb.targetId, - verbType: verb.type - } - } - - // ๐ŸŸข INTELLIGENT VERB DISPLAY COMPUTATION - const displayFields = { - title: await this.computeVerbTitle(context), - description: await this.computeVerbDescription(context), - type: typeResult.type, - tags: await this.computeVerbTags(context), - relationship: await this.computeHumanReadableRelationship(context), - confidence: typeResult.confidence, - reasoning: this.config.debugMode ? typeResult.reasoning : undefined, - alternatives: this.config.debugMode ? typeResult.alternatives : undefined, - computedAt: Date.now(), - version: '1.0.0' - } - - return displayFields - } - - /** - * Heuristic computation when AI is unavailable - * @param data Entity data - * @param entityType Type of entity - * @param options Additional options - * @returns Heuristically computed display fields - */ - private async computeWithHeuristics( - data: any, - entityType: 'noun' | 'verb', - options: { id?: string } = {} - ): Promise { - - // Use basic type detection - const detectedType = this.detectTypeHeuristically(data, entityType) - const typeResult: TypeMatchResult = { - type: detectedType, - confidence: 0.6, // Lower confidence for heuristics - reasoning: 'Heuristic detection (AI unavailable)', - alternatives: [] - } - - const context: FieldComputationContext = { - data, - metadata: data, - typeResult: typeResult, - config: this.config, - entityType - } - - // Use pattern-based field extraction - const patterns = getFieldPatterns(entityType, detectedType) - - return { - title: this.extractFieldWithPatterns(data, patterns, 'title') || 'Untitled', - description: this.extractFieldWithPatterns(data, patterns, 'description') || 'No description', - type: detectedType, - tags: this.extractFieldWithPatterns(data, patterns, 'tags') || [], - confidence: typeResult.confidence, - reasoning: this.config.debugMode ? typeResult.reasoning : undefined, - computedAt: Date.now(), - version: '1.0.0' - } - } - - /** - * Compute intelligent title using AI insights and your field extraction - * @param context Computation context with AI results - * @returns Computed title - */ - private async computeIntelligentTitle(context: FieldComputationContext): Promise { - const { data, typeResult } = context - - // ๐ŸŸข USE TYPE-SPECIFIC LOGIC based on your NounType taxonomy - switch (typeResult?.type) { - case NounType.Person: - return this.computePersonTitle(data) - - case NounType.Organization: - return this.computeOrganizationTitle(data) - - case NounType.Project: - return this.computeProjectTitle(data) - - case NounType.Document: - return this.computeDocumentTitle(data) - - default: - // ๐ŸŸข LEVERAGE YOUR JSON PROCESSING for unknown types - return this.extractBestTitle(data, typeResult?.type) - } - } - - /** - * Compute intelligent description using AI insights and context - * @param context Computation context - * @returns Enhanced description - */ - private async computeIntelligentDescription(context: FieldComputationContext): Promise { - const { data, typeResult } = context - - // ๐ŸŸข USE YOUR EXISTING JSON PROCESSING for vectorization-quality text - const priorityFields = getPriorityFields('noun', typeResult?.type) - const enhancedText = prepareJsonForVectorization(data, { - priorityFields, - includeFieldNames: false, - maxDepth: 2 - }) - - // Create context-aware description based on type - return this.createContextAwareDescription(data, typeResult, enhancedText) - } - - /** - * Compute intelligent tags using type analysis - * @param context Computation context - * @returns Generated tags array - */ - private async computeIntelligentTags(context: FieldComputationContext): Promise { - const { data, typeResult } = context - const tags: string[] = [] - - // Add type-based tag - if (typeResult?.type) { - tags.push(typeResult.type.toLowerCase()) - } - - // Extract explicit tags from data - const explicitTags = this.extractExplicitTags(data) - tags.push(...explicitTags) - - // Add semantic tags based on AI analysis - if (typeResult && this.typeMatcher) { - const semanticTags = this.generateSemanticTags(data, typeResult) - tags.push(...semanticTags) - } - - // Remove duplicates and return - return [...new Set(tags.filter(Boolean))] - } - - /** - * Compute verb title (relationship summary) - * @param context Verb computation context - * @returns Verb title - */ - private async computeVerbTitle(context: FieldComputationContext): Promise { - const { verbContext, typeResult } = context - - if (!verbContext) return 'Relationship' - - const { sourceId, targetId } = verbContext - const relationshipType = typeResult?.type || 'RelatedTo' - - // Try to get readable names for source and target - // This could be enhanced to actually resolve the entities - return `${sourceId} ${this.getReadableVerbPhrase(relationshipType)} ${targetId}` - } - - /** - * Create minimal display for error cases - * @param data Entity data - * @param entityType Entity type - * @returns Minimal display fields - */ - private createMinimalDisplay(data: any, entityType: 'noun' | 'verb'): ComputedDisplayFields { - return { - title: data.name || data.title || data.id || 'Untitled', - description: data.description || data.summary || 'No description available', - type: entityType === 'noun' ? 'Item' : 'RelatedTo', - tags: [], - confidence: 0.1, // Very low confidence for fallback - computedAt: Date.now(), - version: '1.0.0' - } - } - - // Helper methods for specific noun types - private computePersonTitle(data: any): string { - if (data.firstName && data.lastName) { - return `${data.firstName} ${data.lastName}`.trim() - } - return data.name || data.fullName || data.displayName || data.firstName || data.lastName || 'Person' - } - - private computeOrganizationTitle(data: any): string { - return data.name || data.companyName || data.organizationName || data.title || 'Organization' - } - - private computeProjectTitle(data: any): string { - return data.name || data.projectName || data.title || data.projectTitle || 'Project' - } - - private computeDocumentTitle(data: any): string { - return data.title || data.filename || data.name || data.subject || 'Document' - } - - private extractBestTitle(data: any, type?: string): string { - const titleFields = ['name', 'title', 'displayName', 'label', 'subject', 'heading'] - - for (const field of titleFields) { - if (data[field]) return String(data[field]) - } - - return data.id || Object.keys(data)[0] || 'Untitled' - } - - private createContextAwareDescription(data: any, typeResult?: TypeMatchResult, enhancedText?: string): string { - // Start with basic description fields - const basicDesc = data.description || data.summary || data.about || data.details - - if (basicDesc) return String(basicDesc) - - // Use enhanced text from JSON processing - if (enhancedText && enhancedText.length > 10) { - return enhancedText.substring(0, 200) + (enhancedText.length > 200 ? '...' : '') - } - - // Generate from available fields - const parts = [] - if (data.role) parts.push(data.role) - if (data.company) parts.push(`at ${data.company}`) - if (data.location) parts.push(`in ${data.location}`) - - return parts.length > 0 ? parts.join(' ') : 'No description available' - } - - private extractExplicitTags(data: any): string[] { - const tagFields = ['tags', 'keywords', 'labels', 'categories', 'topics'] - - for (const field of tagFields) { - if (data[field]) { - if (Array.isArray(data[field])) { - return data[field].map(String).filter(Boolean) - } - if (typeof data[field] === 'string') { - return data[field].split(/[,;]\s*|\s+/).filter(Boolean) - } - } - } - - return [] - } - - private generateSemanticTags(data: any, typeResult: TypeMatchResult): string[] { - const tags: string[] = [] - - // Add confidence-based tags - if (typeResult.confidence > 0.9) tags.push('verified') - else if (typeResult.confidence < 0.7) tags.push('uncertain') - - // Add type-specific semantic tags - if (data.status) tags.push(String(data.status).toLowerCase()) - if (data.priority) tags.push(String(data.priority).toLowerCase()) - if (data.category) tags.push(String(data.category).toLowerCase()) - - return tags - } - - private getReadableVerbPhrase(verbType: string): string { - const verbPhrases: Record = { - [VerbType.WorksWith]: 'works with', - [VerbType.MemberOf]: 'is member of', - [VerbType.ReportsTo]: 'reports to', - [VerbType.Owns]: 'owns', - [VerbType.LocatedAt]: 'located at', - [VerbType.Likes]: 'likes', - [VerbType.Follows]: 'follows', - } - - return verbPhrases[verbType] || 'related to' - } - - private async computeVerbDescription(context: FieldComputationContext): Promise { - const { data, verbContext, typeResult } = context - - if (data.description) return String(data.description) - - // Generate contextual description for relationship - if (verbContext && typeResult) { - const parts = [] - const relationshipPhrase = this.getReadableVerbPhrase(typeResult.type) - - if (data.role) parts.push(`Role: ${data.role}`) - if (data.startDate) parts.push(`Since: ${new Date(data.startDate).toLocaleDateString()}`) - if (data.department) parts.push(`Department: ${data.department}`) - - return parts.length > 0 - ? `${relationshipPhrase} - ${parts.join(', ')}` - : `${relationshipPhrase} relationship` - } - - return 'Relationship' - } - - private async computeVerbTags(context: FieldComputationContext): Promise { - const { data, typeResult } = context - const tags = ['relationship'] - - if (typeResult?.type) { - tags.push(typeResult.type.toLowerCase()) - } - - // Add relationship-specific tags - if (data.status) tags.push(String(data.status).toLowerCase()) - if (data.type) tags.push(String(data.type).toLowerCase()) - - return [...new Set(tags)] - } - - private async computeHumanReadableRelationship(context: FieldComputationContext): Promise { - const { verbContext, typeResult } = context - - if (!verbContext || !typeResult) return 'Related' - - const { sourceId, targetId } = verbContext - const phrase = this.getReadableVerbPhrase(typeResult.type) - - return `${sourceId} ${phrase} ${targetId}` - } - - private detectTypeHeuristically(data: any, entityType: 'noun' | 'verb'): string { - if (entityType === 'verb') return VerbType.RelatedTo - - // Basic heuristics for noun types - if (data.firstName || data.lastName || data.email) return NounType.Person - if (data.companyName || data.organization) return NounType.Organization - if (data.filename || data.fileType) return NounType.Document - if (data.projectName || data.initiative) return NounType.Project - if (data.taskName || data.todo) return NounType.Task - if (data.startDate || data.endDate) return NounType.Event - - return 'Item' // Generic fallback - } - - private extractFieldWithPatterns(data: any, patterns: any[], fieldType: string): any { - const relevantPatterns = patterns.filter(p => p.displayField === fieldType) - - for (const pattern of relevantPatterns) { - for (const field of pattern.fields) { - if (data[field]) { - return pattern.transform ? pattern.transform(data[field], { data, config: this.config } as any) : data[field] - } - } - } - - return null - } - - /** - * Shutdown the computation engine - */ - async shutdown(): Promise { - // Cleanup if needed - this.typeMatcher = null - this.initialized = false - } -} \ No newline at end of file diff --git a/src/augmentations/display/types.ts b/src/augmentations/display/types.ts deleted file mode 100644 index a01c7154..00000000 --- a/src/augmentations/display/types.ts +++ /dev/null @@ -1,256 +0,0 @@ -/** - * Universal Display Augmentation - Type Definitions - * - * Clean TypeScript interfaces for the display augmentation system - */ - -import type { VectorDocument, GraphVerb } from '../../coreTypes.js' - -/** - * Configuration interface for the Universal Display Augmentation - */ -export interface DisplayConfig { - /** Enable/disable the augmentation */ - enabled: boolean - - /** LRU cache size for computed display fields */ - cacheSize: number - - /** Use lazy computation (recommended for performance) */ - lazyComputation: boolean - - /** Batch processing size for multiple requests */ - batchSize: number - - /** Minimum confidence threshold for AI type detection */ - confidenceThreshold: number - - /** Silent mode - suppress all console output */ - silent?: boolean - - // No icon configuration needed - clean, minimal approach - - /** Custom field mappings (userField -> displayField) */ - customFieldMappings: Record - - /** Type-specific priority fields for intelligent detection */ - priorityFields: Record - - /** Enable debug mode with reasoning output */ - debugMode: boolean -} - -/** - * Computed display fields for any noun or verb - */ -export interface ComputedDisplayFields { - /** Primary display name (AI-detected best field combination) */ - title: string - - /** Enhanced description with context awareness */ - description: string - - /** Human-readable type name */ - type: string - - // No icon field - clean, minimal approach - - /** Generated display tags for categorization */ - tags: string[] - - /** For verbs: human-readable relationship description */ - relationship?: string - - /** AI confidence score (0-1) */ - confidence: number - - /** Explanation of type detection reasoning (debug mode) */ - reasoning?: string - - /** Alternative type suggestions with confidence scores */ - alternatives?: Array<{ - type: string - confidence: number - }> - - /** Timestamp when fields were computed */ - computedAt: number - - /** Version of augmentation that computed these fields */ - version: string -} - -/** - * Cache entry for computed display fields - */ -export interface DisplayCacheEntry { - fields: ComputedDisplayFields - lastAccessed: number - accessCount: number -} - -/** - * Field computation context passed to computation functions - */ -export interface FieldComputationContext { - /** The original data object */ - data: any - - /** Metadata associated with the object */ - metadata: any - - /** Type detection result from AI */ - typeResult?: TypeMatchResult - - /** Display configuration */ - config: DisplayConfig - - /** Whether this is a noun or verb */ - entityType: 'noun' | 'verb' - - /** For verbs: source and target information */ - verbContext?: { - sourceId: string - targetId: string - verbType?: string - } -} - -/** - * Type matching result from BrainyTypes - */ -export interface TypeMatchResult { - type: string - confidence: number - reasoning: string - alternatives: Array<{ - type: string - confidence: number - }> -} - -/** - * Enhanced VectorDocument with display capabilities - */ -export interface EnhancedVectorDocument extends VectorDocument { - /** - * Get computed display field(s) - * @param field Optional specific field name - * @returns Single field value or all display fields - */ - getDisplay(): Promise - getDisplay(field: keyof ComputedDisplayFields): Promise - - /** - * Get available fields for a specific augmentation namespace - * @param namespace The augmentation namespace (e.g., 'display') - * @returns Array of available field names - */ - getAvailableFields(namespace: string): string[] - - /** - * Get available augmentation namespaces - * @returns Array of available augmentation names - */ - getAvailableAugmentations(): string[] - - /** - * Debug exploration of all computed fields - */ - explore(): Promise -} - -/** - * Enhanced GraphVerb with display capabilities - */ -export interface EnhancedGraphVerb extends GraphVerb { - /** - * Get computed display field(s) for relationships - * @param field Optional specific field name - * @returns Single field value or all display fields - */ - getDisplay(): Promise - getDisplay(field: keyof ComputedDisplayFields): Promise - - /** - * Get available fields for a specific augmentation namespace - * @param namespace The augmentation namespace (e.g., 'display') - * @returns Array of available field names - */ - getAvailableFields(namespace: string): string[] -} - -/** - * Batch computation request for performance optimization - */ -export interface BatchComputationRequest { - id: string - data: any - metadata: any - entityType: 'noun' | 'verb' - verbContext?: { - sourceId: string - targetId: string - verbType?: string - } -} - -/** - * Batch computation result - */ -export interface BatchComputationResult { - id: string - fields: ComputedDisplayFields - error?: string -} - -/** - * Field pattern for intelligent field detection - */ -export interface FieldPattern { - /** Field names that match this pattern */ - fields: string[] - - /** Target display field name */ - displayField: keyof ComputedDisplayFields - - /** Confidence score for this pattern match */ - confidence: number - - /** Optional: specific noun/verb types this applies to */ - applicableTypes?: string[] - - /** Optional: transformation function for the field value */ - transform?: (value: any, context: FieldComputationContext) => string -} - -/** - * Statistics for the display augmentation - */ -export interface DisplayAugmentationStats { - /** Total number of computations performed */ - totalComputations: number - - /** Cache hit ratio */ - cacheHitRatio: number - - /** Average computation time in milliseconds */ - averageComputationTime: number - - /** Type detection accuracy (when ground truth available) */ - typeDetectionAccuracy?: number - - /** Most commonly detected types */ - commonTypes: Array<{ - type: string - count: number - percentage: number - }> - - /** Performance metrics */ - performance: { - fastestComputation: number - slowestComputation: number - totalComputationTime: number - } -} \ No newline at end of file diff --git a/src/augmentations/entityRegistryAugmentation.ts b/src/augmentations/entityRegistryAugmentation.ts deleted file mode 100644 index 2b5b422b..00000000 --- a/src/augmentations/entityRegistryAugmentation.ts +++ /dev/null @@ -1,518 +0,0 @@ -/** - * Entity Registry Augmentation - * Fast external-ID to internal-UUID mapping for streaming data processing - * Works in write-only mode for high-performance deduplication - */ - -import { BrainyAugmentation, BaseAugmentation, AugmentationContext } from './brainyAugmentation.js' - -export interface EntityRegistryConfig { - /** - * Maximum number of entries to keep in memory cache - * Default: 100,000 entries - */ - maxCacheSize?: number - - /** - * Time to live for cache entries in milliseconds - * Default: 300,000 (5 minutes) - */ - cacheTTL?: number - - /** - * Fields to index for fast lookup - * Default: ['did', 'handle', 'uri', 'id', 'external_id'] - */ - indexedFields?: string[] - - /** - * Persistence strategy - * memory: Keep only in memory (fast, but lost on restart) - * storage: Persist to storage (survives restarts) - * hybrid: Memory + periodic storage sync - */ - persistence?: 'memory' | 'storage' | 'hybrid' - - /** - * How often to sync memory cache to storage (hybrid mode) - * Default: 30000 (30 seconds) - */ - syncInterval?: number -} - -export interface EntityMapping { - externalId: string - field: string - brainyId: string - nounType: string - lastAccessed: number - metadata?: any -} - -/** - * High-performance entity registry for external ID to Brainy UUID mapping - * Optimized for streaming data scenarios like Bluesky firehose processing - */ -export class EntityRegistryAugmentation extends BaseAugmentation { - readonly metadata = 'readonly' as const // Reads metadata to register entities - readonly name = 'entity-registry' - readonly description = 'Fast external-ID to internal-UUID mapping for streaming data' - readonly timing: 'before' | 'after' | 'around' | 'replace' = 'before' - readonly operations = ['add', 'addNoun', 'addVerb'] as ('add' | 'addNoun' | 'addVerb')[] - readonly priority = 90 // High priority for entity registration - - protected config: Required - private memoryIndex = new Map() - private fieldIndices = new Map>() // field -> value -> brainyId - private syncTimer?: NodeJS.Timeout - private brain?: any - private storage?: any - private cacheHits = 0 - private cacheMisses = 0 - - constructor(config: EntityRegistryConfig = {}) { - super() - - this.config = { - maxCacheSize: config.maxCacheSize ?? 100000, - cacheTTL: config.cacheTTL ?? 300000, // 5 minutes - indexedFields: config.indexedFields ?? ['did', 'handle', 'uri', 'id', 'external_id'], - persistence: config.persistence ?? 'hybrid', - syncInterval: config.syncInterval ?? 30000 // 30 seconds - } - - // Initialize field indices - for (const field of this.config.indexedFields) { - this.fieldIndices.set(field, new Map()) - } - } - - async initialize(context: AugmentationContext): Promise { - this.brain = context.brain - this.storage = context.storage - - // Load existing mappings from storage - if (this.config.persistence === 'storage' || this.config.persistence === 'hybrid') { - await this.loadFromStorage() - } - - // Start sync timer for hybrid mode - if (this.config.persistence === 'hybrid') { - this.syncTimer = setInterval(() => { - this.syncToStorage().catch(console.error) - }, this.config.syncInterval) - } - - console.log(`๐Ÿ” EntityRegistry initialized: ${this.memoryIndex.size} cached mappings`) - } - - async shutdown(): Promise { - // Final sync before shutdown - if (this.config.persistence === 'storage' || this.config.persistence === 'hybrid') { - await this.syncToStorage() - } - - if (this.syncTimer) { - clearInterval(this.syncTimer) - } - } - - /** - * Execute the augmentation - */ - async execute( - operation: string, - params: any, - next: () => Promise - ): Promise { - console.log(`๐Ÿ” [EntityRegistry] execute called: operation=${operation}`) - - // For add operations, check for duplicates first - if (operation === 'add' || operation === 'addNoun') { - const metadata = params.metadata || {} - - // Check if entity already exists - for (const field of this.config.indexedFields) { - const value = this.extractFieldValue(metadata, field) - if (value) { - const existingId = await this.lookupEntity(field, value) - if (existingId) { - // Entity already exists, return the existing one - console.log(`๐Ÿ” Duplicate detected: ${field}:${value} โ†’ ${existingId}`) - return { id: existingId, duplicate: true } as any - } - } - } - } - - // For addVerb operations, resolve external IDs to internal UUIDs - if (operation === 'addVerb') { - const sourceId = params.sourceId - const targetId = params.targetId - - // Try to resolve source and target IDs if they look like external IDs - for (const field of this.config.indexedFields) { - // Check if sourceId matches an external ID pattern - if (typeof sourceId === 'string' && this.looksLikeExternalId(sourceId, field)) { - const resolvedSourceId = await this.lookupEntity(field, sourceId) - if (resolvedSourceId) { - console.log(`๐Ÿ” [EntityRegistry] Resolved source: ${sourceId} โ†’ ${resolvedSourceId}`) - params.sourceId = resolvedSourceId - } - } - - // Check if targetId matches an external ID pattern - if (typeof targetId === 'string' && this.looksLikeExternalId(targetId, field)) { - const resolvedTargetId = await this.lookupEntity(field, targetId) - if (resolvedTargetId) { - console.log(`๐Ÿ” [EntityRegistry] Resolved target: ${targetId} โ†’ ${resolvedTargetId}`) - params.targetId = resolvedTargetId - } - } - } - } - - // Proceed with the operation - const result = await next() - - // Register the entity after successful add - if ((operation === 'add' || operation === 'addNoun' || operation === 'addVerb') && result) { - // Handle both formats: string UUID or object with id property - const brainyId = typeof result === 'string' ? result : (result as any).id - if (brainyId) { - const metadata = params.metadata || {} - const nounType = params.nounType || 'default' - console.log(`๐Ÿ” [EntityRegistry] Registering entity: ${brainyId}`) - await this.registerEntity(brainyId, metadata, nounType) - console.log(`โœ… [EntityRegistry] Entity registered successfully`) - } - } - - return result - } - - /** - * Register a new entity mapping - */ - async registerEntity(brainyId: string, metadata: any, nounType: string): Promise { - const now = Date.now() - - // Extract indexed fields from metadata - for (const field of this.config.indexedFields) { - const value = this.extractFieldValue(metadata, field) - if (value) { - const key = `${field}:${value}` - - // Add to memory index - const mapping: EntityMapping = { - externalId: value, - field, - brainyId, - nounType, - lastAccessed: now, - metadata - } - - this.memoryIndex.set(key, mapping) - - // Add to field-specific index - const fieldIndex = this.fieldIndices.get(field) - if (fieldIndex) { - fieldIndex.set(value, brainyId) - } - } - } - - // Enforce cache size limit (LRU eviction) - await this.evictOldEntries() - } - - /** - * Fast lookup: external ID โ†’ Brainy UUID - * Works in write-only mode without search indexes - */ - async lookupEntity(field: string, value: string): Promise { - const key = `${field}:${value}` - const cached = this.memoryIndex.get(key) - - if (cached) { - // Update last accessed time - cached.lastAccessed = Date.now() - this.cacheHits++ - return cached.brainyId - } - - this.cacheMisses++ - - // If not in cache and using storage persistence, try loading from storage - if (this.config.persistence === 'storage' || this.config.persistence === 'hybrid') { - const stored = await this.loadFromStorageByField(field, value) - if (stored) { - // Add to memory cache - this.memoryIndex.set(key, stored) - const fieldIndex = this.fieldIndices.get(field) - if (fieldIndex) { - fieldIndex.set(value, stored.brainyId) - } - return stored.brainyId - } - } - - return null - } - - /** - * Batch lookup for multiple external IDs - */ - async lookupBatch(lookups: Array<{ field: string; value: string }>): Promise> { - const results = new Map() - const missingKeys: Array<{ field: string; value: string; key: string }> = [] - - // Check memory cache first - for (const lookup of lookups) { - const key = `${lookup.field}:${lookup.value}` - const cached = this.memoryIndex.get(key) - - if (cached) { - cached.lastAccessed = Date.now() - results.set(key, cached.brainyId) - } else { - missingKeys.push({ ...lookup, key }) - results.set(key, null) - } - } - - // Batch load missing keys from storage - if (missingKeys.length > 0 && (this.config.persistence === 'storage' || this.config.persistence === 'hybrid')) { - const stored = await this.loadBatchFromStorage(missingKeys) - - for (const [key, mapping] of stored) { - if (mapping) { - // Add to memory cache - this.memoryIndex.set(key, mapping) - const fieldIndex = this.fieldIndices.get(mapping.field) - if (fieldIndex) { - fieldIndex.set(mapping.externalId, mapping.brainyId) - } - results.set(key, mapping.brainyId) - } - } - } - - return results - } - - /** - * Check if entity exists (faster than lookupEntity for existence checks) - */ - async hasEntity(field: string, value: string): Promise { - const fieldIndex = this.fieldIndices.get(field) - if (fieldIndex && fieldIndex.has(value)) { - return true - } - - return (await this.lookupEntity(field, value)) !== null - } - - /** - * Get all entities by field (e.g., all DIDs) - */ - async getEntitiesByField(field: string): Promise { - const fieldIndex = this.fieldIndices.get(field) - return fieldIndex ? Array.from(fieldIndex.keys()) : [] - } - - /** - * Get registry statistics - */ - getStats(): { - totalMappings: number - fieldCounts: Record - cacheHitRate: number - memoryUsage: number - } { - const fieldCounts: Record = {} - - for (const [field, index] of this.fieldIndices) { - fieldCounts[field] = index.size - } - - return { - totalMappings: this.memoryIndex.size, - fieldCounts, - cacheHitRate: this.cacheHits > 0 ? this.cacheHits / (this.cacheHits + this.cacheMisses) : 0, - memoryUsage: this.estimateMemoryUsage() - } - } - - /** - * Clear all cached mappings - */ - async clearCache(): Promise { - this.memoryIndex.clear() - for (const fieldIndex of this.fieldIndices.values()) { - fieldIndex.clear() - } - } - - // Private helper methods - - /** - * Check if an ID looks like it could be an external ID for a specific field - */ - private looksLikeExternalId(id: string, field: string): boolean { - // Basic heuristics to detect external ID patterns - switch (field) { - case 'did': - return id.startsWith('did:') - case 'handle': - return id.includes('.') && (id.includes('bsky') || id.includes('social')) - case 'external_id': - return !id.match(/^[a-f0-9-]{36}$/i) // Not a UUID - case 'uri': - return id.startsWith('http') || id.startsWith('at://') - case 'id': - return !id.match(/^[a-f0-9-]{36}$/i) // Not a UUID - default: - // For custom fields, assume non-UUID strings might be external IDs - return typeof id === 'string' && id.length > 3 && !id.match(/^[a-f0-9-]{36}$/i) - } - } - - private extractFieldValue(metadata: any, field: string): string | null { - if (!metadata) return null - - // Support nested field access (e.g., "author.did") - const parts = field.split('.') - let value = metadata - - for (const part of parts) { - value = value?.[part] - if (value === undefined || value === null) { - return null - } - } - - return typeof value === 'string' ? value : String(value) - } - - private async evictOldEntries(): Promise { - if (this.memoryIndex.size <= this.config.maxCacheSize) { - return - } - - // Sort by last accessed time and remove oldest entries - const entries = Array.from(this.memoryIndex.entries()) - entries.sort(([, a], [, b]) => a.lastAccessed - b.lastAccessed) - - const toRemove = entries.slice(0, entries.length - this.config.maxCacheSize) - - for (const [key, mapping] of toRemove) { - this.memoryIndex.delete(key) - - const fieldIndex = this.fieldIndices.get(mapping.field) - if (fieldIndex) { - fieldIndex.delete(mapping.externalId) - } - } - } - - private async loadFromStorage(): Promise { - if (!this.brain) return - - try { - // Load registry data from a special storage location - const registryData = await this.brain.storage?.getMetadata('__entity_registry__') - - if (registryData && registryData.mappings) { - for (const mapping of registryData.mappings) { - const key = `${mapping.field}:${mapping.externalId}` - this.memoryIndex.set(key, mapping) - - const fieldIndex = this.fieldIndices.get(mapping.field) - if (fieldIndex) { - fieldIndex.set(mapping.externalId, mapping.brainyId) - } - } - } - } catch (error) { - console.warn('Failed to load entity registry from storage:', error) - } - } - - private async syncToStorage(): Promise { - if (!this.brain) return - - try { - const mappings = Array.from(this.memoryIndex.values()) - - await this.brain.storage?.saveMetadata('__entity_registry__', { - version: 1, - lastSync: Date.now(), - mappings - }) - } catch (error) { - console.warn('Failed to sync entity registry to storage:', error) - } - } - - private async loadFromStorageByField(field: string, value: string): Promise { - // For now, this would require a full load. In production, you'd want - // a more sophisticated storage index system - return null - } - - private async loadBatchFromStorage(keys: Array<{ field: string; value: string; key: string }>): Promise> { - // For now, return empty. In production, implement batch storage lookup - return new Map() - } - - private estimateMemoryUsage(): number { - // Rough estimate: 200 bytes per mapping on average - return this.memoryIndex.size * 200 - } -} - -// Hook into Brainy's add operations to automatically register entities -export class AutoRegisterEntitiesAugmentation extends BaseAugmentation { - readonly metadata = 'readonly' as const // Reads metadata for auto-registration - readonly name = 'auto-register-entities' - readonly description = 'Automatically register entities in the registry when added' - readonly timing: 'before' | 'after' | 'around' | 'replace' = 'after' - readonly operations = ['add', 'addNoun', 'addVerb'] as ('add' | 'addNoun' | 'addVerb')[] - readonly priority = 85 // After entity registry - - private registry?: EntityRegistryAugmentation - private brain?: any - - async initialize(context: AugmentationContext): Promise { - this.brain = context.brain - // Find the entity registry augmentation from the registry - this.registry = this.brain?.augmentations?.augmentations?.find( - (aug: any) => aug instanceof EntityRegistryAugmentation - ) as EntityRegistryAugmentation - } - - async execute( - operation: string, - params: any, - next: () => Promise - ): Promise { - const result = await next() - - // After successful add, register the entity - if ((operation === 'add' || operation === 'addNoun' || operation === 'addVerb') && result) { - if (this.registry) { - // Handle both formats: string UUID or object with id property - const brainyId = typeof result === 'string' ? result : (result as any).id - if (brainyId) { - const metadata = params.metadata || {} - const nounType = params.nounType || 'default' - await this.registry.registerEntity(brainyId, metadata, nounType) - } - } - } - - return result - } -} \ No newline at end of file diff --git a/src/augmentations/indexAugmentation.ts b/src/augmentations/indexAugmentation.ts deleted file mode 100644 index 553ee0b9..00000000 --- a/src/augmentations/indexAugmentation.ts +++ /dev/null @@ -1,337 +0,0 @@ -/** - * Index Augmentation - Optional Metadata Indexing - * - * Replaces the hardcoded MetadataIndex in Brainy with an optional augmentation. - * Provides O(1) metadata filtering and field lookups. - * - * Zero-config: Automatically enabled for better search performance - * Can be disabled or customized via augmentation registry - */ - -import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js' -import { MetadataIndexManager } from '../utils/metadataIndex.js' -import type { BaseStorageAdapter as StorageAdapter } from '../storage/adapters/baseStorageAdapter.js' - -export interface IndexConfig { - enabled?: boolean - maxFieldValues?: number - maxIndexSize?: number // Max number of entries per field value - autoRebuild?: boolean - rebuildThreshold?: number - flushInterval?: number -} - -/** - * IndexAugmentation - Makes metadata indexing optional and pluggable - * - * Features: - * - O(1) metadata field lookups - * - Fast pre-filtering for searches - * - Automatic index maintenance - * - Zero-config with smart defaults - */ -export class IndexAugmentation extends BaseAugmentation { - readonly metadata = 'readonly' as const // Reads metadata to build indexes - readonly name = 'index' - readonly timing = 'after' as const - operations = ['add', 'update', 'updateMetadata', 'delete', 'clear', 'all'] as ('add' | 'update' | 'updateMetadata' | 'delete' | 'clear' | 'all')[] - readonly priority = 60 // Run after data operations - - // Augmentation metadata - readonly category = 'core' as const - readonly description = 'Fast metadata field indexing for O(1) filtering and lookups' - - private metadataIndex: MetadataIndexManager | null = null - protected config: IndexConfig - private flushTimer: NodeJS.Timeout | null = null - - constructor(config: IndexConfig = {}) { - super() - this.config = { - enabled: true, - maxFieldValues: 1000, - autoRebuild: true, - rebuildThreshold: 0.3, // Rebuild if 30% inconsistent - flushInterval: 30000, // Flush every 30 seconds - ...config - } - } - - protected async onInitialize(): Promise { - if (!this.config.enabled) { - this.log('Index augmentation disabled by configuration') - return - } - - // Get storage from context - const storage = this.context?.storage - if (!storage) { - this.log('No storage available, index augmentation inactive', 'warn') - return - } - - // Initialize metadata index - this.metadataIndex = new MetadataIndexManager( - storage as StorageAdapter, - { - maxIndexSize: this.config.maxIndexSize || 10000 - } - ) - - // Check if we need to rebuild - if (this.config.autoRebuild) { - const stats = await this.metadataIndex.getStats() - if (stats.totalEntries === 0) { - // Check if storage has data but index is empty - try { - const storageStats = await storage.getStatistics?.() - if (storageStats && storageStats.totalNouns > 0) { - this.log('Rebuilding metadata index for existing data...') - await this.metadataIndex.rebuild() - const newStats = await this.metadataIndex.getStats() - this.log(`Index rebuilt: ${newStats.totalEntries} entries, ${newStats.fieldsIndexed.length} fields`) - } - } catch (e) { - this.log('Could not check storage statistics', 'info') - } - } - } - - // Start flush timer - if (this.config.flushInterval && this.config.flushInterval > 0) { - this.startFlushTimer() - } - - this.log('Index augmentation initialized') - } - - protected async onShutdown(): Promise { - // Stop flush timer - if (this.flushTimer) { - clearInterval(this.flushTimer) - this.flushTimer = null - } - - // Flush index one last time - if (this.metadataIndex) { - try { - await this.metadataIndex.flush() - } catch (error) { - this.log('Error flushing index during shutdown', 'warn') - } - this.metadataIndex = null - } - - this.log('Index augmentation shut down') - } - - /** - * Execute augmentation - maintain index on data operations - */ - async execute( - operation: string, - params: any, - next: () => Promise - ): Promise { - // Execute the operation first - const result = await next() - - // If index is disabled, just return - if (!this.metadataIndex || !this.config.enabled) { - return result - } - - // Handle index updates after operation completes - switch (operation) { - case 'add': - await this.handleAdd(params) - break - - case 'updateMetadata': - await this.handleUpdate(params) - break - - case 'delete': - await this.handleDelete(params) - break - - case 'clear': - await this.handleClear() - break - } - - return result - } - - /** - * Handle add operation - index new metadata - */ - private async handleAdd(params: any): Promise { - if (!this.metadataIndex) return - - const { id, metadata } = params - if (id && metadata) { - await this.metadataIndex.addToIndex(id, metadata) - this.log(`Indexed metadata for ${id}`, 'info') - } - } - - /** - * Handle update operation - reindex metadata - */ - private async handleUpdate(params: any): Promise { - if (!this.metadataIndex) return - - const { id, oldMetadata, newMetadata } = params - - // Remove old metadata - if (id && oldMetadata) { - await this.metadataIndex.removeFromIndex(id, oldMetadata) - } - - // Add new metadata - if (id && newMetadata) { - await this.metadataIndex.addToIndex(id, newMetadata) - this.log(`Reindexed metadata for ${id}`, 'info') - } - } - - /** - * Handle delete operation - remove from index - */ - private async handleDelete(params: any): Promise { - if (!this.metadataIndex) return - - const { id, metadata } = params - if (id && metadata) { - await this.metadataIndex.removeFromIndex(id, metadata) - this.log(`Removed ${id} from index`, 'info') - } - } - - /** - * Handle clear operation - clear index - */ - private async handleClear(): Promise { - if (!this.metadataIndex) return - - // Clear the index when all data is cleared (rebuild effectively clears it) - await this.metadataIndex.rebuild() - this.log('Index cleared due to clear operation') - } - - /** - * Start periodic flush timer - */ - private startFlushTimer(): void { - if (this.flushTimer) return - - this.flushTimer = setInterval(async () => { - if (this.metadataIndex) { - try { - await this.metadataIndex.flush() - } catch (error) { - this.log('Error during periodic index flush', 'warn') - } - } - }, this.config.flushInterval!) - } - - /** - * Get IDs that match metadata filter (for pre-filtering) - */ - async getIdsForFilter(filter: Record): Promise { - if (!this.metadataIndex) return [] - return this.metadataIndex.getIdsForFilter(filter) - } - - /** - * Get available values for a field - */ - async getFilterValues(field: string): Promise { - if (!this.metadataIndex) return [] - return this.metadataIndex.getFilterValues(field) - } - - /** - * Get all indexed fields - */ - async getFilterFields(): Promise { - if (!this.metadataIndex) return [] - return this.metadataIndex.getFilterFields() - } - - /** - * Get index statistics - */ - async getStats() { - if (!this.metadataIndex) { - return { - enabled: false, - totalEntries: 0, - fieldsIndexed: [], - memoryUsage: 0 - } - } - - const stats = await this.metadataIndex.getStats() - return { - enabled: true, - ...stats - } - } - - /** - * Rebuild the index from storage - */ - async rebuild(): Promise { - if (!this.metadataIndex) { - throw new Error('Index augmentation is not initialized') - } - - this.log('Rebuilding metadata index...') - await this.metadataIndex.rebuild() - const stats = await this.metadataIndex.getStats() - this.log(`Index rebuilt: ${stats.totalEntries} entries, ${stats.fieldsIndexed.length} fields`) - } - - /** - * Flush index to storage - */ - async flush(): Promise { - if (this.metadataIndex) { - await this.metadataIndex.flush() - this.log('Index flushed to storage', 'info') - } - } - - /** - * Add entry to index (public method for direct access) - */ - async addToIndex(id: string, metadata: Record): Promise { - if (!this.metadataIndex) return - await this.metadataIndex.addToIndex(id, metadata) - } - - /** - * Remove entry from index (public method for direct access) - */ - async removeFromIndex(id: string, metadata: Record): Promise { - if (!this.metadataIndex) return - await this.metadataIndex.removeFromIndex(id, metadata) - } - - /** - * Get the underlying MetadataIndexManager instance - */ - getMetadataIndex() { - return this.metadataIndex - } -} - -/** - * Factory function for zero-config index augmentation - */ -export function createIndexAugmentation(config?: IndexConfig): IndexAugmentation { - return new IndexAugmentation(config) -} \ No newline at end of file diff --git a/src/augmentations/intelligentImport/FormatHandlerRegistry.ts b/src/augmentations/intelligentImport/FormatHandlerRegistry.ts deleted file mode 100644 index 4daff432..00000000 --- a/src/augmentations/intelligentImport/FormatHandlerRegistry.ts +++ /dev/null @@ -1,246 +0,0 @@ -/** - * Format Handler Registry - * - * Central registry for format handlers with: - * - MIME type-based routing - * - Lazy loading support - * - Pluggable handler registration - * - Handler lifecycle management - * - * NO MOCKS - Production implementation - */ - -import { FormatHandler, HandlerRegistry as IHandlerRegistry } from './types.js' -import { mimeDetector } from '../../vfs/MimeTypeDetector.js' - -export interface HandlerRegistration { - /** Handler name (e.g., 'csv', 'image', 'pdf') */ - name: string - - /** MIME types this handler supports */ - mimeTypes: string[] - - /** File extensions (fallback if MIME detection fails) */ - extensions: string[] - - /** Lazy loader function */ - loader: () => Promise - - /** Loaded handler instance (lazy-loaded) */ - instance?: FormatHandler -} - -/** - * FormatHandlerRegistry - Central handler management - * - * Implements the HandlerRegistry interface with: - * - MIME type-based routing - * - Lazy loading for performance - * - Multiple handlers per MIME type - * - Priority-based selection - */ -export class FormatHandlerRegistry implements IHandlerRegistry { - // Interface compatibility - handlers: Map Promise> = new Map() - loaded: Map = new Map() - - // Enhanced registry - private registrations: Map = new Map() - private mimeTypeIndex: Map = new Map() // MIME type โ†’ handler names - private extensionIndex: Map = new Map() // Extension โ†’ handler names - - /** - * Register a format handler - * - * @param registration Handler registration details - */ - registerHandler(registration: HandlerRegistration): void { - const { name, mimeTypes, extensions, loader } = registration - - // Store registration - this.registrations.set(name, registration) - - // Index by MIME types - for (const mimeType of mimeTypes) { - const handlers = this.mimeTypeIndex.get(mimeType) || [] - handlers.push(name) - this.mimeTypeIndex.set(mimeType, handlers) - } - - // Index by extensions - for (const ext of extensions) { - const normalized = ext.toLowerCase().replace(/^\./, '') - const handlers = this.extensionIndex.get(normalized) || [] - handlers.push(name) - this.extensionIndex.set(normalized, handlers) - } - - // Interface compatibility - this.handlers.set(name, loader) - } - - /** - * Register a handler (interface compatibility) - */ - register(extensions: string[], loader: () => Promise): void { - const name = extensions[0].replace(/^\./, '') - - // Auto-detect MIME types from extensions for better routing - const mimeTypes: string[] = [] - for (const ext of extensions) { - const mimeType = mimeDetector.detectMimeType(`file${ext}`) - if (mimeType && mimeType !== 'application/octet-stream' && !mimeTypes.includes(mimeType)) { - mimeTypes.push(mimeType) - } - } - - this.registerHandler({ - name, - mimeTypes, - extensions, - loader - }) - } - - /** - * Get handler by filename or extension - * - * Uses MIME detection first, falls back to extension matching - * - * @param filenameOrExt Filename or extension - * @returns Handler instance or null - */ - async getHandler(filenameOrExt: string): Promise { - // Try MIME type detection first - const mimeType = mimeDetector.detectMimeType(filenameOrExt) - const byMime = await this.getHandlerByMimeType(mimeType) - if (byMime) return byMime - - // Fallback to extension matching - const ext = this.extractExtension(filenameOrExt) - if (ext) { - return this.getHandlerByExtension(ext) - } - - return null - } - - /** - * Get handler by MIME type - * - * @param mimeType MIME type string - * @returns Handler instance or null - */ - async getHandlerByMimeType(mimeType: string): Promise { - const handlerNames = this.mimeTypeIndex.get(mimeType) - if (!handlerNames || handlerNames.length === 0) { - return null - } - - // Return first matching handler (could add priority later) - return this.loadHandler(handlerNames[0]) - } - - /** - * Get handler by file extension - * - * @param ext File extension (with or without dot) - * @returns Handler instance or null - */ - async getHandlerByExtension(ext: string): Promise { - const normalized = ext.toLowerCase().replace(/^\./, '') - const handlerNames = this.extensionIndex.get(normalized) - if (!handlerNames || handlerNames.length === 0) { - return null - } - - // Return first matching handler - return this.loadHandler(handlerNames[0]) - } - - /** - * Get handler by name - * - * @param name Handler name - * @returns Handler instance or null - */ - async getHandlerByName(name: string): Promise { - return this.loadHandler(name) - } - - /** - * Load handler (lazy loading) - * - * @param name Handler name - * @returns Loaded handler instance - */ - private async loadHandler(name: string): Promise { - const registration = this.registrations.get(name) - if (!registration) return null - - // Return cached instance if available - if (registration.instance) { - return registration.instance - } - - // Load handler - try { - const handler = await registration.loader() - registration.instance = handler - - // Interface compatibility - this.loaded.set(name, handler) - - return handler - } catch (error) { - console.error(`Failed to load handler ${name}:`, error) - return null - } - } - - /** - * Get all registered handler names - */ - getRegisteredHandlers(): string[] { - return Array.from(this.registrations.keys()) - } - - /** - * Get handlers for a MIME type - */ - getHandlersForMimeType(mimeType: string): string[] { - return this.mimeTypeIndex.get(mimeType) || [] - } - - /** - * Check if handler is registered - */ - hasHandler(name: string): boolean { - return this.registrations.has(name) - } - - /** - * Clear all handlers (for testing) - */ - clear(): void { - this.registrations.clear() - this.mimeTypeIndex.clear() - this.extensionIndex.clear() - this.handlers.clear() - this.loaded.clear() - } - - /** - * Extract file extension from filename - */ - private extractExtension(filename: string): string | null { - const lastDot = filename.lastIndexOf('.') - if (lastDot === -1 || lastDot === 0) return null - return filename.substring(lastDot + 1).toLowerCase() - } -} - -/** - * Global handler registry singleton - */ -export const globalHandlerRegistry = new FormatHandlerRegistry() diff --git a/src/augmentations/intelligentImport/IntelligentImportAugmentation.ts b/src/augmentations/intelligentImport/IntelligentImportAugmentation.ts deleted file mode 100644 index 0e21b585..00000000 --- a/src/augmentations/intelligentImport/IntelligentImportAugmentation.ts +++ /dev/null @@ -1,227 +0,0 @@ -/** - * Intelligent Import Augmentation - * - * Automatically detects and processes CSV, Excel, and PDF files with: - * - Format detection and routing - * - Lazy-loaded handlers - * - Intelligent entity and relationship extraction - * - Integration with NeuralImport augmentation - */ - -import { BaseAugmentation, AugmentationContext } from '../brainyAugmentation.js' -import { FormatHandler, IntelligentImportConfig, ProcessedData } from './types.js' -import { CSVHandler } from './handlers/csvHandler.js' -import { ExcelHandler } from './handlers/excelHandler.js' -import { PDFHandler } from './handlers/pdfHandler.js' -import { ImageHandler } from './handlers/imageHandler.js' - -export class IntelligentImportAugmentation extends BaseAugmentation { - readonly name = 'intelligent-import' - readonly timing = 'before' as const - readonly metadata = { - reads: '*' as '*', - writes: ['_intelligentImport', '_processedFormat', '_extractedData'] as string[] - } - readonly operations = ['import', 'importFile', 'importFromFile', 'importFromURL', 'all'] as any[] - readonly priority = 75 // Before NeuralImport (80), after validation - - protected config: IntelligentImportConfig - private handlers: Map = new Map() - private initialized = false - - constructor(config: Partial = {}) { - super(config) - this.config = { - enableCSV: true, - enableExcel: true, - enablePDF: true, - enableImage: true, // Image handler enabled by default - maxFileSize: 100 * 1024 * 1024, // 100MB default - enableCache: true, - cacheTTL: 24 * 60 * 60 * 1000, // 24 hours - ...config - } - } - - protected async onInitialize(): Promise { - // Initialize handlers based on config - if (this.config.enableCSV) { - this.handlers.set('csv', new CSVHandler()) - } - - if (this.config.enableExcel) { - this.handlers.set('excel', new ExcelHandler()) - } - - if (this.config.enablePDF) { - this.handlers.set('pdf', new PDFHandler()) - } - - if (this.config.enableImage) { - this.handlers.set('image', new ImageHandler()) - } - - this.initialized = true - this.log(`Initialized with ${this.handlers.size} format handlers (CSV: ${this.config.enableCSV}, Excel: ${this.config.enableExcel}, PDF: ${this.config.enablePDF}, Image: ${this.config.enableImage})`) - } - - async execute( - operation: string, - params: any, - next: () => Promise - ): Promise { - // Only process import operations - if (!this.shouldProcess(operation, params)) { - return next() - } - - try { - // Extract file data from params - const fileData = this.extractFileData(params) - if (!fileData) { - return next() - } - - // Check file size limit - if (this.config.maxFileSize && fileData.data.length > this.config.maxFileSize) { - this.log(`File too large (${fileData.data.length} bytes), skipping intelligent import`, 'warn') - return next() - } - - // Detect format and get appropriate handler - const handler = this.detectHandler(fileData.data, fileData.filename) - if (!handler) { - // Not a supported format, pass through - return next() - } - - this.log(`Processing ${fileData.filename || 'file'} with ${handler.format} handler`) - - // Process the file - const processed = await handler.process(fileData.data, { - filename: fileData.filename, - ext: fileData.ext, - ...this.config.csvDefaults, - ...this.config.excelDefaults, - ...this.config.pdfDefaults, - ...this.config.imageDefaults, - ...params.options - }) - - // Enrich params with processed data - params._intelligentImport = true - params._processedFormat = processed.format - params._extractedData = processed.data - params._metadata = { - ...params._metadata, - intelligentImport: processed.metadata - } - - // If this is an import operation, transform params to include the structured data - if (processed.data.length > 0) { - // Store processed data for the neural import augmentation to use - params.data = processed.data - params.metadata = params._metadata - } - - this.log(`Extracted ${processed.data.length} items from ${processed.format} file`) - - return next() - } catch (error) { - this.log(`Intelligent import processing failed: ${error instanceof Error ? error.message : String(error)}`, 'warn') - // Fall through to normal import on error - return next() - } - } - - /** - * Check if we should process this operation - */ - private shouldProcess(operation: string, params: any): boolean { - // Only process if we have handlers initialized - if (!this.initialized || this.handlers.size === 0) { - return false - } - - // Check operation type - const validOps = ['import', 'importFile', 'importFromFile', 'importFromURL'] - if (!validOps.some(op => operation.includes(op))) { - return false - } - - // Must have some data - if (!params || (!params.source && !params.data && !params.filePath && !params.url)) { - return false - } - - return true - } - - /** - * Extract file data from various param formats - */ - private extractFileData(params: any): { data: Buffer, filename?: string, ext?: string } | null { - // From source parameter - if (params.source) { - if (Buffer.isBuffer(params.source)) { - return { data: params.source, filename: params.filename } - } - if (typeof params.source === 'string') { - return { data: Buffer.from(params.source), filename: params.filename } - } - } - - // From data parameter - if (params.data) { - if (Buffer.isBuffer(params.data)) { - return { data: params.data, filename: params.filename } - } - if (typeof params.data === 'string') { - return { data: Buffer.from(params.data), filename: params.filename } - } - } - - // From file path (would need to read - but that should be handled by UniversalImportAPI) - if (params.filePath && typeof params.filePath === 'string') { - const ext = params.filePath.split('.').pop() - return null // File reading handled elsewhere - } - - return null - } - - /** - * Detect which handler can process this file - */ - private detectHandler(data: Buffer, filename?: string): FormatHandler | null { - // Try each handler's canHandle method - for (const handler of this.handlers.values()) { - if (handler.canHandle(data) || (filename && handler.canHandle({ filename }))) { - return handler - } - } - - return null - } - - /** - * Get handler by format name - */ - getHandler(format: string): FormatHandler | undefined { - return this.handlers.get(format.toLowerCase()) - } - - /** - * Get all registered handlers - */ - getHandlers(): FormatHandler[] { - return Array.from(this.handlers.values()) - } - - /** - * Get supported formats - */ - getSupportedFormats(): string[] { - return Array.from(this.handlers.keys()) - } -} diff --git a/src/augmentations/intelligentImport/handlers/imageHandler.ts b/src/augmentations/intelligentImport/handlers/imageHandler.ts deleted file mode 100644 index f896e489..00000000 --- a/src/augmentations/intelligentImport/handlers/imageHandler.ts +++ /dev/null @@ -1,350 +0,0 @@ -/** - * Image Import Handler - * - * Handles image files with: - * - EXIF metadata extraction (camera, GPS, timestamps) via exifr - * - Image metadata (dimensions, format) via probe-image-size - * - Support for JPEG, PNG, WebP, GIF, TIFF, BMP, SVG - * - * NO NATIVE DEPENDENCIES - Pure JavaScript implementation - * Replaces Sharp with lightweight pure-JS alternatives - */ - -import { BaseFormatHandler } from './base.js' -import type { FormatHandlerOptions, ProcessedData } from '../types.js' -import exifr from 'exifr' -import probeImageSize from 'probe-image-size' -import { Readable } from 'stream' - -export interface ImageMetadata { - /** Image dimensions */ - width: number - height: number - - /** Image format (jpeg, png, webp, etc.) */ - format: string - - /** File size in bytes */ - size: number - - /** Orientation (EXIF) */ - orientation?: number - - /** MIME type */ - mimeType?: string -} - -export interface EXIFData { - /** Camera make (e.g., "Canon", "Nikon") */ - make?: string - - /** Camera model */ - model?: string - - /** Lens information */ - lens?: string - - /** Date/time original */ - dateTimeOriginal?: Date - - /** GPS latitude */ - latitude?: number - - /** GPS longitude */ - longitude?: number - - /** GPS altitude in meters */ - altitude?: number - - /** Exposure time (e.g., "1/250") */ - exposureTime?: number - - /** F-number (e.g., 2.8) */ - fNumber?: number - - /** ISO speed */ - iso?: number - - /** Focal length in mm */ - focalLength?: number - - /** Flash fired */ - flash?: boolean - - /** Copyright */ - copyright?: string - - /** Artist/photographer */ - artist?: string - - /** Image description */ - imageDescription?: string - - /** Software used */ - software?: string -} - -export interface ImageHandlerOptions extends FormatHandlerOptions { - /** Extract EXIF data (default: true) */ - extractEXIF?: boolean -} - -/** - * ImageImportHandler - * - * Processes image files and extracts rich metadata including EXIF data. - * Enables developers to import images into the knowledge graph with - * full metadata extraction. - * - * Pure JavaScript implementation (no native dependencies) - */ -export class ImageHandler extends BaseFormatHandler { - readonly format = 'image' - - /** - * Check if this handler can process the given data - */ - canHandle(data: Buffer | string | { filename?: string; ext?: string }): boolean { - // Check by filename/extension - if (typeof data === 'object' && 'filename' in data) { - const mimeType = this.getMimeType(data) - return this.mimeTypeMatches(mimeType, ['image/*']) - } - - // Check by extension - if (typeof data === 'object' && 'ext' in data && data.ext) { - return this.isImageExtension(data.ext) - } - - // Check by data (Buffer magic bytes) - if (Buffer.isBuffer(data)) { - return this.detectImageFormat(data) !== null - } - - return false - } - - /** - * Process image file - */ - async process( - data: Buffer | string, - options: ImageHandlerOptions = {} - ): Promise { - const startTime = Date.now() - - try { - // Convert to Buffer - const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'base64') - - // Extract image metadata - const metadata = await this.extractMetadata(buffer) - - // Extract EXIF data (default: enabled) - let exifData: EXIFData | undefined - if (options.extractEXIF !== false) { - exifData = await this.extractEXIF(buffer) - } - - // Calculate processing time - const processingTime = Date.now() - startTime - - // Generate descriptive name - const imageName = options.filename - ? options.filename.replace(/\.[^/.]+$/, '') // Remove extension - : `${metadata.format.toUpperCase()} Image ${metadata.width}x${metadata.height}` - - // Return structured data - return { - format: 'image', - data: [ - { - name: imageName, - type: 'media', - metadata: { - ...metadata, - subtype: 'image', - exif: exifData - } - } - ], - metadata: { - rowCount: 1, - fields: ['type', 'metadata'], - processingTime, - imageMetadata: metadata, - exifData - }, - filename: options.filename - } - } catch (error) { - throw new Error( - `Image processing failed: ${error instanceof Error ? error.message : String(error)}` - ) - } - } - - /** - * Extract image metadata using probe-image-size (pure JS) - */ - private async extractMetadata(buffer: Buffer): Promise { - try { - // Convert Buffer to Stream for probe-image-size - const stream = Readable.from(buffer) - const result = await probeImageSize(stream) - - return { - width: result.width, - height: result.height, - format: result.type, // 'jpeg', 'png', 'webp', etc. - size: buffer.length, - mimeType: result.mime, - orientation: result.orientation - } - } catch (error) { - // Fallback: Try to detect format from magic bytes - const detectedFormat = this.detectImageFormat(buffer) - - return { - width: 0, - height: 0, - format: detectedFormat || 'unknown', - size: buffer.length, - mimeType: detectedFormat ? `image/${detectedFormat}` : undefined - } - } - } - - /** - * Extract EXIF data using exifr (pure JS) - */ - private async extractEXIF(buffer: Buffer): Promise { - try { - const exif = await exifr.parse(buffer, { - pick: [ - 'Make', - 'Model', - 'LensModel', - 'DateTimeOriginal', - 'latitude', - 'longitude', - 'GPSAltitude', - 'ExposureTime', - 'FNumber', - 'ISO', - 'FocalLength', - 'Flash', - 'Copyright', - 'Artist', - 'ImageDescription', - 'Software' - ] - }) - - if (!exif) return undefined - - return { - make: exif.Make, - model: exif.Model, - lens: exif.LensModel, - dateTimeOriginal: exif.DateTimeOriginal, - latitude: exif.latitude, - longitude: exif.longitude, - altitude: exif.GPSAltitude, - exposureTime: exif.ExposureTime, - fNumber: exif.FNumber, - iso: exif.ISO, - focalLength: exif.FocalLength, - flash: exif.Flash !== undefined ? Boolean(exif.Flash & 1) : undefined, - copyright: exif.Copyright, - artist: exif.Artist, - imageDescription: exif.ImageDescription, - software: exif.Software - } - } catch (error) { - // EXIF extraction can fail for non-JPEG images or corrupt data - return undefined - } - } - - /** - * Detect image format from magic bytes - */ - private detectImageFormat(buffer: Buffer): string | null { - if (buffer.length < 4) return null - - // JPEG: FF D8 FF - if (buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) { - return 'jpeg' - } - - // PNG: 89 50 4E 47 - if ( - buffer[0] === 0x89 && - buffer[1] === 0x50 && - buffer[2] === 0x4e && - buffer[3] === 0x47 - ) { - return 'png' - } - - // GIF: 47 49 46 - if (buffer[0] === 0x47 && buffer[1] === 0x49 && buffer[2] === 0x46) { - return 'gif' - } - - // WebP: 52 49 46 46 ... 57 45 42 50 - if ( - buffer[0] === 0x52 && - buffer[1] === 0x49 && - buffer[2] === 0x46 && - buffer[3] === 0x46 && - buffer.length >= 12 && - buffer[8] === 0x57 && - buffer[9] === 0x45 && - buffer[10] === 0x42 && - buffer[11] === 0x50 - ) { - return 'webp' - } - - // TIFF: 49 49 2A 00 (little-endian) or 4D 4D 00 2A (big-endian) - if ( - (buffer[0] === 0x49 && buffer[1] === 0x49 && buffer[2] === 0x2a && buffer[3] === 0x00) || - (buffer[0] === 0x4d && buffer[1] === 0x4d && buffer[2] === 0x00 && buffer[3] === 0x2a) - ) { - return 'tiff' - } - - // BMP: 42 4D - if (buffer[0] === 0x42 && buffer[1] === 0x4d) { - return 'bmp' - } - - return null - } - - /** - * Detect if extension is an image format - */ - private isImageExtension(ext: string): boolean { - const imageExts = [ - '.jpg', - '.jpeg', - '.png', - '.gif', - '.webp', - '.tiff', - '.tif', - '.bmp', - '.svg', - '.heic', - '.heif', - '.avif' - ] - - const normalized = ext.toLowerCase() - const withDot = normalized.startsWith('.') ? normalized : `.${normalized}` - return imageExts.includes(withDot) - } -} diff --git a/src/augmentations/intelligentImport/index.ts b/src/augmentations/intelligentImport/index.ts deleted file mode 100644 index 5c8c182d..00000000 --- a/src/augmentations/intelligentImport/index.ts +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Intelligent Import Module - * Exports main augmentation and types - */ - -export { IntelligentImportAugmentation } from './IntelligentImportAugmentation.js' -export type { - FormatHandler, - FormatHandlerOptions, - ProcessedData, - IntelligentImportConfig, - HandlerRegistry -} from './types.js' - -// Format Handlers -export { CSVHandler } from './handlers/csvHandler.js' -export { ExcelHandler } from './handlers/excelHandler.js' -export { PDFHandler } from './handlers/pdfHandler.js' -export { ImageHandler } from './handlers/imageHandler.js' - -// Format Handler Registry -export { - FormatHandlerRegistry, - globalHandlerRegistry -} from './FormatHandlerRegistry.js' -export type { HandlerRegistration } from './FormatHandlerRegistry.js' - -// Image Handler Types -export type { - ImageMetadata, - EXIFData, - ImageHandlerOptions -} from './handlers/imageHandler.js' diff --git a/src/augmentations/intelligentVerbScoringAugmentation.ts b/src/augmentations/intelligentVerbScoringAugmentation.ts deleted file mode 100644 index 2ca39a45..00000000 --- a/src/augmentations/intelligentVerbScoringAugmentation.ts +++ /dev/null @@ -1,875 +0,0 @@ -/** - * Intelligent Verb Scoring Augmentation - * - * Enhances relationship quality through intelligent semantic scoring - * Provides context-aware relationship weights based on: - * - Semantic proximity of connected entities - * - Frequency-based amplification - * - Temporal decay modeling - * - Adaptive learning from usage patterns - * - * Critical for enterprise knowledge graphs with millions of relationships - */ - -import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js' - -interface VerbScoringConfig { - enabled?: boolean - - // Semantic Analysis - enableSemanticScoring?: boolean // Use entity embeddings for scoring - semanticThreshold?: number // Minimum semantic similarity - semanticWeight?: number // Weight of semantic component - - // Frequency Analysis - enableFrequencyAmplification?: boolean // Amplify frequently used relationships - frequencyDecay?: number // How quickly frequency importance decays - maxFrequencyBoost?: number // Maximum boost from frequency - - // Temporal Analysis - enableTemporalDecay?: boolean // Apply time-based decay - temporalDecayRate?: number // Decay rate per day (0-1) - temporalWindow?: number // Time window for relevance (days) - - // Learning & Adaptation - enableAdaptiveLearning?: boolean // Learn from usage patterns - learningRate?: number // How quickly to adapt (0-1) - confidenceThreshold?: number // Minimum confidence for relationships - - // Weight Management - minWeight?: number // Minimum relationship weight - maxWeight?: number // Maximum relationship weight - baseWeight?: number // Default weight for new relationships -} - -interface RelationshipMetrics { - count: number // How many times this relationship was created - totalWeight: number // Sum of all weights - averageWeight: number // Average weight - lastUpdated: number // Last time this relationship was scored - semanticScore: number // Semantic similarity score - frequencyScore: number // Frequency-based score - temporalScore: number // Time-based relevance score - confidenceScore: number // Overall confidence -} - -interface ScoringMetrics { - relationshipsScored: number - averageSemanticScore: number - averageFrequencyScore: number - averageTemporalScore: number - averageConfidenceScore: number - adaptiveAdjustments: number - computationTimeMs: number - minScore?: number - maxScore?: number - averageScore?: number - highConfidenceCount?: number -} - -export class IntelligentVerbScoringAugmentation extends BaseAugmentation { - name = 'IntelligentVerbScoring' - timing = 'around' as const - readonly metadata = { - reads: ['type', 'verb', 'source', 'target'] as string[], - writes: ['weight', 'confidence', 'intelligentScoring'] as string[] - } // Adds scoring metadata to verbs - operations = ['addVerb', 'relate'] as ('addVerb' | 'relate')[] - priority = 10 // Enhancement feature - runs after core operations - - // Augmentation metadata - readonly category = 'core' as const - readonly description = 'AI-powered intelligent scoring for relationship strength analysis' - - protected config: Required - private relationshipStats: Map = new Map() - private metrics: ScoringMetrics = { - relationshipsScored: 0, - averageSemanticScore: 0, - averageFrequencyScore: 0, - averageTemporalScore: 0, - averageConfidenceScore: 0, - adaptiveAdjustments: 0, - computationTimeMs: 0 - } - private scoringInstance: any // Will hold IntelligentVerbScoring instance - - constructor(config: VerbScoringConfig = {}) { - super() - this.config = { - enabled: config.enabled ?? true, // Smart by default! - - // Semantic Analysis - enableSemanticScoring: config.enableSemanticScoring ?? true, - semanticThreshold: config.semanticThreshold ?? 0.3, - semanticWeight: config.semanticWeight ?? 0.4, - - // Frequency Analysis - enableFrequencyAmplification: config.enableFrequencyAmplification ?? true, - frequencyDecay: config.frequencyDecay ?? 0.95, // 5% decay per occurrence - maxFrequencyBoost: config.maxFrequencyBoost ?? 2.0, - - // Temporal Analysis - enableTemporalDecay: config.enableTemporalDecay ?? true, - temporalDecayRate: config.temporalDecayRate ?? 0.01, // 1% per day - temporalWindow: config.temporalWindow ?? 365, // 1 year - - // Learning & Adaptation - enableAdaptiveLearning: config.enableAdaptiveLearning ?? true, - learningRate: config.learningRate ?? 0.1, - confidenceThreshold: config.confidenceThreshold ?? 0.3, - - // Weight Management - minWeight: config.minWeight ?? 0.1, - maxWeight: config.maxWeight ?? 1.0, - baseWeight: config.baseWeight ?? 0.5 - } - - // Set enabled property based on config - this.enabled = this.config.enabled - } - - protected async onInitialize(): Promise { - if (this.config.enabled) { - this.log('Intelligent verb scoring initialized for enhanced relationship quality') - } else { - this.log('Intelligent verb scoring disabled') - } - } - - /** - * Get this augmentation instance for API compatibility - * Used by Brainy to access scoring methods - */ - getScoring(): IntelligentVerbScoringAugmentation { - return this - } - - shouldExecute(operation: string, params: any): boolean { - // For addVerb, params are passed as array: [sourceId, targetId, verbType, metadata, weight] - if (operation === 'addVerb' && this.config.enabled) { - return Array.isArray(params) && params.length >= 3 - } - // For relate method, params might be an object - if (operation === 'relate' && this.config.enabled) { - return params.sourceId && params.targetId && params.relationType - } - return false - } - - async execute( - operation: string, - params: any, - next: () => Promise - ): Promise { - if (!this.shouldExecute(operation, params)) { - return next() - } - - const startTime = Date.now() - - try { - let sourceId: string, targetId: string, relationType: string, metadata: any - let scoringResult: { weight: number; confidence: number; reasoning: string[] } | null = null - - // Extract parameters based on operation type - if (operation === 'addVerb' && Array.isArray(params)) { - // addVerb params: [sourceId, targetId, verbType, metadata, weight] - [sourceId, targetId, relationType, metadata] = params - } else if (operation === 'relate') { - // relate params might be an object - sourceId = params.sourceId - targetId = params.targetId - relationType = params.relationType - metadata = params.metadata - } else { - return next() - } - - // Skip if weight is already provided explicitly - if (Array.isArray(params) && params[4] !== undefined && params[4] !== null) { - return next() - } - - // Get the nouns to compute scoring - const sourceNoun = await this.context?.brain.get(sourceId) - const targetNoun = await this.context?.brain.get(targetId) - - // Compute intelligent scores with reasoning - scoringResult = await this.computeVerbScores( - sourceNoun, - targetNoun, - relationType - ) - - // For addVerb, modify the params array - if (operation === 'addVerb' && Array.isArray(params)) { - // Set the weight parameter (index 4) - params[4] = scoringResult.weight - // Enhance metadata with scoring info - params[3] = { - ...params[3], - intelligentScoring: { - weight: scoringResult.weight, - confidence: scoringResult.confidence, - reasoning: scoringResult.reasoning, - scoringMethod: this.getScoringMethodsUsed(), - computedAt: Date.now() - } - } - } - - // Execute with enhanced parameters - const result = await next() - - // Learn from this relationship - if (this.config.enableAdaptiveLearning && scoringResult) { - await this.updateRelationshipLearning( - sourceId, - targetId, - relationType, - scoringResult.weight - ) - } - - // Update metrics - const computationTime = Date.now() - startTime - if (scoringResult) { - this.updateMetrics(scoringResult.weight, computationTime) - } - - return result - - } catch (error) { - this.log(`Intelligent verb scoring error: ${error}`, 'error') - // Fallback to original parameters - return next() - } - } - - private async calculateIntelligentWeight( - sourceId: string, - targetId: string, - relationType: string, - metadata?: any - ): Promise { - let finalWeight = this.config.baseWeight - let scoreComponents: any = {} - - // 1. Semantic Proximity Score - if (this.config.enableSemanticScoring) { - const semanticScore = await this.calculateSemanticScore(sourceId, targetId) - scoreComponents.semantic = semanticScore - finalWeight = finalWeight * (1 + semanticScore * this.config.semanticWeight) - } - - // 2. Frequency Amplification Score - if (this.config.enableFrequencyAmplification) { - const frequencyScore = this.calculateFrequencyScore(sourceId, targetId, relationType) - scoreComponents.frequency = frequencyScore - finalWeight = finalWeight * (1 + frequencyScore) - } - - // 3. Temporal Relevance Score - if (this.config.enableTemporalDecay) { - const temporalScore = this.calculateTemporalScore(sourceId, targetId, relationType) - scoreComponents.temporal = temporalScore - finalWeight = finalWeight * temporalScore - } - - // 4. Context Awareness (from metadata) - const contextScore = this.calculateContextScore(metadata) - scoreComponents.context = contextScore - finalWeight = finalWeight * (1 + contextScore * 0.2) - - // 5. Apply constraints - finalWeight = Math.max(this.config.minWeight, - Math.min(this.config.maxWeight, finalWeight)) - - // Store detailed scoring for analysis - this.storeDetailedScoring(sourceId, targetId, relationType, { - finalWeight, - components: scoreComponents, - timestamp: Date.now() - }) - - return finalWeight - } - - private async calculateSemanticScore(sourceId: string, targetId: string): Promise { - try { - // Get embeddings for both entities - const sourceNoun = await this.context?.brain.get(sourceId) - const targetNoun = await this.context?.brain.get(targetId) - - if (!sourceNoun?.vector || !targetNoun?.vector) { - return 0 - } - - // Get noun types using neural detection (taxonomy-based) - const sourceType = await this.detectNounType(sourceNoun.vector) - const targetType = await this.detectNounType(targetNoun.vector) - - // Calculate direct similarity - const directSimilarity = this.calculateCosineSimilarity(sourceNoun.vector, targetNoun.vector) - - // Calculate taxonomy-based similarity boost - const taxonomyBoost = await this.calculateTaxonomyBoost(sourceType, targetType) - - // Blend direct similarity with taxonomy guidance - // Taxonomy provides consistency while preserving flexibility - const semanticScore = directSimilarity * 0.7 + taxonomyBoost * 0.3 - - return Math.min(1, Math.max(0, semanticScore)) - - } catch (error) { - return 0 - } - } - - /** - * Detect noun type using neural taxonomy matching - */ - private async detectNounType(vector: number[]): Promise { - // Use real neural detection from brain's type detector - if (!this.context?.brain) return 'unknown' - - try { - // Access the brain's neural type detection system - const brain = this.context.brain as any - - // Use the actual neural type detection if available - if (brain.neuralDetector?.detectType) { - const detectedType = await brain.neuralDetector.detectType(vector) - return detectedType || 'unknown' - } - - // Fallback to pattern-based detection using actual embeddings - const patternAnalyzer = brain.patternAnalyzer || brain.neural?.patternAnalyzer - if (patternAnalyzer?.analyzeVector) { - const analysis = await patternAnalyzer.analyzeVector(vector) - return analysis.type || 'unknown' - } - - // Use statistical analysis of vector characteristics - const stats = this.analyzeVectorStatistics(vector) - return this.inferTypeFromStatistics(stats) - } catch (error) { - this.log(`Type detection failed: ${(error as Error).message}`, 'warn') - return 'unknown' - } - } - - /** - * Analyze vector statistics for type inference - */ - private analyzeVectorStatistics(vector: number[]): { - mean: number - variance: number - sparsity: number - entropy: number - magnitude: number - } { - const n = vector.length - if (n === 0) return { mean: 0, variance: 0, sparsity: 0, entropy: 0, magnitude: 0 } - - // Calculate mean - const mean = vector.reduce((sum, val) => sum + val, 0) / n - - // Calculate variance - const variance = vector.reduce((sum, val) => sum + Math.pow(val - mean, 2), 0) / n - - // Calculate sparsity (percentage of non-zero elements) - const nonZeroCount = vector.filter(val => Math.abs(val) > 0.001).length - const sparsity = 1 - (nonZeroCount / n) - - // Calculate entropy (information content) - const absSum = vector.reduce((sum, val) => sum + Math.abs(val), 0) || 1 - const probs = vector.map(val => Math.abs(val) / absSum) - const entropy = -probs.reduce((sum, p) => { - return p > 0 ? sum + p * Math.log2(p) : sum - }, 0) - - // Calculate magnitude - const magnitude = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0)) - - return { mean, variance, sparsity, entropy, magnitude } - } - - /** - * Infer entity type from vector statistics using neural patterns - */ - private inferTypeFromStatistics(stats: { - mean: number - variance: number - sparsity: number - entropy: number - magnitude: number - }): string { - // Neural pattern recognition based on empirical analysis - // These thresholds are derived from analyzing actual embeddings - - // High entropy and low sparsity often indicate abstract concepts - if (stats.entropy > 4.5 && stats.sparsity < 0.3) { - return 'concept' - } - - // Moderate entropy with high magnitude indicates concrete entities - if (stats.magnitude > 8 && stats.entropy > 3 && stats.entropy < 4.5) { - return 'entity' - } - - // High sparsity with focused magnitude indicates specific objects - if (stats.sparsity > 0.6 && stats.magnitude > 3) { - return 'object' - } - - // Documents tend to have balanced statistics - if (stats.entropy > 3.5 && stats.entropy < 4.2 && stats.variance > 0.1) { - return 'document' - } - - // Person entities have characteristic patterns - if (stats.magnitude > 5 && stats.magnitude < 10 && stats.variance > 0.15) { - return 'person' - } - - // Tools and functions have lower entropy - if (stats.entropy < 3 && stats.magnitude > 4) { - return 'tool' - } - - // Default to generic item for unclear patterns - return 'item' - } - - /** - * Calculate taxonomy-based similarity boost - */ - private async calculateTaxonomyBoost(sourceType: string, targetType: string): Promise { - // Define valid relationship patterns in taxonomy - const validPatterns: Record> = { - 'person': { 'concept': 0.9, 'skill': 0.85, 'organization': 0.8, 'person': 0.7 }, - 'concept': { 'concept': 0.9, 'example': 0.85, 'application': 0.8 }, - 'entity': { 'entity': 0.8, 'property': 0.85, 'action': 0.75 }, - 'object': { 'object': 0.7, 'property': 0.8, 'location': 0.75 }, - 'document': { 'topic': 0.9, 'author': 0.85, 'document': 0.7 }, - 'tool': { 'output': 0.9, 'input': 0.85, 'user': 0.8 }, - 'unknown': { 'unknown': 0.5 } // Fallback - } - - // Get boost from taxonomy patterns - const patterns = validPatterns[sourceType] || validPatterns['unknown'] - const boost = patterns[targetType] || 0.3 // Low score for unrecognized patterns - - return boost - } - - private calculateCosineSimilarity(vectorA: number[], vectorB: number[]): number { - if (vectorA.length !== vectorB.length) return 0 - - let dotProduct = 0 - let normA = 0 - let normB = 0 - - for (let i = 0; i < vectorA.length; i++) { - dotProduct += vectorA[i] * vectorB[i] - normA += vectorA[i] * vectorA[i] - normB += vectorB[i] * vectorB[i] - } - - const magnitude = Math.sqrt(normA) * Math.sqrt(normB) - return magnitude ? dotProduct / magnitude : 0 - } - - private calculateFrequencyScore(sourceId: string, targetId: string, relationType: string): number { - const relationshipKey = `${sourceId}:${relationType}:${targetId}` - const stats = this.relationshipStats.get(relationshipKey) - - if (!stats || stats.count <= 1) return 0 - - // Frequency boost diminishes with each occurrence - const frequencyBoost = Math.log(stats.count) * this.config.frequencyDecay - return Math.min(this.config.maxFrequencyBoost, frequencyBoost) - } - - private calculateTemporalScore(sourceId: string, targetId: string, relationType: string): number { - const relationshipKey = `${sourceId}:${relationType}:${targetId}` - const stats = this.relationshipStats.get(relationshipKey) - - if (!stats) return 1.0 // New relationship - full temporal score - - const daysSinceUpdate = (Date.now() - stats.lastUpdated) / (1000 * 60 * 60 * 24) - const decayFactor = Math.pow(1 - this.config.temporalDecayRate, daysSinceUpdate) - - // Relationships older than temporal window get minimum score - if (daysSinceUpdate > this.config.temporalWindow) { - return this.config.minWeight / this.config.baseWeight - } - - return Math.max(0.1, decayFactor) - } - - private calculateContextScore(metadata?: any): number { - if (!metadata) return 0 - - let contextScore = 0 - - // Boost for explicit importance - if (metadata.importance) { - contextScore += Math.min(0.5, metadata.importance) - } - - // Boost for confidence - if (metadata.confidence) { - contextScore += Math.min(0.3, metadata.confidence) - } - - // Boost for source quality - if (metadata.sourceQuality) { - contextScore += Math.min(0.2, metadata.sourceQuality) - } - - return contextScore - } - - private async updateRelationshipLearning( - sourceId: string, - targetId: string, - relationType: string, - weight: number - ): Promise { - const relationshipKey = `${sourceId}:${relationType}:${targetId}` - let stats = this.relationshipStats.get(relationshipKey) - - if (!stats) { - stats = { - count: 0, - totalWeight: 0, - averageWeight: this.config.baseWeight, - lastUpdated: Date.now(), - semanticScore: 0, - frequencyScore: 0, - temporalScore: 1.0, - confidenceScore: this.config.baseWeight - } - } - - // Update statistics with learning rate - stats.count++ - stats.totalWeight += weight - stats.averageWeight = stats.averageWeight * (1 - this.config.learningRate) + - weight * this.config.learningRate - stats.lastUpdated = Date.now() - - // Update confidence based on consistency - const weightVariance = Math.abs(weight - stats.averageWeight) - const consistencyScore = 1 - Math.min(1, weightVariance) - stats.confidenceScore = stats.confidenceScore * (1 - this.config.learningRate) + - consistencyScore * this.config.learningRate - - this.relationshipStats.set(relationshipKey, stats) - this.metrics.adaptiveAdjustments++ - } - - private getConfidenceScore(sourceId: string, targetId: string, relationType: string): number { - const relationshipKey = `${sourceId}:${relationType}:${targetId}` - const stats = this.relationshipStats.get(relationshipKey) - - return stats ? stats.confidenceScore : this.config.baseWeight - } - - private getScoringMethodsUsed(): string[] { - const methods = [] - if (this.config.enableSemanticScoring) methods.push('semantic') - if (this.config.enableFrequencyAmplification) methods.push('frequency') - if (this.config.enableTemporalDecay) methods.push('temporal') - if (this.config.enableAdaptiveLearning) methods.push('adaptive') - return methods - } - - private storeDetailedScoring( - sourceId: string, - targetId: string, - relationType: string, - scoring: any - ): void { - // Store detailed scoring for analysis and debugging - // In production, this might be sent to analytics system - } - - private updateMetrics(weight: number, computationTime: number): void { - this.metrics.relationshipsScored++ - - // Update average computation time with exponential moving average - const alpha = 0.1 // Smoothing factor - this.metrics.computationTimeMs = - alpha * computationTime + (1 - alpha) * this.metrics.computationTimeMs - - // Track score distribution for analysis - this.updateScoreDistribution(weight) - } - - /** - * Update score distribution statistics - */ - private updateScoreDistribution(weight: number): void { - // Track min/max scores - if (!this.metrics.minScore || weight < this.metrics.minScore) { - this.metrics.minScore = weight - } - if (!this.metrics.maxScore || weight > this.metrics.maxScore) { - this.metrics.maxScore = weight - } - - // Update average score with running average - const n = this.metrics.relationshipsScored - this.metrics.averageScore = ((this.metrics.averageScore || 0) * (n - 1) + weight) / n - - // Track confidence levels - if (weight > this.config.baseWeight * 1.5) { - this.metrics.highConfidenceCount = (this.metrics.highConfidenceCount || 0) + 1 - } - } - - /** - * Get intelligent verb scoring statistics - */ - getStats(): ScoringMetrics & { - totalRelationships: number - averageConfidence: number - highConfidenceRelationships: number - learningEfficiency: number - } { - let totalConfidence = 0 - let highConfidenceCount = 0 - - for (const stats of this.relationshipStats.values()) { - totalConfidence += stats.confidenceScore - if (stats.confidenceScore >= this.config.confidenceThreshold * 2) { - highConfidenceCount++ - } - } - - const totalRelationships = this.relationshipStats.size - const averageConfidence = totalRelationships > 0 ? totalConfidence / totalRelationships : 0 - const learningEfficiency = this.metrics.adaptiveAdjustments / Math.max(1, this.metrics.relationshipsScored) - - return { - ...this.metrics, - totalRelationships, - averageConfidence, - highConfidenceRelationships: highConfidenceCount, - learningEfficiency - } - } - - /** - * Export relationship statistics for analysis - */ - exportRelationshipStats(): Array<{ - relationship: string - metrics: RelationshipMetrics - }> { - return Array.from(this.relationshipStats.entries()).map(([key, metrics]) => ({ - relationship: key, - metrics - })) - } - - /** - * Import relationship statistics from previous sessions - */ - importRelationshipStats(stats: Array<{ relationship: string, metrics: RelationshipMetrics }>): void { - for (const { relationship, metrics } of stats) { - this.relationshipStats.set(relationship, metrics) - } - this.log(`Imported ${stats.length} relationship statistics`) - } - - /** - * Get learning statistics for monitoring and debugging - * Required for Brainy.getVerbScoringStats() - */ - getLearningStats(): { - totalRelationships: number - averageConfidence: number - feedbackCount: number - topRelationships: Array<{ - relationship: string - count: number - averageWeight: number - }> - } { - const relationships = Array.from(this.relationshipStats.entries()) - const totalRelationships = relationships.length - const feedbackCount = relationships.reduce((sum, [, stats]) => sum + stats.count, 0) - - const averageWeight = relationships.reduce((sum, [, stats]) => sum + stats.averageWeight, 0) / totalRelationships || 0 - const averageConfidence = Math.min(averageWeight + 0.2, 1.0) - - const topRelationships = relationships - .map(([key, stats]) => ({ - relationship: key, - count: stats.count, - averageWeight: stats.averageWeight - })) - .sort((a, b) => b.count - a.count) - .slice(0, 10) - - return { - totalRelationships, - averageConfidence, - feedbackCount, - topRelationships - } - } - - /** - * Export learning data for backup or analysis - * Required for Brainy.exportVerbScoringLearningData() - */ - exportLearningData(): string { - const data = { - config: this.config, - stats: Array.from(this.relationshipStats.entries()).map(([key, stats]) => ({ - relationship: key, - ...stats - })), - exportedAt: new Date().toISOString(), - version: '1.0' - } - return JSON.stringify(data, null, 2) - } - - /** - * Import learning data from backup - * Required for Brainy.importVerbScoringLearningData() - */ - importLearningData(jsonData: string): void { - try { - const data = JSON.parse(jsonData) - - if (data.stats && Array.isArray(data.stats)) { - for (const stat of data.stats) { - if (stat.relationship) { - this.relationshipStats.set(stat.relationship, { - count: stat.count || 1, - totalWeight: stat.totalWeight || stat.averageWeight || 0.5, - averageWeight: stat.averageWeight || 0.5, - lastUpdated: stat.lastUpdated || Date.now(), - semanticScore: stat.semanticScore || 0.5, - frequencyScore: stat.frequencyScore || 0.5, - temporalScore: stat.temporalScore || 1.0, - confidenceScore: stat.confidenceScore || 0.5 - }) - } - } - } - - this.log(`Imported learning data: ${this.relationshipStats.size} relationships`) - } catch (error) { - console.error('Failed to import learning data:', error) - throw new Error(`Failed to import learning data: ${error}`) - } - } - - /** - * Provide feedback on a relationship's weight - * Required for Brainy.provideVerbScoringFeedback() - */ - async provideFeedback( - sourceId: string, - targetId: string, - relationType: string, - feedback: number, - feedbackType: 'correction' | 'validation' | 'enhancement' = 'correction' - ): Promise { - const key = `${sourceId}-${relationType}-${targetId}` - const stats = this.relationshipStats.get(key) || { - count: 0, - totalWeight: 0, - averageWeight: 0.5, - lastUpdated: Date.now(), - semanticScore: 0.5, - frequencyScore: 0.5, - temporalScore: 1.0, - confidenceScore: 0.5 - } - - // Update statistics based on feedback - if (feedbackType === 'correction') { - // Direct correction - heavily weight the feedback - stats.averageWeight = stats.averageWeight * 0.3 + feedback * 0.7 - } else if (feedbackType === 'validation') { - // Validation - slightly adjust towards feedback - stats.averageWeight = stats.averageWeight * 0.8 + feedback * 0.2 - } else { - // Enhancement - minor adjustment - stats.averageWeight = stats.averageWeight * 0.9 + feedback * 0.1 - } - - stats.count++ - stats.totalWeight += feedback - stats.lastUpdated = Date.now() - - this.relationshipStats.set(key, stats) - this.metrics.adaptiveAdjustments++ - } - - /** - * Compute intelligent scores for a verb relationship - * Used internally during verb creation - */ - async computeVerbScores( - sourceNoun: any, - targetNoun: any, - relationType: string - ): Promise<{ - weight: number - confidence: number - reasoning: string[] - }> { - const reasoning: string[] = [] - let totalScore = 0 - let components = 0 - - // Semantic scoring - if (this.config.enableSemanticScoring && sourceNoun?.vector && targetNoun?.vector) { - const similarity = this.calculateCosineSimilarity(sourceNoun.vector, targetNoun.vector) - const semanticScore = Math.max(similarity, this.config.semanticThreshold) - totalScore += semanticScore * this.config.semanticWeight - components++ - reasoning.push(`Semantic similarity: ${(similarity * 100).toFixed(1)}%`) - } - - // Frequency scoring - const key = `${sourceNoun?.id}-${relationType}-${targetNoun?.id}` - const stats = this.relationshipStats.get(key) - if (this.config.enableFrequencyAmplification && stats) { - const frequencyScore = Math.min(1 + (stats.count - 1) * 0.1, this.config.maxFrequencyBoost) - totalScore += frequencyScore * 0.3 - components++ - reasoning.push(`Frequency boost: ${frequencyScore.toFixed(2)}x`) - } - - // Temporal decay scoring - if (this.config.enableTemporalDecay) { - reasoning.push(`Temporal decay applied (rate: ${this.config.temporalDecayRate})`) - } - - // Calculate final weight - const weight = components > 0 - ? Math.min(Math.max(totalScore / components, this.config.minWeight), this.config.maxWeight) - : this.config.baseWeight - - const confidence = Math.min(weight + 0.2, 1.0) - - return { weight, confidence, reasoning } - } - - protected async onShutdown(): Promise { - const stats = this.getStats() - this.log(`Intelligent verb scoring shutdown: ${stats.relationshipsScored} relationships scored, ${Math.round(stats.averageConfidence * 100)}% avg confidence`) - } -} \ No newline at end of file diff --git a/src/augmentations/manifest.ts b/src/augmentations/manifest.ts deleted file mode 100644 index 373e8556..00000000 --- a/src/augmentations/manifest.ts +++ /dev/null @@ -1,207 +0,0 @@ -/** - * Augmentation Manifest Types - * - * Defines the manifest structure for augmentation discovery and configuration - * Enables tools like brain-cloud to discover and configure augmentations - */ - -/** - * JSON Schema type for configuration validation - */ -export interface JSONSchema { - type?: 'object' | 'array' | 'string' | 'number' | 'boolean' | 'null' - properties?: Record - items?: JSONSchema - required?: string[] - default?: any - description?: string - minimum?: number - maximum?: number - minLength?: number - maxLength?: number - pattern?: string - enum?: any[] - additionalProperties?: boolean | JSONSchema -} - -/** - * Augmentation manifest for discovery and configuration - */ -export interface AugmentationManifest { - /** - * Unique identifier for the augmentation (e.g., 'wal', 'cache') - */ - id: string - - /** - * Display name for the augmentation - */ - name: string - - /** - * Semantic version (e.g., '2.0.0') - */ - version: string - - /** - * Author or organization - */ - author?: string - - /** - * Short description of what the augmentation does - */ - description: string - - /** - * Detailed description for documentation - */ - longDescription?: string - - /** - * Augmentation category for organization - */ - category: 'storage' | 'performance' | 'analytics' | 'integration' | 'internal' | 'core' | 'premium' | 'community' | 'external' - - /** - * JSON Schema for configuration options - * Used to generate configuration UIs and validate configuration - */ - configSchema?: JSONSchema - - /** - * Default configuration values - */ - configDefaults?: Record - - /** - * Configuration examples for documentation - */ - configExamples?: Array<{ - name: string - description: string - config: Record - }> - - /** - * Minimum Brainy version required - */ - minBrainyVersion?: string - - /** - * Maximum Brainy version supported - */ - maxBrainyVersion?: string - - /** - * Other augmentations this one depends on - */ - dependencies?: Array<{ - id: string - version?: string - optional?: boolean - }> - - /** - * Keywords for search and discovery - */ - keywords?: string[] - - /** - * URL to documentation - */ - documentation?: string - - /** - * URL to source code repository - */ - repository?: string - - /** - * License identifier (e.g., 'MIT', 'Apache-2.0') - */ - license?: string - - /** - * UI hints for tools and configuration interfaces - */ - ui?: { - /** - * Icon for the augmentation (emoji or URL) - */ - icon?: string - - /** - * Color theme for UI - */ - color?: string - - /** - * Custom React component name for configuration - */ - configComponent?: string - - /** - * URL to dashboard or control panel - */ - dashboardUrl?: string - - /** - * Hide from UI listings - */ - hidden?: boolean - } - - /** - * Performance characteristics - */ - performance?: { - /** - * Estimated memory usage - */ - memoryUsage?: 'low' | 'medium' | 'high' - - /** - * CPU intensity - */ - cpuUsage?: 'low' | 'medium' | 'high' - - /** - * Network usage - */ - networkUsage?: 'none' | 'low' | 'medium' | 'high' - } - - /** - * Feature flags this augmentation provides - */ - features?: string[] - - /** - * Operations this augmentation enhances - */ - enhancedOperations?: string[] - - /** - * Metrics this augmentation exposes - */ - metrics?: Array<{ - name: string - type: 'counter' | 'gauge' | 'histogram' - description: string - }> - - /** - * Status of the augmentation - */ - status?: 'stable' | 'beta' | 'experimental' | 'deprecated' - - /** - * Deprecation notice if applicable - */ - deprecation?: { - since: string - alternative?: string - removalDate?: string - } -} \ No newline at end of file diff --git a/src/augmentations/metadataEnforcer.ts b/src/augmentations/metadataEnforcer.ts deleted file mode 100644 index b27705a9..00000000 --- a/src/augmentations/metadataEnforcer.ts +++ /dev/null @@ -1,204 +0,0 @@ -/** - * Runtime enforcement of metadata contracts - * Ensures augmentations only access declared fields - */ - -import { BrainyAugmentation, MetadataAccess } from './brainyAugmentation.js' - -export class MetadataEnforcer { - /** - * Enforce metadata access based on augmentation contract - * Returns a wrapped metadata object that enforces the contract - */ - static enforce( - augmentation: BrainyAugmentation, - metadata: any, - operation: 'read' | 'write' = 'write' - ): any { - // Handle simple contracts - if (augmentation.metadata === 'none') { - // No access at all - if (operation === 'read') return null - throw new Error(`Augmentation '${augmentation.name}' has metadata='none' - cannot access metadata`) - } - - if (augmentation.metadata === 'readonly') { - if (operation === 'read') { - // Return frozen deep clone for read-only access - return deepFreeze(deepClone(metadata)) - } - throw new Error(`Augmentation '${augmentation.name}' has metadata='readonly' - cannot write`) - } - - // Handle specific field access - const access = augmentation.metadata as MetadataAccess - - if (operation === 'read') { - // For reads, filter to allowed fields - if (access.reads === '*') { - return deepClone(metadata) // Can read everything - } - - if (!access.reads) { - return {} // No read access - } - - // Filter to specific fields - const filtered: any = {} - for (const field of access.reads) { - if (field.includes('.')) { - // Handle nested fields like '_brainy.deleted' - const parts = field.split('.') - let source = metadata - let target = filtered - - for (let i = 0; i < parts.length - 1; i++) { - const part = parts[i] - if (!source[part]) break - if (!target[part]) target[part] = {} - source = source[part] - target = target[part] - } - - const lastPart = parts[parts.length - 1] - if (source && lastPart in source) { - target[lastPart] = source[lastPart] - } - } else { - // Simple field - if (field in metadata) { - filtered[field] = metadata[field] - } - } - } - - return filtered - } - - // For writes, create a proxy that validates - return new Proxy(metadata, { - set(target, prop, value) { - const field = String(prop) - - // Check if write is allowed - if (access.writes === '*') { - // Can write anything - target[prop] = value - return true - } - - if (!access.writes || !access.writes.includes(field)) { - throw new Error( - `Augmentation '${augmentation.name}' cannot write to field '${field}'. ` + - `Allowed writes: ${access.writes?.join(', ') || 'none'}` - ) - } - - // Check namespace if specified - if (access.namespace && !field.startsWith(access.namespace)) { - console.warn( - `Augmentation '${augmentation.name}' writing outside its namespace. ` + - `Expected: ${access.namespace}.*, got: ${field}` - ) - } - - target[prop] = value - return true - }, - - deleteProperty(target, prop) { - const field = String(prop) - - // Deletion counts as a write - if (access.writes === '*' || access.writes?.includes(field)) { - delete target[prop] - return true - } - - throw new Error( - `Augmentation '${augmentation.name}' cannot delete field '${field}'` - ) - } - }) - } - - /** - * Validate that an augmentation's actual behavior matches its contract - * Used in testing to verify contracts are accurate - */ - static async validateContract( - augmentation: BrainyAugmentation, - testMetadata: any = { test: 'data', _brainy: { deleted: false } } - ): Promise<{ valid: boolean; violations: string[] }> { - const violations: string[] = [] - - // Test read access - try { - const readable = this.enforce(augmentation, testMetadata, 'read') - - if (augmentation.metadata === 'none' && readable !== null) { - violations.push(`Contract says 'none' but got readable metadata`) - } - } catch (error) { - violations.push(`Read enforcement error: ${error}`) - } - - // Test write access - try { - const writable = this.enforce(augmentation, testMetadata, 'write') - - if (augmentation.metadata === 'none') { - violations.push(`Contract says 'none' but got writable metadata`) - } - - if (augmentation.metadata === 'readonly') { - // Try to write - should fail - try { - writable.testWrite = 'value' - violations.push(`Contract says 'readonly' but write succeeded`) - } catch { - // Expected to fail - } - } - } catch (error) { - // Expected for 'none' and 'readonly' on write - if (augmentation.metadata !== 'none' && augmentation.metadata !== 'readonly') { - violations.push(`Write enforcement error: ${error}`) - } - } - - return { - valid: violations.length === 0, - violations - } - } -} - -// Helper functions -function deepClone(obj: any): any { - if (obj === null || typeof obj !== 'object') return obj - if (obj instanceof Date) return new Date(obj) - if (obj instanceof Array) return obj.map(item => deepClone(item)) - - const cloned: any = {} - for (const key in obj) { - if (obj.hasOwnProperty(key)) { - cloned[key] = deepClone(obj[key]) - } - } - return cloned -} - -function deepFreeze(obj: any): any { - Object.freeze(obj) - - Object.getOwnPropertyNames(obj).forEach(prop => { - if (obj[prop] !== null && - (typeof obj[prop] === 'object' || typeof obj[prop] === 'function') && - !Object.isFrozen(obj[prop])) { - deepFreeze(obj[prop]) - } - }) - - return obj -} \ No newline at end of file diff --git a/src/augmentations/metricsAugmentation.ts b/src/augmentations/metricsAugmentation.ts deleted file mode 100644 index f4725594..00000000 --- a/src/augmentations/metricsAugmentation.ts +++ /dev/null @@ -1,356 +0,0 @@ -/** - * Metrics Augmentation - Optional Performance & Usage Metrics - * - * IMPORTANT: This is SEPARATE from core counting (brain.counts.*) which is always enabled. - * - * Core counting provides O(1) entity/relationship counts for scalability. - * This augmentation provides performance analytics and observability. - * - * Features: - * - Search performance tracking (latency, throughput) - * - Cache hit/miss rates - * - Operation timing analysis - * - Usage pattern insights - * - * Zero-config: Automatically enabled for observability - * Can be disabled via augmentation registry without affecting core performance - */ - -import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js' -import { StatisticsCollector } from '../utils/statisticsCollector.js' -import type { BaseStorageAdapter as StorageAdapter } from '../storage/adapters/baseStorageAdapter.js' - -export interface MetricsConfig { - enabled?: boolean - trackSearches?: boolean - trackContentTypes?: boolean - trackVerbTypes?: boolean - trackStorageSizes?: boolean - persistMetrics?: boolean - metricsInterval?: number - silent?: boolean // New: Silent mode support -} - -/** - * MetricsAugmentation - Makes metrics collection optional and pluggable - * - * Features: - * - Performance tracking (search latency, throughput) - * - Usage patterns (content types, verb types) - * - Storage metrics (sizes, counts) - * - Zero-config with smart defaults - */ -export class MetricsAugmentation extends BaseAugmentation { - readonly metadata = 'readonly' as const // Reads metadata for metrics - readonly name = 'metrics' - readonly timing = 'after' as const - operations = ['add', 'update', 'search', 'find', 'similar', 'delete', 'relate', 'unrelate', 'clear', 'all'] as ('add' | 'update' | 'search' | 'find' | 'similar' | 'delete' | 'relate' | 'unrelate' | 'clear' | 'all')[] - readonly priority = 40 // Low priority, runs after other augmentations - - private statisticsCollector: StatisticsCollector | null = null - protected config: MetricsConfig - private metricsTimer: NodeJS.Timeout | null = null - - constructor(config: MetricsConfig = {}) { - super() - this.config = { - enabled: true, - trackSearches: true, - trackContentTypes: true, - trackVerbTypes: true, - trackStorageSizes: true, - persistMetrics: true, - metricsInterval: 60000, // Update metrics every minute - ...config - } - } - - protected async onInitialize(): Promise { - if (!this.config.enabled) { - this.log('Metrics augmentation disabled by configuration') - return - } - - // Initialize statistics collector - this.statisticsCollector = new StatisticsCollector() - - // Load existing metrics from storage if available - if (this.config.persistMetrics && this.context?.storage) { - try { - const storage = this.context.storage as StorageAdapter - const existingStats = await storage.getStatistics?.() - if (existingStats) { - this.statisticsCollector.mergeFromStorage(existingStats) - this.log('Loaded existing metrics from storage') - } - } catch (e) { - this.log('Could not load existing metrics', 'info') - } - } - - // Start metrics update timer - if (this.config.metricsInterval && this.config.metricsInterval > 0) { - this.startMetricsTimer() - } - - this.log('Metrics augmentation initialized') - } - - protected async onShutdown(): Promise { - // Stop metrics timer - if (this.metricsTimer) { - clearInterval(this.metricsTimer) - this.metricsTimer = null - } - - // Persist final metrics - if (this.config.persistMetrics && this.statisticsCollector && this.context?.storage) { - try { - await this.persistMetrics() - } catch (error) { - this.log('Error persisting metrics during shutdown', 'warn') - } - } - - this.statisticsCollector = null - this.log('Metrics augmentation shut down') - } - - /** - * Execute augmentation - track metrics for operations - */ - async execute( - operation: string, - params: any, - next: () => Promise - ): Promise { - // If metrics disabled, just pass through - if (!this.statisticsCollector || !this.config.enabled) { - return next() - } - - // Track operation timing - const startTime = Date.now() - - try { - const result = await next() - const duration = Date.now() - startTime - - // Track metrics based on operation - switch (operation) { - case 'add': - this.handleAdd(params, duration) - break - - case 'search': - this.handleSearch(params, duration) - break - - case 'delete': - this.handleDelete(duration) - break - - case 'clear': - this.handleClear() - break - } - - return result - } catch (error) { - // Error tracking removed - StatisticsCollector doesn't have trackError method - // Could be added later if needed - throw error - } - } - - /** - * Handle add operation metrics - */ - private handleAdd(params: any, duration: number): void { - if (!this.statisticsCollector) return - - // Track update - this.statisticsCollector.trackUpdate() - - // Track content type if available - if (this.config.trackContentTypes && params.metadata?.noun) { - this.statisticsCollector.trackContentType(params.metadata.noun) - } - - // Track verb type if it's a verb operation - if (this.config.trackVerbTypes && params.metadata?.verb) { - this.statisticsCollector.trackVerbType(params.metadata.verb) - } - - this.log(`Add operation completed in ${duration}ms`, 'info') - } - - /** - * Handle search operation metrics - */ - private handleSearch(params: any, duration: number): void { - if (!this.statisticsCollector || !this.config.trackSearches) return - - const { query } = params - this.statisticsCollector.trackSearch(query || '', duration) - this.log(`Search completed in ${duration}ms`, 'info') - } - - /** - * Handle delete operation metrics - */ - private handleDelete(duration: number): void { - if (!this.statisticsCollector) return - - this.statisticsCollector.trackUpdate() - this.log(`Delete operation completed in ${duration}ms`, 'info') - } - - /** - * Handle clear operation - reset metrics - */ - private handleClear(): void { - if (!this.statisticsCollector) return - - // Reset statistics when all data is cleared - this.statisticsCollector = new StatisticsCollector() - this.log('Metrics reset due to clear operation') - } - - /** - * Start periodic metrics update timer - */ - private startMetricsTimer(): void { - if (this.metricsTimer) return - - this.metricsTimer = setInterval(async () => { - await this.updateStorageMetrics() - if (this.config.persistMetrics) { - await this.persistMetrics() - } - }, this.config.metricsInterval!) - } - - /** - * Update storage size metrics - */ - private async updateStorageMetrics(): Promise { - if (!this.statisticsCollector || !this.config.trackStorageSizes) return - if (!this.context?.storage) return - - try { - const storage = this.context.storage as StorageAdapter - const stats = await storage.getStatistics?.() - - if (stats) { - // Estimate sizes based on counts - const avgNounSize = 1024 // 1KB average - const avgVerbSize = 256 // 256B average - - this.statisticsCollector.updateStorageSizes({ - nouns: (stats.totalNodes || 0) * avgNounSize, - verbs: (stats.totalEdges || 0) * avgVerbSize, - metadata: (stats.totalNodes || 0) * 512, // 512B per metadata - index: (stats.hnswIndexSize || 0) // Use HNSW index size from stats - }) - } - } catch (e) { - this.log('Could not update storage metrics', 'info') - } - } - - /** - * Persist metrics to storage - */ - private async persistMetrics(): Promise { - if (!this.statisticsCollector || !this.context?.storage) return - - try { - const stats = this.statisticsCollector.getStatistics() - // Storage adapters can optionally store these metrics - // This is a no-op for adapters that don't support it - this.log('Metrics persisted to storage', 'info') - } catch (e) { - this.log('Could not persist metrics', 'info') - } - } - - /** - * Get current metrics (performance analytics only) - * - * NOTE: For production counting (entities, relationships), use: - * - brain.counts.entities() - O(1) total count - * - brain.counts.byType() - O(1) type-specific counts - * - brain.counts.relationships() - O(1) relationship counts - */ - getStatistics() { - if (!this.statisticsCollector) { - return { - enabled: false, - note: 'For core counting, use brain.counts.* APIs which are always available', - totalSearches: 0, - totalUpdates: 0, - contentTypes: {}, - verbTypes: {}, - searchPerformance: { - averageLatency: 0, - p95Latency: 0, - p99Latency: 0 - } - } - } - - return { - enabled: true, - note: 'Performance analytics only. Core counting available via brain.counts.*', - ...this.statisticsCollector.getStatistics() - } - } - - /** - * Record cache hit (called by cache augmentation) - * Note: Cache metrics are tracked internally by StatisticsCollector - */ - recordCacheHit(): void { - // StatisticsCollector doesn't have trackCacheHit method - // Cache metrics would need to be implemented if needed - this.log('Cache hit recorded', 'info') - } - - /** - * Record cache miss (called by cache augmentation) - * Note: Cache metrics are tracked internally by StatisticsCollector - */ - recordCacheMiss(): void { - // StatisticsCollector doesn't have trackCacheMiss method - // Cache metrics would need to be implemented if needed - this.log('Cache miss recorded', 'info') - } - - /** - * Track custom metric - * Note: Custom metrics would need to be implemented in StatisticsCollector - */ - trackCustomMetric(name: string, value: number): void { - // StatisticsCollector doesn't have trackCustomMetric method - // Could be added later if needed - this.log(`Custom metric recorded: ${name}=${value}`, 'info') - } - - /** - * Reset all metrics - */ - reset(): void { - if (this.statisticsCollector) { - this.statisticsCollector = new StatisticsCollector() - this.log('Metrics manually reset') - } - } -} - -/** - * Factory function for zero-config metrics augmentation - */ -export function createMetricsAugmentation(config?: MetricsConfig): MetricsAugmentation { - return new MetricsAugmentation(config) -} \ No newline at end of file diff --git a/src/augmentations/monitoringAugmentation.ts b/src/augmentations/monitoringAugmentation.ts deleted file mode 100644 index b503dd53..00000000 --- a/src/augmentations/monitoringAugmentation.ts +++ /dev/null @@ -1,270 +0,0 @@ -/** - * Monitoring Augmentation - Optional Health & Performance Monitoring - * - * Replaces the hardcoded HealthMonitor in Brainy with an optional augmentation. - * Provides health checks, performance monitoring, and distributed system tracking. - * - * Zero-config: Automatically enabled for distributed deployments - * Can be disabled or customized via augmentation registry - */ - -import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js' -import { HealthMonitor } from '../distributed/healthMonitor.js' -import { DistributedConfigManager as ConfigManager } from '../distributed/configManager.js' - -export interface MonitoringConfig { - enabled?: boolean - healthCheckInterval?: number - metricsInterval?: number - trackLatency?: boolean - trackErrors?: boolean - trackCacheMetrics?: boolean - exposeHealthEndpoint?: boolean -} - -/** - * MonitoringAugmentation - Makes health monitoring optional and pluggable - * - * Features: - * - Health status tracking - * - Performance monitoring - * - Error rate tracking - * - Distributed system health - * - Zero-config with smart defaults - */ -export class MonitoringAugmentation extends BaseAugmentation { - readonly metadata = 'readonly' as const // Reads metadata for monitoring - readonly name = 'monitoring' - readonly timing = 'after' as const - operations = ['search', 'find', 'similar', 'add', 'update', 'delete', 'relate', 'unrelate', 'all'] as ('search' | 'find' | 'similar' | 'add' | 'update' | 'delete' | 'relate' | 'unrelate' | 'all')[] - readonly priority = 30 // Low priority, observability layer - - private healthMonitor: HealthMonitor | null = null - private configManager: ConfigManager | null = null - protected config: MonitoringConfig - private requestStartTimes = new Map() - - constructor(config: MonitoringConfig = {}) { - super() - this.config = { - enabled: true, - healthCheckInterval: 30000, // 30 seconds - metricsInterval: 60000, // 1 minute - trackLatency: true, - trackErrors: true, - trackCacheMetrics: true, - exposeHealthEndpoint: true, - ...config - } - } - - protected async onInitialize(): Promise { - if (!this.config.enabled) { - this.log('Monitoring augmentation disabled by configuration') - return - } - - // Initialize config manager and health monitor (requires storage) - if (this.context?.storage) { - this.configManager = new ConfigManager(this.context.storage as any) - this.healthMonitor = new HealthMonitor(this.configManager) - this.healthMonitor.start() - } else { - this.log('Storage not available - health monitoring disabled', 'warn') - } - - this.log('Monitoring augmentation initialized') - } - - protected async onShutdown(): Promise { - if (this.healthMonitor) { - this.healthMonitor.stop() - this.healthMonitor = null - } - - this.configManager = null - this.requestStartTimes.clear() - - this.log('Monitoring augmentation shut down') - } - - /** - * Execute augmentation - track health metrics - */ - async execute( - operation: string, - params: any, - next: () => Promise - ): Promise { - // If monitoring disabled, just pass through - if (!this.healthMonitor || !this.config.enabled) { - return next() - } - - // Generate request ID for tracking - const requestId = `${operation}-${Date.now()}-${Math.random()}` - - // Track request start time - if (this.config.trackLatency) { - this.requestStartTimes.set(requestId, Date.now()) - } - - try { - // Execute operation - const result = await next() - - // Track successful operation - if (this.config.trackLatency) { - const startTime = this.requestStartTimes.get(requestId) - if (startTime) { - const latency = Date.now() - startTime - this.healthMonitor.recordRequest(latency, false) - this.requestStartTimes.delete(requestId) - } - } - - // Update vector count for 'add' operations - if (operation === 'add' && this.context?.brain) { - try { - const count = await this.context.brain.getNounCount() - this.healthMonitor.updateVectorCount(count) - } catch (e) { - // Ignore count update errors - } - } - - // Track cache metrics for search operations - if (operation === 'search' && this.config.trackCacheMetrics) { - // Check if result came from cache (would be set by cache augmentation) - const fromCache = (params as any)._fromCache || false - this.healthMonitor.recordCacheAccess(fromCache) - } - - return result - } catch (error) { - // Track error - if (this.config.trackErrors) { - const startTime = this.requestStartTimes.get(requestId) - if (startTime) { - const latency = Date.now() - startTime - this.healthMonitor.recordRequest(latency, true) - this.requestStartTimes.delete(requestId) - } else { - this.healthMonitor.recordRequest(0, true) - } - } - - throw error - } - } - - /** - * Get health status - */ - getHealthStatus() { - if (!this.healthMonitor) { - return { - status: 'disabled', - enabled: false, - uptime: 0, - vectorCount: 0, - requestRate: 0, - errorRate: 0, - cacheHitRate: 0 - } - } - - return { - status: 'healthy', - enabled: true, - ...this.healthMonitor.getHealthEndpointData() - } - } - - /** - * Get health endpoint data (for API exposure) - */ - getHealthEndpointData() { - if (!this.healthMonitor) { - return { - status: 'disabled', - timestamp: new Date().toISOString() - } - } - - return this.healthMonitor.getHealthEndpointData() - } - - /** - * Update vector count manually - */ - updateVectorCount(count: number): void { - if (this.healthMonitor) { - this.healthMonitor.updateVectorCount(count) - } - } - - /** - * Record custom health metric - */ - recordCustomMetric(name: string, value: number): void { - if (this.healthMonitor) { - // Health monitor could be extended to track custom metrics - this.log(`Custom metric recorded: ${name}=${value}`, 'info') - } - } - - /** - * Check if system is healthy - */ - isHealthy(): boolean { - if (!this.healthMonitor) return true // If disabled, assume healthy - - const data = this.healthMonitor.getHealthEndpointData() - - // Define health criteria - const errorRateThreshold = 0.05 // 5% error rate - const minUptime = 60000 // 1 minute - - return ( - data.errorRate < errorRateThreshold && - data.uptime > minUptime - ) - } - - /** - * Get uptime in milliseconds - */ - getUptime(): number { - if (!this.healthMonitor) return 0 - - const data = this.healthMonitor.getHealthEndpointData() - return data.uptime || 0 - } - - /** - * Force health check - */ - async checkHealth(): Promise { - if (!this.healthMonitor) return true - - // Perform active health check - try { - // Could ping storage, check memory, etc. - if (this.context?.storage) { - await this.context.storage.getStatistics?.() - } - return true - } catch (error) { - this.log('Health check failed', 'warn') - return false - } - } -} - -/** - * Factory function for zero-config monitoring augmentation - */ -export function createMonitoringAugmentation(config?: MonitoringConfig): MonitoringAugmentation { - return new MonitoringAugmentation(config) -} \ No newline at end of file diff --git a/src/augmentations/rateLimitAugmentation.ts b/src/augmentations/rateLimitAugmentation.ts deleted file mode 100644 index 84fb8396..00000000 --- a/src/augmentations/rateLimitAugmentation.ts +++ /dev/null @@ -1,384 +0,0 @@ -/** - * Rate Limiting Augmentation - * Provides configurable rate limiting for Brainy operations - */ - -import { BaseAugmentation } from './brainyAugmentation.js' -import { AugmentationManifest } from './manifest.js' - -export interface RateLimitConfig { - enabled?: boolean - limits?: { - searches?: number // Per minute - writes?: number // Per minute - reads?: number // Per minute - deletes?: number // Per minute - } - windowMs?: number // Time window in milliseconds - skipSuccessfulRequests?: boolean - skipFailedRequests?: boolean - keyGenerator?: (context: any) => string -} - -interface RateLimitEntry { - count: number - resetTime: number -} - -/** - * Rate Limit Augmentation - */ -export class RateLimitAugmentation extends BaseAugmentation { - readonly name = 'rateLimiter' - readonly timing = 'before' as const - readonly metadata = 'none' as const - operations = ['search', 'find', 'add', 'update', 'delete', 'get'] as any - readonly priority = 10 // High priority, runs early - - // Augmentation metadata - readonly category = 'core' as const // Use 'core' as security isn't a valid category - readonly description = 'Provides rate limiting for Brainy operations' - - private limiters: Map> = new Map() - private windowMs: number - - constructor(config: RateLimitConfig = {}) { - super(config) - - // Merge with defaults - this.config = { - enabled: config.enabled ?? true, - limits: { - searches: config.limits?.searches ?? 1000, - writes: config.limits?.writes ?? 100, - reads: config.limits?.reads ?? 5000, - deletes: config.limits?.deletes ?? 50 - }, - windowMs: config.windowMs ?? 60000, // 1 minute default - skipSuccessfulRequests: config.skipSuccessfulRequests ?? false, - skipFailedRequests: config.skipFailedRequests ?? true, - keyGenerator: config.keyGenerator || this.defaultKeyGenerator - } - - this.windowMs = this.config.windowMs! - - // Initialize operation limiters - this.initializeLimiters() - } - - getManifest(): AugmentationManifest { - return { - id: 'rate-limiter', - name: 'Rate Limiter', - version: '1.0.0', - description: 'Configurable rate limiting for API operations', - longDescription: 'Provides per-operation rate limiting with configurable windows and limits. Helps prevent abuse and ensures fair resource usage.', - category: 'core', - configSchema: { - type: 'object', - properties: { - enabled: { - type: 'boolean', - default: true, - description: 'Enable or disable rate limiting' - }, - limits: { - type: 'object', - properties: { - searches: { - type: 'number', - default: 1000, - description: 'Search operations per minute' - }, - writes: { - type: 'number', - default: 100, - description: 'Write operations per minute' - }, - reads: { - type: 'number', - default: 5000, - description: 'Read operations per minute' - }, - deletes: { - type: 'number', - default: 50, - description: 'Delete operations per minute' - } - } - }, - windowMs: { - type: 'number', - default: 60000, - description: 'Time window in milliseconds' - } - } - }, - configDefaults: { - enabled: true, - limits: { - searches: 1000, - writes: 100, - reads: 5000, - deletes: 50 - }, - windowMs: 60000 - }, - minBrainyVersion: '3.0.0', - keywords: ['rate-limit', 'security', 'throttle'], - documentation: 'https://docs.brainy.dev/augmentations/rate-limit', - status: 'stable', - performance: { - memoryUsage: 'low', - cpuUsage: 'low', - networkUsage: 'none' - }, - features: ['per-operation-limits', 'configurable-windows', 'key-based-limiting'], - enhancedOperations: ['search', 'add', 'update', 'delete', 'get'], - metrics: [ - { - name: 'rate_limit_exceeded', - type: 'counter', - description: 'Number of rate limit violations' - }, - { - name: 'rate_limit_requests', - type: 'counter', - description: 'Total requests checked' - } - ] - } - } - - /** - * Initialize rate limiters for each operation type - */ - private initializeLimiters(): void { - const operations = ['searches', 'writes', 'reads', 'deletes'] - for (const op of operations) { - this.limiters.set(op, new Map()) - } - } - - /** - * Default key generator (could be IP, user ID, etc.) - */ - private defaultKeyGenerator(_context: any): string { - // In a real implementation, this would extract IP or user ID - return 'default' - } - - /** - * Check if request should be rate limited - */ - private checkRateLimit(operation: string, key: string): boolean { - const limiter = this.limiters.get(operation) - if (!limiter) return false - - const limit = (this.config.limits as any)[operation] - if (!limit) return false - - const now = Date.now() - let entry = limiter.get(key) - - // Initialize or reset entry - if (!entry || now >= entry.resetTime) { - entry = { - count: 0, - resetTime: now + this.windowMs - } - limiter.set(key, entry) - } - - // Check if limit exceeded - if (entry.count >= limit) { - return true // Rate limited - } - - // Increment counter - entry.count++ - return false - } - - /** - * Get remaining requests for an operation - */ - private getRemainingRequests(operation: string, key: string): number { - const limiter = this.limiters.get(operation) - if (!limiter) return -1 - - const limit = (this.config.limits as any)[operation] - if (!limit) return -1 - - const entry = limiter.get(key) - if (!entry) return limit - - const now = Date.now() - if (now >= entry.resetTime) return limit - - return Math.max(0, limit - entry.count) - } - - /** - * Get time until reset - */ - private getResetTime(operation: string, key: string): number { - const limiter = this.limiters.get(operation) - if (!limiter) return 0 - - const entry = limiter.get(key) - if (!entry) return 0 - - const now = Date.now() - return Math.max(0, entry.resetTime - now) - } - - protected async onInitialize(): Promise { - if (!this.config.enabled) { - this.log('Rate limiter disabled by configuration') - return - } - - this.log(`Rate limiter initialized (window: ${this.windowMs}ms)`) - - // Start cleanup timer - setInterval(() => { - this.cleanup() - }, this.windowMs) - } - - protected async onShutdown(): Promise { - this.clear() - this.log('Rate limiter shut down') - } - - /** - * Execute augmentation - apply rate limiting - */ - async execute( - operation: string, - params: any, - next: () => Promise - ): Promise { - // If rate limiting is disabled, just pass through - if (!this.config.enabled) { - return next() - } - - // Map operations to rate limit categories - let rateLimitOperation: string - switch (operation) { - case 'search': - case 'find': - case 'similar': - rateLimitOperation = 'searches' - break - case 'add': - case 'update': - rateLimitOperation = 'writes' - break - case 'delete': - rateLimitOperation = 'deletes' - break - case 'get': - rateLimitOperation = 'reads' - break - default: - return next() // Don't rate limit unknown operations - } - - const key = (this.config.keyGenerator as any)(params) - - if (this.checkRateLimit(rateLimitOperation, key)) { - const error = new Error(`Rate limit exceeded for ${operation}`) - ;(error as any).statusCode = 429 - ;(error as any).retryAfter = this.getResetTime(rateLimitOperation, key) - ;(error as any).rateLimit = { - limit: (this.config.limits as any)[rateLimitOperation], - remaining: 0, - reset: Date.now() + this.getResetTime(rateLimitOperation, key) - } - throw error - } - - try { - const result = await next() - - // Add rate limit info to result if possible - if (result && typeof result === 'object' && !Array.isArray(result)) { - (result as any)._rateLimit = { - limit: (this.config.limits as any)[rateLimitOperation], - remaining: this.getRemainingRequests(rateLimitOperation, key), - reset: Date.now() + this.getResetTime(rateLimitOperation, key) - } - } - - return result - } catch (error) { - // Optionally don't count failed requests - if (this.config.skipFailedRequests) { - const limiter = this.limiters.get(rateLimitOperation)! - const entry = limiter.get(key) - if (entry && entry.count > 0) entry.count-- - } - throw error - } - } - - /** - * Get rate limit statistics - */ - getStats(): { - operations: Record - } { - const stats: any = { operations: {} } - - for (const [operation, limiter] of this.limiters) { - let totalRequests = 0 - for (const entry of limiter.values()) { - totalRequests += entry.count - } - - stats.operations[operation] = { - activeKeys: limiter.size, - totalRequests - } - } - - return stats - } - - /** - * Clear all rate limit entries - */ - clear(): void { - for (const limiter of this.limiters.values()) { - limiter.clear() - } - } - - /** - * Clear expired entries (cleanup) - */ - cleanup(): void { - const now = Date.now() - - for (const limiter of this.limiters.values()) { - for (const [key, entry] of limiter) { - if (now >= entry.resetTime) { - limiter.delete(key) - } - } - } - } -} - -/** - * Create rate limit augmentation - */ -export function createRateLimitAugmentation(config?: RateLimitConfig): RateLimitAugmentation { - return new RateLimitAugmentation(config) -} \ No newline at end of file diff --git a/src/augmentations/requestDeduplicatorAugmentation.ts b/src/augmentations/requestDeduplicatorAugmentation.ts deleted file mode 100644 index 707b71ce..00000000 --- a/src/augmentations/requestDeduplicatorAugmentation.ts +++ /dev/null @@ -1,216 +0,0 @@ -/** - * Request Deduplicator Augmentation - * - * Prevents duplicate concurrent requests to improve performance by 3x - * Automatically deduplicates identical operations - */ - -import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js' - -interface PendingRequest { - promise: Promise - timestamp: number - count: number -} - -interface DeduplicatorConfig { - enabled?: boolean - ttl?: number // Time to live for cached requests (ms) - maxSize?: number // Maximum number of cached requests -} - -export class RequestDeduplicatorAugmentation extends BaseAugmentation { - name = 'RequestDeduplicator' - timing = 'around' as const - metadata = 'none' as const // Doesn't access metadata - operations = ['search', 'find', 'similar', 'searchText', 'searchByNounTypes', 'findSimilar', 'get'] as ('search' | 'find' | 'similar' | 'searchText' | 'searchByNounTypes' | 'findSimilar' | 'get')[] - priority = 50 // Performance optimization - - private pendingRequests: Map> = new Map() - protected config: Required - private cleanupInterval?: NodeJS.Timeout - - constructor(config: DeduplicatorConfig = {}) { - super() - this.config = { - enabled: config.enabled ?? true, - ttl: config.ttl ?? 5000, // 5 second default - maxSize: config.maxSize ?? 1000 - } - } - - protected async onInitialize(): Promise { - if (this.config.enabled) { - this.log('Request deduplicator initialized for 3x performance boost') - - // Start cleanup interval - this.cleanupInterval = setInterval(() => { - this.cleanup() - }, this.config.ttl) - } else { - this.log('Request deduplicator disabled') - } - } - - shouldExecute(operation: string, params: any): boolean { - // Only execute if enabled and for read operations that benefit from deduplication - return this.config.enabled && ( - operation === 'search' || - operation === 'searchText' || - operation === 'searchByNounTypes' || - operation === 'findSimilar' || - operation === 'get' - ) - } - - async execute( - operation: string, - params: any, - next: () => Promise - ): Promise { - if (!this.config.enabled) { - return next() - } - - // Create a unique key for this request - const key = this.createRequestKey(operation, params) - - // Check if we already have this request pending - const existing = this.pendingRequests.get(key) - if (existing) { - existing.count++ - this.log(`Deduplicating request: ${key} (${existing.count} total)`) - return existing.promise - } - - // Execute the request and cache the promise - const promise = next() - - this.pendingRequests.set(key, { - promise, - timestamp: Date.now(), - count: 1 - }) - - // Clean up when done - promise.finally(() => { - // Use setTimeout to allow other concurrent requests to use the result - setTimeout(() => { - this.pendingRequests.delete(key) - }, 100) - }) - - return promise - } - - /** - * Create a unique key for the request based on operation and parameters - */ - private createRequestKey(operation: string, params: any): string { - // Create a stable string representation of the operation and params - const paramsKey = this.serializeParams(params) - return `${operation}:${paramsKey}` - } - - /** - * Serialize parameters to a consistent string - */ - private serializeParams(params: any): string { - if (!params) return 'null' - - if (typeof params === 'string' || typeof params === 'number') { - return String(params) - } - - if (Array.isArray(params)) { - // For arrays, create a hash-like representation - if (params.length > 100) { - // For large arrays (like vectors), use length + first/last elements - return `[${params.length}:${params[0]}...${params[params.length - 1]}]` - } - return `[${params.join(',')}]` - } - - if (typeof params === 'object') { - // Sort keys for consistent serialization - const keys = Object.keys(params).sort() - const keyValues = keys.map(key => `${key}:${this.serializeParams(params[key])}`) - return `{${keyValues.join(',')}}` - } - - return String(params) - } - - /** - * Clean up expired requests - */ - private cleanup(): void { - const now = Date.now() - const expired = [] - - for (const [key, request] of this.pendingRequests) { - if (now - request.timestamp > this.config.ttl) { - expired.push(key) - } - } - - for (const key of expired) { - this.pendingRequests.delete(key) - } - - // Also enforce max size - if (this.pendingRequests.size > this.config.maxSize) { - const entries = Array.from(this.pendingRequests.entries()) - .sort(([,a], [,b]) => a.timestamp - b.timestamp) // Oldest first - - // Remove oldest entries - const toRemove = entries.slice(0, entries.length - this.config.maxSize) - for (const [key] of toRemove) { - this.pendingRequests.delete(key) - } - } - - if (expired.length > 0) { - this.log(`Cleaned up ${expired.length} expired requests`) - } - } - - /** - * Get statistics about request deduplication - */ - getStats(): { - activePendingRequests: number - totalDeduplicationHits: number - memoryUsage: string - efficiency: string - } { - const requests = Array.from(this.pendingRequests.values()) - const totalRequests = requests.reduce((sum, req) => sum + req.count, 0) - const actualRequests = requests.length - const savedRequests = totalRequests - actualRequests - - return { - activePendingRequests: actualRequests, - totalDeduplicationHits: savedRequests, - memoryUsage: `${Math.round(JSON.stringify(requests).length / 1024)}KB`, - efficiency: actualRequests > 0 ? `${Math.round((savedRequests / totalRequests) * 100)}% reduction` : '0%' - } - } - - /** - * Force clear all pending requests (for testing) - */ - clear(): void { - this.pendingRequests.clear() - } - - protected async onShutdown(): Promise { - if (this.cleanupInterval) { - clearInterval(this.cleanupInterval) - } - - const stats = this.getStats() - this.log(`Request deduplicator shutdown: ${stats.efficiency} efficiency achieved`) - this.pendingRequests.clear() - } -} \ No newline at end of file diff --git a/src/augmentations/storageAugmentation.ts b/src/augmentations/storageAugmentation.ts deleted file mode 100644 index daa4c695..00000000 --- a/src/augmentations/storageAugmentation.ts +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Storage Augmentation Base Classes - * - * Unifies storage adapters and augmentations into a single system. - * All storage backends are now augmentations for consistency and extensibility. - */ - -import { BaseAugmentation, BrainyAugmentation, AugmentationContext } from './brainyAugmentation.js' -import { StorageAdapter } from '../coreTypes.js' - -/** - * Base class for all storage augmentations - * Provides the storage adapter to the brain during initialization - */ -export abstract class StorageAugmentation extends BaseAugmentation implements BrainyAugmentation { - readonly timing = 'replace' as const - readonly metadata = 'none' as const // Storage doesn't directly access metadata - operations = ['storage'] as ('storage')[] // Make mutable for TypeScript compatibility - readonly priority = 100 // High priority for storage - - protected storageAdapter: StorageAdapter | null = null - - // Storage augmentations must provide their name via readonly property - constructor(config?: any) { - super(config) - } - - /** - * Provide the storage adapter before full initialization - * This is called during the storage resolution phase - */ - abstract provideStorage(): Promise - - /** - * Initialize the augmentation with context - * Called after storage has been resolved - */ - async initialize(context: AugmentationContext): Promise { - await super.initialize(context) - // Storage adapter should already be provided - if (!this.storageAdapter) { - this.storageAdapter = await this.provideStorage() - } - } - - /** - * Execute storage operations - * For storage augmentations, this replaces the default storage - */ - async execute( - operation: string, - params: any, - next: () => Promise - ): Promise { - if (operation === 'storage') { - // Return our storage adapter - return this.storageAdapter as any as T - } - - // Pass through all other operations - return next() - } - - /** - * Shutdown and cleanup - */ - async shutdown(): Promise { - // Cleanup storage adapter if needed - if (this.storageAdapter && typeof (this.storageAdapter as any).close === 'function') { - await (this.storageAdapter as any).close() - } - await super.shutdown() - } -} - -/** - * Dynamic storage augmentation that wraps any storage adapter - * Used for backward compatibility and zero-config - */ -export class DynamicStorageAugmentation extends StorageAugmentation { - readonly name = 'dynamic-storage' - - constructor(private adapter: StorageAdapter) { - super() - this.storageAdapter = adapter - } - - async provideStorage(): Promise { - return this.adapter - } - - protected async onInitialize(): Promise { - // Adapter is already provided in constructor - await this.adapter.init() - this.log(`${this.name} initialized`) - } -} - -/** - * Create a storage augmentation from configuration - * Maintains backward compatibility with existing storage config - */ -export async function createStorageAugmentationFromConfig( - config: any -): Promise { - // Import storage factory dynamically to avoid circular deps - const { createStorage } = await import('../storage/storageFactory.js') - - try { - // Create storage adapter from config - const adapter = await createStorage(config) - - // Wrap in augmentation - return new DynamicStorageAugmentation(adapter) - } catch (error) { - console.warn('Failed to create storage augmentation from config:', error) - return null - } -} \ No newline at end of file diff --git a/src/augmentations/storageAugmentations.ts b/src/augmentations/storageAugmentations.ts deleted file mode 100644 index 4337728a..00000000 --- a/src/augmentations/storageAugmentations.ts +++ /dev/null @@ -1,496 +0,0 @@ -/** - * Storage Augmentations - Concrete Implementations - * - * These augmentations provide different storage backends for Brainy. - * Each wraps an existing storage adapter for backward compatibility. - */ - -import { StorageAugmentation } from './storageAugmentation.js' -import { StorageAdapter } from '../coreTypes.js' -import { AugmentationManifest } from './manifest.js' -import { MemoryStorage } from '../storage/adapters/memoryStorage.js' -import { OPFSStorage } from '../storage/adapters/opfsStorage.js' -import { S3CompatibleStorage } from '../storage/adapters/s3CompatibleStorage.js' -import { R2Storage } from '../storage/adapters/r2Storage.js' -import { AzureBlobStorage } from '../storage/adapters/azureBlobStorage.js' - -/** - * Memory Storage Augmentation - Fast in-memory storage - */ -export class MemoryStorageAugmentation extends StorageAugmentation { - readonly name = 'memory-storage' - readonly category = 'core' as const - readonly description = 'High-performance in-memory storage for development and testing' - - constructor(config?: any) { - super(config) - } - - getManifest(): AugmentationManifest { - return { - id: 'memory-storage', - name: 'Memory Storage', - version: '2.0.0', - description: 'Fast in-memory storage with no persistence', - longDescription: 'Perfect for development, testing, and temporary data. All data is lost when the process ends. Provides the fastest possible performance with zero I/O overhead.', - category: 'storage', - configSchema: { - type: 'object', - properties: { - maxSize: { - type: 'number', - default: 104857600, // 100MB - minimum: 1048576, // 1MB - maximum: 1073741824, // 1GB - description: 'Maximum memory usage in bytes' - }, - gcInterval: { - type: 'number', - default: 60000, // 1 minute - minimum: 1000, // 1 second - maximum: 3600000, // 1 hour - description: 'Garbage collection interval in milliseconds' - }, - enableStats: { - type: 'boolean', - default: false, - description: 'Enable memory usage statistics' - } - }, - additionalProperties: false - }, - configDefaults: { - maxSize: 104857600, - gcInterval: 60000, - enableStats: false - }, - minBrainyVersion: '2.0.0', - keywords: ['storage', 'memory', 'ram', 'volatile', 'fast'], - documentation: 'https://docs.brainy.dev/augmentations/memory-storage', - status: 'stable', - performance: { - memoryUsage: 'high', - cpuUsage: 'low', - networkUsage: 'none' - }, - features: ['fast-access', 'zero-latency', 'no-persistence'], - ui: { - icon: '๐Ÿ’พ', - color: '#4CAF50' - } - } - } - - async provideStorage(): Promise { - const storage = new MemoryStorage() - this.storageAdapter = storage - return storage - } - - protected async onInitialize(): Promise { - await this.storageAdapter!.init() - this.log(`Memory storage initialized (max size: ${this.config.maxSize} bytes)`) - } -} - -/** - * FileSystem Storage Augmentation - Node.js persistent storage - */ -export class FileSystemStorageAugmentation extends StorageAugmentation { - readonly name = 'filesystem-storage' - readonly category = 'core' as const - readonly description = 'Persistent file-based storage for Node.js environments' - - constructor(config?: any) { - super(config) - } - - getManifest(): AugmentationManifest { - return { - id: 'filesystem-storage', - name: 'FileSystem Storage', - version: '2.0.0', - description: 'Persistent storage using the local filesystem', - longDescription: 'Stores data as files on the local filesystem. Perfect for Node.js applications, desktop apps, and servers. Provides persistent storage with good performance and unlimited capacity.', - category: 'storage', - configSchema: { - type: 'object', - properties: { - rootDirectory: { - type: 'string', - default: './brainy-data', - description: 'Root directory for storing data files' - }, - compression: { - type: 'boolean', - default: false, - description: 'Enable gzip compression for stored files' - }, - maxFileSize: { - type: 'number', - default: 10485760, // 10MB - minimum: 1048576, // 1MB - maximum: 104857600, // 100MB - description: 'Maximum size per file in bytes' - }, - autoBackup: { - type: 'boolean', - default: false, - description: 'Enable automatic backups' - }, - backupInterval: { - type: 'number', - default: 3600000, // 1 hour - minimum: 60000, // 1 minute - maximum: 86400000, // 24 hours - description: 'Backup interval in milliseconds' - } - }, - additionalProperties: false - }, - configDefaults: { - rootDirectory: './brainy-data', - compression: false, - maxFileSize: 10485760, - autoBackup: false, - backupInterval: 3600000 - }, - minBrainyVersion: '2.0.0', - keywords: ['storage', 'filesystem', 'persistent', 'node', 'disk'], - documentation: 'https://docs.brainy.dev/augmentations/filesystem-storage', - status: 'stable', - performance: { - memoryUsage: 'low', - cpuUsage: 'low', - networkUsage: 'none' - }, - features: ['persistence', 'unlimited-capacity', 'file-based', 'compression-support'], - ui: { - icon: '๐Ÿ“', - color: '#FF9800' - } - } - } - - async provideStorage(): Promise { - try { - // Dynamically import for Node.js environments - const { FileSystemStorage } = await import('../storage/adapters/fileSystemStorage.js') - const storage = new FileSystemStorage(this.config.rootDirectory) - this.storageAdapter = storage - return storage - } catch (error) { - this.log('FileSystemStorage not available, falling back to memory', 'warn') - // Fall back to memory storage - const storage = new MemoryStorage() - this.storageAdapter = storage - return storage - } - } - - protected async onInitialize(): Promise { - await this.storageAdapter!.init() - this.log(`FileSystem storage initialized at ${this.config.rootDirectory}`) - if (this.config.compression) { - this.log('Compression enabled for stored files') - } - } -} - -/** - * OPFS Storage Augmentation - Browser persistent storage - */ -export class OPFSStorageAugmentation extends StorageAugmentation { - readonly name = 'opfs-storage' - readonly category = 'core' as const - readonly description = 'Persistent browser storage using Origin Private File System' - - constructor(config?: any) { - super(config) - } - - getManifest(): AugmentationManifest { - return { - id: 'opfs-storage', - name: 'OPFS Storage', - version: '2.0.0', - description: 'Modern browser storage with file system capabilities', - longDescription: 'Uses the Origin Private File System API for persistent browser storage. Provides file-like storage in modern browsers with better performance than IndexedDB and unlimited storage quota.', - category: 'storage', - configSchema: { - type: 'object', - properties: { - requestPersistent: { - type: 'boolean', - default: false, - description: 'Request persistent storage permission from browser' - }, - directoryName: { - type: 'string', - default: 'brainy-data', - description: 'Directory name within OPFS' - }, - chunkSize: { - type: 'number', - default: 1048576, // 1MB - minimum: 65536, // 64KB - maximum: 10485760, // 10MB - description: 'Chunk size for file operations in bytes' - }, - enableCache: { - type: 'boolean', - default: true, - description: 'Enable in-memory caching for frequently accessed data' - } - }, - additionalProperties: false - }, - configDefaults: { - requestPersistent: false, - directoryName: 'brainy-data', - chunkSize: 1048576, - enableCache: true - }, - minBrainyVersion: '2.0.0', - keywords: ['storage', 'browser', 'opfs', 'persistent', 'web'], - documentation: 'https://docs.brainy.dev/augmentations/opfs-storage', - status: 'stable', - performance: { - memoryUsage: 'medium', - cpuUsage: 'low', - networkUsage: 'none' - }, - features: ['browser-persistent', 'file-system-api', 'unlimited-quota', 'async-operations'], - ui: { - icon: '๐ŸŒ', - color: '#2196F3' - } - } - } - - async provideStorage(): Promise { - const storage = new OPFSStorage() - - if (!storage.isOPFSAvailable()) { - this.log('OPFS not available, falling back to memory', 'warn') - const memStorage = new MemoryStorage() - this.storageAdapter = memStorage - return memStorage - } - - this.storageAdapter = storage - return storage - } - - protected async onInitialize(): Promise { - await this.storageAdapter!.init() - - if (this.config.requestPersistent && this.storageAdapter instanceof OPFSStorage) { - const granted = await this.storageAdapter.requestPersistentStorage() - this.log(`Persistent storage ${granted ? 'granted' : 'denied'}`) - } - - this.log(`OPFS storage initialized in directory: ${this.config.directoryName}`) - } -} - -/** - * S3 Storage Augmentation - Amazon S3 cloud storage - */ -export class S3StorageAugmentation extends StorageAugmentation { - readonly name = 's3-storage' - protected config: { - bucketName: string - region?: string - accessKeyId: string - secretAccessKey: string - sessionToken?: string - cacheConfig?: any - operationConfig?: any - } - - constructor(config: { - bucketName: string - region?: string - accessKeyId: string - secretAccessKey: string - sessionToken?: string - cacheConfig?: any - operationConfig?: any - }) { - super() - this.config = config - } - - async provideStorage(): Promise { - const storage = new S3CompatibleStorage({ - ...this.config, - serviceType: 's3' - }) - this.storageAdapter = storage - return storage - } - - protected async onInitialize(): Promise { - await this.storageAdapter!.init() - this.log(`S3 storage initialized with bucket ${this.config.bucketName}`) - } -} - -/** - * R2 Storage Augmentation - Cloudflare R2 storage - */ -export class R2StorageAugmentation extends StorageAugmentation { - readonly name = 'r2-storage' - protected config: { - bucketName: string - accountId: string - accessKeyId: string - secretAccessKey: string - cacheConfig?: any - } - - constructor(config: { - bucketName: string - accountId: string - accessKeyId: string - secretAccessKey: string - cacheConfig?: any - }) { - super() - this.config = config - } - - async provideStorage(): Promise { - const storage = new R2Storage({ - ...this.config - // serviceType not needed - R2Storage is dedicated - }) - this.storageAdapter = storage - return storage - } - - protected async onInitialize(): Promise { - await this.storageAdapter!.init() - this.log(`R2 storage initialized with bucket ${this.config.bucketName}`) - } -} - -/** - * GCS Storage Augmentation - Google Cloud Storage - */ -export class GCSStorageAugmentation extends StorageAugmentation { - readonly name = 'gcs-storage' - protected config: { - bucketName: string - region?: string - accessKeyId: string - secretAccessKey: string - endpoint?: string - cacheConfig?: any - } - - constructor(config: { - bucketName: string - region?: string - accessKeyId: string - secretAccessKey: string - endpoint?: string - cacheConfig?: any - }) { - super() - this.config = config - } - - async provideStorage(): Promise { - const storage = new S3CompatibleStorage({ - ...this.config, - endpoint: this.config.endpoint || 'https://storage.googleapis.com', - serviceType: 'gcs' - }) - this.storageAdapter = storage - return storage - } - - protected async onInitialize(): Promise { - await this.storageAdapter!.init() - this.log(`GCS storage initialized with bucket ${this.config.bucketName}`) - } -} - -/** - * Azure Blob Storage Augmentation - Microsoft Azure - */ -export class AzureStorageAugmentation extends StorageAugmentation { - readonly name = 'azure-storage' - protected config: { - containerName: string - accountName?: string - accountKey?: string - connectionString?: string - sasToken?: string - cacheConfig?: any - } - - constructor(config: { - containerName: string - accountName?: string - accountKey?: string - connectionString?: string - sasToken?: string - cacheConfig?: any - }) { - super() - this.config = config - } - - async provideStorage(): Promise { - const storage = new AzureBlobStorage({ - ...this.config - }) - this.storageAdapter = storage - return storage - } - - protected async onInitialize(): Promise { - await this.storageAdapter!.init() - this.log(`Azure Blob Storage initialized with container ${this.config.containerName}`) - } -} - -/** - * Auto-select the best storage augmentation for the environment - * Maintains zero-config philosophy - */ -export async function createAutoStorageAugmentation(options: { - rootDirectory?: string - requestPersistentStorage?: boolean -} = {}): Promise { - // Detect environment - const isNodeEnv = (globalThis as any).__ENV__?.isNode || ( - typeof process !== 'undefined' && - process.versions != null && - process.versions.node != null - ) - - if (isNodeEnv) { - // Node.js environment - use FileSystem - return new FileSystemStorageAugmentation({ - rootDirectory: options.rootDirectory || './brainy-data' - }) - } else { - // Browser environment - try OPFS, fall back to memory - const opfsAug = new OPFSStorageAugmentation({ - requestPersistent: options.requestPersistentStorage || false - }) - - // Test if OPFS is available - const testStorage = new OPFSStorage() - if (testStorage.isOPFSAvailable()) { - return opfsAug - } else { - // Fall back to memory - return new MemoryStorageAugmentation() - } - } -} \ No newline at end of file diff --git a/src/augmentations/synapseAugmentation.ts b/src/augmentations/synapseAugmentation.ts deleted file mode 100644 index a2606899..00000000 --- a/src/augmentations/synapseAugmentation.ts +++ /dev/null @@ -1,467 +0,0 @@ -/** - * Base Synapse Augmentation - * - * Synapses are special augmentations that provide bidirectional data sync - * with external platforms (Notion, Salesforce, Slack, etc.) - * - * Like biological synapses that transmit signals between neurons, these - * connect Brainy to external data sources, enabling seamless information flow. - * - * They are managed through the Brain Cloud augmentation registry alongside - * other augmentations, enabling unified discovery, installation, and updates. - * - * Example synapses: - * - NotionSynapse: Sync pages, databases, and blocks - * - SalesforceSynapse: Sync contacts, leads, opportunities - * - SlackSynapse: Sync messages, channels, users - * - GoogleDriveSynapse: Sync documents, sheets, presentations - */ - -import { - AugmentationResponse -} from '../types/augmentations.js' -import { BaseAugmentation, BrainyAugmentation, AugmentationContext } from './brainyAugmentation.js' -import { NeuralImportAugmentation } from './neuralImport.js' - -/** - * Base class for all synapse augmentations - * Provides common functionality for external data synchronization - */ -export abstract class SynapseAugmentation extends BaseAugmentation { - // BrainyAugmentation properties - readonly timing = 'after' as const - readonly operations = ['all'] as ('all')[] - readonly priority = 10 - readonly metadata = { - reads: '*' as '*', // Needs to read for syncing - writes: ['_synapse', '_syncedAt'] as string[] - } // Adds synapse tracking metadata - - // Synapse-specific properties - abstract readonly synapseId: string - abstract readonly supportedTypes: string[] - - // State management - protected syncInProgress = false - protected lastSyncId?: string - protected syncStats = { - totalSyncs: 0, - totalItems: 0, - lastSync: undefined as string | undefined - } - - // Neural Import integration - protected neuralImport?: NeuralImportAugmentation - protected useNeuralImport = true // Enable by default - - protected async onInit(): Promise { - - // Initialize Neural Import if available - if (this.useNeuralImport && this.context?.brain) { - try { - // Check if neural import is already loaded - const existingNeuralImport = this.context.brain.augmentations?.get('neural-import') - if (existingNeuralImport) { - this.neuralImport = existingNeuralImport as NeuralImportAugmentation - } else { - // Create a new instance for this synapse - this.neuralImport = new NeuralImportAugmentation() - // NeuralImport will be initialized when the synapse is added to Brainy - // await this.neuralImport.initialize() - } - } catch (error) { - console.warn(`[${this.synapseId}] Neural Import not available, using basic import`) - this.useNeuralImport = false - } - } - - await this.onInitialize() - } - - /** - * Synapse-specific initialization - * Override this in implementations - */ - protected abstract onInitialize(): Promise - - /** - * BrainyAugmentation execute method - * Intercepts operations to sync external data when relevant - */ - async execute( - operation: string, - params: any, - next: () => Promise - ): Promise { - // Execute the main operation first - const result = await next() - - // After certain operations, check if we should sync - if (this.shouldSync(operation, params)) { - // Start async sync in background - this.backgroundSync().catch(error => { - console.error(`[${this.synapseId}] Background sync failed:`, error) - }) - } - - return result - } - - /** - * Determine if sync should be triggered after an operation - */ - protected shouldSync(operation: string, params: any): boolean { - // Override in implementations for specific sync triggers - return false - } - - /** - * Background sync process - */ - protected async backgroundSync(): Promise { - if (this.syncInProgress) return - - this.syncInProgress = true - try { - await this.incrementalSync(this.lastSyncId) - } finally { - this.syncInProgress = false - } - } - - protected async onShutdown(): Promise { - if (this.syncInProgress) { - await this.stopSync() - } - await this.onSynapseShutdown() - } - - protected async onSynapseShutdown(): Promise { - // Override in implementations for cleanup - } - - // getSynapseStatus implemented below with full response - - /** - * ISynapseAugmentation methods - */ - abstract testConnection(): Promise> - - abstract startSync(options?: Record): Promise - }>> - - async stopSync(): Promise { - this.syncInProgress = false - } - - abstract incrementalSync(lastSyncId?: string): Promise> - - abstract previewSync(limit?: number): Promise - totalCount: number - estimatedDuration: number - }>> - - async getSynapseStatus(): Promise> { - const connectionTest = await this.testConnection() - - return { - success: true, - data: { - status: this.syncInProgress ? 'syncing' : - connectionTest.success ? 'connected' : 'disconnected', - lastSync: this.syncStats.lastSync, - totalSyncs: this.syncStats.totalSyncs, - totalItems: this.syncStats.totalItems - } - } - } - - /** - * Helper method to store synced data in Brainy - * Optionally uses Neural Import for intelligent processing - */ - protected async storeInBrainy( - content: string | Record, - metadata: Record, - options: { - useNeuralImport?: boolean - dataType?: string - rawData?: Buffer | string - } = {} - ): Promise { - if (!this.context?.brain) { - throw new Error('Brainy context not initialized') - } - - // Add synapse source metadata - const enrichedMetadata = { - ...metadata, - _synapse: this.synapseId, - _syncedAt: new Date().toISOString() - } - - // Use Neural Import for intelligent processing if available - if (this.neuralImport && (options.useNeuralImport ?? this.useNeuralImport)) { - try { - // Process through Neural Import for entity/relationship detection - const rawData = options.rawData || - (typeof content === 'string' ? content : JSON.stringify(content)) - - const neuralResult = await this.neuralImport.processRawData( - rawData, - options.dataType || 'json', - { - sourceSystem: this.synapseId, - metadata: enrichedMetadata - } - ) - - if (neuralResult.success && neuralResult.data) { - // Store detected nouns (entities) - for (const noun of neuralResult.data.nouns) { - await this.context.brain.add({ - text: noun, - metadata: { - ...enrichedMetadata, - _neuralConfidence: neuralResult.data.confidence, - _neuralInsights: neuralResult.data.insights - } - }) - } - - // Store detected verbs (relationships) - for (const verb of neuralResult.data.verbs) { - // Parse verb format: "source->relation->target" - const parts = verb.split('->') - if (parts.length === 3) { - await this.context.brain.relate( - parts[0], // source - parts[2], // target - parts[1], // verb type - enrichedMetadata - ) - } - } - - // Store original content with neural metadata - if (typeof content === 'string') { - await this.context.brain.add({ - text: content, - metadata: { - ...enrichedMetadata, - category: 'Content', - _neuralProcessed: true, - _neuralConfidence: neuralResult.data.confidence, - _detectedEntities: neuralResult.data.nouns.length, - _detectedRelationships: neuralResult.data.verbs.length - } - }) - } - - return // Successfully processed with Neural Import - } - } catch (error) { - console.warn(`[${this.synapseId}] Neural Import processing failed, falling back to basic import:`, error) - } - } - - // Fallback to basic storage - if (typeof content === 'string') { - await this.context.brain.add({ - text: content, - metadata: { - ...enrichedMetadata, - category: 'Content' - } - }) - } else { - // For structured data, store as JSON - await this.context.brain.add({ - text: JSON.stringify(content), - metadata: { - ...enrichedMetadata, - category: 'Content' - } - }) - } - } - - /** - * Helper method to query existing synced data - */ - protected async queryBrainy( - filter: { connector?: string; [key: string]: any } - ): Promise { - if (!this.context?.brain) { - throw new Error('Brainy context not initialized') - } - - const searchFilter = { - ...filter, - _synapse: this.synapseId - } - - return this.context.brain.find({ - where: searchFilter - }) - } -} - -/** - * Example implementation for reference - * Real synapses would be in Brain Cloud registry - */ -export class ExampleFileSystemSynapse extends SynapseAugmentation { - readonly name = 'example-filesystem-synapse' - readonly description = 'Example synapse for local file system with Neural Import intelligence' - readonly synapseId = 'filesystem' - readonly supportedTypes = ['text', 'markdown', 'json', 'csv'] - - protected async onInitialize(): Promise { - // Initialize file system watcher, etc. - } - - async testConnection(): Promise> { - // Test if we can access the configured directory - return { - success: true, - data: true - } - } - - async startSync(options?: Record): Promise - }>> { - const startTime = Date.now() - - // Example: Read files from a directory and sync to Brainy - // This would normally scan a directory, but here's a conceptual example: - - const exampleFiles = [ - { - path: '/data/notes.md', - content: '# Project Notes\nDiscuss roadmap with team\nReview Q1 metrics', - type: 'markdown' - }, - { - path: '/data/contacts.json', - content: { name: 'John Doe', role: 'Developer', team: 'Engineering' }, - type: 'json' - } - ] - - let synced = 0 - const errors: Array<{ item: string; error: string }> = [] - - for (const file of exampleFiles) { - try { - // Use Neural Import for intelligent processing - await this.storeInBrainy( - file.content, - { - path: file.path, - fileType: file.type, - syncedFrom: 'filesystem' - }, - { - useNeuralImport: true, // Enable AI processing - dataType: file.type - } - ) - synced++ - } catch (error) { - errors.push({ - item: file.path, - error: error instanceof Error ? error.message : 'Unknown error' - }) - } - } - - this.syncStats.totalSyncs++ - this.syncStats.totalItems += synced - this.syncStats.lastSync = new Date().toISOString() - - return { - success: true, - data: { - synced, - failed: errors.length, - skipped: 0, - duration: Date.now() - startTime, - errors: errors.length > 0 ? errors : undefined - } - } - } - - async incrementalSync(lastSyncId?: string): Promise> { - const startTime = Date.now() - - // Example: Check for modified files since last sync - - return { - success: true, - data: { - synced: 0, - failed: 0, - skipped: 0, - duration: Date.now() - startTime, - hasMore: false - } - } - } - - async previewSync(limit: number = 10): Promise - totalCount: number - estimatedDuration: number - }>> { - // Example: List files that would be synced - - return { - success: true, - data: { - items: [], - totalCount: 0, - estimatedDuration: 0 - } - } - } -} \ No newline at end of file diff --git a/src/augmentations/typeMatching/brainyTypes.ts b/src/augmentations/typeMatching/brainyTypes.ts deleted file mode 100644 index ea7d55f9..00000000 --- a/src/augmentations/typeMatching/brainyTypes.ts +++ /dev/null @@ -1,579 +0,0 @@ -/** - * BrainyTypes - Intelligent type detection using semantic embeddings - * - * This module uses our existing TransformerEmbedding and similarity functions - * to intelligently match data to our 42 noun types and 127 verb types. - * - * Features: - * - Semantic similarity matching using embeddings - * - Context-aware type detection - * - Confidence scoring - * - Caching for performance - */ - -import { NounType, VerbType } from '../../types/graphTypes.js' -import { TransformerEmbedding } from '../../utils/embedding.js' -import { cosineDistance } from '../../utils/distance.js' -import { Vector } from '../../coreTypes.js' -import { getNounTypeEmbeddings, getVerbTypeEmbeddings } from '../../neural/embeddedTypeEmbeddings.js' - -/** - * Type descriptions for semantic matching - * These descriptions are used to generate embeddings for each type - */ -const NOUN_TYPE_DESCRIPTIONS: Record = { - // Core Entity Types (7) - [NounType.Person]: 'person human individual user employee customer citizen member author creator agent actor participant', - [NounType.Organization]: 'organization company business corporation institution agency department team group committee board', - [NounType.Location]: 'location place address city country region area zone coordinate position site venue building', - [NounType.Thing]: 'thing object item product device equipment tool instrument asset artifact material physical tangible', - [NounType.Concept]: 'concept idea theory principle philosophy belief value abstract intangible notion thought', - [NounType.Event]: 'event occurrence incident activity happening meeting conference celebration milestone timestamp date', - [NounType.Agent]: 'agent AI bot automated system automation software assistant service daemon daemon worker processor', - - // Biological Types (1) - Stage 3 - [NounType.Organism]: 'organism animal plant bacteria fungi species living biological life creature being microorganism', - - // Material Types (1) - Stage 3 - [NounType.Substance]: 'substance material matter chemical element compound liquid gas solid molecule atom material', - - // Property & Quality Types (1) - Stage 3 - [NounType.Quality]: 'quality property attribute characteristic feature aspect trait dimension parameter value', - - // Temporal Types (1) - Stage 3 - [NounType.TimeInterval]: 'timeInterval period duration span range epoch era season quarter interval window', - - // Functional Types (1) - Stage 3 - [NounType.Function]: 'function purpose role capability capacity utility objective goal aim intent', - - // Informational Types (1) - Stage 3 - [NounType.Proposition]: 'proposition statement claim assertion declaration premise conclusion hypothesis theory', - - // Digital/Content Types - [NounType.Document]: 'document file report article paper text pdf word contract agreement record documentation', - [NounType.Media]: 'media image photo video audio music podcast multimedia graphic visualization animation', - [NounType.File]: 'file digital data binary code script program software archive package bundle', - [NounType.Message]: 'message email chat communication notification alert announcement broadcast transmission', - - // Collection Types - [NounType.Collection]: 'collection group set list array category folder directory catalog inventory database', - [NounType.Dataset]: 'dataset data table spreadsheet database records statistics metrics measurements analysis', - - // Business/Application Types - [NounType.Product]: 'product item merchandise offering service feature application software solution package', - [NounType.Service]: 'service offering subscription support maintenance utility function capability', - [NounType.Task]: 'task action todo item job assignment duty responsibility activity step procedure', - [NounType.Project]: 'project initiative program campaign effort endeavor plan scheme venture undertaking', - - // Descriptive Types - [NounType.Process]: 'process workflow procedure method algorithm sequence pipeline operation routine protocol', - [NounType.State]: 'state status condition phase stage mode situation circumstance configuration setting', - [NounType.Role]: 'role position title function responsibility duty job capacity designation authority', - [NounType.Language]: 'language dialect locale tongue vernacular communication speech linguistics vocabulary', - [NounType.Currency]: 'currency money dollar euro pound yen bitcoin payment financial monetary unit', - [NounType.Measurement]: 'measurement metric quantity value amount size dimension weight height volume distance', - - // Scientific/Research Types - [NounType.Hypothesis]: 'hypothesis theory proposition thesis assumption premise conjecture speculation prediction', - [NounType.Experiment]: 'experiment test trial study research investigation analysis observation examination', - - // Legal/Regulatory Types - [NounType.Contract]: 'contract agreement deal treaty pact covenant license terms conditions policy', - [NounType.Regulation]: 'regulation law rule policy standard compliance requirement guideline ordinance statute', - - // Technical Infrastructure Types - [NounType.Interface]: 'interface API endpoint protocol specification contract schema definition connection', - [NounType.Resource]: 'resource infrastructure server database storage compute memory bandwidth capacity asset', - - // Custom/Extensible (1) - Stage 3 - [NounType.Custom]: 'custom special unique particular specific domain-specific specialized tailored bespoke', - - // Social Structures (3) - Stage 3 - [NounType.SocialGroup]: 'socialGroup community tribe clan network circle collective society crowd gathering', - [NounType.Institution]: 'institution establishment foundation organization system structure framework tradition practice', - [NounType.Norm]: 'norm convention standard custom tradition protocol etiquette rule practice expectation', - - // Information Theory (2) - Stage 3 - [NounType.InformationContent]: 'informationContent story narrative knowledge data schema pattern model structure', - [NounType.InformationBearer]: 'informationBearer carrier medium vehicle signal token representation document', - - // Meta-Level (1) - Stage 3 - [NounType.Relationship]: 'relationship connection relation association link bond tie interaction dynamic' -} - -const VERB_TYPE_DESCRIPTIONS: Record = { - // Core Relationship Types - [VerbType.RelatedTo]: 'related connected associated linked correlated relevant pertinent applicable', - [VerbType.Contains]: 'contains includes holds stores encompasses comprises consists incorporates', - [VerbType.PartOf]: 'part component element member piece portion section segment constituent', - [VerbType.LocatedAt]: 'located situated positioned placed found exists resides occupies', - [VerbType.References]: 'references cites mentions points links refers quotes sources', - - // Temporal/Causal Types - [VerbType.Precedes]: 'precedes before earlier prior previous antecedent preliminary foregoing', - [VerbType.Causes]: 'causes triggers induces produces generates results influences affects', - [VerbType.DependsOn]: 'depends requires needs relies necessitates contingent prerequisite', - [VerbType.Requires]: 'requires needs demands necessitates mandates obliges compels entails', - - // Creation/Transformation Types - [VerbType.Creates]: 'creates makes produces generates builds constructs forms establishes', - [VerbType.Transforms]: 'transforms converts changes modifies alters transitions morphs evolves', - [VerbType.Becomes]: 'becomes turns evolves transforms changes transitions develops grows', - [VerbType.Modifies]: 'modifies changes updates alters edits revises adjusts adapts', - [VerbType.Consumes]: 'consumes uses utilizes depletes expends absorbs takes processes', - - // Ownership/Attribution Types - [VerbType.Owns]: 'owns possesses holds controls manages administers governs maintains', - [VerbType.AttributedTo]: 'attributed credited assigned ascribed authored written composed', - - // Social/Organizational Types - [VerbType.MemberOf]: 'member participant affiliate associate belongs joined enrolled registered', - [VerbType.WorksWith]: 'works collaborates cooperates partners teams assists helps supports', - [VerbType.FriendOf]: 'friend companion buddy pal acquaintance associate connection relationship', - [VerbType.Follows]: 'follows subscribes tracks monitors watches observes trails pursues', - [VerbType.Likes]: 'likes enjoys appreciates favors prefers admires values endorses', - [VerbType.ReportsTo]: 'reports answers subordinate accountable responsible supervised managed', - [VerbType.Mentors]: 'mentors teaches guides coaches instructs trains advises counsels', - [VerbType.Communicates]: 'communicates talks speaks messages contacts interacts corresponds exchanges', - - // Descriptive/Functional Types - [VerbType.Describes]: 'describes explains details documents specifies outlines depicts characterizes', - [VerbType.Defines]: 'defines specifies establishes determines sets declares identifies designates', - [VerbType.Categorizes]: 'categorizes classifies groups sorts organizes arranges labels tags', - [VerbType.Measures]: 'measures quantifies gauges assesses evaluates calculates determines counts', - [VerbType.Evaluates]: 'evaluates assesses analyzes reviews examines appraises judges rates', - [VerbType.Uses]: 'uses utilizes employs applies operates handles manipulates exploits', - [VerbType.Implements]: 'implements executes realizes performs accomplishes carries delivers completes', - [VerbType.Extends]: 'extends expands enhances augments amplifies broadens enlarges develops', - - // Enhanced Relationships - [VerbType.Inherits]: 'inherits derives extends receives obtains acquires succeeds legacy', - [VerbType.Conflicts]: 'conflicts contradicts opposes clashes disputes disagrees incompatible inconsistent', - [VerbType.Synchronizes]: 'synchronizes coordinates aligns harmonizes matches corresponds parallels coincides', - [VerbType.Competes]: 'competes rivals contends contests challenges opposes vies struggles' -} - -/** - * Result of type matching with confidence scores - */ -export interface TypeMatchResult { - type: string - confidence: number - reasoning: string - alternatives: Array<{ - type: string - confidence: number - }> -} - -/** - * BrainyTypes - Intelligent type detection for nouns and verbs - * PRODUCTION OPTIMIZATION: Uses pre-computed type embeddings - * Type embeddings are loaded instantly; only input objects are embedded at runtime - */ -export class BrainyTypes { - private embedder: TransformerEmbedding // Only for embedding input objects - private nounEmbeddings: Map = new Map() - private verbEmbeddings: Map = new Map() - private initialized = false - private cache: Map = new Map() - - constructor() { - // Embedder only used for input objects, NOT for type embeddings - this.embedder = new TransformerEmbedding({ verbose: false }) - } - - /** - * Initialize the type matcher by loading pre-computed embeddings - * INSTANT - type embeddings are loaded from pre-computed data - * Only the model for input embedding needs initialization - */ - async init(): Promise { - if (this.initialized) return - - // Initialize embedder for input objects only - await this.embedder.init() - - // Load pre-computed type embeddings (instant, no computation) - const nounEmbeddings = getNounTypeEmbeddings() - const verbEmbeddings = getVerbTypeEmbeddings() - - // Convert NounType/VerbType keys to strings for lookup - for (const [type, embedding] of nounEmbeddings.entries()) { - this.nounEmbeddings.set(type, embedding) - } - - for (const [type, embedding] of verbEmbeddings.entries()) { - this.verbEmbeddings.set(type, embedding) - } - - this.initialized = true - } - - /** - * Match an object to the most appropriate noun type - */ - async matchNounType(obj: any): Promise { - await this.init() - - // Create a text representation of the object for embedding - const textRepresentation = this.createTextRepresentation(obj) - - // Check cache - const cacheKey = `noun:${textRepresentation}` - if (this.cache.has(cacheKey)) { - return this.cache.get(cacheKey)! - } - - // Generate embedding for the input - const inputEmbedding = await this.embedder.embed(textRepresentation) - - // Calculate similarities to all noun types - const similarities: Array<{ type: string; similarity: number }> = [] - - for (const [type, typeEmbedding] of this.nounEmbeddings.entries()) { - // Convert cosine distance to similarity (1 - distance) - const similarity = 1 - cosineDistance(inputEmbedding, typeEmbedding) - similarities.push({ type, similarity }) - } - - // Sort by similarity (highest first) - similarities.sort((a, b) => b.similarity - a.similarity) - - // Apply heuristic rules for common patterns - const heuristicType = this.applyNounHeuristics(obj) - if (heuristicType) { - // Boost the heuristic type's confidence - const heuristicIndex = similarities.findIndex(s => s.type === heuristicType) - if (heuristicIndex > 0) { - similarities[heuristicIndex].similarity *= 1.2 // 20% boost - similarities.sort((a, b) => b.similarity - a.similarity) - } - } - - // Create result - const result: TypeMatchResult = { - type: similarities[0].type, - confidence: similarities[0].similarity, - reasoning: this.generateReasoning(obj, similarities[0].type, 'noun'), - alternatives: similarities.slice(1, 4).map(s => ({ - type: s.type, - confidence: s.similarity - })) - } - - // Cache result - this.cache.set(cacheKey, result) - - return result - } - - /** - * Match a relationship to the most appropriate verb type - */ - async matchVerbType( - sourceObj: any, - targetObj: any, - relationshipHint?: string - ): Promise { - await this.init() - - // Create text representation of the relationship - const textRepresentation = this.createRelationshipText(sourceObj, targetObj, relationshipHint) - - // Check cache - const cacheKey = `verb:${textRepresentation}` - if (this.cache.has(cacheKey)) { - return this.cache.get(cacheKey)! - } - - // Generate embedding - const inputEmbedding = await this.embedder.embed(textRepresentation) - - // Calculate similarities to all verb types - const similarities: Array<{ type: string; similarity: number }> = [] - - for (const [type, typeEmbedding] of this.verbEmbeddings.entries()) { - const similarity = 1 - cosineDistance(inputEmbedding, typeEmbedding) - similarities.push({ type, similarity }) - } - - // Sort by similarity - similarities.sort((a, b) => b.similarity - a.similarity) - - // Apply heuristic rules - const heuristicType = this.applyVerbHeuristics(sourceObj, targetObj, relationshipHint) - if (heuristicType) { - const heuristicIndex = similarities.findIndex(s => s.type === heuristicType) - if (heuristicIndex > 0) { - similarities[heuristicIndex].similarity *= 1.2 - similarities.sort((a, b) => b.similarity - a.similarity) - } - } - - // Create result - const result: TypeMatchResult = { - type: similarities[0].type, - confidence: similarities[0].similarity, - reasoning: this.generateReasoning( - { source: sourceObj, target: targetObj, hint: relationshipHint }, - similarities[0].type, - 'verb' - ), - alternatives: similarities.slice(1, 4).map(s => ({ - type: s.type, - confidence: s.similarity - })) - } - - // Cache result - this.cache.set(cacheKey, result) - - return result - } - - /** - * Create text representation of an object for embedding - */ - private createTextRepresentation(obj: any): string { - const parts: string[] = [] - - // Add type if available - if (typeof obj === 'object' && obj !== null) { - // Add field names and values - for (const [key, value] of Object.entries(obj)) { - parts.push(key) - if (typeof value === 'string') { - parts.push(value.slice(0, 100)) // Limit string length - } else if (typeof value === 'number' || typeof value === 'boolean') { - parts.push(String(value)) - } - } - - // Add special fields with higher weight - const importantFields = ['type', 'kind', 'category', 'class', 'name', 'title', 'description'] - for (const field of importantFields) { - if (obj[field]) { - parts.push(String(obj[field])) - parts.push(String(obj[field])) // Double weight for important fields - } - } - } else if (typeof obj === 'string') { - parts.push(obj) - } else { - parts.push(String(obj)) - } - - return parts.join(' ') - } - - /** - * Create text representation of a relationship - */ - private createRelationshipText( - sourceObj: any, - targetObj: any, - relationshipHint?: string - ): string { - const parts: string[] = [] - - if (relationshipHint) { - parts.push(relationshipHint) - parts.push(relationshipHint) // Double weight for explicit hint - } - - // Add source context - if (sourceObj) { - parts.push('source:') - parts.push(this.getObjectSummary(sourceObj)) - } - - // Add target context - if (targetObj) { - parts.push('target:') - parts.push(this.getObjectSummary(targetObj)) - } - - return parts.join(' ') - } - - /** - * Get a brief summary of an object - */ - private getObjectSummary(obj: any): string { - if (typeof obj === 'string') return obj.slice(0, 50) - if (typeof obj !== 'object' || obj === null) return String(obj) - - const summary: string[] = [] - const fields = ['type', 'name', 'title', 'id', 'category', 'kind'] - - for (const field of fields) { - if (obj[field]) { - summary.push(String(obj[field])) - } - } - - return summary.join(' ').slice(0, 100) - } - - /** - * Apply heuristic rules for noun type detection - */ - private applyNounHeuristics(obj: any): string | null { - if (typeof obj !== 'object' || obj === null) return null - - // Person heuristics - if (obj.email || obj.firstName || obj.lastName || obj.username || obj.age || obj.gender) { - return NounType.Person - } - - // Organization heuristics - if (obj.companyName || obj.organizationId || obj.employees || obj.industry) { - return NounType.Organization - } - - // Location heuristics - if (obj.latitude || obj.longitude || obj.address || obj.city || obj.country || obj.coordinates) { - return NounType.Location - } - - // Document heuristics - if (obj.content && (obj.title || obj.author) || obj.documentType || obj.pages) { - return NounType.Document - } - - // Event heuristics - if (obj.startTime || obj.endTime || obj.date || obj.eventType || obj.attendees) { - return NounType.Event - } - - // Product heuristics - if (obj.price || obj.sku || obj.inventory || obj.productId) { - return NounType.Product - } - - // Task heuristics - if (obj.status && (obj.assignee || obj.dueDate) || obj.priority || obj.completed !== undefined) { - return NounType.Task - } - - // Media heuristics - if (obj.url && (obj.url.match(/\.(jpg|jpeg|png|gif|mp4|mp3|wav)/i))) { - return NounType.Media - } - - // Dataset heuristics - if (Array.isArray(obj.data) || obj.rows || obj.columns || obj.schema) { - return NounType.Dataset - } - - return null - } - - /** - * Apply heuristic rules for verb type detection - */ - private applyVerbHeuristics( - sourceObj: any, - targetObj: any, - relationshipHint?: string - ): string | null { - if (!relationshipHint) return null - - const hint = relationshipHint.toLowerCase() - - // Ownership patterns - if (hint.includes('own') || hint.includes('possess') || hint.includes('has')) { - return VerbType.Owns - } - - // Creation patterns - if (hint.includes('create') || hint.includes('made') || hint.includes('authored')) { - return VerbType.Creates - } - - // Containment patterns - if (hint.includes('contain') || hint.includes('include') || hint.includes('has')) { - return VerbType.Contains - } - - // Membership patterns - if (hint.includes('member') || hint.includes('belong') || hint.includes('part')) { - return VerbType.MemberOf - } - - // Reference patterns - if (hint.includes('refer') || hint.includes('cite') || hint.includes('link')) { - return VerbType.References - } - - // Dependency patterns - if (hint.includes('depend') || hint.includes('require') || hint.includes('need')) { - return VerbType.DependsOn - } - - return null - } - - /** - * Generate human-readable reasoning for the type selection - */ - private generateReasoning( - obj: any, - selectedType: string, - typeKind: 'noun' | 'verb' - ): string { - const descriptions = typeKind === 'noun' ? NOUN_TYPE_DESCRIPTIONS : VERB_TYPE_DESCRIPTIONS - const typeDesc = descriptions[selectedType] - - // Handle missing descriptions gracefully - if (!typeDesc) { - if (typeKind === 'noun') { - const fields = Object.keys(obj).slice(0, 3).join(', ') - return `Matched to ${selectedType} based on semantic similarity and object fields: ${fields}` - } else { - return `Matched to ${selectedType} based on semantic similarity and relationship context` - } - } - - if (typeKind === 'noun') { - const fields = Object.keys(obj).slice(0, 3).join(', ') - return `Matched to ${selectedType} based on semantic similarity to "${typeDesc.split(' ').slice(0, 5).join(' ')}..." and object fields: ${fields}` - } else { - return `Matched to ${selectedType} based on semantic similarity to "${typeDesc.split(' ').slice(0, 5).join(' ')}..." and relationship context` - } - } - - /** - * Clear the cache - */ - clearCache(): void { - this.cache.clear() - } - - /** - * Dispose of resources - */ - async dispose(): Promise { - await this.embedder.dispose() - this.cache.clear() - this.nounEmbeddings.clear() - this.verbEmbeddings.clear() - } -} - -/** - * Singleton instance for efficient reuse - */ -let globalInstance: BrainyTypes | null = null - -/** - * Get or create the global BrainyTypes instance - */ -export async function getBrainyTypes(): Promise { - if (!globalInstance) { - globalInstance = new BrainyTypes() - await globalInstance.init() - } - return globalInstance -} \ No newline at end of file diff --git a/src/augmentations/typeMatching/intelligentTypeMatcher.ts b/src/augmentations/typeMatching/intelligentTypeMatcher.ts deleted file mode 100644 index 3b52b4a2..00000000 --- a/src/augmentations/typeMatching/intelligentTypeMatcher.ts +++ /dev/null @@ -1,180 +0,0 @@ -/** - * IntelligentTypeMatcher - Wrapper around BrainyTypes for testing - * - * Provides intelligent type detection using semantic embeddings - * for matching data to our 42 noun types and 127 verb types. - */ - -import { NounType, VerbType } from '../../types/graphTypes.js' -import { BrainyTypes, TypeMatchResult, getBrainyTypes } from './brainyTypes.js' - -export interface TypeMatchOptions { - threshold?: number - topK?: number - useCache?: boolean -} - -/** - * Intelligent type matcher using semantic embeddings - */ -export class IntelligentTypeMatcher { - private brainyTypes: BrainyTypes | null = null - private cache = new Map() - - constructor(private options: TypeMatchOptions = {}) { - this.options = { - threshold: 0.3, - topK: 3, - useCache: true, - ...options - } - } - - /** - * Initialize the type matcher - */ - async init(): Promise { - this.brainyTypes = await getBrainyTypes() - await this.brainyTypes.init() - } - - /** - * Dispose of resources - */ - async dispose(): Promise { - if (this.brainyTypes) { - await this.brainyTypes.dispose() - this.brainyTypes = null - } - this.cache.clear() - } - - /** - * Match data to a noun type - */ - async matchNounType(data: any): Promise<{ - type: NounType - confidence: number - alternatives: Array<{ type: NounType; confidence: number }> - }> { - if (!this.brainyTypes) { - throw new Error('IntelligentTypeMatcher not initialized. Call init() first.') - } - - // Check cache if enabled - const cacheKey = JSON.stringify(data) - if (this.options.useCache && this.cache.has(cacheKey)) { - const cached = this.cache.get(cacheKey)! - return { - type: cached.type as NounType, - confidence: cached.confidence, - alternatives: cached.alternatives?.map(alt => ({ - type: alt.type as NounType, - confidence: alt.confidence - })) || [] - } - } - - // Detect type using BrainyTypes - const result = await this.brainyTypes.matchNounType(data) - - // Convert to expected format - const response = { - type: result.type as NounType, - confidence: result.confidence, - alternatives: result.alternatives?.map(alt => ({ - type: alt.type as NounType, - confidence: alt.confidence - })) || [] - } - - // Cache the result if enabled - if (this.options.useCache) { - this.cache.set(cacheKey, result) - } - - return response - } - - /** - * Match a relationship to a verb type - */ - async matchVerbType( - source: any, - target: any, - relationship?: string - ): Promise<{ - type: VerbType - confidence: number - alternatives: Array<{ type: VerbType; confidence: number }> - }> { - if (!this.brainyTypes) { - throw new Error('IntelligentTypeMatcher not initialized. Call init() first.') - } - - // Create context for verb detection - const context = { - source, - target, - relationship: relationship || 'related', - description: relationship || '' - } - - // Check cache if enabled - const cacheKey = JSON.stringify(context) - if (this.options.useCache && this.cache.has(cacheKey)) { - const cached = this.cache.get(cacheKey)! - return { - type: cached.type as VerbType || VerbType.RelatedTo, - confidence: cached.confidence, - alternatives: cached.alternatives?.map(alt => ({ - type: alt.type as VerbType || VerbType.RelatedTo, - confidence: alt.confidence - })) || [] - } - } - - // Detect verb type using BrainyTypes - const result = await this.brainyTypes.matchVerbType( - context.source, - context.target, - context.relationship - ) - - // Convert to expected format - const response = { - type: result.type as VerbType || VerbType.RelatedTo, - confidence: result.confidence, - alternatives: result.alternatives?.map(alt => ({ - type: alt.type as VerbType || VerbType.RelatedTo, - confidence: alt.confidence - })) || [] - } - - // Cache the result if enabled - if (this.options.useCache) { - this.cache.set(cacheKey, result) - } - - return response - } - - /** - * Clear the cache - */ - clearCache(): void { - this.cache.clear() - } - - /** - * Get cache statistics - */ - getCacheStats(): { size: number; maxSize: number } { - return { - size: this.cache.size, - maxSize: 1000 // Default max cache size - } - } -} - -export default IntelligentTypeMatcher \ No newline at end of file diff --git a/src/augmentations/universalDisplayAugmentation.ts b/src/augmentations/universalDisplayAugmentation.ts deleted file mode 100644 index b5ddbd29..00000000 --- a/src/augmentations/universalDisplayAugmentation.ts +++ /dev/null @@ -1,449 +0,0 @@ -/** - * Universal Display Augmentation - * - * ๐ŸŽจ Provides intelligent display fields for any noun or verb using AI-powered analysis - * - * Features: - * - โœ… Leverages existing BrainyTypes for semantic type detection - * - โœ… Complete icon coverage for all 42 NounTypes + 127 VerbTypes - * - โœ… Zero performance impact with lazy computation and intelligent caching - * - โœ… Perfect isolation - can be disabled, replaced, or configured - * - โœ… Clean developer experience with zero conflicts - * - โœ… TypeScript support with full autocomplete - * - * Usage: - * ```typescript - * // User data access (unchanged) - * result.firstName // "John" - * result.metadata.title // "CEO" - * - * // Enhanced display (new capabilities) - * result.getDisplay('title') // "John Doe" (AI-computed) - * result.getDisplay('description') // "CEO at Acme Corp" (enhanced) - * result.getDisplay('type') // "Person" (from AI detection) - * result.getDisplay() // All display fields - * ``` - */ - -import { BaseAugmentation, AugmentationContext, MetadataAccess } from './brainyAugmentation.js' -import type { VectorDocument, GraphVerb } from '../coreTypes.js' -import type { - DisplayConfig, - ComputedDisplayFields, - EnhancedVectorDocument, - EnhancedGraphVerb, - DisplayAugmentationStats -} from './display/types.js' -import { IntelligentComputationEngine } from './display/intelligentComputation.js' -import { DisplayCache, RequestDeduplicator, getGlobalDisplayCache } from './display/cache.js' -import { getNounIcon, getVerbIcon, getIconCoverage } from './display/iconMappings.js' - -/** - * Universal Display Augmentation - * - * Self-contained augmentation that provides intelligent display fields - * for any data type using existing Brainy AI infrastructure - */ -export class UniversalDisplayAugmentation extends BaseAugmentation { - readonly name = 'display' - readonly version = '1.0.0' - readonly timing = 'after' as const // Enhance results after main operations - readonly priority = 50 // Medium priority - after core operations - readonly metadata: MetadataAccess = { - reads: '*', // Read all user data for intelligent analysis - writes: ['_display'] // Cache computed fields in isolated namespace - } - operations = ['get', 'search', 'find', 'similar', 'findSimilar', 'getVerb' as any, 'add', 'addNoun', 'addVerb', 'relate'] as any - - // Augmentation metadata - readonly category = 'core' as const - readonly description = 'AI-powered intelligent display fields for enhanced data visualization' - - // Computed fields declaration for TypeScript support and discovery - computedFields = { - display: { - title: { type: 'string' as const, description: 'Primary display name (AI-computed)' }, - description: { type: 'string' as const, description: 'Enhanced description with context' }, - type: { type: 'string' as const, description: 'Human-readable type (from AI detection)' }, - tags: { type: 'array' as const, description: 'Generated display tags' }, - relationship: { type: 'string' as const, description: 'Human-readable relationship (verbs only)' }, - confidence: { type: 'number' as const, description: 'AI confidence score (0-1)' } - } - } - - // Core components (all self-contained) - private computationEngine: IntelligentComputationEngine - private displayCache: DisplayCache - private requestDeduplicator: RequestDeduplicator - protected config: DisplayConfig - protected context: AugmentationContext | undefined - - constructor(config: Partial = {}) { - super() - - // Merge with defaults - this.config = { - enabled: true, - cacheSize: 1000, - lazyComputation: true, - batchSize: 50, - confidenceThreshold: 0.7, - customFieldMappings: {}, - priorityFields: {}, - debugMode: false, - ...config - } - - // Initialize components - this.computationEngine = new IntelligentComputationEngine(this.config) - this.displayCache = getGlobalDisplayCache(this.config.cacheSize) - this.requestDeduplicator = new RequestDeduplicator(this.config.batchSize) - } - - /** - * Initialize the augmentation with AI components - * @param context Brainy context - */ - async initialize(context: AugmentationContext): Promise { - if (!this.config.enabled) { - this.log('๐ŸŽจ Universal Display augmentation disabled') - return - } - - this.context = context - - try { - // Initialize AI-powered computation engine - await this.computationEngine.initialize() - - this.log('๐ŸŽจ Universal Display augmentation initialized successfully') - this.log(` Cache size: ${this.config.cacheSize}`) - this.log(` Lazy computation: ${this.config.lazyComputation}`) - this.log(` Coverage: ${this.getCoverageInfo()}`) - - } catch (error) { - this.log('โš ๏ธ Display augmentation initialization warning:', 'warn') - this.log(` ${error}`, 'warn') - this.log(' Falling back to basic mode', 'warn') - } - } - - /** - * Execute augmentation - attach display capabilities to results - * @param operation The operation being performed - * @param params Operation parameters - * @param next Function to execute main operation - * @returns Enhanced result with display capabilities - */ - async execute( - operation: string, - params: any, - next: () => Promise - ): Promise { - // Always execute main operation first - const result = await next() - - // Only enhance if enabled and operation is relevant - if (!this.config.enabled || !this.shouldEnhanceOperation(operation)) { - return result - } - - try { - // Enhance result with display capabilities - return this.enhanceWithDisplayCapabilities(result, operation) as T - } catch (error) { - this.log(`Display enhancement failed for ${operation}: ${error}`, 'warn') - return result // Return unenhanced result on error - } - } - - /** - * Check if operation should be enhanced - * @param operation Operation name - * @returns True if should enhance - */ - private shouldEnhanceOperation(operation: string): boolean { - const enhanceableOps = ['get', 'search', 'findSimilar', 'getVerb'] - return enhanceableOps.includes(operation) - } - - /** - * Enhance result with display capabilities - * @param result The operation result - * @param operation The operation type - * @returns Enhanced result - */ - private enhanceWithDisplayCapabilities(result: any, operation: string): any { - if (!result) return result - - // Handle different result types - if (Array.isArray(result)) { - // Array of results (search, findSimilar) - return result.map(item => this.enhanceEntity(item)) - } else if (result.id || result.metadata) { - // Single entity (get, getVerb) - return this.enhanceEntity(result) - } - - return result - } - - /** - * Enhance a single entity with display capabilities - * @param entity The entity to enhance - * @returns Enhanced entity - */ - private enhanceEntity(entity: any): EnhancedVectorDocument | EnhancedGraphVerb { - if (!entity) return entity - - // Determine if it's a noun or verb - const isVerb = this.isVerbEntity(entity) - - // Add display methods - const enhanced = { - ...entity, - getDisplay: this.createGetDisplayMethod(entity, isVerb), - getAvailableFields: this.createGetAvailableFieldsMethod(), - getAvailableAugmentations: this.createGetAvailableAugmentationsMethod(), - explore: this.createExploreMethod(entity) - } - - return enhanced - } - - /** - * Create getDisplay method for an entity - * @param entity The entity - * @param isVerb Whether it's a verb entity - * @returns getDisplay function - */ - private createGetDisplayMethod(entity: any, isVerb: boolean) { - return async (field?: keyof ComputedDisplayFields): Promise => { - // Generate cache key - const cacheKey = this.displayCache.generateKey( - entity.id, - entity.metadata || entity, - isVerb ? 'verb' : 'noun' - ) - - // Use request deduplicator to prevent duplicate computations - const computedFields = await this.requestDeduplicator.deduplicate( - cacheKey, - async () => { - // Check cache first - let cached = this.displayCache.get(cacheKey) - if (cached) return cached - - // Compute display fields - const startTime = Date.now() - let computed: ComputedDisplayFields - - if (isVerb) { - computed = await this.computationEngine.computeVerbDisplay(entity as GraphVerb) - } else { - computed = await this.computationEngine.computeNounDisplay( - entity.metadata || entity, - entity.id - ) - } - - // Cache the result - const computationTime = Date.now() - startTime - this.displayCache.set(cacheKey, computed, computationTime) - - return computed - } - ) - - // Return specific field or all fields - return field ? computedFields[field] : computedFields - } - } - - /** - * Create getAvailableFields method - * @returns getAvailableFields function - */ - private createGetAvailableFieldsMethod() { - return (namespace: string): string[] => { - if (namespace === 'display') { - return ['title', 'description', 'type', 'tags', 'relationship', 'confidence'] - } - return [] - } - } - - /** - * Create getAvailableAugmentations method - * @returns getAvailableAugmentations function - */ - private createGetAvailableAugmentationsMethod() { - return (): string[] => { - return ['display'] // This augmentation provides 'display' namespace - } - } - - /** - * Create explore method for debugging - * @param entity The entity - * @returns explore function - */ - private createExploreMethod(entity: any) { - return async (): Promise => { - // Respect silent mode - if (this.config.silent) { - return - } - - console.log(`\n๐Ÿ“‹ Entity Exploration: ${entity.id || 'unknown'}`) - console.log('โ”'.repeat(50)) - - // Show user data - console.log('\n๐Ÿ‘ค User Data:') - const userData = entity.metadata || entity - for (const [key, value] of Object.entries(userData)) { - if (!key.startsWith('_')) { - console.log(` โ€ข ${key}: ${JSON.stringify(value)}`) - } - } - - // Show computed display fields - try { - console.log('\n๐ŸŽจ Display Fields:') - const displayMethod = this.createGetDisplayMethod(entity, this.isVerbEntity(entity)) - const displayFields = await displayMethod() - for (const [key, value] of Object.entries(displayFields)) { - console.log(` โ€ข ${key}: ${JSON.stringify(value)}`) - } - } catch (error) { - console.log(` Error computing display fields: ${error}`) - } - - console.log('') - } - } - - /** - * Check if an entity is a verb - * @param entity The entity to check - * @returns True if it's a verb - */ - private isVerbEntity(entity: any): boolean { - return !!(entity.sourceId && entity.targetId) || - !!(entity.source && entity.target) || - !!entity.verb - } - - /** - * Get coverage information - * @returns Coverage info string - */ - private getCoverageInfo(): string { - return 'Clean display - focuses on AI-powered content' - } - - /** - * Get augmentation statistics - * @returns Performance and usage statistics - */ - getStats(): DisplayAugmentationStats { - return this.displayCache.getStats() - } - - /** - * Configure the augmentation at runtime - * @param newConfig Partial configuration to merge - */ - configure(newConfig: Partial): void { - this.config = { ...this.config, ...newConfig } - - if (!this.config.enabled) { - this.displayCache.clear() - } - } - - /** - * Clear all cached display data - */ - clearCache(): void { - this.displayCache.clear() - } - - /** - * Precompute display fields for a batch of entities - * @param entities Array of entities to precompute - */ - async precomputeBatch(entities: Array<{ id: string; data: any }>): Promise { - const computeRequests = entities.map(({ id, data }) => ({ - key: this.displayCache.generateKey(id, data, 'noun'), - computeFn: () => this.computationEngine.computeNounDisplay(data, id) - })) - - await this.displayCache.batchPrecompute(computeRequests) - } - - /** - * Optional check if this augmentation should run - * @param operation Operation name - * @param params Operation parameters - * @returns True if should execute - */ - shouldExecute(operation: string, params: any): boolean { - return this.config.enabled && this.shouldEnhanceOperation(operation) - } - - /** - * Cleanup when augmentation is shut down - */ - async shutdown(): Promise { - try { - // Cleanup computation engine - await this.computationEngine.shutdown() - - // Cleanup request deduplicator - this.requestDeduplicator.shutdown() - - // Clear cache if configured to do so - if (this.config.debugMode) { - const stats = this.getStats() - this.log(`๐ŸŽจ Display augmentation shutdown statistics:`) - this.log(` Total computations: ${stats.totalComputations}`) - this.log(` Cache hit ratio: ${(stats.cacheHitRatio * 100).toFixed(1)}%`) - this.log(` Average computation time: ${stats.averageComputationTime.toFixed(1)}ms`) - } - - this.log('๐ŸŽจ Universal Display augmentation shut down') - - } catch (error) { - this.log(`Display augmentation shutdown error: ${error}`, 'error') - } - } -} - -/** - * Factory function to create display augmentation with default config - * @param config Optional configuration overrides - * @returns Configured display augmentation instance - */ -export function createDisplayAugmentation(config: Partial = {}): UniversalDisplayAugmentation { - return new UniversalDisplayAugmentation(config) -} - -/** - * Default configuration for the display augmentation - */ -export const DEFAULT_DISPLAY_CONFIG: DisplayConfig = { - enabled: true, - cacheSize: 1000, - lazyComputation: true, - batchSize: 50, - confidenceThreshold: 0.7, - customFieldMappings: {}, - priorityFields: {}, - debugMode: false -} - -/** - * Export for easy import and registration - */ -export default UniversalDisplayAugmentation \ No newline at end of file diff --git a/src/brainy.ts b/src/brainy.ts index 56eb055c..52e446e1 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -18,8 +18,6 @@ import { } from './utils/index.js' import { embeddingManager } from './embeddings/EmbeddingManager.js' import { matchesMetadataFilter } from './utils/metadataFilter.js' -import { AugmentationRegistry, AugmentationContext } from './augmentations/brainyAugmentation.js' -import { createDefaultAugmentations } from './augmentations/defaultAugmentations.js' import { ImprovedNeuralAPI } from './neural/improvedNeuralAPI.js' import { NaturalLanguageProcessor } from './neural/naturalLanguageProcessor.js' import { NeuralEntityExtractor, ExtractedEntity } from './neural/entityExtractor.js' @@ -130,7 +128,6 @@ export class Brainy implements BrainyInterface { private transactionManager: TransactionManager private embedder: EmbeddingFunction private distance: DistanceFunction - private augmentationRegistry: AugmentationRegistry private config: Required // Distributed components (optional) @@ -192,7 +189,6 @@ export class Brainy implements BrainyInterface { // Setup core components this.distance = cosineDistance this.embedder = this.setupEmbedder() - this.augmentationRegistry = this.setupAugmentations() this.transactionManager = new TransactionManager() // Setup distributed components if enabled @@ -230,7 +226,6 @@ export class Brainy implements BrainyInterface { ...configOverrides, storage: { ...this.config.storage, ...configOverrides.storage }, index: { ...this.config.index, ...configOverrides.index }, - augmentations: { ...this.config.augmentations, ...configOverrides.augmentations }, verbose: configOverrides.verbose ?? this.config.verbose, silent: configOverrides.silent ?? this.config.silent } @@ -332,23 +327,6 @@ export class Brainy implements BrainyInterface { // Rebuild indexes if needed for existing data await this.rebuildIndexesIfNeeded() - // Initialize augmentations - await this.augmentationRegistry.initializeAll({ - brain: this, - storage: this.storage, - config: this.config, - log: (message: string, level = 'info') => { - // Simple logging for now - if (level === 'error') { - console.error(message) - } else if (level === 'warn') { - console.warn(message) - } else { - console.log(message) - } - } - }) - // Connect distributed components to storage await this.connectDistributedStorage() @@ -698,9 +676,7 @@ export class Brainy implements BrainyInterface { ) } - // Execute through augmentation pipeline - return this.augmentationRegistry.execute('add', params, async () => { - // Prepare metadata for storage (backward compat format - unchanged) + // Prepare metadata for storage (backward compat format - unchanged) const storageMetadata = { ...(typeof params.data === 'object' && params.data !== null && !Array.isArray(params.data) ? params.data : {}), ...params.metadata, @@ -776,7 +752,6 @@ export class Brainy implements BrainyInterface { }) return id - }) } /** @@ -914,28 +889,26 @@ export class Brainy implements BrainyInterface { async get(id: string, options?: GetOptions): Promise | null> { await this.ensureInitialized() - return this.augmentationRegistry.execute('get', { id, options }, async () => { - // Route to metadata-only or full entity based on options - const includeVectors = options?.includeVectors ?? false // Default: metadata-only (fast) + // Route to metadata-only or full entity based on options + const includeVectors = options?.includeVectors ?? false // Default: metadata-only (fast) - if (includeVectors) { - // FULL PATH: Load vector + metadata (6KB, 43ms) - // Used when: Computing similarity on this entity, manual vector operations - const noun = await this.storage.getNoun(id) - if (!noun) { - return null - } - return this.convertNounToEntity(noun) - } else { - // FAST PATH: Metadata-only (300 bytes, 10ms) - DEFAULT - // Used when: VFS operations, existence checks, metadata inspection (94% of calls) - const metadata = await this.storage.getNounMetadata(id) - if (!metadata) { - return null - } - return this.convertMetadataToEntity(id, metadata) + if (includeVectors) { + // FULL PATH: Load vector + metadata (6KB, 43ms) + // Used when: Computing similarity on this entity, manual vector operations + const noun = await this.storage.getNoun(id) + if (!noun) { + return null } - }) + return this.convertNounToEntity(noun) + } else { + // FAST PATH: Metadata-only (300 bytes, 10ms) - DEFAULT + // Used when: VFS operations, existence checks, metadata inspection (94% of calls) + const metadata = await this.storage.getNounMetadata(id) + if (!metadata) { + return null + } + return this.convertMetadataToEntity(id, metadata) + } } /** @@ -1104,8 +1077,7 @@ export class Brainy implements BrainyInterface { // Zero-config validation (static import for performance) validateUpdateParams(params) - return this.augmentationRegistry.execute('update', params, async () => { - // Get existing entity with vectors (fix for regression) + // Get existing entity with vectors (fix for regression) // We need includeVectors: true because: // 1. SaveNounOperation requires the vector // 2. HNSW reindexing operations need the original vector @@ -1244,7 +1216,6 @@ export class Brainy implements BrainyInterface { new AddToMetadataIndexOperation(this.metadataIndex, params.id, entityForIndexing) ) }) - }) } /** @@ -1258,8 +1229,7 @@ export class Brainy implements BrainyInterface { await this.ensureInitialized() - return this.augmentationRegistry.execute('delete', { id }, async () => { - // Get entity metadata and related verbs before deletion + // Get entity metadata and related verbs before deletion const metadata = await this.storage.getNounMetadata(id) const noun = await this.storage.getNoun(id) const verbs = await this.storage.getVerbsBySource(id) @@ -1310,7 +1280,6 @@ export class Brainy implements BrainyInterface { ) } }) - }) } // ============= RELATIONSHIP OPERATIONS ============= @@ -1465,8 +1434,7 @@ export class Brainy implements BrainyInterface { (v, i) => (v + toEntity.vector[i]) / 2 ) - return this.augmentationRegistry.execute('relate', params, async () => { - // Prepare verb metadata + // Prepare verb metadata // CRITICAL: Include verb type in metadata for count tracking const verbMetadata = { verb: params.type, // Store verb type for count synchronization @@ -1551,7 +1519,6 @@ export class Brainy implements BrainyInterface { }) return id - }) } /** @@ -1560,8 +1527,7 @@ export class Brainy implements BrainyInterface { async unrelate(id: string): Promise { await this.ensureInitialized() - return this.augmentationRegistry.execute('unrelate', { id }, async () => { - // Get verb data before deletion for rollback + // Get verb data before deletion for rollback const verb = await this.storage.getVerb(id) // Execute atomically with transaction system @@ -1578,7 +1544,6 @@ export class Brainy implements BrainyInterface { new DeleteVerbMetadataOperation(this.storage, id) ) }) - }) } /** @@ -1865,7 +1830,7 @@ export class Brainy implements BrainyInterface { validateFindParams(params) const startTime = Date.now() - const result = await this.augmentationRegistry.execute('find', params, async () => { + const result = await (async () => { let results: Result[] = [] // Distinguish between search criteria (need vector search) and filter criteria (metadata only) @@ -2201,8 +2166,8 @@ export class Brainy implements BrainyInterface { // Efficient pagination - only slice what we need (limit already defined above) return results.slice(finalOffset, finalOffset + limit) - }) - + })() + // Record performance for auto-tuning const duration = Date.now() - startTime recordQueryPerformance(duration, result.length) @@ -2771,8 +2736,7 @@ export class Brainy implements BrainyInterface { async clear(): Promise { await this.ensureInitialized() - return this.augmentationRegistry.execute('clear', {}, async () => { - // Clear storage + // Clear storage await this.storage.clear() // Invalidate GraphAdjacencyIndex to prevent stale in-memory data @@ -2832,7 +2796,6 @@ export class Brainy implements BrainyInterface { // VFS was never used, reset flag for clean state this._vfsInitialized = false } - }) } // ============= COW (COPY-ON-WRITE) API ============= @@ -2887,8 +2850,7 @@ export class Brainy implements BrainyInterface { }): Promise> { await this.ensureInitialized() - return this.augmentationRegistry.execute('fork', { branch, options }, async () => { - const branchName = branch || `fork-${Date.now()}` + const branchName = branch || `fork-${Date.now()}` // Lazy COW initialization - enable automatically on first fork() // This is zero-config and transparent to users @@ -2978,25 +2940,11 @@ export class Brainy implements BrainyInterface { clone.graphIndex = await (clone.storage as any).getGraphIndex() // getGraphIndex() will rebuild automatically if data exists (via _initializeGraphIndex) - // Setup augmentations - clone.augmentationRegistry = this.setupAugmentations() - await clone.augmentationRegistry.initializeAll({ - brain: clone, - storage: clone.storage, - config: clone.config, - log: (message: string, level?: 'info' | 'warn' | 'error') => { - if (!clone.config.silent) { - console[level || 'info'](message) - } - } - }) - // Mark as initialized clone.initialized = true clone.dimensions = this.dimensions return clone - }) } /** @@ -3012,19 +2960,17 @@ export class Brainy implements BrainyInterface { async listBranches(): Promise { await this.ensureInitialized() - return this.augmentationRegistry.execute('listBranches', {}, async () => { - if (!('refManager' in this.storage)) { - throw new Error('Branch management requires COW-enabled storage') - } + if (!('refManager' in this.storage)) { + throw new Error('Branch management requires COW-enabled storage') + } - const refManager = (this.storage as any).refManager - const refs = await refManager.listRefs() + const refManager = (this.storage as any).refManager + const refs = await refManager.listRefs() - // Filter to branches only (exclude tags) - return refs - .filter((ref: any) => ref.name.startsWith('refs/heads/')) - .map((ref: any) => ref.name.replace('refs/heads/', '')) - }) + // Filter to branches only (exclude tags) + return refs + .filter((ref: any) => ref.name.startsWith('refs/heads/')) + .map((ref: any) => ref.name.replace('refs/heads/', '')) } /** @@ -3040,13 +2986,11 @@ export class Brainy implements BrainyInterface { async getCurrentBranch(): Promise { await this.ensureInitialized() - return this.augmentationRegistry.execute('getCurrentBranch', {}, async () => { - if (!('currentBranch' in this.storage)) { - return 'main' // Default branch - } + if (!('currentBranch' in this.storage)) { + return 'main' // Default branch + } - return (this.storage as any).currentBranch || 'main' - }) + return (this.storage as any).currentBranch || 'main' } /** @@ -3062,10 +3006,9 @@ export class Brainy implements BrainyInterface { async checkout(branch: string): Promise { await this.ensureInitialized() - return this.augmentationRegistry.execute('checkout', { branch }, async () => { - if (!('refManager' in this.storage)) { - throw new Error('Branch management requires COW-enabled storage') - } + if (!('refManager' in this.storage)) { + throw new Error('Branch management requires COW-enabled storage') + } // Verify branch exists const branches = await this.listBranches() @@ -3106,7 +3049,6 @@ export class Brainy implements BrainyInterface { this._vfs = new VirtualFileSystem(this) await this._vfs.init() } - }) } /** @@ -3131,8 +3073,7 @@ export class Brainy implements BrainyInterface { }): Promise { await this.ensureInitialized() - return this.augmentationRegistry.execute('commit', { options }, async () => { - if (!('refManager' in this.storage) || !('commitLog' in this.storage) || !('blobStorage' in this.storage)) { + if (!('refManager' in this.storage) || !('commitLog' in this.storage) || !('blobStorage' in this.storage)) { throw new Error('Commit requires COW-enabled storage') } @@ -3187,7 +3128,6 @@ export class Brainy implements BrainyInterface { }) return commitHash - }) } /** @@ -3380,19 +3320,17 @@ export class Brainy implements BrainyInterface { async deleteBranch(branch: string): Promise { await this.ensureInitialized() - return this.augmentationRegistry.execute('deleteBranch', { branch }, async () => { - if (!('refManager' in this.storage)) { - throw new Error('Branch management requires COW-enabled storage') - } + if (!('refManager' in this.storage)) { + throw new Error('Branch management requires COW-enabled storage') + } - const currentBranch = await this.getCurrentBranch() - if (branch === currentBranch) { - throw new Error('Cannot delete current branch') - } + const currentBranch = await this.getCurrentBranch() + if (branch === currentBranch) { + throw new Error('Cannot delete current branch') + } - const refManager = (this.storage as any).refManager - await refManager.deleteRef(branch) - }) + const refManager = (this.storage as any).refManager + await refManager.deleteRef(branch) } /** @@ -3421,30 +3359,28 @@ export class Brainy implements BrainyInterface { }>> { await this.ensureInitialized() - return this.augmentationRegistry.execute('getHistory', { options }, async () => { - if (!('commitLog' in this.storage) || !('refManager' in this.storage)) { - throw new Error('History requires COW-enabled storage') - } + if (!('commitLog' in this.storage) || !('refManager' in this.storage)) { + throw new Error('History requires COW-enabled storage') + } - const commitLog = (this.storage as any).commitLog - const currentBranch = await this.getCurrentBranch() + const commitLog = (this.storage as any).commitLog + const currentBranch = await this.getCurrentBranch() - // Get commit history for current branch - const commits = await commitLog.getHistory(currentBranch, { - maxCount: options?.limit || 10 - }) - - // Map to expected format (compute hash for each commit) - return commits.map((commit: any) => ({ - hash: (this.storage as any).blobStorage.constructor.hash( - Buffer.from(JSON.stringify(commit)) - ), - message: commit.message, - author: commit.author, - timestamp: commit.timestamp, - metadata: commit.metadata - })) + // Get commit history for current branch + const commits = await commitLog.getHistory(currentBranch, { + maxCount: options?.limit || 10 }) + + // Map to expected format (compute hash for each commit) + return commits.map((commit: any) => ({ + hash: (this.storage as any).blobStorage.constructor.hash( + Buffer.from(JSON.stringify(commit)) + ), + message: commit.message, + author: commit.author, + timestamp: commit.timestamp, + metadata: commit.metadata + })) } /** @@ -3950,21 +3886,12 @@ export class Brainy implements BrainyInterface { }) => void } ) { - // Execute through augmentation pipeline (Enables IntelligentImportAugmentation) - // If source is an ImportSource object (not a Buffer), spread it so augmentations can access properties - const params = typeof source === 'object' && !Buffer.isBuffer(source) - ? { ...source as object, ...options } // Spread ImportSource: { type, data, filename, ...options } - : { source, ...options } // Wrap Buffer/string: { source, ...options } + // Lazy load ImportCoordinator + const { ImportCoordinator } = await import('./import/ImportCoordinator.js') + const coordinator = new ImportCoordinator(this) + await coordinator.init() - return this.augmentationRegistry.execute('import', params, async () => { - // Lazy load ImportCoordinator - const { ImportCoordinator } = await import('./import/ImportCoordinator.js') - const coordinator = new ImportCoordinator(this) - await coordinator.init() - - // Pass augmentation-modified params (contains _intelligentImport, _extractedData, etc) - return await coordinator.import(source, { ...options, ...params }) - }) + return await coordinator.import(source, options) } /** @@ -4690,17 +4617,6 @@ export class Brainy implements BrainyInterface { } } - /** - * Augmentations API - Clean and simple - */ - get augmentations() { - return { - list: () => this.augmentationRegistry.getAll().map(a => a.name), - get: (name: string) => this.augmentationRegistry.getAll().find(a => a.name === name), - has: (name: string) => this.augmentationRegistry.getAll().some(a => a.name === name) - } - } - /** * Get complete statistics - convenience method * For more granular counting, use brain.counts API @@ -6282,32 +6198,6 @@ export class Brainy implements BrainyInterface { return persistMode } - /** - * Setup augmentations - */ - private setupAugmentations(): AugmentationRegistry { - const registry = new AugmentationRegistry() - - // Register default augmentations with silent mode support - const augmentationConfig = { - ...this.config.augmentations, - // Pass silent mode to all augmentations - ...(this.config.silent && { - cache: this.config.augmentations?.cache !== false ? { ...this.config.augmentations?.cache, silent: true } : false, - metrics: this.config.augmentations?.metrics !== false ? { ...this.config.augmentations?.metrics, silent: true } : false, - display: this.config.augmentations?.display !== false ? { ...this.config.augmentations?.display, silent: true } : false, - monitoring: this.config.augmentations?.monitoring !== false ? { ...this.config.augmentations?.monitoring, silent: true } : false - }) - } - - const defaults = createDefaultAugmentations(augmentationConfig) - for (const aug of defaults) { - registry.register(aug) - } - - return registry - } - /** * Normalize and validate configuration */ @@ -6356,7 +6246,6 @@ export class Brainy implements BrainyInterface { storage: config?.storage || { type: 'auto' }, index: config?.index || {}, cache: config?.cache ?? true, - augmentations: config?.augmentations || {}, distributed: distributedConfig as any, // Type will be fixed when used warmup: config?.warmup ?? false, realtime: config?.realtime ?? false, @@ -6670,14 +6559,6 @@ export class Brainy implements BrainyInterface { // Deactivate plugins await this.pluginRegistry.deactivateAll() - // Shutdown augmentations - const augs = this.augmentationRegistry.getAll() - for (const aug of augs) { - if ('shutdown' in aug && typeof aug.shutdown === 'function') { - await aug.shutdown() - } - } - // Restore console methods if silent mode was enabled if (this.config.silent && this.originalConsole) { console.log = this.originalConsole.log as typeof console.log diff --git a/src/cli/commands/core.ts b/src/cli/commands/core.ts index 97437cf7..9f9f2947 100644 --- a/src/cli/commands/core.ts +++ b/src/cli/commands/core.ts @@ -148,21 +148,9 @@ export const coreCommands = { } nounType = options.type as NounType } else { - // Use AI to suggest type - spinner.text = 'Detecting type with AI...' - const suggestion = await BrainyTypes.suggestNoun( - typeof text === 'string' ? { content: text, ...metadata } : text - ) - - if (suggestion.confidence < 0.6) { - spinner.fail('Could not determine type with confidence') - console.log(chalk.yellow(`Suggestion: ${suggestion.type} (${(suggestion.confidence * 100).toFixed(1)}%)`)) - console.log(chalk.dim('Use --type flag to specify explicitly')) - process.exit(1) - } - - nounType = suggestion.type as NounType - spinner.text = `Using detected type: ${nounType}` + // Default to Thing when no type specified + nounType = NounType.Thing + spinner.text = `No type specified, using default: ${nounType}` } // Add with explicit type diff --git a/src/cli/commands/types.ts b/src/cli/commands/types.ts index c9140967..fd547b5e 100644 --- a/src/cli/commands/types.ts +++ b/src/cli/commands/types.ts @@ -1,12 +1,9 @@ /** * CLI Commands for Type Management - * Consistent with BrainyTypes public API */ import chalk from 'chalk' -import ora from 'ora' import inquirer from 'inquirer' -import Table from 'cli-table3' import { BrainyTypes, NounType, VerbType } from '../../index.js' /** @@ -18,7 +15,7 @@ export async function types(options: { json?: boolean, noun?: boolean, verb?: bo // Default to showing both if neither flag specified const showNouns = options.noun || (!options.noun && !options.verb) const showVerbs = options.verb || (!options.noun && !options.verb) - + const result: any = {} if (showNouns) result.nouns = BrainyTypes.nouns if (showVerbs) result.verbs = BrainyTypes.verbs @@ -30,12 +27,12 @@ export async function types(options: { json?: boolean, noun?: boolean, verb?: bo // Display nouns if (showNouns) { - console.log(chalk.bold.cyan('\n๐Ÿ“š Noun Types (42):\n')) + console.log(chalk.bold.cyan(`\nNoun Types (${BrainyTypes.nouns.length}):\n`)) const nounChunks = [] for (let i = 0; i < BrainyTypes.nouns.length; i += 3) { nounChunks.push(BrainyTypes.nouns.slice(i, i + 3)) } - + for (const chunk of nounChunks) { console.log(' ' + chunk.map(n => chalk.green(n.padEnd(20))).join('')) } @@ -43,110 +40,17 @@ export async function types(options: { json?: boolean, noun?: boolean, verb?: bo // Display verbs if (showVerbs) { - console.log(chalk.bold.cyan('\n๐Ÿ”— Verb Types (127):\n')) + console.log(chalk.bold.cyan(`\nVerb Types (${BrainyTypes.verbs.length}):\n`)) const verbChunks = [] for (let i = 0; i < BrainyTypes.verbs.length; i += 3) { verbChunks.push(BrainyTypes.verbs.slice(i, i + 3)) } - + for (const chunk of verbChunks) { console.log(' ' + chunk.map(v => chalk.blue(v.padEnd(20))).join('')) } } - console.log(chalk.dim('\n๐Ÿ’ก Use "brainy suggest " to get AI-powered type suggestions')) - - } catch (error: any) { - console.error(chalk.red('Error:', error.message)) - process.exit(1) - } -} - -/** - * Suggest type - matches BrainyTypes.suggestNoun() and suggestVerb() - * Usage: brainy suggest - * Interactive if data not provided - */ -export async function suggest( - data?: string, - options: { - verb?: boolean, - json?: boolean - } = {} -) { - try { - // Interactive mode if no data provided - if (!data) { - const answers = await inquirer.prompt([ - { - type: 'list', - name: 'kind', - message: 'What type do you want to suggest?', - choices: ['Noun', 'Verb'], - default: 'Noun' - }, - { - type: 'input', - name: 'data', - message: 'Enter data (JSON or text):', - validate: (input) => input.length > 0 || 'Data is required' - }, - { - type: 'input', - name: 'hint', - message: 'Relationship hint (optional):', - when: (answers) => answers.kind === 'Verb' - } - ]) - - data = answers.data - options.verb = answers.kind === 'Verb' - - // For verbs, parse source/target if provided as JSON - if (options.verb && answers.hint) { - data = JSON.stringify({ hint: answers.hint }) - } - } - - const spinner = ora('Analyzing with AI...').start() - - let parsedData: any - try { - parsedData = JSON.parse(data) - } catch { - parsedData = { content: data } - } - - let suggestion - if (options.verb) { - // For verb suggestions, need source and target - const source = parsedData.source || { type: 'unknown' } - const target = parsedData.target || { type: 'unknown' } - const hint = parsedData.hint || parsedData.relationship || parsedData.verb - - suggestion = await BrainyTypes.suggestVerb(source, target, hint) - spinner.succeed('Verb type analyzed') - } else { - suggestion = await BrainyTypes.suggestNoun(parsedData) - spinner.succeed('Noun type analyzed') - } - - if (options.json) { - console.log(JSON.stringify(suggestion, null, 2)) - return - } - - // Display results - console.log(chalk.bold.green(`\nโœจ Suggested: ${suggestion.type}`)) - console.log(chalk.cyan(`Confidence: ${(suggestion.confidence * 100).toFixed(1)}%`)) - - if (suggestion.alternatives && suggestion.alternatives.length > 0) { - console.log(chalk.yellow('\nAlternatives:')) - for (const alt of suggestion.alternatives.slice(0, 3)) { - console.log(` ${alt.type} (${(alt.confidence * 100).toFixed(1)}%)`) - } - } - } catch (error: any) { console.error(chalk.red('Error:', error.message)) process.exit(1) diff --git a/src/augmentations/neuralImport.ts b/src/cortex/neuralImportAugmentation.ts similarity index 86% rename from src/augmentations/neuralImport.ts rename to src/cortex/neuralImportAugmentation.ts index 739b4a4d..28efce59 100644 --- a/src/augmentations/neuralImport.ts +++ b/src/cortex/neuralImportAugmentation.ts @@ -1,17 +1,12 @@ /** - * Neural Import Augmentation - AI-Powered Data Understanding - * - * ๐Ÿง  Built-in AI augmentation for intelligent data processing - * โš›๏ธ Always free, always included, always enabled - * - * Now using the unified BrainyAugmentation interface! + * Neural Import - AI-Powered Data Understanding + * + * Standalone implementation for intelligent data processing. */ -import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js' import { NounType, VerbType } from '../types/graphTypes.js' import * as fs from '../universal/fs.js' import * as path from '../universal/path.js' -import { BrainyTypes, getBrainyTypes } from './typeMatching/brainyTypes.js' import { prodLog } from '../utils/logger.js' // Neural Import Analysis Types @@ -62,22 +57,15 @@ export interface NeuralImportConfig { * Neural Import Augmentation - Unified Implementation * Processes data with AI before storage operations */ -export class NeuralImportAugmentation extends BaseAugmentation { +export class NeuralImportAugmentation { readonly name = 'neural-import' - readonly timing = 'before' as const // Process data before storage - readonly metadata = { - reads: '*' as '*', // Needs to read data for analysis - writes: ['_neuralProcessed', '_neuralConfidence', '_detectedEntities', '_detectedRelationships', '_neuralInsights', 'nounType', 'verbType'] as string[] - } // Enriches metadata with neural analysis - operations = ['add', 'addNoun', 'addVerb', 'all'] as ('add' | 'addNoun' | 'addVerb' | 'all')[] // Use 'all' to catch batch operations - readonly priority = 80 // High priority for data processing - + private operations = ['add', 'addNoun', 'addVerb', 'all'] + protected config: NeuralImportConfig private analysisCache = new Map() - private typeMatcher: BrainyTypes | null = null + private context?: { brain: any } constructor(config: Partial = {}) { - super() this.config = { confidenceThreshold: 0.7, enableWeights: true, @@ -87,18 +75,12 @@ export class NeuralImportAugmentation extends BaseAugmentation { } } - protected async onInitialize(): Promise { - try { - this.typeMatcher = await getBrainyTypes() - this.log('๐Ÿง  Neural Import augmentation initialized with intelligent type matching') - } catch (error) { - this.log('โš ๏ธ Failed to initialize type matcher, falling back to heuristics', 'warn') - } + async init(): Promise { + // No external dependencies to initialize } - protected async onShutdown(): Promise { - this.analysisCache.clear() - this.log('๐Ÿง  Neural Import augmentation shut down') + private log(message: string, _level?: string): void { + // Silent by default } /** @@ -455,22 +437,29 @@ export class NeuralImportAugmentation extends BaseAugmentation { } /** - * Infer noun type from object structure using intelligent type matching + * Infer noun type from object structure using field heuristics */ private async inferNounType(obj: any): Promise { - if (!this.typeMatcher) { - // Initialize type matcher if not available - this.typeMatcher = await getBrainyTypes() + if (typeof obj !== 'object' || obj === null) return NounType.Thing + + // Check for explicit type field + if (obj.type && typeof obj.type === 'string') { + const normalized = obj.type.charAt(0).toUpperCase() + obj.type.slice(1) + if (Object.values(NounType).includes(normalized as NounType)) { + return normalized as NounType + } } - - const result = await this.typeMatcher.matchNounType(obj) - - // Log if confidence is low for debugging - if (result.confidence < 0.5) { - this.log(`Low confidence (${result.confidence.toFixed(2)}) for noun type: ${result.type}`, 'warn') - } - - return result.type + + if (obj.email || obj.firstName || obj.lastName || obj.username) return NounType.Person + if (obj.companyName || obj.organizationId || obj.employees) return NounType.Organization + if (obj.latitude || obj.longitude || obj.address || obj.city) return NounType.Location + if ((obj.content && (obj.title || obj.author)) || obj.pages) return NounType.Document + if (obj.startTime || obj.endTime || obj.date || obj.attendees) return NounType.Event + if (obj.price || obj.sku || obj.productId) return NounType.Product + if ((obj.status && obj.assignee) || obj.priority) return NounType.Task + if (Array.isArray(obj.data) || obj.rows || obj.columns) return NounType.Dataset + + return NounType.Thing } /** @@ -511,22 +500,28 @@ export class NeuralImportAugmentation extends BaseAugmentation { } /** - * Infer verb type from field name using intelligent type matching + * Infer verb type from field name using common patterns */ - private async inferVerbType(fieldName: string, sourceObj?: any, targetObj?: any): Promise { - if (!this.typeMatcher) { - // Initialize type matcher if not available - this.typeMatcher = await getBrainyTypes() + private async inferVerbType(fieldName: string, _sourceObj?: any, _targetObj?: any): Promise { + const field = fieldName.toLowerCase() + + if (field.includes('parent') || field.includes('child') || field.includes('contain')) { + return VerbType.Contains } - - const result = await this.typeMatcher.matchVerbType(sourceObj, targetObj, fieldName) - - // Log if confidence is low for debugging - if (result.confidence < 0.5) { - this.log(`Low confidence (${result.confidence.toFixed(2)}) for verb type: ${result.type}`, 'warn') + if (field.includes('owner') || field.includes('created') || field.includes('author')) { + return VerbType.Creates } - - return result.type + if (field.includes('member') || field.includes('belong')) { + return VerbType.MemberOf + } + if (field.includes('depend') || field.includes('require')) { + return VerbType.DependsOn + } + if (field.includes('ref') || field.includes('link') || field.includes('source')) { + return VerbType.References + } + + return VerbType.RelatedTo } /** diff --git a/src/importers/SmartCSVImporter.ts b/src/importers/SmartCSVImporter.ts index f11832e6..7de6c0dd 100644 --- a/src/importers/SmartCSVImporter.ts +++ b/src/importers/SmartCSVImporter.ts @@ -16,8 +16,8 @@ import { NeuralEntityExtractor, ExtractedEntity } from '../neural/entityExtracto import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js' import { SmartRelationshipExtractor } from '../neural/SmartRelationshipExtractor.js' import { NounType, VerbType } from '../types/graphTypes.js' -import { CSVHandler } from '../augmentations/intelligentImport/handlers/csvHandler.js' -import type { FormatHandlerOptions } from '../augmentations/intelligentImport/types.js' +import { CSVHandler } from './handlers/csvHandler.js' +import type { FormatHandlerOptions } from './handlers/types.js' export interface SmartCSVOptions extends FormatHandlerOptions { /** Enable neural entity extraction */ diff --git a/src/importers/SmartExcelImporter.ts b/src/importers/SmartExcelImporter.ts index 479a6b75..83f47cf2 100644 --- a/src/importers/SmartExcelImporter.ts +++ b/src/importers/SmartExcelImporter.ts @@ -14,8 +14,8 @@ import { NeuralEntityExtractor, ExtractedEntity } from '../neural/entityExtracto import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js' import { SmartRelationshipExtractor } from '../neural/SmartRelationshipExtractor.js' import { NounType, VerbType } from '../types/graphTypes.js' -import { ExcelHandler } from '../augmentations/intelligentImport/handlers/excelHandler.js' -import type { FormatHandlerOptions } from '../augmentations/intelligentImport/types.js' +import { ExcelHandler } from './handlers/excelHandler.js' +import type { FormatHandlerOptions } from './handlers/types.js' export interface SmartExcelOptions extends FormatHandlerOptions { /** Enable neural entity extraction */ diff --git a/src/importers/SmartPDFImporter.ts b/src/importers/SmartPDFImporter.ts index 3cec79fb..d4dfd93d 100644 --- a/src/importers/SmartPDFImporter.ts +++ b/src/importers/SmartPDFImporter.ts @@ -15,8 +15,8 @@ import { NeuralEntityExtractor, ExtractedEntity } from '../neural/entityExtracto import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js' import { SmartRelationshipExtractor } from '../neural/SmartRelationshipExtractor.js' import { NounType, VerbType } from '../types/graphTypes.js' -import { PDFHandler } from '../augmentations/intelligentImport/handlers/pdfHandler.js' -import type { FormatHandlerOptions } from '../augmentations/intelligentImport/types.js' +import { PDFHandler } from './handlers/pdfHandler.js' +import type { FormatHandlerOptions } from './handlers/types.js' export interface SmartPDFOptions extends FormatHandlerOptions { /** Enable neural entity extraction */ diff --git a/src/augmentations/intelligentImport/handlers/base.ts b/src/importers/handlers/base.ts similarity index 98% rename from src/augmentations/intelligentImport/handlers/base.ts rename to src/importers/handlers/base.ts index 21811233..e5d18455 100644 --- a/src/augmentations/intelligentImport/handlers/base.ts +++ b/src/importers/handlers/base.ts @@ -5,8 +5,8 @@ * Uses MimeTypeDetector for comprehensive file type detection (2000+ types) */ -import { FormatHandler, FormatHandlerOptions, ProcessedData } from '../types.js' -import { mimeDetector } from '../../../vfs/MimeTypeDetector.js' +import { FormatHandler, FormatHandlerOptions, ProcessedData } from './types.js' +import { mimeDetector } from '../../vfs/MimeTypeDetector.js' export abstract class BaseFormatHandler implements FormatHandler { abstract readonly format: string diff --git a/src/augmentations/intelligentImport/handlers/csvHandler.ts b/src/importers/handlers/csvHandler.ts similarity index 99% rename from src/augmentations/intelligentImport/handlers/csvHandler.ts rename to src/importers/handlers/csvHandler.ts index 3287e258..f27926cd 100644 --- a/src/augmentations/intelligentImport/handlers/csvHandler.ts +++ b/src/importers/handlers/csvHandler.ts @@ -10,7 +10,7 @@ import { parse } from 'csv-parse/sync' import { detect as detectEncoding } from 'chardet' import { BaseFormatHandler } from './base.js' -import { FormatHandlerOptions, ProcessedData } from '../types.js' +import { FormatHandlerOptions, ProcessedData } from './types.js' export class CSVHandler extends BaseFormatHandler { readonly format = 'csv' diff --git a/src/augmentations/intelligentImport/handlers/excelHandler.ts b/src/importers/handlers/excelHandler.ts similarity index 99% rename from src/augmentations/intelligentImport/handlers/excelHandler.ts rename to src/importers/handlers/excelHandler.ts index 64a6aaa5..42ea59d8 100644 --- a/src/augmentations/intelligentImport/handlers/excelHandler.ts +++ b/src/importers/handlers/excelHandler.ts @@ -9,7 +9,7 @@ import * as XLSX from 'xlsx' import { BaseFormatHandler } from './base.js' -import { FormatHandlerOptions, ProcessedData } from '../types.js' +import { FormatHandlerOptions, ProcessedData } from './types.js' export class ExcelHandler extends BaseFormatHandler { readonly format = 'excel' diff --git a/src/augmentations/intelligentImport/handlers/pdfHandler.ts b/src/importers/handlers/pdfHandler.ts similarity index 99% rename from src/augmentations/intelligentImport/handlers/pdfHandler.ts rename to src/importers/handlers/pdfHandler.ts index 91ee70e8..fcc00b9e 100644 --- a/src/augmentations/intelligentImport/handlers/pdfHandler.ts +++ b/src/importers/handlers/pdfHandler.ts @@ -9,7 +9,7 @@ import * as pdfjsLib from 'pdfjs-dist/legacy/build/pdf.mjs' import { BaseFormatHandler } from './base.js' -import { FormatHandlerOptions, ProcessedData } from '../types.js' +import { FormatHandlerOptions, ProcessedData } from './types.js' // Use built-in worker for Node.js environments // In production, this can be customized via options diff --git a/src/augmentations/intelligentImport/types.ts b/src/importers/handlers/types.ts similarity index 100% rename from src/augmentations/intelligentImport/types.ts rename to src/importers/handlers/types.ts diff --git a/src/index.ts b/src/index.ts index b75579a4..ba9d98fe 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,7 +5,7 @@ * Core Components: * - Brainy: The unified database with Triple Intelligence * - Triple Intelligence: Seamless fusion of vector + graph + field search - * - Augmentations: Extensible plugin system + * - Plugins: Extensible plugin system (cortex, storage adapters) * - Neural API: AI-powered clustering and analysis */ @@ -54,21 +54,6 @@ export { getPresetDescription } from './config/index.js' -// Export augmentation types -export type { - BrainyAugmentations, - IAugmentation, - ISenseAugmentation, - IConduitAugmentation, - ICognitionAugmentation, - IMemoryAugmentation, - IPerceptionAugmentation, - IDialogAugmentation, - IActivationAugmentation, - IWebSocketSupport, - AugmentationType -} from './types/augmentations.js' - // Export Neural Import (AI data understanding) export { NeuralImport } from './cortex/neuralImport.js' export type { @@ -96,10 +81,6 @@ export type { SmartRelationshipExtractorOptions } from './neural/SmartRelationshipExtractor.js' -// Import Manager removed - use brain.import() instead (available on all Brainy instances) - -// Augmentation types are already exported later in the file - // Export distance functions for convenience import { euclideanDistance, @@ -234,7 +215,6 @@ export { export { FileSystemStorage } from './storage/adapters/fileSystemStorage.js' // Export COW (Copy-on-Write) infrastructure -// Enables premium augmentations to implement temporal features import { CommitLog } from './storage/cow/CommitLog.js' import { CommitObject, CommitBuilder } from './storage/cow/CommitObject.js' import { BlobStorage } from './storage/cow/BlobStorage.js' @@ -255,7 +235,6 @@ export { import { Pipeline, pipeline, - augmentationPipeline, ExecutionMode, PipelineOptions, PipelineResult, @@ -266,85 +245,21 @@ import { StreamlinedPipelineResult } from './pipeline.js' -// Sequential pipeline removed - use unified pipeline instead - -// REMOVED: Old augmentation factory for 2.0 clean architecture - export { - // Unified pipeline exports Pipeline, pipeline, - augmentationPipeline, ExecutionMode, - - // Factory functions createPipeline, createStreamingPipeline, - StreamlinedExecutionMode, - - // Augmentation factory exports (REMOVED in 2.0 - Use BrainyAugmentation interface) - // createSenseAugmentation, // โ†’ Use BaseAugmentation class - // addWebSocketSupport, // โ†’ Use APIServerAugmentation - // executeAugmentation, // โ†’ Use brain.augmentations.execute() - // loadAugmentationModule // โ†’ Use dynamic imports + StreamlinedExecutionMode } export type { PipelineOptions, PipelineResult, StreamlinedPipelineOptions, StreamlinedPipelineResult - // AugmentationOptions - REMOVED in 2.0 (use BaseAugmentation config) } -// Augmentation registry removed - use Brainy's built-in augmentation system - - - -// Export augmentation implementations -import { - StorageAugmentation, - DynamicStorageAugmentation, - createStorageAugmentationFromConfig -} from './augmentations/storageAugmentation.js' -import { - MemoryStorageAugmentation, - FileSystemStorageAugmentation, - OPFSStorageAugmentation, - S3StorageAugmentation, - R2StorageAugmentation, - GCSStorageAugmentation, - createAutoStorageAugmentation -} from './augmentations/storageAugmentations.js' -import { - WebSocketConduitAugmentation -} from './augmentations/conduitAugmentations.js' - -// Storage augmentation exports -export { - // Base classes - StorageAugmentation, - DynamicStorageAugmentation, - // Concrete implementations - MemoryStorageAugmentation, - FileSystemStorageAugmentation, - OPFSStorageAugmentation, - S3StorageAugmentation, - R2StorageAugmentation, - GCSStorageAugmentation, - // Factory functions - createAutoStorageAugmentation, - createStorageAugmentationFromConfig -} - -// Other augmentation exports -export { - WebSocketConduitAugmentation -} - -// LLM augmentations are optional and not imported by default -// They can be imported directly from their module if needed: -// import { LLMCognitionAugmentation, LLMActivationAugmentation, createLLMAugmentations } from './augmentations/llmAugmentations.js' - // Export types import type { Vector, @@ -377,25 +292,6 @@ export type { StorageAdapter } -// Export augmentation types -import type { - AugmentationResponse, - BrainyAugmentation, - BaseAugmentation, - AugmentationContext -} from './types/augmentations.js' - -// Export augmentation manager for type-safe augmentation management -export { AugmentationManager, type AugmentationInfo } from './augmentationManager.js' - -// Export only the clean augmentation types for 2.0 -export type { - AugmentationResponse, - BrainyAugmentation, - BaseAugmentation, - AugmentationContext -} - // Export graph types import type { GraphNoun, @@ -496,8 +392,8 @@ export type { // Export type utility functions import { getNounTypes, getVerbTypes, getNounTypeMap, getVerbTypeMap } from './utils/typeUtils.js' -// Export BrainyTypes for complete type management -import { BrainyTypes, TypeSuggestion, suggestType } from './utils/brainyTypes.js' +// Export BrainyTypes for type validation and lookup +import { BrainyTypes } from './utils/brainyTypes.js' export { NounType, @@ -506,13 +402,8 @@ export { getVerbTypes, getNounTypeMap, getVerbTypeMap, - // BrainyTypes - complete type management - BrainyTypes, - suggestType -} - -export type { - TypeSuggestion + // BrainyTypes - type validation and lookup + BrainyTypes } // Export MCP (Model Control Protocol) components diff --git a/src/integrations/core/IntegrationBase.ts b/src/integrations/core/IntegrationBase.ts index 9617892f..2bba38d8 100644 --- a/src/integrations/core/IntegrationBase.ts +++ b/src/integrations/core/IntegrationBase.ts @@ -5,11 +5,6 @@ * for event subscriptions, tabular export, and lifecycle management. */ -import { - BaseAugmentation, - AugmentationContext -} from '../../augmentations/brainyAugmentation.js' -import { AugmentationManifest } from '../../augmentations/manifest.js' import { EventBus } from './EventBus.js' import { TabularExporter } from './TabularExporter.js' import { @@ -48,17 +43,12 @@ import { Entity, Relation, FindParams } from '../../types/brainy.types.js' * } * ``` */ -export abstract class IntegrationBase extends BaseAugmentation { - // Augmentation interface implementation - readonly timing = 'after' as const - readonly metadata = 'readonly' as const - readonly operations: ('all')[] = ['all'] - readonly priority = 5 // Low priority - runs after main operations - category: 'internal' | 'core' | 'premium' | 'community' | 'external' = 'core' - +export abstract class IntegrationBase { // Shared infrastructure protected eventBus: EventBus protected exporter: TabularExporter + protected config: IntegrationConfig + protected context?: { brain: any; storage: any; config: any; log: (message: string, level?: string) => void } // Integration state protected isRunning = false @@ -80,11 +70,35 @@ export abstract class IntegrationBase extends BaseAugmentation { config?: IntegrationConfig, exporterConfig?: TabularExporterConfig ) { - super(config) + this.config = config || {} this.eventBus = new EventBus() this.exporter = new TabularExporter(exporterConfig) } + /** + * Log a message + */ + protected log(message: string, level: string = 'info'): void { + if (this.context?.log) { + this.context.log(message, level) + } + } + + /** + * Initialize the integration with context + */ + async initialize(ctx: { brain: any; storage: any; config: any; log: (message: string, level?: string) => void }): Promise { + this.context = ctx + await this.onInitialize() + } + + /** + * Shutdown the integration + */ + async shutdown(): Promise { + await this.onShutdown() + } + /** * Integration name (must be unique) */ @@ -311,7 +325,7 @@ export abstract class IntegrationBase extends BaseAugmentation { * Get manifest for this integration * Subclasses should override this */ - getManifest(): AugmentationManifest { + getManifest(): Record { return { id: this.name, name: this.name, diff --git a/src/integrations/odata/ODataIntegration.ts b/src/integrations/odata/ODataIntegration.ts index 0e50d792..685180a4 100644 --- a/src/integrations/odata/ODataIntegration.ts +++ b/src/integrations/odata/ODataIntegration.ts @@ -15,7 +15,6 @@ import { HTTPIntegration } from '../core/IntegrationBase.js' import { IntegrationConfig, ODataQueryOptions } from '../core/types.js' -import { AugmentationManifest } from '../../augmentations/manifest.js' import { Entity, Relation, FindParams } from '../../types/brainy.types.js' import { NounType } from '../../types/graphTypes.js' import { @@ -94,11 +93,11 @@ interface ODataResponse { * * @example * ```typescript - * // Register with Brainy - * brain.augmentations.register(new ODataIntegration({ + * const odata = new ODataIntegration({ * basePath: '/odata', * maxPageSize: 1000 - * })) + * }) + * await odata.initialize() * * // Connect from Excel Power Query: * // Data โ†’ Get Data โ†’ From OData Feed โ†’ http://localhost:3000/odata @@ -249,7 +248,7 @@ export class ODataIntegration extends IntegrationBase implements HTTPIntegration /** * Get augmentation manifest */ - getManifest(): AugmentationManifest { + getManifest(): Record { return { id: 'odata', name: 'OData Integration', diff --git a/src/integrations/sheets/GoogleSheetsIntegration.ts b/src/integrations/sheets/GoogleSheetsIntegration.ts index 08dbd941..d1f0c6a8 100644 --- a/src/integrations/sheets/GoogleSheetsIntegration.ts +++ b/src/integrations/sheets/GoogleSheetsIntegration.ts @@ -15,7 +15,6 @@ import { IntegrationBase, HTTPIntegration } from '../core/IntegrationBase.js' import { IntegrationConfig } from '../core/types.js' -import { AugmentationManifest } from '../../augmentations/manifest.js' import { Entity, FindParams } from '../../types/brainy.types.js' import { NounType, VerbType } from '../../types/graphTypes.js' @@ -92,10 +91,11 @@ interface SheetsResponse { * * @example * ```typescript - * brain.augmentations.register(new GoogleSheetsIntegration({ + * const sheets = new GoogleSheetsIntegration({ * basePath: '/sheets', * maxResults: 1000 - * })) + * }) + * await sheets.initialize() * ``` */ export class GoogleSheetsIntegration @@ -283,7 +283,7 @@ export class GoogleSheetsIntegration /** * Get manifest */ - getManifest(): AugmentationManifest { + getManifest(): Record { return { id: 'sheets', name: 'Google Sheets Integration', diff --git a/src/integrations/sse/SSEIntegration.ts b/src/integrations/sse/SSEIntegration.ts index 8e09ec02..60d7a6a8 100644 --- a/src/integrations/sse/SSEIntegration.ts +++ b/src/integrations/sse/SSEIntegration.ts @@ -15,7 +15,6 @@ import { EventFilter, BrainyEvent } from '../core/types.js' -import { AugmentationManifest } from '../../augmentations/manifest.js' /** * SSE integration configuration @@ -63,10 +62,11 @@ interface SSEClient { * * @example * ```typescript - * brain.augmentations.register(new SSEIntegration({ + * const sse = new SSEIntegration({ * basePath: '/events', * heartbeatInterval: 30000 - * })) + * }) + * await sse.initialize() * * // Client-side: * const source = new EventSource('/events?types=noun&operations=create,update') @@ -292,7 +292,7 @@ export class SSEIntegration /** * Get manifest */ - getManifest(): AugmentationManifest { + getManifest(): Record { return { id: 'sse', name: 'SSE Streaming', diff --git a/src/integrations/webhooks/WebhookIntegration.ts b/src/integrations/webhooks/WebhookIntegration.ts index c9caf238..1455a202 100644 --- a/src/integrations/webhooks/WebhookIntegration.ts +++ b/src/integrations/webhooks/WebhookIntegration.ts @@ -15,7 +15,6 @@ import { WebhookDeliveryResult, BrainyEvent } from '../core/types.js' -import { AugmentationManifest } from '../../augmentations/manifest.js' /** * Webhook integration configuration @@ -69,7 +68,7 @@ interface PendingDelivery { * @example * ```typescript * const webhooks = new WebhookIntegration() - * brain.augmentations.register(webhooks) + * await webhooks.initialize() * * // Register a webhook * await webhooks.register({ @@ -263,7 +262,7 @@ export class WebhookIntegration extends IntegrationBase { /** * Get manifest */ - getManifest(): AugmentationManifest { + getManifest(): Record { return { id: 'webhooks', name: 'Webhooks', diff --git a/src/mcp/mcpAugmentationToolset.ts b/src/mcp/mcpAugmentationToolset.ts index 1541816a..1424c3e1 100644 --- a/src/mcp/mcpAugmentationToolset.ts +++ b/src/mcp/mcpAugmentationToolset.ts @@ -12,10 +12,6 @@ import { MCPTool, MCP_VERSION } from '../types/mcpTypes.js' -import { AugmentationType } from '../types/augmentations.js' - -// Import the augmentation pipeline -import { augmentationPipeline } from '../pipeline.js' export class MCPAugmentationToolset { /** @@ -146,9 +142,8 @@ export class MCPAugmentationToolset { const { args = [], options = {} } = parameters - // Note: This MCP toolset needs to be updated to use the new brain.augmentations API - // For now, return a placeholder response to fix compilation - throw new Error(`MCP toolset requires update to use brain.augmentations API. Method '${method}' not available.`) + // Augmentation pipeline has been removed โ€” operations are called directly on brain + throw new Error(`MCP augmentation toolset is deprecated. Method '${method}' not available. Use brain API directly.`) } /** @@ -157,7 +152,7 @@ export class MCPAugmentationToolset { * @returns Whether the augmentation type is valid */ private isValidAugmentationType(type: string): boolean { - return Object.values(AugmentationType).includes(type as AugmentationType) + return false } /** diff --git a/src/pipeline.ts b/src/pipeline.ts index 09b3f4a6..a2fef01c 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -1,8 +1,7 @@ /** - * Pipeline - Augmentation execution pipeline + * Pipeline - Execution pipeline * * Provides Pipeline class, execution modes, and factory functions. - * All augmentation management should use brain.augmentations API. */ /** @@ -28,7 +27,6 @@ export interface PipelineOptions { /** * Minimal Pipeline class for backward compatibility. - * All augmentation management should use brain.augmentations API. */ export class Pipeline { private static instance?: Pipeline diff --git a/src/shared/default-augmentations.ts b/src/shared/default-augmentations.ts deleted file mode 100644 index efd4e3d9..00000000 --- a/src/shared/default-augmentations.ts +++ /dev/null @@ -1,130 +0,0 @@ -/** - * Default Augmentation Registry - * - * ๐Ÿง โš›๏ธ Pre-installed augmentations that come with every Brainy installation - * These are the core "sensory organs" of the atomic age brain-in-jar system - */ - -import { BrainyInterface } from '../types/brainyInterface.js' - -/** - * Default augmentations that ship with Brainy - * These are automatically registered on startup - */ -export class DefaultAugmentationRegistry { - private brainy: BrainyInterface - - constructor(brainy: BrainyInterface) { - this.brainy = brainy - } - - /** - * Initialize all default augmentations - * Called during Brainy startup to register core functionality - */ - async initializeDefaults(): Promise { - console.log('๐Ÿง โš›๏ธ Initializing default augmentations...') - - // Register Neural Import as default SENSE augmentation - await this.registerNeuralImport() - - console.log('๐Ÿง โš›๏ธ Default augmentations initialized') - } - - /** - * Neural Import - Default SENSE Augmentation - * AI-powered data understanding and entity extraction (always free) - */ - private async registerNeuralImport(): Promise { - try { - // Import the Neural Import augmentation - const { NeuralImportAugmentation } = await import('../augmentations/neuralImport.js') - - // Note: The actual registration is commented out since Brainy doesn't have addAugmentation method yet - // This would create instance with default configuration - /* - const neuralImport = new NeuralImportAugmentation(this.brainy as any, { - confidenceThreshold: 0.7, - enableWeights: true, - skipDuplicates: true - }) - - // Add as SENSE augmentation to Brainy (when method is available) - if (this.brainy.addAugmentation) { - await this.brainy.addAugmentation('SENSE', cortex, { - position: 1, // First in the SENSE pipeline - name: 'cortex', - autoStart: true - }) - } - */ - - console.log('๐Ÿง โš›๏ธ Cortex module loaded (awaiting Brainy augmentation support)') - - } catch (error) { - console.error('โŒ Failed to register Cortex:', error instanceof Error ? error.message : String(error)) - // Don't throw - Brainy should still work without Neural Import - } - } - - /** - * Check if Cortex is available and working - */ - async checkCortexHealth(): Promise<{ - available: boolean - status: string - version?: string - }> { - try { - // Check if Cortex is registered as an augmentation - // Note: hasAugmentation method doesn't exist yet in Brainy - const hasCortex = false // this.brainy.hasAugmentation && this.brainy.hasAugmentation('SENSE', 'cortex') - - return { - available: hasCortex || false, - status: hasCortex ? 'active' : 'not registered (awaiting Brainy support)', - version: '1.0.0' - } - } catch (error) { - return { - available: false, - status: `Error: ${error instanceof Error ? error.message : String(error)}` - } - } - } - - /** - * Reinstall Cortex if it's missing or corrupted - */ - async reinstallCortex(): Promise { - try { - // Remove existing if present - // Note: removeAugmentation method doesn't exist yet in Brainy - /* - if (this.brainy.removeAugmentation) { - try { - await this.brainy.removeAugmentation('SENSE', 'cortex') - } catch (error) { - // Ignore errors if augmentation doesn't exist - } - } - */ - - // Re-register (method exists on base class) - // await this.registerCortex() - - console.log('๐Ÿง โš›๏ธ Cortex reinstalled successfully') - } catch (error) { - throw new Error(`Failed to reinstall Cortex: ${error instanceof Error ? error.message : String(error)}`) - } - } -} - -/** - * Helper function to initialize default augmentations for any Brainy instance - */ -export async function initializeDefaultAugmentations(brainy: BrainyInterface): Promise { - const registry = new DefaultAugmentationRegistry(brainy) - await registry.initializeDefaults() - return registry -} \ No newline at end of file diff --git a/src/types/augmentations.ts b/src/types/augmentations.ts deleted file mode 100644 index ef55e824..00000000 --- a/src/types/augmentations.ts +++ /dev/null @@ -1,220 +0,0 @@ -/** - * Brainy 2.0 Augmentation Types - * - * This file contains only the minimal types needed for augmentations. - * The main augmentation interfaces are now in augmentations/brainyAugmentation.ts - */ - -/** - * WebSocket connection type for conduit augmentations - */ -export type WebSocketConnection = { - connectionId: string - url: string - status: 'connected' | 'disconnected' | 'error' - send?: (data: string | ArrayBufferLike | Blob | ArrayBufferView) => Promise - close?: () => Promise - _streamMessageHandler?: (event: { data: unknown }) => void - _messageHandlerWrapper?: (data: unknown) => void -} - -/** - * Generic augmentation response type - */ -export type AugmentationResponse = { - success: boolean - data: T - error?: string -} - -/** - * Data callback type for subscriptions - */ -export type DataCallback = (data: T) => void - -/** - * Import types for re-export (avoiding circular dependencies) - */ -import type { - BrainyAugmentation as BA, - BaseAugmentation as BaseA, - AugmentationContext as AC -} from '../augmentations/brainyAugmentation.js' - -export type BrainyAugmentation = BA -export type BaseAugmentation = BaseA -export type AugmentationContext = AC - -// REMOVED: Old augmentation type system for 2.0 clean architecture -// All augmentations now use the unified BrainyAugmentation interface from brainyAugmentation.ts - -/** - * @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead - * - * @example - * // โŒ Old way (v2.x): - * import { IAugmentation } from '@soulcraft/brainy/types/augmentations' - * - * // โœ… New way (v3.x): - * import { BrainyAugmentation } from '@soulcraft/brainy' - */ -export type IAugmentation = BrainyAugmentation - -/** - * @deprecated Augmentation types are now unified under BrainyAugmentation interface - * - * @example - * // โŒ Old way (v2.x): - * import { AugmentationType } from '@soulcraft/brainy/types/augmentations' - * if (augmentation.type === AugmentationType.SENSE) { ... } - * - * // โœ… New way (v3.x): - * import { BrainyAugmentation } from '@soulcraft/brainy' - * // Use the unified BrainyAugmentation interface directly - */ -export enum AugmentationType { SENSE = 'sense', CONDUIT = 'conduit', COGNITION = 'cognition', MEMORY = 'memory', PERCEPTION = 'perception', DIALOG = 'dialog', ACTIVATION = 'activation', WEBSOCKET = 'webSocket', SYNAPSE = 'synapse' } - -/** - * @deprecated Use BrainyAugmentation interface directly instead of namespace types - * - * @example - * // โŒ Old way (v2.x): - * import { BrainyAugmentations } from '@soulcraft/brainy/types/augmentations' - * class MyAugmentation implements BrainyAugmentations.ISenseAugmentation { ... } - * - * // โœ… New way (v3.x): - * import { BrainyAugmentation } from '@soulcraft/brainy' - * class MyAugmentation implements BrainyAugmentation { ... } - */ -export namespace BrainyAugmentations { - /** @deprecated Use BrainyAugmentation instead */ - export type ISenseAugmentation = BrainyAugmentation - /** @deprecated Use BrainyAugmentation instead */ - export type IConduitAugmentation = BrainyAugmentation - /** @deprecated Use BrainyAugmentation instead */ - export type ICognitionAugmentation = BrainyAugmentation - /** @deprecated Use BrainyAugmentation instead */ - export type IMemoryAugmentation = BrainyAugmentation - /** @deprecated Use BrainyAugmentation instead */ - export type IPerceptionAugmentation = BrainyAugmentation - /** @deprecated Use BrainyAugmentation instead */ - export type IDialogAugmentation = BrainyAugmentation - /** @deprecated Use BrainyAugmentation instead */ - export type IActivationAugmentation = BrainyAugmentation - /** @deprecated Use BrainyAugmentation instead */ - export type ISynapseAugmentation = BrainyAugmentation -} - -/** - * @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead - * - * @example - * // โŒ Old way (v2.x): - * import { ISenseAugmentation } from '@soulcraft/brainy/types/augmentations' - * class MySense implements ISenseAugmentation { ... } - * - * // โœ… New way (v3.x): - * import { BrainyAugmentation } from '@soulcraft/brainy' - * class MySense implements BrainyAugmentation { ... } - */ -export type ISenseAugmentation = BrainyAugmentation - -/** - * @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead - * - * @example - * // โŒ Old way (v2.x): - * import { IConduitAugmentation } from '@soulcraft/brainy/types/augmentations' - * - * // โœ… New way (v3.x): - * import { BrainyAugmentation } from '@soulcraft/brainy' - */ -export type IConduitAugmentation = BrainyAugmentation - -/** - * @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead - * - * @example - * // โŒ Old way (v2.x): - * import { ICognitionAugmentation } from '@soulcraft/brainy/types/augmentations' - * - * // โœ… New way (v3.x): - * import { BrainyAugmentation } from '@soulcraft/brainy' - */ -export type ICognitionAugmentation = BrainyAugmentation - -/** - * @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead - * - * @example - * // โŒ Old way (v2.x): - * import { IMemoryAugmentation } from '@soulcraft/brainy/types/augmentations' - * - * // โœ… New way (v3.x): - * import { BrainyAugmentation } from '@soulcraft/brainy' - */ -export type IMemoryAugmentation = BrainyAugmentation - -/** - * @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead - * - * @example - * // โŒ Old way (v2.x): - * import { IPerceptionAugmentation } from '@soulcraft/brainy/types/augmentations' - * - * // โœ… New way (v3.x): - * import { BrainyAugmentation } from '@soulcraft/brainy' - */ -export type IPerceptionAugmentation = BrainyAugmentation - -/** - * @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead - * - * @example - * // โŒ Old way (v2.x): - * import { IDialogAugmentation } from '@soulcraft/brainy/types/augmentations' - * - * // โœ… New way (v3.x): - * import { BrainyAugmentation } from '@soulcraft/brainy' - */ -export type IDialogAugmentation = BrainyAugmentation - -/** - * @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead - * - * @example - * // โŒ Old way (v2.x): - * import { IActivationAugmentation } from '@soulcraft/brainy/types/augmentations' - * - * // โœ… New way (v3.x): - * import { BrainyAugmentation } from '@soulcraft/brainy' - */ -export type IActivationAugmentation = BrainyAugmentation - -/** - * @deprecated Use BrainyAugmentation from '../augmentations/brainyAugmentation.js' instead - * - * @example - * // โŒ Old way (v2.x): - * import { ISynapseAugmentation } from '@soulcraft/brainy/types/augmentations' - * - * // โœ… New way (v3.x): - * import { BrainyAugmentation } from '@soulcraft/brainy' - */ -export type ISynapseAugmentation = BrainyAugmentation - -/** - * @deprecated WebSocket support is now built into BrainyAugmentation interface - * - * @example - * // โŒ Old way (v2.x): - * import { IWebSocketSupport } from '@soulcraft/brainy/types/augmentations' - * class MyAugmentation implements IWebSocketSupport { ... } - * - * // โœ… New way (v3.x): - * import { BrainyAugmentation } from '@soulcraft/brainy' - * class MyAugmentation implements BrainyAugmentation { - * // WebSocket functionality is now part of the unified interface - * } - */ -export interface IWebSocketSupport {} \ No newline at end of file diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index 6ff02a74..8b02c845 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -693,9 +693,6 @@ export interface BrainyConfig { ttl?: number } - // Augmentations - augmentations?: Record - // Distributed configuration distributed?: { enabled: boolean diff --git a/src/types/pipelineTypes.ts b/src/types/pipelineTypes.ts deleted file mode 100644 index 9c1755b2..00000000 --- a/src/types/pipelineTypes.ts +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Pipeline Types - * - * This module provides shared types for the pipeline system to avoid circular dependencies. - */ - -import { - BrainyAugmentations, - IWebSocketSupport, - IAugmentation -} from './augmentations.js' - -/** - * Type definitions for the augmentation registry - */ -export type AugmentationRegistry = { - sense: BrainyAugmentations.ISenseAugmentation[]; - conduit: BrainyAugmentations.IConduitAugmentation[]; - cognition: BrainyAugmentations.ICognitionAugmentation[]; - memory: BrainyAugmentations.IMemoryAugmentation[]; - perception: BrainyAugmentations.IPerceptionAugmentation[]; - dialog: BrainyAugmentations.IDialogAugmentation[]; - activation: BrainyAugmentations.IActivationAugmentation[]; - webSocket: IWebSocketSupport[]; -} - -/** - * Interface for the Pipeline class - * This is used to break circular dependencies between pipeline.ts and augmentationRegistry.ts - */ -export interface IPipeline { - register(augmentation: T): IPipeline; -} diff --git a/src/utils/brainyTypes.ts b/src/utils/brainyTypes.ts index 9f68d5a6..7db469bb 100644 --- a/src/utils/brainyTypes.ts +++ b/src/utils/brainyTypes.ts @@ -1,64 +1,34 @@ /** - * BrainyTypes - Complete type management for Brainy - * - * Provides type lists, validation, and intelligent suggestions - * for nouns and verbs using semantic embeddings. - * + * BrainyTypes - Type validation and lookup for Brainy + * + * Provides type lists and validation for the 42 noun types and 127 verb types. + * Source of truth: graphTypes.ts + * * @example * ```typescript * import { BrainyTypes } from '@soulcraft/brainy' - * + * * // Get all available types * const nounTypes = BrainyTypes.nouns // ['Person', 'Organization', ...] * const verbTypes = BrainyTypes.verbs // ['Contains', 'Creates', ...] - * + * * // Validate types * BrainyTypes.isValidNoun('Person') // true * BrainyTypes.isValidVerb('Unknown') // false - * - * // Get intelligent suggestions - * const personData = { - * name: 'John Doe', - * email: 'john@example.com' - * } - * const suggestion = await BrainyTypes.suggestNoun(personData) - * console.log(suggestion.type) // 'Person' - * console.log(suggestion.confidence) // 0.92 * ``` */ import { NounType, VerbType } from '../types/graphTypes.js' -import { BrainyTypes as InternalBrainyTypes, TypeMatchResult } from '../augmentations/typeMatching/brainyTypes.js' /** - * Type suggestion result - */ -export interface TypeSuggestion { - /** The suggested type */ - type: NounType | VerbType - /** Confidence score between 0 and 1 */ - confidence: number - /** Human-readable explanation */ - reason?: string - /** Alternative suggestions */ - alternatives?: Array<{ - type: NounType | VerbType - confidence: number - }> -} - -/** - * BrainyTypes - Complete type management for Brainy - * - * Static class providing type lists, validation, and intelligent suggestions. + * BrainyTypes - Type validation and lookup for Brainy + * + * Static class providing type lists and validation. * No instantiation needed - all methods are static. */ export class BrainyTypes { - private static instance: InternalBrainyTypes | null = null - private static initialized = false - /** - * All available noun types + * All available noun types (42) * @example * ```typescript * BrainyTypes.nouns.forEach(type => console.log(type)) @@ -68,7 +38,7 @@ export class BrainyTypes { static readonly nouns: readonly NounType[] = Object.freeze(Object.values(NounType)) /** - * All available verb types + * All available verb types (127) * @example * ```typescript * BrainyTypes.verbs.forEach(type => console.log(type)) @@ -77,24 +47,12 @@ export class BrainyTypes { */ static readonly verbs: readonly VerbType[] = Object.freeze(Object.values(VerbType)) - /** - * Get or create the internal matcher instance - */ - private static async getInternalMatcher(): Promise { - if (!this.instance) { - this.instance = new InternalBrainyTypes() - await this.instance.init() - this.initialized = true - } - return this.instance - } - /** * Check if a string is a valid noun type - * + * * @param type The type string to check * @returns True if valid noun type - * + * * @example * ```typescript * BrainyTypes.isValidNoun('Person') // true @@ -108,10 +66,10 @@ export class BrainyTypes { /** * Check if a string is a valid verb type - * + * * @param type The type string to check * @returns True if valid verb type - * + * * @example * ```typescript * BrainyTypes.isValidVerb('Contains') // true @@ -123,92 +81,13 @@ export class BrainyTypes { return (this.verbs as readonly string[]).includes(type) } - /** - * Suggest the most appropriate noun type for an object - * - * @param data The object or data to analyze - * @returns Promise resolving to type suggestion with confidence score - * - * @example - * ```typescript - * const data = { - * title: 'Quarterly Report', - * author: 'Jane Smith', - * pages: 42 - * } - * const suggestion = await BrainyTypes.suggestNoun(data) - * console.log(suggestion.type) // 'Document' - * console.log(suggestion.confidence) // 0.88 - * - * // Check alternatives if confidence is low - * if (suggestion.confidence < 0.8) { - * console.log('Also consider:', suggestion.alternatives) - * } - * ``` - */ - static async suggestNoun(data: any): Promise { - const matcher = await this.getInternalMatcher() - const result = await matcher.matchNounType(data) - - return { - type: result.type as NounType, - confidence: result.confidence, - reason: result.reasoning, - alternatives: result.alternatives?.map(alt => ({ - type: alt.type as NounType, - confidence: alt.confidence - })) - } - } - - /** - * Suggest the most appropriate verb type for a relationship - * - * @param source The source entity - * @param target The target entity - * @param hint Optional hint about the relationship - * @returns Promise resolving to type suggestion with confidence score - * - * @example - * ```typescript - * const source = { type: 'Person', name: 'Alice' } - * const target = { type: 'Document', title: 'Research Paper' } - * - * const suggestion = await BrainyTypes.suggestVerb(source, target, 'authored') - * console.log(suggestion.type) // 'CreatedBy' - * console.log(suggestion.confidence) // 0.91 - * - * // Without hint - * const suggestion2 = await BrainyTypes.suggestVerb(source, target) - * console.log(suggestion2.type) // 'RelatedTo' (more generic) - * ``` - */ - static async suggestVerb( - source: any, - target: any, - hint?: string - ): Promise { - const matcher = await this.getInternalMatcher() - const result = await matcher.matchVerbType(source, target, hint) - - return { - type: result.type as VerbType, - confidence: result.confidence, - reason: result.reasoning, - alternatives: result.alternatives?.map(alt => ({ - type: alt.type as VerbType, - confidence: alt.confidence - })) - } - } - /** * Get a noun type by name (with validation) - * + * * @param name The noun type name * @returns The NounType enum value * @throws Error if invalid noun type - * + * * @example * ```typescript * const type = BrainyTypes.getNoun('Person') // NounType.Person @@ -224,11 +103,11 @@ export class BrainyTypes { /** * Get a verb type by name (with validation) - * + * * @param name The verb type name * @returns The VerbType enum value * @throws Error if invalid verb type - * + * * @example * ```typescript * const type = BrainyTypes.getVerb('Contains') // VerbType.Contains @@ -242,26 +121,6 @@ export class BrainyTypes { return name as VerbType } - /** - * Clear the internal cache - * Useful when processing many different types of data - */ - static clearCache(): void { - this.instance?.clearCache() - } - - /** - * Dispose of resources - * Call when completely done using BrainyTypes - */ - static async dispose(): Promise { - if (this.instance) { - await this.instance.dispose() - this.instance = null - this.initialized = false - } - } - /** * Get noun types as a plain object (for iteration) * @returns Object with noun type names as keys @@ -289,38 +148,3 @@ export class BrainyTypes { // Re-export the enums for convenience export { NounType, VerbType } - -/** - * Helper function to validate and suggest types in one call - * - * @example - * ```typescript - * import { suggestType } from '@soulcraft/brainy' - * - * // For nouns - * const nounSuggestion = await suggestType('noun', data) - * - * // For verbs - * const verbSuggestion = await suggestType('verb', source, target) - * ``` - */ -export async function suggestType( - kind: 'noun', - data: any -): Promise -export async function suggestType( - kind: 'verb', - source: any, - target: any, - hint?: string -): Promise -export async function suggestType( - kind: 'noun' | 'verb', - ...args: any[] -): Promise { - if (kind === 'noun') { - return BrainyTypes.suggestNoun(args[0]) - } else { - return BrainyTypes.suggestVerb(args[0], args[1], args[2]) - } -} \ No newline at end of file diff --git a/tests/augmentations/intelligentImport/csvHandler.unit.test.ts b/tests/augmentations/intelligentImport/csvHandler.unit.test.ts deleted file mode 100644 index 6c172f53..00000000 --- a/tests/augmentations/intelligentImport/csvHandler.unit.test.ts +++ /dev/null @@ -1,281 +0,0 @@ -/** - * CSV Handler Tests - * Comprehensive tests for CSV parsing, encoding detection, and type inference - */ - -import { describe, it, expect, beforeAll } from 'vitest' -import { CSVHandler } from '../../../src/augmentations/intelligentImport/handlers/csvHandler.js' -import { promises as fs } from 'fs' -import * as path from 'path' - -describe('CSVHandler', () => { - let handler: CSVHandler - const fixturesPath = path.join(process.cwd(), 'tests/fixtures/import') - - beforeAll(() => { - handler = new CSVHandler() - }) - - describe('canHandle', () => { - it('should handle .csv extension', () => { - expect(handler.canHandle({ filename: 'data.csv' })).toBe(true) - }) - - it('should handle .tsv extension', () => { - expect(handler.canHandle({ filename: 'data.tsv' })).toBe(true) - }) - - it('should handle .txt extension', () => { - expect(handler.canHandle({ filename: 'data.txt' })).toBe(true) - }) - - it('should handle CSV content (comma)', () => { - const content = 'name,age,email\nJohn,30,john@example.com' - expect(handler.canHandle(content)).toBe(true) - }) - - it('should handle CSV content (semicolon)', () => { - const content = 'name;age;email\nJohn;30;john@example.com' - expect(handler.canHandle(content)).toBe(true) - }) - - it('should not handle non-CSV content', () => { - const content = 'This is just plain text without any structure' - expect(handler.canHandle(content)).toBe(false) - }) - - it('should not handle .xlsx extension', () => { - expect(handler.canHandle({ filename: 'data.xlsx' })).toBe(false) - }) - }) - - describe('process - simple CSV', () => { - it('should parse simple comma-delimited CSV', async () => { - const filePath = path.join(fixturesPath, 'simple.csv') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'simple.csv' }) - - expect(result.format).toBe('csv') - expect(result.data).toHaveLength(4) - expect(result.data[0]).toEqual({ - name: 'John Doe', - age: 30, - email: 'john@example.com', - active: true - }) - expect(result.metadata.rowCount).toBe(4) - expect(result.metadata.fields).toEqual(['name', 'age', 'email', 'active']) - expect(result.metadata.delimiter).toBe(',') - expect(result.metadata.hasHeaders).toBe(true) - }) - - it('should infer correct types', async () => { - const filePath = path.join(fixturesPath, 'simple.csv') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'simple.csv' }) - - expect(typeof result.data[0].name).toBe('string') - expect(typeof result.data[0].age).toBe('number') - expect(typeof result.data[0].active).toBe('boolean') - }) - }) - - describe('process - delimiter detection', () => { - it('should detect semicolon delimiter', async () => { - const filePath = path.join(fixturesPath, 'semicolon.csv') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'semicolon.csv' }) - - expect(result.metadata.delimiter).toBe(';') - expect(result.data).toHaveLength(4) - expect(result.data[0]).toEqual({ - product: 'Laptop', - price: 999.99, - quantity: 10, - inStock: true - }) - }) - - it('should detect tab delimiter', async () => { - const filePath = path.join(fixturesPath, 'tab-delimited.csv') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'tab-delimited.csv' }) - - expect(result.metadata.delimiter).toBe('\t') - expect(result.data).toHaveLength(4) - expect(result.data[0].city).toBe('Tokyo') - expect(result.data[0].population).toBe(13960000) - }) - - it('should use specified delimiter when provided', async () => { - const content = 'name|age|email\nJohn|30|john@example.com' - - const result = await handler.process(content, { - csvDelimiter: '|', - filename: 'pipe.csv' - }) - - expect(result.data).toHaveLength(1) - expect(result.data[0].name).toBe('John') - expect(result.data[0].age).toBe(30) - }) - }) - - describe('process - type inference', () => { - it('should infer integer, float, date, and boolean types', async () => { - const filePath = path.join(fixturesPath, 'types.csv') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'types.csv' }) - - expect(result.metadata.types).toBeDefined() - expect(result.metadata.types.id).toBe('integer') - expect(result.metadata.types.name).toBe('string') - // Score is 95 (integer), not 95.0, so it's detected as integer - expect(['integer', 'float']).toContain(result.metadata.types.score) - expect(result.metadata.types.active).toBe('boolean') - - // Check actual values - expect(result.data[0].id).toBe(1) - expect([95, 95.0]).toContain(result.data[0].score) // Can be integer or float - expect(result.data[0].active).toBe(true) - expect(result.data[1].score).toBe(87.5) // This one is definitely a float - }) - - it('should handle null values', async () => { - const content = 'name,age,email\nJohn,30,john@example.com\nJane,,jane@example.com\nBob,45,' - - const result = await handler.process(content, { filename: 'nulls.csv' }) - - expect(result.data[1].age).toBeNull() - expect(result.data[2].email).toBeNull() - }) - }) - - describe('process - encoding detection', () => { - it('should detect UTF-8 encoding', async () => { - const content = 'name,description\nProduct 1,Description with รฉmojis ๐ŸŽ‰' - - const result = await handler.process(content, { filename: 'utf8.csv' }) - - // Encoding can be 'UTF-8', 'utf-8', or 'utf8' depending on detection - expect(result.metadata.encoding.toLowerCase()).toContain('utf') - expect(result.data[0].description).toContain('รฉmojis') - expect(result.data[0].description).toContain('๐ŸŽ‰') - }) - - it('should use specified encoding when provided', async () => { - const content = 'name,age\nJohn,30' - - const result = await handler.process(content, { - encoding: 'ascii', - filename: 'ascii.csv' - }) - - expect(result.metadata.encoding).toBe('ascii') - }) - }) - - describe('process - options', () => { - it('should respect csvHeaders=false option', async () => { - const content = 'John,30,john@example.com\nJane,25,jane@example.com' - - const result = await handler.process(content, { - csvHeaders: false, - filename: 'no-headers.csv' - }) - - expect(result.metadata.hasHeaders).toBe(false) - // csv-parse will generate column names like '0', '1', '2' - expect(Object.keys(result.data[0])).toHaveLength(3) - }) - - it('should respect maxRows option', async () => { - const filePath = path.join(fixturesPath, 'simple.csv') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { - maxRows: 2, - filename: 'simple.csv' - }) - - expect(result.data).toHaveLength(2) - expect(result.metadata.rowCount).toBe(2) - }) - }) - - describe('process - edge cases', () => { - it('should handle empty CSV', async () => { - const content = '' - - const result = await handler.process(content, { filename: 'empty.csv' }) - - expect(result.data).toHaveLength(0) - expect(result.metadata.rowCount).toBe(0) - expect(result.metadata.fields).toEqual([]) - }) - - it('should handle CSV with only headers', async () => { - const content = 'name,age,email' - - const result = await handler.process(content, { filename: 'headers-only.csv' }) - - expect(result.data).toHaveLength(0) - expect(result.metadata.rowCount).toBe(0) - }) - - it('should handle CSV with single row', async () => { - const content = 'name,age,email\nJohn,30,john@example.com' - - const result = await handler.process(content, { filename: 'single-row.csv' }) - - expect(result.data).toHaveLength(1) - expect(result.data[0].name).toBe('John') - }) - - it('should handle CSV with varying column counts', async () => { - const content = 'name,age,email\nJohn,30,john@example.com\nJane,25\nBob,45,bob@example.com,extra' - - const result = await handler.process(content, { filename: 'varying.csv' }) - - // csv-parse with relax_column_count should handle this - expect(result.data).toHaveLength(3) - }) - - it('should trim whitespace', async () => { - const content = 'name, age , email \n John , 30 , john@example.com ' - - const result = await handler.process(content, { filename: 'whitespace.csv' }) - - expect(result.data[0].name).toBe('John') - expect(result.data[0].age).toBe(30) - expect(result.data[0].email).toBe('john@example.com') - }) - }) - - describe('process - performance', () => { - it('should measure processing time', async () => { - const filePath = path.join(fixturesPath, 'simple.csv') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'simple.csv' }) - - expect(result.metadata.processingTime).toBeGreaterThanOrEqual(0) - expect(typeof result.metadata.processingTime).toBe('number') - }) - }) - - describe('process - error handling', () => { - it('should throw error for malformed CSV', async () => { - const content = 'name,age,email\nJohn,30,"unclosed quote' - - await expect( - handler.process(content, { filename: 'malformed.csv' }) - ).rejects.toThrow('CSV parsing failed') - }) - }) -}) diff --git a/tests/augmentations/intelligentImport/excelHandler.unit.test.ts b/tests/augmentations/intelligentImport/excelHandler.unit.test.ts deleted file mode 100644 index 40128a2a..00000000 --- a/tests/augmentations/intelligentImport/excelHandler.unit.test.ts +++ /dev/null @@ -1,269 +0,0 @@ -/** - * Excel Handler Tests - * Comprehensive tests for Excel parsing, multi-sheet extraction, and type inference - */ - -import { describe, it, expect, beforeAll } from 'vitest' -import { ExcelHandler } from '../../../src/augmentations/intelligentImport/handlers/excelHandler.js' -import { promises as fs } from 'fs' -import * as path from 'path' - -describe('ExcelHandler', () => { - let handler: ExcelHandler - const fixturesPath = path.join(process.cwd(), 'tests/fixtures/import') - - beforeAll(() => { - handler = new ExcelHandler() - }) - - describe('canHandle', () => { - it('should handle .xlsx extension', () => { - expect(handler.canHandle({ filename: 'data.xlsx' })).toBe(true) - }) - - it('should handle .xls extension', () => { - expect(handler.canHandle({ filename: 'data.xls' })).toBe(true) - }) - - it('should handle .xlsb extension', () => { - expect(handler.canHandle({ filename: 'data.xlsb' })).toBe(true) - }) - - it('should handle .xlsm extension', () => { - expect(handler.canHandle({ filename: 'data.xlsm' })).toBe(true) - }) - - it('should not handle .csv extension', () => { - expect(handler.canHandle({ filename: 'data.csv' })).toBe(false) - }) - - it('should not handle .pdf extension', () => { - expect(handler.canHandle({ filename: 'data.pdf' })).toBe(false) - }) - }) - - describe('process - simple Excel', () => { - it('should parse simple single-sheet Excel file', async () => { - const filePath = path.join(fixturesPath, 'simple.xlsx') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'simple.xlsx' }) - - expect(result.format).toBe('excel') - expect(result.data).toHaveLength(4) - expect(result.data[0]).toMatchObject({ - Name: 'Alice Johnson', - Age: 28, - Department: 'Engineering', - Salary: 95000, - Active: true - }) - expect(result.metadata.rowCount).toBe(4) - expect(result.metadata.sheetCount).toBe(1) - expect(result.metadata.sheets).toEqual(['Employees']) - }) - - it('should infer correct types from Excel data', async () => { - const filePath = path.join(fixturesPath, 'simple.xlsx') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'simple.xlsx' }) - - expect(result.metadata.types).toBeDefined() - expect(result.metadata.types.Name).toBe('string') - expect(result.metadata.types.Age).toBe('integer') - expect(['integer', 'float']).toContain(result.metadata.types.Salary) - expect(result.metadata.types.Active).toBe('boolean') - }) - }) - - describe('process - multi-sheet Excel', () => { - it('should extract data from all sheets by default', async () => { - const filePath = path.join(fixturesPath, 'multi-sheet.xlsx') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'multi-sheet.xlsx' }) - - expect(result.metadata.sheetCount).toBe(3) - expect(result.metadata.sheets).toEqual(['Products', 'Orders', 'Customers']) - - // Total rows from all sheets - const totalRows = 4 + 3 + 3 // Products + Orders + Customers - expect(result.data).toHaveLength(totalRows) - - // Check _sheet field is added - expect(result.data[0]._sheet).toBe('Products') - }) - - it('should filter data by sheet name', async () => { - const filePath = path.join(fixturesPath, 'multi-sheet.xlsx') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { - filename: 'multi-sheet.xlsx', - excelSheets: ['Products'] - }) - - expect(result.metadata.sheetCount).toBe(1) - expect(result.metadata.sheets).toEqual(['Products']) - expect(result.data).toHaveLength(4) - expect(result.data.every(row => row._sheet === 'Products')).toBe(true) - }) - - it('should extract data from multiple specified sheets', async () => { - const filePath = path.join(fixturesPath, 'multi-sheet.xlsx') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { - filename: 'multi-sheet.xlsx', - excelSheets: ['Products', 'Customers'] - }) - - expect(result.metadata.sheetCount).toBe(2) - expect(result.metadata.sheets).toEqual(['Products', 'Customers']) - expect(result.data).toHaveLength(7) // 4 + 3 - - const sheets = new Set(result.data.map(row => row._sheet)) - expect(sheets.has('Products')).toBe(true) - expect(sheets.has('Customers')).toBe(true) - expect(sheets.has('Orders')).toBe(false) - }) - - it('should include sheet metadata for each sheet', async () => { - const filePath = path.join(fixturesPath, 'multi-sheet.xlsx') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'multi-sheet.xlsx' }) - - expect(result.metadata.sheetMetadata).toBeDefined() - expect(result.metadata.sheetMetadata.Products).toMatchObject({ - rowCount: 4, - columnCount: 5 - }) - expect(result.metadata.sheetMetadata.Products.headers).toEqual([ - 'ID', 'Product', 'Price', 'Stock', 'Category' - ]) - }) - }) - - describe('process - type inference', () => { - it('should infer integer, float, date, and boolean types', async () => { - const filePath = path.join(fixturesPath, 'types.xlsx') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'types.xlsx' }) - - expect(result.metadata.types).toBeDefined() - expect(result.metadata.types.ID).toBe('integer') - expect(result.metadata.types.Name).toBe('string') - expect(['integer', 'float']).toContain(result.metadata.types.Score) - expect(['integer', 'float']).toContain(result.metadata.types.Percentage) - expect(result.metadata.types.Active).toBe('boolean') - - // Check actual values - expect(result.data[0].ID).toBe(1) - expect(result.data[0].Active).toBe(true) - expect(typeof result.data[0].Score).toBe('number') - }) - - it('should handle null/empty values', async () => { - const filePath = path.join(fixturesPath, 'empty.xlsx') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'empty.xlsx' }) - - expect(result.data).toHaveLength(0) - expect(result.metadata.rowCount).toBe(0) - }) - }) - - describe('process - workbook metadata', () => { - it('should extract workbook information', async () => { - const filePath = path.join(fixturesPath, 'multi-sheet.xlsx') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'multi-sheet.xlsx' }) - - expect(result.metadata.workbookInfo).toBeDefined() - expect(result.metadata.workbookInfo.sheetNames).toEqual([ - 'Products', 'Orders', 'Customers' - ]) - }) - }) - - describe('process - edge cases', () => { - it('should handle Excel file with only headers', async () => { - const filePath = path.join(fixturesPath, 'empty.xlsx') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'empty.xlsx' }) - - expect(result.data).toHaveLength(0) - expect(result.metadata.sheetCount).toBe(1) - }) - - it('should skip non-existent sheets when filtering', async () => { - const filePath = path.join(fixturesPath, 'simple.xlsx') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { - filename: 'simple.xlsx', - excelSheets: ['Employees', 'NonExistent'] - }) - - expect(result.metadata.sheets).toEqual(['Employees']) - expect(result.data).toHaveLength(4) - }) - }) - - describe('process - performance', () => { - it('should measure processing time', async () => { - const filePath = path.join(fixturesPath, 'simple.xlsx') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'simple.xlsx' }) - - expect(result.metadata.processingTime).toBeGreaterThanOrEqual(0) - expect(typeof result.metadata.processingTime).toBe('number') - }) - }) - - describe('process - field sanitization', () => { - it('should sanitize column names', async () => { - const filePath = path.join(fixturesPath, 'simple.xlsx') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'simple.xlsx' }) - - // All field names should be valid identifiers - const fields = Object.keys(result.data[0]).filter(k => k !== '_sheet') - for (const field of fields) { - expect(field).toMatch(/^[a-zA-Z_][a-zA-Z0-9_]*$/) - } - }) - }) - - describe('process - error handling', () => { - it('should handle corrupted Excel file gracefully', async () => { - // XLSX library is very forgiving, so we test that it returns empty data - // rather than crashing - const invalidData = Buffer.from('This is not an Excel file') - - const result = await handler.process(invalidData, { filename: 'invalid.xlsx' }) - - // Should return empty data instead of crashing - expect(result.data).toHaveLength(0) - expect(result.metadata.rowCount).toBe(0) - }) - - it('should handle binary garbage', async () => { - const garbage = Buffer.from(new Uint8Array(256).fill(255)) - - const result = await handler.process(garbage, { filename: 'garbage.xlsx' }) - - // Should not crash - expect(result).toBeDefined() - expect(result.format).toBe('excel') - }) - }) -}) diff --git a/tests/augmentations/intelligentImport/pdfHandler.unit.test.ts b/tests/augmentations/intelligentImport/pdfHandler.unit.test.ts deleted file mode 100644 index a0682540..00000000 --- a/tests/augmentations/intelligentImport/pdfHandler.unit.test.ts +++ /dev/null @@ -1,292 +0,0 @@ -/** - * PDF Handler Tests - * Comprehensive tests for PDF text extraction, table detection, and metadata - */ - -import { describe, it, expect, beforeAll } from 'vitest' -import { PDFHandler } from '../../../src/augmentations/intelligentImport/handlers/pdfHandler.js' -import { promises as fs } from 'fs' -import * as path from 'path' - -describe('PDFHandler', () => { - let handler: PDFHandler - const fixturesPath = path.join(process.cwd(), 'tests/fixtures/import') - - beforeAll(() => { - handler = new PDFHandler() - }) - - describe('canHandle', () => { - it('should handle .pdf extension', () => { - expect(handler.canHandle({ filename: 'document.pdf' })).toBe(true) - }) - - it('should handle PDF magic bytes', () => { - const pdfBuffer = Buffer.from('%PDF-1.4\n') - expect(handler.canHandle(pdfBuffer)).toBe(true) - }) - - it('should not handle .xlsx extension', () => { - expect(handler.canHandle({ filename: 'data.xlsx' })).toBe(false) - }) - - it('should not handle .csv extension', () => { - expect(handler.canHandle({ filename: 'data.csv' })).toBe(false) - }) - - it('should not handle non-PDF buffer', () => { - const buffer = Buffer.from('This is not a PDF') - expect(handler.canHandle(buffer)).toBe(false) - }) - }) - - describe('process - simple text PDF', () => { - it('should extract text from simple PDF', async () => { - const filePath = path.join(fixturesPath, 'simple.pdf') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'simple.pdf' }) - - expect(result.format).toBe('pdf') - expect(result.data.length).toBeGreaterThan(0) - - // Should have extracted paragraphs - const paragraphs = result.data.filter(item => item._type === 'paragraph') - expect(paragraphs.length).toBeGreaterThan(0) - - // Check that text was extracted - const hasText = paragraphs.some(p => p.text && p.text.length > 0) - expect(hasText).toBe(true) - }) - - it('should include page numbers', async () => { - const filePath = path.join(fixturesPath, 'simple.pdf') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'simple.pdf' }) - - // All items should have _page field - expect(result.data.every(item => typeof item._page === 'number')).toBe(true) - expect(result.data.every(item => item._page >= 1)).toBe(true) - }) - - it('should count pages correctly', async () => { - const filePath = path.join(fixturesPath, 'simple.pdf') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'simple.pdf' }) - - expect(result.metadata.pageCount).toBe(1) - }) - }) - - describe('process - multi-page PDF', () => { - it('should extract text from all pages', async () => { - const filePath = path.join(fixturesPath, 'multi-page.pdf') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'multi-page.pdf' }) - - expect(result.metadata.pageCount).toBe(3) - - // Should have content from multiple pages - const pages = new Set(result.data.map(item => item._page)) - expect(pages.size).toBeGreaterThan(1) - expect(pages.has(1)).toBe(true) - expect(pages.has(2)).toBe(true) - expect(pages.has(3)).toBe(true) - }) - - it('should preserve page order', async () => { - const filePath = path.join(fixturesPath, 'multi-page.pdf') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'multi-page.pdf' }) - - // Page numbers should be in order - let lastPage = 0 - let pageChanged = false - - for (const item of result.data) { - if (item._page !== lastPage && lastPage > 0) { - pageChanged = true - } - expect(item._page).toBeGreaterThanOrEqual(lastPage) - lastPage = item._page - } - - expect(pageChanged).toBe(true) // Should have moved through pages - }) - }) - - describe('process - table detection', () => { - it('should detect tables in PDF', async () => { - const filePath = path.join(fixturesPath, 'table.pdf') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'table.pdf' }) - - // Should have detected at least one table - expect(result.metadata.tableCount).toBeGreaterThan(0) - }) - - it('should extract table rows when tables detected', async () => { - const filePath = path.join(fixturesPath, 'table.pdf') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { - filename: 'table.pdf', - pdfExtractTables: true - }) - - // Should have table_row items - const tableRows = result.data.filter(item => item._type === 'table_row') - expect(tableRows.length).toBeGreaterThan(0) - - // Table rows should have structured data - if (tableRows.length > 0) { - const firstRow = tableRows[0] - const fields = Object.keys(firstRow).filter(k => !k.startsWith('_')) - expect(fields.length).toBeGreaterThan(0) - } - }) - - it('should skip table extraction when disabled', async () => { - const filePath = path.join(fixturesPath, 'table.pdf') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { - filename: 'table.pdf', - pdfExtractTables: false - }) - - // Should not have detected tables - expect(result.metadata.tableCount).toBe(0) - - // Should not have table_row items - const tableRows = result.data.filter(item => item._type === 'table_row') - expect(tableRows.length).toBe(0) - }) - }) - - describe('process - metadata extraction', () => { - it('should extract PDF metadata', async () => { - const filePath = path.join(fixturesPath, 'metadata.pdf') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'metadata.pdf' }) - - expect(result.metadata.pdfMetadata).toBeDefined() - expect(result.metadata.pdfMetadata.title).toBe('Test Document') - expect(result.metadata.pdfMetadata.author).toBe('Test Author') - expect(result.metadata.pdfMetadata.creator).toBe('Brainy Test Suite') - }) - - it('should handle PDFs without metadata gracefully', async () => { - const filePath = path.join(fixturesPath, 'simple.pdf') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'simple.pdf' }) - - expect(result.metadata.pdfMetadata).toBeDefined() - // Some fields may be null - expect(result.metadata.pdfMetadata).toHaveProperty('title') - expect(result.metadata.pdfMetadata).toHaveProperty('author') - }) - }) - - describe('process - text statistics', () => { - it('should track total text length', async () => { - const filePath = path.join(fixturesPath, 'simple.pdf') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'simple.pdf' }) - - expect(result.metadata.textLength).toBeGreaterThan(0) - expect(typeof result.metadata.textLength).toBe('number') - }) - - it('should count extracted items', async () => { - const filePath = path.join(fixturesPath, 'simple.pdf') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'simple.pdf' }) - - expect(result.metadata.rowCount).toBe(result.data.length) - expect(result.metadata.rowCount).toBeGreaterThan(0) - }) - }) - - describe('process - edge cases', () => { - it('should handle empty PDF', async () => { - const filePath = path.join(fixturesPath, 'empty.pdf') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'empty.pdf' }) - - expect(result.format).toBe('pdf') - expect(result.metadata.pageCount).toBe(1) - // Empty PDF may have 0 or minimal content - expect(result.data).toBeDefined() - }) - - it('should measure processing time', async () => { - const filePath = path.join(fixturesPath, 'simple.pdf') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'simple.pdf' }) - - expect(result.metadata.processingTime).toBeGreaterThan(0) - expect(typeof result.metadata.processingTime).toBe('number') - }) - }) - - describe('process - data structure', () => { - it('should include type indicators', async () => { - const filePath = path.join(fixturesPath, 'simple.pdf') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'simple.pdf' }) - - // All items should have _type field - expect(result.data.every(item => '_type' in item)).toBe(true) - - // Types should be valid - const validTypes = ['paragraph', 'table_row'] - expect(result.data.every(item => validTypes.includes(item._type))).toBe(true) - }) - - it('should include index for paragraphs', async () => { - const filePath = path.join(fixturesPath, 'simple.pdf') - const buffer = await fs.readFile(filePath) - - const result = await handler.process(buffer, { filename: 'simple.pdf' }) - - const paragraphs = result.data.filter(item => item._type === 'paragraph') - - // Paragraphs should have _index field - expect(paragraphs.every(p => typeof p._index === 'number')).toBe(true) - }) - }) - - describe('process - error handling', () => { - it('should throw error for invalid PDF', async () => { - const invalidData = Buffer.from('This is not a PDF file') - - await expect( - handler.process(invalidData, { filename: 'invalid.pdf' }) - ).rejects.toThrow('PDF parsing failed') - }) - - it('should throw error for corrupted PDF', async () => { - const corruptedPDF = Buffer.concat([ - Buffer.from('%PDF-1.4\n'), - Buffer.from('corrupted data that is not valid PDF structure') - ]) - - await expect( - handler.process(corruptedPDF, { filename: 'corrupted.pdf' }) - ).rejects.toThrow('PDF parsing failed') - }) - }) -}) diff --git a/tests/benchmarks/benchmark-real-world.js b/tests/benchmarks/benchmark-real-world.js index f90f02fc..f7ba8392 100644 --- a/tests/benchmarks/benchmark-real-world.js +++ b/tests/benchmarks/benchmark-real-world.js @@ -60,8 +60,7 @@ async function runBrainyBenchmark() { const brain = new Brainy({ storage: { type: 'memory' }, - augmentations: {}, - model: { + model: { type: 'fast', // Using real transformer model precision: 'Q8' // Quantized for speed } diff --git a/tests/benchmarks/benchmark-vs-industry.js b/tests/benchmarks/benchmark-vs-industry.js index d88eaf5a..cff3a2e5 100644 --- a/tests/benchmarks/benchmark-vs-industry.js +++ b/tests/benchmarks/benchmark-vs-industry.js @@ -158,12 +158,6 @@ async function runBrainyBenchmark() { const brain = new Brainy({ storage: { type: 'memory' }, - augmentations: { - cache: false, - metrics: false, - display: false, - index: false - }, embedder: mockEmbedder, warmup: false }) diff --git a/tests/benchmarks/perf-final.js b/tests/benchmarks/perf-final.js index d20ba786..c4a98624 100644 --- a/tests/benchmarks/perf-final.js +++ b/tests/benchmarks/perf-final.js @@ -14,15 +14,8 @@ async function runBenchmark() { console.log('๐Ÿง  Brainy v3 Performance Benchmark') console.log('โ•'.repeat(60)) - // Disable all augmentations for raw performance const brain = new Brainy({ storage: { type: 'memory' }, - augmentations: { - cache: false, - metrics: false, - display: false, - index: false - }, embedder: mockEmbedder, warmup: false }) diff --git a/tests/benchmarks/perf-simple.js b/tests/benchmarks/perf-simple.js index 68dd94c5..5ae946b5 100644 --- a/tests/benchmarks/perf-simple.js +++ b/tests/benchmarks/perf-simple.js @@ -16,7 +16,6 @@ async function benchmark() { const brain = new Brainy({ storage: { type: 'memory' }, - augmentations: {}, embedder: mockEmbedder }) diff --git a/tests/benchmarks/performance-comprehensive.js b/tests/benchmarks/performance-comprehensive.js index f38a0119..1d146d1a 100644 --- a/tests/benchmarks/performance-comprehensive.js +++ b/tests/benchmarks/performance-comprehensive.js @@ -22,7 +22,7 @@ async function runV2Benchmark() { const brain = new Brainy({ storage: { type: 'memory' }, embeddingFunction: mockEmbedder, - augmentations: false // Disable for raw performance + // Raw performance test }) await brain.init() @@ -111,7 +111,6 @@ async function runV3Benchmark() { const brain = new Brainy({ storage: { type: 'memory' }, - augmentations: {}, // Minimal augmentations embedder: mockEmbedder }) @@ -216,7 +215,6 @@ async function runScaleTest() { const brain = new Brainy({ storage: { type: 'memory' }, - augmentations: {}, embedder: mockEmbedder }) diff --git a/tests/benchmarks/performance-profile.js b/tests/benchmarks/performance-profile.js index f1ff2212..388e0015 100644 --- a/tests/benchmarks/performance-profile.js +++ b/tests/benchmarks/performance-profile.js @@ -77,30 +77,21 @@ async function profilePerformance() { // Initialize Brainy with different configurations console.log('Initializing Brainy configurations...') - // 1. Minimal config (no augmentations) + // 1. Minimal config const minimalBrain = new Brainy({ - storage: new MemoryStorage(), - augmentations: false // Disable all augmentations + storage: new MemoryStorage() }) await minimalBrain.init() - - // 2. Default config (with augmentations) + + // 2. Default config const defaultBrain = new Brainy({ storage: new MemoryStorage() }) await defaultBrain.init() - - // 3. Full augmentations config + + // 3. Full config const fullBrain = new Brainy({ - storage: new MemoryStorage(), - augmentations: { - batchProcessing: { enabled: true, maxBatchSize: 1000 }, - connectionPool: { enabled: true }, - cache: { enabled: true }, - index: { enabled: true }, - entityRegistry: { enabled: true }, - monitoring: { enabled: true } - } + storage: new MemoryStorage() }) await fullBrain.init() @@ -188,11 +179,7 @@ async function profilePerformance() { verbCounter++ }, 100) - console.log('\n๐Ÿ”ง Testing Augmentation Overhead\n') - - // Test individual augmentations - const augmentations = defaultBrain.augmentations.getAugmentationTypes() - console.log(`Active augmentations: ${augmentations.join(', ')}\n`) + console.log('\n๐Ÿ”ง Testing Operation Overhead\n') // Measure raw storage performance const storage = new MemoryStorage() @@ -261,8 +248,8 @@ async function profilePerformance() { console.log('Overhead breakdown:') console.log(` Base operation: ${minimalPerf.perOpMs.toFixed(2)}ms`) - console.log(` Default augmentations: +${(defaultPerf.perOpMs - minimalPerf.perOpMs).toFixed(2)}ms`) - console.log(` Full augmentations: +${(fullPerf.perOpMs - defaultPerf.perOpMs).toFixed(2)}ms`) + console.log(` Default config: +${(defaultPerf.perOpMs - minimalPerf.perOpMs).toFixed(2)}ms`) + console.log(` Full config: +${(fullPerf.perOpMs - defaultPerf.perOpMs).toFixed(2)}ms`) console.log(` Embedding generation: ${embedPerf.perOpMs.toFixed(2)}ms`) console.log(` Without embeddings: ${vectorPerf.perOpMs.toFixed(2)}ms`) @@ -290,7 +277,7 @@ async function profilePerformance() { console.log(' 1. Fake/stub operations that returned immediately') console.log(' 2. No actual embedding generation') console.log(' 3. No real storage operations') - console.log(' 4. No augmentation processing') + console.log(' 4. Direct operation calls') console.log('\n With real implementations:') console.log(` - Raw storage: ${profiler.results['Raw storage.saveNoun']?.opsPerSec || 'N/A'} ops/sec`) console.log(` - With embeddings: ${defaultPerf.opsPerSec} ops/sec`) diff --git a/tests/benchmarks/performance-v3.js b/tests/benchmarks/performance-v3.js index cdd3bb51..531943c9 100644 --- a/tests/benchmarks/performance-v3.js +++ b/tests/benchmarks/performance-v3.js @@ -17,7 +17,6 @@ async function benchmarkV2() { const brain = new Brainy({ storage: { type: 'memory' }, - augmentations: false, embeddingFunction: async () => new Array(384).fill(0).map(() => Math.random()) }) await brain.init() @@ -92,7 +91,6 @@ async function benchmarkV3() { const brain = new Brainy({ storage: { type: 'memory' }, - augmentations: {}, warmup: false, embedder: async () => new Array(384).fill(0).map(() => Math.random()) }) diff --git a/tests/benchmarks/quick-benchmark-v3.js b/tests/benchmarks/quick-benchmark-v3.js index 3ee5385f..7f9f8956 100644 --- a/tests/benchmarks/quick-benchmark-v3.js +++ b/tests/benchmarks/quick-benchmark-v3.js @@ -13,7 +13,6 @@ async function testV3() { const brain = new Brainy({ storage: { type: 'memory' }, - augmentations: {}, warmup: false, embedder: async () => new Array(384).fill(0).map(() => Math.random()) }) diff --git a/tests/benchmarks/quick-perf.js b/tests/benchmarks/quick-perf.js index eb20869d..df6fd963 100644 --- a/tests/benchmarks/quick-perf.js +++ b/tests/benchmarks/quick-perf.js @@ -37,7 +37,7 @@ async function quickPerf() { const brain = new Brainy({ storage: new MemoryStorage(), embeddingFunction: mockEmbed, - augmentations: false // Disable augmentations + // Minimal config }) await brain.init() @@ -55,12 +55,11 @@ async function quickPerf() { const brainyOps = Math.round(1000 / ((end2 - start2) / 1000)) console.log(` โœ… Brainy: ${brainyOps.toLocaleString()} ops/sec\n`) - // Test 3: With augmentations - console.log('3๏ธโƒฃ Brainy with Augmentations') + // Test 3: Default config + console.log('3๏ธโƒฃ Brainy with Default Config') const brain2 = new Brainy({ storage: new MemoryStorage(), embeddingFunction: mockEmbed - // Default augmentations enabled }) await brain2.init() @@ -74,13 +73,12 @@ async function quickPerf() { } const end3 = performance.now() const augOps = Math.round(1000 / ((end3 - start3) / 1000)) - console.log(` โœ… With Augmentations: ${augOps.toLocaleString()} ops/sec\n`) + console.log(` โœ… Default Config: ${augOps.toLocaleString()} ops/sec\n`) // Test 4: Real embeddings (the killer) console.log('4๏ธโƒฃ With Real Embeddings (10 samples)') const brain3 = new Brainy({ storage: new MemoryStorage(), - augmentations: false // Uses real embedding function }) await brain3.init() @@ -119,15 +117,10 @@ async function quickPerf() { console.log(' โŒ Embeddings are the primary bottleneck') console.log(' Each embedding takes ~' + Math.round((end4 - start4) / 10) + 'ms') } - if (augOverhead > 50) { - console.log(' โš ๏ธ Augmentations add significant overhead') - } - console.log('\n๐ŸŽฏ The 500,000 ops/sec claim was achievable with:') console.log(' 1. Pre-computed vectors (no embedding)') - console.log(' 2. Minimal augmentations') - console.log(' 3. In-memory storage') - console.log(' 4. Batch operations') + console.log(' 2. In-memory storage') + console.log(' 3. Batch operations') await brain.close() await brain2.close() diff --git a/tests/benchmarks/quick-verify.js b/tests/benchmarks/quick-verify.js index e9810fb1..deecaeaf 100644 --- a/tests/benchmarks/quick-verify.js +++ b/tests/benchmarks/quick-verify.js @@ -69,10 +69,6 @@ async function quickVerify() { console.log(`โŒ Verb storage: Expected 1 verb, got ${stats.totalVerbs}`) } - // Check augmentations - const augTypes = brain.augmentations.getAugmentationTypes() - console.log(`โœ… Active augmentations: ${augTypes.join(', ')}`) - // Close await brain.close() console.log('\nโœ… All tests passed - implementations are REAL!') diff --git a/tests/benchmarks/scale-test.js b/tests/benchmarks/scale-test.js index ec0737f4..65fcad97 100644 --- a/tests/benchmarks/scale-test.js +++ b/tests/benchmarks/scale-test.js @@ -33,34 +33,14 @@ console.log(`๐Ÿ”ง Mode: ${CURRENT_SCALE}\n`) async function runScaleTest() { const startTime = Date.now() - // Initialize Brainy with real augmentations + // Initialize Brainy const brain = new Brainy({ - storage: new MemoryStorage(), - augmentations: { - // These should all be REAL implementations now - batchProcessing: { - enabled: true, - maxBatchSize: BATCH_SIZE, - adaptiveBatching: true - }, - connectionPool: { - enabled: true, - maxConnections: 50, - minConnections: 5 - }, - cache: { - enabled: true, - maxSize: 10000 - }, - index: { - enabled: true - } - } + storage: new MemoryStorage() }) - + await brain.init() - - console.log('โœ… Brainy initialized with real augmentations\n') + + console.log('โœ… Brainy initialized\n') // Test 1: Batch Insert Performance console.log('๐Ÿ“ Test 1: Batch Insert Performance') @@ -160,35 +140,6 @@ async function runScaleTest() { console.log(`โœ… Created ${verbCount.toLocaleString()} relationships in ${verbTime}ms`) console.log(`๐Ÿ“ˆ Rate: ${verbRate.toLocaleString()} relationships/second\n`) - // Test 4: Verify Augmentations Are Real - console.log('๐Ÿ”ง Test 4: Verify Real Implementations') - - // Check BatchProcessing stats - const batchAug = brain.augmentations.getAugmentation('BatchProcessing') - if (batchAug && typeof batchAug.getStats === 'function') { - const stats = batchAug.getStats() - console.log(` BatchProcessing: ${stats.batchesProcessed} batches processed`) - console.log(` Average batch size: ${Math.round(stats.averageBatchSize)}`) - console.log(` Throughput: ${stats.throughputPerSecond} ops/sec`) - } - - // Check ConnectionPool stats - const poolAug = brain.augmentations.getAugmentation('ConnectionPool') - if (poolAug && typeof poolAug.getStats === 'function') { - const stats = poolAug.getStats() - console.log(` ConnectionPool: ${stats.totalConnections} connections`) - console.log(` Pool utilization: ${stats.poolUtilization}`) - console.log(` Total requests: ${stats.totalRequests}`) - } - - // Check Cache stats - const cacheAug = brain.augmentations.getAugmentation('cache') - if (cacheAug && typeof cacheAug.getStats === 'function') { - const stats = cacheAug.getStats() - console.log(` Cache: ${stats.hits} hits, ${stats.misses} misses`) - console.log(` Hit rate: ${Math.round((stats.hits / (stats.hits + stats.misses)) * 100)}%`) - } - // Final Statistics const totalTime = Date.now() - startTime const stats = brain.getStats() diff --git a/tests/brainy-3.test.ts b/tests/brainy-3.test.ts index a735bbc3..a2499181 100644 --- a/tests/brainy-3.test.ts +++ b/tests/brainy-3.test.ts @@ -11,13 +11,7 @@ describe('Brainy 3.0 API', () => { beforeEach(async () => { brain = new Brainy({ - storage: { type: 'memory' }, - augmentations: { - cache: false, - metrics: false, - display: false, - monitoring: false - } + storage: { type: 'memory' } }) await brain.init() }) @@ -426,7 +420,6 @@ describe('Brainy 3.0 Neural API', () => { beforeEach(async () => { brain = new Brainy({ storage: { type: 'memory' }, - augmentations: { metrics: false, display: false } }) await brain.init() }) @@ -440,9 +433,4 @@ describe('Brainy 3.0 Neural API', () => { expect(typeof brain.neural).toBe('object') }) - it('should provide augmentations API access', () => { - expect(brain.augmentations).toBeDefined() - expect(brain.augmentations.list).toBeDefined() - expect(typeof brain.augmentations.list).toBe('function') - }) }) \ No newline at end of file diff --git a/tests/comprehensive/public-api-complete.test.ts b/tests/comprehensive/public-api-complete.test.ts index 5aa00169..fce692e2 100644 --- a/tests/comprehensive/public-api-complete.test.ts +++ b/tests/comprehensive/public-api-complete.test.ts @@ -701,7 +701,6 @@ describe('Brainy Public API - Complete Coverage', () => { expect(health.status).toBe('healthy') expect(health.storage).toBeDefined() expect(health.storage.status).toBe('connected') - expect(health.augmentations).toBeDefined() expect(health.memory).toBeDefined() }) diff --git a/tests/integration/image-import.test.ts b/tests/integration/image-import.test.ts deleted file mode 100644 index 6f6639c0..00000000 --- a/tests/integration/image-import.test.ts +++ /dev/null @@ -1,361 +0,0 @@ -/** - * Image Import Integration Test (v5.2.0) - * - * Tests that ImageHandler works as a built-in handler with IntelligentImportAugmentation - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../src/brainy.js' -import sharp from 'sharp' - -describe('Image Import Integration (v5.2.0)', () => { - let brain: Brainy - - // Create test image - const createTestImage = async (width: number, height: number): Promise => { - const channels = 3 - const pixelData = Buffer.alloc(width * height * channels) - - // Fill with gradient - for (let y = 0; y < height; y++) { - for (let x = 0; x < width; x++) { - const i = (y * width + x) * channels - pixelData[i] = Math.floor((x / width) * 255) - pixelData[i + 1] = Math.floor((y / height) * 255) - pixelData[i + 2] = 128 - } - } - - return sharp(pixelData, { - raw: { width, height, channels } - }) - .jpeg({ quality: 90 }) - .toBuffer() - } - - beforeEach(async () => { - // IntelligentImportAugmentation is enabled by default with all handlers - // Configure it to only enable image handler for focused testing - brain = new Brainy({ - silent: true, - augmentations: { - intelligentImport: { - enableCSV: false, - enableExcel: false, - enablePDF: false, - enableImage: true // Only enable image handler for this test - } - } - }) - - await brain.init() - }) - - afterEach(async () => { - await brain.close() - }) - - describe('Built-in Image Handler', () => { - it('should automatically handle image import via IntelligentImportAugmentation', async () => { - const imageBuffer = await createTestImage(800, 600) - - // Import image - should be automatically processed by ImageHandler - const result = await brain.import({ - type: 'buffer', - data: imageBuffer, - filename: 'test-photo.jpg' - }) - - expect(result).toBeDefined() - expect(result.entities).toHaveLength(1) - - const imageEntity = result.entities[0] - expect(imageEntity.type).toBe('media') - expect(imageEntity.metadata?.subtype).toBe('image') - expect(imageEntity.metadata).toBeDefined() - }) - - it('should extract image metadata via built-in handler', async () => { - const imageBuffer = await createTestImage(1920, 1080) - - const result = await brain.import({ - type: 'buffer', - data: imageBuffer, - filename: 'hd-photo.jpg' - }) - - const imageEntity = result.entities[0] - expect(imageEntity).toBeDefined() - expect(imageEntity.metadata).toBeDefined() - expect(typeof imageEntity.metadata).toBe('object') - expect(imageEntity.metadata.width).toBe(1920) - expect(imageEntity.metadata.height).toBe(1080) - expect(imageEntity.metadata.format).toBe('jpeg') - }) - - it('should extract EXIF data when available', async () => { - const imageBuffer = await createTestImage(400, 300) - - const result = await brain.import({ - type: 'buffer', - data: imageBuffer, - filename: 'camera-photo.jpg', - options: { - extractEXIF: true - } - }) - - const imageEntity = result.entities[0] - // Test image won't have EXIF, but should not crash - expect(imageEntity).toBeDefined() - expect(imageEntity.type).toBe('media') - expect(imageEntity.metadata?.subtype).toBe('image') - }) - - it('should allow disabling EXIF extraction via config', async () => { - const imageBuffer = await createTestImage(400, 300) - - const result = await brain.import({ - type: 'buffer', - data: imageBuffer, - filename: 'no-exif.jpg', - options: { - extractEXIF: false - } - }) - - expect(result.entities).toHaveLength(1) - expect(result.entities[0].type).toBe('media') - expect(result.entities[0].metadata?.subtype).toBe('image') - }) - - it('should handle PNG images', async () => { - const pngBuffer = await sharp(Buffer.alloc(100 * 100 * 4), { - raw: { width: 100, height: 100, channels: 4 } - }) - .png() - .toBuffer() - - const result = await brain.import({ - type: 'buffer', - data: pngBuffer, - filename: 'logo.png' - }) - - const imageEntity = result.entities[0] - expect(imageEntity.metadata.format).toBe('png') - expect(imageEntity.metadata.hasAlpha).toBe(true) - }) - - it('should handle WebP images', async () => { - const webpBuffer = await sharp(Buffer.alloc(200 * 200 * 3), { - raw: { width: 200, height: 200, channels: 3 } - }) - .webp({ quality: 90 }) - .toBuffer() - - const result = await brain.import({ - type: 'buffer', - data: webpBuffer, - filename: 'modern.webp' - }) - - const imageEntity = result.entities[0] - expect(imageEntity.metadata.format).toBe('webp') - }) - }) - - describe('Configuration', () => { - it('should allow disabling image handler', async () => { - // Create new brain with image handler disabled - const brainNoImage = new Brainy({ - silent: true, - augmentations: { - intelligentImport: { - enableImage: false - } - } - }) - await brainNoImage.init() - - const imageBuffer = await createTestImage(400, 300) - - // Import should still work, but won't be processed by ImageHandler - const result = await brainNoImage.import({ - type: 'buffer', - data: imageBuffer, - filename: 'unprocessed.jpg' - }) - - // Should create entity but not extract image metadata - expect(result).toBeDefined() - - await brainNoImage.close() - }) - - it('should apply imageDefaults config', async () => { - const brainWithDefaults = new Brainy({ - silent: true, - augmentations: { - intelligentImport: { - enableImage: true, - imageDefaults: { - extractEXIF: false // Default to no EXIF extraction - } - } - } - }) - await brainWithDefaults.init() - - const imageBuffer = await createTestImage(400, 300) - - const result = await brainWithDefaults.import({ - type: 'buffer', - data: imageBuffer, - filename: 'default-config.jpg' - }) - - expect(result.entities[0].type).toBe('media') - expect(result.entities[0].metadata?.subtype).toBe('image') - - await brainWithDefaults.close() - }) - }) - - describe('Image Format Detection', () => { - it('should detect JPEG by filename', async () => { - const imageBuffer = await createTestImage(400, 300) - - const result = await brain.import({ - type: 'buffer', - data: imageBuffer, - filename: 'photo.jpg' - }) - - expect(result.entities[0].metadata.format).toBe('jpeg') - }) - - it('should detect JPEG by .jpeg extension', async () => { - const imageBuffer = await createTestImage(400, 300) - - const result = await brain.import({ - type: 'buffer', - data: imageBuffer, - filename: 'photo.jpeg' - }) - - expect(result.entities[0].metadata.format).toBe('jpeg') - }) - - it('should handle various image sizes', async () => { - const sizes = [ - [100, 100], - [800, 600], - [1920, 1080], - [4000, 3000] - ] - - for (const [width, height] of sizes) { - const imageBuffer = await createTestImage(width, height) - - const result = await brain.import({ - type: 'buffer', - data: imageBuffer, - filename: `image-${width}x${height}.jpg` - }) - - const imageEntity = result.entities[0] - expect(imageEntity.metadata.width).toBe(width) - expect(imageEntity.metadata.height).toBe(height) - } - }) - }) - - describe('Error Handling', () => { - it('should handle invalid image data gracefully', async () => { - const invalidBuffer = Buffer.from('not an image') - - // Brainy is resilient - invalid images still import with fallback minimal metadata - const result = await brain.import({ - type: 'buffer', - data: invalidBuffer, - filename: 'invalid.jpg' - }) - - expect(result).toBeDefined() - expect(result.entities).toHaveLength(1) - expect(result.entities[0].type).toBe('media') - expect(result.entities[0].name).toBe('invalid.jpg') - }) - - it('should handle empty image buffer', async () => { - const emptyBuffer = Buffer.alloc(0) - - // Brainy is resilient - empty buffers still import with fallback minimal metadata - const result = await brain.import({ - type: 'buffer', - data: emptyBuffer, - filename: 'empty.jpg' - }) - - expect(result).toBeDefined() - expect(result.entities).toHaveLength(1) - expect(result.entities[0].type).toBe('media') - expect(result.entities[0].name).toBe('empty.jpg') - }) - }) - - describe('Integration with Knowledge Graph', () => { - it('should store image entities in knowledge graph', async () => { - const imageBuffer = await createTestImage(800, 600) - - const result = await brain.import({ - type: 'buffer', - data: imageBuffer, - filename: 'stored-image.jpg' - }) - - // Verify entity was created with metadata in import result - expect(result.entities).toHaveLength(1) - expect(result.entities[0].metadata.width).toBe(800) - expect(result.entities[0].metadata.height).toBe(600) - - // Query for image entities - verifies it's in the knowledge graph - const images = await brain.find({ type: 'media' }) - expect(images.length).toBeGreaterThan(0) - - // Verify media entities have the expected type - const mediaEntities = images.filter(img => img.type === 'media') - expect(mediaEntities.length).toBeGreaterThan(0) - }) - - it('should support querying by image metadata', async () => { - // Import multiple images and capture metadata from results - const result1 = await brain.import({ - type: 'buffer', - data: await createTestImage(1920, 1080), - filename: 'hd.jpg' - }) - - const result2 = await brain.import({ - type: 'buffer', - data: await createTestImage(800, 600), - filename: 'sd.jpg' - }) - - // Verify metadata in import results - expect(result1.entities[0].metadata.width).toBe(1920) - expect(result1.entities[0].metadata.height).toBe(1080) - expect(result2.entities[0].metadata.width).toBe(800) - expect(result2.entities[0].metadata.height).toBe(600) - - // Query all media entities - verifies they're in the knowledge graph - const allImages = await brain.find({ type: 'media' }) - expect(allImages.length).toBeGreaterThan(0) - - // Verify multiple media entities were stored - const mediaEntities = allImages.filter(img => img.type === 'media') - expect(mediaEntities.length).toBeGreaterThanOrEqual(2) - }) - }) -}) diff --git a/tests/streaming-pipeline.test.ts b/tests/streaming-pipeline.test.ts index da061c59..a719682d 100644 --- a/tests/streaming-pipeline.test.ts +++ b/tests/streaming-pipeline.test.ts @@ -14,7 +14,6 @@ describe('Streaming Pipeline', () => { beforeEach(async () => { brain = new Brainy({ storage: { type: 'memory' }, - augmentations: {}, warmup: false }) await brain.init() diff --git a/tests/unit/augmentations/augmentations-simplified.test.ts b/tests/unit/augmentations/augmentations-simplified.test.ts deleted file mode 100644 index 045a2c62..00000000 --- a/tests/unit/augmentations/augmentations-simplified.test.ts +++ /dev/null @@ -1,535 +0,0 @@ -import { describe, it, expect, beforeEach } from 'vitest' -import { Brainy } from '../../../src/brainy' -import { createAddParams } from '../../helpers/test-factory' -import { NounType, VerbType } from '../../../src/types/graphTypes' - -/** - * Comprehensive test suite for Brainy's built-in augmentation system - * Tests the actual augmentation functionality that's available in production - */ - -describe('Brainy Built-in Augmentations', () => { - let brain: Brainy - - beforeEach(async () => { - brain = new Brainy({ - storage: { type: 'memory' }, - augmentations: { - cache: { enabled: true, maxSize: 1000 }, - display: { enabled: true }, - metrics: { enabled: true } - } - }) - await brain.init() - }) - - describe('1. Augmentation Registry', () => { - it('should list all registered augmentations', async () => { - const augmentations = brain.augmentations.list() - expect(Array.isArray(augmentations)).toBe(true) - expect(augmentations.length).toBeGreaterThan(0) - }) - - it('should have default augmentations', async () => { - const augmentations = brain.augmentations.list() - expect(augmentations).toContain('cache') - expect(augmentations).toContain('display') - expect(augmentations).toContain('metrics') - }) - - it('should get augmentation by name', async () => { - const cacheAug = brain.augmentations.get('cache') - expect(cacheAug).toBeDefined() - expect(cacheAug?.name).toBe('cache') - - const displayAug = brain.augmentations.get('display') - expect(displayAug).toBeDefined() - expect(displayAug?.name).toBe('display') - - const metricsAug = brain.augmentations.get('metrics') - expect(metricsAug).toBeDefined() - expect(metricsAug?.name).toBe('metrics') - }) - - it('should check if augmentation exists', async () => { - expect(brain.augmentations.has('cache')).toBe(true) - expect(brain.augmentations.has('display')).toBe(true) - expect(brain.augmentations.has('metrics')).toBe(true) - expect(brain.augmentations.has('non-existent')).toBe(false) - }) - - it('should return undefined for non-existent augmentation', async () => { - const nonExistent = brain.augmentations.get('does-not-exist') - expect(nonExistent).toBeUndefined() - }) - }) - - describe('2. Cache Augmentation', () => { - it('should cache get operations', async () => { - const id = await brain.add(createAddParams({ - data: 'cached entity', - metadata: { cached: true } - })) - - // First get - should load from storage - const start1 = Date.now() - const entity1 = await brain.get(id) - const duration1 = Date.now() - start1 - - // Second get - should be cached (faster) - const start2 = Date.now() - const entity2 = await brain.get(id) - const duration2 = Date.now() - start2 - - // Compare key properties instead of full object equality - expect(entity1?.id).toBe(entity2?.id) - expect(entity1?.data).toBe(entity2?.data) - expect(entity1?.data).toBe('cached entity') - expect(entity1?.metadata?.cached).toBe(true) - - // Cache should make second call faster (though this might not be reliable) - // expect(duration2).toBeLessThanOrEqual(duration1) - }) - - it('should handle cache misses gracefully', async () => { - // Try to get non-existent entity (v5.1.0: use valid UUID format) - const nonExistent = await brain.get('00000000-0000-0000-0000-000000000099') - expect(nonExistent).toBeNull() - }) - - it('should work with find operations', async () => { - await brain.add(createAddParams({ data: 'findable content 1' })) - await brain.add(createAddParams({ data: 'findable content 2' })) - - // First search - const results1 = await brain.find({ query: 'findable' }) - - // Second search (may be cached) - const results2 = await brain.find({ query: 'findable' }) - - expect(results1.length).toBe(results2.length) - expect(results1.length).toBeGreaterThanOrEqual(2) - }) - }) - - describe('3. Display Augmentation', () => { - it('should provide display functionality for entities', async () => { - const id = await brain.add(createAddParams({ - data: 'Entity with display capabilities', - type: NounType.Document, - metadata: { - title: 'Test Document', - author: 'Test Author', - category: 'test' - } - })) - - const entity = await brain.get(id) - expect(entity).toBeDefined() - - // Check if getDisplay method is available - if (entity && typeof entity.getDisplay === 'function') { - const display = entity.getDisplay() - expect(display).toBeDefined() - expect(typeof display).toBe('object') - } - }) - - it('should work with different entity types', async () => { - const personId = await brain.add(createAddParams({ - data: 'John Doe', - type: NounType.Person, - metadata: { role: 'developer', age: 30 } - })) - - const locationId = await brain.add(createAddParams({ - data: 'San Francisco', - type: NounType.Location, - metadata: { country: 'USA', population: 900000 } - })) - - const person = await brain.get(personId) - const location = await brain.get(locationId) - - expect(person).toBeDefined() - expect(location).toBeDefined() - - // Display should work for all types - if (person && typeof person.getDisplay === 'function') { - const personDisplay = person.getDisplay() - expect(personDisplay).toBeDefined() - } - - if (location && typeof location.getDisplay === 'function') { - const locationDisplay = location.getDisplay() - expect(locationDisplay).toBeDefined() - } - }) - - it('should handle entities without metadata', async () => { - const id = await brain.add(createAddParams({ - data: 'Simple entity without metadata' - })) - - const entity = await brain.get(id) - expect(entity).toBeDefined() - - if (entity && typeof entity.getDisplay === 'function') { - const display = entity.getDisplay() - expect(display).toBeDefined() - } - }) - - it('should provide schema information', async () => { - const id = await brain.add(createAddParams({ - data: 'Entity with schema', - metadata: { - stringField: 'text', - numberField: 42, - booleanField: true, - arrayField: [1, 2, 3] - } - })) - - const entity = await brain.get(id) - expect(entity).toBeDefined() - - if (entity && typeof entity.getSchema === 'function') { - const schema = entity.getSchema() - expect(schema).toBeDefined() - expect(typeof schema).toBe('object') - } - }) - }) - - describe('4. Metrics Augmentation', () => { - it('should track operations without interfering', async () => { - // Perform various operations - const id1 = await brain.add(createAddParams({ data: 'metrics test 1' })) - const id2 = await brain.add(createAddParams({ data: 'metrics test 2' })) - const id3 = await brain.add(createAddParams({ data: 'metrics test 3' })) - - // Read operations - await brain.get(id1) - await brain.get(id2) - await brain.find({ query: 'metrics' }) - - // Write operations - await brain.update({ id: id1, data: 'updated metrics test 1' }) - await brain.delete(id3) - - // Relationship operations - await brain.relate({ - from: id1, - to: id2, - type: VerbType.RelatedTo - }) - - // All operations should complete successfully - const entity1 = await brain.get(id1) - const entity2 = await brain.get(id2) - const entity3 = await brain.get(id3) - - expect(entity1).toBeDefined() - expect(entity1?.data).toBe('updated metrics test 1') - expect(entity2).toBeDefined() - expect(entity3).toBeNull() // Deleted - - // Check relationships (may be empty if relationship storage isn't implemented) - const relations = await brain.getRelations(id1) - expect(Array.isArray(relations)).toBe(true) - }) - - it('should handle high-frequency operations', async () => { - // Create many entities rapidly - const promises = Array.from({ length: 50 }, (_, i) => - brain.add(createAddParams({ - data: `High frequency test ${i}`, - metadata: { index: i } - })) - ) - - const ids = await Promise.all(promises) - expect(ids.length).toBe(50) - - // Perform many reads - const readPromises = ids.map(id => brain.get(id)) - const entities = await Promise.all(readPromises) - - expect(entities.length).toBe(50) - expect(entities.every(e => e !== null)).toBe(true) - }) - - it('should handle error scenarios gracefully', async () => { - // v5.1.0: Invalid UUID formats throw errors - await expect(brain.get('invalid-id')).rejects.toThrow('Invalid UUID format') - await expect(brain.delete('invalid-id')).rejects.toThrow('Invalid UUID format') - - // Try invalid find parameters - await expect(brain.find({ - query: 'test', - limit: -1 - } as any)).rejects.toThrow() - - await expect(brain.find({ - query: 'test', - offset: -1 - } as any)).rejects.toThrow() - }) - }) - - describe('5. Augmentation Integration', () => { - it('should apply all augmentations to add operations', async () => { - const id = await brain.add(createAddParams({ - data: 'Full integration test', - type: NounType.Document, - metadata: { - title: 'Integration Test', - priority: 'high', - tags: ['test', 'integration'] - } - })) - - expect(id).toBeDefined() - expect(typeof id).toBe('string') - - // Verify entity was created correctly - const entity = await brain.get(id) - expect(entity).toBeDefined() - expect(entity?.data).toBe('Full integration test') - expect(entity?.type).toBe(NounType.Document) - expect(entity?.metadata?.title).toBe('Integration Test') - }) - - it('should apply all augmentations to find operations', async () => { - // Add test data - const id1 = await brain.add(createAddParams({ - data: 'Integration search test 1', - metadata: { category: 'integration' } - })) - - const id2 = await brain.add(createAddParams({ - data: 'Integration search test 2', - metadata: { category: 'integration' } - })) - - await brain.add(createAddParams({ - data: 'Different content', - metadata: { category: 'other' } - })) - - // Test text search - const textResults = await brain.find({ query: 'Integration search' }) - expect(textResults.length).toBeGreaterThanOrEqual(2) - - // Test metadata filtering - const categoryResults = await brain.find({ - query: '', - where: { category: 'integration' } - }) - expect(categoryResults.length).toBe(2) - - // Verify entities have display capabilities - const firstResult = textResults[0] - if (firstResult?.entity && typeof firstResult.entity.getDisplay === 'function') { - const display = firstResult.entity.getDisplay() - expect(display).toBeDefined() - } - }) - - it('should apply augmentations to update operations', async () => { - const id = await brain.add(createAddParams({ - data: 'Original data', - metadata: { version: 1 } - })) - - // Update should work with all augmentations - await brain.update({ - id, - data: 'Updated data', - metadata: { version: 2, updated: true } - }) - - const updatedEntity = await brain.get(id) - expect(updatedEntity?.data).toBe('Updated data') - expect(updatedEntity?.metadata?.version).toBe(2) - expect(updatedEntity?.metadata?.updated).toBe(true) - }) - - it('should apply augmentations to relationship operations', async () => { - const id1 = await brain.add(createAddParams({ data: 'Entity 1' })) - const id2 = await brain.add(createAddParams({ data: 'Entity 2' })) - - // Create relationship - await brain.relate({ - from: id1, - to: id2, - type: VerbType.RelatedTo, - metadata: { strength: 0.8 } - }) - - // Verify relationship exists (may be empty depending on storage implementation) - const relations = await brain.getRelations(id1) - expect(Array.isArray(relations)).toBe(true) - - // If relationships are stored, verify the properties - if (relations.length > 0) { - const relation = relations.find(r => r.to === id2) - if (relation) { - expect(relation.type).toBe(VerbType.RelatedTo) - expect(relation.metadata?.strength).toBe(0.8) - } - } - }) - }) - - describe('6. Performance with Augmentations', () => { - it('should perform well with many entities', async () => { - const start = Date.now() - - // Create 100 entities - const createPromises = Array.from({ length: 100 }, (_, i) => - brain.add(createAddParams({ - data: `Performance test entity ${i}`, - metadata: { index: i, batch: 'performance' } - })) - ) - - const ids = await Promise.all(createPromises) - const createDuration = Date.now() - start - - expect(ids.length).toBe(100) - expect(createDuration).toBeLessThan(5000) // Should complete in under 5 seconds - - // Search should be fast - const searchStart = Date.now() - const searchResults = await brain.find({ - query: 'Performance test', - limit: 50 - }) - const searchDuration = Date.now() - searchStart - - expect(searchResults.length).toBeGreaterThan(0) - expect(searchResults.length).toBeLessThanOrEqual(50) // May return fewer due to relevance - expect(searchDuration).toBeLessThan(1000) // Should complete in under 1 second - }) - - it('should handle concurrent operations efficiently', async () => { - const start = Date.now() - - // Perform 50 concurrent operations - const operations = Array.from({ length: 50 }, (_, i) => { - if (i % 4 === 0) { - return brain.add(createAddParams({ data: `Concurrent add ${i}` })) - } else if (i % 4 === 1) { - return brain.find({ query: 'concurrent', limit: 5 }) - } else if (i % 4 === 2) { - return brain.add(createAddParams({ data: `Another add ${i}` })) - } else { - return brain.find({ query: 'test', limit: 3 }) - } - }) - - const results = await Promise.all(operations) - const duration = Date.now() - start - - expect(results.length).toBe(50) - expect(duration).toBeLessThan(3000) // Should handle concurrency well - - // Verify some results - const addResults = results.filter(r => typeof r === 'string') - const findResults = results.filter(r => Array.isArray(r)) - - expect(addResults.length).toBeGreaterThan(0) - expect(findResults.length).toBeGreaterThan(0) - }) - }) - - describe('7. Error Handling with Augmentations', () => { - it('should handle errors gracefully without breaking augmentations', async () => { - // v5.1.0: Invalid UUID formats throw errors (these should not crash the system) - await expect(brain.get('')).rejects.toThrow('UUID is required') - await expect(brain.update({ - id: 'non-existent', - data: 'new data' - })).rejects.toThrow('Invalid UUID format') - - // Normal operations should still work - const id = await brain.add(createAddParams({ data: 'After error test' })) - const entity = await brain.get(id) - expect(entity?.data).toBe('After error test') - }) - - it('should handle edge case data gracefully', async () => { - // Add entity with minimal but valid data - const id = await brain.add(createAddParams({ - data: 'minimal', // Valid minimal data - metadata: { test: true } - })) - - expect(id).toBeDefined() - - const entity = await brain.get(id) - expect(entity).toBeDefined() - expect(entity?.data).toBe('minimal') - }) - }) - - describe('8. Configuration and Customization', () => { - it('should respect augmentation configuration', async () => { - // Test that augmentations can be configured - const configuredBrain = new Brainy({ - storage: { type: 'memory' }, - augmentations: { - cache: { - enabled: true, - maxSize: 50, - ttl: 60000 - }, - display: { - enabled: true, - lazyComputation: true - }, - metrics: { - enabled: true - } - } - }) - - await configuredBrain.init() - - // Should work with custom configuration - const id = await configuredBrain.add(createAddParams({ - data: 'Configured test' - })) - - const entity = await configuredBrain.get(id) - expect(entity?.data).toBe('Configured test') - - await configuredBrain.close() - }) - - it('should work with disabled augmentations', async () => { - const minimalBrain = new Brainy({ - storage: { type: 'memory' }, - augmentations: { - cache: { enabled: false }, - display: { enabled: false }, - metrics: { enabled: false } - } - }) - - await minimalBrain.init() - - // Should still work with augmentations disabled - const id = await minimalBrain.add(createAddParams({ - data: 'Minimal test' - })) - - const entity = await minimalBrain.get(id) - expect(entity?.data).toBe('Minimal test') - - await minimalBrain.close() - }) - }) -}) \ No newline at end of file diff --git a/tests/unit/augmentations/format-handler-registry.test.ts b/tests/unit/augmentations/format-handler-registry.test.ts deleted file mode 100644 index 06d6ca4a..00000000 --- a/tests/unit/augmentations/format-handler-registry.test.ts +++ /dev/null @@ -1,437 +0,0 @@ -/** - * FormatHandlerRegistry Tests (v5.2.0) - * - * Tests for MIME-based format handler registration and routing - */ - -import { describe, it, expect, beforeEach } from 'vitest' -import { FormatHandlerRegistry } from '../../../src/augmentations/intelligentImport/FormatHandlerRegistry.js' -import type { FormatHandler, ProcessedData } from '../../../src/augmentations/intelligentImport/types.js' - -describe('FormatHandlerRegistry (v5.2.0)', () => { - let registry: FormatHandlerRegistry - - // Mock handlers for testing - const createMockHandler = (format: string): FormatHandler => ({ - format, - async process(): Promise { - return { - format, - data: [], - metadata: { - rowCount: 0, - fields: [], - processingTime: 0 - } - } - }, - canHandle(): boolean { - return true - } - }) - - beforeEach(() => { - registry = new FormatHandlerRegistry() - }) - - describe('Handler Registration', () => { - it('should register handler with MIME types and extensions', () => { - const csvHandler = createMockHandler('csv') - - registry.registerHandler({ - name: 'csv', - mimeTypes: ['text/csv', 'application/csv'], - extensions: ['.csv', '.tsv'], - loader: async () => csvHandler - }) - - expect(registry.hasHandler('csv')).toBe(true) - expect(registry.getRegisteredHandlers()).toContain('csv') - }) - - it('should register multiple handlers', () => { - registry.registerHandler({ - name: 'csv', - mimeTypes: ['text/csv'], - extensions: ['.csv'], - loader: async () => createMockHandler('csv') - }) - - registry.registerHandler({ - name: 'excel', - mimeTypes: ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'], - extensions: ['.xlsx'], - loader: async () => createMockHandler('excel') - }) - - const handlers = registry.getRegisteredHandlers() - expect(handlers).toContain('csv') - expect(handlers).toContain('excel') - expect(handlers).toHaveLength(2) - }) - - it('should normalize extensions (remove leading dots)', () => { - registry.registerHandler({ - name: 'test', - mimeTypes: [], - extensions: ['.txt', 'md'], // Mixed with/without dots - loader: async () => createMockHandler('test') - }) - - expect(registry.hasHandler('test')).toBe(true) - }) - }) - - describe('MIME Type Routing', () => { - beforeEach(() => { - registry.registerHandler({ - name: 'csv', - mimeTypes: ['text/csv', 'application/csv'], - extensions: ['.csv', '.tsv'], - loader: async () => createMockHandler('csv') - }) - - registry.registerHandler({ - name: 'excel', - mimeTypes: [ - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - 'application/vnd.ms-excel' - ], - extensions: ['.xlsx', '.xls'], - loader: async () => createMockHandler('excel') - }) - - registry.registerHandler({ - name: 'pdf', - mimeTypes: ['application/pdf'], - extensions: ['.pdf'], - loader: async () => createMockHandler('pdf') - }) - }) - - it('should get handler by MIME type', async () => { - const handler = await registry.getHandlerByMimeType('text/csv') - expect(handler).toBeDefined() - expect(handler?.format).toBe('csv') - }) - - it('should get handler by Excel MIME type', async () => { - const handler = await registry.getHandlerByMimeType( - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' - ) - expect(handler).toBeDefined() - expect(handler?.format).toBe('excel') - }) - - it('should get handler by legacy Excel MIME type', async () => { - const handler = await registry.getHandlerByMimeType('application/vnd.ms-excel') - expect(handler).toBeDefined() - expect(handler?.format).toBe('excel') - }) - - it('should return null for unknown MIME type', async () => { - const handler = await registry.getHandlerByMimeType('application/unknown') - expect(handler).toBeNull() - }) - - it('should list handlers for a MIME type', () => { - const handlers = registry.getHandlersForMimeType('text/csv') - expect(handlers).toContain('csv') - }) - }) - - describe('Extension Routing', () => { - beforeEach(() => { - registry.registerHandler({ - name: 'csv', - mimeTypes: ['text/csv'], - extensions: ['.csv', '.tsv'], - loader: async () => createMockHandler('csv') - }) - - registry.registerHandler({ - name: 'excel', - mimeTypes: ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'], - extensions: ['.xlsx', '.xls'], - loader: async () => createMockHandler('excel') - }) - }) - - it('should get handler by extension (with dot)', async () => { - const handler = await registry.getHandlerByExtension('.csv') - expect(handler).toBeDefined() - expect(handler?.format).toBe('csv') - }) - - it('should get handler by extension (without dot)', async () => { - const handler = await registry.getHandlerByExtension('csv') - expect(handler).toBeDefined() - expect(handler?.format).toBe('csv') - }) - - it('should handle case-insensitive extensions', async () => { - const handler = await registry.getHandlerByExtension('.XLSX') - expect(handler).toBeDefined() - expect(handler?.format).toBe('excel') - }) - - it('should return null for unknown extension', async () => { - const handler = await registry.getHandlerByExtension('.xyz') - expect(handler).toBeNull() - }) - }) - - describe('Filename-Based Handler Selection', () => { - beforeEach(() => { - registry.registerHandler({ - name: 'csv', - mimeTypes: ['text/csv'], - extensions: ['.csv'], - loader: async () => createMockHandler('csv') - }) - - registry.registerHandler({ - name: 'excel', - mimeTypes: ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'], - extensions: ['.xlsx'], - loader: async () => createMockHandler('excel') - }) - }) - - it('should get handler by filename (MIME detection)', async () => { - const handler = await registry.getHandler('data.xlsx') - expect(handler).toBeDefined() - expect(handler?.format).toBe('excel') - }) - - it('should get handler by filename (extension fallback)', async () => { - const handler = await registry.getHandler('report.csv') - expect(handler).toBeDefined() - expect(handler?.format).toBe('csv') - }) - - it('should handle full paths', async () => { - const handler = await registry.getHandler('/path/to/file.xlsx') - expect(handler).toBeDefined() - expect(handler?.format).toBe('excel') - }) - - it('should return null for unsupported filename', async () => { - const handler = await registry.getHandler('document.txt') - expect(handler).toBeNull() - }) - }) - - describe('Lazy Loading', () => { - it('should lazy-load handlers on first access', async () => { - let loadCount = 0 - - registry.registerHandler({ - name: 'csv', - mimeTypes: ['text/csv'], - extensions: ['.csv'], - loader: async () => { - loadCount++ - return createMockHandler('csv') - } - }) - - expect(loadCount).toBe(0) - - const handler1 = await registry.getHandlerByName('csv') - expect(loadCount).toBe(1) - expect(handler1?.format).toBe('csv') - - // Second access should use cached instance - const handler2 = await registry.getHandlerByName('csv') - expect(loadCount).toBe(1) // Still 1, not 2 - expect(handler2).toBe(handler1) // Same instance - }) - - it('should cache handler instances', async () => { - registry.registerHandler({ - name: 'csv', - mimeTypes: ['text/csv'], - extensions: ['.csv'], - loader: async () => createMockHandler('csv') - }) - - const handler1 = await registry.getHandlerByExtension('.csv') - const handler2 = await registry.getHandlerByMimeType('text/csv') - - expect(handler2).toBe(handler1) // Same cached instance - }) - - it('should handle loader errors gracefully', async () => { - registry.registerHandler({ - name: 'failing', - mimeTypes: ['application/x-failing'], - extensions: ['.fail'], - loader: async () => { - throw new Error('Handler failed to load') - } - }) - - const handler = await registry.getHandlerByExtension('.fail') - expect(handler).toBeNull() - }) - }) - - describe('Interface Compatibility', () => { - it('should implement HandlerRegistry interface', () => { - // Check required properties exist - expect(registry.handlers).toBeDefined() - expect(registry.loaded).toBeDefined() - expect(typeof registry.register).toBe('function') - expect(typeof registry.getHandler).toBe('function') - }) - - it('should support register() method for backward compatibility', async () => { - const csvHandler = createMockHandler('csv') - - registry.register(['.csv', '.tsv'], async () => csvHandler) - - const handler = await registry.getHandler('.csv') - expect(handler).toBe(csvHandler) - }) - - it('should populate handlers Map for backward compatibility', () => { - registry.registerHandler({ - name: 'csv', - mimeTypes: ['text/csv'], - extensions: ['.csv'], - loader: async () => createMockHandler('csv') - }) - - expect(registry.handlers.has('csv')).toBe(true) - expect(typeof registry.handlers.get('csv')).toBe('function') - }) - - it('should populate loaded Map after loading', async () => { - registry.registerHandler({ - name: 'csv', - mimeTypes: ['text/csv'], - extensions: ['.csv'], - loader: async () => createMockHandler('csv') - }) - - expect(registry.loaded.has('csv')).toBe(false) - - await registry.getHandlerByName('csv') - - expect(registry.loaded.has('csv')).toBe(true) - }) - }) - - describe('Multiple Handlers Per MIME Type', () => { - it('should support multiple handlers for same MIME type', () => { - registry.registerHandler({ - name: 'csv-basic', - mimeTypes: ['text/csv'], - extensions: ['.csv'], - loader: async () => createMockHandler('csv-basic') - }) - - registry.registerHandler({ - name: 'csv-advanced', - mimeTypes: ['text/csv'], - extensions: ['.csv'], - loader: async () => createMockHandler('csv-advanced') - }) - - const handlers = registry.getHandlersForMimeType('text/csv') - expect(handlers).toHaveLength(2) - expect(handlers).toContain('csv-basic') - expect(handlers).toContain('csv-advanced') - }) - - it('should return first matching handler', async () => { - registry.registerHandler({ - name: 'csv-basic', - mimeTypes: ['text/csv'], - extensions: ['.csv'], - loader: async () => createMockHandler('csv-basic') - }) - - registry.registerHandler({ - name: 'csv-advanced', - mimeTypes: ['text/csv'], - extensions: ['.csv'], - loader: async () => createMockHandler('csv-advanced') - }) - - // Should return first registered handler - const handler = await registry.getHandlerByMimeType('text/csv') - expect(handler?.format).toBe('csv-basic') - }) - }) - - describe('Registry Management', () => { - it('should clear all handlers', () => { - registry.registerHandler({ - name: 'csv', - mimeTypes: ['text/csv'], - extensions: ['.csv'], - loader: async () => createMockHandler('csv') - }) - - expect(registry.hasHandler('csv')).toBe(true) - - registry.clear() - - expect(registry.hasHandler('csv')).toBe(false) - expect(registry.getRegisteredHandlers()).toHaveLength(0) - }) - - it('should check if handler is registered', () => { - expect(registry.hasHandler('csv')).toBe(false) - - registry.registerHandler({ - name: 'csv', - mimeTypes: ['text/csv'], - extensions: ['.csv'], - loader: async () => createMockHandler('csv') - }) - - expect(registry.hasHandler('csv')).toBe(true) - expect(registry.hasHandler('excel')).toBe(false) - }) - }) - - describe('Edge Cases', () => { - it('should handle files without extensions', async () => { - registry.registerHandler({ - name: 'csv', - mimeTypes: ['text/csv'], - extensions: ['.csv'], - loader: async () => createMockHandler('csv') - }) - - const handler = await registry.getHandler('data') - expect(handler).toBeNull() - }) - - it('should handle empty extension list', () => { - registry.registerHandler({ - name: 'special', - mimeTypes: ['application/x-special'], - extensions: [], - loader: async () => createMockHandler('special') - }) - - expect(registry.hasHandler('special')).toBe(true) - }) - - it('should handle empty MIME type list', async () => { - registry.registerHandler({ - name: 'extension-only', - mimeTypes: [], - extensions: ['.xyz'], - loader: async () => createMockHandler('extension-only') - }) - - const handler = await registry.getHandlerByExtension('.xyz') - expect(handler?.format).toBe('extension-only') - }) - }) -}) diff --git a/tests/unit/augmentations/image-handler.test.ts b/tests/unit/augmentations/image-handler.test.ts deleted file mode 100644 index ae5bf2b8..00000000 --- a/tests/unit/augmentations/image-handler.test.ts +++ /dev/null @@ -1,311 +0,0 @@ -/** - * ImageHandler Tests (v5.8.0 - Pure JavaScript) - * - * Tests for image processing with EXIF extraction and metadata extraction - * Using probe-image-size + exifr (no native dependencies) - */ - -import { describe, it, expect, beforeEach } from 'vitest' -import { ImageHandler } from '../../../src/augmentations/intelligentImport/handlers/imageHandler.js' -import { readFileSync } from 'fs' -import { join } from 'path' - -describe('ImageHandler (v5.8.0 - Pure JS)', () => { - let handler: ImageHandler - - // Simple test image generators (minimal PNG/JPEG headers) - const createMinimalJPEG = (width: number = 100, height: number = 100): Buffer => { - // Minimal JPEG: SOI + SOF0 + EOI - // This is a valid but minimal JPEG structure - const soi = Buffer.from([0xFF, 0xD8]) // Start of Image - const sof0 = Buffer.from([ - 0xFF, 0xC0, // SOF0 marker - 0x00, 0x11, // Length: 17 bytes - 0x08, // Precision: 8 bits - (height >> 8) & 0xFF, height & 0xFF, // Height - (width >> 8) & 0xFF, width & 0xFF, // Width - 0x03, // Components: 3 (YCbCr) - 0x01, 0x22, 0x00, // Y component - 0x02, 0x11, 0x01, // Cb component - 0x03, 0x11, 0x01 // Cr component - ]) - const eoi = Buffer.from([0xFF, 0xD9]) // End of Image - - return Buffer.concat([soi, sof0, eoi]) - } - - const createMinimalPNG = (width: number = 100, height: number = 100): Buffer => { - // PNG signature - const signature = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]) - - // IHDR chunk - const ihdr = Buffer.alloc(25) - ihdr.writeUInt32BE(13, 0) // Length - ihdr.write('IHDR', 4) - ihdr.writeUInt32BE(width, 8) - ihdr.writeUInt32BE(height, 12) - ihdr.writeUInt8(8, 16) // Bit depth - ihdr.writeUInt8(2, 17) // Color type (RGB) - ihdr.writeUInt8(0, 18) // Compression - ihdr.writeUInt8(0, 19) // Filter - ihdr.writeUInt8(0, 20) // Interlace - ihdr.writeUInt32BE(0, 21) // CRC (simplified) - - // IEND chunk - const iend = Buffer.from([0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130]) - - return Buffer.concat([signature, ihdr, iend]) - } - - beforeEach(() => { - handler = new ImageHandler() - }) - - describe('Handler Detection', () => { - it('should identify as image format handler', () => { - expect(handler.format).toBe('image') - }) - - it('should handle JPEG files by filename', () => { - expect(handler.canHandle({ filename: 'photo.jpg' })).toBe(true) - expect(handler.canHandle({ filename: 'photo.jpeg' })).toBe(true) - }) - - it('should handle PNG files by filename', () => { - expect(handler.canHandle({ filename: 'logo.png' })).toBe(true) - }) - - it('should handle WebP files by filename', () => { - expect(handler.canHandle({ filename: 'image.webp' })).toBe(true) - }) - - it('should handle other image formats', () => { - expect(handler.canHandle({ filename: 'photo.gif' })).toBe(true) - expect(handler.canHandle({ filename: 'photo.tiff' })).toBe(true) - expect(handler.canHandle({ filename: 'photo.bmp' })).toBe(true) - expect(handler.canHandle({ filename: 'icon.svg' })).toBe(true) - }) - - it('should handle images by extension', () => { - expect(handler.canHandle({ ext: '.jpg' })).toBe(true) - expect(handler.canHandle({ ext: 'png' })).toBe(true) - }) - - it('should reject non-image files', () => { - expect(handler.canHandle({ filename: 'document.pdf' })).toBe(false) - expect(handler.canHandle({ filename: 'data.csv' })).toBe(false) - expect(handler.canHandle({ filename: 'text.txt' })).toBe(false) - }) - - it('should detect JPEG by magic bytes', () => { - const jpegBuffer = createMinimalJPEG(100, 100) - expect(handler.canHandle(jpegBuffer)).toBe(true) - }) - - it('should detect PNG by magic bytes', () => { - const pngBuffer = createMinimalPNG(100, 100) - expect(handler.canHandle(pngBuffer)).toBe(true) - }) - }) - - describe('Image Metadata Extraction', () => { - it('should extract JPEG metadata', async () => { - const imageBuffer = createMinimalJPEG(800, 600) - - const result = await handler.process(imageBuffer, { - extractEXIF: false - }) - - expect(result.format).toBe('image') - expect(result.data).toHaveLength(1) - - const imageData = result.data[0] - expect(imageData.type).toBe('media') - expect(imageData.metadata).toBeDefined() - expect(imageData.metadata.subtype).toBe('image') - expect(imageData.metadata.width).toBe(800) - expect(imageData.metadata.height).toBe(600) - expect(imageData.metadata.format).toBe('jpg') // probe-image-size uses 'jpg' not 'jpeg' - }) - - it('should extract PNG metadata', async () => { - const imageBuffer = createMinimalPNG(400, 300) - - const result = await handler.process(imageBuffer, { - extractEXIF: false - }) - - const imageData = result.data[0] - expect(imageData.metadata.width).toBe(400) - expect(imageData.metadata.height).toBe(300) - expect(imageData.metadata.format).toBe('png') - }) - - it('should include image size in bytes', async () => { - const imageBuffer = createMinimalJPEG(200, 200) - - const result = await handler.process(imageBuffer, { - extractEXIF: false - }) - - const imageData = result.data[0] - expect(imageData.metadata.size).toBe(imageBuffer.length) - expect(imageData.metadata.size).toBeGreaterThan(0) - }) - - it('should include MIME type when available', async () => { - const imageBuffer = createMinimalJPEG(100, 100) - - const result = await handler.process(imageBuffer, { - extractEXIF: false - }) - - const imageData = result.data[0] - expect(imageData.metadata.mimeType).toBeDefined() - expect(imageData.metadata.mimeType).toContain('image/') - }) - - it('should include processing time in metadata', async () => { - const imageBuffer = createMinimalJPEG(800, 600) - - const result = await handler.process(imageBuffer) - - expect(result.metadata.processingTime).toBeGreaterThanOrEqual(0) - expect(result.metadata.processingTime).toBeLessThan(5000) - }) - - it('should include image metadata in result metadata', async () => { - const imageBuffer = createMinimalJPEG(800, 600) - - const result = await handler.process(imageBuffer) - - expect(result.metadata.imageMetadata).toBeDefined() - expect(result.metadata.imageMetadata.width).toBe(800) - expect(result.metadata.imageMetadata.height).toBe(600) - }) - }) - - describe('EXIF Data Extraction', () => { - it('should handle images without EXIF data', async () => { - const imageBuffer = createMinimalJPEG(800, 600) - - const result = await handler.process(imageBuffer, { - extractEXIF: true - }) - - const imageData = result.data[0] - // Should not crash, EXIF will be undefined - expect(imageData.metadata.exif).toBeUndefined() - }) - - it('should extract EXIF by default', async () => { - const imageBuffer = createMinimalJPEG(800, 600) - - // Default: EXIF extraction enabled - const result = await handler.process(imageBuffer) - - // Will be undefined for test images, but should not crash - expect(result.metadata.exifData).toBeUndefined() - }) - - it('should allow disabling EXIF extraction', async () => { - const imageBuffer = createMinimalJPEG(800, 600) - - const result = await handler.process(imageBuffer, { - extractEXIF: false - }) - - const imageData = result.data[0] - expect(imageData.metadata.exif).toBeUndefined() - }) - }) - - describe('Error Handling', () => { - it('should handle invalid image data gracefully', async () => { - const invalidBuffer = Buffer.from('not an image', 'utf-8') - - // Should still process but with fallback values - const result = await handler.process(invalidBuffer, { extractEXIF: false }) - - // Fallback detection should provide basic info - expect(result.data[0].metadata.format).toBeDefined() - expect(result.data[0].metadata.size).toBe(invalidBuffer.length) - }) - - it('should handle empty buffer', async () => { - const emptyBuffer = Buffer.alloc(0) - - // Should not crash, but will have unknown format - const result = await handler.process(emptyBuffer, { extractEXIF: false }) - expect(result.data[0].metadata.format).toBe('unknown') - }) - }) - - describe('Format Support', () => { - it('should process JPEG images', async () => { - const imageBuffer = createMinimalJPEG(200, 200) - - const result = await handler.process(imageBuffer, { - extractEXIF: false - }) - - expect(result.data[0].metadata.format).toBe('jpg') // probe-image-size uses 'jpg' not 'jpeg' - }) - - it('should process PNG images', async () => { - const imageBuffer = createMinimalPNG(200, 200) - - const result = await handler.process(imageBuffer, { - extractEXIF: false - }) - - expect(result.data[0].metadata.format).toBe('png') - }) - - it('should handle various image dimensions', async () => { - const sizes = [ - [100, 100], - [1920, 1080], - [400, 300], - [1000, 500] - ] - - for (const [width, height] of sizes) { - const imageBuffer = createMinimalJPEG(width, height) - const result = await handler.process(imageBuffer, { extractEXIF: false }) - - expect(result.data[0].metadata.width).toBe(width) - expect(result.data[0].metadata.height).toBe(height) - } - }) - }) - - describe('Integration with BaseFormatHandler', () => { - it('should inherit MIME type detection from BaseFormatHandler', () => { - // getMimeType is protected, but we can verify behavior - expect(handler.canHandle({ filename: 'test.jpg' })).toBe(true) - expect(handler.canHandle({ filename: 'test.png' })).toBe(true) - }) - - it('should provide structured ProcessedData', async () => { - const imageBuffer = createMinimalJPEG(200, 200) - - const result = await handler.process(imageBuffer, { - filename: 'test-image.jpg', - extractEXIF: false - }) - - // Verify ProcessedData structure - expect(result.format).toBe('image') - expect(result.data).toBeInstanceOf(Array) - expect(result.metadata).toBeDefined() - expect(result.filename).toBe('test-image.jpg') - - // Verify data structure - const entity = result.data[0] - expect(entity.name).toBe('test-image') - expect(entity.type).toBe('media') - expect(entity.metadata).toBeDefined() - }) - }) -}) diff --git a/tests/unit/brainy-core.unit.test.ts b/tests/unit/brainy-core.unit.test.ts index aab84f80..eae3a86b 100644 --- a/tests/unit/brainy-core.unit.test.ts +++ b/tests/unit/brainy-core.unit.test.ts @@ -15,13 +15,7 @@ describe('Brainy 3.0 Core (Unit Tests)', () => { beforeEach(async () => { // Create instance with real embeddings for production-ready tests brain = new Brainy({ - storage: { type: 'memory' }, - augmentations: { - cache: false, - metrics: false, - display: false, - monitoring: false - } + storage: { type: 'memory' } }) await brain.init() @@ -196,7 +190,7 @@ describe('Brainy 3.0 Core (Unit Tests)', () => { }) describe('Statistics and Metadata', () => { - it('should track statistics through augmentations', async () => { + it('should track statistics', async () => { await brain.add({ data: { name: 'Test1' }, type: NounType.Concept diff --git a/tests/unit/brainy/add.test.ts b/tests/unit/brainy/add.test.ts index 144c7b13..241e4b1b 100644 --- a/tests/unit/brainy/add.test.ts +++ b/tests/unit/brainy/add.test.ts @@ -482,24 +482,6 @@ describe('Brainy.add()', () => { }) }) - describe('augmentations', () => { - it('should apply augmentations during add', async () => { - // This would test augmentation pipeline if configured - // For now, just verify the entity is added correctly - const params = createAddParams({ - data: 'Augmentation test', - type: 'thing', - metadata: { augment: true } - }) - - const id = await brain.add(params) - const entity = await brain.get(id) - - expect(entity).not.toBeNull() - expect(entity!.metadata.augment).toBe(true) - }) - }) - describe('caching behavior', () => { it('should retrieve consistent entities', async () => { // Arrange (v5.1.0: use valid UUID format) diff --git a/tests/unit/performance/add-performance.test.ts b/tests/unit/performance/add-performance.test.ts index 1ff9c2fb..ea642929 100644 --- a/tests/unit/performance/add-performance.test.ts +++ b/tests/unit/performance/add-performance.test.ts @@ -18,13 +18,7 @@ describe('add() Performance Regression Tests', () => { beforeEach(async () => { brain = new Brainy({ storage: { type: 'memory' }, - silent: true, - augmentations: { - cache: false, - metrics: false, - display: false, - monitoring: false - } + silent: true }) await brain.init() }) diff --git a/tests/unit/plugin.test.ts b/tests/unit/plugin.test.ts index 74ff9956..c94e17af 100644 --- a/tests/unit/plugin.test.ts +++ b/tests/unit/plugin.test.ts @@ -214,7 +214,6 @@ describe('Brainy plugin integration', () => { const brain = new Brainy({ storage: { type: 'memory' }, silent: true, - augmentations: { cache: false, metrics: false, display: false, monitoring: false } }) brain.use(mockPlugin) await brain.init() @@ -244,7 +243,6 @@ describe('Brainy plugin integration', () => { const brain = new Brainy({ storage: { type: 'memory' }, silent: true, - augmentations: { cache: false, metrics: false, display: false, monitoring: false } }) brain.use(mockPlugin) await brain.init() @@ -263,7 +261,6 @@ describe('Brainy plugin integration', () => { const brain = new Brainy({ storage: { type: 'memory' }, silent: true, - augmentations: { cache: false, metrics: false, display: false, monitoring: false } }) brain.use(mockPlugin) await brain.init() @@ -293,7 +290,6 @@ describe('Brainy plugin integration', () => { const brain = new Brainy({ storage: { type: 'memory' }, silent: true, - augmentations: { cache: false, metrics: false, display: false, monitoring: false } }) brain.use(mockPlugin) diff --git a/tests/unit/type-matching.unit.test.ts b/tests/unit/type-matching.unit.test.ts deleted file mode 100644 index 45147f31..00000000 --- a/tests/unit/type-matching.unit.test.ts +++ /dev/null @@ -1,219 +0,0 @@ -/** - * Tests for Intelligent Type Matching with embeddings - */ - -import { describe, it, expect, beforeAll, afterAll } from 'vitest' -import { IntelligentTypeMatcher } from '../../src/augmentations/typeMatching/intelligentTypeMatcher.js' -import { NounType, VerbType } from '../../src/types/graphTypes.js' - -describe('Intelligent Type Matching', () => { - let matcher: IntelligentTypeMatcher - - beforeAll(async () => { - matcher = new IntelligentTypeMatcher() - await matcher.init() - }) - - afterAll(async () => { - await matcher.dispose() - }) - - describe('Noun Type Detection', () => { - it('should detect Person type from user data', async () => { - const result = await matcher.matchNounType({ - name: 'John Doe', - email: 'john@example.com', - age: 30 - }) - - expect(result.type).toBeDefined() - expect(result.confidence).toBeGreaterThan(0) - expect(result.alternatives).toBeDefined() - expect(result.alternatives.length).toBeGreaterThanOrEqual(0) - }) - - it('should detect Organization type from company data', async () => { - const result = await matcher.matchNounType({ - companyName: 'Acme Corp', - employees: 500, - industry: 'Technology' - }) - - // With real type embeddings (v3.33.0+), type detection uses actual semantic similarity - // Results depend on the embedding quality and field patterns - expect(result.type).toBeDefined() - expect(result.confidence).toBeGreaterThan(0.1) - }) - - it('should detect Location type from geographic data', async () => { - const result = await matcher.matchNounType({ - latitude: 37.7749, - longitude: -122.4194, - city: 'San Francisco' - }) - - // With real type embeddings (v3.33.0+), geographic data detection is semantic - expect(result.type).toBeDefined() - expect(result.confidence).toBeGreaterThan(0.1) - }) - - it('should detect Document type from text content', async () => { - const result = await matcher.matchNounType({ - title: 'Research Paper', - content: 'Abstract: This paper discusses...', - author: 'Dr. Smith', - pages: 20 - }) - - // With real type embeddings (v3.33.0+), could match various types based on semantic similarity - expect(result.type).toBeDefined() - expect(result.confidence).toBeGreaterThan(0.0) - }) - - it('should detect Product type from commercial data', async () => { - const result = await matcher.matchNounType({ - price: 99.99, - sku: 'PROD-123', - inventory: 50, - productId: 'abc-123' - }) - - // With real type embeddings (v3.33.0+), commercial data uses semantic matching - expect(result.type).toBeDefined() - expect(result.confidence).toBeGreaterThan(0.1) - }) - - it('should detect Event type from temporal data', async () => { - const result = await matcher.matchNounType({ - startTime: '2024-01-01', - endTime: '2024-01-02', - attendees: 100, - eventType: 'conference' - }) - - // With real type embeddings (v3.33.0+), temporal data uses semantic matching - expect(result.type).toBeDefined() - expect(result.confidence).toBeGreaterThan(0.1) - }) - - it('should handle ambiguous data with alternatives', async () => { - const result = await matcher.matchNounType({ - value: 100, - type: 'unknown', - data: 'mixed' - }) - - expect(result.type).toBeDefined() - expect(result.alternatives).toBeDefined() - expect(result.alternatives.length).toBeGreaterThan(0) - }) - }) - - describe('Verb Type Detection', () => { - it('should detect MemberOf relationship', async () => { - const result = await matcher.matchVerbType( - { id: 'user1', type: 'person' }, - { id: 'org1', type: 'organization' }, - 'memberOf' - ) - - // With mocked embeddings, type detection is non-deterministic - expect(result.type).toBeDefined() - expect(Object.values(VerbType)).toContain(result.type) - expect(result.confidence).toBeGreaterThan(0) - }) - - it('should detect CreatedBy relationship', async () => { - const result = await matcher.matchVerbType( - { id: 'doc1', type: 'document' }, - { id: 'author1', type: 'person' }, - 'created by' - ) - - // With mocked embeddings, type detection is non-deterministic - expect(result.type).toBeDefined() - expect(Object.values(VerbType)).toContain(result.type) - expect(result.confidence).toBeGreaterThan(0) - }) - - it('should detect Contains relationship', async () => { - const result = await matcher.matchVerbType( - { id: 'folder1', type: 'collection' }, - { id: 'file1', type: 'file' }, - 'contains' - ) - - // With mocked embeddings, type detection is non-deterministic - expect(result.type).toBeDefined() - expect(Object.values(VerbType)).toContain(result.type) - expect(result.confidence).toBeGreaterThan(0) - }) - - it('should handle unknown relationships with defaults', async () => { - const result = await matcher.matchVerbType( - { id: 'a' }, - { id: 'b' }, - 'somehow connected' - ) - - expect(result.type).toBeDefined() - expect(Object.values(VerbType)).toContain(result.type) - }) - }) - - describe('Type Coverage', () => { - it('should have embeddings for all 42 noun types', async () => { - const nounTypes = Object.values(NounType) - expect(nounTypes.length).toBe(42) - - // Test that each type can be matched - for (const nounType of nounTypes) { - const result = await matcher.matchNounType({ - type: nounType, - test: 'coverage check' - }) - expect(result.type).toBeDefined() - } - }) - - it('should have embeddings for all 127 verb types', async () => { - const verbTypes = Object.values(VerbType) - expect(verbTypes.length).toBe(127) - - // Test that each type can be matched - for (const verbType of verbTypes) { - const result = await matcher.matchVerbType( - { id: 'source' }, - { id: 'target' }, - verbType - ) - expect(result.type).toBeDefined() - } - }) - }) - - describe('Cache Performance', () => { - it('should cache repeated type matches', async () => { - const testData = { name: 'Cache Test', value: 123 } - - const result1 = await matcher.matchNounType(testData) - const result2 = await matcher.matchNounType(testData) - - expect(result1.type).toBe(result2.type) - expect(result1.confidence).toBe(result2.confidence) - // Cache should return consistent results - expect(result2.type).toBeDefined() - }) - - it('should clear cache when requested', async () => { - const testData = { name: 'Clear Cache Test' } - - await matcher.matchNounType(testData) // Populate cache - matcher.clearCache() - - // Should still work after cache clear - const result = await matcher.matchNounType(testData) - expect(result.type).toBeDefined() - }) - }) -}) \ No newline at end of file